@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -0
- package/AGENTS.md +67 -0
- package/build.mjs +10 -0
- package/dist/contract/adapter.js +13 -0
- package/dist/contract/adapter.js.map +7 -0
- package/dist/contract/errors.js +28 -0
- package/dist/contract/errors.js.map +7 -0
- package/dist/contract/http.js +7 -0
- package/dist/contract/http.js.map +7 -0
- package/dist/contract/index.js +9 -0
- package/dist/contract/index.js.map +7 -0
- package/dist/contract/outcomes.js +58 -0
- package/dist/contract/outcomes.js.map +7 -0
- package/dist/contract/policy.js +91 -0
- package/dist/contract/policy.js.map +7 -0
- package/dist/contract/results.js +13 -0
- package/dist/contract/results.js.map +7 -0
- package/dist/contract/steps.js +1 -0
- package/dist/contract/steps.js.map +7 -0
- package/dist/contract/version.js +11 -0
- package/dist/contract/version.js.map +7 -0
- package/dist/engine/cache.js +29 -0
- package/dist/engine/cache.js.map +7 -0
- package/dist/engine/deadline.js +54 -0
- package/dist/engine/deadline.js.map +7 -0
- package/dist/engine/engine.js +481 -0
- package/dist/engine/engine.js.map +7 -0
- package/dist/engine/index.js +20 -0
- package/dist/engine/index.js.map +7 -0
- package/dist/engine/runAdapter.js +57 -0
- package/dist/engine/runAdapter.js.map +7 -0
- package/dist/engine/types.js +1 -0
- package/dist/engine/types.js.map +7 -0
- package/dist/extract/classify.js +51 -0
- package/dist/extract/classify.js.map +7 -0
- package/dist/extract/content.js +104 -0
- package/dist/extract/content.js.map +7 -0
- package/dist/extract/entities.js +95 -0
- package/dist/extract/entities.js.map +7 -0
- package/dist/extract/index.js +24 -0
- package/dist/extract/index.js.map +7 -0
- package/dist/extract/text.js +114 -0
- package/dist/extract/text.js.map +7 -0
- package/dist/extract/tokenizer.js +141 -0
- package/dist/extract/tokenizer.js.map +7 -0
- package/dist/fusion/canonicalize.js +88 -0
- package/dist/fusion/canonicalize.js.map +7 -0
- package/dist/fusion/fuse.js +88 -0
- package/dist/fusion/fuse.js.map +7 -0
- package/dist/fusion/index.js +11 -0
- package/dist/fusion/index.js.map +7 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +7 -0
- package/dist/loader/describe.js +92 -0
- package/dist/loader/describe.js.map +7 -0
- package/dist/loader/index.js +31 -0
- package/dist/loader/index.js.map +7 -0
- package/dist/loader/manifest.js +32 -0
- package/dist/loader/manifest.js.map +7 -0
- package/dist/loader/registry.js +98 -0
- package/dist/loader/registry.js.map +7 -0
- package/dist/net/client.js +298 -0
- package/dist/net/client.js.map +7 -0
- package/dist/net/ssrf.js +165 -0
- package/dist/net/ssrf.js.map +7 -0
- package/dist/net/url.js +9 -0
- package/dist/net/url.js.map +7 -0
- package/dist/testing/contract.js +118 -0
- package/dist/testing/contract.js.map +7 -0
- package/dist/testing/fakeAdapter.js +44 -0
- package/dist/testing/fakeAdapter.js.map +7 -0
- package/dist/testing/index.js +15 -0
- package/dist/testing/index.js.map +7 -0
- package/jest.config.cjs +24 -0
- package/package.json +65 -0
- package/src/contract/adapter.ts +100 -0
- package/src/contract/errors.ts +48 -0
- package/src/contract/http.ts +46 -0
- package/src/contract/index.ts +8 -0
- package/src/contract/outcomes.ts +91 -0
- package/src/contract/policy.ts +157 -0
- package/src/contract/results.ts +79 -0
- package/src/contract/steps.ts +45 -0
- package/src/contract/version.ts +25 -0
- package/src/engine/__tests__/engine.test.ts +597 -0
- package/src/engine/cache.ts +46 -0
- package/src/engine/deadline.ts +65 -0
- package/src/engine/engine.ts +633 -0
- package/src/engine/index.ts +22 -0
- package/src/engine/runAdapter.ts +89 -0
- package/src/engine/types.ts +137 -0
- package/src/extract/__tests__/content.test.ts +81 -0
- package/src/extract/__tests__/text.test.ts +61 -0
- package/src/extract/__tests__/tokenizer.test.ts +85 -0
- package/src/extract/classify.ts +81 -0
- package/src/extract/content.ts +136 -0
- package/src/extract/entities.ts +98 -0
- package/src/extract/index.ts +11 -0
- package/src/extract/text.ts +124 -0
- package/src/extract/tokenizer.ts +197 -0
- package/src/fusion/__tests__/fusion.test.ts +198 -0
- package/src/fusion/canonicalize.ts +119 -0
- package/src/fusion/fuse.ts +145 -0
- package/src/fusion/index.ts +8 -0
- package/src/index.ts +15 -0
- package/src/loader/__tests__/describe.test.ts +119 -0
- package/src/loader/__tests__/registry.test.ts +134 -0
- package/src/loader/describe.ts +145 -0
- package/src/loader/index.ts +25 -0
- package/src/loader/manifest.ts +44 -0
- package/src/loader/registry.ts +156 -0
- package/src/net/__tests__/client.test.ts +273 -0
- package/src/net/__tests__/ssrf.test.ts +165 -0
- package/src/net/client.ts +434 -0
- package/src/net/ssrf.ts +235 -0
- package/src/net/url.ts +13 -0
- package/src/testing/contract.ts +172 -0
- package/src/testing/fakeAdapter.ts +69 -0
- package/src/testing/index.ts +9 -0
- package/tsconfig.json +10 -0
- package/watch.mjs +7 -0
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { silentLogger } from "../contract/http.js";
|
|
2
|
+
import { CONTRACT_VERSION } from "../contract/version.js";
|
|
3
|
+
function createStubHttpClient(handler) {
|
|
4
|
+
const calls = [];
|
|
5
|
+
return {
|
|
6
|
+
calls,
|
|
7
|
+
async request(url, init) {
|
|
8
|
+
calls.push(url);
|
|
9
|
+
const stub = await handler(url, init);
|
|
10
|
+
return {
|
|
11
|
+
url: stub.url ?? url,
|
|
12
|
+
status: stub.status ?? 200,
|
|
13
|
+
headers: stub.headers ?? /* @__PURE__ */ new Map(),
|
|
14
|
+
contentType: stub.contentType ?? "text/html",
|
|
15
|
+
body: stub.body,
|
|
16
|
+
truncated: stub.truncated ?? false
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function createTestContext(overrides = {}) {
|
|
22
|
+
return {
|
|
23
|
+
signal: new AbortController().signal,
|
|
24
|
+
http: createStubHttpClient(() => ({ body: "" })),
|
|
25
|
+
report: () => {
|
|
26
|
+
},
|
|
27
|
+
deadlineAt: Date.now() + 1e4,
|
|
28
|
+
logger: silentLogger,
|
|
29
|
+
...overrides
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
const SAMPLE_REQUEST = { query: "open mercato", limit: 5 };
|
|
33
|
+
const VALID_STATUSES = ["ok", "empty", "unavailable", "blocked", "timeout", "error"];
|
|
34
|
+
function describeAdapterContract(module, testCase, api) {
|
|
35
|
+
const { describe, it, expect } = api;
|
|
36
|
+
describe(`${module.id} adapter contract`, () => {
|
|
37
|
+
it("declares the contract version it was built against", () => {
|
|
38
|
+
expect(module.contractVersion).toBe(CONTRACT_VERSION);
|
|
39
|
+
});
|
|
40
|
+
it("exposes a stable id and kind", () => {
|
|
41
|
+
expect(typeof module.id).toBe("string");
|
|
42
|
+
expect(module.id.length > 0).toBe(true);
|
|
43
|
+
expect(["serp", "api", "model", "browser"].includes(module.kind)).toBe(true);
|
|
44
|
+
});
|
|
45
|
+
it("validates its own options", () => {
|
|
46
|
+
const parsed = module.optionsSchema.safeParse(testCase.configuredOptions);
|
|
47
|
+
expect(parsed.success).toBe(true);
|
|
48
|
+
});
|
|
49
|
+
it("reports readiness without performing I/O", () => {
|
|
50
|
+
const adapter = module.createAdapter(testCase.configuredOptions);
|
|
51
|
+
const readiness = adapter.readiness();
|
|
52
|
+
expect(typeof readiness.ready).toBe("boolean");
|
|
53
|
+
});
|
|
54
|
+
const unconfigured = testCase.unconfiguredOptions;
|
|
55
|
+
if (unconfigured !== void 0) {
|
|
56
|
+
it("is unavailable rather than broken when unconfigured", async () => {
|
|
57
|
+
const adapter = module.createAdapter(unconfigured);
|
|
58
|
+
const readiness = adapter.readiness();
|
|
59
|
+
expect(readiness.ready).toBe(false);
|
|
60
|
+
if (!readiness.ready) expect(readiness.reason.length > 0).toBe(true);
|
|
61
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext());
|
|
62
|
+
expect(outcome.status).toBe("unavailable");
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
it("returns an outcome rather than throwing when egress fails", async () => {
|
|
66
|
+
const adapter = module.createAdapter(testCase.configuredOptions);
|
|
67
|
+
const context = createTestContext({
|
|
68
|
+
http: createStubHttpClient(() => {
|
|
69
|
+
throw new Error("network exploded");
|
|
70
|
+
})
|
|
71
|
+
});
|
|
72
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, context);
|
|
73
|
+
expect(VALID_STATUSES.includes(outcome.status)).toBe(true);
|
|
74
|
+
});
|
|
75
|
+
it("honours an already-aborted signal without hanging", async () => {
|
|
76
|
+
const adapter = module.createAdapter(testCase.configuredOptions);
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
controller.abort();
|
|
79
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext({ signal: controller.signal }));
|
|
80
|
+
expect(typeof outcome.status).toBe("string");
|
|
81
|
+
});
|
|
82
|
+
it("never returns a result without a url", async () => {
|
|
83
|
+
const adapter = module.createAdapter(testCase.configuredOptions);
|
|
84
|
+
const context = testCase.context ? testCase.context() : createTestContext();
|
|
85
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, context);
|
|
86
|
+
if (outcome.status !== "ok") return;
|
|
87
|
+
for (const result of outcome.results) {
|
|
88
|
+
expect(typeof result.url).toBe("string");
|
|
89
|
+
expect(result.url.length > 0).toBe(true);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
it("tears down cleanly even when it was never used", async () => {
|
|
93
|
+
const adapter = module.createAdapter(testCase.configuredOptions);
|
|
94
|
+
if (!adapter.dispose) return;
|
|
95
|
+
await adapter.dispose();
|
|
96
|
+
await adapter.dispose();
|
|
97
|
+
expect(true).toBe(true);
|
|
98
|
+
});
|
|
99
|
+
it("answers a health check without throwing", async () => {
|
|
100
|
+
const adapter = module.createAdapter(testCase.configuredOptions);
|
|
101
|
+
if (!adapter.healthCheck) return;
|
|
102
|
+
const context = createTestContext({
|
|
103
|
+
http: createStubHttpClient(() => {
|
|
104
|
+
throw new Error("network exploded");
|
|
105
|
+
})
|
|
106
|
+
});
|
|
107
|
+
const health = await adapter.healthCheck(context);
|
|
108
|
+
expect(typeof health.ok).toBe("boolean");
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
export {
|
|
113
|
+
SAMPLE_REQUEST,
|
|
114
|
+
createStubHttpClient,
|
|
115
|
+
createTestContext,
|
|
116
|
+
describeAdapterContract
|
|
117
|
+
};
|
|
118
|
+
//# sourceMappingURL=contract.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/testing/contract.ts"],
|
|
4
|
+
"sourcesContent": ["import type { AdapterContext, AdapterModule, SearchAdapter } from '../contract/adapter'\nimport { silentLogger, type HttpClient, type HttpResponse } from '../contract/http'\nimport { CONTRACT_VERSION } from '../contract/version'\nimport type { SearchRequest } from '../contract/results'\n\nexport type StubResponse = Partial<HttpResponse> & { readonly body: string }\n\n/**\n * Minimal HTTP double. Adapters are contractually forbidden from reaching for\n * `fetch`, so a test that passes with this client has also proved the adapter\n * routes every byte through the hardened transport.\n */\nexport function createStubHttpClient(\n handler: (url: string, init: Parameters<HttpClient['request']>[1]) => StubResponse | Promise<StubResponse>,\n): HttpClient & { readonly calls: readonly string[] } {\n const calls: string[] = []\n return {\n calls,\n async request(url, init) {\n calls.push(url)\n const stub = await handler(url, init)\n return {\n url: stub.url ?? url,\n status: stub.status ?? 200,\n headers: stub.headers ?? new Map<string, string>(),\n contentType: stub.contentType ?? 'text/html',\n body: stub.body,\n truncated: stub.truncated ?? false,\n }\n },\n }\n}\n\nexport function createTestContext(overrides: Partial<AdapterContext> = {}): AdapterContext {\n return {\n signal: new AbortController().signal,\n http: createStubHttpClient(() => ({ body: '' })),\n report: () => {},\n deadlineAt: Date.now() + 10_000,\n logger: silentLogger,\n ...overrides,\n }\n}\n\nexport const SAMPLE_REQUEST: SearchRequest = { query: 'open mercato', limit: 5 }\n\nconst VALID_STATUSES = ['ok', 'empty', 'unavailable', 'blocked', 'timeout', 'error']\n\nexport type AdapterContractCase<TOptions> = {\n /** Options that leave the adapter deliberately unconfigured. */\n readonly unconfiguredOptions?: TOptions\n /** Options sufficient for the adapter to consider itself ready. */\n readonly configuredOptions: TOptions\n /** Context used for the \"returns results\" probe; stub the HTTP client here. */\n readonly context?: () => AdapterContext\n}\n\ntype TestApi = {\n describe: (name: string, body: () => void) => void\n it: (name: string, body: () => void | Promise<void>) => void\n expect: (value: unknown) => {\n toBe(expected: unknown): void\n toBeDefined(): void\n toContain(expected: string): void\n }\n}\n\n/**\n * Behavioural suite every adapter must pass, ours and third parties'. It pins the\n * parts of the contract the scheduler actually depends on \u2014 most importantly that\n * an adapter reports rather than throws, because a throwing adapter would\n * otherwise take down an entire search.\n */\nexport function describeAdapterContract<TOptions>(\n module: AdapterModule<TOptions>,\n testCase: AdapterContractCase<TOptions>,\n api: TestApi,\n): void {\n const { describe, it, expect } = api\n\n describe(`${module.id} adapter contract`, () => {\n it('declares the contract version it was built against', () => {\n expect(module.contractVersion).toBe(CONTRACT_VERSION)\n })\n\n it('exposes a stable id and kind', () => {\n expect(typeof module.id).toBe('string')\n expect(module.id.length > 0).toBe(true)\n expect(['serp', 'api', 'model', 'browser'].includes(module.kind)).toBe(true)\n })\n\n it('validates its own options', () => {\n const parsed = module.optionsSchema.safeParse(testCase.configuredOptions)\n expect(parsed.success).toBe(true)\n })\n\n it('reports readiness without performing I/O', () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n const readiness = adapter.readiness()\n expect(typeof readiness.ready).toBe('boolean')\n })\n\n const unconfigured = testCase.unconfiguredOptions\n if (unconfigured !== undefined) {\n it('is unavailable rather than broken when unconfigured', async () => {\n const adapter = module.createAdapter(unconfigured)\n const readiness = adapter.readiness()\n expect(readiness.ready).toBe(false)\n if (!readiness.ready) expect(readiness.reason.length > 0).toBe(true)\n\n const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext())\n expect(outcome.status).toBe('unavailable')\n })\n }\n\n // The contract is \"never throws\", not \"always fails when HTTP does\" \u2014 a model\n // or browser adapter may not touch the HTTP client at all.\n it('returns an outcome rather than throwing when egress fails', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n const context = createTestContext({\n http: createStubHttpClient(() => {\n throw new Error('network exploded')\n }),\n })\n const outcome = await adapter.search(SAMPLE_REQUEST, context)\n expect(VALID_STATUSES.includes(outcome.status)).toBe(true)\n })\n\n it('honours an already-aborted signal without hanging', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n const controller = new AbortController()\n controller.abort()\n const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext({ signal: controller.signal }))\n expect(typeof outcome.status).toBe('string')\n })\n\n it('never returns a result without a url', async () => {\n const adapter: SearchAdapter = module.createAdapter(testCase.configuredOptions)\n const context = testCase.context ? testCase.context() : createTestContext()\n const outcome = await adapter.search(SAMPLE_REQUEST, context)\n if (outcome.status !== 'ok') return\n for (const result of outcome.results) {\n expect(typeof result.url).toBe('string')\n expect(result.url.length > 0).toBe(true)\n }\n })\n\n // The host builds an engine per request and disposes it after, so a `dispose`\n // that throws on an adapter which was never used, or on a second call, breaks\n // teardown for every other adapter in the same engine. This is the surface\n // that leaked a browser sidecar and a Chromium per tool call.\n it('tears down cleanly even when it was never used', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n if (!adapter.dispose) return\n await adapter.dispose()\n await adapter.dispose()\n expect(true).toBe(true)\n })\n\n it('answers a health check without throwing', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n if (!adapter.healthCheck) return\n const context = createTestContext({\n http: createStubHttpClient(() => {\n throw new Error('network exploded')\n }),\n })\n const health = await adapter.healthCheck(context)\n expect(typeof health.ok).toBe('boolean')\n })\n })\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,oBAAwD;AACjE,SAAS,wBAAwB;AAU1B,SAAS,qBACd,SACoD;AACpD,QAAM,QAAkB,CAAC;AACzB,SAAO;AAAA,IACL;AAAA,IACA,MAAM,QAAQ,KAAK,MAAM;AACvB,YAAM,KAAK,GAAG;AACd,YAAM,OAAO,MAAM,QAAQ,KAAK,IAAI;AACpC,aAAO;AAAA,QACL,KAAK,KAAK,OAAO;AAAA,QACjB,QAAQ,KAAK,UAAU;AAAA,QACvB,SAAS,KAAK,WAAW,oBAAI,IAAoB;AAAA,QACjD,aAAa,KAAK,eAAe;AAAA,QACjC,MAAM,KAAK;AAAA,QACX,WAAW,KAAK,aAAa;AAAA,MAC/B;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,kBAAkB,YAAqC,CAAC,GAAmB;AACzF,SAAO;AAAA,IACL,QAAQ,IAAI,gBAAgB,EAAE;AAAA,IAC9B,MAAM,qBAAqB,OAAO,EAAE,MAAM,GAAG,EAAE;AAAA,IAC/C,QAAQ,MAAM;AAAA,IAAC;AAAA,IACf,YAAY,KAAK,IAAI,IAAI;AAAA,IACzB,QAAQ;AAAA,IACR,GAAG;AAAA,EACL;AACF;AAEO,MAAM,iBAAgC,EAAE,OAAO,gBAAgB,OAAO,EAAE;AAE/E,MAAM,iBAAiB,CAAC,MAAM,SAAS,eAAe,WAAW,WAAW,OAAO;AA2B5E,SAAS,wBACd,QACA,UACA,KACM;AACN,QAAM,EAAE,UAAU,IAAI,OAAO,IAAI;AAEjC,WAAS,GAAG,OAAO,EAAE,qBAAqB,MAAM;AAC9C,OAAG,sDAAsD,MAAM;AAC7D,aAAO,OAAO,eAAe,EAAE,KAAK,gBAAgB;AAAA,IACtD,CAAC;AAED,OAAG,gCAAgC,MAAM;AACvC,aAAO,OAAO,OAAO,EAAE,EAAE,KAAK,QAAQ;AACtC,aAAO,OAAO,GAAG,SAAS,CAAC,EAAE,KAAK,IAAI;AACtC,aAAO,CAAC,QAAQ,OAAO,SAAS,SAAS,EAAE,SAAS,OAAO,IAAI,CAAC,EAAE,KAAK,IAAI;AAAA,IAC7E,CAAC;AAED,OAAG,6BAA6B,MAAM;AACpC,YAAM,SAAS,OAAO,cAAc,UAAU,SAAS,iBAAiB;AACxE,aAAO,OAAO,OAAO,EAAE,KAAK,IAAI;AAAA,IAClC,CAAC;AAED,OAAG,4CAA4C,MAAM;AACnD,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,YAAM,YAAY,QAAQ,UAAU;AACpC,aAAO,OAAO,UAAU,KAAK,EAAE,KAAK,SAAS;AAAA,IAC/C,CAAC;AAED,UAAM,eAAe,SAAS;AAC9B,QAAI,iBAAiB,QAAW;AAC9B,SAAG,uDAAuD,YAAY;AACpE,cAAM,UAAU,OAAO,cAAc,YAAY;AACjD,cAAM,YAAY,QAAQ,UAAU;AACpC,eAAO,UAAU,KAAK,EAAE,KAAK,KAAK;AAClC,YAAI,CAAC,UAAU,MAAO,QAAO,UAAU,OAAO,SAAS,CAAC,EAAE,KAAK,IAAI;AAEnE,cAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,kBAAkB,CAAC;AACxE,eAAO,QAAQ,MAAM,EAAE,KAAK,aAAa;AAAA,MAC3C,CAAC;AAAA,IACH;AAIA,OAAG,6DAA6D,YAAY;AAC1E,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,YAAM,UAAU,kBAAkB;AAAA,QAChC,MAAM,qBAAqB,MAAM;AAC/B,gBAAM,IAAI,MAAM,kBAAkB;AAAA,QACpC,CAAC;AAAA,MACH,CAAC;AACD,YAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,OAAO;AAC5D,aAAO,eAAe,SAAS,QAAQ,MAAM,CAAC,EAAE,KAAK,IAAI;AAAA,IAC3D,CAAC;AAED,OAAG,qDAAqD,YAAY;AAClE,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,YAAM,aAAa,IAAI,gBAAgB;AACvC,iBAAW,MAAM;AACjB,YAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,kBAAkB,EAAE,QAAQ,WAAW,OAAO,CAAC,CAAC;AACrG,aAAO,OAAO,QAAQ,MAAM,EAAE,KAAK,QAAQ;AAAA,IAC7C,CAAC;AAED,OAAG,wCAAwC,YAAY;AACrD,YAAM,UAAyB,OAAO,cAAc,SAAS,iBAAiB;AAC9E,YAAM,UAAU,SAAS,UAAU,SAAS,QAAQ,IAAI,kBAAkB;AAC1E,YAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,OAAO;AAC5D,UAAI,QAAQ,WAAW,KAAM;AAC7B,iBAAW,UAAU,QAAQ,SAAS;AACpC,eAAO,OAAO,OAAO,GAAG,EAAE,KAAK,QAAQ;AACvC,eAAO,OAAO,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI;AAAA,MACzC;AAAA,IACF,CAAC;AAMD,OAAG,kDAAkD,YAAY;AAC/D,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,UAAI,CAAC,QAAQ,QAAS;AACtB,YAAM,QAAQ,QAAQ;AACtB,YAAM,QAAQ,QAAQ;AACtB,aAAO,IAAI,EAAE,KAAK,IAAI;AAAA,IACxB,CAAC;AAED,OAAG,2CAA2C,YAAY;AACxD,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,UAAI,CAAC,QAAQ,YAAa;AAC1B,YAAM,UAAU,kBAAkB;AAAA,QAChC,MAAM,qBAAqB,MAAM;AAC/B,gBAAM,IAAI,MAAM,kBAAkB;AAAA,QACpC,CAAC;AAAA,MACH,CAAC;AACD,YAAM,SAAS,MAAM,QAAQ,YAAY,OAAO;AAChD,aAAO,OAAO,OAAO,EAAE,EAAE,KAAK,SAAS;AAAA,IACzC,CAAC;AAAA,EACH,CAAC;AACH;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { READY, notReady } from "../contract/adapter.js";
|
|
2
|
+
const BASE_CAPABILITIES = {
|
|
3
|
+
search: true,
|
|
4
|
+
fetch: false,
|
|
5
|
+
snippets: true,
|
|
6
|
+
content: false,
|
|
7
|
+
freshness: false,
|
|
8
|
+
siteFilter: false,
|
|
9
|
+
cost: "free"
|
|
10
|
+
};
|
|
11
|
+
function createFakeAdapter(options) {
|
|
12
|
+
const capabilities = { ...BASE_CAPABILITIES, ...options.capabilities };
|
|
13
|
+
const outcome = options.outcome ?? (options.results && options.results.length > 0 ? { status: "ok", results: options.results } : { status: "empty" });
|
|
14
|
+
return {
|
|
15
|
+
id: options.id,
|
|
16
|
+
kind: options.kind ?? "api",
|
|
17
|
+
capabilities,
|
|
18
|
+
readiness: () => options.ready === false ? notReady("fake adapter configured as not ready") : typeof options.ready === "string" ? notReady(options.ready) : READY,
|
|
19
|
+
async search(_request, context) {
|
|
20
|
+
if (options.throws) throw new Error(`${options.id} exploded`);
|
|
21
|
+
const latency = options.latencyMs ?? 0;
|
|
22
|
+
if (latency > 0) {
|
|
23
|
+
await new Promise((resolve) => {
|
|
24
|
+
const timer = setTimeout(resolve, latency);
|
|
25
|
+
timer.unref?.();
|
|
26
|
+
context.signal.addEventListener(
|
|
27
|
+
"abort",
|
|
28
|
+
() => {
|
|
29
|
+
clearTimeout(timer);
|
|
30
|
+
resolve();
|
|
31
|
+
},
|
|
32
|
+
{ once: true }
|
|
33
|
+
);
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
if (context.signal.aborted) return { status: "timeout", reason: "aborted" };
|
|
37
|
+
return outcome;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export {
|
|
42
|
+
createFakeAdapter
|
|
43
|
+
};
|
|
44
|
+
//# sourceMappingURL=fakeAdapter.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/testing/fakeAdapter.ts"],
|
|
4
|
+
"sourcesContent": ["import type { AdapterCapabilities, AdapterKind, SearchAdapter } from '../contract/adapter'\nimport { READY, notReady } from '../contract/adapter'\nimport type { SearchOutcome } from '../contract/outcomes'\nimport type { RawResult } from '../contract/results'\n\nexport type FakeAdapterOptions = {\n readonly id: string\n readonly kind?: AdapterKind\n readonly ready?: boolean | string\n /** Milliseconds of simulated work before the outcome resolves. */\n readonly latencyMs?: number\n readonly outcome?: SearchOutcome\n readonly results?: readonly RawResult[]\n readonly capabilities?: Partial<AdapterCapabilities>\n /** Set to make the adapter violate the contract, for isolation tests. */\n readonly throws?: boolean\n}\n\nconst BASE_CAPABILITIES: AdapterCapabilities = {\n search: true,\n fetch: false,\n snippets: true,\n content: false,\n freshness: false,\n siteFilter: false,\n cost: 'free',\n}\n\n/** Deterministic adapter double for scheduler and fusion tests. */\nexport function createFakeAdapter(options: FakeAdapterOptions): SearchAdapter {\n const capabilities: AdapterCapabilities = { ...BASE_CAPABILITIES, ...options.capabilities }\n const outcome: SearchOutcome =\n options.outcome ??\n (options.results && options.results.length > 0\n ? { status: 'ok', results: options.results }\n : { status: 'empty' })\n\n return {\n id: options.id,\n kind: options.kind ?? 'api',\n capabilities,\n readiness: () =>\n options.ready === false\n ? notReady('fake adapter configured as not ready')\n : typeof options.ready === 'string'\n ? notReady(options.ready)\n : READY,\n async search(_request, context) {\n if (options.throws) throw new Error(`${options.id} exploded`)\n const latency = options.latencyMs ?? 0\n if (latency > 0) {\n await new Promise<void>((resolve) => {\n const timer = setTimeout(resolve, latency)\n timer.unref?.()\n context.signal.addEventListener(\n 'abort',\n () => {\n clearTimeout(timer)\n resolve()\n },\n { once: true },\n )\n })\n }\n if (context.signal.aborted) return { status: 'timeout', reason: 'aborted' }\n return outcome\n },\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,OAAO,gBAAgB;AAiBhC,MAAM,oBAAyC;AAAA,EAC7C,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,UAAU;AAAA,EACV,SAAS;AAAA,EACT,WAAW;AAAA,EACX,YAAY;AAAA,EACZ,MAAM;AACR;AAGO,SAAS,kBAAkB,SAA4C;AAC5E,QAAM,eAAoC,EAAE,GAAG,mBAAmB,GAAG,QAAQ,aAAa;AAC1F,QAAM,UACJ,QAAQ,YACP,QAAQ,WAAW,QAAQ,QAAQ,SAAS,IACzC,EAAE,QAAQ,MAAM,SAAS,QAAQ,QAAQ,IACzC,EAAE,QAAQ,QAAQ;AAExB,SAAO;AAAA,IACL,IAAI,QAAQ;AAAA,IACZ,MAAM,QAAQ,QAAQ;AAAA,IACtB;AAAA,IACA,WAAW,MACT,QAAQ,UAAU,QACd,SAAS,sCAAsC,IAC/C,OAAO,QAAQ,UAAU,WACvB,SAAS,QAAQ,KAAK,IACtB;AAAA,IACR,MAAM,OAAO,UAAU,SAAS;AAC9B,UAAI,QAAQ,OAAQ,OAAM,IAAI,MAAM,GAAG,QAAQ,EAAE,WAAW;AAC5D,YAAM,UAAU,QAAQ,aAAa;AACrC,UAAI,UAAU,GAAG;AACf,cAAM,IAAI,QAAc,CAAC,YAAY;AACnC,gBAAM,QAAQ,WAAW,SAAS,OAAO;AACzC,gBAAM,QAAQ;AACd,kBAAQ,OAAO;AAAA,YACb;AAAA,YACA,MAAM;AACJ,2BAAa,KAAK;AAClB,sBAAQ;AAAA,YACV;AAAA,YACA,EAAE,MAAM,KAAK;AAAA,UACf;AAAA,QACF,CAAC;AAAA,MACH;AACA,UAAI,QAAQ,OAAO,QAAS,QAAO,EAAE,QAAQ,WAAW,QAAQ,UAAU;AAC1E,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import {
|
|
2
|
+
SAMPLE_REQUEST,
|
|
3
|
+
createStubHttpClient,
|
|
4
|
+
createTestContext,
|
|
5
|
+
describeAdapterContract
|
|
6
|
+
} from "./contract.js";
|
|
7
|
+
import { createFakeAdapter } from "./fakeAdapter.js";
|
|
8
|
+
export {
|
|
9
|
+
SAMPLE_REQUEST,
|
|
10
|
+
createFakeAdapter,
|
|
11
|
+
createStubHttpClient,
|
|
12
|
+
createTestContext,
|
|
13
|
+
describeAdapterContract
|
|
14
|
+
};
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/testing/index.ts"],
|
|
4
|
+
"sourcesContent": ["export {\n SAMPLE_REQUEST,\n createStubHttpClient,\n createTestContext,\n describeAdapterContract,\n type AdapterContractCase,\n type StubResponse,\n} from './contract'\nexport { createFakeAdapter, type FakeAdapterOptions } from './fakeAdapter'\n"],
|
|
5
|
+
"mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAGK;AACP,SAAS,yBAAkD;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/jest.config.cjs
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** @type {import('jest').Config} */
|
|
2
|
+
const base = require('../../jest.config.base.cjs')
|
|
3
|
+
|
|
4
|
+
module.exports = {
|
|
5
|
+
...base,
|
|
6
|
+
preset: 'ts-jest',
|
|
7
|
+
testEnvironment: 'node',
|
|
8
|
+
watchman: false,
|
|
9
|
+
rootDir: '.',
|
|
10
|
+
moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
|
|
11
|
+
transform: {
|
|
12
|
+
'^.+\\.(t|j)sx?$': [
|
|
13
|
+
'ts-jest',
|
|
14
|
+
{
|
|
15
|
+
tsconfig: {
|
|
16
|
+
rootDir: '.',
|
|
17
|
+
ignoreDeprecations: '6.0',
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
},
|
|
22
|
+
testMatch: ['<rootDir>/src/**/__tests__/**/*.test.(ts|tsx)'],
|
|
23
|
+
passWithNoTests: true,
|
|
24
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@open-mercato/web-research",
|
|
3
|
+
"version": "0.7.1-develop.7194.1.ab4fc81f82",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"build": "node build.mjs",
|
|
9
|
+
"watch": "node watch.mjs",
|
|
10
|
+
"test": "jest --config jest.config.cjs",
|
|
11
|
+
"typecheck": "tsc --noEmit"
|
|
12
|
+
},
|
|
13
|
+
"exports": {
|
|
14
|
+
".": {
|
|
15
|
+
"types": "./src/index.ts",
|
|
16
|
+
"default": "./dist/index.js"
|
|
17
|
+
},
|
|
18
|
+
"./*.ts": {
|
|
19
|
+
"types": "./src/*.ts",
|
|
20
|
+
"default": "./dist/*.js"
|
|
21
|
+
},
|
|
22
|
+
"./*": {
|
|
23
|
+
"types": [
|
|
24
|
+
"./src/*.ts",
|
|
25
|
+
"./src/*.tsx"
|
|
26
|
+
],
|
|
27
|
+
"default": "./dist/*.js"
|
|
28
|
+
},
|
|
29
|
+
"./*/*": {
|
|
30
|
+
"types": [
|
|
31
|
+
"./src/*/*.ts",
|
|
32
|
+
"./src/*/*.tsx"
|
|
33
|
+
],
|
|
34
|
+
"default": "./dist/*/*.js"
|
|
35
|
+
},
|
|
36
|
+
"./*/*/*": {
|
|
37
|
+
"types": [
|
|
38
|
+
"./src/*/*/*.ts",
|
|
39
|
+
"./src/*/*/*.tsx"
|
|
40
|
+
],
|
|
41
|
+
"default": "./dist/*/*/*.js"
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
"dependencies": {
|
|
45
|
+
"zod": "^4.4.3"
|
|
46
|
+
},
|
|
47
|
+
"devDependencies": {
|
|
48
|
+
"@types/jest": "^30.0.0",
|
|
49
|
+
"@types/node": "^26.0.1",
|
|
50
|
+
"esbuild": "^0.28.1",
|
|
51
|
+
"glob": "^13.0.6",
|
|
52
|
+
"jest": "^30.4.2",
|
|
53
|
+
"ts-jest": "^29.4.11",
|
|
54
|
+
"typescript": "^6.0.3"
|
|
55
|
+
},
|
|
56
|
+
"publishConfig": {
|
|
57
|
+
"access": "public"
|
|
58
|
+
},
|
|
59
|
+
"repository": {
|
|
60
|
+
"type": "git",
|
|
61
|
+
"url": "https://github.com/open-mercato/open-mercato",
|
|
62
|
+
"directory": "packages/web-research"
|
|
63
|
+
},
|
|
64
|
+
"stableVersion": "0.7.0"
|
|
65
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { z } from 'zod'
|
|
2
|
+
import type { HttpClient, Logger } from './http'
|
|
3
|
+
import type { FetchOutcome, SearchOutcome } from './outcomes'
|
|
4
|
+
import type { FetchRequest, SearchRequest } from './results'
|
|
5
|
+
import type { StepReporter } from './steps'
|
|
6
|
+
|
|
7
|
+
export type AdapterKind = 'serp' | 'api' | 'model' | 'browser'
|
|
8
|
+
|
|
9
|
+
export type AdapterCost = 'free' | 'metered' | 'llm-tokens'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* What calling `healthCheck()` costs — which is NOT what searching costs.
|
|
13
|
+
*
|
|
14
|
+
* Exa's probe reads a local key and Model-native's resolves configuration, both
|
|
15
|
+
* free, though both are priced adapters to search with. Firecrawl's and Tavily's
|
|
16
|
+
* probes issue a real query and bill for it. The browser tier's spawns an OS
|
|
17
|
+
* process: no money, but not something to run on every page view either.
|
|
18
|
+
*
|
|
19
|
+
* A status surface may run `free` probes unattended; `heavy` and `billable` need
|
|
20
|
+
* a human to ask for them.
|
|
21
|
+
*/
|
|
22
|
+
export type ProbeCost = 'free' | 'heavy' | 'billable'
|
|
23
|
+
|
|
24
|
+
export type AdapterCapabilities = {
|
|
25
|
+
readonly search: boolean
|
|
26
|
+
readonly fetch: boolean
|
|
27
|
+
/** Returns real snippets rather than bare URLs. */
|
|
28
|
+
readonly snippets: boolean
|
|
29
|
+
/** Returns full page text inline, sparing a follow-up fetch. */
|
|
30
|
+
readonly content: boolean
|
|
31
|
+
readonly freshness: boolean
|
|
32
|
+
readonly siteFilter: boolean
|
|
33
|
+
readonly cost: AdapterCost
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export type AdapterReadiness =
|
|
37
|
+
| { readonly ready: true }
|
|
38
|
+
| { readonly ready: false; readonly reason: string }
|
|
39
|
+
|
|
40
|
+
export type AdapterHealth = {
|
|
41
|
+
readonly ok: boolean
|
|
42
|
+
readonly detail?: string
|
|
43
|
+
readonly latencyMs?: number
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export type AdapterContext = {
|
|
47
|
+
readonly signal: AbortSignal
|
|
48
|
+
readonly http: HttpClient
|
|
49
|
+
/** Narrates progress; the engine supplies the sequence, timestamp and adapter id. */
|
|
50
|
+
readonly report: StepReporter
|
|
51
|
+
/** Epoch ms after which the engine stops caring about this adapter's answer. */
|
|
52
|
+
readonly deadlineAt: number
|
|
53
|
+
readonly logger: Logger
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface SearchAdapter {
|
|
57
|
+
readonly id: string
|
|
58
|
+
readonly kind: AdapterKind
|
|
59
|
+
readonly capabilities: AdapterCapabilities
|
|
60
|
+
/**
|
|
61
|
+
* Synchronous and I/O-free by contract, so planning a wave never blocks and an
|
|
62
|
+
* unconfigured adapter costs nothing to skip.
|
|
63
|
+
*/
|
|
64
|
+
readiness(): AdapterReadiness
|
|
65
|
+
/**
|
|
66
|
+
* What `healthCheck()` costs to call. Omitting it is safe but pessimistic: the
|
|
67
|
+
* engine then derives `billable` from `capabilities.cost === 'metered'` and
|
|
68
|
+
* `free` otherwise, so an adapter that never declares it is simply left out of
|
|
69
|
+
* unattended probing. Declare it whenever the probe costs money.
|
|
70
|
+
*/
|
|
71
|
+
readonly probeCost?: ProbeCost
|
|
72
|
+
search(request: SearchRequest, context: AdapterContext): Promise<SearchOutcome>
|
|
73
|
+
fetch?(request: FetchRequest, context: AdapterContext): Promise<FetchOutcome>
|
|
74
|
+
healthCheck?(context: AdapterContext): Promise<AdapterHealth>
|
|
75
|
+
dispose?(): Promise<void>
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* What an adapter package default-exports. A descriptor rather than an instance,
|
|
80
|
+
* because the engine builds one instance per tenant from that tenant's options.
|
|
81
|
+
*/
|
|
82
|
+
export type AdapterModule<TOptions = unknown> = {
|
|
83
|
+
readonly id: string
|
|
84
|
+
readonly kind: AdapterKind
|
|
85
|
+
readonly contractVersion: number
|
|
86
|
+
readonly optionsSchema: z.ZodType<TOptions>
|
|
87
|
+
createAdapter(options: TOptions): SearchAdapter
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function defineAdapterModule<TOptions>(
|
|
91
|
+
module: AdapterModule<TOptions>,
|
|
92
|
+
): AdapterModule<TOptions> {
|
|
93
|
+
return module
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export const READY: AdapterReadiness = { ready: true }
|
|
97
|
+
|
|
98
|
+
export function notReady(reason: string): AdapterReadiness {
|
|
99
|
+
return { ready: false, reason }
|
|
100
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export type WebResearchErrorCode =
|
|
2
|
+
| 'ssrf_blocked'
|
|
3
|
+
| 'timeout'
|
|
4
|
+
| 'aborted'
|
|
5
|
+
| 'blocked'
|
|
6
|
+
| 'bad_response'
|
|
7
|
+
| 'too_many_redirects'
|
|
8
|
+
| 'unsupported_content_type'
|
|
9
|
+
| 'network'
|
|
10
|
+
|
|
11
|
+
export type WebResearchErrorInit = {
|
|
12
|
+
readonly retryAfterMs?: number
|
|
13
|
+
readonly status?: number
|
|
14
|
+
readonly cause?: unknown
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class WebResearchError extends Error {
|
|
18
|
+
/**
|
|
19
|
+
* Brand checked by {@link isWebResearchError} instead of `instanceof`. The
|
|
20
|
+
* package resolves as raw `src` for typecheck and as built `dist` at runtime,
|
|
21
|
+
* so a value can cross a realm where the two class identities differ.
|
|
22
|
+
*/
|
|
23
|
+
readonly __webResearchError = true as const
|
|
24
|
+
readonly code: WebResearchErrorCode
|
|
25
|
+
readonly retryAfterMs?: number
|
|
26
|
+
readonly status?: number
|
|
27
|
+
|
|
28
|
+
constructor(code: WebResearchErrorCode, message: string, init: WebResearchErrorInit = {}) {
|
|
29
|
+
super(message, init.cause === undefined ? undefined : { cause: init.cause })
|
|
30
|
+
this.name = 'WebResearchError'
|
|
31
|
+
this.code = code
|
|
32
|
+
if (init.retryAfterMs !== undefined) this.retryAfterMs = init.retryAfterMs
|
|
33
|
+
if (init.status !== undefined) this.status = init.status
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function isWebResearchError(value: unknown): value is WebResearchError {
|
|
38
|
+
return (
|
|
39
|
+
typeof value === 'object' &&
|
|
40
|
+
value !== null &&
|
|
41
|
+
(value as { __webResearchError?: unknown }).__webResearchError === true
|
|
42
|
+
)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function errorMessage(value: unknown): string {
|
|
46
|
+
if (value instanceof Error) return value.message
|
|
47
|
+
return String(value)
|
|
48
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export type HttpMethod = 'GET' | 'POST'
|
|
2
|
+
|
|
3
|
+
export type HttpRequestInit = {
|
|
4
|
+
readonly method?: HttpMethod
|
|
5
|
+
readonly headers?: Readonly<Record<string, string>>
|
|
6
|
+
readonly body?: string
|
|
7
|
+
readonly maxBytes?: number
|
|
8
|
+
readonly timeoutMs?: number
|
|
9
|
+
readonly signal?: AbortSignal
|
|
10
|
+
/**
|
|
11
|
+
* Content-type prefixes the caller can handle. A response outside the list is
|
|
12
|
+
* rejected with `unsupported_content_type` instead of being decoded as text —
|
|
13
|
+
* without this a PDF or image comes back as mojibake the model then tries to read.
|
|
14
|
+
*/
|
|
15
|
+
readonly accept?: readonly string[]
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export type HttpResponse = {
|
|
19
|
+
/** Final URL after redirects, each of which was re-vetted. */
|
|
20
|
+
readonly url: string
|
|
21
|
+
readonly status: number
|
|
22
|
+
readonly headers: ReadonlyMap<string, string>
|
|
23
|
+
readonly contentType: string | null
|
|
24
|
+
readonly body: string
|
|
25
|
+
readonly truncated: boolean
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The single egress path. Adapters receive one of these and must never reach for
|
|
30
|
+
* `fetch` themselves — the SSRF guard, DNS pinning, redirect re-validation, byte
|
|
31
|
+
* caps and politeness live here, and a bypass silently loses all of them.
|
|
32
|
+
*/
|
|
33
|
+
export interface HttpClient {
|
|
34
|
+
request(url: string, init?: HttpRequestInit): Promise<HttpResponse>
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type Logger = {
|
|
38
|
+
debug(message: string, meta?: Record<string, unknown>): void
|
|
39
|
+
info(message: string, meta?: Record<string, unknown>): void
|
|
40
|
+
warn(message: string, meta?: Record<string, unknown>): void
|
|
41
|
+
error(message: string, meta?: Record<string, unknown>): void
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const noop = (): void => {}
|
|
45
|
+
|
|
46
|
+
export const silentLogger: Logger = { debug: noop, info: noop, warn: noop, error: noop }
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { isWebResearchError, errorMessage } from './errors'
|
|
2
|
+
import type { FetchedPage, RawResult } from './results'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Failure shapes shared by search and fetch. Keeping `unavailable`, `blocked`
|
|
6
|
+
* and `error` distinct is the point of the whole contract: the scheduler demotes
|
|
7
|
+
* an `unavailable` adapter for the rest of the run, escalates a `blocked` one to
|
|
8
|
+
* the browser tier, and retries only a retriable `error`. Collapsing them into a
|
|
9
|
+
* thrown exception — as the previous implementation did — makes all three
|
|
10
|
+
* indistinguishable and forces the caller to guess.
|
|
11
|
+
*/
|
|
12
|
+
export type OutcomeFailure =
|
|
13
|
+
| { readonly status: 'unavailable'; readonly reason: string }
|
|
14
|
+
| { readonly status: 'blocked'; readonly reason: string; readonly retryAfterMs?: number }
|
|
15
|
+
| { readonly status: 'timeout'; readonly reason?: string }
|
|
16
|
+
| { readonly status: 'error'; readonly reason: string; readonly retriable: boolean }
|
|
17
|
+
|
|
18
|
+
export type SearchOutcome =
|
|
19
|
+
| { readonly status: 'ok'; readonly results: readonly RawResult[]; readonly answer?: string }
|
|
20
|
+
| { readonly status: 'empty' }
|
|
21
|
+
| OutcomeFailure
|
|
22
|
+
|
|
23
|
+
export type FetchOutcome = { readonly status: 'ok'; readonly page: FetchedPage } | OutcomeFailure
|
|
24
|
+
|
|
25
|
+
export type OutcomeStatus = SearchOutcome['status']
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* A prose answer with no citations is still a successful search — the model
|
|
29
|
+
* knew something, it just did not link anywhere. Only a turn that produced
|
|
30
|
+
* neither links nor prose is `empty`.
|
|
31
|
+
*/
|
|
32
|
+
export function searchOk(results: readonly RawResult[], answer?: string): SearchOutcome {
|
|
33
|
+
const hasAnswer = answer !== undefined && answer.trim().length > 0
|
|
34
|
+
if (results.length === 0 && !hasAnswer) return { status: 'empty' }
|
|
35
|
+
return hasAnswer ? { status: 'ok', results, answer } : { status: 'ok', results }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function searchEmpty(): SearchOutcome {
|
|
39
|
+
return { status: 'empty' }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function unavailable(reason: string): OutcomeFailure {
|
|
43
|
+
return { status: 'unavailable', reason }
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function blocked(reason: string, retryAfterMs?: number): OutcomeFailure {
|
|
47
|
+
return retryAfterMs === undefined
|
|
48
|
+
? { status: 'blocked', reason }
|
|
49
|
+
: { status: 'blocked', reason, retryAfterMs }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function timedOut(reason?: string): OutcomeFailure {
|
|
53
|
+
return reason === undefined ? { status: 'timeout' } : { status: 'timeout', reason }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function failed(reason: string, retriable = false): OutcomeFailure {
|
|
57
|
+
return { status: 'error', reason, retriable }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function isFailure(outcome: SearchOutcome | FetchOutcome): outcome is OutcomeFailure {
|
|
61
|
+
return outcome.status !== 'ok' && outcome.status !== 'empty'
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Maps anything thrown by the shared HTTP client onto the outcome vocabulary, so
|
|
66
|
+
* an adapter body reduces to `try { … } catch (err) { return toOutcomeFailure(err) }`
|
|
67
|
+
* and never leaks an exception to the scheduler.
|
|
68
|
+
*/
|
|
69
|
+
export function toOutcomeFailure(error: unknown): OutcomeFailure {
|
|
70
|
+
if (isWebResearchError(error)) {
|
|
71
|
+
switch (error.code) {
|
|
72
|
+
case 'timeout':
|
|
73
|
+
return timedOut(error.message)
|
|
74
|
+
case 'aborted':
|
|
75
|
+
return timedOut(error.message)
|
|
76
|
+
case 'blocked':
|
|
77
|
+
return blocked(error.message, error.retryAfterMs)
|
|
78
|
+
case 'ssrf_blocked':
|
|
79
|
+
case 'unsupported_content_type':
|
|
80
|
+
return failed(error.message, false)
|
|
81
|
+
case 'network':
|
|
82
|
+
case 'bad_response':
|
|
83
|
+
case 'too_many_redirects':
|
|
84
|
+
return failed(error.message, true)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (error instanceof Error && error.name === 'AbortError') {
|
|
88
|
+
return timedOut(error.message)
|
|
89
|
+
}
|
|
90
|
+
return failed(errorMessage(error), false)
|
|
91
|
+
}
|