@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/.turbo/turbo-build.log +2 -0
  2. package/AGENTS.md +67 -0
  3. package/build.mjs +10 -0
  4. package/dist/contract/adapter.js +13 -0
  5. package/dist/contract/adapter.js.map +7 -0
  6. package/dist/contract/errors.js +28 -0
  7. package/dist/contract/errors.js.map +7 -0
  8. package/dist/contract/http.js +7 -0
  9. package/dist/contract/http.js.map +7 -0
  10. package/dist/contract/index.js +9 -0
  11. package/dist/contract/index.js.map +7 -0
  12. package/dist/contract/outcomes.js +58 -0
  13. package/dist/contract/outcomes.js.map +7 -0
  14. package/dist/contract/policy.js +91 -0
  15. package/dist/contract/policy.js.map +7 -0
  16. package/dist/contract/results.js +13 -0
  17. package/dist/contract/results.js.map +7 -0
  18. package/dist/contract/steps.js +1 -0
  19. package/dist/contract/steps.js.map +7 -0
  20. package/dist/contract/version.js +11 -0
  21. package/dist/contract/version.js.map +7 -0
  22. package/dist/engine/cache.js +29 -0
  23. package/dist/engine/cache.js.map +7 -0
  24. package/dist/engine/deadline.js +54 -0
  25. package/dist/engine/deadline.js.map +7 -0
  26. package/dist/engine/engine.js +481 -0
  27. package/dist/engine/engine.js.map +7 -0
  28. package/dist/engine/index.js +20 -0
  29. package/dist/engine/index.js.map +7 -0
  30. package/dist/engine/runAdapter.js +57 -0
  31. package/dist/engine/runAdapter.js.map +7 -0
  32. package/dist/engine/types.js +1 -0
  33. package/dist/engine/types.js.map +7 -0
  34. package/dist/extract/classify.js +51 -0
  35. package/dist/extract/classify.js.map +7 -0
  36. package/dist/extract/content.js +104 -0
  37. package/dist/extract/content.js.map +7 -0
  38. package/dist/extract/entities.js +95 -0
  39. package/dist/extract/entities.js.map +7 -0
  40. package/dist/extract/index.js +24 -0
  41. package/dist/extract/index.js.map +7 -0
  42. package/dist/extract/text.js +114 -0
  43. package/dist/extract/text.js.map +7 -0
  44. package/dist/extract/tokenizer.js +141 -0
  45. package/dist/extract/tokenizer.js.map +7 -0
  46. package/dist/fusion/canonicalize.js +88 -0
  47. package/dist/fusion/canonicalize.js.map +7 -0
  48. package/dist/fusion/fuse.js +88 -0
  49. package/dist/fusion/fuse.js.map +7 -0
  50. package/dist/fusion/index.js +11 -0
  51. package/dist/fusion/index.js.map +7 -0
  52. package/dist/index.js +21 -0
  53. package/dist/index.js.map +7 -0
  54. package/dist/loader/describe.js +92 -0
  55. package/dist/loader/describe.js.map +7 -0
  56. package/dist/loader/index.js +31 -0
  57. package/dist/loader/index.js.map +7 -0
  58. package/dist/loader/manifest.js +32 -0
  59. package/dist/loader/manifest.js.map +7 -0
  60. package/dist/loader/registry.js +98 -0
  61. package/dist/loader/registry.js.map +7 -0
  62. package/dist/net/client.js +298 -0
  63. package/dist/net/client.js.map +7 -0
  64. package/dist/net/ssrf.js +165 -0
  65. package/dist/net/ssrf.js.map +7 -0
  66. package/dist/net/url.js +9 -0
  67. package/dist/net/url.js.map +7 -0
  68. package/dist/testing/contract.js +118 -0
  69. package/dist/testing/contract.js.map +7 -0
  70. package/dist/testing/fakeAdapter.js +44 -0
  71. package/dist/testing/fakeAdapter.js.map +7 -0
  72. package/dist/testing/index.js +15 -0
  73. package/dist/testing/index.js.map +7 -0
  74. package/jest.config.cjs +24 -0
  75. package/package.json +65 -0
  76. package/src/contract/adapter.ts +100 -0
  77. package/src/contract/errors.ts +48 -0
  78. package/src/contract/http.ts +46 -0
  79. package/src/contract/index.ts +8 -0
  80. package/src/contract/outcomes.ts +91 -0
  81. package/src/contract/policy.ts +157 -0
  82. package/src/contract/results.ts +79 -0
  83. package/src/contract/steps.ts +45 -0
  84. package/src/contract/version.ts +25 -0
  85. package/src/engine/__tests__/engine.test.ts +597 -0
  86. package/src/engine/cache.ts +46 -0
  87. package/src/engine/deadline.ts +65 -0
  88. package/src/engine/engine.ts +633 -0
  89. package/src/engine/index.ts +22 -0
  90. package/src/engine/runAdapter.ts +89 -0
  91. package/src/engine/types.ts +137 -0
  92. package/src/extract/__tests__/content.test.ts +81 -0
  93. package/src/extract/__tests__/text.test.ts +61 -0
  94. package/src/extract/__tests__/tokenizer.test.ts +85 -0
  95. package/src/extract/classify.ts +81 -0
  96. package/src/extract/content.ts +136 -0
  97. package/src/extract/entities.ts +98 -0
  98. package/src/extract/index.ts +11 -0
  99. package/src/extract/text.ts +124 -0
  100. package/src/extract/tokenizer.ts +197 -0
  101. package/src/fusion/__tests__/fusion.test.ts +198 -0
  102. package/src/fusion/canonicalize.ts +119 -0
  103. package/src/fusion/fuse.ts +145 -0
  104. package/src/fusion/index.ts +8 -0
  105. package/src/index.ts +15 -0
  106. package/src/loader/__tests__/describe.test.ts +119 -0
  107. package/src/loader/__tests__/registry.test.ts +134 -0
  108. package/src/loader/describe.ts +145 -0
  109. package/src/loader/index.ts +25 -0
  110. package/src/loader/manifest.ts +44 -0
  111. package/src/loader/registry.ts +156 -0
  112. package/src/net/__tests__/client.test.ts +273 -0
  113. package/src/net/__tests__/ssrf.test.ts +165 -0
  114. package/src/net/client.ts +434 -0
  115. package/src/net/ssrf.ts +235 -0
  116. package/src/net/url.ts +13 -0
  117. package/src/testing/contract.ts +172 -0
  118. package/src/testing/fakeAdapter.ts +69 -0
  119. package/src/testing/index.ts +9 -0
  120. package/tsconfig.json +10 -0
  121. package/watch.mjs +7 -0
@@ -0,0 +1,118 @@
1
+ import { silentLogger } from "../contract/http.js";
2
+ import { CONTRACT_VERSION } from "../contract/version.js";
3
+ function createStubHttpClient(handler) {
4
+ const calls = [];
5
+ return {
6
+ calls,
7
+ async request(url, init) {
8
+ calls.push(url);
9
+ const stub = await handler(url, init);
10
+ return {
11
+ url: stub.url ?? url,
12
+ status: stub.status ?? 200,
13
+ headers: stub.headers ?? /* @__PURE__ */ new Map(),
14
+ contentType: stub.contentType ?? "text/html",
15
+ body: stub.body,
16
+ truncated: stub.truncated ?? false
17
+ };
18
+ }
19
+ };
20
+ }
21
+ function createTestContext(overrides = {}) {
22
+ return {
23
+ signal: new AbortController().signal,
24
+ http: createStubHttpClient(() => ({ body: "" })),
25
+ report: () => {
26
+ },
27
+ deadlineAt: Date.now() + 1e4,
28
+ logger: silentLogger,
29
+ ...overrides
30
+ };
31
+ }
32
+ const SAMPLE_REQUEST = { query: "open mercato", limit: 5 };
33
+ const VALID_STATUSES = ["ok", "empty", "unavailable", "blocked", "timeout", "error"];
34
+ function describeAdapterContract(module, testCase, api) {
35
+ const { describe, it, expect } = api;
36
+ describe(`${module.id} adapter contract`, () => {
37
+ it("declares the contract version it was built against", () => {
38
+ expect(module.contractVersion).toBe(CONTRACT_VERSION);
39
+ });
40
+ it("exposes a stable id and kind", () => {
41
+ expect(typeof module.id).toBe("string");
42
+ expect(module.id.length > 0).toBe(true);
43
+ expect(["serp", "api", "model", "browser"].includes(module.kind)).toBe(true);
44
+ });
45
+ it("validates its own options", () => {
46
+ const parsed = module.optionsSchema.safeParse(testCase.configuredOptions);
47
+ expect(parsed.success).toBe(true);
48
+ });
49
+ it("reports readiness without performing I/O", () => {
50
+ const adapter = module.createAdapter(testCase.configuredOptions);
51
+ const readiness = adapter.readiness();
52
+ expect(typeof readiness.ready).toBe("boolean");
53
+ });
54
+ const unconfigured = testCase.unconfiguredOptions;
55
+ if (unconfigured !== void 0) {
56
+ it("is unavailable rather than broken when unconfigured", async () => {
57
+ const adapter = module.createAdapter(unconfigured);
58
+ const readiness = adapter.readiness();
59
+ expect(readiness.ready).toBe(false);
60
+ if (!readiness.ready) expect(readiness.reason.length > 0).toBe(true);
61
+ const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext());
62
+ expect(outcome.status).toBe("unavailable");
63
+ });
64
+ }
65
+ it("returns an outcome rather than throwing when egress fails", async () => {
66
+ const adapter = module.createAdapter(testCase.configuredOptions);
67
+ const context = createTestContext({
68
+ http: createStubHttpClient(() => {
69
+ throw new Error("network exploded");
70
+ })
71
+ });
72
+ const outcome = await adapter.search(SAMPLE_REQUEST, context);
73
+ expect(VALID_STATUSES.includes(outcome.status)).toBe(true);
74
+ });
75
+ it("honours an already-aborted signal without hanging", async () => {
76
+ const adapter = module.createAdapter(testCase.configuredOptions);
77
+ const controller = new AbortController();
78
+ controller.abort();
79
+ const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext({ signal: controller.signal }));
80
+ expect(typeof outcome.status).toBe("string");
81
+ });
82
+ it("never returns a result without a url", async () => {
83
+ const adapter = module.createAdapter(testCase.configuredOptions);
84
+ const context = testCase.context ? testCase.context() : createTestContext();
85
+ const outcome = await adapter.search(SAMPLE_REQUEST, context);
86
+ if (outcome.status !== "ok") return;
87
+ for (const result of outcome.results) {
88
+ expect(typeof result.url).toBe("string");
89
+ expect(result.url.length > 0).toBe(true);
90
+ }
91
+ });
92
+ it("tears down cleanly even when it was never used", async () => {
93
+ const adapter = module.createAdapter(testCase.configuredOptions);
94
+ if (!adapter.dispose) return;
95
+ await adapter.dispose();
96
+ await adapter.dispose();
97
+ expect(true).toBe(true);
98
+ });
99
+ it("answers a health check without throwing", async () => {
100
+ const adapter = module.createAdapter(testCase.configuredOptions);
101
+ if (!adapter.healthCheck) return;
102
+ const context = createTestContext({
103
+ http: createStubHttpClient(() => {
104
+ throw new Error("network exploded");
105
+ })
106
+ });
107
+ const health = await adapter.healthCheck(context);
108
+ expect(typeof health.ok).toBe("boolean");
109
+ });
110
+ });
111
+ }
112
+ export {
113
+ SAMPLE_REQUEST,
114
+ createStubHttpClient,
115
+ createTestContext,
116
+ describeAdapterContract
117
+ };
118
+ //# sourceMappingURL=contract.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/testing/contract.ts"],
4
+ "sourcesContent": ["import type { AdapterContext, AdapterModule, SearchAdapter } from '../contract/adapter'\nimport { silentLogger, type HttpClient, type HttpResponse } from '../contract/http'\nimport { CONTRACT_VERSION } from '../contract/version'\nimport type { SearchRequest } from '../contract/results'\n\nexport type StubResponse = Partial<HttpResponse> & { readonly body: string }\n\n/**\n * Minimal HTTP double. Adapters are contractually forbidden from reaching for\n * `fetch`, so a test that passes with this client has also proved the adapter\n * routes every byte through the hardened transport.\n */\nexport function createStubHttpClient(\n handler: (url: string, init: Parameters<HttpClient['request']>[1]) => StubResponse | Promise<StubResponse>,\n): HttpClient & { readonly calls: readonly string[] } {\n const calls: string[] = []\n return {\n calls,\n async request(url, init) {\n calls.push(url)\n const stub = await handler(url, init)\n return {\n url: stub.url ?? url,\n status: stub.status ?? 200,\n headers: stub.headers ?? new Map<string, string>(),\n contentType: stub.contentType ?? 'text/html',\n body: stub.body,\n truncated: stub.truncated ?? false,\n }\n },\n }\n}\n\nexport function createTestContext(overrides: Partial<AdapterContext> = {}): AdapterContext {\n return {\n signal: new AbortController().signal,\n http: createStubHttpClient(() => ({ body: '' })),\n report: () => {},\n deadlineAt: Date.now() + 10_000,\n logger: silentLogger,\n ...overrides,\n }\n}\n\nexport const SAMPLE_REQUEST: SearchRequest = { query: 'open mercato', limit: 5 }\n\nconst VALID_STATUSES = ['ok', 'empty', 'unavailable', 'blocked', 'timeout', 'error']\n\nexport type AdapterContractCase<TOptions> = {\n /** Options that leave the adapter deliberately unconfigured. */\n readonly unconfiguredOptions?: TOptions\n /** Options sufficient for the adapter to consider itself ready. */\n readonly configuredOptions: TOptions\n /** Context used for the \"returns results\" probe; stub the HTTP client here. */\n readonly context?: () => AdapterContext\n}\n\ntype TestApi = {\n describe: (name: string, body: () => void) => void\n it: (name: string, body: () => void | Promise<void>) => void\n expect: (value: unknown) => {\n toBe(expected: unknown): void\n toBeDefined(): void\n toContain(expected: string): void\n }\n}\n\n/**\n * Behavioural suite every adapter must pass, ours and third parties'. It pins the\n * parts of the contract the scheduler actually depends on \u2014 most importantly that\n * an adapter reports rather than throws, because a throwing adapter would\n * otherwise take down an entire search.\n */\nexport function describeAdapterContract<TOptions>(\n module: AdapterModule<TOptions>,\n testCase: AdapterContractCase<TOptions>,\n api: TestApi,\n): void {\n const { describe, it, expect } = api\n\n describe(`${module.id} adapter contract`, () => {\n it('declares the contract version it was built against', () => {\n expect(module.contractVersion).toBe(CONTRACT_VERSION)\n })\n\n it('exposes a stable id and kind', () => {\n expect(typeof module.id).toBe('string')\n expect(module.id.length > 0).toBe(true)\n expect(['serp', 'api', 'model', 'browser'].includes(module.kind)).toBe(true)\n })\n\n it('validates its own options', () => {\n const parsed = module.optionsSchema.safeParse(testCase.configuredOptions)\n expect(parsed.success).toBe(true)\n })\n\n it('reports readiness without performing I/O', () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n const readiness = adapter.readiness()\n expect(typeof readiness.ready).toBe('boolean')\n })\n\n const unconfigured = testCase.unconfiguredOptions\n if (unconfigured !== undefined) {\n it('is unavailable rather than broken when unconfigured', async () => {\n const adapter = module.createAdapter(unconfigured)\n const readiness = adapter.readiness()\n expect(readiness.ready).toBe(false)\n if (!readiness.ready) expect(readiness.reason.length > 0).toBe(true)\n\n const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext())\n expect(outcome.status).toBe('unavailable')\n })\n }\n\n // The contract is \"never throws\", not \"always fails when HTTP does\" \u2014 a model\n // or browser adapter may not touch the HTTP client at all.\n it('returns an outcome rather than throwing when egress fails', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n const context = createTestContext({\n http: createStubHttpClient(() => {\n throw new Error('network exploded')\n }),\n })\n const outcome = await adapter.search(SAMPLE_REQUEST, context)\n expect(VALID_STATUSES.includes(outcome.status)).toBe(true)\n })\n\n it('honours an already-aborted signal without hanging', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n const controller = new AbortController()\n controller.abort()\n const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext({ signal: controller.signal }))\n expect(typeof outcome.status).toBe('string')\n })\n\n it('never returns a result without a url', async () => {\n const adapter: SearchAdapter = module.createAdapter(testCase.configuredOptions)\n const context = testCase.context ? testCase.context() : createTestContext()\n const outcome = await adapter.search(SAMPLE_REQUEST, context)\n if (outcome.status !== 'ok') return\n for (const result of outcome.results) {\n expect(typeof result.url).toBe('string')\n expect(result.url.length > 0).toBe(true)\n }\n })\n\n // The host builds an engine per request and disposes it after, so a `dispose`\n // that throws on an adapter which was never used, or on a second call, breaks\n // teardown for every other adapter in the same engine. This is the surface\n // that leaked a browser sidecar and a Chromium per tool call.\n it('tears down cleanly even when it was never used', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n if (!adapter.dispose) return\n await adapter.dispose()\n await adapter.dispose()\n expect(true).toBe(true)\n })\n\n it('answers a health check without throwing', async () => {\n const adapter = module.createAdapter(testCase.configuredOptions)\n if (!adapter.healthCheck) return\n const context = createTestContext({\n http: createStubHttpClient(() => {\n throw new Error('network exploded')\n }),\n })\n const health = await adapter.healthCheck(context)\n expect(typeof health.ok).toBe('boolean')\n })\n })\n}\n"],
5
+ "mappings": "AACA,SAAS,oBAAwD;AACjE,SAAS,wBAAwB;AAU1B,SAAS,qBACd,SACoD;AACpD,QAAM,QAAkB,CAAC;AACzB,SAAO;AAAA,IACL;AAAA,IACA,MAAM,QAAQ,KAAK,MAAM;AACvB,YAAM,KAAK,GAAG;AACd,YAAM,OAAO,MAAM,QAAQ,KAAK,IAAI;AACpC,aAAO;AAAA,QACL,KAAK,KAAK,OAAO;AAAA,QACjB,QAAQ,KAAK,UAAU;AAAA,QACvB,SAAS,KAAK,WAAW,oBAAI,IAAoB;AAAA,QACjD,aAAa,KAAK,eAAe;AAAA,QACjC,MAAM,KAAK;AAAA,QACX,WAAW,KAAK,aAAa;AAAA,MAC/B;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,kBAAkB,YAAqC,CAAC,GAAmB;AACzF,SAAO;AAAA,IACL,QAAQ,IAAI,gBAAgB,EAAE;AAAA,IAC9B,MAAM,qBAAqB,OAAO,EAAE,MAAM,GAAG,EAAE;AAAA,IAC/C,QAAQ,MAAM;AAAA,IAAC;AAAA,IACf,YAAY,KAAK,IAAI,IAAI;AAAA,IACzB,QAAQ;AAAA,IACR,GAAG;AAAA,EACL;AACF;AAEO,MAAM,iBAAgC,EAAE,OAAO,gBAAgB,OAAO,EAAE;AAE/E,MAAM,iBAAiB,CAAC,MAAM,SAAS,eAAe,WAAW,WAAW,OAAO;AA2B5E,SAAS,wBACd,QACA,UACA,KACM;AACN,QAAM,EAAE,UAAU,IAAI,OAAO,IAAI;AAEjC,WAAS,GAAG,OAAO,EAAE,qBAAqB,MAAM;AAC9C,OAAG,sDAAsD,MAAM;AAC7D,aAAO,OAAO,eAAe,EAAE,KAAK,gBAAgB;AAAA,IACtD,CAAC;AAED,OAAG,gCAAgC,MAAM;AACvC,aAAO,OAAO,OAAO,EAAE,EAAE,KAAK,QAAQ;AACtC,aAAO,OAAO,GAAG,SAAS,CAAC,EAAE,KAAK,IAAI;AACtC,aAAO,CAAC,QAAQ,OAAO,SAAS,SAAS,EAAE,SAAS,OAAO,IAAI,CAAC,EAAE,KAAK,IAAI;AAAA,IAC7E,CAAC;AAED,OAAG,6BAA6B,MAAM;AACpC,YAAM,SAAS,OAAO,cAAc,UAAU,SAAS,iBAAiB;AACxE,aAAO,OAAO,OAAO,EAAE,KAAK,IAAI;AAAA,IAClC,CAAC;AAED,OAAG,4CAA4C,MAAM;AACnD,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,YAAM,YAAY,QAAQ,UAAU;AACpC,aAAO,OAAO,UAAU,KAAK,EAAE,KAAK,SAAS;AAAA,IAC/C,CAAC;AAED,UAAM,eAAe,SAAS;AAC9B,QAAI,iBAAiB,QAAW;AAC9B,SAAG,uDAAuD,YAAY;AACpE,cAAM,UAAU,OAAO,cAAc,YAAY;AACjD,cAAM,YAAY,QAAQ,UAAU;AACpC,eAAO,UAAU,KAAK,EAAE,KAAK,KAAK;AAClC,YAAI,CAAC,UAAU,MAAO,QAAO,UAAU,OAAO,SAAS,CAAC,EAAE,KAAK,IAAI;AAEnE,cAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,kBAAkB,CAAC;AACxE,eAAO,QAAQ,MAAM,EAAE,KAAK,aAAa;AAAA,MAC3C,CAAC;AAAA,IACH;AAIA,OAAG,6DAA6D,YAAY;AAC1E,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,YAAM,UAAU,kBAAkB;AAAA,QAChC,MAAM,qBAAqB,MAAM;AAC/B,gBAAM,IAAI,MAAM,kBAAkB;AAAA,QACpC,CAAC;AAAA,MACH,CAAC;AACD,YAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,OAAO;AAC5D,aAAO,eAAe,SAAS,QAAQ,MAAM,CAAC,EAAE,KAAK,IAAI;AAAA,IAC3D,CAAC;AAED,OAAG,qDAAqD,YAAY;AAClE,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,YAAM,aAAa,IAAI,gBAAgB;AACvC,iBAAW,MAAM;AACjB,YAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,kBAAkB,EAAE,QAAQ,WAAW,OAAO,CAAC,CAAC;AACrG,aAAO,OAAO,QAAQ,MAAM,EAAE,KAAK,QAAQ;AAAA,IAC7C,CAAC;AAED,OAAG,wCAAwC,YAAY;AACrD,YAAM,UAAyB,OAAO,cAAc,SAAS,iBAAiB;AAC9E,YAAM,UAAU,SAAS,UAAU,SAAS,QAAQ,IAAI,kBAAkB;AAC1E,YAAM,UAAU,MAAM,QAAQ,OAAO,gBAAgB,OAAO;AAC5D,UAAI,QAAQ,WAAW,KAAM;AAC7B,iBAAW,UAAU,QAAQ,SAAS;AACpC,eAAO,OAAO,OAAO,GAAG,EAAE,KAAK,QAAQ;AACvC,eAAO,OAAO,IAAI,SAAS,CAAC,EAAE,KAAK,IAAI;AAAA,MACzC;AAAA,IACF,CAAC;AAMD,OAAG,kDAAkD,YAAY;AAC/D,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,UAAI,CAAC,QAAQ,QAAS;AACtB,YAAM,QAAQ,QAAQ;AACtB,YAAM,QAAQ,QAAQ;AACtB,aAAO,IAAI,EAAE,KAAK,IAAI;AAAA,IACxB,CAAC;AAED,OAAG,2CAA2C,YAAY;AACxD,YAAM,UAAU,OAAO,cAAc,SAAS,iBAAiB;AAC/D,UAAI,CAAC,QAAQ,YAAa;AAC1B,YAAM,UAAU,kBAAkB;AAAA,QAChC,MAAM,qBAAqB,MAAM;AAC/B,gBAAM,IAAI,MAAM,kBAAkB;AAAA,QACpC,CAAC;AAAA,MACH,CAAC;AACD,YAAM,SAAS,MAAM,QAAQ,YAAY,OAAO;AAChD,aAAO,OAAO,OAAO,EAAE,EAAE,KAAK,SAAS;AAAA,IACzC,CAAC;AAAA,EACH,CAAC;AACH;",
6
+ "names": []
7
+ }
@@ -0,0 +1,44 @@
1
+ import { READY, notReady } from "../contract/adapter.js";
2
+ const BASE_CAPABILITIES = {
3
+ search: true,
4
+ fetch: false,
5
+ snippets: true,
6
+ content: false,
7
+ freshness: false,
8
+ siteFilter: false,
9
+ cost: "free"
10
+ };
11
+ function createFakeAdapter(options) {
12
+ const capabilities = { ...BASE_CAPABILITIES, ...options.capabilities };
13
+ const outcome = options.outcome ?? (options.results && options.results.length > 0 ? { status: "ok", results: options.results } : { status: "empty" });
14
+ return {
15
+ id: options.id,
16
+ kind: options.kind ?? "api",
17
+ capabilities,
18
+ readiness: () => options.ready === false ? notReady("fake adapter configured as not ready") : typeof options.ready === "string" ? notReady(options.ready) : READY,
19
+ async search(_request, context) {
20
+ if (options.throws) throw new Error(`${options.id} exploded`);
21
+ const latency = options.latencyMs ?? 0;
22
+ if (latency > 0) {
23
+ await new Promise((resolve) => {
24
+ const timer = setTimeout(resolve, latency);
25
+ timer.unref?.();
26
+ context.signal.addEventListener(
27
+ "abort",
28
+ () => {
29
+ clearTimeout(timer);
30
+ resolve();
31
+ },
32
+ { once: true }
33
+ );
34
+ });
35
+ }
36
+ if (context.signal.aborted) return { status: "timeout", reason: "aborted" };
37
+ return outcome;
38
+ }
39
+ };
40
+ }
41
+ export {
42
+ createFakeAdapter
43
+ };
44
+ //# sourceMappingURL=fakeAdapter.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/testing/fakeAdapter.ts"],
4
+ "sourcesContent": ["import type { AdapterCapabilities, AdapterKind, SearchAdapter } from '../contract/adapter'\nimport { READY, notReady } from '../contract/adapter'\nimport type { SearchOutcome } from '../contract/outcomes'\nimport type { RawResult } from '../contract/results'\n\nexport type FakeAdapterOptions = {\n readonly id: string\n readonly kind?: AdapterKind\n readonly ready?: boolean | string\n /** Milliseconds of simulated work before the outcome resolves. */\n readonly latencyMs?: number\n readonly outcome?: SearchOutcome\n readonly results?: readonly RawResult[]\n readonly capabilities?: Partial<AdapterCapabilities>\n /** Set to make the adapter violate the contract, for isolation tests. */\n readonly throws?: boolean\n}\n\nconst BASE_CAPABILITIES: AdapterCapabilities = {\n search: true,\n fetch: false,\n snippets: true,\n content: false,\n freshness: false,\n siteFilter: false,\n cost: 'free',\n}\n\n/** Deterministic adapter double for scheduler and fusion tests. */\nexport function createFakeAdapter(options: FakeAdapterOptions): SearchAdapter {\n const capabilities: AdapterCapabilities = { ...BASE_CAPABILITIES, ...options.capabilities }\n const outcome: SearchOutcome =\n options.outcome ??\n (options.results && options.results.length > 0\n ? { status: 'ok', results: options.results }\n : { status: 'empty' })\n\n return {\n id: options.id,\n kind: options.kind ?? 'api',\n capabilities,\n readiness: () =>\n options.ready === false\n ? notReady('fake adapter configured as not ready')\n : typeof options.ready === 'string'\n ? notReady(options.ready)\n : READY,\n async search(_request, context) {\n if (options.throws) throw new Error(`${options.id} exploded`)\n const latency = options.latencyMs ?? 0\n if (latency > 0) {\n await new Promise<void>((resolve) => {\n const timer = setTimeout(resolve, latency)\n timer.unref?.()\n context.signal.addEventListener(\n 'abort',\n () => {\n clearTimeout(timer)\n resolve()\n },\n { once: true },\n )\n })\n }\n if (context.signal.aborted) return { status: 'timeout', reason: 'aborted' }\n return outcome\n },\n }\n}\n"],
5
+ "mappings": "AACA,SAAS,OAAO,gBAAgB;AAiBhC,MAAM,oBAAyC;AAAA,EAC7C,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,UAAU;AAAA,EACV,SAAS;AAAA,EACT,WAAW;AAAA,EACX,YAAY;AAAA,EACZ,MAAM;AACR;AAGO,SAAS,kBAAkB,SAA4C;AAC5E,QAAM,eAAoC,EAAE,GAAG,mBAAmB,GAAG,QAAQ,aAAa;AAC1F,QAAM,UACJ,QAAQ,YACP,QAAQ,WAAW,QAAQ,QAAQ,SAAS,IACzC,EAAE,QAAQ,MAAM,SAAS,QAAQ,QAAQ,IACzC,EAAE,QAAQ,QAAQ;AAExB,SAAO;AAAA,IACL,IAAI,QAAQ;AAAA,IACZ,MAAM,QAAQ,QAAQ;AAAA,IACtB;AAAA,IACA,WAAW,MACT,QAAQ,UAAU,QACd,SAAS,sCAAsC,IAC/C,OAAO,QAAQ,UAAU,WACvB,SAAS,QAAQ,KAAK,IACtB;AAAA,IACR,MAAM,OAAO,UAAU,SAAS;AAC9B,UAAI,QAAQ,OAAQ,OAAM,IAAI,MAAM,GAAG,QAAQ,EAAE,WAAW;AAC5D,YAAM,UAAU,QAAQ,aAAa;AACrC,UAAI,UAAU,GAAG;AACf,cAAM,IAAI,QAAc,CAAC,YAAY;AACnC,gBAAM,QAAQ,WAAW,SAAS,OAAO;AACzC,gBAAM,QAAQ;AACd,kBAAQ,OAAO;AAAA,YACb;AAAA,YACA,MAAM;AACJ,2BAAa,KAAK;AAClB,sBAAQ;AAAA,YACV;AAAA,YACA,EAAE,MAAM,KAAK;AAAA,UACf;AAAA,QACF,CAAC;AAAA,MACH;AACA,UAAI,QAAQ,OAAO,QAAS,QAAO,EAAE,QAAQ,WAAW,QAAQ,UAAU;AAC1E,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
6
+ "names": []
7
+ }
@@ -0,0 +1,15 @@
1
+ import {
2
+ SAMPLE_REQUEST,
3
+ createStubHttpClient,
4
+ createTestContext,
5
+ describeAdapterContract
6
+ } from "./contract.js";
7
+ import { createFakeAdapter } from "./fakeAdapter.js";
8
+ export {
9
+ SAMPLE_REQUEST,
10
+ createFakeAdapter,
11
+ createStubHttpClient,
12
+ createTestContext,
13
+ describeAdapterContract
14
+ };
15
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1,7 @@
1
+ {
2
+ "version": 3,
3
+ "sources": ["../../src/testing/index.ts"],
4
+ "sourcesContent": ["export {\n SAMPLE_REQUEST,\n createStubHttpClient,\n createTestContext,\n describeAdapterContract,\n type AdapterContractCase,\n type StubResponse,\n} from './contract'\nexport { createFakeAdapter, type FakeAdapterOptions } from './fakeAdapter'\n"],
5
+ "mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAGK;AACP,SAAS,yBAAkD;",
6
+ "names": []
7
+ }
@@ -0,0 +1,24 @@
1
+ /** @type {import('jest').Config} */
2
+ const base = require('../../jest.config.base.cjs')
3
+
4
+ module.exports = {
5
+ ...base,
6
+ preset: 'ts-jest',
7
+ testEnvironment: 'node',
8
+ watchman: false,
9
+ rootDir: '.',
10
+ moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
11
+ transform: {
12
+ '^.+\\.(t|j)sx?$': [
13
+ 'ts-jest',
14
+ {
15
+ tsconfig: {
16
+ rootDir: '.',
17
+ ignoreDeprecations: '6.0',
18
+ },
19
+ },
20
+ ],
21
+ },
22
+ testMatch: ['<rootDir>/src/**/__tests__/**/*.test.(ts|tsx)'],
23
+ passWithNoTests: true,
24
+ }
package/package.json ADDED
@@ -0,0 +1,65 @@
1
+ {
2
+ "name": "@open-mercato/web-research",
3
+ "version": "0.7.1-develop.7194.1.ab4fc81f82",
4
+ "license": "MIT",
5
+ "type": "module",
6
+ "main": "./dist/index.js",
7
+ "scripts": {
8
+ "build": "node build.mjs",
9
+ "watch": "node watch.mjs",
10
+ "test": "jest --config jest.config.cjs",
11
+ "typecheck": "tsc --noEmit"
12
+ },
13
+ "exports": {
14
+ ".": {
15
+ "types": "./src/index.ts",
16
+ "default": "./dist/index.js"
17
+ },
18
+ "./*.ts": {
19
+ "types": "./src/*.ts",
20
+ "default": "./dist/*.js"
21
+ },
22
+ "./*": {
23
+ "types": [
24
+ "./src/*.ts",
25
+ "./src/*.tsx"
26
+ ],
27
+ "default": "./dist/*.js"
28
+ },
29
+ "./*/*": {
30
+ "types": [
31
+ "./src/*/*.ts",
32
+ "./src/*/*.tsx"
33
+ ],
34
+ "default": "./dist/*/*.js"
35
+ },
36
+ "./*/*/*": {
37
+ "types": [
38
+ "./src/*/*/*.ts",
39
+ "./src/*/*/*.tsx"
40
+ ],
41
+ "default": "./dist/*/*/*.js"
42
+ }
43
+ },
44
+ "dependencies": {
45
+ "zod": "^4.4.3"
46
+ },
47
+ "devDependencies": {
48
+ "@types/jest": "^30.0.0",
49
+ "@types/node": "^26.0.1",
50
+ "esbuild": "^0.28.1",
51
+ "glob": "^13.0.6",
52
+ "jest": "^30.4.2",
53
+ "ts-jest": "^29.4.11",
54
+ "typescript": "^6.0.3"
55
+ },
56
+ "publishConfig": {
57
+ "access": "public"
58
+ },
59
+ "repository": {
60
+ "type": "git",
61
+ "url": "https://github.com/open-mercato/open-mercato",
62
+ "directory": "packages/web-research"
63
+ },
64
+ "stableVersion": "0.7.0"
65
+ }
@@ -0,0 +1,100 @@
1
+ import type { z } from 'zod'
2
+ import type { HttpClient, Logger } from './http'
3
+ import type { FetchOutcome, SearchOutcome } from './outcomes'
4
+ import type { FetchRequest, SearchRequest } from './results'
5
+ import type { StepReporter } from './steps'
6
+
7
+ export type AdapterKind = 'serp' | 'api' | 'model' | 'browser'
8
+
9
+ export type AdapterCost = 'free' | 'metered' | 'llm-tokens'
10
+
11
+ /**
12
+ * What calling `healthCheck()` costs — which is NOT what searching costs.
13
+ *
14
+ * Exa's probe reads a local key and Model-native's resolves configuration, both
15
+ * free, though both are priced adapters to search with. Firecrawl's and Tavily's
16
+ * probes issue a real query and bill for it. The browser tier's spawns an OS
17
+ * process: no money, but not something to run on every page view either.
18
+ *
19
+ * A status surface may run `free` probes unattended; `heavy` and `billable` need
20
+ * a human to ask for them.
21
+ */
22
+ export type ProbeCost = 'free' | 'heavy' | 'billable'
23
+
24
+ export type AdapterCapabilities = {
25
+ readonly search: boolean
26
+ readonly fetch: boolean
27
+ /** Returns real snippets rather than bare URLs. */
28
+ readonly snippets: boolean
29
+ /** Returns full page text inline, sparing a follow-up fetch. */
30
+ readonly content: boolean
31
+ readonly freshness: boolean
32
+ readonly siteFilter: boolean
33
+ readonly cost: AdapterCost
34
+ }
35
+
36
+ export type AdapterReadiness =
37
+ | { readonly ready: true }
38
+ | { readonly ready: false; readonly reason: string }
39
+
40
+ export type AdapterHealth = {
41
+ readonly ok: boolean
42
+ readonly detail?: string
43
+ readonly latencyMs?: number
44
+ }
45
+
46
+ export type AdapterContext = {
47
+ readonly signal: AbortSignal
48
+ readonly http: HttpClient
49
+ /** Narrates progress; the engine supplies the sequence, timestamp and adapter id. */
50
+ readonly report: StepReporter
51
+ /** Epoch ms after which the engine stops caring about this adapter's answer. */
52
+ readonly deadlineAt: number
53
+ readonly logger: Logger
54
+ }
55
+
56
+ export interface SearchAdapter {
57
+ readonly id: string
58
+ readonly kind: AdapterKind
59
+ readonly capabilities: AdapterCapabilities
60
+ /**
61
+ * Synchronous and I/O-free by contract, so planning a wave never blocks and an
62
+ * unconfigured adapter costs nothing to skip.
63
+ */
64
+ readiness(): AdapterReadiness
65
+ /**
66
+ * What `healthCheck()` costs to call. Omitting it is safe but pessimistic: the
67
+ * engine then derives `billable` from `capabilities.cost === 'metered'` and
68
+ * `free` otherwise, so an adapter that never declares it is simply left out of
69
+ * unattended probing. Declare it whenever the probe costs money.
70
+ */
71
+ readonly probeCost?: ProbeCost
72
+ search(request: SearchRequest, context: AdapterContext): Promise<SearchOutcome>
73
+ fetch?(request: FetchRequest, context: AdapterContext): Promise<FetchOutcome>
74
+ healthCheck?(context: AdapterContext): Promise<AdapterHealth>
75
+ dispose?(): Promise<void>
76
+ }
77
+
78
+ /**
79
+ * What an adapter package default-exports. A descriptor rather than an instance,
80
+ * because the engine builds one instance per tenant from that tenant's options.
81
+ */
82
+ export type AdapterModule<TOptions = unknown> = {
83
+ readonly id: string
84
+ readonly kind: AdapterKind
85
+ readonly contractVersion: number
86
+ readonly optionsSchema: z.ZodType<TOptions>
87
+ createAdapter(options: TOptions): SearchAdapter
88
+ }
89
+
90
+ export function defineAdapterModule<TOptions>(
91
+ module: AdapterModule<TOptions>,
92
+ ): AdapterModule<TOptions> {
93
+ return module
94
+ }
95
+
96
+ export const READY: AdapterReadiness = { ready: true }
97
+
98
+ export function notReady(reason: string): AdapterReadiness {
99
+ return { ready: false, reason }
100
+ }
@@ -0,0 +1,48 @@
1
+ export type WebResearchErrorCode =
2
+ | 'ssrf_blocked'
3
+ | 'timeout'
4
+ | 'aborted'
5
+ | 'blocked'
6
+ | 'bad_response'
7
+ | 'too_many_redirects'
8
+ | 'unsupported_content_type'
9
+ | 'network'
10
+
11
+ export type WebResearchErrorInit = {
12
+ readonly retryAfterMs?: number
13
+ readonly status?: number
14
+ readonly cause?: unknown
15
+ }
16
+
17
+ export class WebResearchError extends Error {
18
+ /**
19
+ * Brand checked by {@link isWebResearchError} instead of `instanceof`. The
20
+ * package resolves as raw `src` for typecheck and as built `dist` at runtime,
21
+ * so a value can cross a realm where the two class identities differ.
22
+ */
23
+ readonly __webResearchError = true as const
24
+ readonly code: WebResearchErrorCode
25
+ readonly retryAfterMs?: number
26
+ readonly status?: number
27
+
28
+ constructor(code: WebResearchErrorCode, message: string, init: WebResearchErrorInit = {}) {
29
+ super(message, init.cause === undefined ? undefined : { cause: init.cause })
30
+ this.name = 'WebResearchError'
31
+ this.code = code
32
+ if (init.retryAfterMs !== undefined) this.retryAfterMs = init.retryAfterMs
33
+ if (init.status !== undefined) this.status = init.status
34
+ }
35
+ }
36
+
37
+ export function isWebResearchError(value: unknown): value is WebResearchError {
38
+ return (
39
+ typeof value === 'object' &&
40
+ value !== null &&
41
+ (value as { __webResearchError?: unknown }).__webResearchError === true
42
+ )
43
+ }
44
+
45
+ export function errorMessage(value: unknown): string {
46
+ if (value instanceof Error) return value.message
47
+ return String(value)
48
+ }
@@ -0,0 +1,46 @@
1
+ export type HttpMethod = 'GET' | 'POST'
2
+
3
+ export type HttpRequestInit = {
4
+ readonly method?: HttpMethod
5
+ readonly headers?: Readonly<Record<string, string>>
6
+ readonly body?: string
7
+ readonly maxBytes?: number
8
+ readonly timeoutMs?: number
9
+ readonly signal?: AbortSignal
10
+ /**
11
+ * Content-type prefixes the caller can handle. A response outside the list is
12
+ * rejected with `unsupported_content_type` instead of being decoded as text —
13
+ * without this a PDF or image comes back as mojibake the model then tries to read.
14
+ */
15
+ readonly accept?: readonly string[]
16
+ }
17
+
18
+ export type HttpResponse = {
19
+ /** Final URL after redirects, each of which was re-vetted. */
20
+ readonly url: string
21
+ readonly status: number
22
+ readonly headers: ReadonlyMap<string, string>
23
+ readonly contentType: string | null
24
+ readonly body: string
25
+ readonly truncated: boolean
26
+ }
27
+
28
+ /**
29
+ * The single egress path. Adapters receive one of these and must never reach for
30
+ * `fetch` themselves — the SSRF guard, DNS pinning, redirect re-validation, byte
31
+ * caps and politeness live here, and a bypass silently loses all of them.
32
+ */
33
+ export interface HttpClient {
34
+ request(url: string, init?: HttpRequestInit): Promise<HttpResponse>
35
+ }
36
+
37
+ export type Logger = {
38
+ debug(message: string, meta?: Record<string, unknown>): void
39
+ info(message: string, meta?: Record<string, unknown>): void
40
+ warn(message: string, meta?: Record<string, unknown>): void
41
+ error(message: string, meta?: Record<string, unknown>): void
42
+ }
43
+
44
+ const noop = (): void => {}
45
+
46
+ export const silentLogger: Logger = { debug: noop, info: noop, warn: noop, error: noop }
@@ -0,0 +1,8 @@
1
+ export * from './adapter'
2
+ export * from './errors'
3
+ export * from './http'
4
+ export * from './outcomes'
5
+ export * from './policy'
6
+ export * from './results'
7
+ export * from './steps'
8
+ export * from './version'
@@ -0,0 +1,91 @@
1
+ import { isWebResearchError, errorMessage } from './errors'
2
+ import type { FetchedPage, RawResult } from './results'
3
+
4
+ /**
5
+ * Failure shapes shared by search and fetch. Keeping `unavailable`, `blocked`
6
+ * and `error` distinct is the point of the whole contract: the scheduler demotes
7
+ * an `unavailable` adapter for the rest of the run, escalates a `blocked` one to
8
+ * the browser tier, and retries only a retriable `error`. Collapsing them into a
9
+ * thrown exception — as the previous implementation did — makes all three
10
+ * indistinguishable and forces the caller to guess.
11
+ */
12
+ export type OutcomeFailure =
13
+ | { readonly status: 'unavailable'; readonly reason: string }
14
+ | { readonly status: 'blocked'; readonly reason: string; readonly retryAfterMs?: number }
15
+ | { readonly status: 'timeout'; readonly reason?: string }
16
+ | { readonly status: 'error'; readonly reason: string; readonly retriable: boolean }
17
+
18
+ export type SearchOutcome =
19
+ | { readonly status: 'ok'; readonly results: readonly RawResult[]; readonly answer?: string }
20
+ | { readonly status: 'empty' }
21
+ | OutcomeFailure
22
+
23
+ export type FetchOutcome = { readonly status: 'ok'; readonly page: FetchedPage } | OutcomeFailure
24
+
25
+ export type OutcomeStatus = SearchOutcome['status']
26
+
27
+ /**
28
+ * A prose answer with no citations is still a successful search — the model
29
+ * knew something, it just did not link anywhere. Only a turn that produced
30
+ * neither links nor prose is `empty`.
31
+ */
32
+ export function searchOk(results: readonly RawResult[], answer?: string): SearchOutcome {
33
+ const hasAnswer = answer !== undefined && answer.trim().length > 0
34
+ if (results.length === 0 && !hasAnswer) return { status: 'empty' }
35
+ return hasAnswer ? { status: 'ok', results, answer } : { status: 'ok', results }
36
+ }
37
+
38
+ export function searchEmpty(): SearchOutcome {
39
+ return { status: 'empty' }
40
+ }
41
+
42
+ export function unavailable(reason: string): OutcomeFailure {
43
+ return { status: 'unavailable', reason }
44
+ }
45
+
46
+ export function blocked(reason: string, retryAfterMs?: number): OutcomeFailure {
47
+ return retryAfterMs === undefined
48
+ ? { status: 'blocked', reason }
49
+ : { status: 'blocked', reason, retryAfterMs }
50
+ }
51
+
52
+ export function timedOut(reason?: string): OutcomeFailure {
53
+ return reason === undefined ? { status: 'timeout' } : { status: 'timeout', reason }
54
+ }
55
+
56
+ export function failed(reason: string, retriable = false): OutcomeFailure {
57
+ return { status: 'error', reason, retriable }
58
+ }
59
+
60
+ export function isFailure(outcome: SearchOutcome | FetchOutcome): outcome is OutcomeFailure {
61
+ return outcome.status !== 'ok' && outcome.status !== 'empty'
62
+ }
63
+
64
+ /**
65
+ * Maps anything thrown by the shared HTTP client onto the outcome vocabulary, so
66
+ * an adapter body reduces to `try { … } catch (err) { return toOutcomeFailure(err) }`
67
+ * and never leaks an exception to the scheduler.
68
+ */
69
+ export function toOutcomeFailure(error: unknown): OutcomeFailure {
70
+ if (isWebResearchError(error)) {
71
+ switch (error.code) {
72
+ case 'timeout':
73
+ return timedOut(error.message)
74
+ case 'aborted':
75
+ return timedOut(error.message)
76
+ case 'blocked':
77
+ return blocked(error.message, error.retryAfterMs)
78
+ case 'ssrf_blocked':
79
+ case 'unsupported_content_type':
80
+ return failed(error.message, false)
81
+ case 'network':
82
+ case 'bad_response':
83
+ case 'too_many_redirects':
84
+ return failed(error.message, true)
85
+ }
86
+ }
87
+ if (error instanceof Error && error.name === 'AbortError') {
88
+ return timedOut(error.message)
89
+ }
90
+ return failed(errorMessage(error), false)
91
+ }