@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/.turbo/turbo-build.log +2 -0
  2. package/AGENTS.md +67 -0
  3. package/build.mjs +10 -0
  4. package/dist/contract/adapter.js +13 -0
  5. package/dist/contract/adapter.js.map +7 -0
  6. package/dist/contract/errors.js +28 -0
  7. package/dist/contract/errors.js.map +7 -0
  8. package/dist/contract/http.js +7 -0
  9. package/dist/contract/http.js.map +7 -0
  10. package/dist/contract/index.js +9 -0
  11. package/dist/contract/index.js.map +7 -0
  12. package/dist/contract/outcomes.js +58 -0
  13. package/dist/contract/outcomes.js.map +7 -0
  14. package/dist/contract/policy.js +91 -0
  15. package/dist/contract/policy.js.map +7 -0
  16. package/dist/contract/results.js +13 -0
  17. package/dist/contract/results.js.map +7 -0
  18. package/dist/contract/steps.js +1 -0
  19. package/dist/contract/steps.js.map +7 -0
  20. package/dist/contract/version.js +11 -0
  21. package/dist/contract/version.js.map +7 -0
  22. package/dist/engine/cache.js +29 -0
  23. package/dist/engine/cache.js.map +7 -0
  24. package/dist/engine/deadline.js +54 -0
  25. package/dist/engine/deadline.js.map +7 -0
  26. package/dist/engine/engine.js +481 -0
  27. package/dist/engine/engine.js.map +7 -0
  28. package/dist/engine/index.js +20 -0
  29. package/dist/engine/index.js.map +7 -0
  30. package/dist/engine/runAdapter.js +57 -0
  31. package/dist/engine/runAdapter.js.map +7 -0
  32. package/dist/engine/types.js +1 -0
  33. package/dist/engine/types.js.map +7 -0
  34. package/dist/extract/classify.js +51 -0
  35. package/dist/extract/classify.js.map +7 -0
  36. package/dist/extract/content.js +104 -0
  37. package/dist/extract/content.js.map +7 -0
  38. package/dist/extract/entities.js +95 -0
  39. package/dist/extract/entities.js.map +7 -0
  40. package/dist/extract/index.js +24 -0
  41. package/dist/extract/index.js.map +7 -0
  42. package/dist/extract/text.js +114 -0
  43. package/dist/extract/text.js.map +7 -0
  44. package/dist/extract/tokenizer.js +141 -0
  45. package/dist/extract/tokenizer.js.map +7 -0
  46. package/dist/fusion/canonicalize.js +88 -0
  47. package/dist/fusion/canonicalize.js.map +7 -0
  48. package/dist/fusion/fuse.js +88 -0
  49. package/dist/fusion/fuse.js.map +7 -0
  50. package/dist/fusion/index.js +11 -0
  51. package/dist/fusion/index.js.map +7 -0
  52. package/dist/index.js +21 -0
  53. package/dist/index.js.map +7 -0
  54. package/dist/loader/describe.js +92 -0
  55. package/dist/loader/describe.js.map +7 -0
  56. package/dist/loader/index.js +31 -0
  57. package/dist/loader/index.js.map +7 -0
  58. package/dist/loader/manifest.js +32 -0
  59. package/dist/loader/manifest.js.map +7 -0
  60. package/dist/loader/registry.js +98 -0
  61. package/dist/loader/registry.js.map +7 -0
  62. package/dist/net/client.js +298 -0
  63. package/dist/net/client.js.map +7 -0
  64. package/dist/net/ssrf.js +165 -0
  65. package/dist/net/ssrf.js.map +7 -0
  66. package/dist/net/url.js +9 -0
  67. package/dist/net/url.js.map +7 -0
  68. package/dist/testing/contract.js +118 -0
  69. package/dist/testing/contract.js.map +7 -0
  70. package/dist/testing/fakeAdapter.js +44 -0
  71. package/dist/testing/fakeAdapter.js.map +7 -0
  72. package/dist/testing/index.js +15 -0
  73. package/dist/testing/index.js.map +7 -0
  74. package/jest.config.cjs +24 -0
  75. package/package.json +65 -0
  76. package/src/contract/adapter.ts +100 -0
  77. package/src/contract/errors.ts +48 -0
  78. package/src/contract/http.ts +46 -0
  79. package/src/contract/index.ts +8 -0
  80. package/src/contract/outcomes.ts +91 -0
  81. package/src/contract/policy.ts +157 -0
  82. package/src/contract/results.ts +79 -0
  83. package/src/contract/steps.ts +45 -0
  84. package/src/contract/version.ts +25 -0
  85. package/src/engine/__tests__/engine.test.ts +597 -0
  86. package/src/engine/cache.ts +46 -0
  87. package/src/engine/deadline.ts +65 -0
  88. package/src/engine/engine.ts +633 -0
  89. package/src/engine/index.ts +22 -0
  90. package/src/engine/runAdapter.ts +89 -0
  91. package/src/engine/types.ts +137 -0
  92. package/src/extract/__tests__/content.test.ts +81 -0
  93. package/src/extract/__tests__/text.test.ts +61 -0
  94. package/src/extract/__tests__/tokenizer.test.ts +85 -0
  95. package/src/extract/classify.ts +81 -0
  96. package/src/extract/content.ts +136 -0
  97. package/src/extract/entities.ts +98 -0
  98. package/src/extract/index.ts +11 -0
  99. package/src/extract/text.ts +124 -0
  100. package/src/extract/tokenizer.ts +197 -0
  101. package/src/fusion/__tests__/fusion.test.ts +198 -0
  102. package/src/fusion/canonicalize.ts +119 -0
  103. package/src/fusion/fuse.ts +145 -0
  104. package/src/fusion/index.ts +8 -0
  105. package/src/index.ts +15 -0
  106. package/src/loader/__tests__/describe.test.ts +119 -0
  107. package/src/loader/__tests__/registry.test.ts +134 -0
  108. package/src/loader/describe.ts +145 -0
  109. package/src/loader/index.ts +25 -0
  110. package/src/loader/manifest.ts +44 -0
  111. package/src/loader/registry.ts +156 -0
  112. package/src/net/__tests__/client.test.ts +273 -0
  113. package/src/net/__tests__/ssrf.test.ts +165 -0
  114. package/src/net/client.ts +434 -0
  115. package/src/net/ssrf.ts +235 -0
  116. package/src/net/url.ts +13 -0
  117. package/src/testing/contract.ts +172 -0
  118. package/src/testing/fakeAdapter.ts +69 -0
  119. package/src/testing/index.ts +9 -0
  120. package/tsconfig.json +10 -0
  121. package/watch.mjs +7 -0
@@ -0,0 +1,597 @@
1
+ import type { AdapterCapabilities, ProbeCost, SearchAdapter } from '../../contract/adapter'
2
+ import type { FetchOutcome } from '../../contract/outcomes'
3
+ import { resolvePolicy, type SearchPolicyInput } from '../../contract/policy'
4
+ import type { RawResult } from '../../contract/results'
5
+ import type { SearchStep } from '../../contract/steps'
6
+ import { createStubHttpClient } from '../../testing/contract'
7
+ import { createFakeAdapter } from '../../testing/fakeAdapter'
8
+ import { createSearchEngine } from '../engine'
9
+ import type { EngineAdapterEntry, SearchEngineResult } from '../types'
10
+
11
+ const http = createStubHttpClient(() => ({ body: '<html><body><p>stub</p></body></html>' }))
12
+
13
+ /**
14
+ * Matches on the host rather than a URL substring: `includes()` would also
15
+ * match a host that merely embeds the name, which is both a weaker assertion
16
+ * and a CodeQL finding.
17
+ */
18
+ const hostOf = (url: string): string => new URL(url).hostname
19
+
20
+ const hits = (host: string, count: number): RawResult[] =>
21
+ Array.from({ length: count }, (_, index) => ({
22
+ url: `https://${host}/${index}`,
23
+ title: `${host} ${index}`,
24
+ snippet: 'snippet',
25
+ }))
26
+
27
+ const entry = (adapter: SearchAdapter, overrides: Partial<EngineAdapterEntry> = {}): EngineAdapterEntry => ({
28
+ adapter,
29
+ weight: 1,
30
+ order: 0,
31
+ enabled: true,
32
+ ...overrides,
33
+ })
34
+
35
+ const engineWith = (
36
+ adapters: readonly EngineAdapterEntry[],
37
+ policy: SearchPolicyInput = {},
38
+ extra: { onStep?: (step: SearchStep) => void } = {},
39
+ ) =>
40
+ createSearchEngine({
41
+ policy: resolvePolicy({ minResults: 3, ...policy }),
42
+ adapters,
43
+ http,
44
+ ...(extra.onStep ? { onStep: extra.onStep } : {}),
45
+ })
46
+
47
+ const statusOf = (result: SearchEngineResult, id: string): string | undefined =>
48
+ result.diagnostics.adapters.find((diagnostic) => diagnostic.id === id)?.status
49
+
50
+ describe('search engine scheduling', () => {
51
+ it('race mode answers from the first adapter and cancels the rest', async () => {
52
+ const engine = engineWith(
53
+ [
54
+ entry(createFakeAdapter({ id: 'fast', latencyMs: 5, results: hits('fast.com', 3) }), { order: 0 }),
55
+ entry(createFakeAdapter({ id: 'slow', latencyMs: 400, results: hits('slow.com', 3) }), { order: 1 }),
56
+ ],
57
+ { settleMode: 'race', concurrency: 2, minResults: 3 },
58
+ )
59
+
60
+ const result = await engine.search({ query: 'x' })
61
+
62
+ expect(result.results.every((hit) => hostOf(hit.url) === 'fast.com')).toBe(true)
63
+ expect(statusOf(result, 'slow')).toBe('cancelled')
64
+ })
65
+
66
+ it('quorum mode waits for a second adapter that a single-adapter quorum would have cancelled', async () => {
67
+ const adapters = () => [
68
+ entry(createFakeAdapter({ id: 'alpha', latencyMs: 5, results: hits('alpha.com', 3) }), { order: 0 }),
69
+ entry(createFakeAdapter({ id: 'beta', latencyMs: 300, results: hits('beta.com', 3) }), { order: 1 }),
70
+ ]
71
+ const base = { settleMode: 'quorum' as const, concurrency: 2, minResults: 3, softDeadlineMs: 2_000 }
72
+
73
+ const strict = await engineWith(adapters(), { ...base, minAdapters: 2 }).search({ query: 'x', limit: 10 })
74
+ const lenient = await engineWith(adapters(), { ...base, minAdapters: 1 }).search({ query: 'x', limit: 10 })
75
+
76
+ expect(statusOf(strict, 'beta')).toBe('ok')
77
+ expect(statusOf(lenient, 'beta')).toBe('cancelled')
78
+ })
79
+
80
+ it('exhaustive mode runs every adapter even once satisfied', async () => {
81
+ const engine = engineWith(
82
+ [
83
+ entry(createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 5) }), { order: 0 }),
84
+ entry(createFakeAdapter({ id: 'beta', latencyMs: 15, results: hits('beta.com', 5) }), { order: 1 }),
85
+ ],
86
+ { settleMode: 'exhaustive', concurrency: 1, minResults: 1 },
87
+ )
88
+
89
+ const result = await engine.search({ query: 'x', limit: 20 })
90
+
91
+ expect(statusOf(result, 'alpha')).toBe('ok')
92
+ expect(statusOf(result, 'beta')).toBe('ok')
93
+ })
94
+
95
+ it('skips an unconfigured adapter and says why', async () => {
96
+ const engine = engineWith([
97
+ entry(createFakeAdapter({ id: 'keyed', ready: 'API key is not set' })),
98
+ entry(createFakeAdapter({ id: 'free', results: hits('free.com', 3) }), { order: 1 }),
99
+ ])
100
+
101
+ const result = await engine.search({ query: 'x' })
102
+
103
+ const skipped = result.diagnostics.adapters.find((diagnostic) => diagnostic.id === 'keyed')
104
+ expect(skipped?.status).toBe('skipped')
105
+ expect(skipped?.reason).toBe('API key is not set')
106
+ expect(result.results).not.toHaveLength(0)
107
+ })
108
+
109
+ it('contains an adapter that throws instead of returning an outcome', async () => {
110
+ const engine = engineWith([
111
+ entry(createFakeAdapter({ id: 'broken', throws: true }), { order: 0 }),
112
+ entry(createFakeAdapter({ id: 'healthy', results: hits('healthy.com', 3) }), { order: 1 }),
113
+ ])
114
+
115
+ const result = await engine.search({ query: 'x' })
116
+
117
+ expect(statusOf(result, 'broken')).toBe('error')
118
+ expect(result.results.length).toBe(3)
119
+ })
120
+
121
+ it('runs the last-resort adapter even when it is disabled', async () => {
122
+ const engine = engineWith(
123
+ [
124
+ entry(createFakeAdapter({ id: 'serp', outcome: { status: 'empty' } }), { order: 0 }),
125
+ entry(createFakeAdapter({ id: 'model-native', results: hits('model.com', 3) }), {
126
+ order: 1,
127
+ enabled: false,
128
+ }),
129
+ ],
130
+ { lastResort: 'model-native', minResults: 3 },
131
+ )
132
+
133
+ const result = await engine.search({ query: 'x' })
134
+
135
+ expect(statusOf(result, 'model-native')).toBe('ok')
136
+ expect(result.results.length).toBe(3)
137
+ })
138
+
139
+ it('does not invoke the last resort when the waves already satisfied the request', async () => {
140
+ const lastResort = createFakeAdapter({ id: 'model-native', results: hits('model.com', 3) })
141
+ const spy = jest.spyOn(lastResort, 'search')
142
+ const engine = engineWith(
143
+ [
144
+ entry(createFakeAdapter({ id: 'serp', results: hits('serp.com', 5) }), { order: 0 }),
145
+ entry(lastResort, { order: 1, enabled: false }),
146
+ ],
147
+ { lastResort: 'model-native', minResults: 3 },
148
+ )
149
+
150
+ await engine.search({ query: 'x' })
151
+
152
+ expect(spy).not.toHaveBeenCalled()
153
+ })
154
+
155
+ it('escalates to a browser adapter when a SERP adapter is blocked', async () => {
156
+ const engine = engineWith(
157
+ [
158
+ entry(
159
+ createFakeAdapter({
160
+ id: 'serp-html',
161
+ kind: 'serp',
162
+ outcome: { status: 'blocked', reason: 'HTTP 429' },
163
+ }),
164
+ { order: 0 },
165
+ ),
166
+ entry(
167
+ createFakeAdapter({ id: 'browser-serp', kind: 'browser', results: hits('browser.com', 3) }),
168
+ { order: 1 },
169
+ ),
170
+ ],
171
+ { escalateToBrowser: true, minResults: 3, lastResort: null },
172
+ )
173
+
174
+ const result = await engine.search({ query: 'x' })
175
+
176
+ expect(result.diagnostics.escalated).toBe(true)
177
+ expect(statusOf(result, 'browser-serp')).toBe('ok')
178
+ expect(result.results.length).toBe(3)
179
+ })
180
+
181
+ it('does not escalate when the wave already produced enough results', async () => {
182
+ const browser = createFakeAdapter({ id: 'browser-serp', kind: 'browser', results: hits('b.com', 3) })
183
+ const spy = jest.spyOn(browser, 'search')
184
+ const engine = engineWith(
185
+ [
186
+ entry(createFakeAdapter({ id: 'serp-html', kind: 'serp', results: hits('serp.com', 5) }), {
187
+ order: 0,
188
+ }),
189
+ entry(browser, { order: 1 }),
190
+ ],
191
+ { escalateToBrowser: true, minResults: 3 },
192
+ )
193
+
194
+ await engine.search({ query: 'x' })
195
+
196
+ expect(spy).not.toHaveBeenCalled()
197
+ })
198
+
199
+ it('does not escalate to a browser adapter the operator disabled', async () => {
200
+ const browser = createFakeAdapter({ id: 'browser-serp', kind: 'browser', results: hits('b.com', 3) })
201
+ const spy = jest.spyOn(browser, 'search')
202
+ const engine = engineWith(
203
+ [
204
+ entry(
205
+ createFakeAdapter({
206
+ id: 'serp-html',
207
+ kind: 'serp',
208
+ outcome: { status: 'blocked', reason: 'HTTP 429' },
209
+ }),
210
+ { order: 0 },
211
+ ),
212
+ entry(browser, { order: 1, enabled: false }),
213
+ ],
214
+ { escalateToBrowser: true, minResults: 3, lastResort: null },
215
+ )
216
+
217
+ const result = await engine.search({ query: 'x' })
218
+
219
+ expect(spy).not.toHaveBeenCalled()
220
+ expect(result.diagnostics.escalated).toBe(false)
221
+ })
222
+
223
+ it('does not escalate a fetch to a browser adapter the operator disabled', async () => {
224
+ // An empty client-side render root is what classifies as `js-shell` and asks
225
+ // for a browser render; without it there is nothing to escalate.
226
+ const shell = createStubHttpClient(() => ({ body: '<html><body><div id="root"></div></body></html>' }))
227
+ const browser = createFakeAdapter({ id: 'browser-fetch', kind: 'browser' })
228
+ const fetchSpy = jest.fn(async () => ({ status: 'unavailable' as const, reason: 'unused' }))
229
+ const engine = createSearchEngine({
230
+ policy: resolvePolicy({ escalateToBrowser: true }),
231
+ adapters: [entry({ ...browser, fetch: fetchSpy }, { enabled: false })],
232
+ http: shell,
233
+ })
234
+
235
+ await engine.fetch({ url: 'https://example.com/app' })
236
+
237
+ expect(fetchSpy).not.toHaveBeenCalled()
238
+ })
239
+
240
+ it('bounds a browser fetch that ignores its abort signal instead of hanging', async () => {
241
+ const browser = createFakeAdapter({ id: 'wedged-browser', kind: 'browser', capabilities: { fetch: true } })
242
+ // A browser adapter whose render neither honours the abort signal nor ever
243
+ // settles — the wedged-Playwright case behind the 66-minute run hang.
244
+ const neverSettles: SearchAdapter['fetch'] = () => new Promise<FetchOutcome>(() => {})
245
+ const engine = createSearchEngine({
246
+ policy: resolvePolicy({ escalateToBrowser: true, hardDeadlineMs: 500 }),
247
+ adapters: [entry({ ...browser, fetch: neverSettles }, { order: 0 })],
248
+ http,
249
+ })
250
+
251
+ const outcome = await engine.fetch({ url: 'https://example.com/app', render: 'always' })
252
+
253
+ expect(outcome.status).toBe('timeout')
254
+ })
255
+
256
+ it('a wedged browser escalation during content enrichment does not hang the search', async () => {
257
+ // A client-side-render shell classifies as `js-shell` and asks for a browser
258
+ // render, so enrichment escalates the top result into the wedged adapter.
259
+ const shell = createStubHttpClient(() => ({ body: '<html><body><div id="root"></div></body></html>' }))
260
+ const browser = createFakeAdapter({ id: 'wedged-browser', kind: 'browser', capabilities: { fetch: true } })
261
+ const neverSettles: SearchAdapter['fetch'] = () => new Promise<FetchOutcome>(() => {})
262
+ const engine = createSearchEngine({
263
+ policy: resolvePolicy({ escalateToBrowser: true, hardDeadlineMs: 500, minResults: 1, lastResort: null }),
264
+ adapters: [
265
+ entry(createFakeAdapter({ id: 'api', results: hits('example.com', 3) }), { order: 0 }),
266
+ entry({ ...browser, fetch: neverSettles }, { order: 1 }),
267
+ ],
268
+ http: shell,
269
+ })
270
+
271
+ const result = await engine.search({ query: 'x', includeContent: true })
272
+
273
+ // The search still returns its results rather than blocking on the escalation.
274
+ expect(result.results.length).toBeGreaterThan(0)
275
+ })
276
+
277
+ it('gives an adapter its own timeout when the policy sets one', async () => {
278
+ const adapters = () => [
279
+ entry(createFakeAdapter({ id: 'slow', latencyMs: 200, results: hits('slow.com', 3) }), {
280
+ timeoutMs: 5_000,
281
+ }),
282
+ ]
283
+
284
+ const shared = await engineWith(
285
+ [entry(createFakeAdapter({ id: 'slow', latencyMs: 200, results: hits('slow.com', 3) }))],
286
+ { adapterTimeoutMs: 50, minResults: 1, lastResort: null },
287
+ ).search({ query: 'x' })
288
+ const overridden = await engineWith(adapters(), {
289
+ adapterTimeoutMs: 50,
290
+ minResults: 1,
291
+ lastResort: null,
292
+ }).search({ query: 'x' })
293
+
294
+ expect(statusOf(shared, 'slow')).toBe('timeout')
295
+ expect(statusOf(overridden, 'slow')).toBe('ok')
296
+ })
297
+
298
+ it('does not call a thin but healthy run degraded', async () => {
299
+ const engine = engineWith(
300
+ [entry(createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 2) }))],
301
+ { minResults: 5, lastResort: null },
302
+ )
303
+
304
+ const result = await engine.search({ query: 'x' })
305
+
306
+ // Two hits under a minimum of five is the web being thin, not the engine
307
+ // failing; an agent told otherwise reports its own findings as unreliable.
308
+ expect(result.results.length).toBe(2)
309
+ expect(result.diagnostics.degraded).toBe(false)
310
+ })
311
+
312
+ it('calls a run degraded when a source was lost', async () => {
313
+ const engine = engineWith(
314
+ [
315
+ entry(createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 2) })),
316
+ entry(createFakeAdapter({ id: 'beta', outcome: { status: 'blocked', reason: 'HTTP 429' } }), {
317
+ order: 1,
318
+ }),
319
+ ],
320
+ { settleMode: 'exhaustive', minResults: 1, lastResort: null },
321
+ )
322
+
323
+ const result = await engine.search({ query: 'x' })
324
+
325
+ expect(result.diagnostics.degraded).toBe(true)
326
+ })
327
+
328
+ it('omits the per-adapter view unless the caller asks for it', async () => {
329
+ const engine = engineWith([entry(createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 3) }))], {
330
+ minResults: 1,
331
+ lastResort: null,
332
+ })
333
+
334
+ expect((await engine.search({ query: 'x' })).byAdapter).toBeUndefined()
335
+ })
336
+
337
+ it('returns each adapter untouched by fusion when asked to compare', async () => {
338
+ const alpha = createFakeAdapter({
339
+ id: 'alpha',
340
+ outcome: { status: 'ok', results: hits('shared.com', 1), answer: 'alpha prose' },
341
+ })
342
+ const beta = createFakeAdapter({
343
+ id: 'beta',
344
+ outcome: { status: 'ok', results: hits('shared.com', 1), answer: 'beta prose' },
345
+ })
346
+ const engine = engineWith([entry(alpha), entry(beta, { order: 1 })], {
347
+ settleMode: 'exhaustive',
348
+ minResults: 1,
349
+ lastResort: null,
350
+ })
351
+
352
+ const result = await engine.search({ query: 'x' }, { includeAdapterResults: true })
353
+
354
+ // Fusion collapses the shared hit into one and keeps only the first prose;
355
+ // the comparison view is what survives that, per adapter.
356
+ expect(result.results).toHaveLength(1)
357
+ expect(result.answer).toBe('alpha prose')
358
+ expect(result.byAdapter?.map((entry) => entry.adapterId)).toEqual(['alpha', 'beta'])
359
+ expect(result.byAdapter?.map((entry) => entry.answer)).toEqual(['alpha prose', 'beta prose'])
360
+ expect(result.byAdapter?.every((entry) => entry.results.length === 1)).toBe(true)
361
+ })
362
+
363
+ it('reports a degraded run rather than failing when every adapter is down', async () => {
364
+ const engine = engineWith(
365
+ [
366
+ entry(createFakeAdapter({ id: 'a', outcome: { status: 'unavailable', reason: 'no key' } })),
367
+ entry(createFakeAdapter({ id: 'b', outcome: { status: 'error', reason: 'boom', retriable: true } }), {
368
+ order: 1,
369
+ }),
370
+ ],
371
+ { lastResort: null },
372
+ )
373
+
374
+ const result = await engine.search({ query: 'x' })
375
+
376
+ expect(result.results).toEqual([])
377
+ expect(result.diagnostics.degraded).toBe(true)
378
+ expect(result.diagnostics.adapters.map((diagnostic) => diagnostic.status).sort()).toEqual([
379
+ 'error',
380
+ 'unavailable',
381
+ ])
382
+ })
383
+
384
+ it('surfaces an adapter answer without inventing one', async () => {
385
+ const withAnswer = engineWith([
386
+ entry(
387
+ createFakeAdapter({
388
+ id: 'model-native',
389
+ outcome: { status: 'ok', results: hits('m.com', 2), answer: 'Because of X.' },
390
+ }),
391
+ ),
392
+ ])
393
+ expect((await withAnswer.search({ query: 'x' })).answer).toBe('Because of X.')
394
+
395
+ const withoutAnswer = engineWith([entry(createFakeAdapter({ id: 'serp', results: hits('s.com', 2) }))])
396
+ expect((await withoutAnswer.search({ query: 'x' })).answer).toBeNull()
397
+ })
398
+
399
+ it('stops waiting at the hard deadline', async () => {
400
+ const engine = engineWith(
401
+ [entry(createFakeAdapter({ id: 'stuck', latencyMs: 5_000, results: hits('stuck.com', 3) }))],
402
+ { hardDeadlineMs: 500, softDeadlineMs: 100, adapterTimeoutMs: 500, lastResort: null },
403
+ )
404
+
405
+ const startedAt = Date.now()
406
+ const result = await engine.search({ query: 'x' })
407
+
408
+ expect(Date.now() - startedAt).toBeLessThan(2_000)
409
+ expect(result.results).toEqual([])
410
+ })
411
+ })
412
+
413
+ describe('search engine narration', () => {
414
+ it('emits an ordered step stream naming each adapter and outcome', async () => {
415
+ const steps: SearchStep[] = []
416
+ const engine = engineWith(
417
+ [
418
+ entry(createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 3) }), { order: 0 }),
419
+ entry(createFakeAdapter({ id: 'beta', ready: 'not configured' }), { order: 1 }),
420
+ ],
421
+ {},
422
+ { onStep: (step) => steps.push(step) },
423
+ )
424
+
425
+ await engine.search({ query: 'x' })
426
+
427
+ expect(steps.map((step) => step.seq)).toEqual(steps.map((_, index) => index))
428
+ expect(steps.some((step) => step.adapterId === 'alpha' && step.event === 'ok')).toBe(true)
429
+ expect(steps.some((step) => step.adapterId === 'beta' && step.event === 'unavailable')).toBe(true)
430
+ expect(steps.at(-1)).toMatchObject({ phase: 'done', event: 'completed' })
431
+ })
432
+
433
+ it('keeps running when a step sink throws', async () => {
434
+ const engine = engineWith(
435
+ [entry(createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 3) }))],
436
+ {},
437
+ {
438
+ onStep: () => {
439
+ throw new Error('sink exploded')
440
+ },
441
+ },
442
+ )
443
+
444
+ await expect(engine.search({ query: 'x' })).resolves.toMatchObject({
445
+ results: expect.any(Array),
446
+ })
447
+ })
448
+ })
449
+
450
+ describe('search engine caching', () => {
451
+ it('collapses concurrent identical queries onto one adapter call', async () => {
452
+ const adapter = createFakeAdapter({ id: 'alpha', latencyMs: 20, results: hits('alpha.com', 3) })
453
+ const spy = jest.spyOn(adapter, 'search')
454
+ const engine = engineWith([entry(adapter)])
455
+
456
+ await Promise.all([engine.search({ query: 'same' }), engine.search({ query: 'same' })])
457
+
458
+ expect(spy).toHaveBeenCalledTimes(1)
459
+ })
460
+
461
+ it('serves a cached result and marks it as cached', async () => {
462
+ const store = new Map<string, SearchEngineResult>()
463
+ const cache = {
464
+ get: async (key: string) => store.get(key) ?? null,
465
+ set: async (key: string, value: SearchEngineResult) => {
466
+ store.set(key, value)
467
+ },
468
+ }
469
+ const adapter = createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 3) })
470
+ const spy = jest.spyOn(adapter, 'search')
471
+ const engine = createSearchEngine({
472
+ policy: resolvePolicy({ minResults: 3 }),
473
+ adapters: [entry(adapter)],
474
+ http,
475
+ cache,
476
+ })
477
+
478
+ const first = await engine.search({ query: 'cached' })
479
+ const second = await engine.search({ query: 'cached' })
480
+
481
+ expect(first.diagnostics.cached).toBe(false)
482
+ expect(second.diagnostics.cached).toBe(true)
483
+ expect(spy).toHaveBeenCalledTimes(1)
484
+ })
485
+
486
+ it('falls back to a live search when the cache throws', async () => {
487
+ const cache = {
488
+ get: async () => {
489
+ throw new Error('cache down')
490
+ },
491
+ set: async () => {
492
+ throw new Error('cache down')
493
+ },
494
+ }
495
+ const engine = createSearchEngine({
496
+ policy: resolvePolicy({ minResults: 3 }),
497
+ adapters: [entry(createFakeAdapter({ id: 'alpha', results: hits('alpha.com', 3) }))],
498
+ http,
499
+ cache,
500
+ })
501
+
502
+ await expect(engine.search({ query: 'x' })).resolves.toMatchObject({ answer: null })
503
+ })
504
+ })
505
+
506
+ describe('search engine health', () => {
507
+ /** A fake that records every probe, so "was it called" is asserted on a counter. */
508
+ const probeAdapter = (
509
+ id: string,
510
+ calls: string[],
511
+ overrides: { probeCost?: ProbeCost; cost?: AdapterCapabilities['cost'] } = {},
512
+ ): SearchAdapter => ({
513
+ ...createFakeAdapter({
514
+ id,
515
+ results: hits(`${id}.com`, 1),
516
+ ...(overrides.cost ? { capabilities: { cost: overrides.cost } } : {}),
517
+ }),
518
+ ...(overrides.probeCost ? { probeCost: overrides.probeCost } : {}),
519
+ healthCheck: async () => {
520
+ calls.push(id)
521
+ return { ok: true }
522
+ },
523
+ })
524
+
525
+ it('reports readiness and health per adapter', async () => {
526
+ const engine = engineWith([
527
+ entry(createFakeAdapter({ id: 'ready', results: hits('r.com', 1) }), { order: 0 }),
528
+ entry(createFakeAdapter({ id: 'unconfigured', ready: 'missing key' }), { order: 1 }),
529
+ ])
530
+
531
+ const reports = await engine.health()
532
+
533
+ expect(reports).toEqual([
534
+ // No `healthCheck` to call, so nothing was probed — the row says so rather
535
+ // than letting a consumer read `ok: true` as "verified".
536
+ { id: 'ready', ready: true, ok: true, probeCost: 'free', probed: false },
537
+ { id: 'unconfigured', ready: false, ok: false, detail: 'missing key', probeCost: 'free', probed: false },
538
+ ])
539
+ })
540
+
541
+ it('spends nothing beyond the probe budget', async () => {
542
+ const calls: string[] = []
543
+ const engine = engineWith([
544
+ entry(probeAdapter('model-native', calls, { probeCost: 'free' }), { order: 0 }),
545
+ entry(probeAdapter('browser', calls, { probeCost: 'heavy' }), { order: 1 }),
546
+ entry(probeAdapter('firecrawl', calls, { probeCost: 'billable' }), { order: 2 }),
547
+ ])
548
+
549
+ const reports = await engine.health({ probe: true, maxProbeCost: 'free' })
550
+
551
+ expect(calls).toEqual(['model-native'])
552
+ expect(reports.map((report) => [report.id, report.probed])).toEqual([
553
+ ['model-native', true],
554
+ ['browser', false],
555
+ ['firecrawl', false],
556
+ ])
557
+ })
558
+
559
+ it('assumes an undeclared probe bills when the adapter is metered', async () => {
560
+ // Pessimistic on purpose: guessing downwards costs a surprise invoice,
561
+ // guessing upwards costs one row that says "not verified" until asked.
562
+ const calls: string[] = []
563
+ const engine = engineWith([
564
+ entry(probeAdapter('paid', calls, { cost: 'metered' }), { order: 0 }),
565
+ entry(probeAdapter('gratis', calls, { cost: 'free' }), { order: 1 }),
566
+ ])
567
+
568
+ const reports = await engine.health({ probe: true, maxProbeCost: 'free' })
569
+
570
+ expect(calls).toEqual(['gratis'])
571
+ expect(reports.find((report) => report.id === 'paid')?.probeCost).toBe('billable')
572
+ })
573
+
574
+ it('limits a targeted re-test to the adapter that was asked for', async () => {
575
+ const calls: string[] = []
576
+ const engine = engineWith([
577
+ entry(probeAdapter('alpha', calls, { probeCost: 'billable' }), { order: 0 }),
578
+ entry(probeAdapter('beta', calls, { probeCost: 'billable' }), { order: 1 }),
579
+ ])
580
+
581
+ await engine.health({ probe: true, only: ['beta'] })
582
+
583
+ expect(calls).toEqual(['beta'])
584
+ })
585
+
586
+ it('probes everything by default, so existing callers keep their behaviour', async () => {
587
+ const calls: string[] = []
588
+ const engine = engineWith([
589
+ entry(probeAdapter('alpha', calls, { probeCost: 'billable' }), { order: 0 }),
590
+ entry(probeAdapter('beta', calls, { probeCost: 'heavy' }), { order: 1 }),
591
+ ])
592
+
593
+ await engine.health()
594
+
595
+ expect(calls.sort()).toEqual(['alpha', 'beta'])
596
+ })
597
+ })
@@ -0,0 +1,46 @@
1
+ import { createHash } from 'node:crypto'
2
+ import type { SearchRequest } from '../contract/results'
3
+
4
+ export type CacheEntry<T> = {
5
+ readonly value: T
6
+ readonly storedAt: number
7
+ }
8
+
9
+ /**
10
+ * Storage seam. Deliberately narrow so a host can back it with any store, and
11
+ * deliberately failable — a cache outage must degrade to a live search, never to
12
+ * an error, so implementations should swallow their own faults.
13
+ */
14
+ export type ResultCache<T> = {
15
+ get(key: string): Promise<T | null>
16
+ set(key: string, value: T, ttlMs: number): Promise<void>
17
+ }
18
+
19
+ export function buildCacheKey(request: SearchRequest, adapterIds: readonly string[]): string {
20
+ const payload = JSON.stringify({
21
+ query: request.query.trim().toLowerCase(),
22
+ limit: request.limit,
23
+ locale: request.locale ?? null,
24
+ freshness: request.freshness ?? null,
25
+ site: request.site ?? null,
26
+ adapters: [...adapterIds].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)),
27
+ })
28
+ return createHash('sha256').update(payload).digest('hex')
29
+ }
30
+
31
+ /**
32
+ * Collapses concurrent identical calls onto one execution. A fan-out of agents
33
+ * asking the same question hits each adapter once instead of once per agent.
34
+ */
35
+ export function createSingleFlight<T>(): (key: string, run: () => Promise<T>) => Promise<T> {
36
+ const inflight = new Map<string, Promise<T>>()
37
+ return (key, run) => {
38
+ const existing = inflight.get(key)
39
+ if (existing) return existing
40
+ const promise = run().finally(() => {
41
+ inflight.delete(key)
42
+ })
43
+ inflight.set(key, promise)
44
+ return promise
45
+ }
46
+ }