@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -0
- package/AGENTS.md +67 -0
- package/build.mjs +10 -0
- package/dist/contract/adapter.js +13 -0
- package/dist/contract/adapter.js.map +7 -0
- package/dist/contract/errors.js +28 -0
- package/dist/contract/errors.js.map +7 -0
- package/dist/contract/http.js +7 -0
- package/dist/contract/http.js.map +7 -0
- package/dist/contract/index.js +9 -0
- package/dist/contract/index.js.map +7 -0
- package/dist/contract/outcomes.js +58 -0
- package/dist/contract/outcomes.js.map +7 -0
- package/dist/contract/policy.js +91 -0
- package/dist/contract/policy.js.map +7 -0
- package/dist/contract/results.js +13 -0
- package/dist/contract/results.js.map +7 -0
- package/dist/contract/steps.js +1 -0
- package/dist/contract/steps.js.map +7 -0
- package/dist/contract/version.js +11 -0
- package/dist/contract/version.js.map +7 -0
- package/dist/engine/cache.js +29 -0
- package/dist/engine/cache.js.map +7 -0
- package/dist/engine/deadline.js +54 -0
- package/dist/engine/deadline.js.map +7 -0
- package/dist/engine/engine.js +481 -0
- package/dist/engine/engine.js.map +7 -0
- package/dist/engine/index.js +20 -0
- package/dist/engine/index.js.map +7 -0
- package/dist/engine/runAdapter.js +57 -0
- package/dist/engine/runAdapter.js.map +7 -0
- package/dist/engine/types.js +1 -0
- package/dist/engine/types.js.map +7 -0
- package/dist/extract/classify.js +51 -0
- package/dist/extract/classify.js.map +7 -0
- package/dist/extract/content.js +104 -0
- package/dist/extract/content.js.map +7 -0
- package/dist/extract/entities.js +95 -0
- package/dist/extract/entities.js.map +7 -0
- package/dist/extract/index.js +24 -0
- package/dist/extract/index.js.map +7 -0
- package/dist/extract/text.js +114 -0
- package/dist/extract/text.js.map +7 -0
- package/dist/extract/tokenizer.js +141 -0
- package/dist/extract/tokenizer.js.map +7 -0
- package/dist/fusion/canonicalize.js +88 -0
- package/dist/fusion/canonicalize.js.map +7 -0
- package/dist/fusion/fuse.js +88 -0
- package/dist/fusion/fuse.js.map +7 -0
- package/dist/fusion/index.js +11 -0
- package/dist/fusion/index.js.map +7 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +7 -0
- package/dist/loader/describe.js +92 -0
- package/dist/loader/describe.js.map +7 -0
- package/dist/loader/index.js +31 -0
- package/dist/loader/index.js.map +7 -0
- package/dist/loader/manifest.js +32 -0
- package/dist/loader/manifest.js.map +7 -0
- package/dist/loader/registry.js +98 -0
- package/dist/loader/registry.js.map +7 -0
- package/dist/net/client.js +298 -0
- package/dist/net/client.js.map +7 -0
- package/dist/net/ssrf.js +165 -0
- package/dist/net/ssrf.js.map +7 -0
- package/dist/net/url.js +9 -0
- package/dist/net/url.js.map +7 -0
- package/dist/testing/contract.js +118 -0
- package/dist/testing/contract.js.map +7 -0
- package/dist/testing/fakeAdapter.js +44 -0
- package/dist/testing/fakeAdapter.js.map +7 -0
- package/dist/testing/index.js +15 -0
- package/dist/testing/index.js.map +7 -0
- package/jest.config.cjs +24 -0
- package/package.json +65 -0
- package/src/contract/adapter.ts +100 -0
- package/src/contract/errors.ts +48 -0
- package/src/contract/http.ts +46 -0
- package/src/contract/index.ts +8 -0
- package/src/contract/outcomes.ts +91 -0
- package/src/contract/policy.ts +157 -0
- package/src/contract/results.ts +79 -0
- package/src/contract/steps.ts +45 -0
- package/src/contract/version.ts +25 -0
- package/src/engine/__tests__/engine.test.ts +597 -0
- package/src/engine/cache.ts +46 -0
- package/src/engine/deadline.ts +65 -0
- package/src/engine/engine.ts +633 -0
- package/src/engine/index.ts +22 -0
- package/src/engine/runAdapter.ts +89 -0
- package/src/engine/types.ts +137 -0
- package/src/extract/__tests__/content.test.ts +81 -0
- package/src/extract/__tests__/text.test.ts +61 -0
- package/src/extract/__tests__/tokenizer.test.ts +85 -0
- package/src/extract/classify.ts +81 -0
- package/src/extract/content.ts +136 -0
- package/src/extract/entities.ts +98 -0
- package/src/extract/index.ts +11 -0
- package/src/extract/text.ts +124 -0
- package/src/extract/tokenizer.ts +197 -0
- package/src/fusion/__tests__/fusion.test.ts +198 -0
- package/src/fusion/canonicalize.ts +119 -0
- package/src/fusion/fuse.ts +145 -0
- package/src/fusion/index.ts +8 -0
- package/src/index.ts +15 -0
- package/src/loader/__tests__/describe.test.ts +119 -0
- package/src/loader/__tests__/registry.test.ts +134 -0
- package/src/loader/describe.ts +145 -0
- package/src/loader/index.ts +25 -0
- package/src/loader/manifest.ts +44 -0
- package/src/loader/registry.ts +156 -0
- package/src/net/__tests__/client.test.ts +273 -0
- package/src/net/__tests__/ssrf.test.ts +165 -0
- package/src/net/client.ts +434 -0
- package/src/net/ssrf.ts +235 -0
- package/src/net/url.ts +13 -0
- package/src/testing/contract.ts +172 -0
- package/src/testing/fakeAdapter.ts +69 -0
- package/src/testing/index.ts +9 -0
- package/tsconfig.json +10 -0
- package/watch.mjs +7 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import type { AdapterContext, AdapterModule, SearchAdapter } from '../contract/adapter'
|
|
2
|
+
import { silentLogger, type HttpClient, type HttpResponse } from '../contract/http'
|
|
3
|
+
import { CONTRACT_VERSION } from '../contract/version'
|
|
4
|
+
import type { SearchRequest } from '../contract/results'
|
|
5
|
+
|
|
6
|
+
export type StubResponse = Partial<HttpResponse> & { readonly body: string }
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Minimal HTTP double. Adapters are contractually forbidden from reaching for
|
|
10
|
+
* `fetch`, so a test that passes with this client has also proved the adapter
|
|
11
|
+
* routes every byte through the hardened transport.
|
|
12
|
+
*/
|
|
13
|
+
export function createStubHttpClient(
|
|
14
|
+
handler: (url: string, init: Parameters<HttpClient['request']>[1]) => StubResponse | Promise<StubResponse>,
|
|
15
|
+
): HttpClient & { readonly calls: readonly string[] } {
|
|
16
|
+
const calls: string[] = []
|
|
17
|
+
return {
|
|
18
|
+
calls,
|
|
19
|
+
async request(url, init) {
|
|
20
|
+
calls.push(url)
|
|
21
|
+
const stub = await handler(url, init)
|
|
22
|
+
return {
|
|
23
|
+
url: stub.url ?? url,
|
|
24
|
+
status: stub.status ?? 200,
|
|
25
|
+
headers: stub.headers ?? new Map<string, string>(),
|
|
26
|
+
contentType: stub.contentType ?? 'text/html',
|
|
27
|
+
body: stub.body,
|
|
28
|
+
truncated: stub.truncated ?? false,
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function createTestContext(overrides: Partial<AdapterContext> = {}): AdapterContext {
|
|
35
|
+
return {
|
|
36
|
+
signal: new AbortController().signal,
|
|
37
|
+
http: createStubHttpClient(() => ({ body: '' })),
|
|
38
|
+
report: () => {},
|
|
39
|
+
deadlineAt: Date.now() + 10_000,
|
|
40
|
+
logger: silentLogger,
|
|
41
|
+
...overrides,
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export const SAMPLE_REQUEST: SearchRequest = { query: 'open mercato', limit: 5 }
|
|
46
|
+
|
|
47
|
+
const VALID_STATUSES = ['ok', 'empty', 'unavailable', 'blocked', 'timeout', 'error']
|
|
48
|
+
|
|
49
|
+
export type AdapterContractCase<TOptions> = {
|
|
50
|
+
/** Options that leave the adapter deliberately unconfigured. */
|
|
51
|
+
readonly unconfiguredOptions?: TOptions
|
|
52
|
+
/** Options sufficient for the adapter to consider itself ready. */
|
|
53
|
+
readonly configuredOptions: TOptions
|
|
54
|
+
/** Context used for the "returns results" probe; stub the HTTP client here. */
|
|
55
|
+
readonly context?: () => AdapterContext
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
type TestApi = {
|
|
59
|
+
describe: (name: string, body: () => void) => void
|
|
60
|
+
it: (name: string, body: () => void | Promise<void>) => void
|
|
61
|
+
expect: (value: unknown) => {
|
|
62
|
+
toBe(expected: unknown): void
|
|
63
|
+
toBeDefined(): void
|
|
64
|
+
toContain(expected: string): void
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Behavioural suite every adapter must pass, ours and third parties'. It pins the
|
|
70
|
+
* parts of the contract the scheduler actually depends on — most importantly that
|
|
71
|
+
* an adapter reports rather than throws, because a throwing adapter would
|
|
72
|
+
* otherwise take down an entire search.
|
|
73
|
+
*/
|
|
74
|
+
export function describeAdapterContract<TOptions>(
|
|
75
|
+
module: AdapterModule<TOptions>,
|
|
76
|
+
testCase: AdapterContractCase<TOptions>,
|
|
77
|
+
api: TestApi,
|
|
78
|
+
): void {
|
|
79
|
+
const { describe, it, expect } = api
|
|
80
|
+
|
|
81
|
+
describe(`${module.id} adapter contract`, () => {
|
|
82
|
+
it('declares the contract version it was built against', () => {
|
|
83
|
+
expect(module.contractVersion).toBe(CONTRACT_VERSION)
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
it('exposes a stable id and kind', () => {
|
|
87
|
+
expect(typeof module.id).toBe('string')
|
|
88
|
+
expect(module.id.length > 0).toBe(true)
|
|
89
|
+
expect(['serp', 'api', 'model', 'browser'].includes(module.kind)).toBe(true)
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
it('validates its own options', () => {
|
|
93
|
+
const parsed = module.optionsSchema.safeParse(testCase.configuredOptions)
|
|
94
|
+
expect(parsed.success).toBe(true)
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
it('reports readiness without performing I/O', () => {
|
|
98
|
+
const adapter = module.createAdapter(testCase.configuredOptions)
|
|
99
|
+
const readiness = adapter.readiness()
|
|
100
|
+
expect(typeof readiness.ready).toBe('boolean')
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
const unconfigured = testCase.unconfiguredOptions
|
|
104
|
+
if (unconfigured !== undefined) {
|
|
105
|
+
it('is unavailable rather than broken when unconfigured', async () => {
|
|
106
|
+
const adapter = module.createAdapter(unconfigured)
|
|
107
|
+
const readiness = adapter.readiness()
|
|
108
|
+
expect(readiness.ready).toBe(false)
|
|
109
|
+
if (!readiness.ready) expect(readiness.reason.length > 0).toBe(true)
|
|
110
|
+
|
|
111
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext())
|
|
112
|
+
expect(outcome.status).toBe('unavailable')
|
|
113
|
+
})
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// The contract is "never throws", not "always fails when HTTP does" — a model
|
|
117
|
+
// or browser adapter may not touch the HTTP client at all.
|
|
118
|
+
it('returns an outcome rather than throwing when egress fails', async () => {
|
|
119
|
+
const adapter = module.createAdapter(testCase.configuredOptions)
|
|
120
|
+
const context = createTestContext({
|
|
121
|
+
http: createStubHttpClient(() => {
|
|
122
|
+
throw new Error('network exploded')
|
|
123
|
+
}),
|
|
124
|
+
})
|
|
125
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, context)
|
|
126
|
+
expect(VALID_STATUSES.includes(outcome.status)).toBe(true)
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
it('honours an already-aborted signal without hanging', async () => {
|
|
130
|
+
const adapter = module.createAdapter(testCase.configuredOptions)
|
|
131
|
+
const controller = new AbortController()
|
|
132
|
+
controller.abort()
|
|
133
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext({ signal: controller.signal }))
|
|
134
|
+
expect(typeof outcome.status).toBe('string')
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
it('never returns a result without a url', async () => {
|
|
138
|
+
const adapter: SearchAdapter = module.createAdapter(testCase.configuredOptions)
|
|
139
|
+
const context = testCase.context ? testCase.context() : createTestContext()
|
|
140
|
+
const outcome = await adapter.search(SAMPLE_REQUEST, context)
|
|
141
|
+
if (outcome.status !== 'ok') return
|
|
142
|
+
for (const result of outcome.results) {
|
|
143
|
+
expect(typeof result.url).toBe('string')
|
|
144
|
+
expect(result.url.length > 0).toBe(true)
|
|
145
|
+
}
|
|
146
|
+
})
|
|
147
|
+
|
|
148
|
+
// The host builds an engine per request and disposes it after, so a `dispose`
|
|
149
|
+
// that throws on an adapter which was never used, or on a second call, breaks
|
|
150
|
+
// teardown for every other adapter in the same engine. This is the surface
|
|
151
|
+
// that leaked a browser sidecar and a Chromium per tool call.
|
|
152
|
+
it('tears down cleanly even when it was never used', async () => {
|
|
153
|
+
const adapter = module.createAdapter(testCase.configuredOptions)
|
|
154
|
+
if (!adapter.dispose) return
|
|
155
|
+
await adapter.dispose()
|
|
156
|
+
await adapter.dispose()
|
|
157
|
+
expect(true).toBe(true)
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
it('answers a health check without throwing', async () => {
|
|
161
|
+
const adapter = module.createAdapter(testCase.configuredOptions)
|
|
162
|
+
if (!adapter.healthCheck) return
|
|
163
|
+
const context = createTestContext({
|
|
164
|
+
http: createStubHttpClient(() => {
|
|
165
|
+
throw new Error('network exploded')
|
|
166
|
+
}),
|
|
167
|
+
})
|
|
168
|
+
const health = await adapter.healthCheck(context)
|
|
169
|
+
expect(typeof health.ok).toBe('boolean')
|
|
170
|
+
})
|
|
171
|
+
})
|
|
172
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type { AdapterCapabilities, AdapterKind, SearchAdapter } from '../contract/adapter'
|
|
2
|
+
import { READY, notReady } from '../contract/adapter'
|
|
3
|
+
import type { SearchOutcome } from '../contract/outcomes'
|
|
4
|
+
import type { RawResult } from '../contract/results'
|
|
5
|
+
|
|
6
|
+
export type FakeAdapterOptions = {
|
|
7
|
+
readonly id: string
|
|
8
|
+
readonly kind?: AdapterKind
|
|
9
|
+
readonly ready?: boolean | string
|
|
10
|
+
/** Milliseconds of simulated work before the outcome resolves. */
|
|
11
|
+
readonly latencyMs?: number
|
|
12
|
+
readonly outcome?: SearchOutcome
|
|
13
|
+
readonly results?: readonly RawResult[]
|
|
14
|
+
readonly capabilities?: Partial<AdapterCapabilities>
|
|
15
|
+
/** Set to make the adapter violate the contract, for isolation tests. */
|
|
16
|
+
readonly throws?: boolean
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const BASE_CAPABILITIES: AdapterCapabilities = {
|
|
20
|
+
search: true,
|
|
21
|
+
fetch: false,
|
|
22
|
+
snippets: true,
|
|
23
|
+
content: false,
|
|
24
|
+
freshness: false,
|
|
25
|
+
siteFilter: false,
|
|
26
|
+
cost: 'free',
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Deterministic adapter double for scheduler and fusion tests. */
|
|
30
|
+
export function createFakeAdapter(options: FakeAdapterOptions): SearchAdapter {
|
|
31
|
+
const capabilities: AdapterCapabilities = { ...BASE_CAPABILITIES, ...options.capabilities }
|
|
32
|
+
const outcome: SearchOutcome =
|
|
33
|
+
options.outcome ??
|
|
34
|
+
(options.results && options.results.length > 0
|
|
35
|
+
? { status: 'ok', results: options.results }
|
|
36
|
+
: { status: 'empty' })
|
|
37
|
+
|
|
38
|
+
return {
|
|
39
|
+
id: options.id,
|
|
40
|
+
kind: options.kind ?? 'api',
|
|
41
|
+
capabilities,
|
|
42
|
+
readiness: () =>
|
|
43
|
+
options.ready === false
|
|
44
|
+
? notReady('fake adapter configured as not ready')
|
|
45
|
+
: typeof options.ready === 'string'
|
|
46
|
+
? notReady(options.ready)
|
|
47
|
+
: READY,
|
|
48
|
+
async search(_request, context) {
|
|
49
|
+
if (options.throws) throw new Error(`${options.id} exploded`)
|
|
50
|
+
const latency = options.latencyMs ?? 0
|
|
51
|
+
if (latency > 0) {
|
|
52
|
+
await new Promise<void>((resolve) => {
|
|
53
|
+
const timer = setTimeout(resolve, latency)
|
|
54
|
+
timer.unref?.()
|
|
55
|
+
context.signal.addEventListener(
|
|
56
|
+
'abort',
|
|
57
|
+
() => {
|
|
58
|
+
clearTimeout(timer)
|
|
59
|
+
resolve()
|
|
60
|
+
},
|
|
61
|
+
{ once: true },
|
|
62
|
+
)
|
|
63
|
+
})
|
|
64
|
+
}
|
|
65
|
+
if (context.signal.aborted) return { status: 'timeout', reason: 'aborted' }
|
|
66
|
+
return outcome
|
|
67
|
+
},
|
|
68
|
+
}
|
|
69
|
+
}
|
package/tsconfig.json
ADDED