@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/.turbo/turbo-build.log +2 -0
  2. package/AGENTS.md +67 -0
  3. package/build.mjs +10 -0
  4. package/dist/contract/adapter.js +13 -0
  5. package/dist/contract/adapter.js.map +7 -0
  6. package/dist/contract/errors.js +28 -0
  7. package/dist/contract/errors.js.map +7 -0
  8. package/dist/contract/http.js +7 -0
  9. package/dist/contract/http.js.map +7 -0
  10. package/dist/contract/index.js +9 -0
  11. package/dist/contract/index.js.map +7 -0
  12. package/dist/contract/outcomes.js +58 -0
  13. package/dist/contract/outcomes.js.map +7 -0
  14. package/dist/contract/policy.js +91 -0
  15. package/dist/contract/policy.js.map +7 -0
  16. package/dist/contract/results.js +13 -0
  17. package/dist/contract/results.js.map +7 -0
  18. package/dist/contract/steps.js +1 -0
  19. package/dist/contract/steps.js.map +7 -0
  20. package/dist/contract/version.js +11 -0
  21. package/dist/contract/version.js.map +7 -0
  22. package/dist/engine/cache.js +29 -0
  23. package/dist/engine/cache.js.map +7 -0
  24. package/dist/engine/deadline.js +54 -0
  25. package/dist/engine/deadline.js.map +7 -0
  26. package/dist/engine/engine.js +481 -0
  27. package/dist/engine/engine.js.map +7 -0
  28. package/dist/engine/index.js +20 -0
  29. package/dist/engine/index.js.map +7 -0
  30. package/dist/engine/runAdapter.js +57 -0
  31. package/dist/engine/runAdapter.js.map +7 -0
  32. package/dist/engine/types.js +1 -0
  33. package/dist/engine/types.js.map +7 -0
  34. package/dist/extract/classify.js +51 -0
  35. package/dist/extract/classify.js.map +7 -0
  36. package/dist/extract/content.js +104 -0
  37. package/dist/extract/content.js.map +7 -0
  38. package/dist/extract/entities.js +95 -0
  39. package/dist/extract/entities.js.map +7 -0
  40. package/dist/extract/index.js +24 -0
  41. package/dist/extract/index.js.map +7 -0
  42. package/dist/extract/text.js +114 -0
  43. package/dist/extract/text.js.map +7 -0
  44. package/dist/extract/tokenizer.js +141 -0
  45. package/dist/extract/tokenizer.js.map +7 -0
  46. package/dist/fusion/canonicalize.js +88 -0
  47. package/dist/fusion/canonicalize.js.map +7 -0
  48. package/dist/fusion/fuse.js +88 -0
  49. package/dist/fusion/fuse.js.map +7 -0
  50. package/dist/fusion/index.js +11 -0
  51. package/dist/fusion/index.js.map +7 -0
  52. package/dist/index.js +21 -0
  53. package/dist/index.js.map +7 -0
  54. package/dist/loader/describe.js +92 -0
  55. package/dist/loader/describe.js.map +7 -0
  56. package/dist/loader/index.js +31 -0
  57. package/dist/loader/index.js.map +7 -0
  58. package/dist/loader/manifest.js +32 -0
  59. package/dist/loader/manifest.js.map +7 -0
  60. package/dist/loader/registry.js +98 -0
  61. package/dist/loader/registry.js.map +7 -0
  62. package/dist/net/client.js +298 -0
  63. package/dist/net/client.js.map +7 -0
  64. package/dist/net/ssrf.js +165 -0
  65. package/dist/net/ssrf.js.map +7 -0
  66. package/dist/net/url.js +9 -0
  67. package/dist/net/url.js.map +7 -0
  68. package/dist/testing/contract.js +118 -0
  69. package/dist/testing/contract.js.map +7 -0
  70. package/dist/testing/fakeAdapter.js +44 -0
  71. package/dist/testing/fakeAdapter.js.map +7 -0
  72. package/dist/testing/index.js +15 -0
  73. package/dist/testing/index.js.map +7 -0
  74. package/jest.config.cjs +24 -0
  75. package/package.json +65 -0
  76. package/src/contract/adapter.ts +100 -0
  77. package/src/contract/errors.ts +48 -0
  78. package/src/contract/http.ts +46 -0
  79. package/src/contract/index.ts +8 -0
  80. package/src/contract/outcomes.ts +91 -0
  81. package/src/contract/policy.ts +157 -0
  82. package/src/contract/results.ts +79 -0
  83. package/src/contract/steps.ts +45 -0
  84. package/src/contract/version.ts +25 -0
  85. package/src/engine/__tests__/engine.test.ts +597 -0
  86. package/src/engine/cache.ts +46 -0
  87. package/src/engine/deadline.ts +65 -0
  88. package/src/engine/engine.ts +633 -0
  89. package/src/engine/index.ts +22 -0
  90. package/src/engine/runAdapter.ts +89 -0
  91. package/src/engine/types.ts +137 -0
  92. package/src/extract/__tests__/content.test.ts +81 -0
  93. package/src/extract/__tests__/text.test.ts +61 -0
  94. package/src/extract/__tests__/tokenizer.test.ts +85 -0
  95. package/src/extract/classify.ts +81 -0
  96. package/src/extract/content.ts +136 -0
  97. package/src/extract/entities.ts +98 -0
  98. package/src/extract/index.ts +11 -0
  99. package/src/extract/text.ts +124 -0
  100. package/src/extract/tokenizer.ts +197 -0
  101. package/src/fusion/__tests__/fusion.test.ts +198 -0
  102. package/src/fusion/canonicalize.ts +119 -0
  103. package/src/fusion/fuse.ts +145 -0
  104. package/src/fusion/index.ts +8 -0
  105. package/src/index.ts +15 -0
  106. package/src/loader/__tests__/describe.test.ts +119 -0
  107. package/src/loader/__tests__/registry.test.ts +134 -0
  108. package/src/loader/describe.ts +145 -0
  109. package/src/loader/index.ts +25 -0
  110. package/src/loader/manifest.ts +44 -0
  111. package/src/loader/registry.ts +156 -0
  112. package/src/net/__tests__/client.test.ts +273 -0
  113. package/src/net/__tests__/ssrf.test.ts +165 -0
  114. package/src/net/client.ts +434 -0
  115. package/src/net/ssrf.ts +235 -0
  116. package/src/net/url.ts +13 -0
  117. package/src/testing/contract.ts +172 -0
  118. package/src/testing/fakeAdapter.ts +69 -0
  119. package/src/testing/index.ts +9 -0
  120. package/tsconfig.json +10 -0
  121. package/watch.mjs +7 -0
@@ -0,0 +1,172 @@
1
+ import type { AdapterContext, AdapterModule, SearchAdapter } from '../contract/adapter'
2
+ import { silentLogger, type HttpClient, type HttpResponse } from '../contract/http'
3
+ import { CONTRACT_VERSION } from '../contract/version'
4
+ import type { SearchRequest } from '../contract/results'
5
+
6
+ export type StubResponse = Partial<HttpResponse> & { readonly body: string }
7
+
8
+ /**
9
+ * Minimal HTTP double. Adapters are contractually forbidden from reaching for
10
+ * `fetch`, so a test that passes with this client has also proved the adapter
11
+ * routes every byte through the hardened transport.
12
+ */
13
+ export function createStubHttpClient(
14
+ handler: (url: string, init: Parameters<HttpClient['request']>[1]) => StubResponse | Promise<StubResponse>,
15
+ ): HttpClient & { readonly calls: readonly string[] } {
16
+ const calls: string[] = []
17
+ return {
18
+ calls,
19
+ async request(url, init) {
20
+ calls.push(url)
21
+ const stub = await handler(url, init)
22
+ return {
23
+ url: stub.url ?? url,
24
+ status: stub.status ?? 200,
25
+ headers: stub.headers ?? new Map<string, string>(),
26
+ contentType: stub.contentType ?? 'text/html',
27
+ body: stub.body,
28
+ truncated: stub.truncated ?? false,
29
+ }
30
+ },
31
+ }
32
+ }
33
+
34
+ export function createTestContext(overrides: Partial<AdapterContext> = {}): AdapterContext {
35
+ return {
36
+ signal: new AbortController().signal,
37
+ http: createStubHttpClient(() => ({ body: '' })),
38
+ report: () => {},
39
+ deadlineAt: Date.now() + 10_000,
40
+ logger: silentLogger,
41
+ ...overrides,
42
+ }
43
+ }
44
+
45
+ export const SAMPLE_REQUEST: SearchRequest = { query: 'open mercato', limit: 5 }
46
+
47
+ const VALID_STATUSES = ['ok', 'empty', 'unavailable', 'blocked', 'timeout', 'error']
48
+
49
+ export type AdapterContractCase<TOptions> = {
50
+ /** Options that leave the adapter deliberately unconfigured. */
51
+ readonly unconfiguredOptions?: TOptions
52
+ /** Options sufficient for the adapter to consider itself ready. */
53
+ readonly configuredOptions: TOptions
54
+ /** Context used for the "returns results" probe; stub the HTTP client here. */
55
+ readonly context?: () => AdapterContext
56
+ }
57
+
58
+ type TestApi = {
59
+ describe: (name: string, body: () => void) => void
60
+ it: (name: string, body: () => void | Promise<void>) => void
61
+ expect: (value: unknown) => {
62
+ toBe(expected: unknown): void
63
+ toBeDefined(): void
64
+ toContain(expected: string): void
65
+ }
66
+ }
67
+
68
+ /**
69
+ * Behavioural suite every adapter must pass, ours and third parties'. It pins the
70
+ * parts of the contract the scheduler actually depends on — most importantly that
71
+ * an adapter reports rather than throws, because a throwing adapter would
72
+ * otherwise take down an entire search.
73
+ */
74
+ export function describeAdapterContract<TOptions>(
75
+ module: AdapterModule<TOptions>,
76
+ testCase: AdapterContractCase<TOptions>,
77
+ api: TestApi,
78
+ ): void {
79
+ const { describe, it, expect } = api
80
+
81
+ describe(`${module.id} adapter contract`, () => {
82
+ it('declares the contract version it was built against', () => {
83
+ expect(module.contractVersion).toBe(CONTRACT_VERSION)
84
+ })
85
+
86
+ it('exposes a stable id and kind', () => {
87
+ expect(typeof module.id).toBe('string')
88
+ expect(module.id.length > 0).toBe(true)
89
+ expect(['serp', 'api', 'model', 'browser'].includes(module.kind)).toBe(true)
90
+ })
91
+
92
+ it('validates its own options', () => {
93
+ const parsed = module.optionsSchema.safeParse(testCase.configuredOptions)
94
+ expect(parsed.success).toBe(true)
95
+ })
96
+
97
+ it('reports readiness without performing I/O', () => {
98
+ const adapter = module.createAdapter(testCase.configuredOptions)
99
+ const readiness = adapter.readiness()
100
+ expect(typeof readiness.ready).toBe('boolean')
101
+ })
102
+
103
+ const unconfigured = testCase.unconfiguredOptions
104
+ if (unconfigured !== undefined) {
105
+ it('is unavailable rather than broken when unconfigured', async () => {
106
+ const adapter = module.createAdapter(unconfigured)
107
+ const readiness = adapter.readiness()
108
+ expect(readiness.ready).toBe(false)
109
+ if (!readiness.ready) expect(readiness.reason.length > 0).toBe(true)
110
+
111
+ const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext())
112
+ expect(outcome.status).toBe('unavailable')
113
+ })
114
+ }
115
+
116
+ // The contract is "never throws", not "always fails when HTTP does" — a model
117
+ // or browser adapter may not touch the HTTP client at all.
118
+ it('returns an outcome rather than throwing when egress fails', async () => {
119
+ const adapter = module.createAdapter(testCase.configuredOptions)
120
+ const context = createTestContext({
121
+ http: createStubHttpClient(() => {
122
+ throw new Error('network exploded')
123
+ }),
124
+ })
125
+ const outcome = await adapter.search(SAMPLE_REQUEST, context)
126
+ expect(VALID_STATUSES.includes(outcome.status)).toBe(true)
127
+ })
128
+
129
+ it('honours an already-aborted signal without hanging', async () => {
130
+ const adapter = module.createAdapter(testCase.configuredOptions)
131
+ const controller = new AbortController()
132
+ controller.abort()
133
+ const outcome = await adapter.search(SAMPLE_REQUEST, createTestContext({ signal: controller.signal }))
134
+ expect(typeof outcome.status).toBe('string')
135
+ })
136
+
137
+ it('never returns a result without a url', async () => {
138
+ const adapter: SearchAdapter = module.createAdapter(testCase.configuredOptions)
139
+ const context = testCase.context ? testCase.context() : createTestContext()
140
+ const outcome = await adapter.search(SAMPLE_REQUEST, context)
141
+ if (outcome.status !== 'ok') return
142
+ for (const result of outcome.results) {
143
+ expect(typeof result.url).toBe('string')
144
+ expect(result.url.length > 0).toBe(true)
145
+ }
146
+ })
147
+
148
+ // The host builds an engine per request and disposes it after, so a `dispose`
149
+ // that throws on an adapter which was never used, or on a second call, breaks
150
+ // teardown for every other adapter in the same engine. This is the surface
151
+ // that leaked a browser sidecar and a Chromium per tool call.
152
+ it('tears down cleanly even when it was never used', async () => {
153
+ const adapter = module.createAdapter(testCase.configuredOptions)
154
+ if (!adapter.dispose) return
155
+ await adapter.dispose()
156
+ await adapter.dispose()
157
+ expect(true).toBe(true)
158
+ })
159
+
160
+ it('answers a health check without throwing', async () => {
161
+ const adapter = module.createAdapter(testCase.configuredOptions)
162
+ if (!adapter.healthCheck) return
163
+ const context = createTestContext({
164
+ http: createStubHttpClient(() => {
165
+ throw new Error('network exploded')
166
+ }),
167
+ })
168
+ const health = await adapter.healthCheck(context)
169
+ expect(typeof health.ok).toBe('boolean')
170
+ })
171
+ })
172
+ }
@@ -0,0 +1,69 @@
1
+ import type { AdapterCapabilities, AdapterKind, SearchAdapter } from '../contract/adapter'
2
+ import { READY, notReady } from '../contract/adapter'
3
+ import type { SearchOutcome } from '../contract/outcomes'
4
+ import type { RawResult } from '../contract/results'
5
+
6
+ export type FakeAdapterOptions = {
7
+ readonly id: string
8
+ readonly kind?: AdapterKind
9
+ readonly ready?: boolean | string
10
+ /** Milliseconds of simulated work before the outcome resolves. */
11
+ readonly latencyMs?: number
12
+ readonly outcome?: SearchOutcome
13
+ readonly results?: readonly RawResult[]
14
+ readonly capabilities?: Partial<AdapterCapabilities>
15
+ /** Set to make the adapter violate the contract, for isolation tests. */
16
+ readonly throws?: boolean
17
+ }
18
+
19
+ const BASE_CAPABILITIES: AdapterCapabilities = {
20
+ search: true,
21
+ fetch: false,
22
+ snippets: true,
23
+ content: false,
24
+ freshness: false,
25
+ siteFilter: false,
26
+ cost: 'free',
27
+ }
28
+
29
+ /** Deterministic adapter double for scheduler and fusion tests. */
30
+ export function createFakeAdapter(options: FakeAdapterOptions): SearchAdapter {
31
+ const capabilities: AdapterCapabilities = { ...BASE_CAPABILITIES, ...options.capabilities }
32
+ const outcome: SearchOutcome =
33
+ options.outcome ??
34
+ (options.results && options.results.length > 0
35
+ ? { status: 'ok', results: options.results }
36
+ : { status: 'empty' })
37
+
38
+ return {
39
+ id: options.id,
40
+ kind: options.kind ?? 'api',
41
+ capabilities,
42
+ readiness: () =>
43
+ options.ready === false
44
+ ? notReady('fake adapter configured as not ready')
45
+ : typeof options.ready === 'string'
46
+ ? notReady(options.ready)
47
+ : READY,
48
+ async search(_request, context) {
49
+ if (options.throws) throw new Error(`${options.id} exploded`)
50
+ const latency = options.latencyMs ?? 0
51
+ if (latency > 0) {
52
+ await new Promise<void>((resolve) => {
53
+ const timer = setTimeout(resolve, latency)
54
+ timer.unref?.()
55
+ context.signal.addEventListener(
56
+ 'abort',
57
+ () => {
58
+ clearTimeout(timer)
59
+ resolve()
60
+ },
61
+ { once: true },
62
+ )
63
+ })
64
+ }
65
+ if (context.signal.aborted) return { status: 'timeout', reason: 'aborted' }
66
+ return outcome
67
+ },
68
+ }
69
+ }
@@ -0,0 +1,9 @@
1
+ export {
2
+ SAMPLE_REQUEST,
3
+ createStubHttpClient,
4
+ createTestContext,
5
+ describeAdapterContract,
6
+ type AdapterContractCase,
7
+ type StubResponse,
8
+ } from './contract'
9
+ export { createFakeAdapter, type FakeAdapterOptions } from './fakeAdapter'
package/tsconfig.json ADDED
@@ -0,0 +1,10 @@
1
+ {
2
+ "$schema": "https://json.schemastore.org/tsconfig",
3
+ "extends": "../../tsconfig.base.json",
4
+ "compilerOptions": {
5
+ "noEmit": true,
6
+ "types": ["node"]
7
+ },
8
+ "include": ["src/**/*"],
9
+ "exclude": ["node_modules", "dist", "**/__tests__/**"]
10
+ }
package/watch.mjs ADDED
@@ -0,0 +1,7 @@
1
+ import { dirname } from 'node:path'
2
+ import { fileURLToPath } from 'node:url'
3
+ import { watch } from '../../scripts/watch.mjs'
4
+
5
+ const __dirname = dirname(fileURLToPath(import.meta.url))
6
+
7
+ watch(__dirname)