@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -0
- package/AGENTS.md +67 -0
- package/build.mjs +10 -0
- package/dist/contract/adapter.js +13 -0
- package/dist/contract/adapter.js.map +7 -0
- package/dist/contract/errors.js +28 -0
- package/dist/contract/errors.js.map +7 -0
- package/dist/contract/http.js +7 -0
- package/dist/contract/http.js.map +7 -0
- package/dist/contract/index.js +9 -0
- package/dist/contract/index.js.map +7 -0
- package/dist/contract/outcomes.js +58 -0
- package/dist/contract/outcomes.js.map +7 -0
- package/dist/contract/policy.js +91 -0
- package/dist/contract/policy.js.map +7 -0
- package/dist/contract/results.js +13 -0
- package/dist/contract/results.js.map +7 -0
- package/dist/contract/steps.js +1 -0
- package/dist/contract/steps.js.map +7 -0
- package/dist/contract/version.js +11 -0
- package/dist/contract/version.js.map +7 -0
- package/dist/engine/cache.js +29 -0
- package/dist/engine/cache.js.map +7 -0
- package/dist/engine/deadline.js +54 -0
- package/dist/engine/deadline.js.map +7 -0
- package/dist/engine/engine.js +481 -0
- package/dist/engine/engine.js.map +7 -0
- package/dist/engine/index.js +20 -0
- package/dist/engine/index.js.map +7 -0
- package/dist/engine/runAdapter.js +57 -0
- package/dist/engine/runAdapter.js.map +7 -0
- package/dist/engine/types.js +1 -0
- package/dist/engine/types.js.map +7 -0
- package/dist/extract/classify.js +51 -0
- package/dist/extract/classify.js.map +7 -0
- package/dist/extract/content.js +104 -0
- package/dist/extract/content.js.map +7 -0
- package/dist/extract/entities.js +95 -0
- package/dist/extract/entities.js.map +7 -0
- package/dist/extract/index.js +24 -0
- package/dist/extract/index.js.map +7 -0
- package/dist/extract/text.js +114 -0
- package/dist/extract/text.js.map +7 -0
- package/dist/extract/tokenizer.js +141 -0
- package/dist/extract/tokenizer.js.map +7 -0
- package/dist/fusion/canonicalize.js +88 -0
- package/dist/fusion/canonicalize.js.map +7 -0
- package/dist/fusion/fuse.js +88 -0
- package/dist/fusion/fuse.js.map +7 -0
- package/dist/fusion/index.js +11 -0
- package/dist/fusion/index.js.map +7 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +7 -0
- package/dist/loader/describe.js +92 -0
- package/dist/loader/describe.js.map +7 -0
- package/dist/loader/index.js +31 -0
- package/dist/loader/index.js.map +7 -0
- package/dist/loader/manifest.js +32 -0
- package/dist/loader/manifest.js.map +7 -0
- package/dist/loader/registry.js +98 -0
- package/dist/loader/registry.js.map +7 -0
- package/dist/net/client.js +298 -0
- package/dist/net/client.js.map +7 -0
- package/dist/net/ssrf.js +165 -0
- package/dist/net/ssrf.js.map +7 -0
- package/dist/net/url.js +9 -0
- package/dist/net/url.js.map +7 -0
- package/dist/testing/contract.js +118 -0
- package/dist/testing/contract.js.map +7 -0
- package/dist/testing/fakeAdapter.js +44 -0
- package/dist/testing/fakeAdapter.js.map +7 -0
- package/dist/testing/index.js +15 -0
- package/dist/testing/index.js.map +7 -0
- package/jest.config.cjs +24 -0
- package/package.json +65 -0
- package/src/contract/adapter.ts +100 -0
- package/src/contract/errors.ts +48 -0
- package/src/contract/http.ts +46 -0
- package/src/contract/index.ts +8 -0
- package/src/contract/outcomes.ts +91 -0
- package/src/contract/policy.ts +157 -0
- package/src/contract/results.ts +79 -0
- package/src/contract/steps.ts +45 -0
- package/src/contract/version.ts +25 -0
- package/src/engine/__tests__/engine.test.ts +597 -0
- package/src/engine/cache.ts +46 -0
- package/src/engine/deadline.ts +65 -0
- package/src/engine/engine.ts +633 -0
- package/src/engine/index.ts +22 -0
- package/src/engine/runAdapter.ts +89 -0
- package/src/engine/types.ts +137 -0
- package/src/extract/__tests__/content.test.ts +81 -0
- package/src/extract/__tests__/text.test.ts +61 -0
- package/src/extract/__tests__/tokenizer.test.ts +85 -0
- package/src/extract/classify.ts +81 -0
- package/src/extract/content.ts +136 -0
- package/src/extract/entities.ts +98 -0
- package/src/extract/index.ts +11 -0
- package/src/extract/text.ts +124 -0
- package/src/extract/tokenizer.ts +197 -0
- package/src/fusion/__tests__/fusion.test.ts +198 -0
- package/src/fusion/canonicalize.ts +119 -0
- package/src/fusion/fuse.ts +145 -0
- package/src/fusion/index.ts +8 -0
- package/src/index.ts +15 -0
- package/src/loader/__tests__/describe.test.ts +119 -0
- package/src/loader/__tests__/registry.test.ts +134 -0
- package/src/loader/describe.ts +145 -0
- package/src/loader/index.ts +25 -0
- package/src/loader/manifest.ts +44 -0
- package/src/loader/registry.ts +156 -0
- package/src/net/__tests__/client.test.ts +273 -0
- package/src/net/__tests__/ssrf.test.ts +165 -0
- package/src/net/client.ts +434 -0
- package/src/net/ssrf.ts +235 -0
- package/src/net/url.ts +13 -0
- package/src/testing/contract.ts +172 -0
- package/src/testing/fakeAdapter.ts +69 -0
- package/src/testing/index.ts +9 -0
- package/tsconfig.json +10 -0
- package/watch.mjs +7 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/extract/tokenizer.ts"],
|
|
4
|
+
"sourcesContent": ["import { decodeEntities } from './entities'\n\nexport type HtmlToken =\n | { readonly kind: 'text'; readonly value: string }\n | {\n readonly kind: 'open'\n readonly name: string\n readonly attributes: ReadonlyMap<string, string>\n readonly selfClosing: boolean\n }\n | { readonly kind: 'close'; readonly name: string }\n\n/**\n * Bounds memory for a pathological document. The tokenizer itself is linear and\n * cannot stall the event loop, so this is an allocation guard rather than the\n * ReDoS mitigation a regex-based extractor would need it to be.\n */\nexport const MAX_HTML_INPUT = 8 * 1024 * 1024\n\nconst VOID_ELEMENTS = new Set([\n 'area',\n 'base',\n 'br',\n 'col',\n 'embed',\n 'hr',\n 'img',\n 'input',\n 'link',\n 'meta',\n 'param',\n 'source',\n 'track',\n 'wbr',\n])\n\nconst RAW_TEXT_ELEMENTS = new Set(['script', 'style'])\n\nfunction isNameStart(character: string): boolean {\n return (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')\n}\n\nfunction isWhitespace(character: string): boolean {\n return character === ' ' || character === '\\t' || character === '\\n' || character === '\\r' || character === '\\f'\n}\n\ntype ParsedTag = {\n readonly name: string\n readonly attributes: Map<string, string>\n readonly selfClosing: boolean\n readonly end: number\n}\n\n/**\n * Reads one start tag beginning at the character after `<`. Quote-aware so an\n * attribute value containing `>` does not truncate the tag.\n */\nfunction readStartTag(html: string, start: number): ParsedTag | null {\n let position = start\n while (position < html.length && !isWhitespace(html[position]) && html[position] !== '>' && html[position] !== '/') {\n position += 1\n }\n const name = html.slice(start, position).toLowerCase()\n if (name.length === 0) return null\n\n const attributes = new Map<string, string>()\n let selfClosing = false\n\n while (position < html.length) {\n while (position < html.length && isWhitespace(html[position])) position += 1\n if (position >= html.length) break\n if (html[position] === '>') {\n position += 1\n break\n }\n if (html[position] === '/') {\n selfClosing = true\n position += 1\n continue\n }\n\n const nameStart = position\n while (\n position < html.length &&\n !isWhitespace(html[position]) &&\n html[position] !== '=' &&\n html[position] !== '>' &&\n html[position] !== '/'\n ) {\n position += 1\n }\n const attributeName = html.slice(nameStart, position).toLowerCase()\n if (attributeName.length === 0) {\n position += 1\n continue\n }\n\n while (position < html.length && isWhitespace(html[position])) position += 1\n if (html[position] !== '=') {\n attributes.set(attributeName, '')\n continue\n }\n position += 1\n while (position < html.length && isWhitespace(html[position])) position += 1\n\n const quote = html[position]\n if (quote === '\"' || quote === \"'\") {\n const valueStart = position + 1\n const valueEnd = html.indexOf(quote, valueStart)\n if (valueEnd === -1) {\n attributes.set(attributeName, decodeEntities(html.slice(valueStart)))\n position = html.length\n break\n }\n attributes.set(attributeName, decodeEntities(html.slice(valueStart, valueEnd)))\n position = valueEnd + 1\n continue\n }\n\n const valueStart = position\n while (position < html.length && !isWhitespace(html[position]) && html[position] !== '>') position += 1\n attributes.set(attributeName, decodeEntities(html.slice(valueStart, position)))\n }\n\n return { name, attributes, selfClosing: selfClosing || VOID_ELEMENTS.has(name), end: position }\n}\n\n/**\n * Single forward scan over HTML emitting structural tokens. Every advance is\n * driven by `indexOf` or a bounded character walk, so there is no backtracking\n * and runtime is linear in the input length.\n */\nexport function* tokenizeHtml(rawHtml: string): Generator<HtmlToken> {\n const html = rawHtml.length > MAX_HTML_INPUT ? rawHtml.slice(0, MAX_HTML_INPUT) : rawHtml\n // Lowercased once: searching for a raw-text close tag inside the loop would\n // otherwise copy the whole document per <script>/<style>.\n let lowerHtml: string | null = null\n let position = 0\n\n while (position < html.length) {\n const angle = html.indexOf('<', position)\n if (angle === -1) {\n const text = html.slice(position)\n if (text.length > 0) yield { kind: 'text', value: decodeEntities(text) }\n return\n }\n if (angle > position) {\n yield { kind: 'text', value: decodeEntities(html.slice(position, angle)) }\n }\n\n const next = html[angle + 1]\n\n if (next === '!') {\n if (html.startsWith('<!--', angle)) {\n const close = html.indexOf('-->', angle + 4)\n position = close === -1 ? html.length : close + 3\n continue\n }\n const close = html.indexOf('>', angle + 2)\n position = close === -1 ? html.length : close + 1\n continue\n }\n\n if (next === '/') {\n const close = html.indexOf('>', angle + 2)\n const end = close === -1 ? html.length : close\n const name = html.slice(angle + 2, end).trim().toLowerCase()\n if (name.length > 0) yield { kind: 'close', name }\n position = close === -1 ? html.length : close + 1\n continue\n }\n\n if (next === undefined || !isNameStart(next)) {\n yield { kind: 'text', value: '<' }\n position = angle + 1\n continue\n }\n\n const tag = readStartTag(html, angle + 1)\n if (tag === null) {\n yield { kind: 'text', value: '<' }\n position = angle + 1\n continue\n }\n yield { kind: 'open', name: tag.name, attributes: tag.attributes, selfClosing: tag.selfClosing }\n position = tag.end\n\n if (RAW_TEXT_ELEMENTS.has(tag.name) && !tag.selfClosing) {\n lowerHtml ??= html.toLowerCase()\n const close = lowerHtml.indexOf(`</${tag.name}`, position)\n position = close === -1 ? html.length : close\n continue\n }\n\n if (tag.selfClosing) yield { kind: 'close', name: tag.name }\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,sBAAsB;AAiBxB,MAAM,iBAAiB,IAAI,OAAO;AAEzC,MAAM,gBAAgB,oBAAI,IAAI;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,MAAM,oBAAoB,oBAAI,IAAI,CAAC,UAAU,OAAO,CAAC;AAErD,SAAS,YAAY,WAA4B;AAC/C,SAAQ,aAAa,OAAO,aAAa,OAAS,aAAa,OAAO,aAAa;AACrF;AAEA,SAAS,aAAa,WAA4B;AAChD,SAAO,cAAc,OAAO,cAAc,OAAQ,cAAc,QAAQ,cAAc,QAAQ,cAAc;AAC9G;AAaA,SAAS,aAAa,MAAc,OAAiC;AACnE,MAAI,WAAW;AACf,SAAO,WAAW,KAAK,UAAU,CAAC,aAAa,KAAK,QAAQ,CAAC,KAAK,KAAK,QAAQ,MAAM,OAAO,KAAK,QAAQ,MAAM,KAAK;AAClH,gBAAY;AAAA,EACd;AACA,QAAM,OAAO,KAAK,MAAM,OAAO,QAAQ,EAAE,YAAY;AACrD,MAAI,KAAK,WAAW,EAAG,QAAO;AAE9B,QAAM,aAAa,oBAAI,IAAoB;AAC3C,MAAI,cAAc;AAElB,SAAO,WAAW,KAAK,QAAQ;AAC7B,WAAO,WAAW,KAAK,UAAU,aAAa,KAAK,QAAQ,CAAC,EAAG,aAAY;AAC3E,QAAI,YAAY,KAAK,OAAQ;AAC7B,QAAI,KAAK,QAAQ,MAAM,KAAK;AAC1B,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,KAAK,QAAQ,MAAM,KAAK;AAC1B,oBAAc;AACd,kBAAY;AACZ;AAAA,IACF;AAEA,UAAM,YAAY;AAClB,WACE,WAAW,KAAK,UAChB,CAAC,aAAa,KAAK,QAAQ,CAAC,KAC5B,KAAK,QAAQ,MAAM,OACnB,KAAK,QAAQ,MAAM,OACnB,KAAK,QAAQ,MAAM,KACnB;AACA,kBAAY;AAAA,IACd;AACA,UAAM,gBAAgB,KAAK,MAAM,WAAW,QAAQ,EAAE,YAAY;AAClE,QAAI,cAAc,WAAW,GAAG;AAC9B,kBAAY;AACZ;AAAA,IACF;AAEA,WAAO,WAAW,KAAK,UAAU,aAAa,KAAK,QAAQ,CAAC,EAAG,aAAY;AAC3E,QAAI,KAAK,QAAQ,MAAM,KAAK;AAC1B,iBAAW,IAAI,eAAe,EAAE;AAChC;AAAA,IACF;AACA,gBAAY;AACZ,WAAO,WAAW,KAAK,UAAU,aAAa,KAAK,QAAQ,CAAC,EAAG,aAAY;AAE3E,UAAM,QAAQ,KAAK,QAAQ;AAC3B,QAAI,UAAU,OAAO,UAAU,KAAK;AAClC,YAAMA,cAAa,WAAW;AAC9B,YAAM,WAAW,KAAK,QAAQ,OAAOA,WAAU;AAC/C,UAAI,aAAa,IAAI;AACnB,mBAAW,IAAI,eAAe,eAAe,KAAK,MAAMA,WAAU,CAAC,CAAC;AACpE,mBAAW,KAAK;AAChB;AAAA,MACF;AACA,iBAAW,IAAI,eAAe,eAAe,KAAK,MAAMA,aAAY,QAAQ,CAAC,CAAC;AAC9E,iBAAW,WAAW;AACtB;AAAA,IACF;AAEA,UAAM,aAAa;AACnB,WAAO,WAAW,KAAK,UAAU,CAAC,aAAa,KAAK,QAAQ,CAAC,KAAK,KAAK,QAAQ,MAAM,IAAK,aAAY;AACtG,eAAW,IAAI,eAAe,eAAe,KAAK,MAAM,YAAY,QAAQ,CAAC,CAAC;AAAA,EAChF;AAEA,SAAO,EAAE,MAAM,YAAY,aAAa,eAAe,cAAc,IAAI,IAAI,GAAG,KAAK,SAAS;AAChG;AAOO,UAAU,aAAa,SAAuC;AACnE,QAAM,OAAO,QAAQ,SAAS,iBAAiB,QAAQ,MAAM,GAAG,cAAc,IAAI;AAGlF,MAAI,YAA2B;AAC/B,MAAI,WAAW;AAEf,SAAO,WAAW,KAAK,QAAQ;AAC7B,UAAM,QAAQ,KAAK,QAAQ,KAAK,QAAQ;AACxC,QAAI,UAAU,IAAI;AAChB,YAAM,OAAO,KAAK,MAAM,QAAQ;AAChC,UAAI,KAAK,SAAS,EAAG,OAAM,EAAE,MAAM,QAAQ,OAAO,eAAe,IAAI,EAAE;AACvE;AAAA,IACF;AACA,QAAI,QAAQ,UAAU;AACpB,YAAM,EAAE,MAAM,QAAQ,OAAO,eAAe,KAAK,MAAM,UAAU,KAAK,CAAC,EAAE;AAAA,IAC3E;AAEA,UAAM,OAAO,KAAK,QAAQ,CAAC;AAE3B,QAAI,SAAS,KAAK;AAChB,UAAI,KAAK,WAAW,QAAQ,KAAK,GAAG;AAClC,cAAMC,SAAQ,KAAK,QAAQ,OAAO,QAAQ,CAAC;AAC3C,mBAAWA,WAAU,KAAK,KAAK,SAASA,SAAQ;AAChD;AAAA,MACF;AACA,YAAM,QAAQ,KAAK,QAAQ,KAAK,QAAQ,CAAC;AACzC,iBAAW,UAAU,KAAK,KAAK,SAAS,QAAQ;AAChD;AAAA,IACF;AAEA,QAAI,SAAS,KAAK;AAChB,YAAM,QAAQ,KAAK,QAAQ,KAAK,QAAQ,CAAC;AACzC,YAAM,MAAM,UAAU,KAAK,KAAK,SAAS;AACzC,YAAM,OAAO,KAAK,MAAM,QAAQ,GAAG,GAAG,EAAE,KAAK,EAAE,YAAY;AAC3D,UAAI,KAAK,SAAS,EAAG,OAAM,EAAE,MAAM,SAAS,KAAK;AACjD,iBAAW,UAAU,KAAK,KAAK,SAAS,QAAQ;AAChD;AAAA,IACF;AAEA,QAAI,SAAS,UAAa,CAAC,YAAY,IAAI,GAAG;AAC5C,YAAM,EAAE,MAAM,QAAQ,OAAO,IAAI;AACjC,iBAAW,QAAQ;AACnB;AAAA,IACF;AAEA,UAAM,MAAM,aAAa,MAAM,QAAQ,CAAC;AACxC,QAAI,QAAQ,MAAM;AAChB,YAAM,EAAE,MAAM,QAAQ,OAAO,IAAI;AACjC,iBAAW,QAAQ;AACnB;AAAA,IACF;AACA,UAAM,EAAE,MAAM,QAAQ,MAAM,IAAI,MAAM,YAAY,IAAI,YAAY,aAAa,IAAI,YAAY;AAC/F,eAAW,IAAI;AAEf,QAAI,kBAAkB,IAAI,IAAI,IAAI,KAAK,CAAC,IAAI,aAAa;AACvD,oBAAc,KAAK,YAAY;AAC/B,YAAM,QAAQ,UAAU,QAAQ,KAAK,IAAI,IAAI,IAAI,QAAQ;AACzD,iBAAW,UAAU,KAAK,KAAK,SAAS;AACxC;AAAA,IACF;AAEA,QAAI,IAAI,YAAa,OAAM,EAAE,MAAM,SAAS,MAAM,IAAI,KAAK;AAAA,EAC7D;AACF;",
|
|
6
|
+
"names": ["valueStart", "close"]
|
|
7
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
const TRACKING_PARAMS = /* @__PURE__ */ new Set([
|
|
2
|
+
"fbclid",
|
|
3
|
+
"gclid",
|
|
4
|
+
"gclsrc",
|
|
5
|
+
"dclid",
|
|
6
|
+
"msclkid",
|
|
7
|
+
"yclid",
|
|
8
|
+
"twclid",
|
|
9
|
+
"igshid",
|
|
10
|
+
"mc_cid",
|
|
11
|
+
"mc_eid",
|
|
12
|
+
"ref_src",
|
|
13
|
+
"ref_url",
|
|
14
|
+
"s_kwcid",
|
|
15
|
+
"vero_id",
|
|
16
|
+
"wickedid",
|
|
17
|
+
"_ga",
|
|
18
|
+
"_gl",
|
|
19
|
+
"spm"
|
|
20
|
+
]);
|
|
21
|
+
const REDIRECT_HOSTS = [
|
|
22
|
+
{ host: /(^|\.)duckduckgo\.com$/, path: /^\/l\/?$/, params: ["uddg"] },
|
|
23
|
+
{ host: /(^|\.)google\.[a-z.]+$/, path: /^\/url$/, params: ["q", "url"] },
|
|
24
|
+
{ host: /(^|\.)out\.reddit\.com$/, params: ["url"] },
|
|
25
|
+
{ host: /(^|\.)steamcommunity\.com$/, path: /^\/linkfilter\/?$/, params: ["url"] }
|
|
26
|
+
];
|
|
27
|
+
const AD_ENDPOINTS = [
|
|
28
|
+
{ host: /(^|\.)duckduckgo\.com$/, path: /^\/y\.js$/ }
|
|
29
|
+
];
|
|
30
|
+
function isAdEndpoint(url) {
|
|
31
|
+
return AD_ENDPOINTS.some((rule) => rule.host.test(url.hostname) && rule.path.test(url.pathname));
|
|
32
|
+
}
|
|
33
|
+
function isTracking(key) {
|
|
34
|
+
const lower = key.toLowerCase();
|
|
35
|
+
return lower.startsWith("utm_") || TRACKING_PARAMS.has(lower);
|
|
36
|
+
}
|
|
37
|
+
function unwrapRedirect(url) {
|
|
38
|
+
for (const rule of REDIRECT_HOSTS) {
|
|
39
|
+
if (!rule.host.test(url.hostname)) continue;
|
|
40
|
+
if (rule.path && !rule.path.test(url.pathname)) continue;
|
|
41
|
+
for (const param of rule.params) {
|
|
42
|
+
const target = url.searchParams.get(param);
|
|
43
|
+
if (!target) continue;
|
|
44
|
+
try {
|
|
45
|
+
const parsed = new URL(target);
|
|
46
|
+
if (parsed.protocol === "http:" || parsed.protocol === "https:") return parsed;
|
|
47
|
+
} catch {
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
function canonicalizeUrl(raw) {
|
|
55
|
+
let parsed;
|
|
56
|
+
try {
|
|
57
|
+
parsed = new URL(raw.trim());
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
for (let hop = 0; hop < 3; hop += 1) {
|
|
62
|
+
const unwrapped = unwrapRedirect(parsed);
|
|
63
|
+
if (!unwrapped) break;
|
|
64
|
+
parsed = unwrapped;
|
|
65
|
+
}
|
|
66
|
+
if (isAdEndpoint(parsed)) return null;
|
|
67
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
68
|
+
if (parsed.hostname.length === 0) return null;
|
|
69
|
+
parsed.hash = "";
|
|
70
|
+
parsed.username = "";
|
|
71
|
+
parsed.password = "";
|
|
72
|
+
const retained = [...parsed.searchParams.entries()].filter(([key2]) => !isTracking(key2));
|
|
73
|
+
retained.sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0);
|
|
74
|
+
parsed.search = "";
|
|
75
|
+
for (const [key2, value] of retained) parsed.searchParams.append(key2, value);
|
|
76
|
+
const host = parsed.hostname.toLowerCase();
|
|
77
|
+
parsed.hostname = host;
|
|
78
|
+
if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
|
|
79
|
+
parsed.pathname = parsed.pathname.replace(/\/+$/, "");
|
|
80
|
+
}
|
|
81
|
+
const domain = host.startsWith("www.") ? host.slice(4) : host;
|
|
82
|
+
const key = `${domain}${parsed.pathname}${parsed.search}`;
|
|
83
|
+
return { url: parsed.toString(), key, domain };
|
|
84
|
+
}
|
|
85
|
+
export {
|
|
86
|
+
canonicalizeUrl
|
|
87
|
+
};
|
|
88
|
+
//# sourceMappingURL=canonicalize.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/fusion/canonicalize.ts"],
|
|
4
|
+
"sourcesContent": ["export type CanonicalUrl = {\n /** Cleaned URL suitable for display and re-fetching. */\n readonly url: string\n /** Aggressively normalized form used only for dedup. */\n readonly key: string\n readonly domain: string\n}\n\nconst TRACKING_PARAMS = new Set([\n 'fbclid',\n 'gclid',\n 'gclsrc',\n 'dclid',\n 'msclkid',\n 'yclid',\n 'twclid',\n 'igshid',\n 'mc_cid',\n 'mc_eid',\n 'ref_src',\n 'ref_url',\n 's_kwcid',\n 'vero_id',\n 'wickedid',\n '_ga',\n '_gl',\n 'spm',\n])\n\n/** Query keys carrying the real destination on known redirect hops. */\nconst REDIRECT_HOSTS: ReadonlyArray<{ host: RegExp; path?: RegExp; params: readonly string[] }> = [\n { host: /(^|\\.)duckduckgo\\.com$/, path: /^\\/l\\/?$/, params: ['uddg'] },\n { host: /(^|\\.)google\\.[a-z.]+$/, path: /^\\/url$/, params: ['q', 'url'] },\n { host: /(^|\\.)out\\.reddit\\.com$/, params: ['url'] },\n { host: /(^|\\.)steamcommunity\\.com$/, path: /^\\/linkfilter\\/?$/, params: ['url'] },\n]\n\n/**\n * Endpoints that are never a real result. DuckDuckGo renders sponsored slots as\n * `y.js` click-tracking links carrying `ad_domain`, and they sit in the same\n * markup as organic hits \u2014 so a SERP parser picks them up, they survive a\n * `site:` filter that should have excluded them, and an agent ends up citing or\n * fetching an advert. There is nothing to unwrap here that is worth keeping.\n */\nconst AD_ENDPOINTS: ReadonlyArray<{ host: RegExp; path: RegExp }> = [\n { host: /(^|\\.)duckduckgo\\.com$/, path: /^\\/y\\.js$/ },\n]\n\nfunction isAdEndpoint(url: URL): boolean {\n return AD_ENDPOINTS.some((rule) => rule.host.test(url.hostname) && rule.path.test(url.pathname))\n}\n\nfunction isTracking(key: string): boolean {\n const lower = key.toLowerCase()\n return lower.startsWith('utm_') || TRACKING_PARAMS.has(lower)\n}\n\nfunction unwrapRedirect(url: URL): URL | null {\n for (const rule of REDIRECT_HOSTS) {\n if (!rule.host.test(url.hostname)) continue\n if (rule.path && !rule.path.test(url.pathname)) continue\n for (const param of rule.params) {\n const target = url.searchParams.get(param)\n if (!target) continue\n try {\n const parsed = new URL(target)\n if (parsed.protocol === 'http:' || parsed.protocol === 'https:') return parsed\n } catch {\n continue\n }\n }\n }\n return null\n}\n\n/**\n * Normalizes a result URL for display and produces a separate dedup key. The two\n * differ on purpose: the key folds `www.` and the scheme so the same page found\n * by two adapters collapses into one hit, while the displayed URL stays faithful\n * enough to fetch.\n */\nexport function canonicalizeUrl(raw: string): CanonicalUrl | null {\n let parsed: URL\n try {\n parsed = new URL(raw.trim())\n } catch {\n return null\n }\n\n for (let hop = 0; hop < 3; hop += 1) {\n const unwrapped = unwrapRedirect(parsed)\n if (!unwrapped) break\n parsed = unwrapped\n }\n\n if (isAdEndpoint(parsed)) return null\n if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null\n if (parsed.hostname.length === 0) return null\n\n parsed.hash = ''\n parsed.username = ''\n parsed.password = ''\n\n const retained = [...parsed.searchParams.entries()].filter(([key]) => !isTracking(key))\n retained.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))\n parsed.search = ''\n for (const [key, value] of retained) parsed.searchParams.append(key, value)\n\n const host = parsed.hostname.toLowerCase()\n parsed.hostname = host\n if (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {\n parsed.pathname = parsed.pathname.replace(/\\/+$/, '')\n }\n\n const domain = host.startsWith('www.') ? host.slice(4) : host\n const key = `${domain}${parsed.pathname}${parsed.search}`\n\n return { url: parsed.toString(), key, domain }\n}\n"],
|
|
5
|
+
"mappings": "AAQA,MAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAGD,MAAM,iBAA4F;AAAA,EAChG,EAAE,MAAM,0BAA0B,MAAM,YAAY,QAAQ,CAAC,MAAM,EAAE;AAAA,EACrE,EAAE,MAAM,0BAA0B,MAAM,WAAW,QAAQ,CAAC,KAAK,KAAK,EAAE;AAAA,EACxE,EAAE,MAAM,2BAA2B,QAAQ,CAAC,KAAK,EAAE;AAAA,EACnD,EAAE,MAAM,8BAA8B,MAAM,qBAAqB,QAAQ,CAAC,KAAK,EAAE;AACnF;AASA,MAAM,eAA8D;AAAA,EAClE,EAAE,MAAM,0BAA0B,MAAM,YAAY;AACtD;AAEA,SAAS,aAAa,KAAmB;AACvC,SAAO,aAAa,KAAK,CAAC,SAAS,KAAK,KAAK,KAAK,IAAI,QAAQ,KAAK,KAAK,KAAK,KAAK,IAAI,QAAQ,CAAC;AACjG;AAEA,SAAS,WAAW,KAAsB;AACxC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,MAAM,WAAW,MAAM,KAAK,gBAAgB,IAAI,KAAK;AAC9D;AAEA,SAAS,eAAe,KAAsB;AAC5C,aAAW,QAAQ,gBAAgB;AACjC,QAAI,CAAC,KAAK,KAAK,KAAK,IAAI,QAAQ,EAAG;AACnC,QAAI,KAAK,QAAQ,CAAC,KAAK,KAAK,KAAK,IAAI,QAAQ,EAAG;AAChD,eAAW,SAAS,KAAK,QAAQ;AAC/B,YAAM,SAAS,IAAI,aAAa,IAAI,KAAK;AACzC,UAAI,CAAC,OAAQ;AACb,UAAI;AACF,cAAM,SAAS,IAAI,IAAI,MAAM;AAC7B,YAAI,OAAO,aAAa,WAAW,OAAO,aAAa,SAAU,QAAO;AAAA,MAC1E,QAAQ;AACN;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAQO,SAAS,gBAAgB,KAAkC;AAChE,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,IAAI,KAAK,CAAC;AAAA,EAC7B,QAAQ;AACN,WAAO;AAAA,EACT;AAEA,WAAS,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG;AACnC,UAAM,YAAY,eAAe,MAAM;AACvC,QAAI,CAAC,UAAW;AAChB,aAAS;AAAA,EACX;AAEA,MAAI,aAAa,MAAM,EAAG,QAAO;AACjC,MAAI,OAAO,aAAa,WAAW,OAAO,aAAa,SAAU,QAAO;AACxE,MAAI,OAAO,SAAS,WAAW,EAAG,QAAO;AAEzC,SAAO,OAAO;AACd,SAAO,WAAW;AAClB,SAAO,WAAW;AAElB,QAAM,WAAW,CAAC,GAAG,OAAO,aAAa,QAAQ,CAAC,EAAE,OAAO,CAAC,CAACA,IAAG,MAAM,CAAC,WAAWA,IAAG,CAAC;AACtF,WAAS,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,MAAO,OAAO,QAAQ,KAAK,OAAO,QAAQ,IAAI,CAAE;AAC7E,SAAO,SAAS;AAChB,aAAW,CAACA,MAAK,KAAK,KAAK,SAAU,QAAO,aAAa,OAAOA,MAAK,KAAK;AAE1E,QAAM,OAAO,OAAO,SAAS,YAAY;AACzC,SAAO,WAAW;AAClB,MAAI,OAAO,SAAS,SAAS,KAAK,OAAO,SAAS,SAAS,GAAG,GAAG;AAC/D,WAAO,WAAW,OAAO,SAAS,QAAQ,QAAQ,EAAE;AAAA,EACtD;AAEA,QAAM,SAAS,KAAK,WAAW,MAAM,IAAI,KAAK,MAAM,CAAC,IAAI;AACzD,QAAM,MAAM,GAAG,MAAM,GAAG,OAAO,QAAQ,GAAG,OAAO,MAAM;AAEvD,SAAO,EAAE,KAAK,OAAO,SAAS,GAAG,KAAK,OAAO;AAC/C;",
|
|
6
|
+
"names": ["key"]
|
|
7
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { canonicalizeUrl } from "./canonicalize.js";
|
|
2
|
+
const RRF_K = 60;
|
|
3
|
+
function longest(current, candidate) {
|
|
4
|
+
const next = candidate?.trim();
|
|
5
|
+
if (!next) return current;
|
|
6
|
+
if (current === null || next.length > current.length) return next;
|
|
7
|
+
return current;
|
|
8
|
+
}
|
|
9
|
+
function fuseResults(inputs, options) {
|
|
10
|
+
const accumulators = /* @__PURE__ */ new Map();
|
|
11
|
+
for (const input of inputs) {
|
|
12
|
+
const weight = input.weight > 0 ? input.weight : 1;
|
|
13
|
+
const contributed = /* @__PURE__ */ new Set();
|
|
14
|
+
let rank = 0;
|
|
15
|
+
for (const result of input.results) {
|
|
16
|
+
const canonical = canonicalizeUrl(result.url);
|
|
17
|
+
if (canonical === null) continue;
|
|
18
|
+
if (contributed.has(canonical.key)) continue;
|
|
19
|
+
contributed.add(canonical.key);
|
|
20
|
+
rank += 1;
|
|
21
|
+
const existing = accumulators.get(canonical.key);
|
|
22
|
+
const accumulator = existing ?? {
|
|
23
|
+
key: canonical.key,
|
|
24
|
+
domain: canonical.domain,
|
|
25
|
+
url: canonical.url,
|
|
26
|
+
title: null,
|
|
27
|
+
snippet: null,
|
|
28
|
+
content: null,
|
|
29
|
+
publishedAt: null,
|
|
30
|
+
score: 0,
|
|
31
|
+
sources: []
|
|
32
|
+
};
|
|
33
|
+
accumulator.title = longest(accumulator.title, result.title);
|
|
34
|
+
accumulator.snippet = longest(accumulator.snippet, result.snippet);
|
|
35
|
+
accumulator.content = accumulator.content ?? (result.content?.trim() || null);
|
|
36
|
+
accumulator.publishedAt = accumulator.publishedAt ?? (result.publishedAt?.trim() || null);
|
|
37
|
+
accumulator.score += weight / (RRF_K + rank);
|
|
38
|
+
accumulator.sources.push({ adapterId: input.adapterId, rank });
|
|
39
|
+
if (!existing) accumulators.set(canonical.key, accumulator);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (accumulators.size === 0) return { results: [], degraded: false };
|
|
43
|
+
const maxScore = Math.max(...[...accumulators.values()].map((entry) => entry.score));
|
|
44
|
+
const ranked = [...accumulators.values()].map((entry) => ({
|
|
45
|
+
entry,
|
|
46
|
+
result: {
|
|
47
|
+
url: entry.url,
|
|
48
|
+
title: entry.title ?? entry.url,
|
|
49
|
+
snippet: entry.snippet,
|
|
50
|
+
content: entry.content,
|
|
51
|
+
publishedAt: entry.publishedAt,
|
|
52
|
+
confidence: maxScore > 0 ? Number((entry.score / maxScore).toFixed(6)) : 0,
|
|
53
|
+
sources: entry.sources
|
|
54
|
+
}
|
|
55
|
+
})).sort((left, right) => {
|
|
56
|
+
if (right.result.confidence !== left.result.confidence) {
|
|
57
|
+
return right.result.confidence - left.result.confidence;
|
|
58
|
+
}
|
|
59
|
+
return left.entry.key < right.entry.key ? -1 : left.entry.key > right.entry.key ? 1 : 0;
|
|
60
|
+
});
|
|
61
|
+
const perDomain = /* @__PURE__ */ new Map();
|
|
62
|
+
const primary = [];
|
|
63
|
+
const overflow = [];
|
|
64
|
+
for (const { entry, result } of ranked) {
|
|
65
|
+
const used = perDomain.get(entry.domain) ?? 0;
|
|
66
|
+
if (used >= options.maxPerDomain || result.confidence < options.minConfidence) {
|
|
67
|
+
overflow.push(result);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
perDomain.set(entry.domain, used + 1);
|
|
71
|
+
primary.push(result);
|
|
72
|
+
}
|
|
73
|
+
let degraded = false;
|
|
74
|
+
const selected = primary.slice(0, options.limit);
|
|
75
|
+
if (selected.length < Math.min(options.minResults, ranked.length)) {
|
|
76
|
+
const needed = Math.min(options.minResults, options.limit) - selected.length;
|
|
77
|
+
if (needed > 0 && overflow.length > 0) {
|
|
78
|
+
degraded = true;
|
|
79
|
+
selected.push(...overflow.slice(0, needed));
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return { results: selected, degraded };
|
|
83
|
+
}
|
|
84
|
+
export {
|
|
85
|
+
RRF_K,
|
|
86
|
+
fuseResults
|
|
87
|
+
};
|
|
88
|
+
//# sourceMappingURL=fuse.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/fusion/fuse.ts"],
|
|
4
|
+
"sourcesContent": ["import type { RawResult, ResultProvenance, SearchResult } from '../contract/results'\nimport { canonicalizeUrl } from './canonicalize'\n\n/**\n * Reciprocal-rank-fusion damping constant. 60 is the value from the original\n * Cormack et al. formulation and is deliberately large relative to typical result\n * counts, so a hit ranked 1 by one adapter cannot dominate a hit ranked 2 by two.\n */\nexport const RRF_K = 60\n\nexport type FusionInput = {\n readonly adapterId: string\n readonly weight: number\n readonly results: readonly RawResult[]\n}\n\nexport type FusionOptions = {\n readonly limit: number\n readonly minResults: number\n readonly minConfidence: number\n readonly maxPerDomain: number\n}\n\nexport type FusionOutput = {\n readonly results: readonly SearchResult[]\n /** True when the confidence threshold or domain cap had to be relaxed. */\n readonly degraded: boolean\n}\n\ntype Accumulator = {\n readonly key: string\n readonly domain: string\n url: string\n title: string | null\n snippet: string | null\n content: string | null\n publishedAt: string | null\n score: number\n readonly sources: ResultProvenance[]\n}\n\nfunction longest(current: string | null, candidate: string | null | undefined): string | null {\n const next = candidate?.trim()\n if (!next) return current\n if (current === null || next.length > current.length) return next\n return current\n}\n\n/**\n * Merges per-adapter result lists into one ranked set.\n *\n * RRF is used rather than blending native scores because the sources are not\n * comparable: an LLM citation list has no score at all, a SERP has an opaque\n * one, and a paid API has its own scale. Rank is the only signal every adapter\n * genuinely agrees on.\n */\nexport function fuseResults(inputs: readonly FusionInput[], options: FusionOptions): FusionOutput {\n const accumulators = new Map<string, Accumulator>()\n\n for (const input of inputs) {\n const weight = input.weight > 0 ? input.weight : 1\n // Two spellings of one URL inside a single adapter's list must not each earn\n // that adapter a rank slot: the duplicate would compound its own score and\n // let one source outvote the agreement between two independent ones.\n const contributed = new Set<string>()\n let rank = 0\n for (const result of input.results) {\n const canonical = canonicalizeUrl(result.url)\n if (canonical === null) continue\n if (contributed.has(canonical.key)) continue\n contributed.add(canonical.key)\n rank += 1\n const existing = accumulators.get(canonical.key)\n const accumulator: Accumulator = existing ?? {\n key: canonical.key,\n domain: canonical.domain,\n url: canonical.url,\n title: null,\n snippet: null,\n content: null,\n publishedAt: null,\n score: 0,\n sources: [],\n }\n accumulator.title = longest(accumulator.title, result.title)\n accumulator.snippet = longest(accumulator.snippet, result.snippet)\n accumulator.content = accumulator.content ?? (result.content?.trim() || null)\n accumulator.publishedAt = accumulator.publishedAt ?? (result.publishedAt?.trim() || null)\n accumulator.score += weight / (RRF_K + rank)\n accumulator.sources.push({ adapterId: input.adapterId, rank })\n if (!existing) accumulators.set(canonical.key, accumulator)\n }\n }\n\n if (accumulators.size === 0) return { results: [], degraded: false }\n\n const maxScore = Math.max(...[...accumulators.values()].map((entry) => entry.score))\n const ranked = [...accumulators.values()]\n .map((entry) => ({\n entry,\n result: {\n url: entry.url,\n title: entry.title ?? entry.url,\n snippet: entry.snippet,\n content: entry.content,\n publishedAt: entry.publishedAt,\n confidence: maxScore > 0 ? Number((entry.score / maxScore).toFixed(6)) : 0,\n sources: entry.sources,\n } satisfies SearchResult,\n }))\n .sort((left, right) => {\n if (right.result.confidence !== left.result.confidence) {\n return right.result.confidence - left.result.confidence\n }\n return left.entry.key < right.entry.key ? -1 : left.entry.key > right.entry.key ? 1 : 0\n })\n\n const perDomain = new Map<string, number>()\n const primary: SearchResult[] = []\n const overflow: SearchResult[] = []\n\n for (const { entry, result } of ranked) {\n const used = perDomain.get(entry.domain) ?? 0\n if (used >= options.maxPerDomain || result.confidence < options.minConfidence) {\n overflow.push(result)\n continue\n }\n perDomain.set(entry.domain, used + 1)\n primary.push(result)\n }\n\n // A confidence threshold or a domain cap must never manufacture an empty\n // answer: fall back to the best rejected hits and report the relaxation.\n let degraded = false\n const selected = primary.slice(0, options.limit)\n if (selected.length < Math.min(options.minResults, ranked.length)) {\n const needed = Math.min(options.minResults, options.limit) - selected.length\n if (needed > 0 && overflow.length > 0) {\n degraded = true\n selected.push(...overflow.slice(0, needed))\n }\n }\n\n return { results: selected, degraded }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,uBAAuB;AAOzB,MAAM,QAAQ;AAiCrB,SAAS,QAAQ,SAAwB,WAAqD;AAC5F,QAAM,OAAO,WAAW,KAAK;AAC7B,MAAI,CAAC,KAAM,QAAO;AAClB,MAAI,YAAY,QAAQ,KAAK,SAAS,QAAQ,OAAQ,QAAO;AAC7D,SAAO;AACT;AAUO,SAAS,YAAY,QAAgC,SAAsC;AAChG,QAAM,eAAe,oBAAI,IAAyB;AAElD,aAAW,SAAS,QAAQ;AAC1B,UAAM,SAAS,MAAM,SAAS,IAAI,MAAM,SAAS;AAIjD,UAAM,cAAc,oBAAI,IAAY;AACpC,QAAI,OAAO;AACX,eAAW,UAAU,MAAM,SAAS;AAClC,YAAM,YAAY,gBAAgB,OAAO,GAAG;AAC5C,UAAI,cAAc,KAAM;AACxB,UAAI,YAAY,IAAI,UAAU,GAAG,EAAG;AACpC,kBAAY,IAAI,UAAU,GAAG;AAC7B,cAAQ;AACR,YAAM,WAAW,aAAa,IAAI,UAAU,GAAG;AAC/C,YAAM,cAA2B,YAAY;AAAA,QAC3C,KAAK,UAAU;AAAA,QACf,QAAQ,UAAU;AAAA,QAClB,KAAK,UAAU;AAAA,QACf,OAAO;AAAA,QACP,SAAS;AAAA,QACT,SAAS;AAAA,QACT,aAAa;AAAA,QACb,OAAO;AAAA,QACP,SAAS,CAAC;AAAA,MACZ;AACA,kBAAY,QAAQ,QAAQ,YAAY,OAAO,OAAO,KAAK;AAC3D,kBAAY,UAAU,QAAQ,YAAY,SAAS,OAAO,OAAO;AACjE,kBAAY,UAAU,YAAY,YAAY,OAAO,SAAS,KAAK,KAAK;AACxE,kBAAY,cAAc,YAAY,gBAAgB,OAAO,aAAa,KAAK,KAAK;AACpF,kBAAY,SAAS,UAAU,QAAQ;AACvC,kBAAY,QAAQ,KAAK,EAAE,WAAW,MAAM,WAAW,KAAK,CAAC;AAC7D,UAAI,CAAC,SAAU,cAAa,IAAI,UAAU,KAAK,WAAW;AAAA,IAC5D;AAAA,EACF;AAEA,MAAI,aAAa,SAAS,EAAG,QAAO,EAAE,SAAS,CAAC,GAAG,UAAU,MAAM;AAEnE,QAAM,WAAW,KAAK,IAAI,GAAG,CAAC,GAAG,aAAa,OAAO,CAAC,EAAE,IAAI,CAAC,UAAU,MAAM,KAAK,CAAC;AACnF,QAAM,SAAS,CAAC,GAAG,aAAa,OAAO,CAAC,EACrC,IAAI,CAAC,WAAW;AAAA,IACf;AAAA,IACA,QAAQ;AAAA,MACN,KAAK,MAAM;AAAA,MACX,OAAO,MAAM,SAAS,MAAM;AAAA,MAC5B,SAAS,MAAM;AAAA,MACf,SAAS,MAAM;AAAA,MACf,aAAa,MAAM;AAAA,MACnB,YAAY,WAAW,IAAI,QAAQ,MAAM,QAAQ,UAAU,QAAQ,CAAC,CAAC,IAAI;AAAA,MACzE,SAAS,MAAM;AAAA,IACjB;AAAA,EACF,EAAE,EACD,KAAK,CAAC,MAAM,UAAU;AACrB,QAAI,MAAM,OAAO,eAAe,KAAK,OAAO,YAAY;AACtD,aAAO,MAAM,OAAO,aAAa,KAAK,OAAO;AAAA,IAC/C;AACA,WAAO,KAAK,MAAM,MAAM,MAAM,MAAM,MAAM,KAAK,KAAK,MAAM,MAAM,MAAM,MAAM,MAAM,IAAI;AAAA,EACxF,CAAC;AAEH,QAAM,YAAY,oBAAI,IAAoB;AAC1C,QAAM,UAA0B,CAAC;AACjC,QAAM,WAA2B,CAAC;AAElC,aAAW,EAAE,OAAO,OAAO,KAAK,QAAQ;AACtC,UAAM,OAAO,UAAU,IAAI,MAAM,MAAM,KAAK;AAC5C,QAAI,QAAQ,QAAQ,gBAAgB,OAAO,aAAa,QAAQ,eAAe;AAC7E,eAAS,KAAK,MAAM;AACpB;AAAA,IACF;AACA,cAAU,IAAI,MAAM,QAAQ,OAAO,CAAC;AACpC,YAAQ,KAAK,MAAM;AAAA,EACrB;AAIA,MAAI,WAAW;AACf,QAAM,WAAW,QAAQ,MAAM,GAAG,QAAQ,KAAK;AAC/C,MAAI,SAAS,SAAS,KAAK,IAAI,QAAQ,YAAY,OAAO,MAAM,GAAG;AACjE,UAAM,SAAS,KAAK,IAAI,QAAQ,YAAY,QAAQ,KAAK,IAAI,SAAS;AACtE,QAAI,SAAS,KAAK,SAAS,SAAS,GAAG;AACrC,iBAAW;AACX,eAAS,KAAK,GAAG,SAAS,MAAM,GAAG,MAAM,CAAC;AAAA,IAC5C;AAAA,EACF;AAEA,SAAO,EAAE,SAAS,UAAU,SAAS;AACvC;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/fusion/index.ts"],
|
|
4
|
+
"sourcesContent": ["export { canonicalizeUrl, type CanonicalUrl } from './canonicalize'\nexport {\n RRF_K,\n fuseResults,\n type FusionInput,\n type FusionOptions,\n type FusionOutput,\n} from './fuse'\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,uBAA0C;AACnD;AAAA,EACE;AAAA,EACA;AAAA,OAIK;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export * from "./contract/index.js";
|
|
2
|
+
export * from "./engine/index.js";
|
|
3
|
+
export * from "./extract/index.js";
|
|
4
|
+
export * from "./fusion/index.js";
|
|
5
|
+
export * from "./loader/index.js";
|
|
6
|
+
import { createHttpClient, createPinnedLookup } from "./net/client.js";
|
|
7
|
+
import { stripTrailingSlashes } from "./net/url.js";
|
|
8
|
+
import {
|
|
9
|
+
assertPublicUrl,
|
|
10
|
+
isLoopbackHostname,
|
|
11
|
+
isPrivateAddress
|
|
12
|
+
} from "./net/ssrf.js";
|
|
13
|
+
export {
|
|
14
|
+
assertPublicUrl,
|
|
15
|
+
createHttpClient,
|
|
16
|
+
createPinnedLookup,
|
|
17
|
+
isLoopbackHostname,
|
|
18
|
+
isPrivateAddress,
|
|
19
|
+
stripTrailingSlashes
|
|
20
|
+
};
|
|
21
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/index.ts"],
|
|
4
|
+
"sourcesContent": ["export * from './contract'\nexport * from './engine'\nexport * from './extract'\nexport * from './fusion'\nexport * from './loader'\nexport { createHttpClient, createPinnedLookup, type HttpClientOptions } from './net/client'\nexport { stripTrailingSlashes } from './net/url'\nexport {\n assertPublicUrl,\n isLoopbackHostname,\n isPrivateAddress,\n type AssertPublicUrlOptions,\n type LookupFn,\n type VettedUrl,\n} from './net/ssrf'\n"],
|
|
5
|
+
"mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,SAAS,kBAAkB,0BAAkD;AAC7E,SAAS,4BAA4B;AACrC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OAIK;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
const SECRET_WORD = /(^|_)(api_?key|key|token|secret|password|passwd|credential)s?(_|$)/;
|
|
2
|
+
function isSecretName(name) {
|
|
3
|
+
return SECRET_WORD.test(name.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase());
|
|
4
|
+
}
|
|
5
|
+
function unwrap(schema) {
|
|
6
|
+
let def = schema.def;
|
|
7
|
+
let required = true;
|
|
8
|
+
while (def && (def.type === "optional" || def.type === "nullable" || def.type === "default")) {
|
|
9
|
+
if (def.type !== "nullable") required = false;
|
|
10
|
+
def = def.innerType?.def;
|
|
11
|
+
}
|
|
12
|
+
return { inner: def ?? {}, required };
|
|
13
|
+
}
|
|
14
|
+
function boundsOf(inner) {
|
|
15
|
+
const bounds = {};
|
|
16
|
+
for (const check of inner.checks ?? []) {
|
|
17
|
+
const def = check?._zod?.def;
|
|
18
|
+
if (!def || typeof def.value !== "number") continue;
|
|
19
|
+
if (def.check === "greater_than" || def.check === "min_length") bounds.min = def.value;
|
|
20
|
+
if (def.check === "less_than" || def.check === "max_length") bounds.max = def.value;
|
|
21
|
+
}
|
|
22
|
+
return bounds;
|
|
23
|
+
}
|
|
24
|
+
function kindOf(inner) {
|
|
25
|
+
switch (inner.type) {
|
|
26
|
+
case "string":
|
|
27
|
+
return "string";
|
|
28
|
+
case "number":
|
|
29
|
+
case "int":
|
|
30
|
+
return "number";
|
|
31
|
+
case "boolean":
|
|
32
|
+
return "boolean";
|
|
33
|
+
case "enum":
|
|
34
|
+
return "enum";
|
|
35
|
+
case "array":
|
|
36
|
+
return inner.element?.def?.type === "string" ? "stringList" : null;
|
|
37
|
+
default:
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
function describeOptionsSchema(module) {
|
|
42
|
+
const schema = module.optionsSchema;
|
|
43
|
+
const shape = schema.shape ?? schema.def?.shape;
|
|
44
|
+
if (!shape) return [];
|
|
45
|
+
const fields = [];
|
|
46
|
+
for (const [name, entry] of Object.entries(shape)) {
|
|
47
|
+
const { inner, required } = unwrap(entry);
|
|
48
|
+
const kind = kindOf(inner);
|
|
49
|
+
if (kind === null) continue;
|
|
50
|
+
const bounds = boundsOf(inner);
|
|
51
|
+
fields.push({
|
|
52
|
+
name,
|
|
53
|
+
kind,
|
|
54
|
+
required,
|
|
55
|
+
secret: isSecretName(name),
|
|
56
|
+
...kind === "enum" && inner.entries ? { choices: Object.keys(inner.entries) } : {},
|
|
57
|
+
...inner.format ? { format: inner.format } : {},
|
|
58
|
+
...bounds.min === void 0 ? {} : { min: bounds.min },
|
|
59
|
+
...bounds.max === void 0 ? {} : { max: bounds.max }
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
return fields;
|
|
63
|
+
}
|
|
64
|
+
const SECRET_PLACEHOLDER = "__om_secret_unchanged__";
|
|
65
|
+
function maskSecrets(values, fields) {
|
|
66
|
+
const masked = { ...values };
|
|
67
|
+
for (const field of fields) {
|
|
68
|
+
if (!field.secret) continue;
|
|
69
|
+
const value = masked[field.name];
|
|
70
|
+
if (typeof value === "string" && value.length > 0) masked[field.name] = SECRET_PLACEHOLDER;
|
|
71
|
+
}
|
|
72
|
+
return masked;
|
|
73
|
+
}
|
|
74
|
+
function unmaskSecrets(incoming, stored, fields) {
|
|
75
|
+
const merged = { ...incoming };
|
|
76
|
+
for (const field of fields) {
|
|
77
|
+
if (!field.secret) continue;
|
|
78
|
+
if (merged[field.name] === SECRET_PLACEHOLDER) {
|
|
79
|
+
const previous = stored[field.name];
|
|
80
|
+
if (previous === void 0) delete merged[field.name];
|
|
81
|
+
else merged[field.name] = previous;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return merged;
|
|
85
|
+
}
|
|
86
|
+
export {
|
|
87
|
+
SECRET_PLACEHOLDER,
|
|
88
|
+
describeOptionsSchema,
|
|
89
|
+
maskSecrets,
|
|
90
|
+
unmaskSecrets
|
|
91
|
+
};
|
|
92
|
+
//# sourceMappingURL=describe.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/loader/describe.ts"],
|
|
4
|
+
"sourcesContent": ["import type { AdapterModule } from '../contract/adapter'\n\nexport type OptionFieldKind = 'string' | 'number' | 'boolean' | 'enum' | 'stringList'\n\nexport type OptionField = {\n readonly name: string\n readonly kind: OptionFieldKind\n readonly required: boolean\n /** True when the value must never be echoed back to a browser. */\n readonly secret: boolean\n readonly choices?: readonly string[]\n readonly format?: string\n readonly min?: number\n readonly max?: number\n}\n\nconst SECRET_WORD = /(^|_)(api_?key|key|token|secret|password|passwd|credential)s?(_|$)/\n\n/**\n * Whether a field name looks like a credential. The name is split at camelCase\n * boundaries first, so `accessToken` and `access_token` are treated alike \u2014 a\n * plain word-boundary match misses the camelCase spelling entirely.\n */\nfunction isSecretName(name: string): boolean {\n return SECRET_WORD.test(name.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase())\n}\n\ntype ZodInternals = {\n type?: string\n innerType?: { def?: ZodInternals }\n entries?: Record<string, unknown>\n format?: string\n element?: { def?: ZodInternals }\n checks?: Array<{ _zod?: { def?: { check?: string; value?: number } } }>\n}\n\ntype ZodLike = { def?: ZodInternals; shape?: Record<string, ZodLike> }\n\nfunction unwrap(schema: ZodLike): { inner: ZodInternals; required: boolean } {\n let def = schema.def\n let required = true\n // `.optional()` / `.nullable()` / `.default()` wrap the real type.\n while (def && (def.type === 'optional' || def.type === 'nullable' || def.type === 'default')) {\n if (def.type !== 'nullable') required = false\n def = def.innerType?.def\n }\n return { inner: def ?? {}, required }\n}\n\nfunction boundsOf(inner: ZodInternals): { min?: number; max?: number } {\n const bounds: { min?: number; max?: number } = {}\n for (const check of inner.checks ?? []) {\n const def = check?._zod?.def\n if (!def || typeof def.value !== 'number') continue\n if (def.check === 'greater_than' || def.check === 'min_length') bounds.min = def.value\n if (def.check === 'less_than' || def.check === 'max_length') bounds.max = def.value\n }\n return bounds\n}\n\nfunction kindOf(inner: ZodInternals): OptionFieldKind | null {\n switch (inner.type) {\n case 'string':\n return 'string'\n case 'number':\n case 'int':\n return 'number'\n case 'boolean':\n return 'boolean'\n case 'enum':\n return 'enum'\n case 'array':\n // Only string arrays are renderable as a plain comma-separated input.\n return inner.element?.def?.type === 'string' ? 'stringList' : null\n default:\n // `custom` is how an adapter declares a host-supplied capability\n // (a model resolver, a sidecar handle). Not user-configurable, so it must\n // never reach the admin form.\n return null\n }\n}\n\n/**\n * Describes an adapter's options as renderable fields, so the settings UI can\n * build a form for an adapter it has never heard of \u2014 including one published by\n * a third party \u2014 without the adapter shipping any UI of its own.\n */\nexport function describeOptionsSchema(module: AdapterModule<unknown>): OptionField[] {\n const schema = module.optionsSchema as unknown as ZodLike\n const shape = schema.shape ?? (schema.def as { shape?: Record<string, ZodLike> } | undefined)?.shape\n if (!shape) return []\n\n const fields: OptionField[] = []\n for (const [name, entry] of Object.entries(shape)) {\n const { inner, required } = unwrap(entry)\n const kind = kindOf(inner)\n if (kind === null) continue\n const bounds = boundsOf(inner)\n fields.push({\n name,\n kind,\n required,\n secret: isSecretName(name),\n ...(kind === 'enum' && inner.entries ? { choices: Object.keys(inner.entries) } : {}),\n ...(inner.format ? { format: inner.format } : {}),\n ...(bounds.min === undefined ? {} : { min: bounds.min }),\n ...(bounds.max === undefined ? {} : { max: bounds.max }),\n })\n }\n return fields\n}\n\n/** Placeholder returned instead of a stored secret, and accepted back unchanged. */\nexport const SECRET_PLACEHOLDER = '__om_secret_unchanged__'\n\nexport function maskSecrets(\n values: Record<string, unknown>,\n fields: readonly OptionField[],\n): Record<string, unknown> {\n const masked: Record<string, unknown> = { ...values }\n for (const field of fields) {\n if (!field.secret) continue\n const value = masked[field.name]\n if (typeof value === 'string' && value.length > 0) masked[field.name] = SECRET_PLACEHOLDER\n }\n return masked\n}\n\n/** Restores stored secrets the client echoed back as the placeholder. */\nexport function unmaskSecrets(\n incoming: Record<string, unknown>,\n stored: Record<string, unknown>,\n fields: readonly OptionField[],\n): Record<string, unknown> {\n const merged: Record<string, unknown> = { ...incoming }\n for (const field of fields) {\n if (!field.secret) continue\n if (merged[field.name] === SECRET_PLACEHOLDER) {\n const previous = stored[field.name]\n if (previous === undefined) delete merged[field.name]\n else merged[field.name] = previous\n }\n }\n return merged\n}\n"],
|
|
5
|
+
"mappings": "AAgBA,MAAM,cAAc;AAOpB,SAAS,aAAa,MAAuB;AAC3C,SAAO,YAAY,KAAK,KAAK,QAAQ,sBAAsB,OAAO,EAAE,YAAY,CAAC;AACnF;AAaA,SAAS,OAAO,QAA6D;AAC3E,MAAI,MAAM,OAAO;AACjB,MAAI,WAAW;AAEf,SAAO,QAAQ,IAAI,SAAS,cAAc,IAAI,SAAS,cAAc,IAAI,SAAS,YAAY;AAC5F,QAAI,IAAI,SAAS,WAAY,YAAW;AACxC,UAAM,IAAI,WAAW;AAAA,EACvB;AACA,SAAO,EAAE,OAAO,OAAO,CAAC,GAAG,SAAS;AACtC;AAEA,SAAS,SAAS,OAAqD;AACrE,QAAM,SAAyC,CAAC;AAChD,aAAW,SAAS,MAAM,UAAU,CAAC,GAAG;AACtC,UAAM,MAAM,OAAO,MAAM;AACzB,QAAI,CAAC,OAAO,OAAO,IAAI,UAAU,SAAU;AAC3C,QAAI,IAAI,UAAU,kBAAkB,IAAI,UAAU,aAAc,QAAO,MAAM,IAAI;AACjF,QAAI,IAAI,UAAU,eAAe,IAAI,UAAU,aAAc,QAAO,MAAM,IAAI;AAAA,EAChF;AACA,SAAO;AACT;AAEA,SAAS,OAAO,OAA6C;AAC3D,UAAQ,MAAM,MAAM;AAAA,IAClB,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAEH,aAAO,MAAM,SAAS,KAAK,SAAS,WAAW,eAAe;AAAA,IAChE;AAIE,aAAO;AAAA,EACX;AACF;AAOO,SAAS,sBAAsB,QAA+C;AACnF,QAAM,SAAS,OAAO;AACtB,QAAM,QAAQ,OAAO,SAAU,OAAO,KAAyD;AAC/F,MAAI,CAAC,MAAO,QAAO,CAAC;AAEpB,QAAM,SAAwB,CAAC;AAC/B,aAAW,CAAC,MAAM,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AACjD,UAAM,EAAE,OAAO,SAAS,IAAI,OAAO,KAAK;AACxC,UAAM,OAAO,OAAO,KAAK;AACzB,QAAI,SAAS,KAAM;AACnB,UAAM,SAAS,SAAS,KAAK;AAC7B,WAAO,KAAK;AAAA,MACV;AAAA,MACA;AAAA,MACA;AAAA,MACA,QAAQ,aAAa,IAAI;AAAA,MACzB,GAAI,SAAS,UAAU,MAAM,UAAU,EAAE,SAAS,OAAO,KAAK,MAAM,OAAO,EAAE,IAAI,CAAC;AAAA,MAClF,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;AAAA,MAC/C,GAAI,OAAO,QAAQ,SAAY,CAAC,IAAI,EAAE,KAAK,OAAO,IAAI;AAAA,MACtD,GAAI,OAAO,QAAQ,SAAY,CAAC,IAAI,EAAE,KAAK,OAAO,IAAI;AAAA,IACxD,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAGO,MAAM,qBAAqB;AAE3B,SAAS,YACd,QACA,QACyB;AACzB,QAAM,SAAkC,EAAE,GAAG,OAAO;AACpD,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,MAAM,OAAQ;AACnB,UAAM,QAAQ,OAAO,MAAM,IAAI;AAC/B,QAAI,OAAO,UAAU,YAAY,MAAM,SAAS,EAAG,QAAO,MAAM,IAAI,IAAI;AAAA,EAC1E;AACA,SAAO;AACT;AAGO,SAAS,cACd,UACA,QACA,QACyB;AACzB,QAAM,SAAkC,EAAE,GAAG,SAAS;AACtD,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,MAAM,OAAQ;AACnB,QAAI,OAAO,MAAM,IAAI,MAAM,oBAAoB;AAC7C,YAAM,WAAW,OAAO,MAAM,IAAI;AAClC,UAAI,aAAa,OAAW,QAAO,OAAO,MAAM,IAAI;AAAA,UAC/C,QAAO,MAAM,IAAI,IAAI;AAAA,IAC5B;AAAA,EACF;AACA,SAAO;AACT;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import {
|
|
2
|
+
SECRET_PLACEHOLDER,
|
|
3
|
+
describeOptionsSchema,
|
|
4
|
+
maskSecrets,
|
|
5
|
+
unmaskSecrets
|
|
6
|
+
} from "./describe.js";
|
|
7
|
+
import {
|
|
8
|
+
ADAPTER_MANIFEST_KEY,
|
|
9
|
+
MANIFEST_NAMESPACE,
|
|
10
|
+
adapterManifestSchema,
|
|
11
|
+
readAdapterManifest
|
|
12
|
+
} from "./manifest.js";
|
|
13
|
+
import {
|
|
14
|
+
createUnavailableAdapter,
|
|
15
|
+
instantiateAdapter,
|
|
16
|
+
resolveAdapterModules
|
|
17
|
+
} from "./registry.js";
|
|
18
|
+
export {
|
|
19
|
+
ADAPTER_MANIFEST_KEY,
|
|
20
|
+
MANIFEST_NAMESPACE,
|
|
21
|
+
SECRET_PLACEHOLDER,
|
|
22
|
+
adapterManifestSchema,
|
|
23
|
+
createUnavailableAdapter,
|
|
24
|
+
describeOptionsSchema,
|
|
25
|
+
instantiateAdapter,
|
|
26
|
+
maskSecrets,
|
|
27
|
+
readAdapterManifest,
|
|
28
|
+
resolveAdapterModules,
|
|
29
|
+
unmaskSecrets
|
|
30
|
+
};
|
|
31
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/loader/index.ts"],
|
|
4
|
+
"sourcesContent": ["export {\n SECRET_PLACEHOLDER,\n describeOptionsSchema,\n maskSecrets,\n unmaskSecrets,\n type OptionField,\n type OptionFieldKind,\n} from './describe'\nexport {\n ADAPTER_MANIFEST_KEY,\n MANIFEST_NAMESPACE,\n adapterManifestSchema,\n readAdapterManifest,\n type AdapterManifest,\n} from './manifest'\nexport {\n createUnavailableAdapter,\n instantiateAdapter,\n resolveAdapterModules,\n type AdapterModuleRegistry,\n type AdapterRegistryEntry,\n type InstantiatedAdapter,\n type LoadedAdapterModule,\n type RejectedAdapterModule,\n} from './registry'\n"],
|
|
5
|
+
"mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAGK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OAMK;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
const MANIFEST_NAMESPACE = "openMercato";
|
|
3
|
+
const ADAPTER_MANIFEST_KEY = "webResearchAdapter";
|
|
4
|
+
const adapterManifestSchema = z.object({
|
|
5
|
+
id: z.string().min(1).max(64),
|
|
6
|
+
/** Import specifier relative to the package root; defaults to the package main. */
|
|
7
|
+
entry: z.string().min(1).optional()
|
|
8
|
+
});
|
|
9
|
+
function readAdapterManifest(packageJson) {
|
|
10
|
+
if (typeof packageJson !== "object" || packageJson === null) return null;
|
|
11
|
+
const parsed = packageJson;
|
|
12
|
+
const namespace = parsed[MANIFEST_NAMESPACE];
|
|
13
|
+
if (typeof namespace !== "object" || namespace === null) return null;
|
|
14
|
+
const raw = namespace[ADAPTER_MANIFEST_KEY];
|
|
15
|
+
if (raw === void 0) return null;
|
|
16
|
+
const packageName = typeof parsed.name === "string" ? parsed.name : "";
|
|
17
|
+
if (packageName.length === 0) {
|
|
18
|
+
return { error: "package declares a web-research adapter but has no name" };
|
|
19
|
+
}
|
|
20
|
+
const result = adapterManifestSchema.safeParse(raw);
|
|
21
|
+
if (!result.success) {
|
|
22
|
+
return { error: `${packageName} has an invalid ${MANIFEST_NAMESPACE}.${ADAPTER_MANIFEST_KEY}: ${result.error.message}` };
|
|
23
|
+
}
|
|
24
|
+
return { manifest: result.data, packageName };
|
|
25
|
+
}
|
|
26
|
+
export {
|
|
27
|
+
ADAPTER_MANIFEST_KEY,
|
|
28
|
+
MANIFEST_NAMESPACE,
|
|
29
|
+
adapterManifestSchema,
|
|
30
|
+
readAdapterManifest
|
|
31
|
+
};
|
|
32
|
+
//# sourceMappingURL=manifest.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/loader/manifest.ts"],
|
|
4
|
+
"sourcesContent": ["import { z } from 'zod'\n\n/** `package.json` \u2192 `openMercato.webResearchAdapter`. */\nexport const MANIFEST_NAMESPACE = 'openMercato'\nexport const ADAPTER_MANIFEST_KEY = 'webResearchAdapter'\n\nexport const adapterManifestSchema = z.object({\n id: z.string().min(1).max(64),\n /** Import specifier relative to the package root; defaults to the package main. */\n entry: z.string().min(1).optional(),\n})\n\nexport type AdapterManifest = z.infer<typeof adapterManifestSchema>\n\ntype PackageJsonLike = {\n readonly name?: unknown\n readonly [MANIFEST_NAMESPACE]?: unknown\n}\n\n/**\n * Reads the adapter manifest out of a parsed `package.json`. Returns null for any\n * package that simply is not an adapter, and only reports a problem when a\n * package claims to be one but declares it wrongly.\n */\nexport function readAdapterManifest(\n packageJson: unknown,\n): { manifest: AdapterManifest; packageName: string } | { error: string } | null {\n if (typeof packageJson !== 'object' || packageJson === null) return null\n const parsed = packageJson as PackageJsonLike\n const namespace = parsed[MANIFEST_NAMESPACE]\n if (typeof namespace !== 'object' || namespace === null) return null\n const raw = (namespace as Record<string, unknown>)[ADAPTER_MANIFEST_KEY]\n if (raw === undefined) return null\n\n const packageName = typeof parsed.name === 'string' ? parsed.name : ''\n if (packageName.length === 0) {\n return { error: 'package declares a web-research adapter but has no name' }\n }\n const result = adapterManifestSchema.safeParse(raw)\n if (!result.success) {\n return { error: `${packageName} has an invalid ${MANIFEST_NAMESPACE}.${ADAPTER_MANIFEST_KEY}: ${result.error.message}` }\n }\n return { manifest: result.data, packageName }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,SAAS;AAGX,MAAM,qBAAqB;AAC3B,MAAM,uBAAuB;AAE7B,MAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE;AAAA;AAAA,EAE5B,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,SAAS;AACpC,CAAC;AAcM,SAAS,oBACd,aAC+E;AAC/E,MAAI,OAAO,gBAAgB,YAAY,gBAAgB,KAAM,QAAO;AACpE,QAAM,SAAS;AACf,QAAM,YAAY,OAAO,kBAAkB;AAC3C,MAAI,OAAO,cAAc,YAAY,cAAc,KAAM,QAAO;AAChE,QAAM,MAAO,UAAsC,oBAAoB;AACvE,MAAI,QAAQ,OAAW,QAAO;AAE9B,QAAM,cAAc,OAAO,OAAO,SAAS,WAAW,OAAO,OAAO;AACpE,MAAI,YAAY,WAAW,GAAG;AAC5B,WAAO,EAAE,OAAO,0DAA0D;AAAA,EAC5E;AACA,QAAM,SAAS,sBAAsB,UAAU,GAAG;AAClD,MAAI,CAAC,OAAO,SAAS;AACnB,WAAO,EAAE,OAAO,GAAG,WAAW,mBAAmB,kBAAkB,IAAI,oBAAoB,KAAK,OAAO,MAAM,OAAO,GAAG;AAAA,EACzH;AACA,SAAO,EAAE,UAAU,OAAO,MAAM,YAAY;AAC9C;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { notReady } from "../contract/adapter.js";
|
|
2
|
+
import { errorMessage } from "../contract/errors.js";
|
|
3
|
+
import { unavailable } from "../contract/outcomes.js";
|
|
4
|
+
import { CONTRACT_VERSION, MIN_SUPPORTED_CONTRACT_VERSION, isSupportedContractVersion } from "../contract/version.js";
|
|
5
|
+
const ADAPTER_KINDS = /* @__PURE__ */ new Set(["serp", "api", "model", "browser"]);
|
|
6
|
+
function createUnavailableAdapter(id, reason) {
|
|
7
|
+
return {
|
|
8
|
+
id,
|
|
9
|
+
kind: "api",
|
|
10
|
+
capabilities: {
|
|
11
|
+
search: false,
|
|
12
|
+
fetch: false,
|
|
13
|
+
snippets: false,
|
|
14
|
+
content: false,
|
|
15
|
+
freshness: false,
|
|
16
|
+
siteFilter: false,
|
|
17
|
+
cost: "free"
|
|
18
|
+
},
|
|
19
|
+
readiness: () => notReady(reason),
|
|
20
|
+
search: async () => unavailable(reason),
|
|
21
|
+
healthCheck: async () => ({ ok: false, detail: reason })
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
function describeModule(candidate) {
|
|
25
|
+
const unwrapped = typeof candidate === "object" && candidate !== null && "default" in candidate && !("createAdapter" in candidate) ? candidate.default : candidate;
|
|
26
|
+
if (typeof unwrapped !== "object" || unwrapped === null) {
|
|
27
|
+
return "module does not export an adapter descriptor";
|
|
28
|
+
}
|
|
29
|
+
const module = unwrapped;
|
|
30
|
+
if (typeof module.id !== "string" || module.id.length === 0) return "adapter descriptor has no id";
|
|
31
|
+
if (typeof module.kind !== "string" || !ADAPTER_KINDS.has(module.kind)) {
|
|
32
|
+
return `adapter "${module.id}" declares an unknown kind`;
|
|
33
|
+
}
|
|
34
|
+
if (typeof module.createAdapter !== "function") {
|
|
35
|
+
return `adapter "${module.id}" does not expose createAdapter()`;
|
|
36
|
+
}
|
|
37
|
+
if (typeof module.optionsSchema !== "object" || module.optionsSchema === null || typeof module.optionsSchema.safeParse !== "function") {
|
|
38
|
+
return `adapter "${module.id}" does not expose a zod optionsSchema`;
|
|
39
|
+
}
|
|
40
|
+
if (!isSupportedContractVersion(module.contractVersion)) {
|
|
41
|
+
return `adapter "${module.id}" targets contract version ${String(module.contractVersion)}, this engine speaks ${MIN_SUPPORTED_CONTRACT_VERSION}..${CONTRACT_VERSION}`;
|
|
42
|
+
}
|
|
43
|
+
return module;
|
|
44
|
+
}
|
|
45
|
+
function resolveAdapterModules(entries) {
|
|
46
|
+
const loaded = [];
|
|
47
|
+
const rejected = [];
|
|
48
|
+
const seen = /* @__PURE__ */ new Map();
|
|
49
|
+
for (const entry of entries) {
|
|
50
|
+
const described = describeModule(entry.module);
|
|
51
|
+
if (typeof described === "string") {
|
|
52
|
+
rejected.push({ packageName: entry.packageName, id: null, reason: described });
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
const owner = seen.get(described.id);
|
|
56
|
+
if (owner !== void 0) {
|
|
57
|
+
rejected.push({
|
|
58
|
+
packageName: entry.packageName,
|
|
59
|
+
id: described.id,
|
|
60
|
+
reason: `adapter id "${described.id}" is already provided by ${owner}`
|
|
61
|
+
});
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
seen.set(described.id, entry.packageName);
|
|
65
|
+
loaded.push({ packageName: entry.packageName, module: described });
|
|
66
|
+
}
|
|
67
|
+
return { loaded, rejected };
|
|
68
|
+
}
|
|
69
|
+
function instantiateAdapter(entry, rawOptions) {
|
|
70
|
+
const { module, packageName } = entry;
|
|
71
|
+
const parsed = module.optionsSchema.safeParse(rawOptions ?? {});
|
|
72
|
+
if (!parsed.success) {
|
|
73
|
+
const reason = `invalid options for "${module.id}": ${parsed.error.message}`;
|
|
74
|
+
return { id: module.id, packageName, adapter: createUnavailableAdapter(module.id, reason), error: reason };
|
|
75
|
+
}
|
|
76
|
+
try {
|
|
77
|
+
const adapter = module.createAdapter(parsed.data);
|
|
78
|
+
if (typeof adapter?.search !== "function" || typeof adapter?.readiness !== "function") {
|
|
79
|
+
const reason = `adapter "${module.id}" returned an object that does not implement SearchAdapter`;
|
|
80
|
+
return {
|
|
81
|
+
id: module.id,
|
|
82
|
+
packageName,
|
|
83
|
+
adapter: createUnavailableAdapter(module.id, reason),
|
|
84
|
+
error: reason
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
return { id: module.id, packageName, adapter };
|
|
88
|
+
} catch (error) {
|
|
89
|
+
const reason = `adapter "${module.id}" threw while being constructed: ${errorMessage(error)}`;
|
|
90
|
+
return { id: module.id, packageName, adapter: createUnavailableAdapter(module.id, reason), error: reason };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
export {
|
|
94
|
+
createUnavailableAdapter,
|
|
95
|
+
instantiateAdapter,
|
|
96
|
+
resolveAdapterModules
|
|
97
|
+
};
|
|
98
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/loader/registry.ts"],
|
|
4
|
+
"sourcesContent": ["import type { AdapterModule, SearchAdapter } from '../contract/adapter'\nimport { notReady } from '../contract/adapter'\nimport { errorMessage } from '../contract/errors'\nimport { unavailable } from '../contract/outcomes'\nimport { CONTRACT_VERSION, MIN_SUPPORTED_CONTRACT_VERSION, isSupportedContractVersion } from '../contract/version'\n\nexport type AdapterRegistryEntry = {\n readonly packageName: string\n /** The statically imported module namespace or its default export. */\n readonly module: unknown\n}\n\nexport type LoadedAdapterModule = {\n readonly packageName: string\n readonly module: AdapterModule<unknown>\n}\n\nexport type RejectedAdapterModule = {\n readonly packageName: string\n readonly id: string | null\n readonly reason: string\n}\n\nexport type AdapterModuleRegistry = {\n readonly loaded: readonly LoadedAdapterModule[]\n readonly rejected: readonly RejectedAdapterModule[]\n}\n\nconst ADAPTER_KINDS = new Set(['serp', 'api', 'model', 'browser'])\n\n/**\n * A stand-in for an adapter that could not be built. Returning one of these\n * instead of throwing is what keeps a broken third-party package from taking the\n * whole tool set down: the scheduler simply reports it as skipped, with the\n * reason, alongside every adapter that did work.\n */\nexport function createUnavailableAdapter(id: string, reason: string): SearchAdapter {\n return {\n id,\n kind: 'api',\n capabilities: {\n search: false,\n fetch: false,\n snippets: false,\n content: false,\n freshness: false,\n siteFilter: false,\n cost: 'free',\n },\n readiness: () => notReady(reason),\n search: async () => unavailable(reason),\n healthCheck: async () => ({ ok: false, detail: reason }),\n }\n}\n\nfunction describeModule(candidate: unknown): AdapterModule<unknown> | string {\n const unwrapped =\n typeof candidate === 'object' &&\n candidate !== null &&\n 'default' in candidate &&\n !('createAdapter' in candidate)\n ? (candidate as { default: unknown }).default\n : candidate\n\n if (typeof unwrapped !== 'object' || unwrapped === null) {\n return 'module does not export an adapter descriptor'\n }\n const module = unwrapped as Partial<AdapterModule<unknown>>\n if (typeof module.id !== 'string' || module.id.length === 0) return 'adapter descriptor has no id'\n if (typeof module.kind !== 'string' || !ADAPTER_KINDS.has(module.kind)) {\n return `adapter \"${module.id}\" declares an unknown kind`\n }\n if (typeof module.createAdapter !== 'function') {\n return `adapter \"${module.id}\" does not expose createAdapter()`\n }\n if (\n typeof module.optionsSchema !== 'object' ||\n module.optionsSchema === null ||\n typeof (module.optionsSchema as { safeParse?: unknown }).safeParse !== 'function'\n ) {\n return `adapter \"${module.id}\" does not expose a zod optionsSchema`\n }\n if (!isSupportedContractVersion(module.contractVersion)) {\n return `adapter \"${module.id}\" targets contract version ${String(module.contractVersion)}, this engine speaks ${MIN_SUPPORTED_CONTRACT_VERSION}..${CONTRACT_VERSION}`\n }\n return module as AdapterModule<unknown>\n}\n\n/**\n * Validates statically imported adapter modules. Discovery itself is done by the\n * generator so imports stay static \u2014 a bundler cannot follow an `import()` with a\n * runtime-computed specifier.\n */\nexport function resolveAdapterModules(entries: readonly AdapterRegistryEntry[]): AdapterModuleRegistry {\n const loaded: LoadedAdapterModule[] = []\n const rejected: RejectedAdapterModule[] = []\n const seen = new Map<string, string>()\n\n for (const entry of entries) {\n const described = describeModule(entry.module)\n if (typeof described === 'string') {\n rejected.push({ packageName: entry.packageName, id: null, reason: described })\n continue\n }\n const owner = seen.get(described.id)\n if (owner !== undefined) {\n rejected.push({\n packageName: entry.packageName,\n id: described.id,\n reason: `adapter id \"${described.id}\" is already provided by ${owner}`,\n })\n continue\n }\n seen.set(described.id, entry.packageName)\n loaded.push({ packageName: entry.packageName, module: described })\n }\n\n return { loaded, rejected }\n}\n\nexport type InstantiatedAdapter = {\n readonly id: string\n readonly packageName: string\n readonly adapter: SearchAdapter\n /** Present when the adapter could not be built and a placeholder is standing in. */\n readonly error?: string\n}\n\n/** Builds one adapter, converting any options or constructor fault into a placeholder. */\nexport function instantiateAdapter(\n entry: LoadedAdapterModule,\n rawOptions: unknown,\n): InstantiatedAdapter {\n const { module, packageName } = entry\n const parsed = module.optionsSchema.safeParse(rawOptions ?? {})\n if (!parsed.success) {\n const reason = `invalid options for \"${module.id}\": ${parsed.error.message}`\n return { id: module.id, packageName, adapter: createUnavailableAdapter(module.id, reason), error: reason }\n }\n try {\n const adapter = module.createAdapter(parsed.data)\n if (typeof adapter?.search !== 'function' || typeof adapter?.readiness !== 'function') {\n const reason = `adapter \"${module.id}\" returned an object that does not implement SearchAdapter`\n return {\n id: module.id,\n packageName,\n adapter: createUnavailableAdapter(module.id, reason),\n error: reason,\n }\n }\n return { id: module.id, packageName, adapter }\n } catch (error) {\n const reason = `adapter \"${module.id}\" threw while being constructed: ${errorMessage(error)}`\n return { id: module.id, packageName, adapter: createUnavailableAdapter(module.id, reason), error: reason }\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,gBAAgB;AACzB,SAAS,oBAAoB;AAC7B,SAAS,mBAAmB;AAC5B,SAAS,kBAAkB,gCAAgC,kCAAkC;AAwB7F,MAAM,gBAAgB,oBAAI,IAAI,CAAC,QAAQ,OAAO,SAAS,SAAS,CAAC;AAQ1D,SAAS,yBAAyB,IAAY,QAA+B;AAClF,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN,cAAc;AAAA,MACZ,QAAQ;AAAA,MACR,OAAO;AAAA,MACP,UAAU;AAAA,MACV,SAAS;AAAA,MACT,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,MAAM;AAAA,IACR;AAAA,IACA,WAAW,MAAM,SAAS,MAAM;AAAA,IAChC,QAAQ,YAAY,YAAY,MAAM;AAAA,IACtC,aAAa,aAAa,EAAE,IAAI,OAAO,QAAQ,OAAO;AAAA,EACxD;AACF;AAEA,SAAS,eAAe,WAAqD;AAC3E,QAAM,YACJ,OAAO,cAAc,YACrB,cAAc,QACd,aAAa,aACb,EAAE,mBAAmB,aAChB,UAAmC,UACpC;AAEN,MAAI,OAAO,cAAc,YAAY,cAAc,MAAM;AACvD,WAAO;AAAA,EACT;AACA,QAAM,SAAS;AACf,MAAI,OAAO,OAAO,OAAO,YAAY,OAAO,GAAG,WAAW,EAAG,QAAO;AACpE,MAAI,OAAO,OAAO,SAAS,YAAY,CAAC,cAAc,IAAI,OAAO,IAAI,GAAG;AACtE,WAAO,YAAY,OAAO,EAAE;AAAA,EAC9B;AACA,MAAI,OAAO,OAAO,kBAAkB,YAAY;AAC9C,WAAO,YAAY,OAAO,EAAE;AAAA,EAC9B;AACA,MACE,OAAO,OAAO,kBAAkB,YAChC,OAAO,kBAAkB,QACzB,OAAQ,OAAO,cAA0C,cAAc,YACvE;AACA,WAAO,YAAY,OAAO,EAAE;AAAA,EAC9B;AACA,MAAI,CAAC,2BAA2B,OAAO,eAAe,GAAG;AACvD,WAAO,YAAY,OAAO,EAAE,8BAA8B,OAAO,OAAO,eAAe,CAAC,wBAAwB,8BAA8B,KAAK,gBAAgB;AAAA,EACrK;AACA,SAAO;AACT;AAOO,SAAS,sBAAsB,SAAiE;AACrG,QAAM,SAAgC,CAAC;AACvC,QAAM,WAAoC,CAAC;AAC3C,QAAM,OAAO,oBAAI,IAAoB;AAErC,aAAW,SAAS,SAAS;AAC3B,UAAM,YAAY,eAAe,MAAM,MAAM;AAC7C,QAAI,OAAO,cAAc,UAAU;AACjC,eAAS,KAAK,EAAE,aAAa,MAAM,aAAa,IAAI,MAAM,QAAQ,UAAU,CAAC;AAC7E;AAAA,IACF;AACA,UAAM,QAAQ,KAAK,IAAI,UAAU,EAAE;AACnC,QAAI,UAAU,QAAW;AACvB,eAAS,KAAK;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,IAAI,UAAU;AAAA,QACd,QAAQ,eAAe,UAAU,EAAE,4BAA4B,KAAK;AAAA,MACtE,CAAC;AACD;AAAA,IACF;AACA,SAAK,IAAI,UAAU,IAAI,MAAM,WAAW;AACxC,WAAO,KAAK,EAAE,aAAa,MAAM,aAAa,QAAQ,UAAU,CAAC;AAAA,EACnE;AAEA,SAAO,EAAE,QAAQ,SAAS;AAC5B;AAWO,SAAS,mBACd,OACA,YACqB;AACrB,QAAM,EAAE,QAAQ,YAAY,IAAI;AAChC,QAAM,SAAS,OAAO,cAAc,UAAU,cAAc,CAAC,CAAC;AAC9D,MAAI,CAAC,OAAO,SAAS;AACnB,UAAM,SAAS,wBAAwB,OAAO,EAAE,MAAM,OAAO,MAAM,OAAO;AAC1E,WAAO,EAAE,IAAI,OAAO,IAAI,aAAa,SAAS,yBAAyB,OAAO,IAAI,MAAM,GAAG,OAAO,OAAO;AAAA,EAC3G;AACA,MAAI;AACF,UAAM,UAAU,OAAO,cAAc,OAAO,IAAI;AAChD,QAAI,OAAO,SAAS,WAAW,cAAc,OAAO,SAAS,cAAc,YAAY;AACrF,YAAM,SAAS,YAAY,OAAO,EAAE;AACpC,aAAO;AAAA,QACL,IAAI,OAAO;AAAA,QACX;AAAA,QACA,SAAS,yBAAyB,OAAO,IAAI,MAAM;AAAA,QACnD,OAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO,EAAE,IAAI,OAAO,IAAI,aAAa,QAAQ;AAAA,EAC/C,SAAS,OAAO;AACd,UAAM,SAAS,YAAY,OAAO,EAAE,oCAAoC,aAAa,KAAK,CAAC;AAC3F,WAAO,EAAE,IAAI,OAAO,IAAI,aAAa,SAAS,yBAAyB,OAAO,IAAI,MAAM,GAAG,OAAO,OAAO;AAAA,EAC3G;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|