@open-mercato/web-research-browser 0.7.1-develop.7194.1.ab4fc81f82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -0
- package/build.mjs +10 -0
- package/dist/adapter.js +162 -0
- package/dist/adapter.js.map +7 -0
- package/dist/client.js +189 -0
- package/dist/client.js.map +7 -0
- package/dist/dispatch.js +161 -0
- package/dist/dispatch.js.map +7 -0
- package/dist/index.js +18 -0
- package/dist/index.js.map +7 -0
- package/dist/playwright.js +143 -0
- package/dist/playwright.js.map +7 -0
- package/dist/protocol.js +28 -0
- package/dist/protocol.js.map +7 -0
- package/dist/sidecar.js +79 -0
- package/dist/sidecar.js.map +7 -0
- package/jest.config.cjs +31 -0
- package/package.json +64 -0
- package/src/__tests__/browser.test.ts +242 -0
- package/src/__tests__/install.test.ts +72 -0
- package/src/__tests__/leases.test.ts +50 -0
- package/src/__tests__/sandbox.test.ts +67 -0
- package/src/adapter.ts +196 -0
- package/src/client.ts +245 -0
- package/src/dispatch.ts +205 -0
- package/src/index.ts +10 -0
- package/src/playwright.ts +265 -0
- package/src/protocol.ts +51 -0
- package/src/sidecar.ts +108 -0
- package/tsconfig.json +10 -0
- package/watch.mjs +7 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { assertPublicUrl } from "@open-mercato/web-research";
|
|
3
|
+
import { SidecarError } from "./protocol.js";
|
|
4
|
+
function isModuleNotFound(message) {
|
|
5
|
+
return /cannot find (package|module) ['"]?playwright/i.test(message) || /ERR_MODULE_NOT_FOUND/i.test(message) || /failed to resolve ['"]?playwright/i.test(message);
|
|
6
|
+
}
|
|
7
|
+
async function importChromium() {
|
|
8
|
+
try {
|
|
9
|
+
const playwright = await import("playwright");
|
|
10
|
+
return playwright.chromium;
|
|
11
|
+
} catch (error) {
|
|
12
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
13
|
+
throw new SidecarError(
|
|
14
|
+
`Playwright could not be loaded even though it is a dependency of this package \u2014 reinstall dependencies. Underlying: ${message}`,
|
|
15
|
+
isModuleNotFound(message) ? "needs-install" : "init"
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
function isMissingBrowserError(error) {
|
|
20
|
+
return error instanceof Error && /Executable doesn'?t exist at/i.test(error.message);
|
|
21
|
+
}
|
|
22
|
+
const INSTALL_TIMEOUT_MS = 10 * 6e4;
|
|
23
|
+
const defaultInstallRunner = async (onProgress) => new Promise((resolve, reject) => {
|
|
24
|
+
const child = spawn("npx", ["playwright", "install", "chromium"], { stdio: ["ignore", "pipe", "pipe"] });
|
|
25
|
+
let tail = "";
|
|
26
|
+
const collect = (chunk) => {
|
|
27
|
+
tail = `${tail}${chunk.toString("utf8")}`.slice(-4e3);
|
|
28
|
+
};
|
|
29
|
+
child.stdout?.on("data", collect);
|
|
30
|
+
child.stderr?.on("data", collect);
|
|
31
|
+
const timer = setTimeout(() => {
|
|
32
|
+
child.kill("SIGKILL");
|
|
33
|
+
reject(new Error(`\`npx playwright install chromium\` exceeded ${INSTALL_TIMEOUT_MS}ms`));
|
|
34
|
+
}, INSTALL_TIMEOUT_MS);
|
|
35
|
+
child.once("error", (error) => {
|
|
36
|
+
clearTimeout(timer);
|
|
37
|
+
reject(error);
|
|
38
|
+
});
|
|
39
|
+
child.once("close", (code) => {
|
|
40
|
+
clearTimeout(timer);
|
|
41
|
+
if (code === 0) {
|
|
42
|
+
onProgress("chromium install complete");
|
|
43
|
+
resolve();
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
reject(new Error(`\`npx playwright install chromium\` failed (exit ${code}): ${tail.trim()}`));
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
async function launchChromium(browserType, options, deps = {}) {
|
|
50
|
+
try {
|
|
51
|
+
return await browserType.launch(options);
|
|
52
|
+
} catch (error) {
|
|
53
|
+
if (isSandboxFailure(error)) {
|
|
54
|
+
throw new SidecarError(
|
|
55
|
+
`Chromium could not start its sandbox, and this adapter renders untrusted pages so it will not run without one. Fix the container (run as a non-root user, or grant SYS_ADMIN, or enable user namespaces), or accept the risk explicitly with ${SANDBOX_OPT_OUT_ENV}=1. Underlying: ${error instanceof Error ? error.message : String(error)}`,
|
|
56
|
+
"init"
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
if (!isMissingBrowserError(error)) throw error;
|
|
60
|
+
const onProgress = deps.onProgress ?? (() => {
|
|
61
|
+
});
|
|
62
|
+
onProgress("chromium binary missing, running `npx playwright install chromium` (one-time, ~150MB)");
|
|
63
|
+
try {
|
|
64
|
+
await (deps.install ?? defaultInstallRunner)(onProgress);
|
|
65
|
+
} catch (installError) {
|
|
66
|
+
const detail = installError instanceof Error ? installError.message : String(installError);
|
|
67
|
+
throw new SidecarError(
|
|
68
|
+
`Playwright browser auto-install failed: ${detail}. Run \`npx playwright install chromium\` manually.`,
|
|
69
|
+
"init"
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
return await browserType.launch(options);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const MAX_VERDICT_CACHE = 2048;
|
|
76
|
+
async function installNavigationGuard(context, options = {}) {
|
|
77
|
+
if (typeof context.route !== "function") {
|
|
78
|
+
options.warn?.("context.route() unavailable \u2014 in-page navigation SSRF guard NOT installed");
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const filterSubresources = options.filterSubresources ?? false;
|
|
82
|
+
const verdicts = /* @__PURE__ */ new Map();
|
|
83
|
+
const remember = (host, blocked) => {
|
|
84
|
+
if (verdicts.size >= MAX_VERDICT_CACHE) {
|
|
85
|
+
const oldest = verdicts.keys().next().value;
|
|
86
|
+
if (oldest !== void 0) verdicts.delete(oldest);
|
|
87
|
+
}
|
|
88
|
+
verdicts.set(host, blocked);
|
|
89
|
+
};
|
|
90
|
+
await context.route("**/*", async (route) => {
|
|
91
|
+
const request = route.request();
|
|
92
|
+
const isNavigation = request.isNavigationRequest();
|
|
93
|
+
if (!isNavigation && !filterSubresources) return route.continue();
|
|
94
|
+
const url = request.url();
|
|
95
|
+
let host;
|
|
96
|
+
try {
|
|
97
|
+
host = new URL(url).host;
|
|
98
|
+
} catch {
|
|
99
|
+
return isNavigation ? route.abort("blockedbyclient") : route.continue();
|
|
100
|
+
}
|
|
101
|
+
const cached = isNavigation ? void 0 : verdicts.get(host);
|
|
102
|
+
if (cached === true) return route.abort("blockedbyclient");
|
|
103
|
+
if (cached === false) return route.continue();
|
|
104
|
+
try {
|
|
105
|
+
await assertPublicUrl(url, isNavigation ? "browser navigation" : "browser subresource", {
|
|
106
|
+
failClosed: true
|
|
107
|
+
});
|
|
108
|
+
remember(host, false);
|
|
109
|
+
} catch {
|
|
110
|
+
remember(host, true);
|
|
111
|
+
return route.abort("blockedbyclient");
|
|
112
|
+
}
|
|
113
|
+
return route.continue();
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
const LAUNCH_ARGS = [
|
|
117
|
+
"--disable-blink-features=AutomationControlled",
|
|
118
|
+
"--disable-dev-shm-usage"
|
|
119
|
+
];
|
|
120
|
+
const SANDBOX_OPT_OUT_ENV = "OM_WEB_RESEARCH_BROWSER_NO_SANDBOX";
|
|
121
|
+
function resolveLaunchArgs(env = process.env) {
|
|
122
|
+
return env[SANDBOX_OPT_OUT_ENV] === "1" ? [...LAUNCH_ARGS, "--no-sandbox"] : [...LAUNCH_ARGS];
|
|
123
|
+
}
|
|
124
|
+
function isSandboxDisabled(env = process.env) {
|
|
125
|
+
return env[SANDBOX_OPT_OUT_ENV] === "1";
|
|
126
|
+
}
|
|
127
|
+
function isSandboxFailure(error) {
|
|
128
|
+
if (!(error instanceof Error)) return false;
|
|
129
|
+
return /running as root without --no-sandbox/i.test(error.message) || /No usable sandbox/i.test(error.message) || /clone helper|namespace sandbox|SUID sandbox/i.test(error.message);
|
|
130
|
+
}
|
|
131
|
+
const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
|
132
|
+
export {
|
|
133
|
+
DEFAULT_USER_AGENT,
|
|
134
|
+
LAUNCH_ARGS,
|
|
135
|
+
SANDBOX_OPT_OUT_ENV,
|
|
136
|
+
importChromium,
|
|
137
|
+
installNavigationGuard,
|
|
138
|
+
isSandboxDisabled,
|
|
139
|
+
isSandboxFailure,
|
|
140
|
+
launchChromium,
|
|
141
|
+
resolveLaunchArgs
|
|
142
|
+
};
|
|
143
|
+
//# sourceMappingURL=playwright.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/playwright.ts"],
|
|
4
|
+
"sourcesContent": ["import { spawn } from 'node:child_process'\nimport { assertPublicUrl } from '@open-mercato/web-research'\nimport { SidecarError } from './protocol'\n\n/**\n * Structural projections of the Playwright surface we touch.\n *\n * Playwright is a real dependency of this package \u2014 installing the adapter gets\n * you the tooling it needs \u2014 but it is still reached through a dynamic import\n * inside the sidecar, so the app and MCP processes never load it. The types stay\n * structural for the same reason: nothing here should pull the SDK into a bundle\n * that only ever spawns the child.\n */\nexport type PageHandle = {\n goto(url: string, options?: { waitUntil?: string; timeout?: number }): Promise<unknown>\n content(): Promise<string>\n url(): string\n close(): Promise<void>\n}\n\nexport type ContextHandle = {\n newPage(): Promise<unknown>\n close(): Promise<void>\n route?(pattern: string, handler: (route: RouteHandle) => Promise<void> | void): Promise<void>\n}\n\nexport type RouteHandle = {\n request(): { url(): string; isNavigationRequest(): boolean }\n abort(reason?: string): Promise<void>\n continue(): Promise<void>\n}\n\nexport type BrowserHandle = {\n newContext(options?: Record<string, unknown>): Promise<ContextHandle>\n close(): Promise<void>\n}\n\nexport type BrowserType = {\n launch(options?: { headless?: boolean; args?: string[] }): Promise<BrowserHandle>\n}\n\nfunction isModuleNotFound(message: string): boolean {\n return (\n /cannot find (package|module) ['\"]?playwright/i.test(message) ||\n /ERR_MODULE_NOT_FOUND/i.test(message) ||\n /failed to resolve ['\"]?playwright/i.test(message)\n )\n}\n\nexport async function importChromium(): Promise<BrowserType> {\n try {\n const playwright = (await import('playwright')) as unknown as { chromium: BrowserType }\n return playwright.chromium\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error)\n // Playwright is a dependency of this package, so a module-not-found here\n // means a broken install (hoisting, a pruned image layer), not a missing opt-in.\n throw new SidecarError(\n `Playwright could not be loaded even though it is a dependency of this package \u2014 reinstall dependencies. Underlying: ${message}`,\n isModuleNotFound(message) ? 'needs-install' : 'init',\n )\n }\n}\n\n/**\n * Playwright's launch error when the npm package is installed but the browser\n * binary was never downloaded. The wording has been stable across 1.x, and it is\n * the only signal Playwright gives for this case.\n */\nfunction isMissingBrowserError(error: unknown): boolean {\n return error instanceof Error && /Executable doesn'?t exist at/i.test(error.message)\n}\n\n/** Generous, but bounded \u2014 an unbounded install hangs the sidecar forever. */\nconst INSTALL_TIMEOUT_MS = 10 * 60_000\n\nexport type InstallRunner = (onProgress: (line: string) => void) => Promise<void>\n\nconst defaultInstallRunner: InstallRunner = async (onProgress) =>\n new Promise((resolve, reject) => {\n const child = spawn('npx', ['playwright', 'install', 'chromium'], { stdio: ['ignore', 'pipe', 'pipe'] })\n let tail = ''\n const collect = (chunk: Buffer): void => {\n tail = `${tail}${chunk.toString('utf8')}`.slice(-4000)\n }\n child.stdout?.on('data', collect)\n child.stderr?.on('data', collect)\n\n const timer = setTimeout(() => {\n child.kill('SIGKILL')\n reject(new Error(`\\`npx playwright install chromium\\` exceeded ${INSTALL_TIMEOUT_MS}ms`))\n }, INSTALL_TIMEOUT_MS)\n\n child.once('error', (error) => {\n clearTimeout(timer)\n reject(error)\n })\n child.once('close', (code) => {\n clearTimeout(timer)\n if (code === 0) {\n onProgress('chromium install complete')\n resolve()\n return\n }\n reject(new Error(`\\`npx playwright install chromium\\` failed (exit ${code}): ${tail.trim()}`))\n })\n })\n\n/**\n * Launches Chromium, downloading the binary once if it is missing.\n *\n * Installing the npm package does not fetch the ~150MB browser, so an operator\n * who enables this tier would otherwise hit a launch failure with no obvious\n * remedy. One retry only \u2014 a second failure is a real fault, not a cold cache.\n */\nexport async function launchChromium(\n browserType: BrowserType,\n options: { headless?: boolean; args?: string[] },\n deps: { install?: InstallRunner; onProgress?: (line: string) => void } = {},\n): Promise<BrowserHandle> {\n try {\n return await browserType.launch(options)\n } catch (error) {\n // Deliberately not retried without the sandbox. Falling back silently would\n // mean the security posture depends on whether a container happened to be\n // configured, with nobody told which one they got.\n if (isSandboxFailure(error)) {\n throw new SidecarError(\n `Chromium could not start its sandbox, and this adapter renders untrusted pages so it will not run without one. ` +\n `Fix the container (run as a non-root user, or grant SYS_ADMIN, or enable user namespaces), ` +\n `or accept the risk explicitly with ${SANDBOX_OPT_OUT_ENV}=1. Underlying: ${error instanceof Error ? error.message : String(error)}`,\n 'init',\n )\n }\n if (!isMissingBrowserError(error)) throw error\n const onProgress = deps.onProgress ?? (() => {})\n onProgress('chromium binary missing, running `npx playwright install chromium` (one-time, ~150MB)')\n try {\n await (deps.install ?? defaultInstallRunner)(onProgress)\n } catch (installError) {\n const detail = installError instanceof Error ? installError.message : String(installError)\n throw new SidecarError(\n `Playwright browser auto-install failed: ${detail}. Run \\`npx playwright install chromium\\` manually.`,\n 'init',\n )\n }\n return await browserType.launch(options)\n }\n}\n\nconst MAX_VERDICT_CACHE = 2048\n\n/**\n * Blocks private-origin navigations for the life of the context.\n *\n * The parent and the sidecar both vet the URL before a `goto`, but a page reached\n * through a legitimate public navigation can redirect or script-navigate itself\n * to a metadata endpoint, and those hops never pass through the RPC layer.\n *\n * Fails closed: Chromium resolves names with its own resolver, so a hostname\n * `node:dns` cannot vet must not be allowed through on the assumption it is dead.\n */\nexport async function installNavigationGuard(\n context: ContextHandle,\n options: { filterSubresources?: boolean; warn?: (message: string) => void } = {},\n): Promise<void> {\n if (typeof context.route !== 'function') {\n options.warn?.('context.route() unavailable \u2014 in-page navigation SSRF guard NOT installed')\n return\n }\n const filterSubresources = options.filterSubresources ?? false\n const verdicts = new Map<string, boolean>()\n const remember = (host: string, blocked: boolean): void => {\n if (verdicts.size >= MAX_VERDICT_CACHE) {\n const oldest = verdicts.keys().next().value\n if (oldest !== undefined) verdicts.delete(oldest)\n }\n verdicts.set(host, blocked)\n }\n\n await context.route('**/*', async (route) => {\n const request = route.request()\n const isNavigation = request.isNavigationRequest()\n if (!isNavigation && !filterSubresources) return route.continue()\n\n const url = request.url()\n let host: string\n try {\n host = new URL(url).host\n } catch {\n return isNavigation ? route.abort('blockedbyclient') : route.continue()\n }\n\n const cached = isNavigation ? undefined : verdicts.get(host)\n if (cached === true) return route.abort('blockedbyclient')\n if (cached === false) return route.continue()\n\n try {\n await assertPublicUrl(url, isNavigation ? 'browser navigation' : 'browser subresource', {\n failClosed: true,\n })\n remember(host, false)\n } catch {\n remember(host, true)\n return route.abort('blockedbyclient')\n }\n return route.continue()\n })\n}\n\n/**\n * Launch flags aimed at not announcing automation. Playwright's defaults set\n * `navigator.webdriver` and a headless UA, which the SERP endpoints this adapter\n * exists to reach specifically look for.\n *\n * `--disable-dev-shm-usage` is a sizing workaround, not a security control: it\n * makes Chromium write shared memory to /tmp instead of a container's small\n * /dev/shm. The sandbox is deliberately NOT disabled here \u2014 see\n * {@link resolveLaunchArgs}.\n */\nexport const LAUNCH_ARGS = [\n '--disable-blink-features=AutomationControlled',\n '--disable-dev-shm-usage',\n]\n\nexport const SANDBOX_OPT_OUT_ENV = 'OM_WEB_RESEARCH_BROWSER_NO_SANDBOX'\n\n/**\n * Launch flags for this process, sandbox included unless an operator opted out.\n *\n * This adapter exists to render pages chosen by a search engine or a model, so\n * the renderer is by definition parsing hostile input. `--no-sandbox` turns a\n * renderer compromise into code running as whatever user the app runs as, which\n * is the one mitigation Chromium provides for exactly this threat.\n *\n * It used to be on unconditionally because the sandbox needs privileges a\n * root-in-Docker container does not grant by default. That is a container\n * problem with container fixes (run as a non-root user, or grant `SYS_ADMIN`, or\n * enable user namespaces); silently shipping every deployment the insecure\n * variant to spare those is the wrong trade. The opt-out stays available for\n * environments that genuinely cannot, and it is loud.\n */\nexport function resolveLaunchArgs(env: NodeJS.ProcessEnv = process.env): string[] {\n return env[SANDBOX_OPT_OUT_ENV] === '1' ? [...LAUNCH_ARGS, '--no-sandbox'] : [...LAUNCH_ARGS]\n}\n\nexport function isSandboxDisabled(env: NodeJS.ProcessEnv = process.env): boolean {\n return env[SANDBOX_OPT_OUT_ENV] === '1'\n}\n\n/**\n * Chromium's refusal when it cannot set up its sandbox. The wording differs\n * between the root check and the namespace failure, so both shapes are matched.\n */\nexport function isSandboxFailure(error: unknown): boolean {\n if (!(error instanceof Error)) return false\n return (\n /running as root without --no-sandbox/i.test(error.message) ||\n /No usable sandbox/i.test(error.message) ||\n /clone helper|namespace sandbox|SUID sandbox/i.test(error.message)\n )\n}\n\nexport const DEFAULT_USER_AGENT =\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,aAAa;AACtB,SAAS,uBAAuB;AAChC,SAAS,oBAAoB;AAuC7B,SAAS,iBAAiB,SAA0B;AAClD,SACE,gDAAgD,KAAK,OAAO,KAC5D,wBAAwB,KAAK,OAAO,KACpC,qCAAqC,KAAK,OAAO;AAErD;AAEA,eAAsB,iBAAuC;AAC3D,MAAI;AACF,UAAM,aAAc,MAAM,OAAO,YAAY;AAC7C,WAAO,WAAW;AAAA,EACpB,SAAS,OAAO;AACd,UAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAGrE,UAAM,IAAI;AAAA,MACR,4HAAuH,OAAO;AAAA,MAC9H,iBAAiB,OAAO,IAAI,kBAAkB;AAAA,IAChD;AAAA,EACF;AACF;AAOA,SAAS,sBAAsB,OAAyB;AACtD,SAAO,iBAAiB,SAAS,gCAAgC,KAAK,MAAM,OAAO;AACrF;AAGA,MAAM,qBAAqB,KAAK;AAIhC,MAAM,uBAAsC,OAAO,eACjD,IAAI,QAAQ,CAAC,SAAS,WAAW;AAC/B,QAAM,QAAQ,MAAM,OAAO,CAAC,cAAc,WAAW,UAAU,GAAG,EAAE,OAAO,CAAC,UAAU,QAAQ,MAAM,EAAE,CAAC;AACvG,MAAI,OAAO;AACX,QAAM,UAAU,CAAC,UAAwB;AACvC,WAAO,GAAG,IAAI,GAAG,MAAM,SAAS,MAAM,CAAC,GAAG,MAAM,IAAK;AAAA,EACvD;AACA,QAAM,QAAQ,GAAG,QAAQ,OAAO;AAChC,QAAM,QAAQ,GAAG,QAAQ,OAAO;AAEhC,QAAM,QAAQ,WAAW,MAAM;AAC7B,UAAM,KAAK,SAAS;AACpB,WAAO,IAAI,MAAM,gDAAgD,kBAAkB,IAAI,CAAC;AAAA,EAC1F,GAAG,kBAAkB;AAErB,QAAM,KAAK,SAAS,CAAC,UAAU;AAC7B,iBAAa,KAAK;AAClB,WAAO,KAAK;AAAA,EACd,CAAC;AACD,QAAM,KAAK,SAAS,CAAC,SAAS;AAC5B,iBAAa,KAAK;AAClB,QAAI,SAAS,GAAG;AACd,iBAAW,2BAA2B;AACtC,cAAQ;AACR;AAAA,IACF;AACA,WAAO,IAAI,MAAM,oDAAoD,IAAI,MAAM,KAAK,KAAK,CAAC,EAAE,CAAC;AAAA,EAC/F,CAAC;AACH,CAAC;AASH,eAAsB,eACpB,aACA,SACA,OAAyE,CAAC,GAClD;AACxB,MAAI;AACF,WAAO,MAAM,YAAY,OAAO,OAAO;AAAA,EACzC,SAAS,OAAO;AAId,QAAI,iBAAiB,KAAK,GAAG;AAC3B,YAAM,IAAI;AAAA,QACR,gPAEwC,mBAAmB,mBAAmB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAAA,QACpI;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,sBAAsB,KAAK,EAAG,OAAM;AACzC,UAAM,aAAa,KAAK,eAAe,MAAM;AAAA,IAAC;AAC9C,eAAW,uFAAuF;AAClG,QAAI;AACF,aAAO,KAAK,WAAW,sBAAsB,UAAU;AAAA,IACzD,SAAS,cAAc;AACrB,YAAM,SAAS,wBAAwB,QAAQ,aAAa,UAAU,OAAO,YAAY;AACzF,YAAM,IAAI;AAAA,QACR,2CAA2C,MAAM;AAAA,QACjD;AAAA,MACF;AAAA,IACF;AACA,WAAO,MAAM,YAAY,OAAO,OAAO;AAAA,EACzC;AACF;AAEA,MAAM,oBAAoB;AAY1B,eAAsB,uBACpB,SACA,UAA8E,CAAC,GAChE;AACf,MAAI,OAAO,QAAQ,UAAU,YAAY;AACvC,YAAQ,OAAO,gFAA2E;AAC1F;AAAA,EACF;AACA,QAAM,qBAAqB,QAAQ,sBAAsB;AACzD,QAAM,WAAW,oBAAI,IAAqB;AAC1C,QAAM,WAAW,CAAC,MAAc,YAA2B;AACzD,QAAI,SAAS,QAAQ,mBAAmB;AACtC,YAAM,SAAS,SAAS,KAAK,EAAE,KAAK,EAAE;AACtC,UAAI,WAAW,OAAW,UAAS,OAAO,MAAM;AAAA,IAClD;AACA,aAAS,IAAI,MAAM,OAAO;AAAA,EAC5B;AAEA,QAAM,QAAQ,MAAM,QAAQ,OAAO,UAAU;AAC3C,UAAM,UAAU,MAAM,QAAQ;AAC9B,UAAM,eAAe,QAAQ,oBAAoB;AACjD,QAAI,CAAC,gBAAgB,CAAC,mBAAoB,QAAO,MAAM,SAAS;AAEhE,UAAM,MAAM,QAAQ,IAAI;AACxB,QAAI;AACJ,QAAI;AACF,aAAO,IAAI,IAAI,GAAG,EAAE;AAAA,IACtB,QAAQ;AACN,aAAO,eAAe,MAAM,MAAM,iBAAiB,IAAI,MAAM,SAAS;AAAA,IACxE;AAEA,UAAM,SAAS,eAAe,SAAY,SAAS,IAAI,IAAI;AAC3D,QAAI,WAAW,KAAM,QAAO,MAAM,MAAM,iBAAiB;AACzD,QAAI,WAAW,MAAO,QAAO,MAAM,SAAS;AAE5C,QAAI;AACF,YAAM,gBAAgB,KAAK,eAAe,uBAAuB,uBAAuB;AAAA,QACtF,YAAY;AAAA,MACd,CAAC;AACD,eAAS,MAAM,KAAK;AAAA,IACtB,QAAQ;AACN,eAAS,MAAM,IAAI;AACnB,aAAO,MAAM,MAAM,iBAAiB;AAAA,IACtC;AACA,WAAO,MAAM,SAAS;AAAA,EACxB,CAAC;AACH;AAYO,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AACF;AAEO,MAAM,sBAAsB;AAiB5B,SAAS,kBAAkB,MAAyB,QAAQ,KAAe;AAChF,SAAO,IAAI,mBAAmB,MAAM,MAAM,CAAC,GAAG,aAAa,cAAc,IAAI,CAAC,GAAG,WAAW;AAC9F;AAEO,SAAS,kBAAkB,MAAyB,QAAQ,KAAc;AAC/E,SAAO,IAAI,mBAAmB,MAAM;AACtC;AAMO,SAAS,iBAAiB,OAAyB;AACxD,MAAI,EAAE,iBAAiB,OAAQ,QAAO;AACtC,SACE,wCAAwC,KAAK,MAAM,OAAO,KAC1D,qBAAqB,KAAK,MAAM,OAAO,KACvC,+CAA+C,KAAK,MAAM,OAAO;AAErE;AAEO,MAAM,qBACX;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/dist/protocol.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
class SidecarError extends Error {
|
|
2
|
+
constructor(message, kind = "runtime") {
|
|
3
|
+
super(message);
|
|
4
|
+
this.name = "SidecarError";
|
|
5
|
+
this.kind = kind;
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
function encodeLine(value) {
|
|
9
|
+
return `${JSON.stringify(value)}
|
|
10
|
+
`;
|
|
11
|
+
}
|
|
12
|
+
function decodeLine(line) {
|
|
13
|
+
try {
|
|
14
|
+
return JSON.parse(line);
|
|
15
|
+
} catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
function isReply(value) {
|
|
20
|
+
return typeof value === "object" && value !== null && typeof value.id === "string" && typeof value.ok === "boolean";
|
|
21
|
+
}
|
|
22
|
+
export {
|
|
23
|
+
SidecarError,
|
|
24
|
+
decodeLine,
|
|
25
|
+
encodeLine,
|
|
26
|
+
isReply
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=protocol.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/protocol.ts"],
|
|
4
|
+
"sourcesContent": ["export type SidecarErrorKind = 'init' | 'needs-install' | 'navigation' | 'runtime' | 'timeout' | 'unknown'\n\nexport type SidecarRequest = {\n readonly id: string\n readonly method: string\n readonly params?: Record<string, unknown>\n}\n\nexport type SidecarReply =\n | { readonly id: string; readonly ok: true; readonly result?: unknown }\n | {\n readonly id: string\n readonly ok: false\n readonly error: { readonly message: string; readonly kind: SidecarErrorKind }\n }\n\nexport class SidecarError extends Error {\n readonly kind: SidecarErrorKind\n\n constructor(message: string, kind: SidecarErrorKind = 'runtime') {\n super(message)\n this.name = 'SidecarError'\n this.kind = kind\n }\n}\n\n/**\n * Line-delimited JSON, one object per line, correlated by `id`. Deliberately not\n * JSON-RPC 2.0: the surface is four methods over a pipe we own both ends of, and\n * an `ok` discriminant is easier to narrow than presence-of-`error`.\n */\nexport function encodeLine(value: SidecarRequest | SidecarReply): string {\n return `${JSON.stringify(value)}\\n`\n}\n\nexport function decodeLine(line: string): unknown {\n try {\n return JSON.parse(line)\n } catch {\n return null\n }\n}\n\nexport function isReply(value: unknown): value is SidecarReply {\n return (\n typeof value === 'object' &&\n value !== null &&\n typeof (value as { id?: unknown }).id === 'string' &&\n typeof (value as { ok?: unknown }).ok === 'boolean'\n )\n}\n"],
|
|
5
|
+
"mappings": "AAgBO,MAAM,qBAAqB,MAAM;AAAA,EAGtC,YAAY,SAAiB,OAAyB,WAAW;AAC/D,UAAM,OAAO;AACb,SAAK,OAAO;AACZ,SAAK,OAAO;AAAA,EACd;AACF;AAOO,SAAS,WAAW,OAA8C;AACvE,SAAO,GAAG,KAAK,UAAU,KAAK,CAAC;AAAA;AACjC;AAEO,SAAS,WAAW,MAAuB;AAChD,MAAI;AACF,WAAO,KAAK,MAAM,IAAI;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,QAAQ,OAAuC;AAC7D,SACE,OAAO,UAAU,YACjB,UAAU,QACV,OAAQ,MAA2B,OAAO,YAC1C,OAAQ,MAA2B,OAAO;AAE9C;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/dist/sidecar.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { createState, dispatch, teardown } from "./dispatch.js";
|
|
2
|
+
import { decodeLine, encodeLine } from "./protocol.js";
|
|
3
|
+
const state = createState();
|
|
4
|
+
function write(reply) {
|
|
5
|
+
try {
|
|
6
|
+
process.stdout.write(encodeLine(reply));
|
|
7
|
+
} catch {
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
const chains = /* @__PURE__ */ new Map();
|
|
11
|
+
function chainKey(request) {
|
|
12
|
+
const leaseId = request.params?.leaseId;
|
|
13
|
+
return typeof leaseId === "string" ? leaseId : "__global__";
|
|
14
|
+
}
|
|
15
|
+
function enqueue(line) {
|
|
16
|
+
const parsed = decodeLine(line);
|
|
17
|
+
if (typeof parsed !== "object" || parsed === null) return;
|
|
18
|
+
const request = parsed;
|
|
19
|
+
if (typeof request.id !== "string" || typeof request.method !== "string") return;
|
|
20
|
+
const key = chainKey(request);
|
|
21
|
+
const previous = chains.get(key) ?? Promise.resolve();
|
|
22
|
+
const next = previous.then(async () => {
|
|
23
|
+
write(await dispatch(state, request));
|
|
24
|
+
}).catch((error) => {
|
|
25
|
+
write({
|
|
26
|
+
id: request.id,
|
|
27
|
+
ok: false,
|
|
28
|
+
error: { message: error instanceof Error ? error.message : String(error), kind: "unknown" }
|
|
29
|
+
});
|
|
30
|
+
}).finally(() => {
|
|
31
|
+
if (chains.get(key) === next) chains.delete(key);
|
|
32
|
+
});
|
|
33
|
+
chains.set(key, next);
|
|
34
|
+
}
|
|
35
|
+
async function shutdownAndExit(code) {
|
|
36
|
+
try {
|
|
37
|
+
await teardown(state);
|
|
38
|
+
} catch {
|
|
39
|
+
}
|
|
40
|
+
process.exit(code);
|
|
41
|
+
}
|
|
42
|
+
function startParentWatchdog() {
|
|
43
|
+
const parentPid = process.ppid;
|
|
44
|
+
if (!parentPid || parentPid <= 1) return;
|
|
45
|
+
const interval = setInterval(() => {
|
|
46
|
+
try {
|
|
47
|
+
process.kill(parentPid, 0);
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (error.code === "ESRCH") {
|
|
50
|
+
clearInterval(interval);
|
|
51
|
+
void shutdownAndExit(0);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}, 2e3);
|
|
55
|
+
interval.unref?.();
|
|
56
|
+
}
|
|
57
|
+
function main() {
|
|
58
|
+
let buffer = "";
|
|
59
|
+
process.stdin.setEncoding("utf8");
|
|
60
|
+
process.stdin.on("data", (chunk) => {
|
|
61
|
+
buffer += chunk;
|
|
62
|
+
let newline = buffer.indexOf("\n");
|
|
63
|
+
while (newline !== -1) {
|
|
64
|
+
const line = buffer.slice(0, newline);
|
|
65
|
+
buffer = buffer.slice(newline + 1);
|
|
66
|
+
if (line.trim()) enqueue(line);
|
|
67
|
+
newline = buffer.indexOf("\n");
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
process.stdin.on("end", () => void shutdownAndExit(0));
|
|
71
|
+
process.stdout.on("error", () => void shutdownAndExit(0));
|
|
72
|
+
startParentWatchdog();
|
|
73
|
+
}
|
|
74
|
+
if (process.env.OM_WEB_RESEARCH_SIDECAR_TEST !== "1") main();
|
|
75
|
+
export {
|
|
76
|
+
enqueue,
|
|
77
|
+
main
|
|
78
|
+
};
|
|
79
|
+
//# sourceMappingURL=sidecar.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/sidecar.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Playwright sidecar entry point.\n *\n * Runs as a child process so Playwright never enters the app or MCP bundle and a\n * crashed browser cannot take the server down. Speaks line-delimited JSON on\n * stdio; every request is serialized per lease so a `goto` cannot race a\n * `release` on the same context.\n */\n\nimport { createState, dispatch, teardown, type SidecarState } from './dispatch'\nimport { decodeLine, encodeLine, type SidecarReply, type SidecarRequest } from './protocol'\n\nconst state: SidecarState = createState()\n\nfunction write(reply: SidecarReply): void {\n try {\n process.stdout.write(encodeLine(reply))\n } catch {\n // stdout is gone; the parent will time the call out and reap us.\n }\n}\n\n/** Per-lease chains: work on different contexts proceeds concurrently. */\nconst chains = new Map<string, Promise<void>>()\n\nfunction chainKey(request: SidecarRequest): string {\n const leaseId = request.params?.leaseId\n return typeof leaseId === 'string' ? leaseId : '__global__'\n}\n\nexport function enqueue(line: string): void {\n const parsed = decodeLine(line)\n if (typeof parsed !== 'object' || parsed === null) return\n const request = parsed as SidecarRequest\n if (typeof request.id !== 'string' || typeof request.method !== 'string') return\n\n const key = chainKey(request)\n const previous = chains.get(key) ?? Promise.resolve()\n const next: Promise<void> = previous\n .then(async () => {\n write(await dispatch(state, request))\n })\n .catch((error: unknown) => {\n write({\n id: request.id,\n ok: false,\n error: { message: error instanceof Error ? error.message : String(error), kind: 'unknown' },\n })\n })\n .finally(() => {\n // Every render takes a fresh lease id, so without this the map grows one\n // settled promise per request for the life of the process. Only the tail is\n // dropped; a later request on the same lease owns the entry by then.\n if (chains.get(key) === next) chains.delete(key)\n })\n chains.set(key, next)\n}\n\nasync function shutdownAndExit(code: number): Promise<void> {\n try {\n await teardown(state)\n } catch {\n // Exiting regardless.\n }\n process.exit(code)\n}\n\n/**\n * A SIGKILL'd parent produces no stdin EOF, so without this poll a Chromium can\n * outlive the server that spawned it and leak memory until the box is restarted.\n */\nfunction startParentWatchdog(): void {\n const parentPid = process.ppid\n if (!parentPid || parentPid <= 1) return\n const interval = setInterval(() => {\n try {\n process.kill(parentPid, 0)\n } catch (error) {\n if ((error as NodeJS.ErrnoException).code === 'ESRCH') {\n clearInterval(interval)\n void shutdownAndExit(0)\n }\n // EPERM means it exists but we may not signal it \u2014 still alive.\n }\n }, 2000)\n interval.unref?.()\n}\n\nexport function main(): void {\n let buffer = ''\n process.stdin.setEncoding('utf8')\n process.stdin.on('data', (chunk: string) => {\n buffer += chunk\n let newline = buffer.indexOf('\\n')\n while (newline !== -1) {\n const line = buffer.slice(0, newline)\n buffer = buffer.slice(newline + 1)\n if (line.trim()) enqueue(line)\n newline = buffer.indexOf('\\n')\n }\n })\n process.stdin.on('end', () => void shutdownAndExit(0))\n process.stdout.on('error', () => void shutdownAndExit(0))\n startParentWatchdog()\n}\n\n// Guarded so tests can import the module without starting the stdio loop.\nif (process.env.OM_WEB_RESEARCH_SIDECAR_TEST !== '1') main()\n"],
|
|
5
|
+
"mappings": "AASA,SAAS,aAAa,UAAU,gBAAmC;AACnE,SAAS,YAAY,kBAA0D;AAE/E,MAAM,QAAsB,YAAY;AAExC,SAAS,MAAM,OAA2B;AACxC,MAAI;AACF,YAAQ,OAAO,MAAM,WAAW,KAAK,CAAC;AAAA,EACxC,QAAQ;AAAA,EAER;AACF;AAGA,MAAM,SAAS,oBAAI,IAA2B;AAE9C,SAAS,SAAS,SAAiC;AACjD,QAAM,UAAU,QAAQ,QAAQ;AAChC,SAAO,OAAO,YAAY,WAAW,UAAU;AACjD;AAEO,SAAS,QAAQ,MAAoB;AAC1C,QAAM,SAAS,WAAW,IAAI;AAC9B,MAAI,OAAO,WAAW,YAAY,WAAW,KAAM;AACnD,QAAM,UAAU;AAChB,MAAI,OAAO,QAAQ,OAAO,YAAY,OAAO,QAAQ,WAAW,SAAU;AAE1E,QAAM,MAAM,SAAS,OAAO;AAC5B,QAAM,WAAW,OAAO,IAAI,GAAG,KAAK,QAAQ,QAAQ;AACpD,QAAM,OAAsB,SACzB,KAAK,YAAY;AAChB,UAAM,MAAM,SAAS,OAAO,OAAO,CAAC;AAAA,EACtC,CAAC,EACA,MAAM,CAAC,UAAmB;AACzB,UAAM;AAAA,MACJ,IAAI,QAAQ;AAAA,MACZ,IAAI;AAAA,MACJ,OAAO,EAAE,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAAG,MAAM,UAAU;AAAA,IAC5F,CAAC;AAAA,EACH,CAAC,EACA,QAAQ,MAAM;AAIb,QAAI,OAAO,IAAI,GAAG,MAAM,KAAM,QAAO,OAAO,GAAG;AAAA,EACjD,CAAC;AACH,SAAO,IAAI,KAAK,IAAI;AACtB;AAEA,eAAe,gBAAgB,MAA6B;AAC1D,MAAI;AACF,UAAM,SAAS,KAAK;AAAA,EACtB,QAAQ;AAAA,EAER;AACA,UAAQ,KAAK,IAAI;AACnB;AAMA,SAAS,sBAA4B;AACnC,QAAM,YAAY,QAAQ;AAC1B,MAAI,CAAC,aAAa,aAAa,EAAG;AAClC,QAAM,WAAW,YAAY,MAAM;AACjC,QAAI;AACF,cAAQ,KAAK,WAAW,CAAC;AAAA,IAC3B,SAAS,OAAO;AACd,UAAK,MAAgC,SAAS,SAAS;AACrD,sBAAc,QAAQ;AACtB,aAAK,gBAAgB,CAAC;AAAA,MACxB;AAAA,IAEF;AAAA,EACF,GAAG,GAAI;AACP,WAAS,QAAQ;AACnB;AAEO,SAAS,OAAa;AAC3B,MAAI,SAAS;AACb,UAAQ,MAAM,YAAY,MAAM;AAChC,UAAQ,MAAM,GAAG,QAAQ,CAAC,UAAkB;AAC1C,cAAU;AACV,QAAI,UAAU,OAAO,QAAQ,IAAI;AACjC,WAAO,YAAY,IAAI;AACrB,YAAM,OAAO,OAAO,MAAM,GAAG,OAAO;AACpC,eAAS,OAAO,MAAM,UAAU,CAAC;AACjC,UAAI,KAAK,KAAK,EAAG,SAAQ,IAAI;AAC7B,gBAAU,OAAO,QAAQ,IAAI;AAAA,IAC/B;AAAA,EACF,CAAC;AACD,UAAQ,MAAM,GAAG,OAAO,MAAM,KAAK,gBAAgB,CAAC,CAAC;AACrD,UAAQ,OAAO,GAAG,SAAS,MAAM,KAAK,gBAAgB,CAAC,CAAC;AACxD,sBAAoB;AACtB;AAGA,IAAI,QAAQ,IAAI,iCAAiC,IAAK,MAAK;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/jest.config.cjs
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** @type {import('jest').Config} */
|
|
2
|
+
const base = require('../../jest.config.base.cjs')
|
|
3
|
+
|
|
4
|
+
module.exports = {
|
|
5
|
+
...base,
|
|
6
|
+
testEnvironment: 'node',
|
|
7
|
+
watchman: false,
|
|
8
|
+
rootDir: '.',
|
|
9
|
+
moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
|
|
10
|
+
moduleNameMapper: {
|
|
11
|
+
'^@open-mercato/web-research/(.*)$': '<rootDir>/../web-research/src/$1',
|
|
12
|
+
'^@open-mercato/web-research$': '<rootDir>/../web-research/src/index.ts',
|
|
13
|
+
'^@open-mercato/web-research-serp$': '<rootDir>/../web-research-serp/src/index.ts',
|
|
14
|
+
},
|
|
15
|
+
// Shared transformer rather than plain ts-jest: `client.ts` reads
|
|
16
|
+
// `import.meta.url` to locate the sidecar script, which a CJS test transform
|
|
17
|
+
// cannot parse. This is the same transformer the CLI and enterprise use.
|
|
18
|
+
transform: {
|
|
19
|
+
'^.+\\.(t|j)sx?$': [
|
|
20
|
+
'<rootDir>/../../scripts/jest-mikroorm-transformer.cjs',
|
|
21
|
+
{
|
|
22
|
+
tsconfig: {
|
|
23
|
+
rootDir: '.',
|
|
24
|
+
ignoreDeprecations: '6.0',
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
],
|
|
28
|
+
},
|
|
29
|
+
testMatch: ['<rootDir>/src/**/__tests__/**/*.test.(ts|tsx)'],
|
|
30
|
+
passWithNoTests: true,
|
|
31
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@open-mercato/web-research-browser",
|
|
3
|
+
"version": "0.7.1-develop.7194.1.ab4fc81f82",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"bin": {
|
|
8
|
+
"om-web-research-sidecar": "./dist/sidecar.js"
|
|
9
|
+
},
|
|
10
|
+
"openMercato": {
|
|
11
|
+
"webResearchAdapter": {
|
|
12
|
+
"id": "browser"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"scripts": {
|
|
16
|
+
"build": "node build.mjs",
|
|
17
|
+
"watch": "node watch.mjs",
|
|
18
|
+
"test": "jest --config jest.config.cjs",
|
|
19
|
+
"typecheck": "tsc --noEmit"
|
|
20
|
+
},
|
|
21
|
+
"exports": {
|
|
22
|
+
".": {
|
|
23
|
+
"types": "./src/index.ts",
|
|
24
|
+
"default": "./dist/index.js"
|
|
25
|
+
},
|
|
26
|
+
"./*.ts": {
|
|
27
|
+
"types": "./src/*.ts",
|
|
28
|
+
"default": "./dist/*.js"
|
|
29
|
+
},
|
|
30
|
+
"./*": {
|
|
31
|
+
"types": [
|
|
32
|
+
"./src/*.ts"
|
|
33
|
+
],
|
|
34
|
+
"default": "./dist/*.js"
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"peerDependencies": {
|
|
38
|
+
"@open-mercato/web-research": "^0.6.0"
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"@open-mercato/web-research-serp": "0.7.1-develop.7194.1.ab4fc81f82",
|
|
42
|
+
"playwright": "^1.40.0",
|
|
43
|
+
"zod": "^4.4.3"
|
|
44
|
+
},
|
|
45
|
+
"devDependencies": {
|
|
46
|
+
"@open-mercato/web-research": "0.7.1-develop.7194.1.ab4fc81f82",
|
|
47
|
+
"@types/jest": "^30.0.0",
|
|
48
|
+
"@types/node": "^26.0.1",
|
|
49
|
+
"esbuild": "^0.28.1",
|
|
50
|
+
"glob": "^13.0.6",
|
|
51
|
+
"jest": "^30.4.2",
|
|
52
|
+
"ts-jest": "^29.4.11",
|
|
53
|
+
"typescript": "^6.0.3"
|
|
54
|
+
},
|
|
55
|
+
"publishConfig": {
|
|
56
|
+
"access": "public"
|
|
57
|
+
},
|
|
58
|
+
"repository": {
|
|
59
|
+
"type": "git",
|
|
60
|
+
"url": "https://github.com/open-mercato/open-mercato",
|
|
61
|
+
"directory": "packages/web-research-browser"
|
|
62
|
+
},
|
|
63
|
+
"stableVersion": "0.7.0"
|
|
64
|
+
}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { PassThrough } from 'node:stream'
|
|
2
|
+
import { createTestContext, describeAdapterContract } from '@open-mercato/web-research/testing'
|
|
3
|
+
import { browserAdapterModule, createBrowserAdapter } from '../adapter'
|
|
4
|
+
import { Sidecar, type SidecarStream, type SpawnFn } from '../client'
|
|
5
|
+
import { decodeLine, encodeLine, type SidecarRequest } from '../protocol'
|
|
6
|
+
|
|
7
|
+
const SERP_HTML = `
|
|
8
|
+
<div class="result">
|
|
9
|
+
<a class="result__a" href="https://example.com/one">Rendered result</a>
|
|
10
|
+
<div class="result__snippet">Rendered snippet.</div>
|
|
11
|
+
</div>`
|
|
12
|
+
|
|
13
|
+
type FakeChild = SidecarStream & {
|
|
14
|
+
written: string[]
|
|
15
|
+
emitExit(code: number | null): void
|
|
16
|
+
emitStdinError(error: NodeJS.ErrnoException): void
|
|
17
|
+
killed: NodeJS.Signals[]
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Scripted child process. Playwright is never imported by these tests — the
|
|
22
|
+
* point is the parent-side protocol, not the browser.
|
|
23
|
+
*/
|
|
24
|
+
function makeFakeSpawn(
|
|
25
|
+
respond: (request: SidecarRequest) => Record<string, unknown> | 'silent' | Error,
|
|
26
|
+
): { spawnFn: SpawnFn; children: FakeChild[] } {
|
|
27
|
+
const children: FakeChild[] = []
|
|
28
|
+
const spawnFn: SpawnFn = () => {
|
|
29
|
+
const stdout = new PassThrough({ encoding: 'utf8' })
|
|
30
|
+
const stderr = new PassThrough({ encoding: 'utf8' })
|
|
31
|
+
const exitHandlers: Array<(code: number | null) => void> = []
|
|
32
|
+
const stdinErrorHandlers: Array<(error: NodeJS.ErrnoException) => void> = []
|
|
33
|
+
const child: FakeChild = {
|
|
34
|
+
written: [],
|
|
35
|
+
killed: [],
|
|
36
|
+
stdin: {
|
|
37
|
+
on(event: 'error', handler: (error: NodeJS.ErrnoException) => void) {
|
|
38
|
+
stdinErrorHandlers.push(handler)
|
|
39
|
+
return child.stdin
|
|
40
|
+
},
|
|
41
|
+
write(chunk: string) {
|
|
42
|
+
child.written.push(chunk)
|
|
43
|
+
const request = decodeLine(chunk.trim()) as SidecarRequest
|
|
44
|
+
const reply = respond(request)
|
|
45
|
+
if (reply === 'silent') return true
|
|
46
|
+
if (reply instanceof Error) {
|
|
47
|
+
stdout.write(
|
|
48
|
+
encodeLine({ id: request.id, ok: false, error: { message: reply.message, kind: 'runtime' } }),
|
|
49
|
+
)
|
|
50
|
+
return true
|
|
51
|
+
}
|
|
52
|
+
stdout.write(encodeLine({ id: request.id, ok: true, result: reply }))
|
|
53
|
+
return true
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
stdout,
|
|
57
|
+
stderr,
|
|
58
|
+
once(_event, handler) {
|
|
59
|
+
exitHandlers.push(handler)
|
|
60
|
+
return child
|
|
61
|
+
},
|
|
62
|
+
kill(signal) {
|
|
63
|
+
child.killed.push((signal ?? 'SIGTERM') as NodeJS.Signals)
|
|
64
|
+
return true
|
|
65
|
+
},
|
|
66
|
+
emitExit(code) {
|
|
67
|
+
for (const handler of exitHandlers.splice(0)) handler(code)
|
|
68
|
+
},
|
|
69
|
+
emitStdinError(error) {
|
|
70
|
+
for (const handler of stdinErrorHandlers) handler(error)
|
|
71
|
+
},
|
|
72
|
+
}
|
|
73
|
+
children.push(child)
|
|
74
|
+
return child
|
|
75
|
+
}
|
|
76
|
+
return { spawnFn, children }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const sidecarWith = (spawnFn: SpawnFn, overrides: Partial<Record<string, number>> = {}) =>
|
|
80
|
+
new Sidecar('/fake/sidecar.js', spawnFn, overrides.callTimeoutMs ?? 500, overrides.closeTimeoutMs ?? 50, 4)
|
|
81
|
+
|
|
82
|
+
describe('sidecar client protocol', () => {
|
|
83
|
+
it('correlates a reply to its request', async () => {
|
|
84
|
+
const { spawnFn } = makeFakeSpawn(() => ({ pong: true }))
|
|
85
|
+
await expect(sidecarWith(spawnFn).call('ping')).resolves.toEqual({ pong: true })
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
it('surfaces a sidecar error as a typed rejection', async () => {
|
|
89
|
+
const { spawnFn } = makeFakeSpawn(() => new Error('navigation blocked'))
|
|
90
|
+
await expect(sidecarWith(spawnFn).call('render', { url: 'https://x' })).rejects.toThrow('navigation blocked')
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
// A child that has died leaves its stdin pipe broken, and Node reports that
|
|
94
|
+
// asynchronously on the stream rather than throwing from write(). Unhandled,
|
|
95
|
+
// it escapes the adapter and the call throws where the contract promises an
|
|
96
|
+
// outcome — which is exactly how this surfaced on a runner with no browser.
|
|
97
|
+
it('turns a broken sidecar pipe into a typed rejection, not an unhandled error', async () => {
|
|
98
|
+
const { spawnFn, children } = makeFakeSpawn(() => 'silent')
|
|
99
|
+
const sidecar = sidecarWith(spawnFn)
|
|
100
|
+
const call = sidecar.call('render', { url: 'https://x' })
|
|
101
|
+
const epipe: NodeJS.ErrnoException = Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })
|
|
102
|
+
children[0].emitStdinError(epipe)
|
|
103
|
+
children[0].emitExit(1)
|
|
104
|
+
await expect(call).rejects.toThrow(/exited unexpectedly/)
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
it('times a silent call out instead of hanging forever', async () => {
|
|
108
|
+
const { spawnFn } = makeFakeSpawn(() => 'silent')
|
|
109
|
+
await expect(sidecarWith(spawnFn).call('render')).rejects.toThrow(/timed out/)
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
it('rejects one aborted call without disturbing the others', async () => {
|
|
113
|
+
const { spawnFn } = makeFakeSpawn((request) => (request.method === 'slow' ? 'silent' : { ok: true }))
|
|
114
|
+
const sidecar = sidecarWith(spawnFn)
|
|
115
|
+
const controller = new AbortController()
|
|
116
|
+
|
|
117
|
+
const aborted = sidecar.call('slow', {}, controller.signal)
|
|
118
|
+
controller.abort()
|
|
119
|
+
|
|
120
|
+
await expect(aborted).rejects.toThrow(/aborted/)
|
|
121
|
+
await expect(sidecar.call('ping')).resolves.toEqual({ ok: true })
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
it('degrades to a clean busy error rather than piling up unbounded work', async () => {
|
|
125
|
+
const { spawnFn } = makeFakeSpawn(() => 'silent')
|
|
126
|
+
const sidecar = sidecarWith(spawnFn)
|
|
127
|
+
const inflight = [0, 1, 2, 3].map(() => sidecar.call('slow').catch(() => 'settled'))
|
|
128
|
+
|
|
129
|
+
await expect(sidecar.call('one-too-many')).rejects.toThrow(/busy/)
|
|
130
|
+
await Promise.all(inflight)
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
it('rejects every pending call when the child dies, with the stderr tail', async () => {
|
|
134
|
+
const { spawnFn, children } = makeFakeSpawn(() => 'silent')
|
|
135
|
+
const sidecar = sidecarWith(spawnFn)
|
|
136
|
+
const pending = sidecar.call('render')
|
|
137
|
+
children[0].stderr.write('playwright: Executable does not exist\n')
|
|
138
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
139
|
+
children[0].emitExit(1)
|
|
140
|
+
|
|
141
|
+
await expect(pending).rejects.toThrow(/exited unexpectedly/)
|
|
142
|
+
})
|
|
143
|
+
|
|
144
|
+
// Regression: a child that dies mid-line leaves a JSON fragment behind. If the
|
|
145
|
+
// buffer is not cleared on respawn it prefixes the next spawn's first reply,
|
|
146
|
+
// which then fails to parse and hangs that call for the full timeout.
|
|
147
|
+
it('clears a partial line so the next spawn is not poisoned', async () => {
|
|
148
|
+
const { spawnFn, children } = makeFakeSpawn(() => ({ ok: true }))
|
|
149
|
+
const sidecar = sidecarWith(spawnFn)
|
|
150
|
+
await sidecar.call('ping')
|
|
151
|
+
|
|
152
|
+
children[0].stdout.write('{"id":"truncated","ok":tr')
|
|
153
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
154
|
+
children[0].emitExit(1)
|
|
155
|
+
|
|
156
|
+
await expect(sidecar.call('ping')).resolves.toEqual({ ok: true })
|
|
157
|
+
expect(children).toHaveLength(2)
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
it('escalates to SIGKILL when the child ignores a close', async () => {
|
|
161
|
+
const { spawnFn, children } = makeFakeSpawn(() => 'silent')
|
|
162
|
+
const sidecar = sidecarWith(spawnFn)
|
|
163
|
+
await sidecar.call('ping').catch(() => undefined)
|
|
164
|
+
|
|
165
|
+
await sidecar.close()
|
|
166
|
+
|
|
167
|
+
expect(children[0].killed).toContain('SIGTERM')
|
|
168
|
+
expect(children[0].killed).toContain('SIGKILL')
|
|
169
|
+
})
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
describe('browser adapter', () => {
|
|
173
|
+
const options = (extra: Record<string, unknown> = {}) => ({
|
|
174
|
+
sidecar: { sidecarPath: '/fake/sidecar.js', callTimeoutMs: 500, closeTimeoutMs: 50, ...extra },
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
// Participation is the policy row's job; the scheduler never puts a browser
|
|
178
|
+
// adapter in a normal wave, so it is reachable only via escalation.
|
|
179
|
+
it('is ready without per-adapter opt-in, since the policy row governs it', () => {
|
|
180
|
+
expect(createBrowserAdapter({}).readiness().ready).toBe(true)
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
it('parses a browser-rendered SERP with the shared engine profiles', async () => {
|
|
184
|
+
const { spawnFn } = makeFakeSpawn((request) =>
|
|
185
|
+
request.method === 'render' ? { url: 'https://html.duckduckgo.com/html/', html: SERP_HTML } : {},
|
|
186
|
+
)
|
|
187
|
+
const adapter = createBrowserAdapter(options({ spawnFn }))
|
|
188
|
+
|
|
189
|
+
const outcome = await adapter.search({ query: 'x', limit: 5 }, createTestContext())
|
|
190
|
+
|
|
191
|
+
expect(outcome.status).toBe('ok')
|
|
192
|
+
if (outcome.status !== 'ok') return
|
|
193
|
+
expect(outcome.results[0]).toMatchObject({
|
|
194
|
+
url: 'https://example.com/one',
|
|
195
|
+
title: 'Rendered result',
|
|
196
|
+
snippet: 'Rendered snippet.',
|
|
197
|
+
})
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
it('releases the lease even when the render fails', async () => {
|
|
201
|
+
const { spawnFn, children } = makeFakeSpawn((request) =>
|
|
202
|
+
request.method === 'render' ? new Error('nav failed') : {},
|
|
203
|
+
)
|
|
204
|
+
const adapter = createBrowserAdapter(options({ spawnFn }))
|
|
205
|
+
|
|
206
|
+
await adapter.search({ query: 'x', limit: 5 }, createTestContext())
|
|
207
|
+
|
|
208
|
+
const methods = children[0].written.map((line) => (decodeLine(line.trim()) as SidecarRequest).method)
|
|
209
|
+
expect(methods).toContain('release')
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
it('reports blocked when even a rendered page yields nothing', async () => {
|
|
213
|
+
const { spawnFn } = makeFakeSpawn(() => ({ url: 'https://x', html: '<html><body>nope</body></html>' }))
|
|
214
|
+
const adapter = createBrowserAdapter(options({ spawnFn }))
|
|
215
|
+
|
|
216
|
+
const outcome = await adapter.search({ query: 'x', limit: 5 }, createTestContext())
|
|
217
|
+
|
|
218
|
+
expect(outcome.status).toBe('blocked')
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
it('renders a page into readable text for fetch', async () => {
|
|
222
|
+
const html = `<html><head><title>Doc</title></head><body><article><p>${'word '.repeat(80)}</p></article></body></html>`
|
|
223
|
+
const { spawnFn } = makeFakeSpawn((request) =>
|
|
224
|
+
request.method === 'render' ? { url: 'https://example.com/a', html } : {},
|
|
225
|
+
)
|
|
226
|
+
const adapter = createBrowserAdapter(options({ spawnFn }))
|
|
227
|
+
|
|
228
|
+
const outcome = await adapter.fetch?.({ url: 'https://example.com/a' }, createTestContext())
|
|
229
|
+
|
|
230
|
+
expect(outcome).toMatchObject({ status: 'ok', page: { title: 'Doc', renderedWith: 'browser' } })
|
|
231
|
+
})
|
|
232
|
+
})
|
|
233
|
+
|
|
234
|
+
describeAdapterContract(
|
|
235
|
+
browserAdapterModule,
|
|
236
|
+
{
|
|
237
|
+
configuredOptions: {
|
|
238
|
+
sidecar: { sidecarPath: '/fake/sidecar.js', callTimeoutMs: 200, closeTimeoutMs: 50 },
|
|
239
|
+
},
|
|
240
|
+
},
|
|
241
|
+
{ describe, it, expect: expect as never },
|
|
242
|
+
)
|