@open-mercato/web-research-browser 0.9.1-develop.7573.1.d0ccc51d5b → 0.9.1-develop.7575.1.8480826a9b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/dispatch.js +14 -61
- package/dist/dispatch.js.map +2 -2
- package/dist/playwright.js +7 -9
- package/dist/playwright.js.map +2 -2
- package/package.json +3 -3
- package/src/dispatch.ts +15 -79
- package/src/playwright.ts +17 -42
- package/dist/egressProxy.js +0 -291
- package/dist/egressProxy.js.map +0 -7
- package/src/__tests__/egressProxy.test.ts +0 -481
- package/src/__tests__/launchWiring.test.ts +0 -222
- package/src/__tests__/navigationGuard.test.ts +0 -200
- package/src/egressProxy.ts +0 -345
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
[build:web-research-browser] found
|
|
1
|
+
[build:web-research-browser] found 7 entry points
|
|
2
2
|
[build:web-research-browser] built successfully
|
package/dist/dispatch.js
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { assertPublicUrl } from "@open-mercato/web-research";
|
|
2
|
-
import { startEgressProxy } from "./egressProxy.js";
|
|
3
2
|
import {
|
|
4
3
|
DEFAULT_USER_AGENT,
|
|
5
|
-
RESOLVER_LOCKDOWN_ARGS,
|
|
6
4
|
isSandboxDisabled,
|
|
7
5
|
resolveLaunchArgs,
|
|
8
6
|
importChromium,
|
|
@@ -11,64 +9,33 @@ import {
|
|
|
11
9
|
} from "./playwright.js";
|
|
12
10
|
import { SidecarError } from "./protocol.js";
|
|
13
11
|
const LEASE_IDLE_TIMEOUT_MS = 5 * 6e4;
|
|
14
|
-
const FILTER_SUBRESOURCES_ENV = "OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES";
|
|
15
12
|
const DEFAULT_GOTO_TIMEOUT_MS = 3e4;
|
|
16
13
|
const MAX_GOTO_TIMEOUT_MS = 12e4;
|
|
17
14
|
function createState(overrides = {}) {
|
|
18
15
|
return {
|
|
19
16
|
browser: null,
|
|
20
17
|
leases: /* @__PURE__ */ new Map(),
|
|
21
|
-
filterSubresources:
|
|
18
|
+
filterSubresources: process.env.OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES === "1",
|
|
22
19
|
warn: (message) => process.stderr.write(`om-web-research-sidecar: ${message}
|
|
23
20
|
`),
|
|
24
21
|
now: () => Date.now(),
|
|
25
22
|
...overrides
|
|
26
23
|
};
|
|
27
24
|
}
|
|
28
|
-
function
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
async function launchBrowser(state) {
|
|
32
|
-
const browserType = await (state.loadBrowserType ?? importChromium)();
|
|
25
|
+
async function ensureBrowser(state) {
|
|
26
|
+
if (state.browser) return state.browser;
|
|
27
|
+
const chromium = await importChromium();
|
|
33
28
|
if (isSandboxDisabled()) {
|
|
34
29
|
state.warn(
|
|
35
30
|
"SECURITY: Chromium sandbox is disabled by OM_WEB_RESEARCH_BROWSER_NO_SANDBOX=1 \u2014 a renderer compromise reaches this host"
|
|
36
31
|
);
|
|
37
32
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
});
|
|
45
|
-
let browser;
|
|
46
|
-
try {
|
|
47
|
-
browser = await launchChromium(
|
|
48
|
-
browserType,
|
|
49
|
-
{
|
|
50
|
-
headless: true,
|
|
51
|
-
args: [...resolveLaunchArgs(), ...RESOLVER_LOCKDOWN_ARGS],
|
|
52
|
-
proxy: { server: proxy.url, bypass: "<-loopback>" }
|
|
53
|
-
},
|
|
54
|
-
{ onProgress: state.warn, ...state.installRunner ? { install: state.installRunner } : {} }
|
|
55
|
-
);
|
|
56
|
-
} catch (error) {
|
|
57
|
-
await proxy.close();
|
|
58
|
-
throw error;
|
|
59
|
-
}
|
|
60
|
-
state.egressProxy = proxy;
|
|
61
|
-
state.browser = browser;
|
|
62
|
-
return browser;
|
|
63
|
-
}
|
|
64
|
-
async function ensureBrowser(state) {
|
|
65
|
-
if (state.browser) return state.browser;
|
|
66
|
-
if (!state.launching) {
|
|
67
|
-
state.launching = launchBrowser(state).finally(() => {
|
|
68
|
-
state.launching = null;
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
return state.launching;
|
|
33
|
+
state.browser = await launchChromium(
|
|
34
|
+
chromium,
|
|
35
|
+
{ headless: true, args: resolveLaunchArgs() },
|
|
36
|
+
{ onProgress: state.warn, ...state.installRunner ? { install: state.installRunner } : {} }
|
|
37
|
+
);
|
|
38
|
+
return state.browser;
|
|
72
39
|
}
|
|
73
40
|
async function openLease(state, leaseId, userAgent) {
|
|
74
41
|
await reapIdleLeases(state);
|
|
@@ -78,8 +45,8 @@ async function openLease(state, leaseId, userAgent) {
|
|
|
78
45
|
return existing;
|
|
79
46
|
}
|
|
80
47
|
const browser = await ensureBrowser(state);
|
|
81
|
-
const context = await browser.newContext({ userAgent, locale: "en-US"
|
|
82
|
-
await installNavigationGuard(context, {
|
|
48
|
+
const context = await browser.newContext({ userAgent, locale: "en-US" });
|
|
49
|
+
await installNavigationGuard(context, { filterSubresources: state.filterSubresources, warn: state.warn });
|
|
83
50
|
const page = await context.newPage();
|
|
84
51
|
const lease = { context, page, touchedAt: state.now() };
|
|
85
52
|
state.leases.set(leaseId, lease);
|
|
@@ -103,19 +70,15 @@ async function closeLease(state, leaseId) {
|
|
|
103
70
|
}
|
|
104
71
|
}
|
|
105
72
|
async function teardown(state) {
|
|
106
|
-
if (state.launching) await state.launching.catch(() => null);
|
|
107
73
|
for (const leaseId of [...state.leases.keys()]) await closeLease(state, leaseId);
|
|
108
74
|
const browser = state.browser;
|
|
109
|
-
const proxy = state.egressProxy;
|
|
110
75
|
state.browser = null;
|
|
111
|
-
state.egressProxy = null;
|
|
112
76
|
if (browser) {
|
|
113
77
|
try {
|
|
114
78
|
await browser.close();
|
|
115
79
|
} catch {
|
|
116
80
|
}
|
|
117
81
|
}
|
|
118
|
-
if (proxy) await proxy.close();
|
|
119
82
|
}
|
|
120
83
|
function stringParam(request, key) {
|
|
121
84
|
const value = request.params?.[key];
|
|
@@ -132,7 +95,7 @@ async function dispatchInner(state, request) {
|
|
|
132
95
|
const url = stringParam(request, "url");
|
|
133
96
|
const leaseId = stringParam(request, "leaseId");
|
|
134
97
|
try {
|
|
135
|
-
await assertPublicUrl(url, "browser render", { failClosed: true
|
|
98
|
+
await assertPublicUrl(url, "browser render", { failClosed: true });
|
|
136
99
|
} catch (error) {
|
|
137
100
|
return {
|
|
138
101
|
id: request.id,
|
|
@@ -156,18 +119,8 @@ async function dispatchInner(state, request) {
|
|
|
156
119
|
error: { message: error instanceof Error ? error.message : String(error), kind: "navigation" }
|
|
157
120
|
};
|
|
158
121
|
}
|
|
159
|
-
const finalUrl = lease.page.url();
|
|
160
|
-
try {
|
|
161
|
-
await assertPublicUrl(finalUrl, "browser final URL", { failClosed: true, ...lookupOption(state) });
|
|
162
|
-
} catch (error) {
|
|
163
|
-
return {
|
|
164
|
-
id: request.id,
|
|
165
|
-
ok: false,
|
|
166
|
-
error: { message: error instanceof Error ? error.message : String(error), kind: "navigation" }
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
122
|
const html = await lease.page.content();
|
|
170
|
-
return { id: request.id, ok: true, result: { url:
|
|
123
|
+
return { id: request.id, ok: true, result: { url: lease.page.url(), html } };
|
|
171
124
|
}
|
|
172
125
|
case "release":
|
|
173
126
|
await closeLease(state, stringParam(request, "leaseId"));
|
package/dist/dispatch.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/dispatch.ts"],
|
|
4
|
-
"sourcesContent": ["import { assertPublicUrl
|
|
5
|
-
"mappings": "AAAA,SAAS,
|
|
4
|
+
"sourcesContent": ["import { assertPublicUrl } from '@open-mercato/web-research'\nimport {\n DEFAULT_USER_AGENT,\n isSandboxDisabled,\n resolveLaunchArgs,\n importChromium,\n installNavigationGuard,\n launchChromium,\n type InstallRunner,\n type BrowserHandle,\n type ContextHandle,\n type PageHandle,\n} from './playwright'\nimport { SidecarError, type SidecarReply, type SidecarRequest } from './protocol'\n\nexport type Lease = { context: ContextHandle; page: PageHandle; touchedAt: number }\n\n/**\n * How long an untouched lease may live.\n *\n * `release` normally closes a context, but the caller can die between `render`\n * and `release` \u2014 a killed parent, a dropped socket \u2014 and every orphan is a live\n * BrowserContext holding memory for as long as the sidecar runs.\n */\nexport const LEASE_IDLE_TIMEOUT_MS = 5 * 60_000\n\nexport type SidecarState = {\n browser: BrowserHandle | null\n leases: Map<string, Lease>\n filterSubresources: boolean\n warn: (message: string) => void\n /** Injectable so a lease-expiry test does not have to wait five minutes. */\n now: () => number\n /** Injectable so tests never shell out to `npx playwright install`. */\n installRunner?: InstallRunner\n}\n\nexport const DEFAULT_GOTO_TIMEOUT_MS = 30_000\nconst MAX_GOTO_TIMEOUT_MS = 120_000\n\nexport function createState(overrides: Partial<SidecarState> = {}): SidecarState {\n return {\n browser: null,\n leases: new Map(),\n filterSubresources: process.env.OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES === '1',\n warn: (message) => process.stderr.write(`om-web-research-sidecar: ${message}\\n`),\n now: () => Date.now(),\n ...overrides,\n }\n}\n\nasync function ensureBrowser(state: SidecarState): Promise<BrowserHandle> {\n if (state.browser) return state.browser\n const chromium = await importChromium()\n if (isSandboxDisabled()) {\n // Said once per browser, not once per render, but said: this process renders\n // pages chosen by a search engine or a model with the one mitigation\n // Chromium offers against a hostile renderer turned off.\n state.warn(\n 'SECURITY: Chromium sandbox is disabled by OM_WEB_RESEARCH_BROWSER_NO_SANDBOX=1 \u2014 a renderer compromise reaches this host',\n )\n }\n state.browser = await launchChromium(\n chromium,\n { headless: true, args: resolveLaunchArgs() },\n { onProgress: state.warn, ...(state.installRunner ? { install: state.installRunner } : {}) },\n )\n return state.browser\n}\n\n/**\n * One BrowserContext per lease. Contexts are Playwright's real isolation boundary\n * (cookies, storage, cache) and cost ~10ms, so two concurrent searches never share\n * a cookie jar the way a single shared page would.\n */\nasync function openLease(state: SidecarState, leaseId: string, userAgent: string): Promise<Lease> {\n await reapIdleLeases(state)\n const existing = state.leases.get(leaseId)\n if (existing) {\n existing.touchedAt = state.now()\n return existing\n }\n const browser = await ensureBrowser(state)\n const context = await browser.newContext({ userAgent, locale: 'en-US' })\n await installNavigationGuard(context, { filterSubresources: state.filterSubresources, warn: state.warn })\n const page = (await context.newPage()) as unknown as PageHandle\n const lease: Lease = { context, page, touchedAt: state.now() }\n state.leases.set(leaseId, lease)\n return lease\n}\n\n/** Closes leases nobody released. Swept on open, so an idle sidecar stays idle. */\nexport async function reapIdleLeases(state: SidecarState): Promise<void> {\n const cutoff = state.now() - LEASE_IDLE_TIMEOUT_MS\n for (const [leaseId, lease] of [...state.leases]) {\n if (lease.touchedAt > cutoff) continue\n state.warn(`reaping lease ${leaseId} idle for over ${LEASE_IDLE_TIMEOUT_MS}ms`)\n await closeLease(state, leaseId)\n }\n}\n\nasync function closeLease(state: SidecarState, leaseId: string): Promise<void> {\n const lease = state.leases.get(leaseId)\n if (!lease) return\n state.leases.delete(leaseId)\n try {\n await lease.context.close()\n } catch {\n // A context that already died takes nothing with it.\n }\n}\n\nexport async function teardown(state: SidecarState): Promise<void> {\n for (const leaseId of [...state.leases.keys()]) await closeLease(state, leaseId)\n const browser = state.browser\n state.browser = null\n if (browser) {\n try {\n await browser.close()\n } catch {\n // Best effort; the parent escalates to SIGKILL if we hang.\n }\n }\n}\n\nfunction stringParam(request: SidecarRequest, key: string): string {\n const value = request.params?.[key]\n if (typeof value !== 'string' || value.length === 0) {\n throw new SidecarError(`missing \"${key}\" parameter`, 'runtime')\n }\n return value\n}\n\nasync function dispatchInner(state: SidecarState, request: SidecarRequest): Promise<SidecarReply> {\n switch (request.method) {\n case 'ping':\n return { id: request.id, ok: true, result: { ready: true } }\n\n case 'render': {\n const url = stringParam(request, 'url')\n const leaseId = stringParam(request, 'leaseId')\n // Re-checked here, in a distinct process from the caller, and BEFORE\n // Playwright is imported \u2014 a blocked URL never launches a browser.\n try {\n await assertPublicUrl(url, 'browser render', { failClosed: true })\n } catch (error) {\n return {\n id: request.id,\n ok: false,\n error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },\n }\n }\n\n const userAgent = typeof request.params?.userAgent === 'string' ? request.params.userAgent : DEFAULT_USER_AGENT\n const timeout = Math.min(\n typeof request.params?.timeoutMs === 'number' ? request.params.timeoutMs : DEFAULT_GOTO_TIMEOUT_MS,\n MAX_GOTO_TIMEOUT_MS,\n )\n const waitUntil = typeof request.params?.waitUntil === 'string' ? request.params.waitUntil : 'domcontentloaded'\n\n const lease = await openLease(state, leaseId, userAgent)\n try {\n await lease.page.goto(url, { waitUntil, timeout })\n } catch (error) {\n return {\n id: request.id,\n ok: false,\n error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },\n }\n }\n const html = await lease.page.content()\n return { id: request.id, ok: true, result: { url: lease.page.url(), html } }\n }\n\n case 'release':\n await closeLease(state, stringParam(request, 'leaseId'))\n return { id: request.id, ok: true }\n\n case 'close':\n await teardown(state)\n return { id: request.id, ok: true }\n\n default:\n return {\n id: request.id,\n ok: false,\n error: { message: `unknown method: ${request.method}`, kind: 'runtime' },\n }\n }\n}\n\nexport async function dispatch(state: SidecarState, request: SidecarRequest): Promise<SidecarReply> {\n try {\n return await dispatchInner(state, request)\n } catch (error) {\n return {\n id: request.id,\n ok: false,\n error: {\n message: error instanceof Error ? error.message : String(error),\n kind: error instanceof SidecarError ? error.kind : 'unknown',\n },\n }\n }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,uBAAuB;AAChC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAKK;AACP,SAAS,oBAA4D;AAW9D,MAAM,wBAAwB,IAAI;AAalC,MAAM,0BAA0B;AACvC,MAAM,sBAAsB;AAErB,SAAS,YAAY,YAAmC,CAAC,GAAiB;AAC/E,SAAO;AAAA,IACL,SAAS;AAAA,IACT,QAAQ,oBAAI,IAAI;AAAA,IAChB,oBAAoB,QAAQ,IAAI,gDAAgD;AAAA,IAChF,MAAM,CAAC,YAAY,QAAQ,OAAO,MAAM,4BAA4B,OAAO;AAAA,CAAI;AAAA,IAC/E,KAAK,MAAM,KAAK,IAAI;AAAA,IACpB,GAAG;AAAA,EACL;AACF;AAEA,eAAe,cAAc,OAA6C;AACxE,MAAI,MAAM,QAAS,QAAO,MAAM;AAChC,QAAM,WAAW,MAAM,eAAe;AACtC,MAAI,kBAAkB,GAAG;AAIvB,UAAM;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AACA,QAAM,UAAU,MAAM;AAAA,IACpB;AAAA,IACA,EAAE,UAAU,MAAM,MAAM,kBAAkB,EAAE;AAAA,IAC5C,EAAE,YAAY,MAAM,MAAM,GAAI,MAAM,gBAAgB,EAAE,SAAS,MAAM,cAAc,IAAI,CAAC,EAAG;AAAA,EAC7F;AACA,SAAO,MAAM;AACf;AAOA,eAAe,UAAU,OAAqB,SAAiB,WAAmC;AAChG,QAAM,eAAe,KAAK;AAC1B,QAAM,WAAW,MAAM,OAAO,IAAI,OAAO;AACzC,MAAI,UAAU;AACZ,aAAS,YAAY,MAAM,IAAI;AAC/B,WAAO;AAAA,EACT;AACA,QAAM,UAAU,MAAM,cAAc,KAAK;AACzC,QAAM,UAAU,MAAM,QAAQ,WAAW,EAAE,WAAW,QAAQ,QAAQ,CAAC;AACvE,QAAM,uBAAuB,SAAS,EAAE,oBAAoB,MAAM,oBAAoB,MAAM,MAAM,KAAK,CAAC;AACxG,QAAM,OAAQ,MAAM,QAAQ,QAAQ;AACpC,QAAM,QAAe,EAAE,SAAS,MAAM,WAAW,MAAM,IAAI,EAAE;AAC7D,QAAM,OAAO,IAAI,SAAS,KAAK;AAC/B,SAAO;AACT;AAGA,eAAsB,eAAe,OAAoC;AACvE,QAAM,SAAS,MAAM,IAAI,IAAI;AAC7B,aAAW,CAAC,SAAS,KAAK,KAAK,CAAC,GAAG,MAAM,MAAM,GAAG;AAChD,QAAI,MAAM,YAAY,OAAQ;AAC9B,UAAM,KAAK,iBAAiB,OAAO,kBAAkB,qBAAqB,IAAI;AAC9E,UAAM,WAAW,OAAO,OAAO;AAAA,EACjC;AACF;AAEA,eAAe,WAAW,OAAqB,SAAgC;AAC7E,QAAM,QAAQ,MAAM,OAAO,IAAI,OAAO;AACtC,MAAI,CAAC,MAAO;AACZ,QAAM,OAAO,OAAO,OAAO;AAC3B,MAAI;AACF,UAAM,MAAM,QAAQ,MAAM;AAAA,EAC5B,QAAQ;AAAA,EAER;AACF;AAEA,eAAsB,SAAS,OAAoC;AACjE,aAAW,WAAW,CAAC,GAAG,MAAM,OAAO,KAAK,CAAC,EAAG,OAAM,WAAW,OAAO,OAAO;AAC/E,QAAM,UAAU,MAAM;AACtB,QAAM,UAAU;AAChB,MAAI,SAAS;AACX,QAAI;AACF,YAAM,QAAQ,MAAM;AAAA,IACtB,QAAQ;AAAA,IAER;AAAA,EACF;AACF;AAEA,SAAS,YAAY,SAAyB,KAAqB;AACjE,QAAM,QAAQ,QAAQ,SAAS,GAAG;AAClC,MAAI,OAAO,UAAU,YAAY,MAAM,WAAW,GAAG;AACnD,UAAM,IAAI,aAAa,YAAY,GAAG,eAAe,SAAS;AAAA,EAChE;AACA,SAAO;AACT;AAEA,eAAe,cAAc,OAAqB,SAAgD;AAChG,UAAQ,QAAQ,QAAQ;AAAA,IACtB,KAAK;AACH,aAAO,EAAE,IAAI,QAAQ,IAAI,IAAI,MAAM,QAAQ,EAAE,OAAO,KAAK,EAAE;AAAA,IAE7D,KAAK,UAAU;AACb,YAAM,MAAM,YAAY,SAAS,KAAK;AACtC,YAAM,UAAU,YAAY,SAAS,SAAS;AAG9C,UAAI;AACF,cAAM,gBAAgB,KAAK,kBAAkB,EAAE,YAAY,KAAK,CAAC;AAAA,MACnE,SAAS,OAAO;AACd,eAAO;AAAA,UACL,IAAI,QAAQ;AAAA,UACZ,IAAI;AAAA,UACJ,OAAO,EAAE,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAAG,MAAM,aAAa;AAAA,QAC/F;AAAA,MACF;AAEA,YAAM,YAAY,OAAO,QAAQ,QAAQ,cAAc,WAAW,QAAQ,OAAO,YAAY;AAC7F,YAAM,UAAU,KAAK;AAAA,QACnB,OAAO,QAAQ,QAAQ,cAAc,WAAW,QAAQ,OAAO,YAAY;AAAA,QAC3E;AAAA,MACF;AACA,YAAM,YAAY,OAAO,QAAQ,QAAQ,cAAc,WAAW,QAAQ,OAAO,YAAY;AAE7F,YAAM,QAAQ,MAAM,UAAU,OAAO,SAAS,SAAS;AACvD,UAAI;AACF,cAAM,MAAM,KAAK,KAAK,KAAK,EAAE,WAAW,QAAQ,CAAC;AAAA,MACnD,SAAS,OAAO;AACd,eAAO;AAAA,UACL,IAAI,QAAQ;AAAA,UACZ,IAAI;AAAA,UACJ,OAAO,EAAE,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAAG,MAAM,aAAa;AAAA,QAC/F;AAAA,MACF;AACA,YAAM,OAAO,MAAM,MAAM,KAAK,QAAQ;AACtC,aAAO,EAAE,IAAI,QAAQ,IAAI,IAAI,MAAM,QAAQ,EAAE,KAAK,MAAM,KAAK,IAAI,GAAG,KAAK,EAAE;AAAA,IAC7E;AAAA,IAEA,KAAK;AACH,YAAM,WAAW,OAAO,YAAY,SAAS,SAAS,CAAC;AACvD,aAAO,EAAE,IAAI,QAAQ,IAAI,IAAI,KAAK;AAAA,IAEpC,KAAK;AACH,YAAM,SAAS,KAAK;AACpB,aAAO,EAAE,IAAI,QAAQ,IAAI,IAAI,KAAK;AAAA,IAEpC;AACE,aAAO;AAAA,QACL,IAAI,QAAQ;AAAA,QACZ,IAAI;AAAA,QACJ,OAAO,EAAE,SAAS,mBAAmB,QAAQ,MAAM,IAAI,MAAM,UAAU;AAAA,MACzE;AAAA,EACJ;AACF;AAEA,eAAsB,SAAS,OAAqB,SAAgD;AAClG,MAAI;AACF,WAAO,MAAM,cAAc,OAAO,OAAO;AAAA,EAC3C,SAAS,OAAO;AACd,WAAO;AAAA,MACL,IAAI,QAAQ;AAAA,MACZ,IAAI;AAAA,MACJ,OAAO;AAAA,QACL,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,MAAM,iBAAiB,eAAe,MAAM,OAAO;AAAA,MACrD;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/playwright.js
CHANGED
|
@@ -75,8 +75,10 @@ async function launchChromium(browserType, options, deps = {}) {
|
|
|
75
75
|
const MAX_VERDICT_CACHE = 2048;
|
|
76
76
|
async function installNavigationGuard(context, options = {}) {
|
|
77
77
|
if (typeof context.route !== "function") {
|
|
78
|
-
|
|
78
|
+
options.warn?.("context.route() unavailable \u2014 in-page navigation SSRF guard NOT installed");
|
|
79
|
+
return;
|
|
79
80
|
}
|
|
81
|
+
const filterSubresources = options.filterSubresources ?? false;
|
|
80
82
|
const verdicts = /* @__PURE__ */ new Map();
|
|
81
83
|
const remember = (host, blocked) => {
|
|
82
84
|
if (verdicts.size >= MAX_VERDICT_CACHE) {
|
|
@@ -88,20 +90,20 @@ async function installNavigationGuard(context, options = {}) {
|
|
|
88
90
|
await context.route("**/*", async (route) => {
|
|
89
91
|
const request = route.request();
|
|
90
92
|
const isNavigation = request.isNavigationRequest();
|
|
93
|
+
if (!isNavigation && !filterSubresources) return route.continue();
|
|
91
94
|
const url = request.url();
|
|
92
95
|
let host;
|
|
93
96
|
try {
|
|
94
97
|
host = new URL(url).host;
|
|
95
98
|
} catch {
|
|
96
|
-
return route.abort("blockedbyclient");
|
|
99
|
+
return isNavigation ? route.abort("blockedbyclient") : route.continue();
|
|
97
100
|
}
|
|
98
101
|
const cached = isNavigation ? void 0 : verdicts.get(host);
|
|
99
102
|
if (cached === true) return route.abort("blockedbyclient");
|
|
100
103
|
if (cached === false) return route.continue();
|
|
101
104
|
try {
|
|
102
105
|
await assertPublicUrl(url, isNavigation ? "browser navigation" : "browser subresource", {
|
|
103
|
-
failClosed: true
|
|
104
|
-
...options.lookup ? { lookup: options.lookup } : {}
|
|
106
|
+
failClosed: true
|
|
105
107
|
});
|
|
106
108
|
remember(host, false);
|
|
107
109
|
} catch {
|
|
@@ -111,12 +113,9 @@ async function installNavigationGuard(context, options = {}) {
|
|
|
111
113
|
return route.continue();
|
|
112
114
|
});
|
|
113
115
|
}
|
|
114
|
-
const RESOLVER_LOCKDOWN_ARGS = ["--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE 127.0.0.1"];
|
|
115
116
|
const LAUNCH_ARGS = [
|
|
116
117
|
"--disable-blink-features=AutomationControlled",
|
|
117
|
-
"--disable-dev-shm-usage"
|
|
118
|
-
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
|
|
119
|
-
"--disable-quic"
|
|
118
|
+
"--disable-dev-shm-usage"
|
|
120
119
|
];
|
|
121
120
|
const SANDBOX_OPT_OUT_ENV = "OM_WEB_RESEARCH_BROWSER_NO_SANDBOX";
|
|
122
121
|
function resolveLaunchArgs(env = process.env) {
|
|
@@ -133,7 +132,6 @@ const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Appl
|
|
|
133
132
|
export {
|
|
134
133
|
DEFAULT_USER_AGENT,
|
|
135
134
|
LAUNCH_ARGS,
|
|
136
|
-
RESOLVER_LOCKDOWN_ARGS,
|
|
137
135
|
SANDBOX_OPT_OUT_ENV,
|
|
138
136
|
importChromium,
|
|
139
137
|
installNavigationGuard,
|
package/dist/playwright.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/playwright.ts"],
|
|
4
|
-
"sourcesContent": ["import { spawn } from 'node:child_process'\nimport { assertPublicUrl, type LookupFn } from '@open-mercato/web-research'\nimport { SidecarError } from './protocol'\n\n/**\n * Structural projections of the Playwright surface we touch.\n *\n * Playwright is a real dependency of this package \u2014 installing the adapter gets\n * you the tooling it needs \u2014 but it is still reached through a dynamic import\n * inside the sidecar, so the app and MCP processes never load it. The types stay\n * structural for the same reason: nothing here should pull the SDK into a bundle\n * that only ever spawns the child.\n */\nexport type PageHandle = {\n goto(url: string, options?: { waitUntil?: string; timeout?: number }): Promise<unknown>\n content(): Promise<string>\n url(): string\n close(): Promise<void>\n}\n\nexport type ContextHandle = {\n newPage(): Promise<unknown>\n close(): Promise<void>\n route?(pattern: string, handler: (route: RouteHandle) => Promise<void> | void): Promise<void>\n}\n\nexport type RouteHandle = {\n request(): { url(): string; isNavigationRequest(): boolean }\n abort(reason?: string): Promise<void>\n continue(): Promise<void>\n}\n\nexport type BrowserHandle = {\n newContext(options?: Record<string, unknown>): Promise<ContextHandle>\n close(): Promise<void>\n}\n\nexport type LaunchOptions = {\n headless?: boolean\n args?: string[]\n proxy?: { server: string; bypass?: string }\n}\n\nexport type BrowserType = {\n launch(options?: LaunchOptions): Promise<BrowserHandle>\n}\n\nfunction isModuleNotFound(message: string): boolean {\n return (\n /cannot find (package|module) ['\"]?playwright/i.test(message) ||\n /ERR_MODULE_NOT_FOUND/i.test(message) ||\n /failed to resolve ['\"]?playwright/i.test(message)\n )\n}\n\nexport async function importChromium(): Promise<BrowserType> {\n try {\n const playwright = (await import('playwright')) as unknown as { chromium: BrowserType }\n return playwright.chromium\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error)\n // Playwright is a dependency of this package, so a module-not-found here\n // means a broken install (hoisting, a pruned image layer), not a missing opt-in.\n throw new SidecarError(\n `Playwright could not be loaded even though it is a dependency of this package \u2014 reinstall dependencies. Underlying: ${message}`,\n isModuleNotFound(message) ? 'needs-install' : 'init',\n )\n }\n}\n\n/**\n * Playwright's launch error when the npm package is installed but the browser\n * binary was never downloaded. The wording has been stable across 1.x, and it is\n * the only signal Playwright gives for this case.\n */\nfunction isMissingBrowserError(error: unknown): boolean {\n return error instanceof Error && /Executable doesn'?t exist at/i.test(error.message)\n}\n\n/** Generous, but bounded \u2014 an unbounded install hangs the sidecar forever. */\nconst INSTALL_TIMEOUT_MS = 10 * 60_000\n\nexport type InstallRunner = (onProgress: (line: string) => void) => Promise<void>\n\nconst defaultInstallRunner: InstallRunner = async (onProgress) =>\n new Promise((resolve, reject) => {\n const child = spawn('npx', ['playwright', 'install', 'chromium'], { stdio: ['ignore', 'pipe', 'pipe'] })\n let tail = ''\n const collect = (chunk: Buffer): void => {\n tail = `${tail}${chunk.toString('utf8')}`.slice(-4000)\n }\n child.stdout?.on('data', collect)\n child.stderr?.on('data', collect)\n\n const timer = setTimeout(() => {\n child.kill('SIGKILL')\n reject(new Error(`\\`npx playwright install chromium\\` exceeded ${INSTALL_TIMEOUT_MS}ms`))\n }, INSTALL_TIMEOUT_MS)\n\n child.once('error', (error) => {\n clearTimeout(timer)\n reject(error)\n })\n child.once('close', (code) => {\n clearTimeout(timer)\n if (code === 0) {\n onProgress('chromium install complete')\n resolve()\n return\n }\n reject(new Error(`\\`npx playwright install chromium\\` failed (exit ${code}): ${tail.trim()}`))\n })\n })\n\n/**\n * Launches Chromium, downloading the binary once if it is missing.\n *\n * Installing the npm package does not fetch the ~150MB browser, so an operator\n * who enables this tier would otherwise hit a launch failure with no obvious\n * remedy. One retry only \u2014 a second failure is a real fault, not a cold cache.\n */\nexport async function launchChromium(\n browserType: BrowserType,\n options: LaunchOptions,\n deps: { install?: InstallRunner; onProgress?: (line: string) => void } = {},\n): Promise<BrowserHandle> {\n try {\n return await browserType.launch(options)\n } catch (error) {\n // Deliberately not retried without the sandbox. Falling back silently would\n // mean the security posture depends on whether a container happened to be\n // configured, with nobody told which one they got.\n if (isSandboxFailure(error)) {\n throw new SidecarError(\n `Chromium could not start its sandbox, and this adapter renders untrusted pages so it will not run without one. ` +\n `Fix the container (run as a non-root user, or grant SYS_ADMIN, or enable user namespaces), ` +\n `or accept the risk explicitly with ${SANDBOX_OPT_OUT_ENV}=1. Underlying: ${error instanceof Error ? error.message : String(error)}`,\n 'init',\n )\n }\n if (!isMissingBrowserError(error)) throw error\n const onProgress = deps.onProgress ?? (() => {})\n onProgress('chromium binary missing, running `npx playwright install chromium` (one-time, ~150MB)')\n try {\n await (deps.install ?? defaultInstallRunner)(onProgress)\n } catch (installError) {\n const detail = installError instanceof Error ? installError.message : String(installError)\n throw new SidecarError(\n `Playwright browser auto-install failed: ${detail}. Run \\`npx playwright install chromium\\` manually.`,\n 'init',\n )\n }\n return await browserType.launch(options)\n }\n}\n\nconst MAX_VERDICT_CACHE = 2048\n\nexport type NavigationGuardOptions = {\n /** @deprecated Every routed request is filtered now; this option is ignored. */\n filterSubresources?: boolean\n warn?: (message: string) => void\n lookup?: LookupFn\n}\n\n/**\n * Aborts every routed request whose URL is not public, navigations and\n * subresources alike, for the life of the context.\n *\n * This is an early-abort layer, not the egress boundary: `context.route()` never\n * sees redirect hops, WebSockets, service workers or UDP, and it vets the name\n * with `node:dns` while Chromium dials whatever its own resolver returns. The\n * boundary is the pinned egress proxy every connection is forced through (see\n * `egressProxy.ts`). Navigation verdicts are never cached; subresource verdicts\n * are cached per host for the life of the context.\n *\n * Fails closed: a context without request interception is refused, and a URL\n * that cannot be parsed is aborted.\n */\nexport async function installNavigationGuard(\n context: ContextHandle,\n options: NavigationGuardOptions = {},\n): Promise<void> {\n if (typeof context.route !== 'function') {\n throw new SidecarError('context.route() is unavailable, so the request guard cannot be installed', 'init')\n }\n const verdicts = new Map<string, boolean>()\n const remember = (host: string, blocked: boolean): void => {\n if (verdicts.size >= MAX_VERDICT_CACHE) {\n const oldest = verdicts.keys().next().value\n if (oldest !== undefined) verdicts.delete(oldest)\n }\n verdicts.set(host, blocked)\n }\n\n await context.route('**/*', async (route) => {\n const request = route.request()\n const isNavigation = request.isNavigationRequest()\n const url = request.url()\n let host: string\n try {\n host = new URL(url).host\n } catch {\n return route.abort('blockedbyclient')\n }\n\n const cached = isNavigation ? undefined : verdicts.get(host)\n if (cached === true) return route.abort('blockedbyclient')\n if (cached === false) return route.continue()\n\n try {\n await assertPublicUrl(url, isNavigation ? 'browser navigation' : 'browser subresource', {\n failClosed: true,\n ...(options.lookup ? { lookup: options.lookup } : {}),\n })\n remember(host, false)\n } catch {\n remember(host, true)\n return route.abort('blockedbyclient')\n }\n return route.continue()\n })\n}\n\n/**\n * Denies Chromium its own DNS, so any path that bypasses the egress proxy by\n * hostname fails instead of resolving. The proxy address is a literal, so it is\n * unaffected. Only meaningful alongside the egress proxy, so it is kept out of\n * {@link LAUNCH_ARGS}.\n */\nexport const RESOLVER_LOCKDOWN_ARGS = ['--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE 127.0.0.1']\n\n/**\n * Launch flags aimed at not announcing automation. Playwright's defaults set\n * `navigator.webdriver` and a headless UA, which the SERP endpoints this adapter\n * exists to reach specifically look for.\n *\n * `--disable-dev-shm-usage` is a sizing workaround, not a security control: it\n * makes Chromium write shared memory to /tmp instead of a container's small\n * /dev/shm. The WebRTC policy and `--disable-quic` keep UDP from leaving outside\n * the egress proxy. The sandbox is deliberately NOT disabled here \u2014 see\n * {@link resolveLaunchArgs}.\n */\nexport const LAUNCH_ARGS = [\n '--disable-blink-features=AutomationControlled',\n '--disable-dev-shm-usage',\n '--force-webrtc-ip-handling-policy=disable_non_proxied_udp',\n '--disable-quic',\n]\n\nexport const SANDBOX_OPT_OUT_ENV = 'OM_WEB_RESEARCH_BROWSER_NO_SANDBOX'\n\n/**\n * Launch flags for this process, sandbox included unless an operator opted out.\n *\n * This adapter exists to render pages chosen by a search engine or a model, so\n * the renderer is by definition parsing hostile input. `--no-sandbox` turns a\n * renderer compromise into code running as whatever user the app runs as, which\n * is the one mitigation Chromium provides for exactly this threat.\n *\n * It used to be on unconditionally because the sandbox needs privileges a\n * root-in-Docker container does not grant by default. That is a container\n * problem with container fixes (run as a non-root user, or grant `SYS_ADMIN`, or\n * enable user namespaces); silently shipping every deployment the insecure\n * variant to spare those is the wrong trade. The opt-out stays available for\n * environments that genuinely cannot, and it is loud.\n */\nexport function resolveLaunchArgs(env: NodeJS.ProcessEnv = process.env): string[] {\n return env[SANDBOX_OPT_OUT_ENV] === '1' ? [...LAUNCH_ARGS, '--no-sandbox'] : [...LAUNCH_ARGS]\n}\n\nexport function isSandboxDisabled(env: NodeJS.ProcessEnv = process.env): boolean {\n return env[SANDBOX_OPT_OUT_ENV] === '1'\n}\n\n/**\n * Chromium's refusal when it cannot set up its sandbox. The wording differs\n * between the root check and the namespace failure, so both shapes are matched.\n */\nexport function isSandboxFailure(error: unknown): boolean {\n if (!(error instanceof Error)) return false\n return (\n /running as root without --no-sandbox/i.test(error.message) ||\n /No usable sandbox/i.test(error.message) ||\n /clone helper|namespace sandbox|SUID sandbox/i.test(error.message)\n )\n}\n\nexport const DEFAULT_USER_AGENT =\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,aAAa;AACtB,SAAS,
|
|
4
|
+
"sourcesContent": ["import { spawn } from 'node:child_process'\nimport { assertPublicUrl } from '@open-mercato/web-research'\nimport { SidecarError } from './protocol'\n\n/**\n * Structural projections of the Playwright surface we touch.\n *\n * Playwright is a real dependency of this package \u2014 installing the adapter gets\n * you the tooling it needs \u2014 but it is still reached through a dynamic import\n * inside the sidecar, so the app and MCP processes never load it. The types stay\n * structural for the same reason: nothing here should pull the SDK into a bundle\n * that only ever spawns the child.\n */\nexport type PageHandle = {\n goto(url: string, options?: { waitUntil?: string; timeout?: number }): Promise<unknown>\n content(): Promise<string>\n url(): string\n close(): Promise<void>\n}\n\nexport type ContextHandle = {\n newPage(): Promise<unknown>\n close(): Promise<void>\n route?(pattern: string, handler: (route: RouteHandle) => Promise<void> | void): Promise<void>\n}\n\nexport type RouteHandle = {\n request(): { url(): string; isNavigationRequest(): boolean }\n abort(reason?: string): Promise<void>\n continue(): Promise<void>\n}\n\nexport type BrowserHandle = {\n newContext(options?: Record<string, unknown>): Promise<ContextHandle>\n close(): Promise<void>\n}\n\nexport type BrowserType = {\n launch(options?: { headless?: boolean; args?: string[] }): Promise<BrowserHandle>\n}\n\nfunction isModuleNotFound(message: string): boolean {\n return (\n /cannot find (package|module) ['\"]?playwright/i.test(message) ||\n /ERR_MODULE_NOT_FOUND/i.test(message) ||\n /failed to resolve ['\"]?playwright/i.test(message)\n )\n}\n\nexport async function importChromium(): Promise<BrowserType> {\n try {\n const playwright = (await import('playwright')) as unknown as { chromium: BrowserType }\n return playwright.chromium\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error)\n // Playwright is a dependency of this package, so a module-not-found here\n // means a broken install (hoisting, a pruned image layer), not a missing opt-in.\n throw new SidecarError(\n `Playwright could not be loaded even though it is a dependency of this package \u2014 reinstall dependencies. Underlying: ${message}`,\n isModuleNotFound(message) ? 'needs-install' : 'init',\n )\n }\n}\n\n/**\n * Playwright's launch error when the npm package is installed but the browser\n * binary was never downloaded. The wording has been stable across 1.x, and it is\n * the only signal Playwright gives for this case.\n */\nfunction isMissingBrowserError(error: unknown): boolean {\n return error instanceof Error && /Executable doesn'?t exist at/i.test(error.message)\n}\n\n/** Generous, but bounded \u2014 an unbounded install hangs the sidecar forever. */\nconst INSTALL_TIMEOUT_MS = 10 * 60_000\n\nexport type InstallRunner = (onProgress: (line: string) => void) => Promise<void>\n\nconst defaultInstallRunner: InstallRunner = async (onProgress) =>\n new Promise((resolve, reject) => {\n const child = spawn('npx', ['playwright', 'install', 'chromium'], { stdio: ['ignore', 'pipe', 'pipe'] })\n let tail = ''\n const collect = (chunk: Buffer): void => {\n tail = `${tail}${chunk.toString('utf8')}`.slice(-4000)\n }\n child.stdout?.on('data', collect)\n child.stderr?.on('data', collect)\n\n const timer = setTimeout(() => {\n child.kill('SIGKILL')\n reject(new Error(`\\`npx playwright install chromium\\` exceeded ${INSTALL_TIMEOUT_MS}ms`))\n }, INSTALL_TIMEOUT_MS)\n\n child.once('error', (error) => {\n clearTimeout(timer)\n reject(error)\n })\n child.once('close', (code) => {\n clearTimeout(timer)\n if (code === 0) {\n onProgress('chromium install complete')\n resolve()\n return\n }\n reject(new Error(`\\`npx playwright install chromium\\` failed (exit ${code}): ${tail.trim()}`))\n })\n })\n\n/**\n * Launches Chromium, downloading the binary once if it is missing.\n *\n * Installing the npm package does not fetch the ~150MB browser, so an operator\n * who enables this tier would otherwise hit a launch failure with no obvious\n * remedy. One retry only \u2014 a second failure is a real fault, not a cold cache.\n */\nexport async function launchChromium(\n browserType: BrowserType,\n options: { headless?: boolean; args?: string[] },\n deps: { install?: InstallRunner; onProgress?: (line: string) => void } = {},\n): Promise<BrowserHandle> {\n try {\n return await browserType.launch(options)\n } catch (error) {\n // Deliberately not retried without the sandbox. Falling back silently would\n // mean the security posture depends on whether a container happened to be\n // configured, with nobody told which one they got.\n if (isSandboxFailure(error)) {\n throw new SidecarError(\n `Chromium could not start its sandbox, and this adapter renders untrusted pages so it will not run without one. ` +\n `Fix the container (run as a non-root user, or grant SYS_ADMIN, or enable user namespaces), ` +\n `or accept the risk explicitly with ${SANDBOX_OPT_OUT_ENV}=1. Underlying: ${error instanceof Error ? error.message : String(error)}`,\n 'init',\n )\n }\n if (!isMissingBrowserError(error)) throw error\n const onProgress = deps.onProgress ?? (() => {})\n onProgress('chromium binary missing, running `npx playwright install chromium` (one-time, ~150MB)')\n try {\n await (deps.install ?? defaultInstallRunner)(onProgress)\n } catch (installError) {\n const detail = installError instanceof Error ? installError.message : String(installError)\n throw new SidecarError(\n `Playwright browser auto-install failed: ${detail}. Run \\`npx playwright install chromium\\` manually.`,\n 'init',\n )\n }\n return await browserType.launch(options)\n }\n}\n\nconst MAX_VERDICT_CACHE = 2048\n\n/**\n * Blocks private-origin navigations for the life of the context.\n *\n * The parent and the sidecar both vet the URL before a `goto`, but a page reached\n * through a legitimate public navigation can redirect or script-navigate itself\n * to a metadata endpoint, and those hops never pass through the RPC layer.\n *\n * Fails closed: Chromium resolves names with its own resolver, so a hostname\n * `node:dns` cannot vet must not be allowed through on the assumption it is dead.\n */\nexport async function installNavigationGuard(\n context: ContextHandle,\n options: { filterSubresources?: boolean; warn?: (message: string) => void } = {},\n): Promise<void> {\n if (typeof context.route !== 'function') {\n options.warn?.('context.route() unavailable \u2014 in-page navigation SSRF guard NOT installed')\n return\n }\n const filterSubresources = options.filterSubresources ?? false\n const verdicts = new Map<string, boolean>()\n const remember = (host: string, blocked: boolean): void => {\n if (verdicts.size >= MAX_VERDICT_CACHE) {\n const oldest = verdicts.keys().next().value\n if (oldest !== undefined) verdicts.delete(oldest)\n }\n verdicts.set(host, blocked)\n }\n\n await context.route('**/*', async (route) => {\n const request = route.request()\n const isNavigation = request.isNavigationRequest()\n if (!isNavigation && !filterSubresources) return route.continue()\n\n const url = request.url()\n let host: string\n try {\n host = new URL(url).host\n } catch {\n return isNavigation ? route.abort('blockedbyclient') : route.continue()\n }\n\n const cached = isNavigation ? undefined : verdicts.get(host)\n if (cached === true) return route.abort('blockedbyclient')\n if (cached === false) return route.continue()\n\n try {\n await assertPublicUrl(url, isNavigation ? 'browser navigation' : 'browser subresource', {\n failClosed: true,\n })\n remember(host, false)\n } catch {\n remember(host, true)\n return route.abort('blockedbyclient')\n }\n return route.continue()\n })\n}\n\n/**\n * Launch flags aimed at not announcing automation. Playwright's defaults set\n * `navigator.webdriver` and a headless UA, which the SERP endpoints this adapter\n * exists to reach specifically look for.\n *\n * `--disable-dev-shm-usage` is a sizing workaround, not a security control: it\n * makes Chromium write shared memory to /tmp instead of a container's small\n * /dev/shm. The sandbox is deliberately NOT disabled here \u2014 see\n * {@link resolveLaunchArgs}.\n */\nexport const LAUNCH_ARGS = [\n '--disable-blink-features=AutomationControlled',\n '--disable-dev-shm-usage',\n]\n\nexport const SANDBOX_OPT_OUT_ENV = 'OM_WEB_RESEARCH_BROWSER_NO_SANDBOX'\n\n/**\n * Launch flags for this process, sandbox included unless an operator opted out.\n *\n * This adapter exists to render pages chosen by a search engine or a model, so\n * the renderer is by definition parsing hostile input. `--no-sandbox` turns a\n * renderer compromise into code running as whatever user the app runs as, which\n * is the one mitigation Chromium provides for exactly this threat.\n *\n * It used to be on unconditionally because the sandbox needs privileges a\n * root-in-Docker container does not grant by default. That is a container\n * problem with container fixes (run as a non-root user, or grant `SYS_ADMIN`, or\n * enable user namespaces); silently shipping every deployment the insecure\n * variant to spare those is the wrong trade. The opt-out stays available for\n * environments that genuinely cannot, and it is loud.\n */\nexport function resolveLaunchArgs(env: NodeJS.ProcessEnv = process.env): string[] {\n return env[SANDBOX_OPT_OUT_ENV] === '1' ? [...LAUNCH_ARGS, '--no-sandbox'] : [...LAUNCH_ARGS]\n}\n\nexport function isSandboxDisabled(env: NodeJS.ProcessEnv = process.env): boolean {\n return env[SANDBOX_OPT_OUT_ENV] === '1'\n}\n\n/**\n * Chromium's refusal when it cannot set up its sandbox. The wording differs\n * between the root check and the namespace failure, so both shapes are matched.\n */\nexport function isSandboxFailure(error: unknown): boolean {\n if (!(error instanceof Error)) return false\n return (\n /running as root without --no-sandbox/i.test(error.message) ||\n /No usable sandbox/i.test(error.message) ||\n /clone helper|namespace sandbox|SUID sandbox/i.test(error.message)\n )\n}\n\nexport const DEFAULT_USER_AGENT =\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,aAAa;AACtB,SAAS,uBAAuB;AAChC,SAAS,oBAAoB;AAuC7B,SAAS,iBAAiB,SAA0B;AAClD,SACE,gDAAgD,KAAK,OAAO,KAC5D,wBAAwB,KAAK,OAAO,KACpC,qCAAqC,KAAK,OAAO;AAErD;AAEA,eAAsB,iBAAuC;AAC3D,MAAI;AACF,UAAM,aAAc,MAAM,OAAO,YAAY;AAC7C,WAAO,WAAW;AAAA,EACpB,SAAS,OAAO;AACd,UAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAGrE,UAAM,IAAI;AAAA,MACR,4HAAuH,OAAO;AAAA,MAC9H,iBAAiB,OAAO,IAAI,kBAAkB;AAAA,IAChD;AAAA,EACF;AACF;AAOA,SAAS,sBAAsB,OAAyB;AACtD,SAAO,iBAAiB,SAAS,gCAAgC,KAAK,MAAM,OAAO;AACrF;AAGA,MAAM,qBAAqB,KAAK;AAIhC,MAAM,uBAAsC,OAAO,eACjD,IAAI,QAAQ,CAAC,SAAS,WAAW;AAC/B,QAAM,QAAQ,MAAM,OAAO,CAAC,cAAc,WAAW,UAAU,GAAG,EAAE,OAAO,CAAC,UAAU,QAAQ,MAAM,EAAE,CAAC;AACvG,MAAI,OAAO;AACX,QAAM,UAAU,CAAC,UAAwB;AACvC,WAAO,GAAG,IAAI,GAAG,MAAM,SAAS,MAAM,CAAC,GAAG,MAAM,IAAK;AAAA,EACvD;AACA,QAAM,QAAQ,GAAG,QAAQ,OAAO;AAChC,QAAM,QAAQ,GAAG,QAAQ,OAAO;AAEhC,QAAM,QAAQ,WAAW,MAAM;AAC7B,UAAM,KAAK,SAAS;AACpB,WAAO,IAAI,MAAM,gDAAgD,kBAAkB,IAAI,CAAC;AAAA,EAC1F,GAAG,kBAAkB;AAErB,QAAM,KAAK,SAAS,CAAC,UAAU;AAC7B,iBAAa,KAAK;AAClB,WAAO,KAAK;AAAA,EACd,CAAC;AACD,QAAM,KAAK,SAAS,CAAC,SAAS;AAC5B,iBAAa,KAAK;AAClB,QAAI,SAAS,GAAG;AACd,iBAAW,2BAA2B;AACtC,cAAQ;AACR;AAAA,IACF;AACA,WAAO,IAAI,MAAM,oDAAoD,IAAI,MAAM,KAAK,KAAK,CAAC,EAAE,CAAC;AAAA,EAC/F,CAAC;AACH,CAAC;AASH,eAAsB,eACpB,aACA,SACA,OAAyE,CAAC,GAClD;AACxB,MAAI;AACF,WAAO,MAAM,YAAY,OAAO,OAAO;AAAA,EACzC,SAAS,OAAO;AAId,QAAI,iBAAiB,KAAK,GAAG;AAC3B,YAAM,IAAI;AAAA,QACR,gPAEwC,mBAAmB,mBAAmB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAAA,QACpI;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,sBAAsB,KAAK,EAAG,OAAM;AACzC,UAAM,aAAa,KAAK,eAAe,MAAM;AAAA,IAAC;AAC9C,eAAW,uFAAuF;AAClG,QAAI;AACF,aAAO,KAAK,WAAW,sBAAsB,UAAU;AAAA,IACzD,SAAS,cAAc;AACrB,YAAM,SAAS,wBAAwB,QAAQ,aAAa,UAAU,OAAO,YAAY;AACzF,YAAM,IAAI;AAAA,QACR,2CAA2C,MAAM;AAAA,QACjD;AAAA,MACF;AAAA,IACF;AACA,WAAO,MAAM,YAAY,OAAO,OAAO;AAAA,EACzC;AACF;AAEA,MAAM,oBAAoB;AAY1B,eAAsB,uBACpB,SACA,UAA8E,CAAC,GAChE;AACf,MAAI,OAAO,QAAQ,UAAU,YAAY;AACvC,YAAQ,OAAO,gFAA2E;AAC1F;AAAA,EACF;AACA,QAAM,qBAAqB,QAAQ,sBAAsB;AACzD,QAAM,WAAW,oBAAI,IAAqB;AAC1C,QAAM,WAAW,CAAC,MAAc,YAA2B;AACzD,QAAI,SAAS,QAAQ,mBAAmB;AACtC,YAAM,SAAS,SAAS,KAAK,EAAE,KAAK,EAAE;AACtC,UAAI,WAAW,OAAW,UAAS,OAAO,MAAM;AAAA,IAClD;AACA,aAAS,IAAI,MAAM,OAAO;AAAA,EAC5B;AAEA,QAAM,QAAQ,MAAM,QAAQ,OAAO,UAAU;AAC3C,UAAM,UAAU,MAAM,QAAQ;AAC9B,UAAM,eAAe,QAAQ,oBAAoB;AACjD,QAAI,CAAC,gBAAgB,CAAC,mBAAoB,QAAO,MAAM,SAAS;AAEhE,UAAM,MAAM,QAAQ,IAAI;AACxB,QAAI;AACJ,QAAI;AACF,aAAO,IAAI,IAAI,GAAG,EAAE;AAAA,IACtB,QAAQ;AACN,aAAO,eAAe,MAAM,MAAM,iBAAiB,IAAI,MAAM,SAAS;AAAA,IACxE;AAEA,UAAM,SAAS,eAAe,SAAY,SAAS,IAAI,IAAI;AAC3D,QAAI,WAAW,KAAM,QAAO,MAAM,MAAM,iBAAiB;AACzD,QAAI,WAAW,MAAO,QAAO,MAAM,SAAS;AAE5C,QAAI;AACF,YAAM,gBAAgB,KAAK,eAAe,uBAAuB,uBAAuB;AAAA,QACtF,YAAY;AAAA,MACd,CAAC;AACD,eAAS,MAAM,KAAK;AAAA,IACtB,QAAQ;AACN,eAAS,MAAM,IAAI;AACnB,aAAO,MAAM,MAAM,iBAAiB;AAAA,IACtC;AACA,WAAO,MAAM,SAAS;AAAA,EACxB,CAAC;AACH;AAYO,MAAM,cAAc;AAAA,EACzB;AAAA,EACA;AACF;AAEO,MAAM,sBAAsB;AAiB5B,SAAS,kBAAkB,MAAyB,QAAQ,KAAe;AAChF,SAAO,IAAI,mBAAmB,MAAM,MAAM,CAAC,GAAG,aAAa,cAAc,IAAI,CAAC,GAAG,WAAW;AAC9F;AAEO,SAAS,kBAAkB,MAAyB,QAAQ,KAAc;AAC/E,SAAO,IAAI,mBAAmB,MAAM;AACtC;AAMO,SAAS,iBAAiB,OAAyB;AACxD,MAAI,EAAE,iBAAiB,OAAQ,QAAO;AACtC,SACE,wCAAwC,KAAK,MAAM,OAAO,KAC1D,qBAAqB,KAAK,MAAM,OAAO,KACvC,+CAA+C,KAAK,MAAM,OAAO;AAErE;AAEO,MAAM,qBACX;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/web-research-browser",
|
|
3
|
-
"version": "0.9.1-develop.
|
|
3
|
+
"version": "0.9.1-develop.7575.1.8480826a9b",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -38,12 +38,12 @@
|
|
|
38
38
|
"@open-mercato/web-research": "^0.6.0"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@open-mercato/web-research-serp": "0.9.1-develop.
|
|
41
|
+
"@open-mercato/web-research-serp": "0.9.1-develop.7575.1.8480826a9b",
|
|
42
42
|
"playwright": "^1.40.0",
|
|
43
43
|
"zod": "^4.4.3"
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
|
-
"@open-mercato/web-research": "0.9.1-develop.
|
|
46
|
+
"@open-mercato/web-research": "0.9.1-develop.7575.1.8480826a9b",
|
|
47
47
|
"@types/jest": "^30.0.0",
|
|
48
48
|
"@types/node": "^26.0.1",
|
|
49
49
|
"esbuild": "^0.28.1",
|
package/src/dispatch.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { assertPublicUrl
|
|
2
|
-
import { startEgressProxy, type EgressConnect, type EgressProxy } from './egressProxy'
|
|
1
|
+
import { assertPublicUrl } from '@open-mercato/web-research'
|
|
3
2
|
import {
|
|
4
3
|
DEFAULT_USER_AGENT,
|
|
5
|
-
RESOLVER_LOCKDOWN_ARGS,
|
|
6
4
|
isSandboxDisabled,
|
|
7
5
|
resolveLaunchArgs,
|
|
8
6
|
importChromium,
|
|
@@ -10,7 +8,6 @@ import {
|
|
|
10
8
|
launchChromium,
|
|
11
9
|
type InstallRunner,
|
|
12
10
|
type BrowserHandle,
|
|
13
|
-
type BrowserType,
|
|
14
11
|
type ContextHandle,
|
|
15
12
|
type PageHandle,
|
|
16
13
|
} from './playwright'
|
|
@@ -30,25 +27,14 @@ export const LEASE_IDLE_TIMEOUT_MS = 5 * 60_000
|
|
|
30
27
|
export type SidecarState = {
|
|
31
28
|
browser: BrowserHandle | null
|
|
32
29
|
leases: Map<string, Lease>
|
|
33
|
-
/** @deprecated Subresource filtering is always on; this field is ignored. */
|
|
34
30
|
filterSubresources: boolean
|
|
35
31
|
warn: (message: string) => void
|
|
36
32
|
/** Injectable so a lease-expiry test does not have to wait five minutes. */
|
|
37
33
|
now: () => number
|
|
38
34
|
/** Injectable so tests never shell out to `npx playwright install`. */
|
|
39
35
|
installRunner?: InstallRunner
|
|
40
|
-
/** Injectable DNS for the URL checks and the egress proxy, so tests never resolve real names. */
|
|
41
|
-
lookup?: LookupFn
|
|
42
|
-
/** Injectable dial for the egress proxy, so tests can stand in for public hosts. */
|
|
43
|
-
connect?: EgressConnect
|
|
44
|
-
/** Injectable so tests never import Playwright. */
|
|
45
|
-
loadBrowserType?: () => Promise<BrowserType>
|
|
46
|
-
egressProxy?: EgressProxy | null
|
|
47
|
-
launching?: Promise<BrowserHandle> | null
|
|
48
36
|
}
|
|
49
37
|
|
|
50
|
-
const FILTER_SUBRESOURCES_ENV = 'OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES'
|
|
51
|
-
|
|
52
38
|
export const DEFAULT_GOTO_TIMEOUT_MS = 30_000
|
|
53
39
|
const MAX_GOTO_TIMEOUT_MS = 120_000
|
|
54
40
|
|
|
@@ -56,19 +42,16 @@ export function createState(overrides: Partial<SidecarState> = {}): SidecarState
|
|
|
56
42
|
return {
|
|
57
43
|
browser: null,
|
|
58
44
|
leases: new Map(),
|
|
59
|
-
filterSubresources:
|
|
45
|
+
filterSubresources: process.env.OM_WEB_RESEARCH_BROWSER_FILTER_SUBRESOURCES === '1',
|
|
60
46
|
warn: (message) => process.stderr.write(`om-web-research-sidecar: ${message}\n`),
|
|
61
47
|
now: () => Date.now(),
|
|
62
48
|
...overrides,
|
|
63
49
|
}
|
|
64
50
|
}
|
|
65
51
|
|
|
66
|
-
function
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
async function launchBrowser(state: SidecarState): Promise<BrowserHandle> {
|
|
71
|
-
const browserType = await (state.loadBrowserType ?? importChromium)()
|
|
52
|
+
async function ensureBrowser(state: SidecarState): Promise<BrowserHandle> {
|
|
53
|
+
if (state.browser) return state.browser
|
|
54
|
+
const chromium = await importChromium()
|
|
72
55
|
if (isSandboxDisabled()) {
|
|
73
56
|
// Said once per browser, not once per render, but said: this process renders
|
|
74
57
|
// pages chosen by a search engine or a model with the one mitigation
|
|
@@ -77,45 +60,12 @@ async function launchBrowser(state: SidecarState): Promise<BrowserHandle> {
|
|
|
77
60
|
'SECURITY: Chromium sandbox is disabled by OM_WEB_RESEARCH_BROWSER_NO_SANDBOX=1 — a renderer compromise reaches this host',
|
|
78
61
|
)
|
|
79
62
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
})
|
|
87
|
-
let browser: BrowserHandle
|
|
88
|
-
try {
|
|
89
|
-
browser = await launchChromium(
|
|
90
|
-
browserType,
|
|
91
|
-
{
|
|
92
|
-
headless: true,
|
|
93
|
-
args: [...resolveLaunchArgs(), ...RESOLVER_LOCKDOWN_ARGS],
|
|
94
|
-
proxy: { server: proxy.url, bypass: '<-loopback>' },
|
|
95
|
-
},
|
|
96
|
-
{ onProgress: state.warn, ...(state.installRunner ? { install: state.installRunner } : {}) },
|
|
97
|
-
)
|
|
98
|
-
} catch (error) {
|
|
99
|
-
await proxy.close()
|
|
100
|
-
throw error
|
|
101
|
-
}
|
|
102
|
-
state.egressProxy = proxy
|
|
103
|
-
state.browser = browser
|
|
104
|
-
return browser
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/**
|
|
108
|
-
* Single-flight: two leases opening at once share one launch, so neither a
|
|
109
|
-
* browser nor its egress proxy is orphaned outside `teardown`.
|
|
110
|
-
*/
|
|
111
|
-
async function ensureBrowser(state: SidecarState): Promise<BrowserHandle> {
|
|
112
|
-
if (state.browser) return state.browser
|
|
113
|
-
if (!state.launching) {
|
|
114
|
-
state.launching = launchBrowser(state).finally(() => {
|
|
115
|
-
state.launching = null
|
|
116
|
-
})
|
|
117
|
-
}
|
|
118
|
-
return state.launching
|
|
63
|
+
state.browser = await launchChromium(
|
|
64
|
+
chromium,
|
|
65
|
+
{ headless: true, args: resolveLaunchArgs() },
|
|
66
|
+
{ onProgress: state.warn, ...(state.installRunner ? { install: state.installRunner } : {}) },
|
|
67
|
+
)
|
|
68
|
+
return state.browser
|
|
119
69
|
}
|
|
120
70
|
|
|
121
71
|
/**
|
|
@@ -131,8 +81,8 @@ async function openLease(state: SidecarState, leaseId: string, userAgent: string
|
|
|
131
81
|
return existing
|
|
132
82
|
}
|
|
133
83
|
const browser = await ensureBrowser(state)
|
|
134
|
-
const context = await browser.newContext({ userAgent, locale: 'en-US'
|
|
135
|
-
await installNavigationGuard(context, {
|
|
84
|
+
const context = await browser.newContext({ userAgent, locale: 'en-US' })
|
|
85
|
+
await installNavigationGuard(context, { filterSubresources: state.filterSubresources, warn: state.warn })
|
|
136
86
|
const page = (await context.newPage()) as unknown as PageHandle
|
|
137
87
|
const lease: Lease = { context, page, touchedAt: state.now() }
|
|
138
88
|
state.leases.set(leaseId, lease)
|
|
@@ -161,12 +111,9 @@ async function closeLease(state: SidecarState, leaseId: string): Promise<void> {
|
|
|
161
111
|
}
|
|
162
112
|
|
|
163
113
|
export async function teardown(state: SidecarState): Promise<void> {
|
|
164
|
-
if (state.launching) await state.launching.catch(() => null)
|
|
165
114
|
for (const leaseId of [...state.leases.keys()]) await closeLease(state, leaseId)
|
|
166
115
|
const browser = state.browser
|
|
167
|
-
const proxy = state.egressProxy
|
|
168
116
|
state.browser = null
|
|
169
|
-
state.egressProxy = null
|
|
170
117
|
if (browser) {
|
|
171
118
|
try {
|
|
172
119
|
await browser.close()
|
|
@@ -174,7 +121,6 @@ export async function teardown(state: SidecarState): Promise<void> {
|
|
|
174
121
|
// Best effort; the parent escalates to SIGKILL if we hang.
|
|
175
122
|
}
|
|
176
123
|
}
|
|
177
|
-
if (proxy) await proxy.close()
|
|
178
124
|
}
|
|
179
125
|
|
|
180
126
|
function stringParam(request: SidecarRequest, key: string): string {
|
|
@@ -196,7 +142,7 @@ async function dispatchInner(state: SidecarState, request: SidecarRequest): Prom
|
|
|
196
142
|
// Re-checked here, in a distinct process from the caller, and BEFORE
|
|
197
143
|
// Playwright is imported — a blocked URL never launches a browser.
|
|
198
144
|
try {
|
|
199
|
-
await assertPublicUrl(url, 'browser render', { failClosed: true
|
|
145
|
+
await assertPublicUrl(url, 'browser render', { failClosed: true })
|
|
200
146
|
} catch (error) {
|
|
201
147
|
return {
|
|
202
148
|
id: request.id,
|
|
@@ -222,18 +168,8 @@ async function dispatchInner(state: SidecarState, request: SidecarRequest): Prom
|
|
|
222
168
|
error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },
|
|
223
169
|
}
|
|
224
170
|
}
|
|
225
|
-
const finalUrl = lease.page.url()
|
|
226
|
-
try {
|
|
227
|
-
await assertPublicUrl(finalUrl, 'browser final URL', { failClosed: true, ...lookupOption(state) })
|
|
228
|
-
} catch (error) {
|
|
229
|
-
return {
|
|
230
|
-
id: request.id,
|
|
231
|
-
ok: false,
|
|
232
|
-
error: { message: error instanceof Error ? error.message : String(error), kind: 'navigation' },
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
171
|
const html = await lease.page.content()
|
|
236
|
-
return { id: request.id, ok: true, result: { url:
|
|
172
|
+
return { id: request.id, ok: true, result: { url: lease.page.url(), html } }
|
|
237
173
|
}
|
|
238
174
|
|
|
239
175
|
case 'release':
|