@hraness/kb 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +43 -0
- package/LICENSE +21 -0
- package/README.md +545 -0
- package/dist/agent-context.js +35 -0
- package/dist/agent-guide-audit.js +18 -0
- package/dist/attachments.js +19 -0
- package/dist/authoring.js +33 -0
- package/dist/benchmark.js +19 -0
- package/dist/capture.js +49 -0
- package/dist/cli.js +3646 -0
- package/dist/clip/acquire.js +57 -0
- package/dist/clip/args.js +21 -0
- package/dist/clip/bounded-byte-buffer.js +9 -0
- package/dist/clip/browser-profiles.js +17 -0
- package/dist/clip/bundle-reader.js +11 -0
- package/dist/clip/cli.js +27 -0
- package/dist/clip/cookies.js +21 -0
- package/dist/clip/defuddle-worker.js +84 -0
- package/dist/clip/doctor.js +32 -0
- package/dist/clip/jobs.js +784 -0
- package/dist/clip/network-proxy.js +9 -0
- package/dist/clip/network.js +28 -0
- package/dist/clip/persist.js +30 -0
- package/dist/clip/refresh.js +11 -0
- package/dist/clip/terminal.js +11 -0
- package/dist/evaluation-builder.js +12238 -0
- package/dist/evaluation-kb.js +26 -0
- package/dist/evaluation.js +35 -0
- package/dist/git.js +39 -0
- package/dist/graph.js +39 -0
- package/dist/index-01jj6rbv.js +1103 -0
- package/dist/index-06c9ctr6.js +738 -0
- package/dist/index-0kavxzqj.js +2480 -0
- package/dist/index-1gwbassd.js +743 -0
- package/dist/index-1n418kb9.js +827 -0
- package/dist/index-1vrd1rmn.js +56 -0
- package/dist/index-1xxnjn0d.js +171 -0
- package/dist/index-3rm7cz6h.js +373 -0
- package/dist/index-3v2z4f0q.js +328 -0
- package/dist/index-48pz4jpc.js +392 -0
- package/dist/index-4j3tt0c3.js +234 -0
- package/dist/index-5n05se68.js +307 -0
- package/dist/index-5vwpzb5a.js +764 -0
- package/dist/index-6g2pv9d2.js +370 -0
- package/dist/index-7fzc32gf.js +712 -0
- package/dist/index-84x0vjjp.js +432 -0
- package/dist/index-adx6khj5.js +488 -0
- package/dist/index-b88v3vtm.js +805 -0
- package/dist/index-cv6fh7z5.js +462 -0
- package/dist/index-cxfrakt7.js +1067 -0
- package/dist/index-d13v9ckt.js +150 -0
- package/dist/index-dycjbhw2.js +1943 -0
- package/dist/index-dyqwejk5.js +531 -0
- package/dist/index-e5fbsywq.js +658 -0
- package/dist/index-f984hw45.js +927 -0
- package/dist/index-gh719d91.js +63 -0
- package/dist/index-hgve9rh2.js +2046 -0
- package/dist/index-hya40gb2.js +474 -0
- package/dist/index-j4zgmzjr.js +152 -0
- package/dist/index-jsmvyyvf.js +1166 -0
- package/dist/index-kh0pwvdd.js +1974 -0
- package/dist/index-mqx4nd6v.js +119 -0
- package/dist/index-mxxxytys.js +1254 -0
- package/dist/index-n5dd7r0v.js +711 -0
- package/dist/index-npg9z1a4.js +431 -0
- package/dist/index-pj501bh1.js +140 -0
- package/dist/index-qry4vhxk.js +1041 -0
- package/dist/index-s2gw5aw9.js +199 -0
- package/dist/index-vvdj6kca.js +70 -0
- package/dist/index-w2zc0vwa.js +571 -0
- package/dist/index-x3fthpsc.js +585 -0
- package/dist/index-zr53sf63.js +44 -0
- package/dist/index-zxdy5pby.js +2728 -0
- package/dist/index-zzhgcwyt.js +966 -0
- package/dist/index.js +453 -0
- package/dist/navigation.js +13 -0
- package/dist/pdf.js +85 -0
- package/dist/percolate.js +24 -0
- package/dist/portfolio.js +77 -0
- package/dist/query.js +29 -0
- package/dist/repository-memory.js +60 -0
- package/dist/sdk.js +36 -0
- package/dist/search-rules.js +34 -0
- package/dist/search.js +22 -0
- package/dist/semantic.js +39 -0
- package/dist/source-inbox.js +17 -0
- package/dist/untrusted-content.js +32 -0
- package/dist/url-intelligence.js +74 -0
- package/dist/workflow.js +23 -0
- package/dist/workflows/decision-context.js +20 -0
- package/dist/workflows/explain-change.js +8 -0
- package/dist/workflows/index.js +28 -0
- package/dist/workflows/plan-radar.js +10 -0
- package/package.json +409 -0
- package/skills/kb/AGENTS.md +19 -0
- package/skills/kb/SKILL.md +82 -0
- package/skills/kb/agents/openai.yaml +4 -0
- package/skills/kb/references/pdf-review.md +51 -0
- package/skills/kb/references/percolate.md +144 -0
- package/skills/kb/references/plan-structure.md +105 -0
- package/skills/kb/references/plan.md +115 -0
- package/skills/kb/references/query.md +231 -0
- package/skills/kb/references/refresh.md +133 -0
- package/skills/kb/references/save-pdf.md +125 -0
- package/skills/kb/references/save-url.md +178 -0
- package/skills/kb/references/url-authentication.md +75 -0
- package/skills/kb/references/url-platforms.md +25 -0
- package/src/agent-context.ts +1241 -0
- package/src/agent-guide-audit.ts +750 -0
- package/src/attachments.ts +723 -0
- package/src/authoring.ts +1725 -0
- package/src/benchmark.ts +362 -0
- package/src/capture.ts +68 -0
- package/src/cli.ts +3660 -0
- package/src/clip/acquire.ts +1273 -0
- package/src/clip/archive-today.ts +402 -0
- package/src/clip/args.ts +423 -0
- package/src/clip/assets.ts +262 -0
- package/src/clip/bounded-byte-buffer.ts +75 -0
- package/src/clip/browser-profiles.ts +372 -0
- package/src/clip/bundle-reader.ts +608 -0
- package/src/clip/capture.ts +1054 -0
- package/src/clip/cli.ts +181 -0
- package/src/clip/cookies.ts +512 -0
- package/src/clip/doctor.ts +1110 -0
- package/src/clip/extract.ts +868 -0
- package/src/clip/jobs.ts +959 -0
- package/src/clip/lib.ts +861 -0
- package/src/clip/media.ts +1474 -0
- package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
- package/src/clip/metadata-search-tool/Cargo.toml +18 -0
- package/src/clip/metadata-search-tool/runner.ts +685 -0
- package/src/clip/metadata-search-tool/src/main.rs +389 -0
- package/src/clip/metadata-search.ts +687 -0
- package/src/clip/network-proxy.ts +627 -0
- package/src/clip/network.ts +950 -0
- package/src/clip/package-root.ts +54 -0
- package/src/clip/persist.ts +1472 -0
- package/src/clip/platforms.ts +1824 -0
- package/src/clip/refresh.ts +197 -0
- package/src/clip/structured.ts +432 -0
- package/src/clip/terminal.ts +197 -0
- package/src/clip/url-intelligence.ts +581 -0
- package/src/clip/url-metadata-backfill.ts +616 -0
- package/src/clip/url-metadata-cli.ts +251 -0
- package/src/clip/url-metadata.ts +1397 -0
- package/src/evaluation-analysis-v2.ts +1440 -0
- package/src/evaluation-builder.ts +2539 -0
- package/src/evaluation-corpus-authoring.ts +2052 -0
- package/src/evaluation-evidence.ts +1147 -0
- package/src/evaluation-execution-v2.ts +426 -0
- package/src/evaluation-implementation.ts +131 -0
- package/src/evaluation-kb-closure.ts +1430 -0
- package/src/evaluation-kb-runner-v2.ts +735 -0
- package/src/evaluation-kb-v2.ts +2567 -0
- package/src/evaluation-kb.ts +1018 -0
- package/src/evaluation-measurement-v2.ts +1462 -0
- package/src/evaluation-packing-v2.ts +452 -0
- package/src/evaluation-redaction.ts +28 -0
- package/src/evaluation-v2.ts +3857 -0
- package/src/evaluation.ts +1220 -0
- package/src/git.ts +1276 -0
- package/src/graph.ts +1487 -0
- package/src/index.ts +20 -0
- package/src/init.ts +123 -0
- package/src/navigation.ts +277 -0
- package/src/note-lock.ts +529 -0
- package/src/pdf/args.ts +202 -0
- package/src/pdf/capture.ts +298 -0
- package/src/pdf/cli.ts +159 -0
- package/src/pdf/extract.ts +303 -0
- package/src/pdf/layout.ts +618 -0
- package/src/pdf/markdown.ts +154 -0
- package/src/pdf/model.ts +234 -0
- package/src/pdf/ocr.ts +190 -0
- package/src/pdf/persist.ts +314 -0
- package/src/pdf/source.ts +135 -0
- package/src/pdf/tools.ts +146 -0
- package/src/pdf.ts +92 -0
- package/src/percolate.ts +877 -0
- package/src/portfolio-audit.ts +574 -0
- package/src/portfolio-identity.ts +162 -0
- package/src/portfolio-registry.ts +473 -0
- package/src/portfolio.ts +765 -0
- package/src/query.ts +646 -0
- package/src/repository-memory.ts +1150 -0
- package/src/sdk.ts +1452 -0
- package/src/search-rules.ts +712 -0
- package/src/search.ts +710 -0
- package/src/semantic-runtime.ts +1139 -0
- package/src/semantic.ts +2480 -0
- package/src/source-inbox.ts +232 -0
- package/src/untrusted-content.ts +401 -0
- package/src/url-intelligence.ts +104 -0
- package/src/vault.ts +636 -0
- package/src/workflow.ts +653 -0
- package/src/workflows/decision-context.ts +81 -0
- package/src/workflows/explain-change.ts +73 -0
- package/src/workflows/index.ts +3 -0
- package/src/workflows/plan-radar.ts +101 -0
|
@@ -0,0 +1,1273 @@
|
|
|
1
|
+
import {
|
|
2
|
+
chmodSync,
|
|
3
|
+
existsSync,
|
|
4
|
+
lstatSync,
|
|
5
|
+
mkdtempSync,
|
|
6
|
+
readFileSync,
|
|
7
|
+
realpathSync,
|
|
8
|
+
rmSync,
|
|
9
|
+
statSync,
|
|
10
|
+
writeFileSync,
|
|
11
|
+
} from "node:fs";
|
|
12
|
+
import { homedir, tmpdir } from "node:os";
|
|
13
|
+
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
14
|
+
|
|
15
|
+
import { getCookies, type BrowserName, type GetCookiesOptions } from "@steipete/sweet-cookie";
|
|
16
|
+
|
|
17
|
+
import { captureUrl, type CaptureArguments } from "./args.js";
|
|
18
|
+
import { readBoundedByteStream } from "./bounded-byte-buffer.js";
|
|
19
|
+
import {
|
|
20
|
+
filterCookieProviderResult,
|
|
21
|
+
readCookieFile,
|
|
22
|
+
renderCookieHeader,
|
|
23
|
+
type StrictCookie,
|
|
24
|
+
} from "./cookies.js";
|
|
25
|
+
import { assertSafeNetworkUrl, decodeBytes, safeFetch } from "./network.js";
|
|
26
|
+
import { startNetworkProxy, type LocalNetworkProxy } from "./network-proxy.js";
|
|
27
|
+
import { classifyPlatformUrl } from "./platforms.js";
|
|
28
|
+
import { findKbPackageRoot, resolvePackageDirectory } from "./package-root.js";
|
|
29
|
+
import { sanitizeArtifactUrl } from "./persist.js";
|
|
30
|
+
|
|
31
|
+
const agentBrowserBinDirectory = join(resolvePackageDirectory("agent-browser"), "bin");
|
|
32
|
+
|
|
33
|
+
export function agentBrowserCommand(): readonly string[] {
|
|
34
|
+
// The upstream wrapper selects glibc versus musl and restores the executable
|
|
35
|
+
// bit when package lifecycle scripts were intentionally skipped.
|
|
36
|
+
return [process.execPath, join(agentBrowserBinDirectory, "agent-browser.js")];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type AcquisitionMethod =
|
|
40
|
+
| "file"
|
|
41
|
+
| "http"
|
|
42
|
+
| "cookie-http"
|
|
43
|
+
| "hacker-news-api"
|
|
44
|
+
| "bluesky-api"
|
|
45
|
+
| "reddit-json"
|
|
46
|
+
| "archive-is"
|
|
47
|
+
| "browser-profile"
|
|
48
|
+
| "browser-live"
|
|
49
|
+
| "browser-cdp"
|
|
50
|
+
| "browser-fresh";
|
|
51
|
+
|
|
52
|
+
export type AcquiredPage = {
|
|
53
|
+
readonly body: string;
|
|
54
|
+
readonly contentType: string | null;
|
|
55
|
+
readonly finalUrl: URL;
|
|
56
|
+
readonly method: AcquisitionMethod;
|
|
57
|
+
readonly warnings: readonly string[];
|
|
58
|
+
readonly browserTitle?: string;
|
|
59
|
+
readonly screenshotPath?: string;
|
|
60
|
+
readonly sourceEvidence?: string;
|
|
61
|
+
readonly contentTruncated?: boolean;
|
|
62
|
+
readonly renderedText?: string;
|
|
63
|
+
readonly renderedTextTruncated?: boolean;
|
|
64
|
+
readonly renderedTextByteLimit?: number;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
export type ChromeProfile = {
|
|
68
|
+
readonly directory: string;
|
|
69
|
+
readonly name: string;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
type CommandResult = {
|
|
73
|
+
readonly stdout: string;
|
|
74
|
+
readonly stderr: string;
|
|
75
|
+
readonly exitCode: number;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
type CommandIsolation = {
|
|
79
|
+
readonly cwd: string;
|
|
80
|
+
readonly environment: Readonly<Record<string, string | undefined>>;
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const inheritedProxyKeys = new Set([
|
|
84
|
+
"ALL_PROXY",
|
|
85
|
+
"HTTP_PROXY",
|
|
86
|
+
"HTTPS_PROXY",
|
|
87
|
+
"NO_PROXY",
|
|
88
|
+
"all_proxy",
|
|
89
|
+
"http_proxy",
|
|
90
|
+
"https_proxy",
|
|
91
|
+
"no_proxy",
|
|
92
|
+
]);
|
|
93
|
+
|
|
94
|
+
/** Remove ambient agent-browser auth, provider, startup, and proxy choices before every invocation. */
|
|
95
|
+
export function isolatedAgentBrowserEnvironment(
|
|
96
|
+
source: Readonly<Record<string, string | undefined>>,
|
|
97
|
+
socketDirectory: string,
|
|
98
|
+
): Readonly<Record<string, string | undefined>> {
|
|
99
|
+
const environment: Record<string, string> = {};
|
|
100
|
+
for (const [key, value] of Object.entries(source)) {
|
|
101
|
+
if (value === undefined || key.startsWith("AGENT_BROWSER_") || inheritedProxyKeys.has(key)) continue;
|
|
102
|
+
environment[key] = value;
|
|
103
|
+
}
|
|
104
|
+
environment.AGENT_BROWSER_SOCKET_DIR = socketDirectory;
|
|
105
|
+
return environment;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function createAgentBrowserIsolation(directory: string): CommandIsolation & {
|
|
109
|
+
readonly configPath: string;
|
|
110
|
+
readonly socketDirectory: string;
|
|
111
|
+
} {
|
|
112
|
+
const configPath = join(directory, "agent-browser.config.json");
|
|
113
|
+
// macOS limits Unix-domain socket paths to roughly 104 bytes. Its normal
|
|
114
|
+
// per-user tmpdir is already long, so keep this randomized private path under /tmp.
|
|
115
|
+
const socketRoot = process.platform === "win32" ? tmpdir() : "/tmp";
|
|
116
|
+
const socketDirectory = mkdtempSync(join(socketRoot, "jc-ab-"));
|
|
117
|
+
try {
|
|
118
|
+
chmodSync(socketDirectory, 0o700);
|
|
119
|
+
writeFileSync(configPath, "{}\n", { encoding: "utf8", flag: "wx", mode: 0o600 });
|
|
120
|
+
chmodSync(configPath, 0o600);
|
|
121
|
+
return {
|
|
122
|
+
configPath,
|
|
123
|
+
cwd: directory,
|
|
124
|
+
socketDirectory,
|
|
125
|
+
environment: isolatedAgentBrowserEnvironment(process.env, socketDirectory),
|
|
126
|
+
};
|
|
127
|
+
} catch (error) {
|
|
128
|
+
rmSync(socketDirectory, { recursive: true, force: true });
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
134
|
+
typeof value === "object" && value !== null && !Array.isArray(value);
|
|
135
|
+
|
|
136
|
+
async function readBoundedStream(stream: ReadableStream<Uint8Array>, maxBytes: number): Promise<string> {
|
|
137
|
+
const bytes = await readBoundedByteStream(stream, maxBytes, "process output");
|
|
138
|
+
return new TextDecoder().decode(bytes);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async function runCommand(
|
|
142
|
+
command: readonly string[],
|
|
143
|
+
timeoutMs: number,
|
|
144
|
+
maxOutputBytes: number,
|
|
145
|
+
isolation?: CommandIsolation,
|
|
146
|
+
stdin?: string,
|
|
147
|
+
): Promise<CommandResult> {
|
|
148
|
+
const child = Bun.spawn([...command], {
|
|
149
|
+
stdin: stdin === undefined ? "ignore" : new Blob([stdin]),
|
|
150
|
+
stdout: "pipe",
|
|
151
|
+
stderr: "pipe",
|
|
152
|
+
...(isolation === undefined ? {} : { cwd: isolation.cwd, env: isolation.environment }),
|
|
153
|
+
});
|
|
154
|
+
let timedOut = false;
|
|
155
|
+
let forceKill: ReturnType<typeof setTimeout> | null = null;
|
|
156
|
+
const timeout = setTimeout(() => {
|
|
157
|
+
timedOut = true;
|
|
158
|
+
child.kill("SIGTERM");
|
|
159
|
+
forceKill = setTimeout(() => child.kill("SIGKILL"), 1_000);
|
|
160
|
+
}, timeoutMs);
|
|
161
|
+
try {
|
|
162
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
163
|
+
readBoundedStream(child.stdout, maxOutputBytes),
|
|
164
|
+
readBoundedStream(child.stderr, Math.min(maxOutputBytes, 2 * 1024 * 1024)),
|
|
165
|
+
child.exited,
|
|
166
|
+
]);
|
|
167
|
+
if (timedOut) throw new Error(`command timed out after ${timeoutMs}ms`);
|
|
168
|
+
return { stdout, stderr, exitCode };
|
|
169
|
+
} catch (error) {
|
|
170
|
+
child.kill("SIGKILL");
|
|
171
|
+
await child.exited;
|
|
172
|
+
throw error;
|
|
173
|
+
} finally {
|
|
174
|
+
clearTimeout(timeout);
|
|
175
|
+
if (forceKill !== null) clearTimeout(forceKill);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function parseJsonValueOutput(output: string, label: string): unknown {
|
|
180
|
+
let lineEnd = output.length;
|
|
181
|
+
while (lineEnd >= 0) {
|
|
182
|
+
const newline = output.lastIndexOf("\n", lineEnd - 1);
|
|
183
|
+
const line = output.slice(newline + 1, lineEnd).trim();
|
|
184
|
+
lineEnd = newline;
|
|
185
|
+
if (line[0] !== "{" && line[0] !== "[") continue;
|
|
186
|
+
try {
|
|
187
|
+
return JSON.parse(line) as unknown;
|
|
188
|
+
} catch {
|
|
189
|
+
// Keep looking for the final JSON object after non-JSON diagnostics.
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
throw new Error(`${label} did not return JSON`);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function parseJsonOutput(output: string, label: string): Record<string, unknown> {
|
|
196
|
+
const parsed = parseJsonValueOutput(output, label);
|
|
197
|
+
if (isRecord(parsed)) return parsed;
|
|
198
|
+
throw new Error(`${label} did not return a JSON object`);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function parseAgentBrowserData(output: string, label: string): Record<string, unknown> {
|
|
202
|
+
const parsed = parseJsonOutput(output, label);
|
|
203
|
+
if (parsed.success !== true) {
|
|
204
|
+
throw new Error(`${label} failed`);
|
|
205
|
+
}
|
|
206
|
+
if (!isRecord(parsed.data)) throw new Error(`${label} returned no data`);
|
|
207
|
+
return parsed.data;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async function runAgentBrowser(
|
|
211
|
+
globalArgs: readonly string[],
|
|
212
|
+
command: readonly string[],
|
|
213
|
+
options: CommandIsolation & { readonly timeoutMs: number; readonly maxOutputBytes: number },
|
|
214
|
+
): Promise<Record<string, unknown>> {
|
|
215
|
+
let result: CommandResult;
|
|
216
|
+
try {
|
|
217
|
+
result = await runCommand(
|
|
218
|
+
[...agentBrowserCommand(), ...globalArgs, ...command, "--json"],
|
|
219
|
+
options.timeoutMs,
|
|
220
|
+
options.maxOutputBytes,
|
|
221
|
+
options,
|
|
222
|
+
);
|
|
223
|
+
} catch (error) {
|
|
224
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
225
|
+
throw new Error(`agent-browser ${command[0] ?? "command"} failed: ${message}`, { cause: error });
|
|
226
|
+
}
|
|
227
|
+
if (result.exitCode !== 0) {
|
|
228
|
+
throw new Error(`agent-browser ${command[0] ?? "command"} failed with exit code ${result.exitCode}`);
|
|
229
|
+
}
|
|
230
|
+
return parseAgentBrowserData(result.stdout, `agent-browser ${command[0] ?? "command"}`);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
async function runAgentBrowserBatch(
|
|
234
|
+
globalArgs: readonly string[],
|
|
235
|
+
commands: readonly (readonly string[])[],
|
|
236
|
+
options: CommandIsolation & { readonly timeoutMs: number; readonly maxOutputBytes: number },
|
|
237
|
+
): Promise<void> {
|
|
238
|
+
const result = await runCommand(
|
|
239
|
+
[...agentBrowserCommand(), ...globalArgs, "batch", "--bail", "--json"],
|
|
240
|
+
options.timeoutMs,
|
|
241
|
+
options.maxOutputBytes,
|
|
242
|
+
options,
|
|
243
|
+
JSON.stringify(commands),
|
|
244
|
+
);
|
|
245
|
+
if (result.exitCode !== 0) throw new Error(`agent-browser batch failed with exit code ${result.exitCode}`);
|
|
246
|
+
const parsed = parseJsonValueOutput(result.stdout, "agent-browser batch");
|
|
247
|
+
if (
|
|
248
|
+
!Array.isArray(parsed)
|
|
249
|
+
|| parsed.length !== commands.length
|
|
250
|
+
|| parsed.some((entry) => !isRecord(entry) || entry.success !== true)
|
|
251
|
+
) throw new Error("agent-browser batch failed");
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Discover Chrome profiles without opening a browser or reading cookie values. */
|
|
255
|
+
export async function discoverChromeProfiles(timeoutMs = 15_000): Promise<readonly ChromeProfile[]> {
|
|
256
|
+
const directory = mkdtempSync(join(tmpdir(), "hraness-kb-profiles-"));
|
|
257
|
+
chmodSync(directory, 0o700);
|
|
258
|
+
let socketDirectory: string | null = null;
|
|
259
|
+
try {
|
|
260
|
+
const isolation = createAgentBrowserIsolation(directory);
|
|
261
|
+
socketDirectory = isolation.socketDirectory;
|
|
262
|
+
const result = await runCommand(
|
|
263
|
+
[...agentBrowserCommand(), "--config", isolation.configPath, "profiles", "--json"],
|
|
264
|
+
timeoutMs,
|
|
265
|
+
1024 * 1024,
|
|
266
|
+
isolation,
|
|
267
|
+
);
|
|
268
|
+
if (result.exitCode !== 0) return [];
|
|
269
|
+
const parsed = parseJsonOutput(result.stdout, "agent-browser profiles");
|
|
270
|
+
if (parsed.success !== true || !Array.isArray(parsed.data)) return [];
|
|
271
|
+
const profiles: ChromeProfile[] = [];
|
|
272
|
+
for (const entry of parsed.data) {
|
|
273
|
+
if (!isRecord(entry)) continue;
|
|
274
|
+
if (typeof entry.directory !== "string" || typeof entry.name !== "string") continue;
|
|
275
|
+
profiles.push({ directory: entry.directory, name: entry.name });
|
|
276
|
+
}
|
|
277
|
+
return profiles;
|
|
278
|
+
} finally {
|
|
279
|
+
if (socketDirectory !== null) rmSync(socketDirectory, { recursive: true, force: true });
|
|
280
|
+
rmSync(directory, { recursive: true, force: true });
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function selectedProfile(profiles: readonly ChromeProfile[]): string | undefined {
|
|
285
|
+
const defaultProfile = profiles.find(({ directory }) => directory === "Default");
|
|
286
|
+
if (defaultProfile !== undefined) return defaultProfile.directory;
|
|
287
|
+
return profiles.length === 1 ? profiles[0]?.directory : undefined;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function shouldExpand(url: URL, options: CaptureArguments, method: AcquisitionMethod): boolean {
|
|
291
|
+
if (options.scope === "page") return false;
|
|
292
|
+
const platform = classifyPlatformUrl(url.href)?.platform ?? "generic";
|
|
293
|
+
const hasExplicitCookies = options.cookieSources.length > 0 || options.cookiesFile !== undefined;
|
|
294
|
+
if (platform === "x" && method === "browser-fresh" && !hasExplicitCookies) return false;
|
|
295
|
+
return platform === "x"
|
|
296
|
+
|| platform === "hacker-news"
|
|
297
|
+
|| platform === "reddit"
|
|
298
|
+
|| platform === "bluesky"
|
|
299
|
+
|| platform === "linkedin"
|
|
300
|
+
|| platform === "facebook"
|
|
301
|
+
|| platform === "instagram"
|
|
302
|
+
|| platform === "tiktok"
|
|
303
|
+
|| platform === "threads"
|
|
304
|
+
|| platform === "whatsapp"
|
|
305
|
+
|| platform === "youtube"
|
|
306
|
+
|| platform === "github"
|
|
307
|
+
|| platform === "discourse"
|
|
308
|
+
|| platform === "substack";
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export type RenderedTextSnapshotMerge = {
|
|
312
|
+
readonly content: string;
|
|
313
|
+
readonly truncated: boolean;
|
|
314
|
+
readonly observedSnapshots: number;
|
|
315
|
+
/** Content lines contributed after the first non-empty snapshot. */
|
|
316
|
+
readonly addedLines: number;
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
function renderedTextLines(snapshot: string): readonly string[] {
|
|
320
|
+
const lines = snapshot.replace(/\r\n?/g, "\n").split("\n");
|
|
321
|
+
let start = 0;
|
|
322
|
+
while (start < lines.length && lines[start]?.trim() === "") start += 1;
|
|
323
|
+
let end = lines.length;
|
|
324
|
+
while (end > start && lines[end - 1]?.trim() === "") end -= 1;
|
|
325
|
+
return lines.slice(start, end);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Find the longest suffix of `source` that is also a prefix of `prefix`. */
|
|
329
|
+
function suffixPrefixOverlap(source: readonly string[], prefix: readonly string[]): number {
|
|
330
|
+
if (source.length === 0 || prefix.length === 0) return 0;
|
|
331
|
+
|
|
332
|
+
// KMP keeps snapshot merging linear even for long feeds full of repeated rows.
|
|
333
|
+
const fallback = new Array<number>(prefix.length).fill(0);
|
|
334
|
+
for (let index = 1; index < prefix.length; index += 1) {
|
|
335
|
+
let matched = fallback[index - 1] ?? 0;
|
|
336
|
+
while (matched > 0 && prefix[index] !== prefix[matched]) {
|
|
337
|
+
matched = fallback[matched - 1] ?? 0;
|
|
338
|
+
}
|
|
339
|
+
if (prefix[index] === prefix[matched]) matched += 1;
|
|
340
|
+
fallback[index] = matched;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
let matched = 0;
|
|
344
|
+
for (let index = 0; index < source.length; index += 1) {
|
|
345
|
+
while (matched > 0 && source[index] !== prefix[matched]) {
|
|
346
|
+
matched = fallback[matched - 1] ?? 0;
|
|
347
|
+
}
|
|
348
|
+
if (source[index] === prefix[matched]) matched += 1;
|
|
349
|
+
// A complete match before the end is not a suffix. Continue with its
|
|
350
|
+
// longest proper border so a later match can reach the final source line.
|
|
351
|
+
if (matched === prefix.length && index < source.length - 1) {
|
|
352
|
+
matched = fallback[matched - 1] ?? 0;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return matched;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function truncateUtf8(value: string, maxBytes: number): { readonly content: string; readonly truncated: boolean } {
|
|
359
|
+
const encoded = new TextEncoder().encode(value);
|
|
360
|
+
if (encoded.byteLength <= maxBytes) return { content: value, truncated: false };
|
|
361
|
+
|
|
362
|
+
let end = maxBytes;
|
|
363
|
+
// If the byte just outside the prefix is a UTF-8 continuation byte, the
|
|
364
|
+
// prefix ended part-way through a code point. Drop that incomplete point.
|
|
365
|
+
while (end > 0 && (encoded[end] ?? 0) >>> 6 === 0b10) end -= 1;
|
|
366
|
+
return {
|
|
367
|
+
content: new TextDecoder("utf-8", { fatal: true }).decode(encoded.subarray(0, end)),
|
|
368
|
+
truncated: true,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* Merge consecutive rendered-text observations from one page.
|
|
374
|
+
*
|
|
375
|
+
* Static leading chrome is removed from later observations, while a suffix /
|
|
376
|
+
* prefix overlap preserves virtualized rows that disappeared during scrolling.
|
|
377
|
+
*/
|
|
378
|
+
export function mergeRenderedTextSnapshots(
|
|
379
|
+
snapshots: readonly string[],
|
|
380
|
+
maxBytes: number,
|
|
381
|
+
): RenderedTextSnapshotMerge {
|
|
382
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) {
|
|
383
|
+
throw new RangeError("rendered text byte limit must be a non-negative safe integer");
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
const merged: string[] = [];
|
|
387
|
+
let hasBaseline = false;
|
|
388
|
+
let addedLines = 0;
|
|
389
|
+
for (const snapshot of snapshots) {
|
|
390
|
+
const lines = renderedTextLines(snapshot);
|
|
391
|
+
if (lines.length === 0) continue;
|
|
392
|
+
if (!hasBaseline) {
|
|
393
|
+
for (const line of lines) merged.push(line);
|
|
394
|
+
hasBaseline = true;
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
let commonPrefix = 0;
|
|
399
|
+
const prefixLimit = Math.min(merged.length, lines.length);
|
|
400
|
+
while (commonPrefix < prefixLimit && merged[commonPrefix] === lines[commonPrefix]) {
|
|
401
|
+
commonPrefix += 1;
|
|
402
|
+
}
|
|
403
|
+
const remaining = lines.slice(commonPrefix);
|
|
404
|
+
if (remaining.length === 0) continue;
|
|
405
|
+
|
|
406
|
+
const overlap = suffixPrefixOverlap(merged, remaining);
|
|
407
|
+
const additions = remaining.slice(overlap);
|
|
408
|
+
if (additions.length === 0) continue;
|
|
409
|
+
if (commonPrefix === 0 && overlap === 0 && merged.at(-1) !== "") merged.push("");
|
|
410
|
+
for (const line of additions) merged.push(line);
|
|
411
|
+
addedLines += additions.length;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
const bounded = truncateUtf8(merged.join("\n"), maxBytes);
|
|
415
|
+
return {
|
|
416
|
+
...bounded,
|
|
417
|
+
observedSnapshots: snapshots.length,
|
|
418
|
+
addedLines,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
export type BrowserExpansionLimits = {
|
|
423
|
+
readonly maxScrolls: number;
|
|
424
|
+
/** Total UTF-8 bytes retained across every in-pass rendered-text observation. */
|
|
425
|
+
readonly maxObservedTextBytes: number;
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
export type BrowserExpansionTelemetry = {
|
|
429
|
+
readonly scrolls: number;
|
|
430
|
+
readonly scrollBudgetReached: boolean;
|
|
431
|
+
readonly renderedTextSnapshots: readonly string[];
|
|
432
|
+
readonly renderedTextObservationTruncated: boolean;
|
|
433
|
+
};
|
|
434
|
+
|
|
435
|
+
/** Derive conservative browser-work budgets from the requested conversation size. */
|
|
436
|
+
export function browserExpansionLimits(
|
|
437
|
+
maxItems: number,
|
|
438
|
+
maxObservedTextBytes = 4 * 1024 * 1024,
|
|
439
|
+
): BrowserExpansionLimits {
|
|
440
|
+
const boundedItems = Number.isSafeInteger(maxItems) ? Math.max(1, Math.min(maxItems, 10_000)) : 500;
|
|
441
|
+
const boundedObservationBytes = Number.isSafeInteger(maxObservedTextBytes) && maxObservedTextBytes > 0
|
|
442
|
+
? Math.min(maxObservedTextBytes, 4 * 1024 * 1024)
|
|
443
|
+
: 4 * 1024 * 1024;
|
|
444
|
+
return {
|
|
445
|
+
maxScrolls: Math.max(3, Math.min(40, Math.ceil(boundedItems / 20))),
|
|
446
|
+
maxObservedTextBytes: boundedObservationBytes,
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** Build the bounded, ingestion-only scroll/observation program used by rendered thread capture. */
|
|
451
|
+
export function browserExpansionScript(limits: BrowserExpansionLimits): string {
|
|
452
|
+
return `(async () => {
|
|
453
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
454
|
+
const renderedTextSnapshots = [];
|
|
455
|
+
let observedTextBytes = 0;
|
|
456
|
+
let renderedTextObservationTruncated = false;
|
|
457
|
+
let stable = 0;
|
|
458
|
+
let previousHeight = 0;
|
|
459
|
+
let scrolls = 0;
|
|
460
|
+
let settled = false;
|
|
461
|
+
const boundedRenderedText = (value, maxBytes) => {
|
|
462
|
+
let bytes = 0;
|
|
463
|
+
let end = 0;
|
|
464
|
+
while (end < value.length) {
|
|
465
|
+
const first = value.charCodeAt(end);
|
|
466
|
+
let width = 3;
|
|
467
|
+
let codeUnits = 1;
|
|
468
|
+
if (first <= 0x7f) width = 1;
|
|
469
|
+
else if (first <= 0x7ff) width = 2;
|
|
470
|
+
else if (first >= 0xd800 && first <= 0xdbff) {
|
|
471
|
+
const second = value.charCodeAt(end + 1);
|
|
472
|
+
if (second >= 0xdc00 && second <= 0xdfff) {
|
|
473
|
+
width = 4;
|
|
474
|
+
codeUnits = 2;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
if (bytes + width > maxBytes) break;
|
|
478
|
+
bytes += width;
|
|
479
|
+
end += codeUnits;
|
|
480
|
+
}
|
|
481
|
+
return { text: value.slice(0, end), bytes, truncated: end < value.length };
|
|
482
|
+
};
|
|
483
|
+
for (let pass = 0; pass < ${limits.maxScrolls}; pass += 1) {
|
|
484
|
+
window.scrollTo(0, document.documentElement.scrollHeight);
|
|
485
|
+
scrolls += 1;
|
|
486
|
+
await sleep(700);
|
|
487
|
+
try {
|
|
488
|
+
const root = document.body || document.documentElement;
|
|
489
|
+
const renderedValue = root ? root.innerText : '';
|
|
490
|
+
const rendered = typeof renderedValue === 'string' ? renderedValue : '';
|
|
491
|
+
const remainingBytes = ${limits.maxObservedTextBytes} - observedTextBytes;
|
|
492
|
+
const remainingPasses = ${limits.maxScrolls} - pass;
|
|
493
|
+
const passBytes = Math.max(0, Math.ceil(remainingBytes / remainingPasses));
|
|
494
|
+
const observation = boundedRenderedText(rendered, passBytes);
|
|
495
|
+
if (observation.text.trim() !== '') renderedTextSnapshots.push(observation.text);
|
|
496
|
+
observedTextBytes += observation.bytes;
|
|
497
|
+
if (observation.truncated) renderedTextObservationTruncated = true;
|
|
498
|
+
} catch {
|
|
499
|
+
renderedTextObservationTruncated = true;
|
|
500
|
+
}
|
|
501
|
+
const height = document.documentElement.scrollHeight;
|
|
502
|
+
stable = height === previousHeight ? stable + 1 : 0;
|
|
503
|
+
previousHeight = height;
|
|
504
|
+
if (stable >= 2) {
|
|
505
|
+
settled = true;
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
window.scrollTo(0, 0);
|
|
510
|
+
return {
|
|
511
|
+
scrolls,
|
|
512
|
+
scrollBudgetReached: !settled && scrolls >= ${limits.maxScrolls},
|
|
513
|
+
renderedTextSnapshots,
|
|
514
|
+
renderedTextObservationTruncated
|
|
515
|
+
};
|
|
516
|
+
})()`;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
const nonNegativeInteger = (value: unknown): number | null =>
|
|
520
|
+
typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
521
|
+
|
|
522
|
+
/** Narrow eval telemetry and reject target-controlled or incompatible result shapes. */
|
|
523
|
+
export function readBrowserExpansionTelemetry(
|
|
524
|
+
value: unknown,
|
|
525
|
+
limits: BrowserExpansionLimits,
|
|
526
|
+
): BrowserExpansionTelemetry | null {
|
|
527
|
+
if (!isRecord(value)) return null;
|
|
528
|
+
const scrolls = nonNegativeInteger(value.scrolls);
|
|
529
|
+
if (!Array.isArray(value.renderedTextSnapshots)) return null;
|
|
530
|
+
if (value.renderedTextSnapshots.length > limits.maxScrolls) return null;
|
|
531
|
+
const renderedTextSnapshots: string[] = [];
|
|
532
|
+
let observedTextBytes = 0;
|
|
533
|
+
for (const snapshot of value.renderedTextSnapshots) {
|
|
534
|
+
if (typeof snapshot !== "string") return null;
|
|
535
|
+
if (snapshot.length > limits.maxObservedTextBytes) return null;
|
|
536
|
+
observedTextBytes += new TextEncoder().encode(snapshot).byteLength;
|
|
537
|
+
if (observedTextBytes > limits.maxObservedTextBytes) return null;
|
|
538
|
+
renderedTextSnapshots.push(snapshot);
|
|
539
|
+
}
|
|
540
|
+
if (
|
|
541
|
+
scrolls === null
|
|
542
|
+
|| scrolls > limits.maxScrolls
|
|
543
|
+
|| renderedTextSnapshots.length > scrolls
|
|
544
|
+
|| typeof value.scrollBudgetReached !== "boolean"
|
|
545
|
+
|| typeof value.renderedTextObservationTruncated !== "boolean"
|
|
546
|
+
) return null;
|
|
547
|
+
return {
|
|
548
|
+
scrolls,
|
|
549
|
+
scrollBudgetReached: value.scrollBudgetReached,
|
|
550
|
+
renderedTextSnapshots,
|
|
551
|
+
renderedTextObservationTruncated: value.renderedTextObservationTruncated,
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/** Convert bounded browser-expansion telemetry into explicit completeness warnings. */
|
|
556
|
+
export function browserExpansionWarnings(
|
|
557
|
+
telemetry: BrowserExpansionTelemetry,
|
|
558
|
+
limits: BrowserExpansionLimits,
|
|
559
|
+
): readonly string[] {
|
|
560
|
+
const warnings = [
|
|
561
|
+
"Browser capture left disclosure controls untouched; collapsed content may remain unavailable.",
|
|
562
|
+
];
|
|
563
|
+
if (telemetry.scrollBudgetReached) {
|
|
564
|
+
warnings.push(
|
|
565
|
+
`Browser capture reached its ${limits.maxScrolls}-scroll budget before the document stabilized; lazy content may remain unloaded.`,
|
|
566
|
+
);
|
|
567
|
+
}
|
|
568
|
+
if (telemetry.renderedTextObservationTruncated) {
|
|
569
|
+
warnings.push(
|
|
570
|
+
`Rendered-text observations reached their ${limits.maxObservedTextBytes}-byte capture budget; some virtualized content may be missing.`,
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
return warnings;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
function browserCaptureScript(): string {
|
|
577
|
+
return `({
|
|
578
|
+
url: location.href,
|
|
579
|
+
title: document.title,
|
|
580
|
+
html: '<!doctype html>\\n' + document.documentElement.outerHTML
|
|
581
|
+
})`;
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
function readBrowserContent(data: Record<string, unknown>): {
|
|
585
|
+
readonly content: string;
|
|
586
|
+
readonly finalUrl: URL;
|
|
587
|
+
readonly truncated: boolean;
|
|
588
|
+
} {
|
|
589
|
+
if (typeof data.content !== "string" || data.content.trim() === "") {
|
|
590
|
+
throw new Error("agent-browser read returned no rendered content");
|
|
591
|
+
}
|
|
592
|
+
const url = typeof data.finalUrl === "string" ? data.finalUrl : data.url;
|
|
593
|
+
if (typeof url !== "string") throw new Error("agent-browser read returned no final URL");
|
|
594
|
+
return { content: data.content, finalUrl: new URL(url), truncated: data.truncated === true };
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function readBrowserUrl(data: Record<string, unknown>): URL {
|
|
598
|
+
const value = typeof data.url === "string" ? data.url : data.finalUrl;
|
|
599
|
+
if (typeof value !== "string") throw new Error("agent-browser returned no current URL");
|
|
600
|
+
return new URL(value);
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
function navigationIdentity(url: URL): string {
|
|
604
|
+
const comparable = new URL(url);
|
|
605
|
+
comparable.hash = "";
|
|
606
|
+
return comparable.href;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/** Expansion may change a fragment, but it must never replace the captured document. */
|
|
610
|
+
export function browserExpansionStayedOnPage(before: URL, after: URL): boolean {
|
|
611
|
+
return (after.protocol === "http:" || after.protocol === "https:")
|
|
612
|
+
&& navigationIdentity(before) === navigationIdentity(after);
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
/** Refuse to confuse an unrelated pre-existing attached tab with the requested capture. */
|
|
616
|
+
export function browserNavigationReachedTarget(
|
|
617
|
+
target: URL,
|
|
618
|
+
before: URL | null,
|
|
619
|
+
after: URL,
|
|
620
|
+
navigationCommandSucceeded: boolean,
|
|
621
|
+
): boolean {
|
|
622
|
+
if (after.protocol !== "http:" && after.protocol !== "https:") return false;
|
|
623
|
+
const targetIdentity = navigationIdentity(target);
|
|
624
|
+
const afterIdentity = navigationIdentity(after);
|
|
625
|
+
if (afterIdentity === targetIdentity) return true;
|
|
626
|
+
return navigationCommandSucceeded
|
|
627
|
+
&& before !== null
|
|
628
|
+
&& navigationIdentity(before) !== afterIdentity;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
async function terminateAgentBrowserSession(session: string, socketDirectory: string): Promise<void> {
|
|
632
|
+
const pidPath = join(socketDirectory, `${session}.pid`);
|
|
633
|
+
if (!existsSync(pidPath)) return;
|
|
634
|
+
const rawPid = readFileSync(pidPath, "utf8").trim();
|
|
635
|
+
if (!/^\d+$/.test(rawPid)) return;
|
|
636
|
+
const pid = Number(rawPid);
|
|
637
|
+
if (!Number.isSafeInteger(pid) || pid <= 1 || pid === process.pid) return;
|
|
638
|
+
const signal = (name: NodeJS.Signals): void => {
|
|
639
|
+
try {
|
|
640
|
+
process.kill(process.platform === "win32" ? pid : -pid, name);
|
|
641
|
+
} catch {
|
|
642
|
+
try {
|
|
643
|
+
process.kill(pid, name);
|
|
644
|
+
} catch {
|
|
645
|
+
// The exact, tool-owned process already exited.
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
};
|
|
649
|
+
signal("SIGTERM");
|
|
650
|
+
await Bun.sleep(500);
|
|
651
|
+
signal("SIGKILL");
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
function pathInside(root: string, target: string): boolean {
|
|
655
|
+
const child = relative(root, target);
|
|
656
|
+
return child === "" || (!isAbsolute(child) && child !== ".." && !child.startsWith(`..${sep}`));
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
function canonicalPotentialPath(value: string, label: string): string {
|
|
660
|
+
const suffix: string[] = [];
|
|
661
|
+
let ancestor = resolve(value);
|
|
662
|
+
while (true) {
|
|
663
|
+
try {
|
|
664
|
+
lstatSync(ancestor);
|
|
665
|
+
let canonicalAncestor: string;
|
|
666
|
+
try {
|
|
667
|
+
canonicalAncestor = realpathSync(ancestor);
|
|
668
|
+
} catch {
|
|
669
|
+
throw new Error(`${label} contains an unresolved symbolic link.`);
|
|
670
|
+
}
|
|
671
|
+
return resolve(canonicalAncestor, ...suffix);
|
|
672
|
+
} catch (error) {
|
|
673
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
674
|
+
}
|
|
675
|
+
const parent = dirname(ancestor);
|
|
676
|
+
if (parent === ancestor) throw new Error(`${label} has no resolvable filesystem ancestor.`);
|
|
677
|
+
suffix.unshift(basename(ancestor));
|
|
678
|
+
ancestor = parent;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
function profilePath(value: string): string | null {
|
|
683
|
+
const pathLike = isAbsolute(value) || value.startsWith(`.${sep}`) || value.startsWith(`..${sep}`)
|
|
684
|
+
|| value.startsWith(`~${sep}`) || value.includes("/") || value.includes("\\");
|
|
685
|
+
if (!pathLike) return null;
|
|
686
|
+
const expanded = value.startsWith(`~${sep}`) ? join(homedir(), value.slice(2)) : resolve(value);
|
|
687
|
+
return canonicalPotentialPath(expanded, "Persistent browser profile");
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
export function assertSafePersistentProfile(options: CaptureArguments): string | null {
|
|
691
|
+
if (options.browserProfile === undefined) return null;
|
|
692
|
+
const path = profilePath(options.browserProfile);
|
|
693
|
+
if (path === null) return null;
|
|
694
|
+
const repositoryRoot = realpathSync(findKbPackageRoot());
|
|
695
|
+
const outputRoot = canonicalPotentialPath(options.outputBase, "Capture output root");
|
|
696
|
+
if (pathInside(repositoryRoot, path) || pathInside(outputRoot, path) || pathInside(path, outputRoot)) {
|
|
697
|
+
throw new Error("Persistent browser profiles must live outside the repository and capture output roots.");
|
|
698
|
+
}
|
|
699
|
+
return path;
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/** Build secret-bearing commands for stdin-only batch delivery to an owned fresh browser. */
|
|
703
|
+
export function browserCookieCommands(
|
|
704
|
+
cookies: readonly StrictCookie[],
|
|
705
|
+
target: URL,
|
|
706
|
+
): readonly (readonly string[])[] {
|
|
707
|
+
return cookies.map((cookie) => {
|
|
708
|
+
const command = ["cookies", "set", cookie.name, cookie.value];
|
|
709
|
+
if (cookie.hostOnly) command.push("--url", target.origin);
|
|
710
|
+
else command.push("--domain", `.${cookie.domain}`);
|
|
711
|
+
command.push("--path", cookie.path);
|
|
712
|
+
if (cookie.httpOnly) command.push("--httpOnly");
|
|
713
|
+
if (cookie.secure) command.push("--secure");
|
|
714
|
+
if (cookie.sameSite !== null) command.push("--sameSite", cookie.sameSite);
|
|
715
|
+
if (cookie.expires > 0) command.push("--expires", String(cookie.expires));
|
|
716
|
+
return command;
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
export async function seedOwnedBrowserCookies(
|
|
721
|
+
options: CaptureArguments,
|
|
722
|
+
globalArgs: readonly string[],
|
|
723
|
+
commandOptions: CommandIsolation & { readonly timeoutMs: number; readonly maxOutputBytes: number },
|
|
724
|
+
dependencies: {
|
|
725
|
+
readonly readCookies?: CookieRecordReader;
|
|
726
|
+
readonly runBatch?: typeof runAgentBrowserBatch;
|
|
727
|
+
} = {},
|
|
728
|
+
): Promise<readonly string[]> {
|
|
729
|
+
const selected = options.cookieSources.length > 0 || options.cookiesFile !== undefined;
|
|
730
|
+
if (!selected) return [];
|
|
731
|
+
const target = captureUrl(options);
|
|
732
|
+
const result = await (dependencies.readCookies ?? acquireCookieRecords)(options, target);
|
|
733
|
+
await (dependencies.runBatch ?? runAgentBrowserBatch)(
|
|
734
|
+
globalArgs,
|
|
735
|
+
browserCookieCommands(result.cookies, target),
|
|
736
|
+
commandOptions,
|
|
737
|
+
);
|
|
738
|
+
return [
|
|
739
|
+
...result.warnings,
|
|
740
|
+
"Seeded explicitly selected cookies into the owned browser without broadening their domain, path, Secure, HttpOnly, SameSite, or expiry attributes.",
|
|
741
|
+
];
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
/** Chromium flags that prevent loopback, QUIC, or WebRTC from bypassing the filtering proxy. */
|
|
745
|
+
export function browserProxyArguments(
|
|
746
|
+
proxyUrl: string,
|
|
747
|
+
profileDirectory?: "Default",
|
|
748
|
+
): readonly string[] {
|
|
749
|
+
const chromiumArguments = [
|
|
750
|
+
...(profileDirectory === undefined ? [] : [`--profile-directory=${profileDirectory}`]),
|
|
751
|
+
"--disable-quic",
|
|
752
|
+
"--disable-dns-prefetch",
|
|
753
|
+
"--disable-background-networking",
|
|
754
|
+
"--disable-component-update",
|
|
755
|
+
"--disable-default-apps",
|
|
756
|
+
"--disable-sync",
|
|
757
|
+
"--disable-features=AsyncDns",
|
|
758
|
+
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
|
|
759
|
+
"--proxy-bypass-list=<-loopback>",
|
|
760
|
+
].join("\n");
|
|
761
|
+
return ["--proxy", proxyUrl, "--args", chromiumArguments];
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
export type BrowserAcquisitionDependencies = {
|
|
765
|
+
readonly run?: typeof runAgentBrowser;
|
|
766
|
+
readonly runBatch?: typeof runAgentBrowserBatch;
|
|
767
|
+
readonly sleep?: (milliseconds: number) => Promise<void>;
|
|
768
|
+
readonly assertNetworkUrl?: typeof assertSafeNetworkUrl;
|
|
769
|
+
};
|
|
770
|
+
|
|
771
|
+
export async function acquireBrowser(
|
|
772
|
+
options: CaptureArguments,
|
|
773
|
+
temporaryDirectory: string,
|
|
774
|
+
useDiscoveredProfile = false,
|
|
775
|
+
dependencies: BrowserAcquisitionDependencies = {},
|
|
776
|
+
): Promise<AcquiredPage> {
|
|
777
|
+
if (options.currentTab && (!options.browserLive && options.cdp === undefined)) {
|
|
778
|
+
throw new Error("current-tab capture requires --browser-live or --cdp");
|
|
779
|
+
}
|
|
780
|
+
if (options.currentTab && options.browserProfile !== undefined) {
|
|
781
|
+
throw new Error("current-tab capture cannot use a browser profile; attach with --browser-live or --cdp");
|
|
782
|
+
}
|
|
783
|
+
const assertNetworkUrl = dependencies.assertNetworkUrl ?? assertSafeNetworkUrl;
|
|
784
|
+
const requestedUrl = options.currentTab ? null : captureUrl(options);
|
|
785
|
+
if (requestedUrl !== null) {
|
|
786
|
+
await assertNetworkUrl(requestedUrl, options.allowPrivateNetwork, options.timeoutMs);
|
|
787
|
+
}
|
|
788
|
+
const runBrowser = dependencies.run ?? runAgentBrowser;
|
|
789
|
+
const runBrowserBatch = dependencies.runBatch ?? runAgentBrowserBatch;
|
|
790
|
+
const sleep = dependencies.sleep ?? ((milliseconds: number) => Bun.sleep(milliseconds));
|
|
791
|
+
const warnings: string[] = [];
|
|
792
|
+
const persistentProfilePath = assertSafePersistentProfile(options);
|
|
793
|
+
const ownedProfile = options.browserProfileOwnership === "owned";
|
|
794
|
+
if (ownedProfile && persistentProfilePath === null) {
|
|
795
|
+
throw new Error("owned browser-profile execution requires an explicit path-backed profile");
|
|
796
|
+
}
|
|
797
|
+
const session = `clip-${process.pid}-${Date.now().toString(36)}-${crypto.randomUUID().slice(0, 8)}`;
|
|
798
|
+
const isolation = createAgentBrowserIsolation(temporaryDirectory);
|
|
799
|
+
try {
|
|
800
|
+
const globalArgs = ["--config", isolation.configPath, "--session", session];
|
|
801
|
+
if (options.browserExecutable !== undefined) {
|
|
802
|
+
globalArgs.push("--executable-path", options.browserExecutable);
|
|
803
|
+
}
|
|
804
|
+
let method: AcquisitionMethod = "browser-fresh";
|
|
805
|
+
let ownsBrowser = true;
|
|
806
|
+
|
|
807
|
+
if (options.cdp !== undefined) {
|
|
808
|
+
globalArgs.push("--cdp", options.cdp);
|
|
809
|
+
method = "browser-cdp";
|
|
810
|
+
ownsBrowser = false;
|
|
811
|
+
} else if (options.browserLive) {
|
|
812
|
+
globalArgs.push("--auto-connect");
|
|
813
|
+
method = "browser-live";
|
|
814
|
+
ownsBrowser = false;
|
|
815
|
+
} else {
|
|
816
|
+
const profile = options.browserProfile
|
|
817
|
+
?? (useDiscoveredProfile ? selectedProfile(await discoverChromeProfiles(options.timeoutMs)) : undefined);
|
|
818
|
+
if (profile !== undefined) {
|
|
819
|
+
globalArgs.push("--profile", profile);
|
|
820
|
+
method = "browser-profile";
|
|
821
|
+
warnings.push(ownedProfile
|
|
822
|
+
? "Used an owned private browser-profile snapshot; page activity cannot modify the source profile."
|
|
823
|
+
: persistentProfilePath === null
|
|
824
|
+
? "A named Chrome profile can expose broad all-origin browser state to public subresources loaded by the target page; prefer a dedicated per-site profile."
|
|
825
|
+
: "The selected persistent browser profile can be updated by page activity; keep dedicated capture profiles outside the repository.");
|
|
826
|
+
} else if (options.browserProfile !== undefined || useDiscoveredProfile) {
|
|
827
|
+
warnings.push("No unambiguous Chrome profile was found; used a fresh browser session.");
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
let networkProxy: LocalNetworkProxy | null = null;
|
|
832
|
+
|
|
833
|
+
const commandOptions = {
|
|
834
|
+
cwd: isolation.cwd,
|
|
835
|
+
environment: isolation.environment,
|
|
836
|
+
timeoutMs: options.timeoutMs,
|
|
837
|
+
maxOutputBytes: Math.max(options.maxHtmlBytes * 2 + 1024 * 1024, 4 * 1024 * 1024),
|
|
838
|
+
};
|
|
839
|
+
try {
|
|
840
|
+
if (ownsBrowser) {
|
|
841
|
+
networkProxy = await startNetworkProxy({
|
|
842
|
+
allowPrivateNetwork: options.allowPrivateNetwork,
|
|
843
|
+
timeoutMs: options.timeoutMs,
|
|
844
|
+
maxTransferredBytes: Math.max(
|
|
845
|
+
64 * 1024 * 1024,
|
|
846
|
+
Math.min(Number.MAX_SAFE_INTEGER, (options.maxHtmlBytes + options.maxTotalAssetBytes) * 2),
|
|
847
|
+
),
|
|
848
|
+
});
|
|
849
|
+
globalArgs.push(...browserProxyArguments(networkProxy.url, options.browserProfileDirectory));
|
|
850
|
+
}
|
|
851
|
+
if (!options.currentTab) {
|
|
852
|
+
try {
|
|
853
|
+
await runBrowser(globalArgs, ["open", "about:blank"], commandOptions);
|
|
854
|
+
} catch (error) {
|
|
855
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
856
|
+
warnings.push(`Browser startup did not settle cleanly; attempted to use the isolated session: ${message}`);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
if (ownsBrowser && (method === "browser-fresh" || ownedProfile)) {
|
|
860
|
+
warnings.push(...await seedOwnedBrowserCookies(options, globalArgs, commandOptions));
|
|
861
|
+
} else if (options.cookieSources.length > 0 || options.cookiesFile !== undefined) {
|
|
862
|
+
warnings.push("Explicit cookie input remained a separate HTTP/media lane and was not imported into the selected profile or attached browser.");
|
|
863
|
+
}
|
|
864
|
+
if (!ownsBrowser) {
|
|
865
|
+
warnings.push(options.currentTab
|
|
866
|
+
? "Captured the current attached tab without navigation or interaction; the external browser itself was left open."
|
|
867
|
+
: "Attached browser capture navigated and scrolled the active tab; the external browser itself was left open.");
|
|
868
|
+
}
|
|
869
|
+
let beforeNavigation: URL | null = null;
|
|
870
|
+
try {
|
|
871
|
+
beforeNavigation = readBrowserUrl(await runBrowser(globalArgs, ["get", "url"], commandOptions));
|
|
872
|
+
} catch (error) {
|
|
873
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
874
|
+
warnings.push(`Could not establish a pre-navigation browser URL: ${message}`);
|
|
875
|
+
}
|
|
876
|
+
let navigationCommandSucceeded = false;
|
|
877
|
+
if (!options.currentTab) {
|
|
878
|
+
try {
|
|
879
|
+
// The requested URL can contain one-time query credentials. Deliver the
|
|
880
|
+
// native CDP navigation command over batch stdin so it never appears in
|
|
881
|
+
// a process listing and page JavaScript cannot monkeypatch or race it.
|
|
882
|
+
await runBrowserBatch(
|
|
883
|
+
globalArgs,
|
|
884
|
+
[["open", captureUrl(options).href]],
|
|
885
|
+
commandOptions,
|
|
886
|
+
);
|
|
887
|
+
navigationCommandSucceeded = true;
|
|
888
|
+
} catch (error) {
|
|
889
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
890
|
+
warnings.push(`Browser navigation command ended during page transition: ${message}`);
|
|
891
|
+
}
|
|
892
|
+
await sleep(Math.min(5_000, Math.max(1_500, Math.floor(options.timeoutMs / 6))));
|
|
893
|
+
}
|
|
894
|
+
let readable: ReturnType<typeof readBrowserContent>;
|
|
895
|
+
try {
|
|
896
|
+
readable = readBrowserContent(await runBrowser(globalArgs, ["read"], commandOptions));
|
|
897
|
+
} catch (error) {
|
|
898
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
899
|
+
warnings.push(`Rendered readable text was unavailable; continuing with the bounded DOM: ${message}`);
|
|
900
|
+
readable = {
|
|
901
|
+
content: "",
|
|
902
|
+
finalUrl: readBrowserUrl(await runBrowser(globalArgs, ["get", "url"], commandOptions)),
|
|
903
|
+
truncated: false,
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
if (options.currentTab) {
|
|
907
|
+
if (readable.finalUrl.protocol !== "http:" && readable.finalUrl.protocol !== "https:") {
|
|
908
|
+
throw new Error("the current tab must have an HTTP or HTTPS URL");
|
|
909
|
+
}
|
|
910
|
+
if (beforeNavigation !== null && !browserExpansionStayedOnPage(beforeNavigation, readable.finalUrl)) {
|
|
911
|
+
throw new Error("the current tab changed pages while it was being read; retry on the intended page");
|
|
912
|
+
}
|
|
913
|
+
} else if (!browserNavigationReachedTarget(
|
|
914
|
+
captureUrl(options),
|
|
915
|
+
beforeNavigation,
|
|
916
|
+
readable.finalUrl,
|
|
917
|
+
navigationCommandSucceeded,
|
|
918
|
+
)) {
|
|
919
|
+
throw new Error("browser did not establish the requested navigation; refusing to capture a pre-existing tab");
|
|
920
|
+
}
|
|
921
|
+
let browserPageProvenanceIntact = true;
|
|
922
|
+
if (!options.currentTab && shouldExpand(readable.finalUrl, options, method)) {
|
|
923
|
+
const expansionLimits = browserExpansionLimits(options.maxItems, options.maxHtmlBytes);
|
|
924
|
+
try {
|
|
925
|
+
const expansion = await runBrowser(globalArgs, ["eval", browserExpansionScript(expansionLimits)], {
|
|
926
|
+
...commandOptions,
|
|
927
|
+
timeoutMs: Math.min(commandOptions.timeoutMs, 30_000),
|
|
928
|
+
});
|
|
929
|
+
const telemetry = readBrowserExpansionTelemetry(expansion.result, expansionLimits);
|
|
930
|
+
if (telemetry === null) {
|
|
931
|
+
warnings.push("Browser expansion returned no trustworthy bounded-work telemetry; conversation completeness cannot be confirmed.");
|
|
932
|
+
} else {
|
|
933
|
+
warnings.push(...browserExpansionWarnings(telemetry, expansionLimits));
|
|
934
|
+
}
|
|
935
|
+
const expandedReadable = readBrowserContent(await runBrowser(globalArgs, ["read"], commandOptions));
|
|
936
|
+
if (browserExpansionStayedOnPage(readable.finalUrl, expandedReadable.finalUrl)) {
|
|
937
|
+
const merged = mergeRenderedTextSnapshots(
|
|
938
|
+
[
|
|
939
|
+
readable.content,
|
|
940
|
+
...(telemetry?.renderedTextSnapshots ?? []),
|
|
941
|
+
expandedReadable.content,
|
|
942
|
+
],
|
|
943
|
+
options.maxHtmlBytes,
|
|
944
|
+
);
|
|
945
|
+
readable = {
|
|
946
|
+
content: merged.content,
|
|
947
|
+
finalUrl: expandedReadable.finalUrl,
|
|
948
|
+
truncated: readable.truncated
|
|
949
|
+
|| expandedReadable.truncated
|
|
950
|
+
|| telemetry?.renderedTextObservationTruncated === true
|
|
951
|
+
|| merged.truncated,
|
|
952
|
+
};
|
|
953
|
+
if (merged.addedLines > 0) {
|
|
954
|
+
warnings.push(
|
|
955
|
+
`Merged ${merged.addedLines} newly observed rendered-text line(s) with the pre-expansion snapshot so virtualized content remains available.`,
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
} else {
|
|
959
|
+
browserPageProvenanceIntact = false;
|
|
960
|
+
warnings.push(
|
|
961
|
+
"Browser expansion navigated away from the captured page; preserved the proven baseline and skipped post-expansion DOM and screenshot capture.",
|
|
962
|
+
);
|
|
963
|
+
}
|
|
964
|
+
} catch (error) {
|
|
965
|
+
browserPageProvenanceIntact = false;
|
|
966
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
967
|
+
warnings.push(
|
|
968
|
+
`Browser expansion stopped early; preserved the baseline rendered text and skipped post-expansion DOM and screenshot capture: ${message}`,
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
if (!options.currentTab) {
|
|
973
|
+
await assertNetworkUrl(readable.finalUrl, options.allowPrivateNetwork, options.timeoutMs);
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
const renderedText = readable.content;
|
|
977
|
+
let body = renderedText;
|
|
978
|
+
let contentType = "text/plain; charset=utf-8";
|
|
979
|
+
let contentTruncated = readable.truncated;
|
|
980
|
+
let sourceEvidence: string | undefined;
|
|
981
|
+
let browserTitle: string | undefined;
|
|
982
|
+
if (browserPageProvenanceIntact) {
|
|
983
|
+
try {
|
|
984
|
+
const capture = await runBrowser(globalArgs, ["eval", browserCaptureScript()], commandOptions);
|
|
985
|
+
if (isRecord(capture.result)) {
|
|
986
|
+
const html = capture.result.html;
|
|
987
|
+
const title = capture.result.title;
|
|
988
|
+
const captureUrl = typeof capture.result.url === "string"
|
|
989
|
+
? new URL(capture.result.url)
|
|
990
|
+
: null;
|
|
991
|
+
if (captureUrl === null || !browserExpansionStayedOnPage(readable.finalUrl, captureUrl)) {
|
|
992
|
+
browserPageProvenanceIntact = false;
|
|
993
|
+
warnings.push("Rendered DOM capture changed pages; preserved the proven readable baseline.");
|
|
994
|
+
} else if (typeof html === "string") {
|
|
995
|
+
const byteLength = new TextEncoder().encode(html).byteLength;
|
|
996
|
+
if (byteLength <= options.maxHtmlBytes) {
|
|
997
|
+
body = html;
|
|
998
|
+
contentType = "text/html; charset=utf-8";
|
|
999
|
+
contentTruncated = false;
|
|
1000
|
+
if (options.evidence === "source" || options.evidence === "all") sourceEvidence = html;
|
|
1001
|
+
} else {
|
|
1002
|
+
warnings.push(`Rendered DOM exceeded ${options.maxHtmlBytes} bytes; extracted the bounded readable fallback.`);
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
if (
|
|
1006
|
+
browserPageProvenanceIntact
|
|
1007
|
+
&& typeof title === "string"
|
|
1008
|
+
&& title.trim() !== ""
|
|
1009
|
+
) browserTitle = title;
|
|
1010
|
+
} else {
|
|
1011
|
+
browserPageProvenanceIntact = false;
|
|
1012
|
+
warnings.push("Rendered DOM capture returned no trustworthy page provenance; preserved the readable baseline.");
|
|
1013
|
+
}
|
|
1014
|
+
} catch (error) {
|
|
1015
|
+
browserPageProvenanceIntact = false;
|
|
1016
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1017
|
+
warnings.push(`Rendered DOM was unavailable; extracted the bounded readable fallback: ${message}`);
|
|
1018
|
+
}
|
|
1019
|
+
} else {
|
|
1020
|
+
warnings.push("Rendered DOM capture was skipped because post-expansion page provenance was not established.");
|
|
1021
|
+
}
|
|
1022
|
+
if (body.trim() === "") throw new Error("browser returned neither readable text nor a bounded rendered DOM");
|
|
1023
|
+
if (readable.truncated) warnings.push("Rendered text was truncated at its configured output boundary.");
|
|
1024
|
+
|
|
1025
|
+
let screenshotPath: string | undefined;
|
|
1026
|
+
if ((options.evidence === "screenshot" || options.evidence === "all") && browserPageProvenanceIntact) {
|
|
1027
|
+
const requestedScreenshotPath = join(temporaryDirectory, "page.png");
|
|
1028
|
+
screenshotPath = requestedScreenshotPath;
|
|
1029
|
+
try {
|
|
1030
|
+
await runBrowser(globalArgs, ["screenshot", requestedScreenshotPath], {
|
|
1031
|
+
...commandOptions,
|
|
1032
|
+
timeoutMs: options.timeoutMs,
|
|
1033
|
+
maxOutputBytes: 2 * 1024 * 1024,
|
|
1034
|
+
});
|
|
1035
|
+
const afterScreenshot = readBrowserUrl(await runBrowser(globalArgs, ["get", "url"], commandOptions));
|
|
1036
|
+
if (!browserExpansionStayedOnPage(readable.finalUrl, afterScreenshot)) {
|
|
1037
|
+
rmSync(requestedScreenshotPath, { force: true });
|
|
1038
|
+
warnings.push("Browser screenshot changed pages during capture and was discarded.");
|
|
1039
|
+
screenshotPath = undefined;
|
|
1040
|
+
} else if (!existsSync(requestedScreenshotPath)) {
|
|
1041
|
+
warnings.push("Browser screenshot was requested but agent-browser did not create it.");
|
|
1042
|
+
screenshotPath = undefined;
|
|
1043
|
+
}
|
|
1044
|
+
} catch (error) {
|
|
1045
|
+
rmSync(requestedScreenshotPath, { force: true });
|
|
1046
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1047
|
+
warnings.push(`Browser screenshot was unavailable or had no trustworthy page provenance: ${message}`);
|
|
1048
|
+
screenshotPath = undefined;
|
|
1049
|
+
}
|
|
1050
|
+
} else if (options.evidence === "screenshot" || options.evidence === "all") {
|
|
1051
|
+
warnings.push("Browser screenshot was skipped because post-expansion page provenance was not established.");
|
|
1052
|
+
}
|
|
1053
|
+
return {
|
|
1054
|
+
body,
|
|
1055
|
+
contentType,
|
|
1056
|
+
finalUrl: options.currentTab
|
|
1057
|
+
? new URL(sanitizeArtifactUrl(readable.finalUrl.href))
|
|
1058
|
+
: readable.finalUrl,
|
|
1059
|
+
method,
|
|
1060
|
+
warnings,
|
|
1061
|
+
...(browserTitle === undefined ? {} : { browserTitle }),
|
|
1062
|
+
...(screenshotPath === undefined ? {} : { screenshotPath }),
|
|
1063
|
+
...(sourceEvidence === undefined ? {} : { sourceEvidence }),
|
|
1064
|
+
...(contentTruncated ? { contentTruncated: true } : {}),
|
|
1065
|
+
renderedText,
|
|
1066
|
+
...(readable.truncated ? { renderedTextTruncated: true } : {}),
|
|
1067
|
+
renderedTextByteLimit: options.maxHtmlBytes,
|
|
1068
|
+
};
|
|
1069
|
+
} finally {
|
|
1070
|
+
try {
|
|
1071
|
+
if (ownsBrowser) {
|
|
1072
|
+
try {
|
|
1073
|
+
await runBrowser(["--config", isolation.configPath, "--session", session], ["close"], {
|
|
1074
|
+
cwd: isolation.cwd,
|
|
1075
|
+
environment: isolation.environment,
|
|
1076
|
+
timeoutMs: 15_000,
|
|
1077
|
+
maxOutputBytes: 1024 * 1024,
|
|
1078
|
+
});
|
|
1079
|
+
} catch {
|
|
1080
|
+
warnings.push("Browser session did not close cleanly; terminated its isolated process group.");
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
// Attached sessions still spawn a task-owned agent-browser daemon. Stop
|
|
1084
|
+
// that client without issuing `close`, which could affect the user's browser.
|
|
1085
|
+
await terminateAgentBrowserSession(session, isolation.socketDirectory);
|
|
1086
|
+
} finally {
|
|
1087
|
+
await networkProxy?.close();
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
} finally {
|
|
1091
|
+
rmSync(isolation.socketDirectory, { recursive: true, force: true });
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
export async function acquireHttp(options: CaptureArguments): Promise<AcquiredPage> {
|
|
1096
|
+
const response = await safeFetch(captureUrl(options), {
|
|
1097
|
+
timeoutMs: options.timeoutMs,
|
|
1098
|
+
maxBytes: options.maxHtmlBytes,
|
|
1099
|
+
allowPrivateNetwork: options.allowPrivateNetwork,
|
|
1100
|
+
userAgent: options.userAgent,
|
|
1101
|
+
retries: 2,
|
|
1102
|
+
});
|
|
1103
|
+
return {
|
|
1104
|
+
body: decodeBytes(response.bytes, response.contentType),
|
|
1105
|
+
contentType: response.contentType,
|
|
1106
|
+
finalUrl: response.finalUrl,
|
|
1107
|
+
method: "http",
|
|
1108
|
+
warnings: [],
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
export async function acquireCookieHttp(options: CaptureArguments): Promise<AcquiredPage> {
|
|
1113
|
+
if (options.cookieSources.length === 0 && options.cookiesFile === undefined) {
|
|
1114
|
+
throw new Error("cookie capture requires --cookie-source or --cookies-file");
|
|
1115
|
+
}
|
|
1116
|
+
const target = captureUrl(options);
|
|
1117
|
+
const cookieResult = await acquireCookieHeader(options, target);
|
|
1118
|
+
const response = await safeFetch(target, {
|
|
1119
|
+
timeoutMs: options.timeoutMs,
|
|
1120
|
+
maxBytes: options.maxHtmlBytes,
|
|
1121
|
+
allowPrivateNetwork: options.allowPrivateNetwork,
|
|
1122
|
+
userAgent: options.userAgent,
|
|
1123
|
+
cookieHeader: cookieResult.header,
|
|
1124
|
+
retries: 2,
|
|
1125
|
+
});
|
|
1126
|
+
return {
|
|
1127
|
+
body: decodeBytes(response.bytes, response.contentType),
|
|
1128
|
+
contentType: response.contentType,
|
|
1129
|
+
finalUrl: response.finalUrl,
|
|
1130
|
+
method: "cookie-http",
|
|
1131
|
+
warnings: cookieResult.warnings,
|
|
1132
|
+
};
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
/** Read only cookies matching one requested origin from explicitly selected local state. */
|
|
1136
|
+
export type CookieStoreReader = (options: GetCookiesOptions) => Promise<unknown>;
|
|
1137
|
+
export type CookieSelection = Pick<
|
|
1138
|
+
CaptureArguments,
|
|
1139
|
+
"cookieSources" | "cookiesFile" | "cookieProfile" | "timeoutMs"
|
|
1140
|
+
> & {
|
|
1141
|
+
/** Authenticated API callers must not infer a cookie's host from their request target. */
|
|
1142
|
+
readonly requireExplicitCookieScope?: boolean;
|
|
1143
|
+
};
|
|
1144
|
+
export type CookieHeaderReader = (
|
|
1145
|
+
options: CookieSelection,
|
|
1146
|
+
url: URL,
|
|
1147
|
+
) => Promise<{ readonly header: string; readonly warnings: readonly string[] }>;
|
|
1148
|
+
export type CookieRecordReader = (
|
|
1149
|
+
options: CookieSelection,
|
|
1150
|
+
url: URL,
|
|
1151
|
+
) => Promise<{ readonly cookies: readonly StrictCookie[]; readonly warnings: readonly string[] }>;
|
|
1152
|
+
|
|
1153
|
+
/** Build an injectable strict reader that retains every browser cookie attribute. */
|
|
1154
|
+
export function createCookieRecordReader(reader: CookieStoreReader): CookieRecordReader {
|
|
1155
|
+
return async (options, url) => {
|
|
1156
|
+
if (options.cookieSources.length === 0 && options.cookiesFile === undefined) {
|
|
1157
|
+
throw new Error("cookie capture requires --cookie-source or --cookies-file");
|
|
1158
|
+
}
|
|
1159
|
+
if (options.cookiesFile !== undefined) {
|
|
1160
|
+
const parsed = readCookieFile(options.cookiesFile, url, {
|
|
1161
|
+
requirePrivate: options.requireExplicitCookieScope === true,
|
|
1162
|
+
});
|
|
1163
|
+
if (!parsed.ok) {
|
|
1164
|
+
throw new Error("the explicitly selected cookie file contained no usable cookies for this request");
|
|
1165
|
+
}
|
|
1166
|
+
if (options.requireExplicitCookieScope === true && parsed.scopeProvenance !== "explicit") {
|
|
1167
|
+
throw new Error("authenticated API cookie files require an explicit domain or URL on every cookie record");
|
|
1168
|
+
}
|
|
1169
|
+
const warnings: string[] = [];
|
|
1170
|
+
if (options.cookieSources.length > 0) {
|
|
1171
|
+
warnings.push("The explicit cookie file took precedence over the selected browser cookie source.");
|
|
1172
|
+
}
|
|
1173
|
+
if (parsed.rejected > 0) {
|
|
1174
|
+
warnings.push(`Ignored ${parsed.rejected} malformed, expired, or out-of-scope cookie record(s).`);
|
|
1175
|
+
}
|
|
1176
|
+
if (parsed.format === "cookie-header" || parsed.format === "curl") {
|
|
1177
|
+
warnings.push(
|
|
1178
|
+
"The cookie header did not encode attributes; browser replay inferred restrictive host-only, target-path, HTTPS-Secure, HttpOnly, and SameSite=Strict attributes. Use Cookie-Editor JSON or Netscape format when exact attributes matter.",
|
|
1179
|
+
);
|
|
1180
|
+
}
|
|
1181
|
+
return { cookies: parsed.cookies, warnings };
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
if (options.cookieSources.length === 0) {
|
|
1185
|
+
throw new Error("cookie capture requires at least one explicit browser cookie source");
|
|
1186
|
+
}
|
|
1187
|
+
const chromiumSource = options.cookieSources.find((source) =>
|
|
1188
|
+
source === "chrome" || source === "arc" || source === "brave" || source === "chromium");
|
|
1189
|
+
const selectedBrowsers: BrowserName[] = [];
|
|
1190
|
+
for (const source of options.cookieSources) {
|
|
1191
|
+
const backend: BrowserName = source === "arc" || source === "brave" || source === "chromium" ? "chrome" : source;
|
|
1192
|
+
if (!selectedBrowsers.includes(backend)) selectedBrowsers.push(backend);
|
|
1193
|
+
}
|
|
1194
|
+
const cookieOptions: GetCookiesOptions = {
|
|
1195
|
+
url: url.href,
|
|
1196
|
+
mode: "first",
|
|
1197
|
+
timeoutMs: options.timeoutMs,
|
|
1198
|
+
debug: false,
|
|
1199
|
+
browsers: selectedBrowsers,
|
|
1200
|
+
profile: options.cookieProfile ?? "",
|
|
1201
|
+
chromeProfile: options.cookieProfile ?? "",
|
|
1202
|
+
edgeProfile: options.cookieProfile ?? "",
|
|
1203
|
+
firefoxProfile: options.cookieProfile ?? "",
|
|
1204
|
+
...(chromiumSource === undefined ? {} : { chromiumBrowser: chromiumSource }),
|
|
1205
|
+
...(options.cookieProfile === undefined
|
|
1206
|
+
? {}
|
|
1207
|
+
: {
|
|
1208
|
+
...(options.cookieSources.includes("safari") ? { safariCookiesFile: options.cookieProfile } : {}),
|
|
1209
|
+
}),
|
|
1210
|
+
};
|
|
1211
|
+
let provided: unknown;
|
|
1212
|
+
try {
|
|
1213
|
+
provided = await reader(cookieOptions);
|
|
1214
|
+
} catch {
|
|
1215
|
+
throw new Error("the explicitly selected browser cookie source could not be read");
|
|
1216
|
+
}
|
|
1217
|
+
const filtered = filterCookieProviderResult(provided, url);
|
|
1218
|
+
if (!filtered.validShape) throw new Error("the selected browser cookie provider returned malformed data");
|
|
1219
|
+
if (filtered.cookies.length === 0) {
|
|
1220
|
+
throw new Error(filtered.rejected === 0
|
|
1221
|
+
? "no matching cookies were found in the explicitly selected browser"
|
|
1222
|
+
: `no usable origin-scoped cookies were found; rejected ${filtered.rejected} malformed, expired, or out-of-scope record(s)`);
|
|
1223
|
+
}
|
|
1224
|
+
const warnings: string[] = [];
|
|
1225
|
+
if (filtered.rejected > 0) {
|
|
1226
|
+
warnings.push(`Ignored ${filtered.rejected} malformed, expired, or out-of-scope browser cookie record(s).`);
|
|
1227
|
+
}
|
|
1228
|
+
if (filtered.providerWarningCount > 0) {
|
|
1229
|
+
warnings.push(`The browser cookie provider reported ${filtered.providerWarningCount} non-fatal warning(s).`);
|
|
1230
|
+
}
|
|
1231
|
+
return { cookies: filtered.cookies, warnings };
|
|
1232
|
+
};
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
/** Build an injectable HTTP-header reader from the same attribute-preserving filter. */
|
|
1236
|
+
export function createCookieHeaderReader(reader: CookieStoreReader): CookieHeaderReader {
|
|
1237
|
+
const records = createCookieRecordReader(reader);
|
|
1238
|
+
return async (options, url) => {
|
|
1239
|
+
const result = await records(options, url);
|
|
1240
|
+
return { header: renderCookieHeader(result.cookies), warnings: result.warnings };
|
|
1241
|
+
};
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
export const acquireCookieRecords = createCookieRecordReader((options) => getCookies(options));
|
|
1245
|
+
export const acquireCookieHeader: CookieHeaderReader = async (options, url) => {
|
|
1246
|
+
const result = await acquireCookieRecords(options, url);
|
|
1247
|
+
return { header: renderCookieHeader(result.cookies), warnings: result.warnings };
|
|
1248
|
+
};
|
|
1249
|
+
|
|
1250
|
+
async function readStdinBounded(maxBytes: number): Promise<string> {
|
|
1251
|
+
return readBoundedStream(Bun.stdin.stream(), maxBytes);
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
export async function acquireFile(options: CaptureArguments): Promise<AcquiredPage> {
|
|
1255
|
+
if (options.htmlFile === undefined) throw new Error("file capture requires --html <path|->");
|
|
1256
|
+
const body = options.htmlFile === "-"
|
|
1257
|
+
? await readStdinBounded(options.maxHtmlBytes)
|
|
1258
|
+
: (() => {
|
|
1259
|
+
const stats = statSync(options.htmlFile);
|
|
1260
|
+
if (!stats.isFile()) throw new Error(`HTML input is not a regular file: ${options.htmlFile}`);
|
|
1261
|
+
if (stats.size > options.maxHtmlBytes) {
|
|
1262
|
+
throw new Error(`HTML input is ${stats.size} bytes; limit is ${options.maxHtmlBytes}`);
|
|
1263
|
+
}
|
|
1264
|
+
return readFileSync(options.htmlFile, "utf8");
|
|
1265
|
+
})();
|
|
1266
|
+
return {
|
|
1267
|
+
body,
|
|
1268
|
+
contentType: "text/html; charset=utf-8",
|
|
1269
|
+
finalUrl: captureUrl(options),
|
|
1270
|
+
method: "file",
|
|
1271
|
+
warnings: options.htmlFile === "-" ? [] : [`Parsed rendered HTML from ${basename(options.htmlFile)}.`],
|
|
1272
|
+
};
|
|
1273
|
+
}
|