@taurgis/bonsai 3.2.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/agents/agents/salesforce-docs-researcher.agent.md +4 -4
- package/bin/cli.mjs +7 -4
- package/dist/base-command.d.ts +37 -0
- package/dist/base-command.js +97 -6
- package/dist/commands/config/base.js +1 -0
- package/dist/commands/config/get.js +2 -2
- package/dist/commands/config/list.js +2 -2
- package/dist/commands/config/set.js +2 -2
- package/dist/commands/config/unset.js +2 -2
- package/dist/commands/fetch.js +4 -0
- package/dist/commands/inspect.js +11 -2
- package/dist/commands/list.d.ts +12 -9
- package/dist/commands/list.js +67 -39
- package/dist/commands/prune.d.ts +2 -0
- package/dist/commands/prune.js +35 -4
- package/dist/commands/status.js +10 -0
- package/dist/hooks/command-not-found/suggest.d.ts +1 -1
- package/dist/hooks/command-not-found/suggest.js +12 -2
- package/dist/lib/argv.d.ts +5 -0
- package/dist/lib/argv.js +27 -5
- package/dist/lib/cli-flag-manifest.d.ts +7 -0
- package/dist/lib/cli-flag-manifest.js +7 -0
- package/dist/lib/cli-presentation.d.ts +6 -4
- package/dist/lib/cli-presentation.js +7 -4
- package/dist/lib/cli-result-types.d.ts +25 -1
- package/dist/lib/config/index.d.ts +2 -2
- package/dist/lib/config/index.js +2 -2
- package/dist/lib/config/io.d.ts +9 -0
- package/dist/lib/config/io.js +71 -21
- package/dist/lib/config/schema.d.ts +2 -0
- package/dist/lib/config/schema.js +7 -0
- package/dist/lib/envelope.js +11 -4
- package/dist/lib/prune-flags.d.ts +2 -0
- package/dist/lib/prune-flags.js +18 -6
- package/dist/lib/research/artifact-index.js +7 -1
- package/dist/lib/research/artifact.js +20 -2
- package/dist/lib/research/browser.d.ts +12 -0
- package/dist/lib/research/browser.js +63 -8
- package/dist/lib/research/cli-io.d.ts +7 -0
- package/dist/lib/research/fetch-command-service.js +80 -23
- package/dist/lib/research/fetcher.js +3 -1
- package/dist/lib/research/import-command-service.js +22 -1
- package/dist/lib/research/metadata-filters.d.ts +18 -0
- package/dist/lib/research/metadata-filters.js +37 -0
- package/dist/lib/research/metadata-flags.d.ts +34 -0
- package/dist/lib/research/metadata-flags.js +45 -0
- package/dist/lib/research/prompt-injection.d.ts +2 -1
- package/dist/lib/research/prompt-injection.js +18 -2
- package/dist/lib/research/resolve-target.d.ts +2 -0
- package/dist/lib/research/resolve-target.js +1 -1
- package/dist/lib/research/storage.d.ts +4 -7
- package/dist/lib/research/storage.js +26 -25
- package/dist/lib/text.d.ts +14 -8
- package/dist/lib/text.js +21 -9
- package/oclif.manifest.json +120 -12
- package/package.json +2 -1
package/dist/lib/config/io.js
CHANGED
|
@@ -5,43 +5,93 @@ import { KEY_META, ALL_KEYS } from './schema.js';
|
|
|
5
5
|
/** Global, user-level config lives in the OCLIF config dir; project config sits at the repo root. */
|
|
6
6
|
export const USER_CONFIG_FILENAME = 'config.json';
|
|
7
7
|
export const PROJECT_CONFIG_FILENAME = '.bonsai.json';
|
|
8
|
-
/**
|
|
9
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Parse a config file, keeping only known keys with valid values. Unparseable JSON, a non-object
|
|
10
|
+
* top level, or an invalid value for a known key degrades that part to "absent" rather than
|
|
11
|
+
* throwing — callers that only need values (`readUserConfig`/`readProjectConfig`) ignore
|
|
12
|
+
* `warnings`; {@link invalidConfigFileWarnings} surfaces them so the degradation is never silent.
|
|
13
|
+
*/
|
|
14
|
+
function parseConfigFile(filePath, raw, scopeFlag, bin) {
|
|
15
|
+
let parsed;
|
|
10
16
|
try {
|
|
11
|
-
|
|
12
|
-
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
|
|
13
|
-
return {};
|
|
14
|
-
const obj = parsed;
|
|
15
|
-
const result = {};
|
|
16
|
-
for (const key of ALL_KEYS) {
|
|
17
|
-
const val = obj[key];
|
|
18
|
-
if (val !== undefined && KEY_META[key].isValid(val)) {
|
|
19
|
-
Object.assign(result, { [key]: val });
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
return result;
|
|
17
|
+
parsed = JSON.parse(raw);
|
|
23
18
|
}
|
|
24
19
|
catch {
|
|
25
|
-
return {
|
|
20
|
+
return {
|
|
21
|
+
values: {},
|
|
22
|
+
warnings: [
|
|
23
|
+
`Ignoring ${filePath}: not valid JSON. Using the configured value or default instead. ` +
|
|
24
|
+
`Fix the file by hand, or overwrite it with ${bin} config set <key> <value> ${scopeFlag}.`,
|
|
25
|
+
],
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
|
|
29
|
+
return {
|
|
30
|
+
values: {},
|
|
31
|
+
warnings: [
|
|
32
|
+
`Ignoring ${filePath}: expected a JSON object at the top level. Using the configured value or default instead.`,
|
|
33
|
+
],
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
const obj = parsed;
|
|
37
|
+
const values = {};
|
|
38
|
+
const warnings = [];
|
|
39
|
+
for (const key of ALL_KEYS) {
|
|
40
|
+
const val = obj[key];
|
|
41
|
+
if (val === undefined)
|
|
42
|
+
continue;
|
|
43
|
+
if (KEY_META[key].isValid(val)) {
|
|
44
|
+
Object.assign(values, { [key]: val });
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
const validValues = KEY_META[key].values;
|
|
48
|
+
const validValuesHint = validValues ? ` Valid values: ${validValues.join(', ')}.` : '';
|
|
49
|
+
warnings.push(`Ignoring "${key}" in ${filePath}: invalid value ${JSON.stringify(val)}.${validValuesHint} ` +
|
|
50
|
+
`Using the configured value or default instead.`);
|
|
51
|
+
}
|
|
26
52
|
}
|
|
53
|
+
return { values, warnings };
|
|
27
54
|
}
|
|
28
|
-
|
|
55
|
+
// Only the "fix it" suggestion in a warning message names the binary; every value-only caller
|
|
56
|
+
// below (readUserConfig/readProjectConfig, and transitively write*Config) discards warnings
|
|
57
|
+
// entirely, so this default never reaches a user — it's a placeholder for an unused code path,
|
|
58
|
+
// not a claim about the real bin name. invalidConfigFileWarnings passes the real `this.config.bin`.
|
|
59
|
+
const DEFAULT_BIN = 'bonsai';
|
|
60
|
+
function readConfigFile(filePath, scopeFlag, bin = DEFAULT_BIN) {
|
|
29
61
|
if (!existsSync(filePath))
|
|
30
|
-
return {};
|
|
62
|
+
return { values: {}, warnings: [] };
|
|
63
|
+
let raw;
|
|
31
64
|
try {
|
|
32
|
-
|
|
65
|
+
raw = readFileSync(filePath, 'utf-8');
|
|
33
66
|
}
|
|
34
67
|
catch {
|
|
35
|
-
return {};
|
|
68
|
+
return { values: {}, warnings: [] };
|
|
36
69
|
}
|
|
70
|
+
return parseConfigFile(filePath, raw, scopeFlag, bin);
|
|
37
71
|
}
|
|
38
72
|
export function readUserConfig(configDir) {
|
|
39
73
|
if (!configDir)
|
|
40
74
|
return {};
|
|
41
|
-
return readConfigFile(join(configDir, USER_CONFIG_FILENAME));
|
|
75
|
+
return readConfigFile(join(configDir, USER_CONFIG_FILENAME), '--global').values;
|
|
42
76
|
}
|
|
43
77
|
export function readProjectConfig(cwd) {
|
|
44
|
-
return readConfigFile(join(cwd, PROJECT_CONFIG_FILENAME));
|
|
78
|
+
return readConfigFile(join(cwd, PROJECT_CONFIG_FILENAME), '--local').values;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Warnings for the user and project config files, so a corrupted or hand-edited file that would
|
|
82
|
+
* otherwise silently degrade to `{}` (or drop just the offending key) is never a silent surprise —
|
|
83
|
+
* mirrors {@link invalidEnvOverrideWarnings} for env vars. Empty when both files are missing,
|
|
84
|
+
* unreadable, or valid.
|
|
85
|
+
*
|
|
86
|
+
* @param bin - CLI binary name (`this.config.bin`) for the "fix it" suggestion in a warning.
|
|
87
|
+
*/
|
|
88
|
+
export function invalidConfigFileWarnings(configDir, cwd, bin) {
|
|
89
|
+
const warnings = [];
|
|
90
|
+
if (configDir) {
|
|
91
|
+
warnings.push(...readConfigFile(join(configDir, USER_CONFIG_FILENAME), '--global', bin).warnings);
|
|
92
|
+
}
|
|
93
|
+
warnings.push(...readConfigFile(join(cwd, PROJECT_CONFIG_FILENAME), '--local', bin).warnings);
|
|
94
|
+
return warnings;
|
|
45
95
|
}
|
|
46
96
|
function writeAtomically(filePath, data) {
|
|
47
97
|
atomicWriteFile(filePath, JSON.stringify(data, null, 2) + '\n');
|
|
@@ -29,3 +29,5 @@ export declare function isKnownKey(key: string): key is ConfigKey;
|
|
|
29
29
|
export declare function suggestKey(input: string): ConfigKey | undefined;
|
|
30
30
|
/** Comma-joined valid key list for error messages, kept in one place. */
|
|
31
31
|
export declare function validKeysHint(): string;
|
|
32
|
+
/** Valid keys with their accepted values, e.g. "storage (global|project), summary (…)", for `--help` descriptions. */
|
|
33
|
+
export declare function keyValuesHint(): string;
|
|
@@ -40,3 +40,10 @@ export function suggestKey(input) {
|
|
|
40
40
|
export function validKeysHint() {
|
|
41
41
|
return ALL_KEYS.join(', ');
|
|
42
42
|
}
|
|
43
|
+
/** Valid keys with their accepted values, e.g. "storage (global|project), summary (…)", for `--help` descriptions. */
|
|
44
|
+
export function keyValuesHint() {
|
|
45
|
+
return ALL_KEYS.map((key) => {
|
|
46
|
+
const { values } = KEY_META[key];
|
|
47
|
+
return values ? `${key} (${values.join('|')})` : key;
|
|
48
|
+
}).join(', ');
|
|
49
|
+
}
|
package/dist/lib/envelope.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { EXIT_RUNTIME_FAILURE } from './cli-error-policy.js';
|
|
2
|
+
import { pluralize } from './text.js';
|
|
2
3
|
/** Map oclif parse-time errors to stable Bonsai codes when the throw site did not set one. */
|
|
3
4
|
export function stableErrorCodeFrom(err) {
|
|
4
5
|
const e = err;
|
|
@@ -117,8 +118,9 @@ export function enrichCacheMissEnvelope(envelope, data, bin) {
|
|
|
117
118
|
exitCode: EXIT_RUNTIME_FAILURE,
|
|
118
119
|
message: (first, failures) => {
|
|
119
120
|
const url = first.normalizedUrl ?? '';
|
|
120
|
-
|
|
121
|
-
|
|
121
|
+
const otherCount = failures.length - 1;
|
|
122
|
+
return otherCount > 0
|
|
123
|
+
? `Cache miss for ${url} and ${otherCount} other ${pluralize(otherCount, 'URL', 'URLs')}`
|
|
122
124
|
: `Cache miss for ${url}`;
|
|
123
125
|
},
|
|
124
126
|
// A URL that is already a secondary source of a different cached artifact must not be pointed
|
|
@@ -141,8 +143,13 @@ export function enrichRowErrorEnvelope(envelope, data) {
|
|
|
141
143
|
return row.error ?? null;
|
|
142
144
|
},
|
|
143
145
|
code: (first) => first.code ?? 'FETCH_FAILED',
|
|
144
|
-
message: (first, failures) =>
|
|
145
|
-
|
|
146
|
+
message: (first, failures) => {
|
|
147
|
+
const otherCount = failures.length - 1;
|
|
148
|
+
const suffix = otherCount > 0
|
|
149
|
+
? `\n…and ${otherCount} other URL ${pluralize(otherCount, 'failure', 'failures')}`
|
|
150
|
+
: '';
|
|
151
|
+
return (first.message ?? '') + suffix;
|
|
152
|
+
},
|
|
146
153
|
suggestions: (failures) => failures[0]?.suggestions,
|
|
147
154
|
ref: (first) => first.ref,
|
|
148
155
|
exitCode: EXIT_RUNTIME_FAILURE,
|
package/dist/lib/prune-flags.js
CHANGED
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import { emptyUrlFilterError } from './research/url.js';
|
|
2
2
|
import { durationFlagError } from './research/freshness.js';
|
|
3
|
+
import { emptyTopicFilterError, emptyTagsFilterError } from './research/metadata-filters.js';
|
|
3
4
|
function missingFilterError(input) {
|
|
4
5
|
// Treat an explicitly passed empty string as "filter was attempted" so `--older-than ''`
|
|
5
6
|
// is not misreported as MISSING_FILTER (durationError rejects it as INVALID_DURATION).
|
|
7
|
+
// `tags` is the one array-valued flag here: oclif never parses a bare `--tags` into `[]` (a
|
|
8
|
+
// multiple string flag requires a value per occurrence), so a non-empty array is the correct
|
|
9
|
+
// "was this filter attempted" test, mirroring the `!== undefined` check used for every scalar flag.
|
|
6
10
|
if (input.olderThan !== undefined ||
|
|
7
11
|
input.inactive !== undefined ||
|
|
8
12
|
input.artifactType !== undefined ||
|
|
9
|
-
input.url !== undefined
|
|
13
|
+
input.url !== undefined ||
|
|
14
|
+
input.topic !== undefined ||
|
|
15
|
+
(input.tags?.length ?? 0) > 0) {
|
|
10
16
|
return null;
|
|
11
17
|
}
|
|
12
18
|
return {
|
|
13
|
-
message: 'Must specify at least one pruning filter: --older-than, --inactive, --artifact-type, or --
|
|
19
|
+
message: 'Must specify at least one pruning filter: --older-than, --inactive, --artifact-type, --url, --topic, or --tags.',
|
|
14
20
|
code: 'MISSING_FILTER',
|
|
15
21
|
suggestions: [`Preview age-based pruning: ${input.bin} prune --older-than 30d --dry-run`],
|
|
16
22
|
};
|
|
@@ -41,10 +47,14 @@ function mutationSafetyError(input) {
|
|
|
41
47
|
if (!input.readOnly && !input.dryRun && !input.yes) {
|
|
42
48
|
const olderThanPart = input.olderThan ? ` --older-than ${input.olderThan}` : '';
|
|
43
49
|
const urlPart = input.url ? ` --url "${input.url}"` : '';
|
|
50
|
+
const topicPart = input.topic ? ` --topic "${input.topic}"` : '';
|
|
51
|
+
const tagsPart = (input.tags ?? []).map((tag) => ` --tags "${tag}"`).join('');
|
|
44
52
|
return {
|
|
45
53
|
message: 'Safety check: use --yes to confirm pruning, or --dry-run to preview files that would be deleted.',
|
|
46
54
|
code: 'SAFETY_CHECK_REQUIRED',
|
|
47
|
-
suggestions: [
|
|
55
|
+
suggestions: [
|
|
56
|
+
`Preview first: ${input.bin} prune --dry-run${olderThanPart}${urlPart}${topicPart}${tagsPart}`,
|
|
57
|
+
],
|
|
48
58
|
};
|
|
49
59
|
}
|
|
50
60
|
return null;
|
|
@@ -66,9 +76,11 @@ function durationError(input) {
|
|
|
66
76
|
* @returns Error descriptor for `this.error`, or `null`.
|
|
67
77
|
*/
|
|
68
78
|
export function pruneFlagError(input) {
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
79
|
+
const flagValueErr = emptyUrlFilterError(input.url) ??
|
|
80
|
+
emptyTopicFilterError(input.topic) ??
|
|
81
|
+
emptyTagsFilterError(input.tags);
|
|
82
|
+
if (flagValueErr)
|
|
83
|
+
return { message: flagValueErr, code: 'INVALID_FLAG_VALUE' };
|
|
72
84
|
// Duration before missing-filter so `--older-than ''` reports INVALID_DURATION, not MISSING_FILTER.
|
|
73
85
|
return durationError(input) ?? missingFilterError(input) ?? mutationSafetyError(input);
|
|
74
86
|
}
|
|
@@ -39,7 +39,13 @@ function getOrUpdateIndexEntry(filePath, cached, onChanged) {
|
|
|
39
39
|
return cached;
|
|
40
40
|
}
|
|
41
41
|
try {
|
|
42
|
-
const
|
|
42
|
+
const artifact = parseArtifactShallow(readFileSync(filePath, 'utf-8'));
|
|
43
|
+
// Mirrors the same guard in storage.ts's scanCacheDir: parseArtifact never throws on a foreign
|
|
44
|
+
// or garbled *.md file that merely has `---`/`---` fences, it just defaults cache_key to ''. Treat
|
|
45
|
+
// that the same as a parse failure so it's skipped rather than indexed as a phantom entry.
|
|
46
|
+
if (!artifact.metadata.cache_key)
|
|
47
|
+
return undefined;
|
|
48
|
+
const entry = { sig, artifact };
|
|
43
49
|
onChanged();
|
|
44
50
|
return entry;
|
|
45
51
|
}
|
|
@@ -1,12 +1,30 @@
|
|
|
1
|
+
import { CONTROL_CHAR_PATTERN } from '../text.js';
|
|
2
|
+
// Frontmatter is one field per line. An embedded newline (or tab, treated the same way as
|
|
3
|
+
// legitimate Markdown whitespace elsewhere in this codebase — see sanitizeForTerminal in
|
|
4
|
+
// ../text.ts and stripUnsafeControlChars in ./prompt-injection.ts) in a free-form value (e.g. an
|
|
5
|
+
// agent-supplied --topic or --tags) would inject a bogus extra line — at best a dropped stray
|
|
6
|
+
// line, at worst a fake `key: value` pair, or a bare `---` that closes the frontmatter fence early
|
|
7
|
+
// and splices the remaining metadata fields (and even the closing fence and body sections) into
|
|
8
|
+
// what parseArtifact then treats as free-text content. Collapsing that whitespace to a space
|
|
9
|
+
// (rather than deleting it, which would silently merge adjacent words) keeps every value confined
|
|
10
|
+
// to the single line the format requires, regardless of where the string originated. Other control
|
|
11
|
+
// characters (e.g. a raw ANSI escape byte, whether from an agent-supplied flag or extracted page
|
|
12
|
+
// text) are stripped outright: the cache file is plain text meant to be `cat`ed or displayed by
|
|
13
|
+
// `inspect`/`list`, and an uncaught escape sequence there would reach a real terminal — unlike
|
|
14
|
+
// --json output, which JSON-escapes control characters by construction.
|
|
15
|
+
function sanitizeFrontmatterLine(value) {
|
|
16
|
+
return value.replace(/[\t\r\n]+/g, ' ').replace(CONTROL_CHAR_PATTERN, '');
|
|
17
|
+
}
|
|
1
18
|
function serializeArrayField(lines, name, items) {
|
|
2
19
|
lines.push(`${name}:`);
|
|
3
20
|
for (const item of items) {
|
|
4
|
-
lines.push(` - ${item}`);
|
|
21
|
+
lines.push(` - ${sanitizeFrontmatterLine(item)}`);
|
|
5
22
|
}
|
|
6
23
|
}
|
|
7
24
|
// A scalar frontmatter line, blanking null/undefined. Using `??` (not `||`) preserves a numeric 0.
|
|
8
25
|
function serializeScalarField(lines, name, value) {
|
|
9
|
-
|
|
26
|
+
const safeValue = typeof value === 'string' ? sanitizeFrontmatterLine(value) : value;
|
|
27
|
+
lines.push(`${name}: ${safeValue ?? ''}`);
|
|
10
28
|
}
|
|
11
29
|
/**
|
|
12
30
|
* Serializes artifact metadata to standard YAML frontmatter text.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ChildProcess } from 'node:child_process';
|
|
1
2
|
export interface BrowserFetchOptions {
|
|
2
3
|
timeoutMs?: number;
|
|
3
4
|
bodyLimitBytes?: number;
|
|
@@ -25,6 +26,16 @@ export declare class CdpClient {
|
|
|
25
26
|
}
|
|
26
27
|
export declare function findChromePath(): string;
|
|
27
28
|
export declare function buildChromeArgs(): string[];
|
|
29
|
+
/**
|
|
30
|
+
* Kill Chrome and every child process it spawned (zygote, gpu-process, renderer, crashpad).
|
|
31
|
+
* `chromeProcess.kill()` alone only signals that one PID — per Node's child_process docs, a signal
|
|
32
|
+
* sent to a child is not propagated to *that child's own* children on Linux, so Chrome's
|
|
33
|
+
* multi-process tree survives a plain kill on every rendered fetch, not just an interrupted one.
|
|
34
|
+
* Spawning with `detached: true` makes Chrome the leader of its own process group, so signaling the
|
|
35
|
+
* negative PID reaches the whole tree in one call. Falls back to a direct kill if group-kill fails
|
|
36
|
+
* (already exited, or a platform without POSIX process groups).
|
|
37
|
+
*/
|
|
38
|
+
export declare function killChromeTree(chromeProcess: ChildProcess): void;
|
|
28
39
|
export declare function waitForLoad(client: CdpClient, sessionId: string, timeoutMs: number, settleMs?: number): Promise<void>;
|
|
29
40
|
export declare const BLOCKED_ASSET_URLS: string[];
|
|
30
41
|
export interface CdpPage {
|
|
@@ -106,6 +117,7 @@ export declare const SANDBOX_EGRESS_ERROR_MARKER = "sandboxed execution environm
|
|
|
106
117
|
* against the environment directly and fall back to a manual fetch if it's still unreachable.
|
|
107
118
|
*/
|
|
108
119
|
export declare function describeNavigationFailure(errorText: string, url: string): Error;
|
|
120
|
+
export declare function isTransientBrowserError(err: unknown): boolean;
|
|
109
121
|
/**
|
|
110
122
|
* Navigates an already-open CDP page to `currentUrl` and captures the rendered HTML, honoring the
|
|
111
123
|
* openCdpPage Fetch-domain safety guard at every point a blocked navigation could otherwise be
|
|
@@ -130,13 +130,63 @@ export function buildChromeArgs() {
|
|
|
130
130
|
// CI runners occasionally starve headless Chrome's startup (disk/CPU contention from concurrent
|
|
131
131
|
// jobs), so this is generous enough to absorb that rather than a "typical" local startup time.
|
|
132
132
|
const CHROME_STARTUP_TIMEOUT_MS = 20_000;
|
|
133
|
+
// Node's default SIGINT/SIGTERM disposition kills the process immediately, without waiting for the
|
|
134
|
+
// async `finally { page.close() }` in fetchRenderedHtmlOnce to run — so an interrupted --rendered
|
|
135
|
+
// fetch otherwise abandons a whole Chrome process tree (main + zygote/gpu/renderer children). The
|
|
136
|
+
// plain 'exit' event covers ordinary completion (belt-and-braces alongside close()'s own cleanup);
|
|
137
|
+
// a `once` signal listener runs the same cleanup, then re-sends the signal to this process. Because
|
|
138
|
+
// `once` already removed itself, that re-delivery has no listener left and Node's own default
|
|
139
|
+
// disposition takes over — the process still exits with the normal 128+signum code, with no exit-code
|
|
140
|
+
// bookkeeping of our own.
|
|
141
|
+
const liveChromeProcesses = new Set();
|
|
142
|
+
let chromeCleanupRegistered = false;
|
|
143
|
+
function killLiveChromeProcesses() {
|
|
144
|
+
for (const chromeProcess of liveChromeProcesses)
|
|
145
|
+
killChromeTree(chromeProcess);
|
|
146
|
+
}
|
|
147
|
+
function ensureChromeCleanupRegistered() {
|
|
148
|
+
if (chromeCleanupRegistered)
|
|
149
|
+
return;
|
|
150
|
+
chromeCleanupRegistered = true;
|
|
151
|
+
process.once('exit', killLiveChromeProcesses);
|
|
152
|
+
for (const signal of ['SIGINT', 'SIGTERM']) {
|
|
153
|
+
process.once(signal, () => {
|
|
154
|
+
killLiveChromeProcesses();
|
|
155
|
+
process.kill(process.pid, signal);
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Kill Chrome and every child process it spawned (zygote, gpu-process, renderer, crashpad).
|
|
161
|
+
* `chromeProcess.kill()` alone only signals that one PID — per Node's child_process docs, a signal
|
|
162
|
+
* sent to a child is not propagated to *that child's own* children on Linux, so Chrome's
|
|
163
|
+
* multi-process tree survives a plain kill on every rendered fetch, not just an interrupted one.
|
|
164
|
+
* Spawning with `detached: true` makes Chrome the leader of its own process group, so signaling the
|
|
165
|
+
* negative PID reaches the whole tree in one call. Falls back to a direct kill if group-kill fails
|
|
166
|
+
* (already exited, or a platform without POSIX process groups).
|
|
167
|
+
*/
|
|
168
|
+
export function killChromeTree(chromeProcess) {
|
|
169
|
+
if (typeof chromeProcess.pid === 'number') {
|
|
170
|
+
try {
|
|
171
|
+
process.kill(-chromeProcess.pid, 'SIGKILL');
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// Fall through to a best-effort direct kill below.
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
chromeProcess.kill('SIGKILL');
|
|
179
|
+
}
|
|
133
180
|
async function spawnChrome(chromePath) {
|
|
134
|
-
|
|
181
|
+
ensureChromeCleanupRegistered();
|
|
182
|
+
const chromeProcess = spawn(chromePath, buildChromeArgs(), { detached: true });
|
|
183
|
+
liveChromeProcesses.add(chromeProcess);
|
|
184
|
+
chromeProcess.once('exit', () => liveChromeProcesses.delete(chromeProcess));
|
|
135
185
|
try {
|
|
136
186
|
const wsUrl = await new Promise((resolve, reject) => {
|
|
137
187
|
let output = '';
|
|
138
188
|
const timeoutId = setTimeout(() => {
|
|
139
|
-
chromeProcess
|
|
189
|
+
killChromeTree(chromeProcess);
|
|
140
190
|
reject(new Error('Timed out waiting for Chrome to start.'));
|
|
141
191
|
}, CHROME_STARTUP_TIMEOUT_MS);
|
|
142
192
|
const onData = (data) => {
|
|
@@ -168,7 +218,7 @@ async function spawnChrome(chromePath) {
|
|
|
168
218
|
return { chromeProcess, wsUrl };
|
|
169
219
|
}
|
|
170
220
|
catch (err) {
|
|
171
|
-
chromeProcess
|
|
221
|
+
killChromeTree(chromeProcess);
|
|
172
222
|
throw err;
|
|
173
223
|
}
|
|
174
224
|
}
|
|
@@ -286,7 +336,7 @@ export async function openCdpPage() {
|
|
|
286
336
|
}
|
|
287
337
|
catch { }
|
|
288
338
|
client.close();
|
|
289
|
-
chromeProcess
|
|
339
|
+
killChromeTree(chromeProcess);
|
|
290
340
|
};
|
|
291
341
|
const takeBlockedNavigationError = () => {
|
|
292
342
|
const err = blockedNavigationError;
|
|
@@ -297,7 +347,7 @@ export async function openCdpPage() {
|
|
|
297
347
|
}
|
|
298
348
|
catch (err) {
|
|
299
349
|
client.close();
|
|
300
|
-
chromeProcess
|
|
350
|
+
killChromeTree(chromeProcess);
|
|
301
351
|
throw err;
|
|
302
352
|
}
|
|
303
353
|
}
|
|
@@ -414,8 +464,11 @@ export function describeNavigationFailure(errorText, url) {
|
|
|
414
464
|
`or bypassing the proxy: confirm "${host}" is reachable from this environment (e.g. test the ` +
|
|
415
465
|
`proxy directly), or fetch the page from a network with access and import it instead.`);
|
|
416
466
|
}
|
|
417
|
-
// Chrome can emit
|
|
418
|
-
|
|
467
|
+
// ERR_CERT_VERIFIER_CHANGED: Chrome can emit this on first navigation in headless CI when the cert
|
|
468
|
+
// verifier reloads mid-run. ERR_CONNECTION_CLOSED is Chromium's code for a plain remote TCP FIN
|
|
469
|
+
// (net_error_list.h), which CI runners can hit as a one-off network blip against a real host
|
|
470
|
+
// rather than a real block.
|
|
471
|
+
const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED', 'ERR_CONNECTION_CLOSED'];
|
|
419
472
|
// Resource contention in CI (concurrent jobs competing for CPU/disk) can make Chrome itself fail
|
|
420
473
|
// to come up in time, or die before its devtools websocket URL is printed. Neither reflects a
|
|
421
474
|
// problem with the target page, so it's worth a fresh attempt rather than failing the whole fetch.
|
|
@@ -423,7 +476,9 @@ const TRANSIENT_CHROME_STARTUP_MESSAGES = [
|
|
|
423
476
|
'Timed out waiting for Chrome to start.',
|
|
424
477
|
'Chrome exited prematurely',
|
|
425
478
|
];
|
|
426
|
-
|
|
479
|
+
// Exported so tests can assert the transient-error classification directly rather than only
|
|
480
|
+
// indirectly through a live fetchRenderedHtml retry, which needs a real Chrome navigation to fail.
|
|
481
|
+
export function isTransientBrowserError(err) {
|
|
427
482
|
const message = err instanceof Error ? err.message : String(err);
|
|
428
483
|
return (TRANSIENT_NAV_ERROR_CODES.some((code) => message.includes(code)) ||
|
|
429
484
|
TRANSIENT_CHROME_STARTUP_MESSAGES.some((text) => message.includes(text)));
|
|
@@ -18,6 +18,13 @@ export interface CliIo {
|
|
|
18
18
|
warn(msg: string): void;
|
|
19
19
|
log(msg: string): void;
|
|
20
20
|
error(msg: string, opts: CliErrorOptions): never;
|
|
21
|
+
/**
|
|
22
|
+
* Print one row of a batch failure with the same "Error: … / Code: … / Try this: …" rendering as
|
|
23
|
+
* {@link error}, but without exiting the process — a batch (`fetch url1 url2`) must keep
|
|
24
|
+
* processing the remaining rows after one fails. Using `warn` for this mislabels a real failure
|
|
25
|
+
* as non-fatal even though the row still flips the command's exit code and JSON `ok` to false.
|
|
26
|
+
*/
|
|
27
|
+
errorRow(msg: string, opts: Omit<CliErrorOptions, 'exit'>): void;
|
|
21
28
|
}
|
|
22
29
|
/**
|
|
23
30
|
* Single exit point for a URL that failed normalization, shared by BaseCommand and the command
|
|
@@ -3,10 +3,10 @@ import { mkdtempSync, rmSync } from 'node:fs';
|
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { finalizeBatch } from '../batch.js';
|
|
6
|
-
import { formatErrorForJson } from '../envelope.js';
|
|
7
6
|
import { loadSummaryLevel } from '../config/index.js';
|
|
8
7
|
import { buildFetchFailureFromCaught, buildFetchFailureResult, buildFetchResultData, describeError, extractionQualityWarnings, FETCH_STATUS_LABEL, fetchFailureGuidance, reportCacheStatus, } from './fetch-result.js';
|
|
9
8
|
import { durationFlagError, evaluateFreshnessWithMaxAge } from './freshness.js';
|
|
9
|
+
import { metadataLengthError, metadataNewlineError } from './metadata-flags.js';
|
|
10
10
|
import { fetchRenderedHtml } from './browser.js';
|
|
11
11
|
import { capturePage } from './capture.js';
|
|
12
12
|
import { persistSectionArtifacts } from './docs/section-artifacts.js';
|
|
@@ -15,8 +15,10 @@ import { applyAutoTags } from './keywords.js';
|
|
|
15
15
|
import { persistArtifact } from './persist-artifact.js';
|
|
16
16
|
import { createArtifactFromFetch, revalidateCache, } from './revalidate.js';
|
|
17
17
|
import { resolveResearchTarget } from './resolve-target.js';
|
|
18
|
+
import { locateArtifact } from './storage.js';
|
|
18
19
|
import { failInvalidUrl } from './cli-io.js';
|
|
19
20
|
import { batchSeparator } from '../cache-view.js';
|
|
21
|
+
import { formatTip } from '../text.js';
|
|
20
22
|
import { EXIT_STALE_SERVED } from '../cli-error-policy.js';
|
|
21
23
|
import { detectSite } from '../../sites/index.js';
|
|
22
24
|
import { applySiteFetchProvenance } from '../../sites/types.js';
|
|
@@ -34,6 +36,22 @@ export function validateFetchCommandFlags(io, flags) {
|
|
|
34
36
|
if (msg)
|
|
35
37
|
io.error(msg, { exit: 2, code: 'INVALID_DURATION' });
|
|
36
38
|
}
|
|
39
|
+
const newlineErr = metadataNewlineError(flags);
|
|
40
|
+
if (newlineErr) {
|
|
41
|
+
io.error(newlineErr, {
|
|
42
|
+
exit: 2,
|
|
43
|
+
code: 'INVALID_METADATA_VALUE',
|
|
44
|
+
suggestions: ['Remove line breaks from the value.'],
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
const lengthErr = metadataLengthError(flags);
|
|
48
|
+
if (lengthErr) {
|
|
49
|
+
io.error(lengthErr, {
|
|
50
|
+
exit: 2,
|
|
51
|
+
code: 'INVALID_METADATA_VALUE',
|
|
52
|
+
suggestions: ['Shorten the value.'],
|
|
53
|
+
});
|
|
54
|
+
}
|
|
37
55
|
// --force skips cache lookup; --allow-stale only applies when serving a stale entry after a
|
|
38
56
|
// failed revalidation. Together they are a no-op combination that looks intentional -- reject it.
|
|
39
57
|
if (flags.force && flags.allowStale) {
|
|
@@ -112,7 +130,7 @@ async function executeCacheHit(cached, targetDir, run, secure) {
|
|
|
112
130
|
redirectedToGlobal: Boolean(revalidationResult.redirectedToGlobal),
|
|
113
131
|
};
|
|
114
132
|
}
|
|
115
|
-
async function executeCacheMiss(normalizedUrl, cacheKey, run) {
|
|
133
|
+
async function executeCacheMiss(normalizedUrl, cacheKey, readRoots, run) {
|
|
116
134
|
const { currentTime, flags, summaryLevel } = run;
|
|
117
135
|
const siteModule = detectSite(normalizedUrl);
|
|
118
136
|
const useRendered = flags.rendered || Boolean(siteModule?.defaults?.rendered);
|
|
@@ -141,8 +159,13 @@ async function executeCacheMiss(normalizedUrl, cacheKey, run) {
|
|
|
141
159
|
currentTime,
|
|
142
160
|
summaryLevel,
|
|
143
161
|
});
|
|
144
|
-
|
|
145
|
-
|
|
162
|
+
// --force bypasses the normal revalidation-hit path (it always re-fetches instead of
|
|
163
|
+
// conditionally refreshing), so this "miss" may actually be overwriting a previously curated
|
|
164
|
+
// entry. Read-only lookup, purely to carry topic/tags forward like every other refresh path
|
|
165
|
+
// (revalidate.ts's preserveUserMetadata) — explicit --topic/--tags on this call still win.
|
|
166
|
+
const priorArtifact = locateArtifact(readRoots, cacheKey, true)?.artifact ?? null;
|
|
167
|
+
artifact.metadata.topic = flags.topic || priorArtifact?.metadata.topic || null;
|
|
168
|
+
artifact.metadata.tags = flags.tags?.length ? flags.tags : (priorArtifact?.metadata.tags ?? []);
|
|
146
169
|
artifact.metadata.site_module_id = siteModule?.id ?? null;
|
|
147
170
|
if (capture) {
|
|
148
171
|
applyCaptureMetadata(artifact, capture);
|
|
@@ -177,7 +200,7 @@ async function resolveArtifact(target, run) {
|
|
|
177
200
|
redirectedToGlobal: hit.redirectedToGlobal,
|
|
178
201
|
};
|
|
179
202
|
}
|
|
180
|
-
const artifact = await executeCacheMiss(normalizedUrl, cacheKey, run);
|
|
203
|
+
const artifact = await executeCacheMiss(normalizedUrl, cacheKey, roots.readRoots, run);
|
|
181
204
|
const write = persistArtifact({
|
|
182
205
|
roots,
|
|
183
206
|
cacheKey,
|
|
@@ -234,11 +257,17 @@ function failureRowOrRethrow(url, err, run) {
|
|
|
234
257
|
const row = err instanceof Errors.CLIError
|
|
235
258
|
? buildFetchFailureResult({ url, dryRun, err })
|
|
236
259
|
: buildFetchFailureFromCaught(io.bin, url, err, dryRun);
|
|
237
|
-
// Human batches only get the spinner "failed" label unless we echo the reason.
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
260
|
+
// Human batches only get the spinner "failed" label unless we echo the reason. Render it with
|
|
261
|
+
// the same "Error:" formatting as a single-URL failure (io.errorRow reuses oclif's own
|
|
262
|
+
// exit:false renderer) rather than io.warn, since a row failure still fails the command overall
|
|
263
|
+
// and should never read as merely a warning.
|
|
264
|
+
if (!io.json) {
|
|
265
|
+
io.errorRow(row.error.message, {
|
|
266
|
+
code: row.error.code,
|
|
267
|
+
suggestions: row.error.suggestions,
|
|
268
|
+
ref: row.error.ref,
|
|
269
|
+
});
|
|
270
|
+
}
|
|
242
271
|
return row;
|
|
243
272
|
}
|
|
244
273
|
async function fetchSingleTarget(url, run) {
|
|
@@ -252,6 +281,10 @@ async function fetchSingleTarget(url, run) {
|
|
|
252
281
|
url,
|
|
253
282
|
flagOverride: flags.storage,
|
|
254
283
|
lookup: !flags.force,
|
|
284
|
+
// `dryRun` already merges explicit --dry-run with global read-only/plan mode
|
|
285
|
+
// (BaseCommand.effectiveDryRun), so a corrupt entry hit while resolving the cache target is
|
|
286
|
+
// never archived (renamed) on disk during either kind of no-write fetch.
|
|
287
|
+
readOnly: dryRun,
|
|
255
288
|
});
|
|
256
289
|
}
|
|
257
290
|
catch (err) {
|
|
@@ -279,21 +312,45 @@ async function fetchSingleTarget(url, run) {
|
|
|
279
312
|
redirectedToGlobal,
|
|
280
313
|
dryRun,
|
|
281
314
|
});
|
|
282
|
-
|
|
283
|
-
if (dryRun && cacheStatus !== 'hit' && cacheStatus !== 'stale') {
|
|
284
|
-
io.log('[dry-run] Preview only — cache was not written.');
|
|
285
|
-
}
|
|
286
|
-
for (const warning of extractionQualityWarnings(resultData.source.qualityNotes)) {
|
|
287
|
-
io.warn(`${normalizedUrl}: ${warning}`);
|
|
288
|
-
}
|
|
289
|
-
io.log(resultData.content);
|
|
290
|
-
const separator = batchSeparator(run.urlCount > 1);
|
|
291
|
-
if (separator) {
|
|
292
|
-
io.log(`\n${separator}\n`);
|
|
293
|
-
}
|
|
294
|
-
}
|
|
315
|
+
logHumanFetchResult({ io, resultData, cacheStatus, dryRun, normalizedUrl, run });
|
|
295
316
|
return resultData;
|
|
296
317
|
}
|
|
318
|
+
/** True when this run launched Chrome on its own (no --rendered flag) to capture `url`, rather than
|
|
319
|
+
* just reporting a stored artifact's capture history from an earlier run (cache 'hit'/'stale'/
|
|
320
|
+
* 'revalidated'). Human mode uses this to decide whether the browser-fallback note would be accurate. */
|
|
321
|
+
function usedAutomaticBrowserFallback(cacheStatus, captureMethod, rendered) {
|
|
322
|
+
const capturedThisRun = cacheStatus === 'miss' || cacheStatus === 'refreshed';
|
|
323
|
+
return capturedThisRun && captureMethod === 'browser_fallback' && !rendered;
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* Human-mode-only side effects for one fetch result: dry-run notice, the automatic browser-fallback
|
|
327
|
+
* note, extraction-quality warnings, the content itself, and the batch separator. No-op under
|
|
328
|
+
* `--json` (that output goes through the envelope instead).
|
|
329
|
+
*/
|
|
330
|
+
function logHumanFetchResult(input) {
|
|
331
|
+
const { io, resultData, cacheStatus, dryRun, normalizedUrl, run } = input;
|
|
332
|
+
if (io.json)
|
|
333
|
+
return;
|
|
334
|
+
if (dryRun && cacheStatus !== 'hit' && cacheStatus !== 'stale') {
|
|
335
|
+
io.log('[dry-run] Preview only — cache was not written.');
|
|
336
|
+
}
|
|
337
|
+
// Launching Chrome is real added latency and a real dependency that was otherwise invisible
|
|
338
|
+
// outside --json; an explicit --rendered already tells the caller a browser was used.
|
|
339
|
+
if (usedAutomaticBrowserFallback(cacheStatus, resultData.source.captureMethod, run.flags.rendered)) {
|
|
340
|
+
io.log('Note: used browser-rendered capture (static content was insufficient).');
|
|
341
|
+
}
|
|
342
|
+
for (const warning of extractionQualityWarnings(resultData.source.qualityNotes)) {
|
|
343
|
+
io.warn(`${normalizedUrl}: ${warning}`);
|
|
344
|
+
}
|
|
345
|
+
io.log(resultData.content);
|
|
346
|
+
if (!dryRun) {
|
|
347
|
+
io.warn(formatTip(`${io.bin} inspect ${normalizedUrl} for cached metadata.`));
|
|
348
|
+
}
|
|
349
|
+
const separator = batchSeparator(run.urlCount > 1);
|
|
350
|
+
if (separator) {
|
|
351
|
+
io.log(`\n${separator}\n`);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
297
354
|
// Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
|
|
298
355
|
function applyCaptureMetadata(artifact, capture) {
|
|
299
356
|
const meta = artifact.metadata;
|
|
@@ -156,7 +156,9 @@ function notModifiedResult(res, currentUrl) {
|
|
|
156
156
|
}
|
|
157
157
|
function assertOk(res) {
|
|
158
158
|
if (!res.ok) {
|
|
159
|
-
|
|
159
|
+
// statusText is often empty (e.g. HTTP/2 responses carry no reason phrase); trim so a
|
|
160
|
+
// reasonless status never leaves a dangling trailing space, matching browser.ts's assertOk.
|
|
161
|
+
throw new Error(`Fetch failed with status ${res.status} ${res.statusText}`.trimEnd());
|
|
160
162
|
}
|
|
161
163
|
}
|
|
162
164
|
/**
|