@taurgis/bonsai 3.2.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -6
- package/agents/agents/salesforce-docs-researcher.agent.md +17 -4
- package/agents/instructions/salesforce-research.instructions.md +2 -1
- package/bin/cli.mjs +13 -18
- package/dist/base-command.d.ts +59 -12
- package/dist/base-command.js +179 -51
- package/dist/commands/config/base.js +12 -2
- package/dist/commands/config/get.js +8 -12
- package/dist/commands/config/index.js +2 -2
- package/dist/commands/config/list.js +11 -17
- package/dist/commands/config/set.d.ts +2 -1
- package/dist/commands/config/set.js +35 -28
- package/dist/commands/config/unset.d.ts +2 -1
- package/dist/commands/config/unset.js +26 -25
- package/dist/commands/fetch.d.ts +7 -15
- package/dist/commands/fetch.js +61 -401
- package/dist/commands/import.d.ts +4 -9
- package/dist/commands/import.js +56 -284
- package/dist/commands/inspect.d.ts +7 -1
- package/dist/commands/inspect.js +112 -52
- package/dist/commands/list.d.ts +18 -3
- package/dist/commands/list.js +121 -46
- package/dist/commands/prune.d.ts +6 -1
- package/dist/commands/prune.js +89 -78
- package/dist/commands/status.d.ts +3 -2
- package/dist/commands/status.js +62 -79
- package/dist/hooks/command-not-found/suggest.d.ts +2 -1
- package/dist/hooks/command-not-found/suggest.js +78 -41
- package/dist/lib/argv.d.ts +29 -3
- package/dist/lib/argv.js +147 -54
- package/dist/lib/batch.d.ts +41 -0
- package/dist/lib/batch.js +39 -0
- package/dist/lib/cache-view.d.ts +21 -0
- package/dist/lib/cache-view.js +30 -0
- package/dist/lib/cli-emit.d.ts +28 -0
- package/dist/lib/cli-emit.js +44 -0
- package/dist/lib/cli-error-policy.d.ts +57 -0
- package/dist/lib/cli-error-policy.js +74 -0
- package/dist/lib/cli-flag-manifest.d.ts +12 -0
- package/dist/lib/cli-flag-manifest.js +15 -0
- package/dist/lib/cli-presentation.d.ts +41 -0
- package/dist/lib/cli-presentation.js +53 -0
- package/dist/lib/cli-result-types.d.ts +112 -0
- package/dist/lib/cli-result-types.js +2 -0
- package/dist/lib/config/index.d.ts +23 -5
- package/dist/lib/config/index.js +60 -5
- package/dist/lib/config/io.d.ts +9 -0
- package/dist/lib/config/io.js +71 -21
- package/dist/lib/config/schema.d.ts +2 -0
- package/dist/lib/config/schema.js +7 -0
- package/dist/lib/config-persist.d.ts +28 -0
- package/dist/lib/config-persist.js +29 -0
- package/dist/lib/envelope.d.ts +41 -3
- package/dist/lib/envelope.js +154 -4
- package/dist/lib/help-preflight.d.ts +5 -8
- package/dist/lib/help-preflight.js +13 -32
- package/dist/lib/json-meta.d.ts +0 -2
- package/dist/lib/json-meta.js +21 -25
- package/dist/lib/limit-flag.d.ts +11 -1
- package/dist/lib/limit-flag.js +19 -6
- package/dist/lib/parse-error-ux.d.ts +5 -0
- package/dist/lib/parse-error-ux.js +73 -0
- package/dist/lib/prune-flags.d.ts +30 -0
- package/dist/lib/prune-flags.js +86 -0
- package/dist/lib/research/artifact-index.d.ts +11 -2
- package/dist/lib/research/artifact-index.js +14 -5
- package/dist/lib/research/artifact.js +20 -2
- package/dist/lib/research/browser.d.ts +53 -2
- package/dist/lib/research/browser.js +202 -58
- package/dist/lib/research/capture.d.ts +24 -2
- package/dist/lib/research/capture.js +24 -4
- package/dist/lib/research/cli-io.d.ts +40 -0
- package/dist/lib/research/cli-io.js +27 -0
- package/dist/lib/research/compress.d.ts +7 -0
- package/dist/lib/research/compress.js +7 -3
- package/dist/lib/research/docs/capabilities.d.ts +15 -0
- package/dist/lib/research/docs/capabilities.js +5 -0
- package/dist/lib/research/docs/clean-dom.d.ts +2 -0
- package/dist/lib/research/docs/clean-dom.js +2 -0
- package/dist/lib/research/docs/code-blocks.d.ts +2 -0
- package/dist/lib/research/docs/code-blocks.js +2 -0
- package/dist/lib/research/docs/detect.d.ts +4 -0
- package/dist/lib/research/docs/detect.js +4 -0
- package/dist/lib/research/docs/machine-readable.d.ts +18 -0
- package/dist/lib/research/docs/machine-readable.js +10 -0
- package/dist/lib/research/docs/markdown-source.d.ts +20 -0
- package/dist/lib/research/docs/markdown-source.js +19 -2
- package/dist/lib/research/docs/quality-gates.d.ts +5 -0
- package/dist/lib/research/docs/quality-gates.js +4 -0
- package/dist/lib/research/docs/rsc.d.ts +9 -2
- package/dist/lib/research/docs/rsc.js +5 -2
- package/dist/lib/research/docs/section-artifacts.d.ts +12 -0
- package/dist/lib/research/docs/section-artifacts.js +12 -0
- package/dist/lib/research/docs/sections.d.ts +18 -0
- package/dist/lib/research/docs/sections.js +17 -1
- package/dist/lib/research/docs/source-map.d.ts +28 -0
- package/dist/lib/research/docs/source-map.js +27 -0
- package/dist/lib/research/docs/validate.d.ts +35 -2
- package/dist/lib/research/docs/validate.js +47 -2
- package/dist/lib/research/extract.d.ts +16 -0
- package/dist/lib/research/extract.js +12 -1
- package/dist/lib/research/fetch-command-service.d.ts +36 -0
- package/dist/lib/research/fetch-command-service.js +367 -0
- package/dist/lib/research/fetch-result.d.ts +114 -0
- package/dist/lib/research/fetch-result.js +212 -0
- package/dist/lib/research/fetcher.d.ts +46 -8
- package/dist/lib/research/fetcher.js +61 -31
- package/dist/lib/research/freshness.d.ts +54 -11
- package/dist/lib/research/freshness.js +91 -31
- package/dist/lib/research/import-command-service.d.ts +56 -0
- package/dist/lib/research/import-command-service.js +402 -0
- package/dist/lib/research/keywords.js +5 -1
- package/dist/lib/research/markdown.d.ts +12 -1
- package/dist/lib/research/markdown.js +13 -5
- package/dist/lib/research/metadata-filters.d.ts +18 -0
- package/dist/lib/research/metadata-filters.js +37 -0
- package/dist/lib/research/metadata-flags.d.ts +34 -0
- package/dist/lib/research/metadata-flags.js +45 -0
- package/dist/lib/research/persist-artifact.d.ts +23 -0
- package/dist/lib/research/persist-artifact.js +27 -0
- package/dist/lib/research/prompt-injection.d.ts +9 -0
- package/dist/lib/research/prompt-injection.js +59 -14
- package/dist/lib/research/proxy.d.ts +14 -8
- package/dist/lib/research/proxy.js +14 -10
- package/dist/lib/research/resolve-target.d.ts +2 -0
- package/dist/lib/research/resolve-target.js +1 -1
- package/dist/lib/research/revalidate.d.ts +49 -20
- package/dist/lib/research/revalidate.js +80 -15
- package/dist/lib/research/schema.d.ts +7 -0
- package/dist/lib/research/schema.js +2 -0
- package/dist/lib/research/secret-scan.d.ts +6 -0
- package/dist/lib/research/secret-scan.js +8 -0
- package/dist/lib/research/secure-write.d.ts +4 -0
- package/dist/lib/research/secure-write.js +31 -2
- package/dist/lib/research/storage.d.ts +9 -8
- package/dist/lib/research/storage.js +34 -28
- package/dist/lib/research/summarize.d.ts +15 -0
- package/dist/lib/research/summarize.js +11 -0
- package/dist/lib/research/url.d.ts +9 -6
- package/dist/lib/research/url.js +33 -21
- package/dist/lib/text.d.ts +19 -8
- package/dist/lib/text.js +32 -9
- package/dist/lib/value-flags.d.ts +14 -0
- package/dist/lib/value-flags.js +28 -0
- package/dist/sites/coveo.d.ts +10 -0
- package/dist/sites/coveo.js +10 -0
- package/dist/sites/index.d.ts +16 -0
- package/dist/sites/index.js +16 -4
- package/dist/sites/markdown-twin.d.ts +31 -0
- package/dist/sites/markdown-twin.js +56 -0
- package/dist/sites/salesforce/fetch-page.d.ts +8 -2
- package/dist/sites/salesforce/fetch-page.js +14 -3
- package/dist/sites/salesforce/help-tooling-markdown.d.ts +30 -0
- package/dist/sites/salesforce/help-tooling-markdown.js +74 -0
- package/dist/sites/salesforce/index.d.ts +1 -0
- package/dist/sites/salesforce/index.js +1 -0
- package/dist/sites/salesforce-developer/fetch-page.d.ts +6 -1
- package/dist/sites/salesforce-developer/fetch-page.js +6 -1
- package/dist/sites/salesforce-developer/index.d.ts +1 -0
- package/dist/sites/salesforce-developer/index.js +1 -0
- package/dist/sites/salesforce-developer/route-markdown.d.ts +18 -6
- package/dist/sites/salesforce-developer/route-markdown.js +29 -67
- package/dist/sites/salesforce-doc-fetch.d.ts +39 -5
- package/dist/sites/salesforce-doc-fetch.js +52 -7
- package/dist/sites/salesforce-dom-probe.d.ts +20 -2
- package/dist/sites/salesforce-dom-probe.js +20 -2
- package/dist/sites/types.d.ts +14 -8
- package/dist/sites/types.js +3 -0
- package/oclif.manifest.json +216 -102
- package/package.json +20 -7
- package/agents/agents/official-docs-researcher.agent.md +0 -87
- package/agents/instructions/web-research.instructions.md +0 -44
- package/agents/skills/web-research/SKILL.md +0 -118
|
@@ -47,15 +47,23 @@ function addGfmTableRule(service) {
|
|
|
47
47
|
});
|
|
48
48
|
}
|
|
49
49
|
/**
|
|
50
|
-
*
|
|
51
|
-
*
|
|
50
|
+
* Drops empty anchor links (e.g. `[](url)`) from Markdown. These arise from icon-only heading
|
|
51
|
+
* permalinks and ReadMe query-param chips that carry no text content, wasting tokens.
|
|
52
|
+
* Images (``) are intentionally not matched.
|
|
53
|
+
*
|
|
54
|
+
* @param markdown - Markdown text to clean.
|
|
55
|
+
* @returns Markdown with text-less links removed.
|
|
52
56
|
*/
|
|
53
|
-
// Icon-only/empty anchors (e.g. a heading's "#" permalink, a ReadMe query-param chip) convert to a
|
|
54
|
-
// text-less link `[](url)` that carries no information and just wastes tokens. Drop them, but never
|
|
55
|
-
// match images `` (empty alt is still a real image).
|
|
56
57
|
export function dropEmptyLinks(markdown) {
|
|
57
58
|
return markdown.replace(/(?<!!)\[\s*\]\([^)]*\)/g, '');
|
|
58
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Converts HTML content to clean, deterministic Markdown.
|
|
62
|
+
* Configured to use ATX-style headings (#) and fenced code blocks (```), with GFM tables.
|
|
63
|
+
*
|
|
64
|
+
* @param html - HTML string to convert.
|
|
65
|
+
* @returns Clean Markdown with empty links stripped.
|
|
66
|
+
*/
|
|
59
67
|
export function htmlToMarkdown(html) {
|
|
60
68
|
const turndownService = new TurndownService({
|
|
61
69
|
headingStyle: 'atx',
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ResearchArtifactMetadata } from './schema.js';
|
|
2
|
+
/** Whether an artifact's topic matches a `--topic` filter (case-insensitive exact match). */
|
|
3
|
+
export declare function matchesTopicFilter(meta: Pick<ResearchArtifactMetadata, 'topic'>, topic: string | undefined): boolean;
|
|
4
|
+
/** Whether an artifact's tags satisfy a `--tags` filter (case-insensitive, must match all). */
|
|
5
|
+
export declare function matchesTagsFilter(meta: Pick<ResearchArtifactMetadata, 'tags'>, tags: string[] | undefined): boolean;
|
|
6
|
+
/**
|
|
7
|
+
* Reject an explicitly passed, whitespace-only `--topic` filter. Silently treating it as "no
|
|
8
|
+
* filter" (matching everything, as an empty/falsy value normally would in {@link matchesTopicFilter})
|
|
9
|
+
* would hide what's almost always a shell-quoting mistake — the same reasoning `--url` already
|
|
10
|
+
* applies via `emptyUrlFilterError`.
|
|
11
|
+
*/
|
|
12
|
+
export declare function emptyTopicFilterError(topic: string | undefined): string | undefined;
|
|
13
|
+
/**
|
|
14
|
+
* Reject an explicitly passed, whitespace-only `--tags` filter entry. Unlike a missing/empty
|
|
15
|
+
* `--topic` (which reads as "no filter"), an empty tag string in {@link matchesTagsFilter} never
|
|
16
|
+
* matches any artifact's tags — silently returning zero results with no signal. Fail fast instead.
|
|
17
|
+
*/
|
|
18
|
+
export declare function emptyTagsFilterError(tags: string[] | undefined): string | undefined;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/** Whether an artifact's topic matches a `--topic` filter (case-insensitive exact match). */
|
|
2
|
+
export function matchesTopicFilter(meta, topic) {
|
|
3
|
+
if (!topic)
|
|
4
|
+
return true;
|
|
5
|
+
return !!meta.topic && meta.topic.trim().toLowerCase() === topic.trim().toLowerCase();
|
|
6
|
+
}
|
|
7
|
+
/** Whether an artifact's tags satisfy a `--tags` filter (case-insensitive, must match all). */
|
|
8
|
+
export function matchesTagsFilter(meta, tags) {
|
|
9
|
+
if (!tags || tags.length === 0)
|
|
10
|
+
return true;
|
|
11
|
+
const metaTagsLower = meta.tags.map((t) => t.toLowerCase());
|
|
12
|
+
return tags.every((t) => metaTagsLower.includes(t.toLowerCase()));
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Reject an explicitly passed, whitespace-only `--topic` filter. Silently treating it as "no
|
|
16
|
+
* filter" (matching everything, as an empty/falsy value normally would in {@link matchesTopicFilter})
|
|
17
|
+
* would hide what's almost always a shell-quoting mistake — the same reasoning `--url` already
|
|
18
|
+
* applies via `emptyUrlFilterError`.
|
|
19
|
+
*/
|
|
20
|
+
export function emptyTopicFilterError(topic) {
|
|
21
|
+
if (topic === undefined)
|
|
22
|
+
return undefined;
|
|
23
|
+
if (topic.trim() === '') {
|
|
24
|
+
return '--topic must be a non-empty value (e.g. "React Suspense").';
|
|
25
|
+
}
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Reject an explicitly passed, whitespace-only `--tags` filter entry. Unlike a missing/empty
|
|
30
|
+
* `--topic` (which reads as "no filter"), an empty tag string in {@link matchesTagsFilter} never
|
|
31
|
+
* matches any artifact's tags — silently returning zero results with no signal. Fail fast instead.
|
|
32
|
+
*/
|
|
33
|
+
export function emptyTagsFilterError(tags) {
|
|
34
|
+
if (!tags?.some((t) => t.trim() === ''))
|
|
35
|
+
return undefined;
|
|
36
|
+
return '--tags must be non-empty values (e.g. "react").';
|
|
37
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** Longest `--topic` value accepted; topics are short labels, not free text. */
|
|
2
|
+
export declare const MAX_TOPIC_LENGTH = 200;
|
|
3
|
+
/** Longest single `--tags` value accepted; tags are short taxonomic labels. */
|
|
4
|
+
export declare const MAX_TAG_LENGTH = 100;
|
|
5
|
+
/**
|
|
6
|
+
* Reject a `--topic`/`--tags` value longer than its cap before it ever reaches the cache.
|
|
7
|
+
* Unbounded input here would still round-trip through YAML frontmatter and `list`/`inspect`
|
|
8
|
+
* output correctly, but a single unreasonably long value wraps a human-readable listing line
|
|
9
|
+
* across dozens of terminal rows, so cap it at the flag boundary the same way `metadataNewlineError`
|
|
10
|
+
* catches the newline case.
|
|
11
|
+
*
|
|
12
|
+
* @param flags - The command's parsed `topic`/`tags` flags.
|
|
13
|
+
* @returns An error message naming the offending flag and its length cap, or `null` when both fit.
|
|
14
|
+
*/
|
|
15
|
+
export declare function metadataLengthError(flags: {
|
|
16
|
+
topic?: string;
|
|
17
|
+
tags?: string[];
|
|
18
|
+
}): string | null;
|
|
19
|
+
/**
|
|
20
|
+
* Reject a `--topic`/`--tags` value containing a line break before it ever reaches the cache.
|
|
21
|
+
* Stored as-is, an embedded newline would corrupt the single-line-per-field frontmatter format
|
|
22
|
+
* (see `sanitizeFrontmatterLine` in `artifact.ts`): at best a silently dropped stray line, at
|
|
23
|
+
* worst a spoofed `key: value` pair or a bare `---` that closes the frontmatter fence early and
|
|
24
|
+
* splices the rest of the metadata into the body. Failing fast here — rather than letting the
|
|
25
|
+
* serializer silently rewrite the caller's input — gives the same actionable-error treatment as
|
|
26
|
+
* every other flag validator (`durationFlagError`, etc.).
|
|
27
|
+
*
|
|
28
|
+
* @param flags - The command's parsed `topic`/`tags` flags.
|
|
29
|
+
* @returns An error message naming the offending flag, or `null` when both are clean.
|
|
30
|
+
*/
|
|
31
|
+
export declare function metadataNewlineError(flags: {
|
|
32
|
+
topic?: string;
|
|
33
|
+
tags?: string[];
|
|
34
|
+
}): string | null;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** Longest `--topic` value accepted; topics are short labels, not free text. */
|
|
2
|
+
export const MAX_TOPIC_LENGTH = 200;
|
|
3
|
+
/** Longest single `--tags` value accepted; tags are short taxonomic labels. */
|
|
4
|
+
export const MAX_TAG_LENGTH = 100;
|
|
5
|
+
/**
|
|
6
|
+
* Reject a `--topic`/`--tags` value longer than its cap before it ever reaches the cache.
|
|
7
|
+
* Unbounded input here would still round-trip through YAML frontmatter and `list`/`inspect`
|
|
8
|
+
* output correctly, but a single unreasonably long value wraps a human-readable listing line
|
|
9
|
+
* across dozens of terminal rows, so cap it at the flag boundary the same way `metadataNewlineError`
|
|
10
|
+
* catches the newline case.
|
|
11
|
+
*
|
|
12
|
+
* @param flags - The command's parsed `topic`/`tags` flags.
|
|
13
|
+
* @returns An error message naming the offending flag and its length cap, or `null` when both fit.
|
|
14
|
+
*/
|
|
15
|
+
export function metadataLengthError(flags) {
|
|
16
|
+
if (flags.topic !== undefined && flags.topic.length > MAX_TOPIC_LENGTH) {
|
|
17
|
+
return `--topic must be ${MAX_TOPIC_LENGTH} characters or fewer (got ${flags.topic.length}).`;
|
|
18
|
+
}
|
|
19
|
+
const longTag = flags.tags?.find((tag) => tag.length > MAX_TAG_LENGTH);
|
|
20
|
+
if (longTag !== undefined) {
|
|
21
|
+
return `--tags must be ${MAX_TAG_LENGTH} characters or fewer (got ${longTag.length}).`;
|
|
22
|
+
}
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Reject a `--topic`/`--tags` value containing a line break before it ever reaches the cache.
|
|
27
|
+
* Stored as-is, an embedded newline would corrupt the single-line-per-field frontmatter format
|
|
28
|
+
* (see `sanitizeFrontmatterLine` in `artifact.ts`): at best a silently dropped stray line, at
|
|
29
|
+
* worst a spoofed `key: value` pair or a bare `---` that closes the frontmatter fence early and
|
|
30
|
+
* splices the rest of the metadata into the body. Failing fast here — rather than letting the
|
|
31
|
+
* serializer silently rewrite the caller's input — gives the same actionable-error treatment as
|
|
32
|
+
* every other flag validator (`durationFlagError`, etc.).
|
|
33
|
+
*
|
|
34
|
+
* @param flags - The command's parsed `topic`/`tags` flags.
|
|
35
|
+
* @returns An error message naming the offending flag, or `null` when both are clean.
|
|
36
|
+
*/
|
|
37
|
+
export function metadataNewlineError(flags) {
|
|
38
|
+
if (flags.topic !== undefined && /[\r\n]/.test(flags.topic)) {
|
|
39
|
+
return '--topic cannot contain line breaks.';
|
|
40
|
+
}
|
|
41
|
+
if (flags.tags?.some((tag) => /[\r\n]/.test(tag))) {
|
|
42
|
+
return '--tags cannot contain line breaks.';
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type SecureWriteResult } from './secure-write.js';
|
|
2
|
+
import type { StoreRoots } from './store-roots.js';
|
|
3
|
+
import type { ResearchArtifact } from './schema.js';
|
|
4
|
+
export type ArtifactWriteKind = 'import' | 'fetch';
|
|
5
|
+
export interface PersistArtifactResult extends SecureWriteResult {
|
|
6
|
+
dryRun: boolean;
|
|
7
|
+
/** Secret-redirect warning for stderr, or null when no redirect. */
|
|
8
|
+
redirectWarning: string | null;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Persist (or dry-run preview) an artifact with secret-safe routing. Owns redirect warning copy
|
|
12
|
+
* so fetch/import cannot drift. When `scratchDir` is set (fetch dry-run), also writes a throwaway
|
|
13
|
+
* copy there for downstream extraction while reporting the real would-be `dataDir`.
|
|
14
|
+
*/
|
|
15
|
+
export declare function persistArtifact(opts: {
|
|
16
|
+
roots: StoreRoots;
|
|
17
|
+
cacheKey: string;
|
|
18
|
+
artifact: ResearchArtifact;
|
|
19
|
+
dryRun: boolean;
|
|
20
|
+
kind: ArtifactWriteKind;
|
|
21
|
+
/** Fetch dry-run throwaway directory; real destination still comes from the secure write. */
|
|
22
|
+
scratchDir?: string | null;
|
|
23
|
+
}): PersistArtifactResult;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { writeArtifact } from './storage.js';
|
|
2
|
+
import { writeArtifactSecurely } from './secure-write.js';
|
|
3
|
+
function redirectWarningFor(kind, dryRun, secretLabel) {
|
|
4
|
+
const content = kind === 'import' ? 'imported content' : 'page content';
|
|
5
|
+
const verb = dryRun ? 'would store' : 'stored';
|
|
6
|
+
return `Detected ${secretLabel} in the ${content}; ${verb} in the global cache instead of the project to avoid committing secrets.`;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Persist (or dry-run preview) an artifact with secret-safe routing. Owns redirect warning copy
|
|
10
|
+
* so fetch/import cannot drift. When `scratchDir` is set (fetch dry-run), also writes a throwaway
|
|
11
|
+
* copy there for downstream extraction while reporting the real would-be `dataDir`.
|
|
12
|
+
*/
|
|
13
|
+
export function persistArtifact(opts) {
|
|
14
|
+
const result = writeArtifactSecurely(opts.roots, opts.cacheKey, opts.artifact, {
|
|
15
|
+
dryRun: opts.dryRun,
|
|
16
|
+
});
|
|
17
|
+
if (opts.scratchDir) {
|
|
18
|
+
writeArtifact(opts.scratchDir, opts.cacheKey, opts.artifact);
|
|
19
|
+
}
|
|
20
|
+
return {
|
|
21
|
+
...result,
|
|
22
|
+
dryRun: opts.dryRun,
|
|
23
|
+
redirectWarning: result.redirected
|
|
24
|
+
? redirectWarningFor(opts.kind, opts.dryRun, result.secretLabel)
|
|
25
|
+
: null,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
@@ -1 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scans Markdown for embedded prompt-injection attacks and redacts any block or line that matches.
|
|
3
|
+
* Detects direct patterns, plus base64/hex-encoded variants and typoglycemia obfuscation.
|
|
4
|
+
* HTML comments are checked first; the body is then split on paragraph breaks and scanned line-by-line.
|
|
5
|
+
*
|
|
6
|
+
* @param markdown - Markdown content (possibly untrusted, sourced from fetched pages).
|
|
7
|
+
* @returns Sanitized Markdown with harmful instruction blocks replaced by a redaction placeholder,
|
|
8
|
+
* and ANSI/control bytes stripped so a cached read can never replay them to a terminal.
|
|
9
|
+
*/
|
|
1
10
|
export declare function sanitizePromptInjection(markdown: string): string;
|
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
|
+
import { CONTROL_CHAR_PATTERN } from '../text.js';
|
|
2
3
|
const REDACTION = '[Removed potentially unsafe agent instruction]';
|
|
3
4
|
const ZERO_WIDTH = /[\u200b-\u200f\ufeff]/g;
|
|
5
|
+
// Tab/LF/CR fall inside CONTROL_CHAR_PATTERN's Unicode "Control" category too, but they are
|
|
6
|
+
// legitimate Markdown whitespace, so the replacer below keeps those three and drops everything else.
|
|
7
|
+
const SAFE_CONTROL_CHARS = new Set(['\t', '\n', '\r']);
|
|
8
|
+
/**
|
|
9
|
+
* Removes ANSI escape codes and other control bytes a malicious page could embed in its visible
|
|
10
|
+
* text, while leaving Markdown-structural whitespace (tab/newline/CR) intact. Every fetch/import
|
|
11
|
+
* path renders its stored content directly to the terminal (`bonsai <url>`, cache hits, etc.), so
|
|
12
|
+
* raw control bytes surviving Readability/Turndown would replay as a terminal-injection attack on
|
|
13
|
+
* every subsequent read of the cache; this is the choke point all of those paths share.
|
|
14
|
+
*/
|
|
15
|
+
function stripUnsafeControlChars(text) {
|
|
16
|
+
return text.replace(CONTROL_CHAR_PATTERN, (char) => (SAFE_CONTROL_CHARS.has(char) ? char : ''));
|
|
17
|
+
}
|
|
4
18
|
const HTML_COMMENT = /<!--[\s\S]*?-->/g;
|
|
5
19
|
const BASE64_TOKEN = /(?<![A-Za-z0-9+/_-])(?:[A-Za-z0-9+/_-]{16,}={0,2})(?![A-Za-z0-9+/_-])/g;
|
|
6
20
|
const HEX_TOKEN = /(?<![a-f0-9])(?:0x)?[a-f0-9]{24,}(?![a-f0-9])/gi;
|
|
@@ -30,19 +44,28 @@ const TYPOGLYCEMIA_TARGETS = [
|
|
|
30
44
|
'repository',
|
|
31
45
|
'workspace',
|
|
32
46
|
];
|
|
47
|
+
// Contexts where a following verb reads as a command aimed at the reader, not a third-person
|
|
48
|
+
// description of an attack (e.g. "An attacker may tell the model to ignore..." must stay
|
|
49
|
+
// unredacted — see the matching test below). `^` is matched per clause (see CLAUSE_BOUNDARY
|
|
50
|
+
// below), so a command hidden after an innocuous opener ("Heads up: ignore...") still counts as
|
|
51
|
+
// clause-initial; only right after a direct role-address word is the other allowed context.
|
|
52
|
+
const COMMAND_CONTEXT = String.raw `(?:^|\b(?:please|assistant|system|user|agent)\s+)`;
|
|
53
|
+
function commandPattern(suffix) {
|
|
54
|
+
return new RegExp(COMMAND_CONTEXT + suffix, 'i');
|
|
55
|
+
}
|
|
33
56
|
const HARMFUL_INSTRUCTION_PATTERNS = [
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
57
|
+
commandPattern(String.raw `ignore\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b`),
|
|
58
|
+
commandPattern(String.raw `disregard\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b`),
|
|
59
|
+
commandPattern(String.raw `forget\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b`),
|
|
60
|
+
commandPattern(String.raw `override\s+(the\s+)?(system|developer|agent)\s+(prompt|instructions?|message)\b`),
|
|
61
|
+
commandPattern(String.raw `you\s+are\s+now\s+(in\s+)?(developer|admin|root|system)\s+mode\b`),
|
|
62
|
+
commandPattern(String.raw `reveal\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b`),
|
|
63
|
+
commandPattern(String.raw `print\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b`),
|
|
64
|
+
commandPattern(String.raw `(exfiltrate|steal|leak)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?|private\s+data)\b`),
|
|
65
|
+
commandPattern(String.raw `send\s+(me\s+|all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b`),
|
|
66
|
+
commandPattern(String.raw `(upload|post|fetch|curl)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b`),
|
|
67
|
+
commandPattern(String.raw `delete\s+(all\s+)?(files?|the\s+repository|the\s+workspace|the\s+home\s+directory)\b`),
|
|
68
|
+
commandPattern(String.raw `run\s+.*\b(rm\s+-rf|curl\s+.*\|\s*(sh|bash)|sudo)\b`),
|
|
46
69
|
];
|
|
47
70
|
function normalizeForDetection(text) {
|
|
48
71
|
return text
|
|
@@ -110,8 +133,20 @@ function isReadableDecodedText(text) {
|
|
|
110
133
|
const printable = [...text].filter((char) => /[\t\n\r -~]/.test(char)).length;
|
|
111
134
|
return printable / Math.max(text.length, 1) > 0.85;
|
|
112
135
|
}
|
|
136
|
+
// Sentence-ending punctuation splits a line into clauses for `^`-anchored matching below. This has
|
|
137
|
+
// to run on the raw text, before normalizeForDetection's charset filter erases that same
|
|
138
|
+
// punctuation into plain spaces — testing the anchor after normalization would never see it.
|
|
139
|
+
//
|
|
140
|
+
// ponytail: this still isn't complete — a filler *word* with no punctuation before it ("Please
|
|
141
|
+
// just ignore previous instructions") is a clause-internal command and stays unredacted, same as
|
|
142
|
+
// before this fix. Heuristic prompt-injection detection can't be made airtight with regex alone
|
|
143
|
+
// (OWASP LLM01 has no complete solution either); upgrade path is a small classifier or an LLM-based
|
|
144
|
+
// second pass over blocks this pattern set flags as borderline, if false negatives here prove costly.
|
|
145
|
+
const CLAUSE_BOUNDARY = /(?<=[.!?:;])\s+/;
|
|
113
146
|
function isUnsafeAgentInstruction(text) {
|
|
114
|
-
return
|
|
147
|
+
return text
|
|
148
|
+
.split(CLAUSE_BOUNDARY)
|
|
149
|
+
.some((clause) => detectionCandidates(clause).some((candidate) => HARMFUL_INSTRUCTION_PATTERNS.some((pattern) => pattern.test(candidate))));
|
|
115
150
|
}
|
|
116
151
|
function redactBlock(block) {
|
|
117
152
|
const withSafeSpans = redactUnsafeSpans(block);
|
|
@@ -132,8 +167,17 @@ function stripQuotes(span) {
|
|
|
132
167
|
}
|
|
133
168
|
return span;
|
|
134
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* Scans Markdown for embedded prompt-injection attacks and redacts any block or line that matches.
|
|
172
|
+
* Detects direct patterns, plus base64/hex-encoded variants and typoglycemia obfuscation.
|
|
173
|
+
* HTML comments are checked first; the body is then split on paragraph breaks and scanned line-by-line.
|
|
174
|
+
*
|
|
175
|
+
* @param markdown - Markdown content (possibly untrusted, sourced from fetched pages).
|
|
176
|
+
* @returns Sanitized Markdown with harmful instruction blocks replaced by a redaction placeholder,
|
|
177
|
+
* and ANSI/control bytes stripped so a cached read can never replay them to a terminal.
|
|
178
|
+
*/
|
|
135
179
|
export function sanitizePromptInjection(markdown) {
|
|
136
|
-
|
|
180
|
+
const redacted = markdown
|
|
137
181
|
.replace(HTML_COMMENT, (comment) => redactBlock(comment))
|
|
138
182
|
.split(/(\n{2,})/)
|
|
139
183
|
.map((block) => {
|
|
@@ -146,4 +190,5 @@ export function sanitizePromptInjection(markdown) {
|
|
|
146
190
|
return lineRedacted === block ? redactBlock(block) : lineRedacted;
|
|
147
191
|
})
|
|
148
192
|
.join('');
|
|
193
|
+
return stripUnsafeControlChars(redacted);
|
|
149
194
|
}
|
|
@@ -59,13 +59,19 @@ export declare function getCaBundleSpkiHashes(): string[];
|
|
|
59
59
|
/**
|
|
60
60
|
* Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
|
|
61
61
|
* than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
|
|
62
|
-
* NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so
|
|
63
|
-
* MITMs HTTPS
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
62
|
+
* NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so an environment
|
|
63
|
+
* that MITMs HTTPS makes every other tool trust that CA while Chrome's handshake with it fails
|
|
64
|
+
* with ERR_CERT_AUTHORITY_INVALID. This is scoped narrowly to the specific CA(s) discovered via
|
|
65
|
+
* the CA bundle env vars — never --ignore-certificate-errors, which would disable verification
|
|
66
|
+
* for every host, including ones that have nothing to do with the sandbox.
|
|
67
|
+
*
|
|
68
|
+
* Deliberately independent of isProxyConfigured(): some sandboxes intercept HTTPS transparently
|
|
69
|
+
* (or via a proxy Chrome reaches without an explicit --proxy-server flag) while still exporting a
|
|
70
|
+
* CA bundle env var for CLI tools to trust — HTTPS_PROXY/HTTP_PROXY and the CA bundle vars are
|
|
71
|
+
* two separate signals that usually co-occur but aren't guaranteed to. Gating on proxy detection
|
|
72
|
+
* left Chrome untrusting a CA bundle that was in fact discoverable, so it rejected every
|
|
73
|
+
* browser-rendered fetch with ERR_CERT_AUTHORITY_INVALID in that shape of environment. A no-op
|
|
74
|
+
* (empty array) when no CA bundle is discoverable, so default Chrome trust is untouched on
|
|
75
|
+
* ordinary developer machines.
|
|
70
76
|
*/
|
|
71
77
|
export declare function getChromeSpkiArgs(): string[];
|
|
@@ -169,18 +169,22 @@ export function getCaBundleSpkiHashes() {
|
|
|
169
169
|
/**
|
|
170
170
|
* Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
|
|
171
171
|
* than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
|
|
172
|
-
* NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so
|
|
173
|
-
* MITMs HTTPS
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
172
|
+
* NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so an environment
|
|
173
|
+
* that MITMs HTTPS makes every other tool trust that CA while Chrome's handshake with it fails
|
|
174
|
+
* with ERR_CERT_AUTHORITY_INVALID. This is scoped narrowly to the specific CA(s) discovered via
|
|
175
|
+
* the CA bundle env vars — never --ignore-certificate-errors, which would disable verification
|
|
176
|
+
* for every host, including ones that have nothing to do with the sandbox.
|
|
177
|
+
*
|
|
178
|
+
* Deliberately independent of isProxyConfigured(): some sandboxes intercept HTTPS transparently
|
|
179
|
+
* (or via a proxy Chrome reaches without an explicit --proxy-server flag) while still exporting a
|
|
180
|
+
* CA bundle env var for CLI tools to trust — HTTPS_PROXY/HTTP_PROXY and the CA bundle vars are
|
|
181
|
+
* two separate signals that usually co-occur but aren't guaranteed to. Gating on proxy detection
|
|
182
|
+
* left Chrome untrusting a CA bundle that was in fact discoverable, so it rejected every
|
|
183
|
+
* browser-rendered fetch with ERR_CERT_AUTHORITY_INVALID in that shape of environment. A no-op
|
|
184
|
+
* (empty array) when no CA bundle is discoverable, so default Chrome trust is untouched on
|
|
185
|
+
* ordinary developer machines.
|
|
180
186
|
*/
|
|
181
187
|
export function getChromeSpkiArgs() {
|
|
182
|
-
if (!isProxyConfigured())
|
|
183
|
-
return [];
|
|
184
188
|
const hashes = getCaBundleSpkiHashes();
|
|
185
189
|
if (!hashes.length)
|
|
186
190
|
return [];
|
|
@@ -7,6 +7,8 @@ export interface ResolveResearchTargetOptions {
|
|
|
7
7
|
dataDir: string;
|
|
8
8
|
flagOverride?: StorageMode;
|
|
9
9
|
lookup?: boolean;
|
|
10
|
+
/** Read-only/plan mode: a corrupt entry hit during lookup is reported but never archived on disk. */
|
|
11
|
+
readOnly?: boolean;
|
|
10
12
|
url: string;
|
|
11
13
|
}
|
|
12
14
|
export interface ResolvedResearchTarget {
|
|
@@ -13,7 +13,7 @@ export function resolveResearchTarget(opts) {
|
|
|
13
13
|
});
|
|
14
14
|
return {
|
|
15
15
|
cacheKey,
|
|
16
|
-
located: opts.lookup === false ? null : locateArtifact(roots.readRoots, cacheKey),
|
|
16
|
+
located: opts.lookup === false ? null : locateArtifact(roots.readRoots, cacheKey, opts.readOnly),
|
|
17
17
|
normalizedUrl,
|
|
18
18
|
roots,
|
|
19
19
|
};
|
|
@@ -1,34 +1,63 @@
|
|
|
1
1
|
import type { ResearchArtifact } from './schema.js';
|
|
2
|
+
import { type FetchedContent } from './fetcher.js';
|
|
3
|
+
import { type ExtractionResult } from './extract.js';
|
|
4
|
+
import type { StoreRoots } from './store-roots.js';
|
|
2
5
|
import type { SummaryLevel } from '../config/schema.js';
|
|
6
|
+
/**
|
|
7
|
+
* When set, project-located revalidation writes reuse {@link persistArtifact} (secret-scan +
|
|
8
|
+
* redirect) so a refresh cannot leave credentials in a committable project cache (#90).
|
|
9
|
+
*/
|
|
10
|
+
export interface RevalidateSecureWrite {
|
|
11
|
+
roots: StoreRoots;
|
|
12
|
+
dryRun?: boolean;
|
|
13
|
+
scratchDir?: string | null;
|
|
14
|
+
}
|
|
15
|
+
/** Result of conditional revalidation or a full refresh when validators miss. */
|
|
3
16
|
export interface RevalidationResult {
|
|
17
|
+
/** Outcome of the revalidation attempt. */
|
|
4
18
|
status: 'revalidated' | 'refreshed' | 'stale';
|
|
19
|
+
/** Artifact to serve (updated, refreshed, or the prior stale entry). */
|
|
5
20
|
artifact: ResearchArtifact;
|
|
21
|
+
/** When status is `stale`, whether serving stale content is allowed. */
|
|
6
22
|
allowed?: boolean;
|
|
23
|
+
/** Revalidation failure message when status is `stale`. */
|
|
7
24
|
error?: string;
|
|
25
|
+
/** Data dir the write landed in (or would land in under dry-run), when a write ran. */
|
|
26
|
+
storageDir?: string;
|
|
27
|
+
/** True when a project refresh was redirected to global due to a detected secret. */
|
|
28
|
+
redirectedToGlobal?: boolean;
|
|
29
|
+
/** Secret-redirect warning for stderr, or null/undefined when none. */
|
|
30
|
+
redirectWarning?: string | null;
|
|
31
|
+
}
|
|
32
|
+
type RevalidateOptions = {
|
|
33
|
+
allowStale: boolean;
|
|
34
|
+
ttlOverride?: string | null;
|
|
35
|
+
rendered?: boolean;
|
|
36
|
+
summaryLevel: SummaryLevel;
|
|
37
|
+
/** Present only when the live entry is in the project cache. */
|
|
38
|
+
secure?: RevalidateSecureWrite;
|
|
39
|
+
};
|
|
40
|
+
interface CreateArtifactFromFetchInput {
|
|
41
|
+
url: string;
|
|
42
|
+
normalizedUrl: string;
|
|
43
|
+
cacheKey: string;
|
|
44
|
+
fetchResult: FetchedContent;
|
|
45
|
+
extraction: ExtractionResult;
|
|
46
|
+
tier: 'stable' | 'standard' | 'volatile';
|
|
47
|
+
ttl: string | null;
|
|
48
|
+
currentTime: Date;
|
|
49
|
+
summaryLevel: SummaryLevel;
|
|
8
50
|
}
|
|
9
51
|
/**
|
|
10
|
-
*
|
|
52
|
+
* Construct a ResearchArtifact from a fresh HTML fetch (or error-page marker).
|
|
53
|
+
*
|
|
54
|
+
* @param input - URL, fetch result, extraction, tier/TTL, clock, and summary level.
|
|
55
|
+
* @returns A ready-to-persist research artifact.
|
|
11
56
|
*/
|
|
12
|
-
export declare function createArtifactFromFetch(
|
|
13
|
-
contentType: string | null;
|
|
14
|
-
etag: string | null;
|
|
15
|
-
lastModified: string | null;
|
|
16
|
-
finalUrl: string;
|
|
17
|
-
responseSize: number;
|
|
18
|
-
content: string;
|
|
19
|
-
}, extraction: {
|
|
20
|
-
title: string;
|
|
21
|
-
detailedMarkdown: string;
|
|
22
|
-
confidence: 'high' | 'medium' | 'low';
|
|
23
|
-
qualityNotes: string[];
|
|
24
|
-
}, tier: 'stable' | 'standard' | 'volatile', ttl: string | null, currentTime: Date, summaryLevel: SummaryLevel): ResearchArtifact;
|
|
57
|
+
export declare function createArtifactFromFetch(input: CreateArtifactFromFetchInput): ResearchArtifact;
|
|
25
58
|
/**
|
|
26
59
|
* Revalidates a stale cache artifact using conditional request headers,
|
|
27
60
|
* falling back to stale serving within the grace period if the remote host is offline.
|
|
28
61
|
*/
|
|
29
|
-
export declare function revalidateCache(dataDir: string, existing: ResearchArtifact, currentTime: Date, options:
|
|
30
|
-
|
|
31
|
-
ttlOverride?: string | null;
|
|
32
|
-
rendered?: boolean;
|
|
33
|
-
summaryLevel: SummaryLevel;
|
|
34
|
-
}): Promise<RevalidationResult>;
|
|
62
|
+
export declare function revalidateCache(dataDir: string, existing: ResearchArtifact, currentTime: Date, options: RevalidateOptions): Promise<RevalidationResult>;
|
|
63
|
+
export {};
|