@taurgis/bonsai 3.2.0 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/agents/agents/salesforce-docs-researcher.agent.md +14 -1
- package/agents/instructions/salesforce-research.instructions.md +2 -1
- package/bin/cli.mjs +9 -17
- package/dist/base-command.d.ts +22 -12
- package/dist/base-command.js +86 -49
- package/dist/commands/config/base.js +11 -2
- package/dist/commands/config/get.js +8 -12
- package/dist/commands/config/index.js +2 -2
- package/dist/commands/config/list.js +10 -16
- package/dist/commands/config/set.d.ts +2 -1
- package/dist/commands/config/set.js +34 -27
- package/dist/commands/config/unset.d.ts +2 -1
- package/dist/commands/config/unset.js +25 -24
- package/dist/commands/fetch.d.ts +7 -15
- package/dist/commands/fetch.js +57 -401
- package/dist/commands/import.d.ts +4 -9
- package/dist/commands/import.js +56 -284
- package/dist/commands/inspect.d.ts +7 -1
- package/dist/commands/inspect.js +103 -52
- package/dist/commands/list.d.ts +13 -1
- package/dist/commands/list.js +79 -32
- package/dist/commands/prune.d.ts +4 -1
- package/dist/commands/prune.js +55 -75
- package/dist/commands/status.d.ts +3 -2
- package/dist/commands/status.js +52 -79
- package/dist/hooks/command-not-found/suggest.d.ts +1 -0
- package/dist/hooks/command-not-found/suggest.js +66 -39
- package/dist/lib/argv.d.ts +24 -3
- package/dist/lib/argv.js +124 -53
- package/dist/lib/batch.d.ts +41 -0
- package/dist/lib/batch.js +39 -0
- package/dist/lib/cache-view.d.ts +21 -0
- package/dist/lib/cache-view.js +30 -0
- package/dist/lib/cli-emit.d.ts +28 -0
- package/dist/lib/cli-emit.js +44 -0
- package/dist/lib/cli-error-policy.d.ts +57 -0
- package/dist/lib/cli-error-policy.js +74 -0
- package/dist/lib/cli-flag-manifest.d.ts +5 -0
- package/dist/lib/cli-flag-manifest.js +8 -0
- package/dist/lib/cli-presentation.d.ts +39 -0
- package/dist/lib/cli-presentation.js +50 -0
- package/dist/lib/cli-result-types.d.ts +88 -0
- package/dist/lib/cli-result-types.js +2 -0
- package/dist/lib/config/index.d.ts +21 -3
- package/dist/lib/config/index.js +58 -3
- package/dist/lib/config-persist.d.ts +28 -0
- package/dist/lib/config-persist.js +29 -0
- package/dist/lib/envelope.d.ts +41 -3
- package/dist/lib/envelope.js +147 -4
- package/dist/lib/help-preflight.d.ts +5 -8
- package/dist/lib/help-preflight.js +13 -32
- package/dist/lib/json-meta.d.ts +0 -2
- package/dist/lib/json-meta.js +21 -25
- package/dist/lib/limit-flag.d.ts +11 -1
- package/dist/lib/limit-flag.js +19 -6
- package/dist/lib/parse-error-ux.d.ts +5 -0
- package/dist/lib/parse-error-ux.js +73 -0
- package/dist/lib/prune-flags.d.ts +28 -0
- package/dist/lib/prune-flags.js +74 -0
- package/dist/lib/research/artifact-index.d.ts +11 -2
- package/dist/lib/research/artifact-index.js +7 -4
- package/dist/lib/research/browser.d.ts +41 -2
- package/dist/lib/research/browser.js +139 -50
- package/dist/lib/research/capture.d.ts +24 -2
- package/dist/lib/research/capture.js +24 -4
- package/dist/lib/research/cli-io.d.ts +33 -0
- package/dist/lib/research/cli-io.js +27 -0
- package/dist/lib/research/compress.d.ts +7 -0
- package/dist/lib/research/compress.js +7 -3
- package/dist/lib/research/docs/capabilities.d.ts +15 -0
- package/dist/lib/research/docs/capabilities.js +5 -0
- package/dist/lib/research/docs/clean-dom.d.ts +2 -0
- package/dist/lib/research/docs/clean-dom.js +2 -0
- package/dist/lib/research/docs/code-blocks.d.ts +2 -0
- package/dist/lib/research/docs/code-blocks.js +2 -0
- package/dist/lib/research/docs/detect.d.ts +4 -0
- package/dist/lib/research/docs/detect.js +4 -0
- package/dist/lib/research/docs/machine-readable.d.ts +18 -0
- package/dist/lib/research/docs/machine-readable.js +10 -0
- package/dist/lib/research/docs/markdown-source.d.ts +20 -0
- package/dist/lib/research/docs/markdown-source.js +19 -2
- package/dist/lib/research/docs/quality-gates.d.ts +5 -0
- package/dist/lib/research/docs/quality-gates.js +4 -0
- package/dist/lib/research/docs/rsc.d.ts +9 -2
- package/dist/lib/research/docs/rsc.js +5 -2
- package/dist/lib/research/docs/section-artifacts.d.ts +12 -0
- package/dist/lib/research/docs/section-artifacts.js +12 -0
- package/dist/lib/research/docs/sections.d.ts +18 -0
- package/dist/lib/research/docs/sections.js +17 -1
- package/dist/lib/research/docs/source-map.d.ts +28 -0
- package/dist/lib/research/docs/source-map.js +27 -0
- package/dist/lib/research/docs/validate.d.ts +35 -2
- package/dist/lib/research/docs/validate.js +47 -2
- package/dist/lib/research/extract.d.ts +16 -0
- package/dist/lib/research/extract.js +12 -1
- package/dist/lib/research/fetch-command-service.d.ts +36 -0
- package/dist/lib/research/fetch-command-service.js +310 -0
- package/dist/lib/research/fetch-result.d.ts +114 -0
- package/dist/lib/research/fetch-result.js +212 -0
- package/dist/lib/research/fetcher.d.ts +46 -8
- package/dist/lib/research/fetcher.js +58 -30
- package/dist/lib/research/freshness.d.ts +54 -11
- package/dist/lib/research/freshness.js +91 -31
- package/dist/lib/research/import-command-service.d.ts +56 -0
- package/dist/lib/research/import-command-service.js +381 -0
- package/dist/lib/research/keywords.js +5 -1
- package/dist/lib/research/markdown.d.ts +12 -1
- package/dist/lib/research/markdown.js +13 -5
- package/dist/lib/research/persist-artifact.d.ts +23 -0
- package/dist/lib/research/persist-artifact.js +27 -0
- package/dist/lib/research/prompt-injection.d.ts +8 -0
- package/dist/lib/research/prompt-injection.js +42 -13
- package/dist/lib/research/proxy.d.ts +14 -8
- package/dist/lib/research/proxy.js +14 -10
- package/dist/lib/research/revalidate.d.ts +49 -20
- package/dist/lib/research/revalidate.js +80 -15
- package/dist/lib/research/schema.d.ts +7 -0
- package/dist/lib/research/schema.js +2 -0
- package/dist/lib/research/secret-scan.d.ts +6 -0
- package/dist/lib/research/secret-scan.js +8 -0
- package/dist/lib/research/secure-write.d.ts +4 -0
- package/dist/lib/research/secure-write.js +31 -2
- package/dist/lib/research/storage.d.ts +5 -1
- package/dist/lib/research/storage.js +8 -3
- package/dist/lib/research/summarize.d.ts +15 -0
- package/dist/lib/research/summarize.js +11 -0
- package/dist/lib/research/url.d.ts +9 -6
- package/dist/lib/research/url.js +33 -21
- package/dist/lib/text.d.ts +7 -2
- package/dist/lib/text.js +13 -2
- package/dist/lib/value-flags.d.ts +14 -0
- package/dist/lib/value-flags.js +28 -0
- package/dist/sites/coveo.d.ts +10 -0
- package/dist/sites/coveo.js +10 -0
- package/dist/sites/index.d.ts +16 -0
- package/dist/sites/index.js +16 -4
- package/dist/sites/markdown-twin.d.ts +31 -0
- package/dist/sites/markdown-twin.js +56 -0
- package/dist/sites/salesforce/fetch-page.d.ts +8 -2
- package/dist/sites/salesforce/fetch-page.js +14 -3
- package/dist/sites/salesforce/help-tooling-markdown.d.ts +30 -0
- package/dist/sites/salesforce/help-tooling-markdown.js +74 -0
- package/dist/sites/salesforce/index.d.ts +1 -0
- package/dist/sites/salesforce/index.js +1 -0
- package/dist/sites/salesforce-developer/fetch-page.d.ts +6 -1
- package/dist/sites/salesforce-developer/fetch-page.js +6 -1
- package/dist/sites/salesforce-developer/index.d.ts +1 -0
- package/dist/sites/salesforce-developer/index.js +1 -0
- package/dist/sites/salesforce-developer/route-markdown.d.ts +18 -6
- package/dist/sites/salesforce-developer/route-markdown.js +29 -67
- package/dist/sites/salesforce-doc-fetch.d.ts +39 -5
- package/dist/sites/salesforce-doc-fetch.js +52 -7
- package/dist/sites/salesforce-dom-probe.d.ts +20 -2
- package/dist/sites/salesforce-dom-probe.js +20 -2
- package/dist/sites/types.d.ts +14 -8
- package/dist/sites/types.js +3 -0
- package/oclif.manifest.json +104 -98
- package/package.json +19 -7
- package/agents/agents/official-docs-researcher.agent.md +0 -87
- package/agents/instructions/web-research.instructions.md +0 -44
- package/agents/skills/web-research/SKILL.md +0 -118
|
@@ -10,7 +10,11 @@ const ERROR_PAGE_PATTERNS = [
|
|
|
10
10
|
/something\s+went\s+wrong/i,
|
|
11
11
|
/an?\s+error\s+occurred/i,
|
|
12
12
|
];
|
|
13
|
-
/** True when short text reads like an error/soft-404 page rather than real content.
|
|
13
|
+
/** True when short text reads like an error/soft-404 page rather than real content.
|
|
14
|
+
*
|
|
15
|
+
* @param text - Body text to inspect (only short bodies trigger the heuristic).
|
|
16
|
+
* @returns True when the text matches an error/soft-404 pattern.
|
|
17
|
+
*/
|
|
14
18
|
export function looksLikeErrorPage(text) {
|
|
15
19
|
const trimmed = text.trim();
|
|
16
20
|
// Only treat short bodies as errors; a long article mentioning "404" once is fine.
|
|
@@ -18,7 +22,12 @@ export function looksLikeErrorPage(text) {
|
|
|
18
22
|
return false;
|
|
19
23
|
return ERROR_PAGE_PATTERNS.some((re) => re.test(trimmed));
|
|
20
24
|
}
|
|
21
|
-
/** Same docs origin guard: a probed artifact must live on the same hostname as the page.
|
|
25
|
+
/** Same docs origin guard: a probed artifact must live on the same hostname as the page.
|
|
26
|
+
*
|
|
27
|
+
* @param candidateUrl - URL of the artifact to validate.
|
|
28
|
+
* @param baseUrl - URL of the original page being researched.
|
|
29
|
+
* @returns True when both URLs share the same hostname.
|
|
30
|
+
*/
|
|
22
31
|
export function isSameDocsOrigin(candidateUrl, baseUrl) {
|
|
23
32
|
try {
|
|
24
33
|
return new URL(candidateUrl).hostname === new URL(baseUrl).hostname;
|
|
@@ -31,6 +40,9 @@ const MIN_BODY_CHARS = 10;
|
|
|
31
40
|
/**
|
|
32
41
|
* Validates that an `llms.txt` or route-Markdown body is usable: non-empty, not an HTML page, and
|
|
33
42
|
* not a soft error page. HTML returned for a `.md`/`.txt` probe is the most common false positive.
|
|
43
|
+
*
|
|
44
|
+
* @param body - Raw response body from the probe fetch.
|
|
45
|
+
* @returns `{ ok: true }` when the body is valid, or `{ ok: false, reason }` explaining the failure.
|
|
34
46
|
*/
|
|
35
47
|
export function validateTextArtifact(body) {
|
|
36
48
|
const trimmed = body.trim();
|
|
@@ -42,3 +54,36 @@ export function validateTextArtifact(body) {
|
|
|
42
54
|
return { ok: false, reason: 'body looks like an error page' };
|
|
43
55
|
return { ok: true };
|
|
44
56
|
}
|
|
57
|
+
const MARKDOWN_CONTENT_TYPE = 'text/markdown';
|
|
58
|
+
/**
|
|
59
|
+
* Validates a probed `.md`-route response as a trustworthy Markdown twin, the shared trust
|
|
60
|
+
* contract behind both the developer.salesforce.com and help.salesforce.com twin probes: the
|
|
61
|
+
* server must label it `text/markdown`, the redirect chain must land on the expected host over
|
|
62
|
+
* https (never a plain-http downgrade or an on-path host), and the body must pass
|
|
63
|
+
* {@link validateTextArtifact} (non-empty, non-HTML, not a soft-error page). A 200 HTML "not
|
|
64
|
+
* found" shell — the most common false positive for a `.md` probe — fails the content-type check
|
|
65
|
+
* before the body is even inspected.
|
|
66
|
+
*
|
|
67
|
+
* @param res - The probe response's content-type, final URL after redirects, and body.
|
|
68
|
+
* @param allowedHost - Hostname the redirect chain must land on (compared case-insensitively).
|
|
69
|
+
* @returns True when the response is a trustworthy Markdown twin.
|
|
70
|
+
*/
|
|
71
|
+
export function isValidatedMarkdownTwin(res, allowedHost) {
|
|
72
|
+
if (!res.content)
|
|
73
|
+
return false;
|
|
74
|
+
const mediaType = (res.contentType?.split(';')[0] ?? '').trim().toLowerCase();
|
|
75
|
+
if (mediaType !== MARKDOWN_CONTENT_TYPE)
|
|
76
|
+
return false;
|
|
77
|
+
let finalUrl;
|
|
78
|
+
try {
|
|
79
|
+
finalUrl = new URL(res.finalUrl);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
if (finalUrl.protocol !== 'https:' ||
|
|
85
|
+
finalUrl.hostname.toLowerCase() !== allowedHost.toLowerCase()) {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
return validateTextArtifact(res.content).ok;
|
|
89
|
+
}
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Readability-extracted content from a page, ready to store as a research artifact.
|
|
3
|
+
* `confidence` reflects the extracted text length; `qualityNotes` carry stable `quality:*` codes
|
|
4
|
+
* for downstream agent use. `isIndexHub` is true when the page is a nav hub rather than an article.
|
|
5
|
+
*/
|
|
1
6
|
export interface ExtractionResult {
|
|
2
7
|
title: string;
|
|
3
8
|
detailedMarkdown: string;
|
|
@@ -5,7 +10,18 @@ export interface ExtractionResult {
|
|
|
5
10
|
qualityNotes: string[];
|
|
6
11
|
isIndexHub?: boolean;
|
|
7
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* Prefix marking a `qualityNotes` entry as human-prose worth surfacing on stderr (as opposed to
|
|
15
|
+
* the stable `quality:*` machine codes in the same array, which are for agents parsing `--json`).
|
|
16
|
+
*/
|
|
17
|
+
export declare const QUALITY_WARNING_PREFIX = "warning: ";
|
|
8
18
|
/**
|
|
9
19
|
* Extracts readerable content from an HTML document, sanitizes it, and converts it to detailed Markdown.
|
|
20
|
+
*
|
|
21
|
+
* @param html - Raw HTML of the page to extract.
|
|
22
|
+
* @param finalUrl - The resolved URL after any redirects; used to resolve relative links.
|
|
23
|
+
* @returns Extracted title, Markdown body, confidence level, quality notes, and index-hub flag.
|
|
24
|
+
* @throws {Error} When `finalUrl` is not a valid URL, or when Readability cannot parse any
|
|
25
|
+
* article content from the page.
|
|
10
26
|
*/
|
|
11
27
|
export declare function extractHtmlContent(html: string, finalUrl: string): ExtractionResult;
|
|
@@ -40,12 +40,17 @@ function cleanUnsafeElements(document) {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Prefix marking a `qualityNotes` entry as human-prose worth surfacing on stderr (as opposed to
|
|
45
|
+
* the stable `quality:*` machine codes in the same array, which are for agents parsing `--json`).
|
|
46
|
+
*/
|
|
47
|
+
export const QUALITY_WARNING_PREFIX = 'warning: ';
|
|
43
48
|
function determineConfidence(textLength) {
|
|
44
49
|
const notes = ['readability extracted main article'];
|
|
45
50
|
let confidence = 'high';
|
|
46
51
|
if (textLength < 500) {
|
|
47
52
|
confidence = 'low';
|
|
48
|
-
notes.push(
|
|
53
|
+
notes.push(`${QUALITY_WARNING_PREFIX}extracted content is very short (less than 500 characters)`);
|
|
49
54
|
}
|
|
50
55
|
else if (textLength < 2000) {
|
|
51
56
|
confidence = 'medium';
|
|
@@ -54,6 +59,12 @@ function determineConfidence(textLength) {
|
|
|
54
59
|
}
|
|
55
60
|
/**
|
|
56
61
|
* Extracts readerable content from an HTML document, sanitizes it, and converts it to detailed Markdown.
|
|
62
|
+
*
|
|
63
|
+
* @param html - Raw HTML of the page to extract.
|
|
64
|
+
* @param finalUrl - The resolved URL after any redirects; used to resolve relative links.
|
|
65
|
+
* @returns Extracted title, Markdown body, confidence level, quality notes, and index-hub flag.
|
|
66
|
+
* @throws {Error} When `finalUrl` is not a valid URL, or when Readability cannot parse any
|
|
67
|
+
* article content from the page.
|
|
57
68
|
*/
|
|
58
69
|
export function extractHtmlContent(html, finalUrl) {
|
|
59
70
|
const { document } = parseHTML(html);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { StorageMode } from '../config/index.js';
|
|
2
|
+
import { type CliIo } from './cli-io.js';
|
|
3
|
+
/** Spinner port used while a fetch is in progress. */
|
|
4
|
+
export interface FetchCommandSpinner {
|
|
5
|
+
running(): boolean;
|
|
6
|
+
start(msg: string): void;
|
|
7
|
+
stop(msg?: string): void;
|
|
8
|
+
}
|
|
9
|
+
/** Parsed flags for one fetch command invocation. */
|
|
10
|
+
export interface FetchCommandFlags {
|
|
11
|
+
topic?: string;
|
|
12
|
+
tags?: string[];
|
|
13
|
+
format: 'compressed' | 'detailed';
|
|
14
|
+
tier: 'stable' | 'standard' | 'volatile';
|
|
15
|
+
ttl?: string;
|
|
16
|
+
maxAge?: string;
|
|
17
|
+
force: boolean;
|
|
18
|
+
allowStale: boolean;
|
|
19
|
+
rendered: boolean;
|
|
20
|
+
storage?: StorageMode;
|
|
21
|
+
}
|
|
22
|
+
/** Inputs for {@link runFetchCommandService}. */
|
|
23
|
+
export interface FetchCommandServiceOptions {
|
|
24
|
+
urls: string[];
|
|
25
|
+
flags: FetchCommandFlags;
|
|
26
|
+
dryRun: boolean;
|
|
27
|
+
io: CliIo;
|
|
28
|
+
spinner: FetchCommandSpinner;
|
|
29
|
+
}
|
|
30
|
+
/** Reject invalid duration values and contradictory flag pairs before any URL work starts. */
|
|
31
|
+
export declare function validateFetchCommandFlags(io: CliIo, flags: FetchCommandFlags): void;
|
|
32
|
+
/**
|
|
33
|
+
* Fetch/serve each URL against the cache and return the command's `--json` data payload
|
|
34
|
+
* (a single row, or an array for multi-URL batches with failures kept as rows).
|
|
35
|
+
*/
|
|
36
|
+
export declare function runFetchCommandService(opts: FetchCommandServiceOptions): Promise<unknown>;
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { Errors } from '@oclif/core';
|
|
2
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { finalizeBatch } from '../batch.js';
|
|
6
|
+
import { formatErrorForJson } from '../envelope.js';
|
|
7
|
+
import { loadSummaryLevel } from '../config/index.js';
|
|
8
|
+
import { buildFetchFailureFromCaught, buildFetchFailureResult, buildFetchResultData, describeError, extractionQualityWarnings, FETCH_STATUS_LABEL, fetchFailureGuidance, reportCacheStatus, } from './fetch-result.js';
|
|
9
|
+
import { durationFlagError, evaluateFreshnessWithMaxAge } from './freshness.js';
|
|
10
|
+
import { fetchRenderedHtml } from './browser.js';
|
|
11
|
+
import { capturePage } from './capture.js';
|
|
12
|
+
import { persistSectionArtifacts } from './docs/section-artifacts.js';
|
|
13
|
+
import { fetchStaticHtml, fetchText } from './fetcher.js';
|
|
14
|
+
import { applyAutoTags } from './keywords.js';
|
|
15
|
+
import { persistArtifact } from './persist-artifact.js';
|
|
16
|
+
import { createArtifactFromFetch, revalidateCache, } from './revalidate.js';
|
|
17
|
+
import { resolveResearchTarget } from './resolve-target.js';
|
|
18
|
+
import { failInvalidUrl } from './cli-io.js';
|
|
19
|
+
import { batchSeparator } from '../cache-view.js';
|
|
20
|
+
import { EXIT_STALE_SERVED } from '../cli-error-policy.js';
|
|
21
|
+
import { detectSite } from '../../sites/index.js';
|
|
22
|
+
import { applySiteFetchProvenance } from '../../sites/types.js';
|
|
23
|
+
const CAPTURE_DEPS = {
|
|
24
|
+
fetchStatic: fetchStaticHtml,
|
|
25
|
+
fetchRendered: fetchRenderedHtml,
|
|
26
|
+
fetchText,
|
|
27
|
+
};
|
|
28
|
+
/** Reject invalid duration values and contradictory flag pairs before any URL work starts. */
|
|
29
|
+
export function validateFetchCommandFlags(io, flags) {
|
|
30
|
+
for (const msg of [
|
|
31
|
+
durationFlagError('--ttl', flags.ttl),
|
|
32
|
+
durationFlagError('--max-age', flags.maxAge),
|
|
33
|
+
]) {
|
|
34
|
+
if (msg)
|
|
35
|
+
io.error(msg, { exit: 2, code: 'INVALID_DURATION' });
|
|
36
|
+
}
|
|
37
|
+
// --force skips cache lookup; --allow-stale only applies when serving a stale entry after a
|
|
38
|
+
// failed revalidation. Together they are a no-op combination that looks intentional -- reject it.
|
|
39
|
+
if (flags.force && flags.allowStale) {
|
|
40
|
+
io.error('Cannot combine --force with --allow-stale: --force ignores the cache entirely.', {
|
|
41
|
+
exit: 2,
|
|
42
|
+
code: 'CONFLICTING_FLAGS',
|
|
43
|
+
suggestions: [
|
|
44
|
+
'Use --force to fetch fresh content, or omit --force if you want a stale entry served ' +
|
|
45
|
+
'within grace (with --allow-stale suppressing the exit-5 signal).',
|
|
46
|
+
],
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Fetch/serve each URL against the cache and return the command's `--json` data payload
|
|
52
|
+
* (a single row, or an array for multi-URL batches with failures kept as rows).
|
|
53
|
+
*/
|
|
54
|
+
export async function runFetchCommandService(opts) {
|
|
55
|
+
const { dryRun, flags, io, spinner, urls } = opts;
|
|
56
|
+
const run = {
|
|
57
|
+
flags,
|
|
58
|
+
io,
|
|
59
|
+
spinner,
|
|
60
|
+
currentTime: new Date(),
|
|
61
|
+
tmpDir: dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null,
|
|
62
|
+
summaryLevel: loadSummaryLevel(io.configDir, io.cwd),
|
|
63
|
+
dryRun,
|
|
64
|
+
urlCount: urls.length,
|
|
65
|
+
};
|
|
66
|
+
try {
|
|
67
|
+
const results = [];
|
|
68
|
+
for (const url of urls) {
|
|
69
|
+
try {
|
|
70
|
+
results.push(await fetchSingleTarget(url, run));
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
results.push(failureRowOrRethrow(url, err, run));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return finalizeBatch(results, (row) => 'error' in row);
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
if (run.tmpDir)
|
|
80
|
+
rmSync(run.tmpDir, { recursive: true, force: true });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
async function executeCacheHit(cached, targetDir, run, secure) {
|
|
84
|
+
const { currentTime, flags, io } = run;
|
|
85
|
+
const freshnessState = evaluateFreshnessWithMaxAge(cached, currentTime, {
|
|
86
|
+
ttl: flags.ttl,
|
|
87
|
+
maxAge: flags.maxAge,
|
|
88
|
+
});
|
|
89
|
+
if (freshnessState === 'fresh') {
|
|
90
|
+
return { cacheStatus: 'hit', freshnessState, artifact: cached, redirectedToGlobal: false };
|
|
91
|
+
}
|
|
92
|
+
const revalidationResult = await revalidateCache(targetDir, cached, currentTime, {
|
|
93
|
+
allowStale: flags.allowStale,
|
|
94
|
+
ttlOverride: flags.ttl,
|
|
95
|
+
rendered: flags.rendered,
|
|
96
|
+
summaryLevel: run.summaryLevel,
|
|
97
|
+
secure,
|
|
98
|
+
});
|
|
99
|
+
if (revalidationResult.status === 'stale') {
|
|
100
|
+
const exitSuffix = revalidationResult.allowed ? '' : ` (exit ${EXIT_STALE_SERVED})`;
|
|
101
|
+
io.warn(`Serving stale content within grace period${exitSuffix}: revalidation failed (${revalidationResult.error}).`);
|
|
102
|
+
if (!revalidationResult.allowed)
|
|
103
|
+
process.exitCode = EXIT_STALE_SERVED;
|
|
104
|
+
}
|
|
105
|
+
if (revalidationResult.redirectWarning)
|
|
106
|
+
io.warn(revalidationResult.redirectWarning);
|
|
107
|
+
return {
|
|
108
|
+
cacheStatus: revalidationResult.status,
|
|
109
|
+
freshnessState,
|
|
110
|
+
artifact: revalidationResult.artifact,
|
|
111
|
+
storageDir: revalidationResult.storageDir,
|
|
112
|
+
redirectedToGlobal: Boolean(revalidationResult.redirectedToGlobal),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
async function executeCacheMiss(normalizedUrl, cacheKey, run) {
|
|
116
|
+
const { currentTime, flags, summaryLevel } = run;
|
|
117
|
+
const siteModule = detectSite(normalizedUrl);
|
|
118
|
+
const useRendered = flags.rendered || Boolean(siteModule?.defaults?.rendered);
|
|
119
|
+
let fetchResult;
|
|
120
|
+
let extraction;
|
|
121
|
+
let capture = null;
|
|
122
|
+
let siteFetch = null;
|
|
123
|
+
if (siteModule?.fetchPage) {
|
|
124
|
+
// Custom site modules own their fetch/extract strategy; the generic capture path is skipped.
|
|
125
|
+
siteFetch = await siteModule.fetchPage(normalizedUrl);
|
|
126
|
+
({ fetchResult, extraction } = siteFetch);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
capture = await capturePage(normalizedUrl, { forceRendered: useRendered }, CAPTURE_DEPS);
|
|
130
|
+
fetchResult = capture.fetchResult;
|
|
131
|
+
extraction = capture.extraction;
|
|
132
|
+
}
|
|
133
|
+
const artifact = createArtifactFromFetch({
|
|
134
|
+
url: normalizedUrl,
|
|
135
|
+
normalizedUrl,
|
|
136
|
+
cacheKey,
|
|
137
|
+
fetchResult,
|
|
138
|
+
extraction,
|
|
139
|
+
tier: flags.tier,
|
|
140
|
+
ttl: flags.ttl || null,
|
|
141
|
+
currentTime,
|
|
142
|
+
summaryLevel,
|
|
143
|
+
});
|
|
144
|
+
artifact.metadata.topic = flags.topic || null;
|
|
145
|
+
artifact.metadata.tags = flags.tags || [];
|
|
146
|
+
artifact.metadata.site_module_id = siteModule?.id ?? null;
|
|
147
|
+
if (capture) {
|
|
148
|
+
applyCaptureMetadata(artifact, capture);
|
|
149
|
+
}
|
|
150
|
+
else if (siteFetch?.captureMethod) {
|
|
151
|
+
applySiteFetchProvenance(artifact.metadata, siteFetch);
|
|
152
|
+
}
|
|
153
|
+
else if (useRendered) {
|
|
154
|
+
artifact.metadata.capture_method = 'browser_fallback';
|
|
155
|
+
}
|
|
156
|
+
// Auto-tag from the fetched content when the caller supplied none, keeping cached pages
|
|
157
|
+
// searchable by keyword. Explicit --tags always win (handled in applyAutoTags).
|
|
158
|
+
return applyAutoTags(artifact);
|
|
159
|
+
}
|
|
160
|
+
async function resolveArtifact(target, run) {
|
|
161
|
+
const { cacheKey, located, normalizedUrl, roots } = target;
|
|
162
|
+
if (located) {
|
|
163
|
+
// Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
|
|
164
|
+
// never mutated. Project-located entries reuse persistArtifact (secret-scan + redirect) so a
|
|
165
|
+
// refresh that gains a credential cannot stay in the committable project cache (#90).
|
|
166
|
+
const revalDir = run.tmpDir ?? located.dataDir;
|
|
167
|
+
const projectLocated = located.dataDir === roots.writeRoot && roots.mode === 'project';
|
|
168
|
+
const secure = projectLocated
|
|
169
|
+
? { roots, dryRun: Boolean(run.tmpDir), scratchDir: run.tmpDir }
|
|
170
|
+
: undefined;
|
|
171
|
+
const hit = await executeCacheHit(located.artifact, revalDir, run, secure);
|
|
172
|
+
return {
|
|
173
|
+
cacheStatus: hit.cacheStatus,
|
|
174
|
+
freshnessState: hit.freshnessState,
|
|
175
|
+
artifact: hit.artifact,
|
|
176
|
+
storageDir: hit.storageDir ?? located.dataDir,
|
|
177
|
+
redirectedToGlobal: hit.redirectedToGlobal,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
const artifact = await executeCacheMiss(normalizedUrl, cacheKey, run);
|
|
181
|
+
const write = persistArtifact({
|
|
182
|
+
roots,
|
|
183
|
+
cacheKey,
|
|
184
|
+
artifact,
|
|
185
|
+
dryRun: Boolean(run.tmpDir),
|
|
186
|
+
kind: 'fetch',
|
|
187
|
+
scratchDir: run.tmpDir,
|
|
188
|
+
});
|
|
189
|
+
if (write.redirectWarning)
|
|
190
|
+
run.io.warn(write.redirectWarning);
|
|
191
|
+
// No entry existed at lookup, so there is no prior freshness to report. 'none' (not
|
|
192
|
+
// 'stale_expired') keeps the field honest -- nothing expired; the page was simply uncached and
|
|
193
|
+
// has now been fetched fresh. Mirrors the `status` command's miss reporting.
|
|
194
|
+
return {
|
|
195
|
+
cacheStatus: 'miss',
|
|
196
|
+
freshnessState: 'none',
|
|
197
|
+
artifact,
|
|
198
|
+
storageDir: write.dataDir,
|
|
199
|
+
redirectedToGlobal: write.redirected,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
// Long references are split into searchable/inspectable section children whenever the page artifact
|
|
203
|
+
// is freshly written (T-22).
|
|
204
|
+
// ponytail: section persist is best-effort; failures are swallowed so chunking never fails the
|
|
205
|
+
// parent fetch. Upgrade: surface section errors in warnings/envelope when agents need them.
|
|
206
|
+
function persistSectionsIfFresh(targetDir, artifact, cacheStatus, run) {
|
|
207
|
+
if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
|
|
208
|
+
return;
|
|
209
|
+
try {
|
|
210
|
+
persistSectionArtifacts(targetDir, artifact, run.currentTime, run.summaryLevel);
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
/* ignored — see ponytail note above */
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
function failureRowOrRethrow(url, err, run) {
|
|
217
|
+
const { dryRun, io, spinner } = run;
|
|
218
|
+
// Invalid URLs fail before ux.action.start; stopping a never-started spinner prints noise.
|
|
219
|
+
if (!io.json && spinner.running())
|
|
220
|
+
spinner.stop('failed');
|
|
221
|
+
if (run.urlCount === 1) {
|
|
222
|
+
if (err instanceof Errors.CLIError)
|
|
223
|
+
throw err;
|
|
224
|
+
// Deep fetch/extract throws plain Errors; re-emit with actionable next steps.
|
|
225
|
+
const message = describeError(err);
|
|
226
|
+
const guidance = fetchFailureGuidance(message, url, io.bin);
|
|
227
|
+
io.error(message, {
|
|
228
|
+
exit: 1,
|
|
229
|
+
code: 'FETCH_FAILED',
|
|
230
|
+
suggestions: guidance?.suggestions,
|
|
231
|
+
ref: guidance?.ref,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
const row = err instanceof Errors.CLIError
|
|
235
|
+
? buildFetchFailureResult({ url, dryRun, err })
|
|
236
|
+
: buildFetchFailureFromCaught(io.bin, url, err, dryRun);
|
|
237
|
+
// Human batches only get the spinner "failed" label unless we echo the reason. Mirror the
|
|
238
|
+
// single-URL error format (message + Code: + Try this:) rather than a bare message, so a row
|
|
239
|
+
// failure in a batch is exactly as actionable as it is standalone.
|
|
240
|
+
if (!io.json)
|
|
241
|
+
io.warn(formatErrorForJson(row.error));
|
|
242
|
+
return row;
|
|
243
|
+
}
|
|
244
|
+
async function fetchSingleTarget(url, run) {
|
|
245
|
+
const { dryRun, flags, io, spinner } = run;
|
|
246
|
+
let target;
|
|
247
|
+
try {
|
|
248
|
+
target = resolveResearchTarget({
|
|
249
|
+
configDir: io.configDir,
|
|
250
|
+
cwd: io.cwd,
|
|
251
|
+
dataDir: io.dataDir,
|
|
252
|
+
url,
|
|
253
|
+
flagOverride: flags.storage,
|
|
254
|
+
lookup: !flags.force,
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
catch (err) {
|
|
258
|
+
failInvalidUrl(io.error, url, err.message);
|
|
259
|
+
}
|
|
260
|
+
const { cacheKey, normalizedUrl, roots } = target;
|
|
261
|
+
if (!io.json) {
|
|
262
|
+
spinner.start('Fetching ' + normalizedUrl);
|
|
263
|
+
}
|
|
264
|
+
const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await resolveArtifact(target, run);
|
|
265
|
+
persistSectionsIfFresh(run.tmpDir ?? storageDir, artifact, cacheStatus, run);
|
|
266
|
+
if (!io.json) {
|
|
267
|
+
spinner.stop(FETCH_STATUS_LABEL[reportCacheStatus(cacheStatus, dryRun)]);
|
|
268
|
+
}
|
|
269
|
+
const resultData = buildFetchResultData({
|
|
270
|
+
url,
|
|
271
|
+
normalizedUrl,
|
|
272
|
+
cacheKey,
|
|
273
|
+
storageDir,
|
|
274
|
+
storageMode: roots.mode,
|
|
275
|
+
cacheStatus,
|
|
276
|
+
freshnessState,
|
|
277
|
+
format: flags.format,
|
|
278
|
+
artifact,
|
|
279
|
+
redirectedToGlobal,
|
|
280
|
+
dryRun,
|
|
281
|
+
});
|
|
282
|
+
if (!io.json) {
|
|
283
|
+
if (dryRun && cacheStatus !== 'hit' && cacheStatus !== 'stale') {
|
|
284
|
+
io.log('[dry-run] Preview only — cache was not written.');
|
|
285
|
+
}
|
|
286
|
+
for (const warning of extractionQualityWarnings(resultData.source.qualityNotes)) {
|
|
287
|
+
io.warn(`${normalizedUrl}: ${warning}`);
|
|
288
|
+
}
|
|
289
|
+
io.log(resultData.content);
|
|
290
|
+
const separator = batchSeparator(run.urlCount > 1);
|
|
291
|
+
if (separator) {
|
|
292
|
+
io.log(`\n${separator}\n`);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return resultData;
|
|
296
|
+
}
|
|
297
|
+
// Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
|
|
298
|
+
function applyCaptureMetadata(artifact, capture) {
|
|
299
|
+
const meta = artifact.metadata;
|
|
300
|
+
meta.capture_method = capture.captureMethod;
|
|
301
|
+
meta.docs_engine = capture.capabilities.docsEngine ?? null;
|
|
302
|
+
meta.docs_framework = capture.capabilities.framework ?? null;
|
|
303
|
+
meta.source_doc_url = capture.sourceDocUrl;
|
|
304
|
+
meta.search_provider = capture.capabilities.search?.provider ?? null;
|
|
305
|
+
if (capture.extraction.isIndexHub)
|
|
306
|
+
meta.artifact_type = 'index';
|
|
307
|
+
if (capture.sourceDocUrl && !meta.source_urls.includes(capture.sourceDocUrl)) {
|
|
308
|
+
meta.source_urls.push(capture.sourceDocUrl);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import type { StorageMode } from '../config/index.js';
|
|
2
|
+
import type { CacheHitStatus, FreshnessState } from '../cli-result-types.js';
|
|
3
|
+
import type { ResearchArtifact } from './schema.js';
|
|
4
|
+
import { type CliErrorShape } from '../envelope.js';
|
|
5
|
+
/**
|
|
6
|
+
* Human-prose warnings pulled out of `qualityNotes` (e.g. very-short extraction). The remaining
|
|
7
|
+
* `quality:*` machine codes stay `--json`-only; this is what stderr shows in human-readable mode.
|
|
8
|
+
*/
|
|
9
|
+
export declare function extractionQualityWarnings(qualityNotes: readonly string[]): string[];
|
|
10
|
+
/** Cache status as reported to users: dry-run remaps write-implying statuses to `would_*`. */
|
|
11
|
+
export type ReportedCacheStatus = CacheHitStatus | 'would_fetch' | 'would_refresh' | 'would_revalidate';
|
|
12
|
+
/** Remap write-implying cache statuses when dry-run/read-only skipped persistence. */
|
|
13
|
+
export declare function reportCacheStatus(status: CacheHitStatus, dryRun: boolean): ReportedCacheStatus;
|
|
14
|
+
/** Human spinner labels for reported (possibly remapped) cache statuses. */
|
|
15
|
+
export declare const FETCH_STATUS_LABEL: Record<ReportedCacheStatus, string>;
|
|
16
|
+
export interface FetchResultInput {
|
|
17
|
+
url: string;
|
|
18
|
+
normalizedUrl: string;
|
|
19
|
+
cacheKey: string;
|
|
20
|
+
storageDir: string;
|
|
21
|
+
storageMode: StorageMode;
|
|
22
|
+
cacheStatus: CacheHitStatus;
|
|
23
|
+
freshnessState: FreshnessState;
|
|
24
|
+
format: 'compressed' | 'detailed';
|
|
25
|
+
artifact: Pick<ResearchArtifact, 'metadata' | 'compressed' | 'detailed'>;
|
|
26
|
+
redirectedToGlobal: boolean;
|
|
27
|
+
dryRun: boolean;
|
|
28
|
+
}
|
|
29
|
+
/** Machine-readable success/preview payload for one fetch URL. */
|
|
30
|
+
export declare function buildFetchResultData(input: FetchResultInput): {
|
|
31
|
+
schemaVersion: number;
|
|
32
|
+
command: string;
|
|
33
|
+
dryRun: boolean;
|
|
34
|
+
cache: {
|
|
35
|
+
key: string;
|
|
36
|
+
status: ReportedCacheStatus;
|
|
37
|
+
freshness: FreshnessState;
|
|
38
|
+
path: string;
|
|
39
|
+
storage: StorageMode;
|
|
40
|
+
redirectedToGlobal: boolean;
|
|
41
|
+
};
|
|
42
|
+
source: {
|
|
43
|
+
url: string;
|
|
44
|
+
normalizedUrl: string;
|
|
45
|
+
captureMethod: "static_fetch" | "browser_fallback" | "agent_supplied" | "route_markdown" | "github_source" | null;
|
|
46
|
+
extractionStatus: "agent_supplied" | "extracted" | "failed" | null;
|
|
47
|
+
extractionConfidence: "high" | "medium" | "low" | null;
|
|
48
|
+
qualityNotes: string[];
|
|
49
|
+
fetchedAt: string | null;
|
|
50
|
+
validatedAt: string | null;
|
|
51
|
+
staleAfter: string | null;
|
|
52
|
+
};
|
|
53
|
+
artifactType: "source" | "research_note" | "index" | "section";
|
|
54
|
+
docsEngine: string | null;
|
|
55
|
+
docsFramework: string | null;
|
|
56
|
+
sourceDocUrl: string | null;
|
|
57
|
+
searchProvider: string | null;
|
|
58
|
+
format: "compressed" | "detailed";
|
|
59
|
+
tokenEstimate: number | null;
|
|
60
|
+
content: string;
|
|
61
|
+
};
|
|
62
|
+
export interface FetchFailureGuidance {
|
|
63
|
+
suggestions?: string[];
|
|
64
|
+
ref?: string;
|
|
65
|
+
}
|
|
66
|
+
/** Per-URL failure row for multi-URL batches — keeps prior successes in `data`. */
|
|
67
|
+
export declare function buildFetchFailureResult(input: {
|
|
68
|
+
url: string;
|
|
69
|
+
dryRun: boolean;
|
|
70
|
+
err: CliErrorShape;
|
|
71
|
+
fallbackGuidance?: FetchFailureGuidance;
|
|
72
|
+
}): {
|
|
73
|
+
schemaVersion: number;
|
|
74
|
+
command: string;
|
|
75
|
+
dryRun: boolean;
|
|
76
|
+
error: {
|
|
77
|
+
code: string;
|
|
78
|
+
message: string;
|
|
79
|
+
suggestions?: string[];
|
|
80
|
+
ref?: string;
|
|
81
|
+
};
|
|
82
|
+
cache: null;
|
|
83
|
+
source: {
|
|
84
|
+
url: string;
|
|
85
|
+
normalizedUrl: null;
|
|
86
|
+
};
|
|
87
|
+
content: null;
|
|
88
|
+
};
|
|
89
|
+
/**
|
|
90
|
+
* Shape a caught runtime error into a batch failure row (message + guidance).
|
|
91
|
+
* CLIError rows should call {@link buildFetchFailureResult} directly so oclif fields are preserved.
|
|
92
|
+
*/
|
|
93
|
+
export declare function buildFetchFailureFromCaught(bin: string, url: string, err: unknown, dryRun: boolean): {
|
|
94
|
+
schemaVersion: number;
|
|
95
|
+
command: string;
|
|
96
|
+
dryRun: boolean;
|
|
97
|
+
error: {
|
|
98
|
+
code: string;
|
|
99
|
+
message: string;
|
|
100
|
+
suggestions?: string[];
|
|
101
|
+
ref?: string;
|
|
102
|
+
};
|
|
103
|
+
cache: null;
|
|
104
|
+
source: {
|
|
105
|
+
url: string;
|
|
106
|
+
normalizedUrl: null;
|
|
107
|
+
};
|
|
108
|
+
content: null;
|
|
109
|
+
};
|
|
110
|
+
export declare function describeError(err: unknown): string;
|
|
111
|
+
export declare function fetchFailureGuidance(message: string, url: string, bin?: string): {
|
|
112
|
+
suggestions: string[];
|
|
113
|
+
ref?: string;
|
|
114
|
+
} | undefined;
|