@lincoln504/pi-research 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/cli.mjs +1404 -600
- package/dist/thread-worker.mjs +40 -11
- package/docs/AGENT-SKILL.md +11 -4
- package/docs/ARCHITECTURE.md +19 -9
- package/docs/CHANGELOG.md +107 -1
- package/docs/CONFIGURATION.md +10 -8
- package/docs/KNOWLEDGE-STORE.md +5 -4
- package/docs/PI-EXTENSION.md +1 -1
- package/docs/SDK.md +63 -5
- package/docs/THIRD-PARTY-NOTICES.md +1 -1
- package/package.json +5 -4
- package/scripts/build.cjs +23 -5
- package/scripts/cleanup.cjs +50 -10
- package/scripts/prepare.cjs +15 -5
- package/scripts/setup.cjs +24 -4
- package/skills/pi-research/README.md +9 -2
- package/skills/pi-research/SKILL.md +8 -5
- package/skills/pi-research/references/configuration.md +7 -3
- package/skills/pi-research/scripts/run.mjs +24 -2
- package/src/cli.ts +161 -21
- package/src/config.ts +56 -15
- package/src/constants.ts +31 -0
- package/src/core/interfaces/knowledge-interfaces.ts +6 -1
- package/src/core/interfaces/state-manager-interfaces.ts +1 -1
- package/src/core/llm/model-registry-factory.ts +40 -3
- package/src/core/llm/research-model-resolver.ts +2 -2
- package/src/core/pi-version.ts +136 -0
- package/src/core/service-registry.ts +117 -8
- package/src/index.ts +11 -18
- package/src/infrastructure/browser/browser-client.ts +16 -4
- package/src/infrastructure/browser/browser-error-utils.ts +12 -1
- package/src/infrastructure/browser/browser-server.ts +17 -5
- package/src/infrastructure/browser/browser-task-scheduler.ts +101 -23
- package/src/infrastructure/browser/cleanup-utils.ts +45 -7
- package/src/infrastructure/browser/client-agent.ts +4 -3
- package/src/infrastructure/browser/priority-task-queue.ts +10 -3
- package/src/infrastructure/browser/scheduler-factory.ts +26 -1
- package/src/infrastructure/browser/task-execution-service.ts +92 -28
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
- package/src/infrastructure/browser/thread-worker.mjs +40 -11
- package/src/infrastructure/browser/worker-pool-manager.ts +11 -3
- package/src/infrastructure/embedding/embedding-client.ts +22 -8
- package/src/infrastructure/embedding/embedding-factory.ts +28 -11
- package/src/infrastructure/embedding/embedding-server.ts +65 -4
- package/src/infrastructure/file-lock-service.ts +223 -26
- package/src/infrastructure/process-lifecycle-service.ts +108 -31
- package/src/infrastructure/state/state-backup-manager.ts +5 -7
- package/src/infrastructure/state/state-manager-service.ts +1 -1
- package/src/infrastructure/state/state-manager.ts +19 -20
- package/src/infrastructure/types/state-types.ts +9 -0
- package/src/knowledge/embedder-init.ts +1 -1
- package/src/knowledge/embedder.ts +52 -3
- package/src/knowledge/index.ts +1 -1
- package/src/knowledge/store.ts +35 -9
- package/src/knowledge/webgpu-viability.ts +9 -4
- package/src/knowledge/writer-queue.ts +88 -46
- package/src/orchestration/deep-research-orchestrator.ts +13 -2
- package/src/orchestration/quick-research-orchestrator.ts +2 -2
- package/src/orchestration/research-orchestration-service.ts +2 -4
- package/src/orchestration/research-synthesis-service.ts +34 -92
- package/src/security/cisa-kev.ts +10 -3
- package/src/security/github-advisories.ts +18 -7
- package/src/security/nvd-client.ts +17 -6
- package/src/security/osv-client.ts +14 -4
- package/src/skill-install/skill-installer.ts +135 -9
- package/src/stackexchange/rest-client.ts +6 -2
- package/src/tools/research-knowledge-search.ts +23 -2
- package/src/tools/search.ts +6 -1
- package/src/utils/atomic-replace.ts +101 -0
- package/src/utils/circuit-breaker.ts +20 -0
- package/src/utils/citation-utils.ts +106 -8
- package/src/utils/http-body.ts +176 -0
- package/src/utils/llm-usage.ts +37 -0
- package/src/utils/metrics-summary.ts +25 -2
- package/src/utils/research-export.ts +9 -2
- package/src/web-research/browser-search.ts +50 -4
- package/src/web-research/retry-utils.ts +13 -1
- package/src/web-research/search.ts +20 -5
- package/src/web-research/types.ts +17 -4
- package/src/web-research/web-scraper.ts +7 -50
- package/src/youtube/transcript-client.ts +5 -1
|
@@ -481,7 +481,10 @@ export async function runBackgroundExtraction(
|
|
|
481
481
|
const delay = KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS * attempt;
|
|
482
482
|
metrics.increment('research_knowledge_search_synthesis_retries_total', 1);
|
|
483
483
|
logger.warn(`[research-knowledge-search] Synthesis attempt ${attempt}/${KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS} failed transiently (${msg}); retrying in ${delay}ms`);
|
|
484
|
-
|
|
484
|
+
// keepAlive=true: the retry is already committed and the next attempt must
|
|
485
|
+
// run, so this sleep is foreground work. An unref'd timer here can be the
|
|
486
|
+
// only pending handle and let the process exit mid-retry (see abortableDelay).
|
|
487
|
+
await abortableDelay(delay, signal, true);
|
|
485
488
|
}
|
|
486
489
|
}
|
|
487
490
|
|
|
@@ -625,7 +628,10 @@ export async function triageRelevantUrls(
|
|
|
625
628
|
const delay = KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS * attempt;
|
|
626
629
|
metrics.increment('research_knowledge_search_triage_retries_total', 1);
|
|
627
630
|
logger.warn(`[research-knowledge-search] Triage attempt ${attempt}/${KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS} failed transiently (${msg}); retrying in ${delay}ms`);
|
|
628
|
-
|
|
631
|
+
// keepAlive=true: the retry is already committed and the next attempt must
|
|
632
|
+
// run, so this sleep is foreground work. An unref'd timer here can be the
|
|
633
|
+
// only pending handle and let the process exit mid-retry (see abortableDelay).
|
|
634
|
+
await abortableDelay(delay, signal, true);
|
|
629
635
|
}
|
|
630
636
|
}
|
|
631
637
|
|
|
@@ -956,6 +962,21 @@ export function createResearchKnowledgeSearchTool(iface?: ConfigInterface): Tool
|
|
|
956
962
|
return buildSteeringResult(result, urls);
|
|
957
963
|
} catch (error) {
|
|
958
964
|
const durationMs = Date.now() - startTime;
|
|
965
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
966
|
+
|
|
967
|
+
// A CANCELLED search is not an empty store. Reporting the miss string here
|
|
968
|
+
// told the calling agent, definitively, that the knowledge store has
|
|
969
|
+
// nothing on this topic — so it would go straight to full live research,
|
|
970
|
+
// the very work the user just interrupted. Propagate the abort instead;
|
|
971
|
+
// the run is ending either way. (Same misclassification as the search
|
|
972
|
+
// layer's "browser workers may be unavailable" on Ctrl-C.)
|
|
973
|
+
if (signal?.aborted || (error instanceof Error && error.name === 'AbortError') || /^(aborted|research (aborted|cancelled))$/i.test(msg)) {
|
|
974
|
+
metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'cancelled' });
|
|
975
|
+
metrics.increment('research_knowledge_search_total', 1, { status: 'cancelled' });
|
|
976
|
+
logger.debug('[research-knowledge-search] Cancelled mid-search — propagating rather than reporting an empty store.');
|
|
977
|
+
throw error;
|
|
978
|
+
}
|
|
979
|
+
|
|
959
980
|
metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'error' });
|
|
960
981
|
metrics.increment('research_knowledge_search_total', 1, { status: 'error' });
|
|
961
982
|
|
package/src/tools/search.ts
CHANGED
|
@@ -106,7 +106,12 @@ export function createSearchTool(options: {
|
|
|
106
106
|
results.forEach((r, i) => {
|
|
107
107
|
markdown += `## Query ${i + 1}: ${r.query}\n`;
|
|
108
108
|
if (r.results.length === 0) {
|
|
109
|
-
|
|
109
|
+
// Render the attributed failure, not a bare "no results": a timeout or
|
|
110
|
+
// dead worker says nothing about the query, and presenting it as an
|
|
111
|
+
// empty result sent the model off rewriting perfectly good queries —
|
|
112
|
+
// the misdirection QueryFailure exists to prevent. This is the only
|
|
113
|
+
// surface the researcher agent actually reads.
|
|
114
|
+
markdown += r.error ? `*${r.error.message}*\n\n` : `*No results found.*\n\n`;
|
|
110
115
|
} else {
|
|
111
116
|
r.results.forEach((item, j) => {
|
|
112
117
|
markdown += `[${j + 1}] **${item.title}**\n${item.url}\n${item.content}\n\n`;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic file replacement, with the Windows caveat handled honestly.
|
|
3
|
+
*
|
|
4
|
+
* The temp-file + fsync + rename dance exists so a reader never observes a partial
|
|
5
|
+
* file and a crash never leaves one: rename is atomic, so the destination is either
|
|
6
|
+
* entirely the old content or entirely the new.
|
|
7
|
+
*
|
|
8
|
+
* On Windows that guarantee is easy to lose. The long-standing comment at these call
|
|
9
|
+
* sites — "fs.rename fails on Windows if target exists (NTFS)" — is not true: Node
|
|
10
|
+
* calls `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`, so replacing an existing file
|
|
11
|
+
* is fine. What actually fails is a rename whose TARGET is held open by another
|
|
12
|
+
* process without delete sharing (an AV scanner mid-scan, a backup agent), which
|
|
13
|
+
* surfaces as EPERM/EBUSY/EACCES.
|
|
14
|
+
*
|
|
15
|
+
* The old remedy — fall straight to `copyFile` — traded a transient, retryable error
|
|
16
|
+
* for a permanent hazard: `copyFile` truncates the destination and rewrites it in
|
|
17
|
+
* place, so a concurrent reader or a crash mid-copy sees a torn or zero-length file.
|
|
18
|
+
* For `state.json` (leader election, server auth secrets) and `config.env` (API keys)
|
|
19
|
+
* that is a much worse outcome than the error it was avoiding.
|
|
20
|
+
*
|
|
21
|
+
* So: retry the rename first. An AV lock clears in tens of milliseconds, and a retried
|
|
22
|
+
* rename stays atomic. Only a lock that outlives every retry falls back to copying,
|
|
23
|
+
* which is reported to the caller so it can be logged as the degraded path it is.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import * as fs from 'node:fs/promises';
|
|
27
|
+
import * as fsSync from 'node:fs';
|
|
28
|
+
|
|
29
|
+
/** Transient Windows sharing/locking failures — worth retrying, unlike ENOENT/ENOSPC. */
|
|
30
|
+
const RETRYABLE_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']);
|
|
31
|
+
|
|
32
|
+
const MAX_RENAME_ATTEMPTS = 5;
|
|
33
|
+
const RENAME_RETRY_DELAY_MS = 25;
|
|
34
|
+
|
|
35
|
+
export type ReplaceOutcome = 'renamed' | 'copied';
|
|
36
|
+
|
|
37
|
+
function isRetryable(err: unknown): boolean {
|
|
38
|
+
const code = (err as NodeJS.ErrnoException | undefined)?.code;
|
|
39
|
+
return code !== undefined && RETRYABLE_CODES.has(code);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Replace `destPath` with `tmpPath` atomically where the platform allows it.
|
|
44
|
+
*
|
|
45
|
+
* Returns `'renamed'` for the atomic path and `'copied'` for the degraded Windows
|
|
46
|
+
* fallback. Rethrows on every non-Windows failure, and on Windows failures that are
|
|
47
|
+
* not lock-related.
|
|
48
|
+
*/
|
|
49
|
+
export async function replaceFile(tmpPath: string, destPath: string): Promise<ReplaceOutcome> {
|
|
50
|
+
let lastError: unknown;
|
|
51
|
+
for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
|
|
52
|
+
try {
|
|
53
|
+
await fs.rename(tmpPath, destPath);
|
|
54
|
+
return 'renamed';
|
|
55
|
+
} catch (err) {
|
|
56
|
+
lastError = err;
|
|
57
|
+
// Only a Windows sharing conflict is worth another try; anything else (and any
|
|
58
|
+
// failure on a POSIX platform, where rename-over-existing always works) is real.
|
|
59
|
+
if (process.platform !== 'win32' || !isRetryable(err)) throw err;
|
|
60
|
+
if (attempt < MAX_RENAME_ATTEMPTS - 1) {
|
|
61
|
+
await new Promise((r) => setTimeout(r, RENAME_RETRY_DELAY_MS * (attempt + 1)));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Every retry lost to the same lock. Copying is not atomic — a reader can catch the
|
|
67
|
+
// destination mid-write — but it is the only remaining way to persist the data.
|
|
68
|
+
try {
|
|
69
|
+
await fs.copyFile(tmpPath, destPath);
|
|
70
|
+
return 'copied';
|
|
71
|
+
} catch {
|
|
72
|
+
throw lastError;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Synchronous {@link replaceFile}, for the config writer, which is sync throughout. */
|
|
77
|
+
export function replaceFileSync(tmpPath: string, destPath: string): ReplaceOutcome {
|
|
78
|
+
let lastError: unknown;
|
|
79
|
+
for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
|
|
80
|
+
try {
|
|
81
|
+
fsSync.renameSync(tmpPath, destPath);
|
|
82
|
+
return 'renamed';
|
|
83
|
+
} catch (err) {
|
|
84
|
+
lastError = err;
|
|
85
|
+
if (process.platform !== 'win32' || !isRetryable(err)) throw err;
|
|
86
|
+
if (attempt < MAX_RENAME_ATTEMPTS - 1) {
|
|
87
|
+
// Sync busy-wait: this path is rare, bounded (~250ms worst case) and the
|
|
88
|
+
// caller is already blocking on a synchronous write.
|
|
89
|
+
const until = Date.now() + RENAME_RETRY_DELAY_MS * (attempt + 1);
|
|
90
|
+
while (Date.now() < until) { /* spin */ }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
fsSync.copyFileSync(tmpPath, destPath);
|
|
97
|
+
return 'copied';
|
|
98
|
+
} catch {
|
|
99
|
+
throw lastError;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -78,6 +78,26 @@ export class CircuitBreaker {
|
|
|
78
78
|
metrics.observe('circuit_breaker_call_duration_ms', duration, { breaker: this.options.name, status: 'error' });
|
|
79
79
|
if (this.options.isTransientError(error)) {
|
|
80
80
|
this.onFailure(error);
|
|
81
|
+
} else if (admittedAsProbe && this.state === 'HALF_OPEN') {
|
|
82
|
+
// The trial probe failed for a reason this breaker does not count. That is
|
|
83
|
+
// INCONCLUSIVE, not healthy — so re-open and wait out another reset window
|
|
84
|
+
// rather than staying HALF_OPEN.
|
|
85
|
+
//
|
|
86
|
+
// Without this the breaker pins permanently: onFailure is the only path to
|
|
87
|
+
// OPEN, and it is skipped for non-counted errors, while transitionTo(HALF_OPEN)
|
|
88
|
+
// never re-arms nextAttemptTime. The admission gate above then fast-fails every
|
|
89
|
+
// caller beyond halfOpenMaxCalls (default 1) forever. This is not a corner case
|
|
90
|
+
// for the browser pool: DEFAULT_BREAKER_CONFIG.isTransientError deliberately
|
|
91
|
+
// excludes pool-shutdown, draining, Cloudflare and task-timeout errors — exactly
|
|
92
|
+
// what a recovering or blocked pool returns — so the probe most likely to run is
|
|
93
|
+
// also the one most likely to pin the breaker. Observed effect: 19 of a 20-query
|
|
94
|
+
// parallel burst fast-failing instantly, surfacing to the researcher as
|
|
95
|
+
// "your query may be too narrow".
|
|
96
|
+
logger.warn(
|
|
97
|
+
`[CircuitBreaker] '${this.options.name}' HALF_OPEN probe failed with a non-counted error; ` +
|
|
98
|
+
`treating as inconclusive and re-opening: ${error instanceof Error ? error.message : String(error)}`
|
|
99
|
+
);
|
|
100
|
+
this.transitionTo('OPEN');
|
|
81
101
|
}
|
|
82
102
|
throw error;
|
|
83
103
|
} finally {
|
|
@@ -16,7 +16,9 @@ export interface GlobalCitation extends Citation {
|
|
|
16
16
|
* @param reports - Map of researcher ID to report content
|
|
17
17
|
* @returns Object containing normalized reports and the global citation list
|
|
18
18
|
*/
|
|
19
|
-
|
|
19
|
+
// ReadonlyMap: this only reads `reports` (it returns a NEW normalizedReports map),
|
|
20
|
+
// and callers now pass a read-only view to avoid create-on-read side effects.
|
|
21
|
+
export function normalizeCitations(reports: ReadonlyMap<string, string>): {
|
|
20
22
|
normalizedReports: Map<string, string>;
|
|
21
23
|
globalCitations: GlobalCitation[];
|
|
22
24
|
} {
|
|
@@ -67,13 +69,28 @@ export function normalizeCitations(reports: Map<string, string>): {
|
|
|
67
69
|
const headerIdx = lastCitedLinksHeaderIndex(report);
|
|
68
70
|
let content = headerIdx >= 0 ? report.slice(0, headerIdx) : report;
|
|
69
71
|
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
72
|
+
// Renumber local [N] markers to their global ids, and DELETE the ones that could
|
|
73
|
+
// not be mapped.
|
|
74
|
+
//
|
|
75
|
+
// Leaving an unmapped marker in place (the previous `return match`) was a silent
|
|
76
|
+
// misattribution bug, not a harmless no-op: global ids are assigned densely from 1,
|
|
77
|
+
// so a surviving `[1]` no longer means "this report's first source" — it resolves
|
|
78
|
+
// against the renumbered global list and points at a completely different document.
|
|
79
|
+
// Markers go unmapped whenever parseCitations dropped the entry (e.g. a URL whose
|
|
80
|
+
// hostname has no dot: `localhost`, a bare internal host, a malformed line). The
|
|
81
|
+
// result is fed straight into the evaluator/synthesis prompt, which is explicitly
|
|
82
|
+
// told the numbering is already normalized — so the model faithfully reproduces the
|
|
83
|
+
// wrong attribution, and ensureCitedLinks then regenerates a CITED LINKS list that
|
|
84
|
+
// looks perfectly consistent with it.
|
|
85
|
+
//
|
|
86
|
+
// rewriteCitationMarkers is the same routine the synthesis path already uses: it
|
|
87
|
+
// skips fenced blocks and inline code spans (so a quoted `items[12]` survives),
|
|
88
|
+
// bounds what counts as a marker, and preserves line-leading indentation.
|
|
89
|
+
content = rewriteCitationMarkers(
|
|
90
|
+
content,
|
|
91
|
+
localToGlobal,
|
|
92
|
+
(n) => !localToGlobal.has(n),
|
|
93
|
+
);
|
|
77
94
|
|
|
78
95
|
normalizedReports.set(id, content.trim());
|
|
79
96
|
}
|
|
@@ -95,3 +112,84 @@ export function formatCitedLinks(citations: GlobalCitation[]): string {
|
|
|
95
112
|
|
|
96
113
|
return `CITED LINKS\n${links.join('\n')}`;
|
|
97
114
|
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Largest `[N]` treated as a citation marker. Inline markers index a source list
|
|
118
|
+
* that is realistically well under this; the bound exists so an ordinary bracketed
|
|
119
|
+
* number in prose — a year (`[2023]`), an issue id, an array index — is left alone
|
|
120
|
+
* instead of being silently deleted as a "dangling citation".
|
|
121
|
+
*/
|
|
122
|
+
export const MAX_PLAUSIBLE_CITATION_MARKER = 200;
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Rewrite inline `[N]` citation markers in a prose body: renumber the ones we can
|
|
126
|
+
* map, delete the ones known to dangle, leave everything else untouched.
|
|
127
|
+
*
|
|
128
|
+
* Deletion is whitespace-aware rather than followed by a global space collapse.
|
|
129
|
+
* A blanket `text.replace(/ +/g, ' ')` also eats *leading* indentation, which
|
|
130
|
+
* silently flattens nested list levels into one and turns 4-space indented code
|
|
131
|
+
* blocks into paragraphs — corrupting the structure of every report that contains
|
|
132
|
+
* them. So the pad preceding a deleted marker is consumed only when the marker is
|
|
133
|
+
* mid-line (i.e. real prose spacing); line-leading indentation is preserved.
|
|
134
|
+
*
|
|
135
|
+
* Code is skipped entirely. A report may quote code containing bracketed integers
|
|
136
|
+
* (`items[12]`), and on the no-CITED-LINKS path *every* out-of-range marker is treated
|
|
137
|
+
* as dangling — so without this guard a quoted array index would be silently deleted
|
|
138
|
+
* from the user's code sample. Fenced blocks are tracked across lines; single-backtick
|
|
139
|
+
* spans are detected by an odd backtick count earlier on the same line.
|
|
140
|
+
*
|
|
141
|
+
* @param remap local marker number → verified global id
|
|
142
|
+
* @param isDangling whether a marker not present in `remap` should be deleted
|
|
143
|
+
*/
|
|
144
|
+
export function rewriteCitationMarkers(
|
|
145
|
+
text: string,
|
|
146
|
+
remap: Map<number, number>,
|
|
147
|
+
isDangling: (n: number) => boolean,
|
|
148
|
+
): string {
|
|
149
|
+
const inFence = fencedCodeMask(text);
|
|
150
|
+
return text.replace(/([ \t]*)\[(\d+)\]/g, (match, pad: string, digits: string, offset: number) => {
|
|
151
|
+
const n = parseInt(digits, 10);
|
|
152
|
+
const mapped = remap.get(n);
|
|
153
|
+
if (mapped !== undefined) return `${pad}[${mapped}]`;
|
|
154
|
+
// Citation numbering is 1-based; `[0]` is always an index or a literal, never a marker.
|
|
155
|
+
if (n < 1 || n > MAX_PLAUSIBLE_CITATION_MARKER || !isDangling(n)) return match;
|
|
156
|
+
if (inFence(offset) || inInlineCodeSpan(text, offset)) return match;
|
|
157
|
+
// Preserve indentation when nothing but whitespace precedes on this line;
|
|
158
|
+
// otherwise drop the pad too so no double space is left behind.
|
|
159
|
+
const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
|
|
160
|
+
return text.slice(lineStart, offset).trim() === '' ? pad : '';
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Build an `offset → is inside a ``` fenced block` predicate for `text`.
|
|
166
|
+
*
|
|
167
|
+
* Computed once per rewrite (a single line scan) rather than per marker, so marker
|
|
168
|
+
* rewriting stays linear in the body length.
|
|
169
|
+
*/
|
|
170
|
+
function fencedCodeMask(text: string): (offset: number) => boolean {
|
|
171
|
+
const ranges: Array<[number, number]> = [];
|
|
172
|
+
let open = -1;
|
|
173
|
+
let pos = 0;
|
|
174
|
+
for (const line of text.split('\n')) {
|
|
175
|
+
if (/^\s{0,3}(```|~~~)/.test(line)) {
|
|
176
|
+
if (open < 0) open = pos;
|
|
177
|
+
else {
|
|
178
|
+
ranges.push([open, pos + line.length]);
|
|
179
|
+
open = -1;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
pos += line.length + 1; // +1 for the '\n' consumed by split
|
|
183
|
+
}
|
|
184
|
+
// An unterminated fence runs to the end of the body — treat the remainder as code.
|
|
185
|
+
if (open >= 0) ranges.push([open, text.length]);
|
|
186
|
+
return (offset: number) => ranges.some(([start, end]) => offset >= start && offset < end);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Whether `offset` sits inside a single-backtick code span on its own line. */
|
|
190
|
+
function inInlineCodeSpan(text: string, offset: number): boolean {
|
|
191
|
+
const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
|
|
192
|
+
let ticks = 0;
|
|
193
|
+
for (let i = lineStart; i < offset; i++) if (text[i] === '`') ticks++;
|
|
194
|
+
return ticks % 2 === 1;
|
|
195
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capped response-body reading.
|
|
3
|
+
*
|
|
4
|
+
* `response.json()` and `response.text()` buffer the WHOLE body into the main
|
|
5
|
+
* process before anything can inspect its size — the byte count is only knowable
|
|
6
|
+
* after the memory has already been committed. A hostile or merely broken upstream
|
|
7
|
+
* can therefore drive this process to an OOM kill with a single reply, and because
|
|
8
|
+
* a `Content-Length` header is advisory (absent on chunked encoding, and freely
|
|
9
|
+
* falsifiable) no pre-screen closes that hole. Streaming chunk-by-chunk and
|
|
10
|
+
* aborting the instant the cap is crossed is the only check that actually bounds
|
|
11
|
+
* the allocation.
|
|
12
|
+
*
|
|
13
|
+
* The scraper has read HTML and PDF this way for some time; these helpers exist so
|
|
14
|
+
* the JSON API clients share that one implementation rather than each growing a
|
|
15
|
+
* near-copy of it.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Thrown by {@link readBodyCapped} when a streamed body crosses its byte cap. */
|
|
19
|
+
export class BodyTooLargeError extends Error {
|
|
20
|
+
constructor(public readonly bytesRead: number, public readonly maxBytes: number) {
|
|
21
|
+
super(`Response body exceeded ${maxBytes} byte cap`);
|
|
22
|
+
this.name = 'BodyTooLargeError';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Default cap for a JSON API response.
|
|
28
|
+
*
|
|
29
|
+
* Every endpoint behind these clients answers in kilobytes for a normal query, so
|
|
30
|
+
* 32MB is orders of magnitude of headroom for legitimate traffic while still
|
|
31
|
+
* bounding a runaway body to something a Node heap absorbs without dying. Bulk
|
|
32
|
+
* endpoints that legitimately return more (a full vulnerability catalogue, a
|
|
33
|
+
* maximum-size CVE page) pass their own larger cap explicitly.
|
|
34
|
+
*/
|
|
35
|
+
export const MAX_JSON_BODY_BYTES = 32 * 1024 * 1024;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Cap for bulk catalogue downloads (CISA KEV's full feed, a 2000-item NVD page).
|
|
39
|
+
* These are legitimately tens of megabytes and grow over time, so they get their
|
|
40
|
+
* own ceiling rather than forcing the shared default up for everyone.
|
|
41
|
+
*/
|
|
42
|
+
export const MAX_BULK_JSON_BODY_BYTES = 96 * 1024 * 1024;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Read a response body with a HARD byte cap, streaming chunk-by-chunk so an
|
|
46
|
+
* over-limit body aborts the moment the cap is crossed.
|
|
47
|
+
*
|
|
48
|
+
* A body-less response (test doubles, some mocks) falls back to `arrayBuffer()`;
|
|
49
|
+
* callers that had a post-read size check keep it as the belt for that path.
|
|
50
|
+
*/
|
|
51
|
+
export async function readBodyCapped(response: Response, maxBytes: number): Promise<Uint8Array> {
|
|
52
|
+
const body = response.body;
|
|
53
|
+
// Also tolerate non-standard doubles that expose a body without getReader.
|
|
54
|
+
if (!body || typeof body.getReader !== 'function') {
|
|
55
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const reader = body.getReader();
|
|
59
|
+
const chunks: Uint8Array[] = [];
|
|
60
|
+
let total = 0;
|
|
61
|
+
try {
|
|
62
|
+
for (;;) {
|
|
63
|
+
const { done, value } = await reader.read();
|
|
64
|
+
if (done) break;
|
|
65
|
+
if (value) {
|
|
66
|
+
total += value.byteLength;
|
|
67
|
+
if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
|
|
68
|
+
chunks.push(value);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
} finally {
|
|
72
|
+
// Over-cap or downstream throw: release the connection instead of leaving
|
|
73
|
+
// the socket open (and the server still sending) until GC. No-op after a
|
|
74
|
+
// clean done=true read.
|
|
75
|
+
try { await reader.cancel(); } catch { /* already closed */ }
|
|
76
|
+
}
|
|
77
|
+
const out = new Uint8Array(total);
|
|
78
|
+
let offset = 0;
|
|
79
|
+
for (const c of chunks) { out.set(c, offset); offset += c.byteLength; }
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Whether this response can actually be read as a stream — i.e. whether the cap
|
|
85
|
+
* can be enforced at all.
|
|
86
|
+
*
|
|
87
|
+
* A real `Response` from `fetch()` always exposes a `ReadableStream` body except
|
|
88
|
+
* for the null-body statuses (204/205/304), which have nothing to cap. So in
|
|
89
|
+
* production this is true exactly when there are bytes to bound, and the capped
|
|
90
|
+
* path is never quietly skipped. It returns false only for the null-body statuses
|
|
91
|
+
* and for hand-rolled test doubles, which implement whichever of
|
|
92
|
+
* `arrayBuffer()`/`text()`/`json()` they need and no stream at all.
|
|
93
|
+
*/
|
|
94
|
+
function hasStreamBody(response: Response): boolean {
|
|
95
|
+
const body = response.body;
|
|
96
|
+
return !!body && typeof body.getReader === 'function';
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Capped replacement for `response.text()`.
|
|
101
|
+
*
|
|
102
|
+
* Decodes UTF-8 non-fatally, matching `Response.text()` — a malformed byte becomes
|
|
103
|
+
* U+FFFD rather than throwing, so swapping this in cannot turn a body that used to
|
|
104
|
+
* parse into an exception.
|
|
105
|
+
*/
|
|
106
|
+
export async function readTextCapped(
|
|
107
|
+
response: Response,
|
|
108
|
+
maxBytes: number = MAX_JSON_BODY_BYTES,
|
|
109
|
+
): Promise<string> {
|
|
110
|
+
if (hasStreamBody(response)) {
|
|
111
|
+
return new TextDecoder('utf-8').decode(await readBodyCapped(response, maxBytes));
|
|
112
|
+
}
|
|
113
|
+
// Double/null-body ladder — see hasStreamBody for why this is unreachable for a
|
|
114
|
+
// real body-bearing response.
|
|
115
|
+
if (typeof response.arrayBuffer === 'function') {
|
|
116
|
+
return new TextDecoder('utf-8').decode(new Uint8Array(await response.arrayBuffer()));
|
|
117
|
+
}
|
|
118
|
+
return response.text();
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Capped replacement for `response.json()`.
|
|
123
|
+
*
|
|
124
|
+
* Throws {@link BodyTooLargeError} for an over-cap body and a `SyntaxError` for a
|
|
125
|
+
* malformed one, so callers that already distinguish "malformed JSON" from other
|
|
126
|
+
* failures keep working: the size failure is a distinct, catchable type rather
|
|
127
|
+
* than something that masquerades as a parse error.
|
|
128
|
+
*/
|
|
129
|
+
export async function readJsonCapped<T = unknown>(
|
|
130
|
+
response: Response,
|
|
131
|
+
maxBytes: number = MAX_JSON_BODY_BYTES,
|
|
132
|
+
): Promise<T> {
|
|
133
|
+
if (hasStreamBody(response) || typeof response.arrayBuffer === 'function' || typeof response.text === 'function') {
|
|
134
|
+
return JSON.parse(await readTextCapped(response, maxBytes)) as T;
|
|
135
|
+
}
|
|
136
|
+
// A double that implements json() alone.
|
|
137
|
+
return response.json() as Promise<T>;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Accumulator for a Node `IncomingMessage` body that decodes UTF-8 exactly ONCE,
|
|
142
|
+
* at the end.
|
|
143
|
+
*
|
|
144
|
+
* The obvious `body += chunk` is silently lossy. `+=` stringifies each Buffer on
|
|
145
|
+
* its own, so a multi-byte character straddling a chunk boundary is decoded as two
|
|
146
|
+
* independent fragments and both halves become U+FFFD. Chunk boundaries fall
|
|
147
|
+
* wherever the socket happens to split, so the damage scales with payload size and
|
|
148
|
+
* lands only on non-ASCII text — measured at ~15 mangled characters in a 560KB CJK
|
|
149
|
+
* body. Nothing throws: `JSON.parse` still succeeds, so the corruption reaches the
|
|
150
|
+
* report and the knowledge store unannounced.
|
|
151
|
+
*
|
|
152
|
+
* Buffering the chunks and concatenating before a single decode is what makes a
|
|
153
|
+
* split sequence whole again.
|
|
154
|
+
*
|
|
155
|
+
* `byteLength` counts real bytes, which is also the honest unit for a payload cap —
|
|
156
|
+
* the old `body.length` measured UTF-16 code units of an already-corrupted string.
|
|
157
|
+
*/
|
|
158
|
+
export class Utf8Body {
|
|
159
|
+
private readonly chunks: Buffer[] = [];
|
|
160
|
+
private bytes = 0;
|
|
161
|
+
|
|
162
|
+
/** Total bytes accumulated so far. */
|
|
163
|
+
get byteLength(): number {
|
|
164
|
+
return this.bytes;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
push(chunk: Buffer): void {
|
|
168
|
+
this.chunks.push(chunk);
|
|
169
|
+
this.bytes += chunk.byteLength;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Concatenate and decode. Safe to call once the stream has ended. */
|
|
173
|
+
toString(): string {
|
|
174
|
+
return Buffer.concat(this.chunks).toString('utf8');
|
|
175
|
+
}
|
|
176
|
+
}
|
package/src/utils/llm-usage.ts
CHANGED
|
@@ -13,8 +13,44 @@
|
|
|
13
13
|
|
|
14
14
|
import type { Model } from '@earendil-works/pi-ai';
|
|
15
15
|
import { extractUsage } from '../types/llm.ts';
|
|
16
|
+
import { logger } from '../logger.ts';
|
|
16
17
|
import { metrics } from './metrics.ts';
|
|
17
18
|
|
|
19
|
+
/** Models already warned about, so the notice fires at most once per model per process. */
|
|
20
|
+
const unpricedModelsWarned = new Set<string>();
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Warn once when a model bills tokens but carries an all-zero price table.
|
|
24
|
+
*
|
|
25
|
+
* Cost is never read off the wire — pi computes it as tokens × the model's local
|
|
26
|
+
* price table — so an unpriced table yields $0.00 silently and indefinitely. That
|
|
27
|
+
* is legitimate for flat-rate plans and local servers, but it is also exactly what
|
|
28
|
+
* a misconfiguration looks like: a hand-written `models[]` entry in models.json
|
|
29
|
+
* REPLACES pi's catalog entry for the same id, discarding its pricing. This
|
|
30
|
+
* happened in practice and went unnoticed for weeks because every display site
|
|
31
|
+
* suppressed a zero cost. The message deliberately does not assert a fault — it
|
|
32
|
+
* states the condition and names the usual cause.
|
|
33
|
+
*/
|
|
34
|
+
function warnIfUnpriced(model: Model<any>, tokens: number, cost: number): void {
|
|
35
|
+
if (tokens <= 0 || cost > 0) return;
|
|
36
|
+
const m = model as unknown as { provider?: string; id?: string; cost?: Record<string, number> };
|
|
37
|
+
const key = `${m.provider ?? 'unknown'}/${m.id ?? 'unknown'}`;
|
|
38
|
+
if (unpricedModelsWarned.has(key)) return;
|
|
39
|
+
|
|
40
|
+
const table = m.cost;
|
|
41
|
+
const allZero =
|
|
42
|
+
!table || Object.values(table).every((v) => typeof v !== 'number' || v === 0);
|
|
43
|
+
if (!allZero) return;
|
|
44
|
+
|
|
45
|
+
unpricedModelsWarned.add(key);
|
|
46
|
+
logger.warn(
|
|
47
|
+
`[LlmUsage] Model "${key}" consumed ${tokens} tokens but reports $0.00 — its price table is all zeros. ` +
|
|
48
|
+
`Expected for flat-rate plans and local servers. Otherwise the price data is missing: a hand-written ` +
|
|
49
|
+
`"models" entry in ~/.pi/agent/models.json replaces pi's catalog entry (and its pricing) for the same ` +
|
|
50
|
+
`model id — remove it and let pi's catalog supply the model, or use "modelOverrides", which merges.`
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
18
54
|
/** Structural subset of the run observer — just the token sink. Kept structural (rather
|
|
19
55
|
* than importing ResearchObserver from core/) so this foundation-layer helper does not
|
|
20
56
|
* depend on an upper layer; the full ResearchObserver satisfies it. */
|
|
@@ -43,6 +79,7 @@ export function recordLlmUsage(
|
|
|
43
79
|
): { tokens: number; cost: number } {
|
|
44
80
|
if (!rawUsage) return { tokens: 0, cost: 0 };
|
|
45
81
|
const { tokens, cost } = extractUsage(model, rawUsage);
|
|
82
|
+
warnIfUnpriced(model, tokens, cost);
|
|
46
83
|
if (tokens > 0 || cost > 0) {
|
|
47
84
|
const labels: Record<string, string> = { component: opts.component };
|
|
48
85
|
if (opts.complexity !== undefined) labels['complexity'] = String(opts.complexity);
|
|
@@ -283,8 +283,15 @@ export function buildResearchSummary(stats: ResearchStats): string {
|
|
|
283
283
|
if (stats.tokens > 0) {
|
|
284
284
|
resourceParts.push(`**${formatTokens(stats.tokens)}** tokens`);
|
|
285
285
|
}
|
|
286
|
-
|
|
287
|
-
|
|
286
|
+
// Render the cost whenever tokens were actually billed-through, INCLUDING when it
|
|
287
|
+
// is exactly zero. Gating on `cost > 0` hid a real defect for weeks: a model whose
|
|
288
|
+
// price table is all zeros (an unpriced or catalog-overwritten entry) produced no
|
|
289
|
+
// cost line at all, so a misconfiguration was indistinguishable from a run that
|
|
290
|
+
// simply had not reported yet. "$0.00" beside a large token count is conspicuous;
|
|
291
|
+
// silence is not. Zero is also the honest figure for flat-rate and local providers.
|
|
292
|
+
if (stats.tokens > 0 || stats.cost > 0) {
|
|
293
|
+
const costStr =
|
|
294
|
+
stats.cost === 0 ? '$0.00' : stats.cost < 0.01 ? '<$0.01' : `$${stats.cost.toFixed(2)}`;
|
|
288
295
|
resourceParts.push(`**${costStr}**`);
|
|
289
296
|
}
|
|
290
297
|
if (stats.durationMs > 0) {
|
|
@@ -317,6 +324,9 @@ export interface SessionStats {
|
|
|
317
324
|
totalUrlsDiscovered: number;
|
|
318
325
|
totalSearchQueries: number;
|
|
319
326
|
totalTokens: number;
|
|
327
|
+
/** Summed LLM cost across the session's runs. The Session Metrics UI advertises
|
|
328
|
+
* "API cost estimates"; without this field it had no cost to show. */
|
|
329
|
+
totalCost: number;
|
|
320
330
|
totalToolUsage: {
|
|
321
331
|
searches: number;
|
|
322
332
|
scrapes: number;
|
|
@@ -339,6 +349,7 @@ export function aggregateSessionStats(
|
|
|
339
349
|
let totalUrlsDiscovered = 0;
|
|
340
350
|
let totalSearchQueries = 0;
|
|
341
351
|
let totalTokens = 0;
|
|
352
|
+
let totalCost = 0;
|
|
342
353
|
const totalToolUsage = {
|
|
343
354
|
searches: 0,
|
|
344
355
|
scrapes: 0,
|
|
@@ -359,6 +370,7 @@ export function aggregateSessionStats(
|
|
|
359
370
|
totalUrlsDiscovered += stats.urlsDiscovered;
|
|
360
371
|
totalSearchQueries += stats.searchQueries;
|
|
361
372
|
totalTokens += stats.tokens;
|
|
373
|
+
totalCost += stats.cost;
|
|
362
374
|
totalToolUsage.searches += stats.toolUsage.searches;
|
|
363
375
|
totalToolUsage.scrapes += stats.toolUsage.scrapes;
|
|
364
376
|
totalToolUsage.securitySearches += stats.toolUsage.securitySearches;
|
|
@@ -377,6 +389,7 @@ export function aggregateSessionStats(
|
|
|
377
389
|
totalUrlsDiscovered,
|
|
378
390
|
totalSearchQueries,
|
|
379
391
|
totalTokens,
|
|
392
|
+
totalCost,
|
|
380
393
|
totalToolUsage,
|
|
381
394
|
};
|
|
382
395
|
}
|
|
@@ -403,6 +416,16 @@ export function buildSessionOverview(stats: SessionStats): string {
|
|
|
403
416
|
}
|
|
404
417
|
if (stats.totalTokens > 0) {
|
|
405
418
|
overviewParts.push(`**${formatTokens(stats.totalTokens)}** tokens`);
|
|
419
|
+
// Shown alongside tokens even at exactly $0.00 — the Session Metrics view
|
|
420
|
+
// advertises cost estimates, and a silent omission is what let an unpriced
|
|
421
|
+
// model table go unnoticed. Zero is correct for flat-rate and local providers.
|
|
422
|
+
const costStr =
|
|
423
|
+
stats.totalCost === 0
|
|
424
|
+
? '$0.00'
|
|
425
|
+
: stats.totalCost < 0.01
|
|
426
|
+
? '<$0.01'
|
|
427
|
+
: `$${stats.totalCost.toFixed(2)}`;
|
|
428
|
+
overviewParts.push(`**${costStr}**`);
|
|
406
429
|
}
|
|
407
430
|
if (stats.totalDurationMs > 0) {
|
|
408
431
|
overviewParts.push(`**${formatDuration(stats.totalDurationMs)}** total research time`);
|
|
@@ -207,8 +207,15 @@ export async function exportResearchReport(
|
|
|
207
207
|
*/
|
|
208
208
|
export function appendExportMessage(result: string, filepath: string, totalCost?: number): string {
|
|
209
209
|
let footer = `\n\n---\n\nResearch report saved to: ${filepath}`;
|
|
210
|
-
|
|
211
|
-
|
|
210
|
+
// Report a defined cost even when it is exactly zero — suppressing it made an
|
|
211
|
+
// unpriced/misconfigured model table look identical to "no cost reported yet".
|
|
212
|
+
if (totalCost !== undefined) {
|
|
213
|
+
const costStr =
|
|
214
|
+
totalCost === 0
|
|
215
|
+
? '$0.00'
|
|
216
|
+
: totalCost < 0.01
|
|
217
|
+
? '<$0.01'
|
|
218
|
+
: `$${totalCost.toFixed(4)}`.replace(/0+$/, '').replace(/\.$/, '');
|
|
212
219
|
footer += `\nTotal cost: ${costStr}`;
|
|
213
220
|
}
|
|
214
221
|
return result + footer;
|