@lincoln504/pi-research 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +4 -4
  2. package/dist/cli.mjs +1404 -600
  3. package/dist/thread-worker.mjs +40 -11
  4. package/docs/AGENT-SKILL.md +11 -4
  5. package/docs/ARCHITECTURE.md +19 -9
  6. package/docs/CHANGELOG.md +107 -1
  7. package/docs/CONFIGURATION.md +10 -8
  8. package/docs/KNOWLEDGE-STORE.md +5 -4
  9. package/docs/PI-EXTENSION.md +1 -1
  10. package/docs/SDK.md +63 -5
  11. package/docs/THIRD-PARTY-NOTICES.md +1 -1
  12. package/package.json +5 -4
  13. package/scripts/build.cjs +23 -5
  14. package/scripts/cleanup.cjs +50 -10
  15. package/scripts/prepare.cjs +15 -5
  16. package/scripts/setup.cjs +24 -4
  17. package/skills/pi-research/README.md +9 -2
  18. package/skills/pi-research/SKILL.md +8 -5
  19. package/skills/pi-research/references/configuration.md +7 -3
  20. package/skills/pi-research/scripts/run.mjs +24 -2
  21. package/src/cli.ts +161 -21
  22. package/src/config.ts +56 -15
  23. package/src/constants.ts +31 -0
  24. package/src/core/interfaces/knowledge-interfaces.ts +6 -1
  25. package/src/core/interfaces/state-manager-interfaces.ts +1 -1
  26. package/src/core/llm/model-registry-factory.ts +40 -3
  27. package/src/core/llm/research-model-resolver.ts +2 -2
  28. package/src/core/pi-version.ts +136 -0
  29. package/src/core/service-registry.ts +117 -8
  30. package/src/index.ts +11 -18
  31. package/src/infrastructure/browser/browser-client.ts +16 -4
  32. package/src/infrastructure/browser/browser-error-utils.ts +12 -1
  33. package/src/infrastructure/browser/browser-server.ts +17 -5
  34. package/src/infrastructure/browser/browser-task-scheduler.ts +101 -23
  35. package/src/infrastructure/browser/cleanup-utils.ts +45 -7
  36. package/src/infrastructure/browser/client-agent.ts +4 -3
  37. package/src/infrastructure/browser/priority-task-queue.ts +10 -3
  38. package/src/infrastructure/browser/scheduler-factory.ts +26 -1
  39. package/src/infrastructure/browser/task-execution-service.ts +92 -28
  40. package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
  41. package/src/infrastructure/browser/thread-worker.mjs +40 -11
  42. package/src/infrastructure/browser/worker-pool-manager.ts +11 -3
  43. package/src/infrastructure/embedding/embedding-client.ts +22 -8
  44. package/src/infrastructure/embedding/embedding-factory.ts +28 -11
  45. package/src/infrastructure/embedding/embedding-server.ts +65 -4
  46. package/src/infrastructure/file-lock-service.ts +223 -26
  47. package/src/infrastructure/process-lifecycle-service.ts +108 -31
  48. package/src/infrastructure/state/state-backup-manager.ts +5 -7
  49. package/src/infrastructure/state/state-manager-service.ts +1 -1
  50. package/src/infrastructure/state/state-manager.ts +19 -20
  51. package/src/infrastructure/types/state-types.ts +9 -0
  52. package/src/knowledge/embedder-init.ts +1 -1
  53. package/src/knowledge/embedder.ts +52 -3
  54. package/src/knowledge/index.ts +1 -1
  55. package/src/knowledge/store.ts +35 -9
  56. package/src/knowledge/webgpu-viability.ts +9 -4
  57. package/src/knowledge/writer-queue.ts +88 -46
  58. package/src/orchestration/deep-research-orchestrator.ts +13 -2
  59. package/src/orchestration/quick-research-orchestrator.ts +2 -2
  60. package/src/orchestration/research-orchestration-service.ts +2 -4
  61. package/src/orchestration/research-synthesis-service.ts +34 -92
  62. package/src/security/cisa-kev.ts +10 -3
  63. package/src/security/github-advisories.ts +18 -7
  64. package/src/security/nvd-client.ts +17 -6
  65. package/src/security/osv-client.ts +14 -4
  66. package/src/skill-install/skill-installer.ts +135 -9
  67. package/src/stackexchange/rest-client.ts +6 -2
  68. package/src/tools/research-knowledge-search.ts +23 -2
  69. package/src/tools/search.ts +6 -1
  70. package/src/utils/atomic-replace.ts +101 -0
  71. package/src/utils/circuit-breaker.ts +20 -0
  72. package/src/utils/citation-utils.ts +106 -8
  73. package/src/utils/http-body.ts +176 -0
  74. package/src/utils/llm-usage.ts +37 -0
  75. package/src/utils/metrics-summary.ts +25 -2
  76. package/src/utils/research-export.ts +9 -2
  77. package/src/web-research/browser-search.ts +50 -4
  78. package/src/web-research/retry-utils.ts +13 -1
  79. package/src/web-research/search.ts +20 -5
  80. package/src/web-research/types.ts +17 -4
  81. package/src/web-research/web-scraper.ts +7 -50
  82. package/src/youtube/transcript-client.ts +5 -1
@@ -481,7 +481,10 @@ export async function runBackgroundExtraction(
481
481
  const delay = KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS * attempt;
482
482
  metrics.increment('research_knowledge_search_synthesis_retries_total', 1);
483
483
  logger.warn(`[research-knowledge-search] Synthesis attempt ${attempt}/${KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS} failed transiently (${msg}); retrying in ${delay}ms`);
484
- await abortableDelay(delay, signal);
484
+ // keepAlive=true: the retry is already committed and the next attempt must
485
+ // run, so this sleep is foreground work. An unref'd timer here can be the
486
+ // only pending handle and let the process exit mid-retry (see abortableDelay).
487
+ await abortableDelay(delay, signal, true);
485
488
  }
486
489
  }
487
490
 
@@ -625,7 +628,10 @@ export async function triageRelevantUrls(
625
628
  const delay = KNOWLEDGE_SYNTHESIS_RETRY_BASE_MS * attempt;
626
629
  metrics.increment('research_knowledge_search_triage_retries_total', 1);
627
630
  logger.warn(`[research-knowledge-search] Triage attempt ${attempt}/${KNOWLEDGE_SYNTHESIS_MAX_ATTEMPTS} failed transiently (${msg}); retrying in ${delay}ms`);
628
- await abortableDelay(delay, signal);
631
+ // keepAlive=true: the retry is already committed and the next attempt must
632
+ // run, so this sleep is foreground work. An unref'd timer here can be the
633
+ // only pending handle and let the process exit mid-retry (see abortableDelay).
634
+ await abortableDelay(delay, signal, true);
629
635
  }
630
636
  }
631
637
 
@@ -956,6 +962,21 @@ export function createResearchKnowledgeSearchTool(iface?: ConfigInterface): Tool
956
962
  return buildSteeringResult(result, urls);
957
963
  } catch (error) {
958
964
  const durationMs = Date.now() - startTime;
965
+ const msg = error instanceof Error ? error.message : String(error);
966
+
967
+ // A CANCELLED search is not an empty store. Reporting the miss string here
968
+ // told the calling agent, definitively, that the knowledge store has
969
+ // nothing on this topic — so it would go straight to full live research,
970
+ // the very work the user just interrupted. Propagate the abort instead;
971
+ // the run is ending either way. (Same misclassification as the search
972
+ // layer's "browser workers may be unavailable" on Ctrl-C.)
973
+ if (signal?.aborted || (error instanceof Error && error.name === 'AbortError') || /^(aborted|research (aborted|cancelled))$/i.test(msg)) {
974
+ metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'cancelled' });
975
+ metrics.increment('research_knowledge_search_total', 1, { status: 'cancelled' });
976
+ logger.debug('[research-knowledge-search] Cancelled mid-search — propagating rather than reporting an empty store.');
977
+ throw error;
978
+ }
979
+
959
980
  metrics.observe('research_knowledge_search_duration_ms', durationMs, { status: 'error' });
960
981
  metrics.increment('research_knowledge_search_total', 1, { status: 'error' });
961
982
 
@@ -106,7 +106,12 @@ export function createSearchTool(options: {
106
106
  results.forEach((r, i) => {
107
107
  markdown += `## Query ${i + 1}: ${r.query}\n`;
108
108
  if (r.results.length === 0) {
109
- markdown += `*No results found.*\n\n`;
109
+ // Render the attributed failure, not a bare "no results": a timeout or
110
+ // dead worker says nothing about the query, and presenting it as an
111
+ // empty result sent the model off rewriting perfectly good queries —
112
+ // the misdirection QueryFailure exists to prevent. This is the only
113
+ // surface the researcher agent actually reads.
114
+ markdown += r.error ? `*${r.error.message}*\n\n` : `*No results found.*\n\n`;
110
115
  } else {
111
116
  r.results.forEach((item, j) => {
112
117
  markdown += `[${j + 1}] **${item.title}**\n${item.url}\n${item.content}\n\n`;
@@ -0,0 +1,101 @@
1
+ /**
2
+ * Atomic file replacement, with the Windows caveat handled honestly.
3
+ *
4
+ * The temp-file + fsync + rename dance exists so a reader never observes a partial
5
+ * file and a crash never leaves one: rename is atomic, so the destination is either
6
+ * entirely the old content or entirely the new.
7
+ *
8
+ * On Windows that guarantee is easy to lose. The long-standing comment at these call
9
+ * sites — "fs.rename fails on Windows if target exists (NTFS)" — is not true: Node
10
+ * calls `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`, so replacing an existing file
11
+ * is fine. What actually fails is a rename whose TARGET is held open by another
12
+ * process without delete sharing (an AV scanner mid-scan, a backup agent), which
13
+ * surfaces as EPERM/EBUSY/EACCES.
14
+ *
15
+ * The old remedy — fall straight to `copyFile` — traded a transient, retryable error
16
+ * for a permanent hazard: `copyFile` truncates the destination and rewrites it in
17
+ * place, so a concurrent reader or a crash mid-copy sees a torn or zero-length file.
18
+ * For `state.json` (leader election, server auth secrets) and `config.env` (API keys)
19
+ * that is a much worse outcome than the error it was avoiding.
20
+ *
21
+ * So: retry the rename first. An AV lock clears in tens of milliseconds, and a retried
22
+ * rename stays atomic. Only a lock that outlives every retry falls back to copying,
23
+ * which is reported to the caller so it can be logged as the degraded path it is.
24
+ */
25
+
26
+ import * as fs from 'node:fs/promises';
27
+ import * as fsSync from 'node:fs';
28
+
29
+ /** Transient Windows sharing/locking failures — worth retrying, unlike ENOENT/ENOSPC. */
30
+ const RETRYABLE_CODES = new Set(['EPERM', 'EBUSY', 'EACCES']);
31
+
32
+ const MAX_RENAME_ATTEMPTS = 5;
33
+ const RENAME_RETRY_DELAY_MS = 25;
34
+
35
+ export type ReplaceOutcome = 'renamed' | 'copied';
36
+
37
+ function isRetryable(err: unknown): boolean {
38
+ const code = (err as NodeJS.ErrnoException | undefined)?.code;
39
+ return code !== undefined && RETRYABLE_CODES.has(code);
40
+ }
41
+
42
+ /**
43
+ * Replace `destPath` with `tmpPath` atomically where the platform allows it.
44
+ *
45
+ * Returns `'renamed'` for the atomic path and `'copied'` for the degraded Windows
46
+ * fallback. Rethrows on every non-Windows failure, and on Windows failures that are
47
+ * not lock-related.
48
+ */
49
+ export async function replaceFile(tmpPath: string, destPath: string): Promise<ReplaceOutcome> {
50
+ let lastError: unknown;
51
+ for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
52
+ try {
53
+ await fs.rename(tmpPath, destPath);
54
+ return 'renamed';
55
+ } catch (err) {
56
+ lastError = err;
57
+ // Only a Windows sharing conflict is worth another try; anything else (and any
58
+ // failure on a POSIX platform, where rename-over-existing always works) is real.
59
+ if (process.platform !== 'win32' || !isRetryable(err)) throw err;
60
+ if (attempt < MAX_RENAME_ATTEMPTS - 1) {
61
+ await new Promise((r) => setTimeout(r, RENAME_RETRY_DELAY_MS * (attempt + 1)));
62
+ }
63
+ }
64
+ }
65
+
66
+ // Every retry lost to the same lock. Copying is not atomic — a reader can catch the
67
+ // destination mid-write — but it is the only remaining way to persist the data.
68
+ try {
69
+ await fs.copyFile(tmpPath, destPath);
70
+ return 'copied';
71
+ } catch {
72
+ throw lastError;
73
+ }
74
+ }
75
+
76
+ /** Synchronous {@link replaceFile}, for the config writer, which is sync throughout. */
77
+ export function replaceFileSync(tmpPath: string, destPath: string): ReplaceOutcome {
78
+ let lastError: unknown;
79
+ for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt++) {
80
+ try {
81
+ fsSync.renameSync(tmpPath, destPath);
82
+ return 'renamed';
83
+ } catch (err) {
84
+ lastError = err;
85
+ if (process.platform !== 'win32' || !isRetryable(err)) throw err;
86
+ if (attempt < MAX_RENAME_ATTEMPTS - 1) {
87
+ // Sync busy-wait: this path is rare, bounded (~250ms worst case) and the
88
+ // caller is already blocking on a synchronous write.
89
+ const until = Date.now() + RENAME_RETRY_DELAY_MS * (attempt + 1);
90
+ while (Date.now() < until) { /* spin */ }
91
+ }
92
+ }
93
+ }
94
+
95
+ try {
96
+ fsSync.copyFileSync(tmpPath, destPath);
97
+ return 'copied';
98
+ } catch {
99
+ throw lastError;
100
+ }
101
+ }
@@ -78,6 +78,26 @@ export class CircuitBreaker {
78
78
  metrics.observe('circuit_breaker_call_duration_ms', duration, { breaker: this.options.name, status: 'error' });
79
79
  if (this.options.isTransientError(error)) {
80
80
  this.onFailure(error);
81
+ } else if (admittedAsProbe && this.state === 'HALF_OPEN') {
82
+ // The trial probe failed for a reason this breaker does not count. That is
83
+ // INCONCLUSIVE, not healthy — so re-open and wait out another reset window
84
+ // rather than staying HALF_OPEN.
85
+ //
86
+ // Without this the breaker pins permanently: onFailure is the only path to
87
+ // OPEN, and it is skipped for non-counted errors, while transitionTo(HALF_OPEN)
88
+ // never re-arms nextAttemptTime. The admission gate above then fast-fails every
89
+ // caller beyond halfOpenMaxCalls (default 1) forever. This is not a corner case
90
+ // for the browser pool: DEFAULT_BREAKER_CONFIG.isTransientError deliberately
91
+ // excludes pool-shutdown, draining, Cloudflare and task-timeout errors — exactly
92
+ // what a recovering or blocked pool returns — so the probe most likely to run is
93
+ // also the one most likely to pin the breaker. Observed effect: 19 of a 20-query
94
+ // parallel burst fast-failing instantly, surfacing to the researcher as
95
+ // "your query may be too narrow".
96
+ logger.warn(
97
+ `[CircuitBreaker] '${this.options.name}' HALF_OPEN probe failed with a non-counted error; ` +
98
+ `treating as inconclusive and re-opening: ${error instanceof Error ? error.message : String(error)}`
99
+ );
100
+ this.transitionTo('OPEN');
81
101
  }
82
102
  throw error;
83
103
  } finally {
@@ -16,7 +16,9 @@ export interface GlobalCitation extends Citation {
16
16
  * @param reports - Map of researcher ID to report content
17
17
  * @returns Object containing normalized reports and the global citation list
18
18
  */
19
- export function normalizeCitations(reports: Map<string, string>): {
19
+ // ReadonlyMap: this only reads `reports` (it returns a NEW normalizedReports map),
20
+ // and callers now pass a read-only view to avoid create-on-read side effects.
21
+ export function normalizeCitations(reports: ReadonlyMap<string, string>): {
20
22
  normalizedReports: Map<string, string>;
21
23
  globalCitations: GlobalCitation[];
22
24
  } {
@@ -67,13 +69,28 @@ export function normalizeCitations(reports: Map<string, string>): {
67
69
  const headerIdx = lastCitedLinksHeaderIndex(report);
68
70
  let content = headerIdx >= 0 ? report.slice(0, headerIdx) : report;
69
71
 
70
- // Replace [N] with [GlobalID]
71
- // Also handle [N][M] combinations
72
- content = content.replace(/\[(\d+)\]/g, (match, p1) => {
73
- const localId = parseInt(p1, 10);
74
- const globalId = localToGlobal.get(localId);
75
- return globalId !== undefined ? `[${globalId}]` : match;
76
- });
72
+ // Renumber local [N] markers to their global ids, and DELETE the ones that could
73
+ // not be mapped.
74
+ //
75
+ // Leaving an unmapped marker in place (the previous `return match`) was a silent
76
+ // misattribution bug, not a harmless no-op: global ids are assigned densely from 1,
77
+ // so a surviving `[1]` no longer means "this report's first source" — it resolves
78
+ // against the renumbered global list and points at a completely different document.
79
+ // Markers go unmapped whenever parseCitations dropped the entry (e.g. a URL whose
80
+ // hostname has no dot: `localhost`, a bare internal host, a malformed line). The
81
+ // result is fed straight into the evaluator/synthesis prompt, which is explicitly
82
+ // told the numbering is already normalized — so the model faithfully reproduces the
83
+ // wrong attribution, and ensureCitedLinks then regenerates a CITED LINKS list that
84
+ // looks perfectly consistent with it.
85
+ //
86
+ // rewriteCitationMarkers is the same routine the synthesis path already uses: it
87
+ // skips fenced blocks and inline code spans (so a quoted `items[12]` survives),
88
+ // bounds what counts as a marker, and preserves line-leading indentation.
89
+ content = rewriteCitationMarkers(
90
+ content,
91
+ localToGlobal,
92
+ (n) => !localToGlobal.has(n),
93
+ );
77
94
 
78
95
  normalizedReports.set(id, content.trim());
79
96
  }
@@ -95,3 +112,84 @@ export function formatCitedLinks(citations: GlobalCitation[]): string {
95
112
 
96
113
  return `CITED LINKS\n${links.join('\n')}`;
97
114
  }
115
+
116
+ /**
117
+ * Largest `[N]` treated as a citation marker. Inline markers index a source list
118
+ * that is realistically well under this; the bound exists so an ordinary bracketed
119
+ * number in prose — a year (`[2023]`), an issue id, an array index — is left alone
120
+ * instead of being silently deleted as a "dangling citation".
121
+ */
122
+ export const MAX_PLAUSIBLE_CITATION_MARKER = 200;
123
+
124
+ /**
125
+ * Rewrite inline `[N]` citation markers in a prose body: renumber the ones we can
126
+ * map, delete the ones known to dangle, leave everything else untouched.
127
+ *
128
+ * Deletion is whitespace-aware rather than followed by a global space collapse.
129
+ * A blanket `text.replace(/ +/g, ' ')` also eats *leading* indentation, which
130
+ * silently flattens nested list levels into one and turns 4-space indented code
131
+ * blocks into paragraphs — corrupting the structure of every report that contains
132
+ * them. So the pad preceding a deleted marker is consumed only when the marker is
133
+ * mid-line (i.e. real prose spacing); line-leading indentation is preserved.
134
+ *
135
+ * Code is skipped entirely. A report may quote code containing bracketed integers
136
+ * (`items[12]`), and on the no-CITED-LINKS path *every* out-of-range marker is treated
137
+ * as dangling — so without this guard a quoted array index would be silently deleted
138
+ * from the user's code sample. Fenced blocks are tracked across lines; single-backtick
139
+ * spans are detected by an odd backtick count earlier on the same line.
140
+ *
141
+ * @param remap local marker number → verified global id
142
+ * @param isDangling whether a marker not present in `remap` should be deleted
143
+ */
144
+ export function rewriteCitationMarkers(
145
+ text: string,
146
+ remap: Map<number, number>,
147
+ isDangling: (n: number) => boolean,
148
+ ): string {
149
+ const inFence = fencedCodeMask(text);
150
+ return text.replace(/([ \t]*)\[(\d+)\]/g, (match, pad: string, digits: string, offset: number) => {
151
+ const n = parseInt(digits, 10);
152
+ const mapped = remap.get(n);
153
+ if (mapped !== undefined) return `${pad}[${mapped}]`;
154
+ // Citation numbering is 1-based; `[0]` is always an index or a literal, never a marker.
155
+ if (n < 1 || n > MAX_PLAUSIBLE_CITATION_MARKER || !isDangling(n)) return match;
156
+ if (inFence(offset) || inInlineCodeSpan(text, offset)) return match;
157
+ // Preserve indentation when nothing but whitespace precedes on this line;
158
+ // otherwise drop the pad too so no double space is left behind.
159
+ const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
160
+ return text.slice(lineStart, offset).trim() === '' ? pad : '';
161
+ });
162
+ }
163
+
164
+ /**
165
+ * Build an `offset → is inside a ``` fenced block` predicate for `text`.
166
+ *
167
+ * Computed once per rewrite (a single line scan) rather than per marker, so marker
168
+ * rewriting stays linear in the body length.
169
+ */
170
+ function fencedCodeMask(text: string): (offset: number) => boolean {
171
+ const ranges: Array<[number, number]> = [];
172
+ let open = -1;
173
+ let pos = 0;
174
+ for (const line of text.split('\n')) {
175
+ if (/^\s{0,3}(```|~~~)/.test(line)) {
176
+ if (open < 0) open = pos;
177
+ else {
178
+ ranges.push([open, pos + line.length]);
179
+ open = -1;
180
+ }
181
+ }
182
+ pos += line.length + 1; // +1 for the '\n' consumed by split
183
+ }
184
+ // An unterminated fence runs to the end of the body — treat the remainder as code.
185
+ if (open >= 0) ranges.push([open, text.length]);
186
+ return (offset: number) => ranges.some(([start, end]) => offset >= start && offset < end);
187
+ }
188
+
189
+ /** Whether `offset` sits inside a single-backtick code span on its own line. */
190
+ function inInlineCodeSpan(text: string, offset: number): boolean {
191
+ const lineStart = text.lastIndexOf('\n', offset - 1) + 1;
192
+ let ticks = 0;
193
+ for (let i = lineStart; i < offset; i++) if (text[i] === '`') ticks++;
194
+ return ticks % 2 === 1;
195
+ }
@@ -0,0 +1,176 @@
1
+ /**
2
+ * Capped response-body reading.
3
+ *
4
+ * `response.json()` and `response.text()` buffer the WHOLE body into the main
5
+ * process before anything can inspect its size — the byte count is only knowable
6
+ * after the memory has already been committed. A hostile or merely broken upstream
7
+ * can therefore drive this process to an OOM kill with a single reply, and because
8
+ * a `Content-Length` header is advisory (absent on chunked encoding, and freely
9
+ * falsifiable) no pre-screen closes that hole. Streaming chunk-by-chunk and
10
+ * aborting the instant the cap is crossed is the only check that actually bounds
11
+ * the allocation.
12
+ *
13
+ * The scraper has read HTML and PDF this way for some time; these helpers exist so
14
+ * the JSON API clients share that one implementation rather than each growing a
15
+ * near-copy of it.
16
+ */
17
+
18
+ /** Thrown by {@link readBodyCapped} when a streamed body crosses its byte cap. */
19
+ export class BodyTooLargeError extends Error {
20
+ constructor(public readonly bytesRead: number, public readonly maxBytes: number) {
21
+ super(`Response body exceeded ${maxBytes} byte cap`);
22
+ this.name = 'BodyTooLargeError';
23
+ }
24
+ }
25
+
26
+ /**
27
+ * Default cap for a JSON API response.
28
+ *
29
+ * Every endpoint behind these clients answers in kilobytes for a normal query, so
30
+ * 32MB is orders of magnitude of headroom for legitimate traffic while still
31
+ * bounding a runaway body to something a Node heap absorbs without dying. Bulk
32
+ * endpoints that legitimately return more (a full vulnerability catalogue, a
33
+ * maximum-size CVE page) pass their own larger cap explicitly.
34
+ */
35
+ export const MAX_JSON_BODY_BYTES = 32 * 1024 * 1024;
36
+
37
+ /**
38
+ * Cap for bulk catalogue downloads (CISA KEV's full feed, a 2000-item NVD page).
39
+ * These are legitimately tens of megabytes and grow over time, so they get their
40
+ * own ceiling rather than forcing the shared default up for everyone.
41
+ */
42
+ export const MAX_BULK_JSON_BODY_BYTES = 96 * 1024 * 1024;
43
+
44
+ /**
45
+ * Read a response body with a HARD byte cap, streaming chunk-by-chunk so an
46
+ * over-limit body aborts the moment the cap is crossed.
47
+ *
48
+ * A body-less response (test doubles, some mocks) falls back to `arrayBuffer()`;
49
+ * callers that had a post-read size check keep it as the belt for that path.
50
+ */
51
+ export async function readBodyCapped(response: Response, maxBytes: number): Promise<Uint8Array> {
52
+ const body = response.body;
53
+ // Also tolerate non-standard doubles that expose a body without getReader.
54
+ if (!body || typeof body.getReader !== 'function') {
55
+ return new Uint8Array(await response.arrayBuffer());
56
+ }
57
+
58
+ const reader = body.getReader();
59
+ const chunks: Uint8Array[] = [];
60
+ let total = 0;
61
+ try {
62
+ for (;;) {
63
+ const { done, value } = await reader.read();
64
+ if (done) break;
65
+ if (value) {
66
+ total += value.byteLength;
67
+ if (total > maxBytes) throw new BodyTooLargeError(total, maxBytes);
68
+ chunks.push(value);
69
+ }
70
+ }
71
+ } finally {
72
+ // Over-cap or downstream throw: release the connection instead of leaving
73
+ // the socket open (and the server still sending) until GC. No-op after a
74
+ // clean done=true read.
75
+ try { await reader.cancel(); } catch { /* already closed */ }
76
+ }
77
+ const out = new Uint8Array(total);
78
+ let offset = 0;
79
+ for (const c of chunks) { out.set(c, offset); offset += c.byteLength; }
80
+ return out;
81
+ }
82
+
83
+ /**
84
+ * Whether this response can actually be read as a stream — i.e. whether the cap
85
+ * can be enforced at all.
86
+ *
87
+ * A real `Response` from `fetch()` always exposes a `ReadableStream` body except
88
+ * for the null-body statuses (204/205/304), which have nothing to cap. So in
89
+ * production this is true exactly when there are bytes to bound, and the capped
90
+ * path is never quietly skipped. It returns false only for the null-body statuses
91
+ * and for hand-rolled test doubles, which implement whichever of
92
+ * `arrayBuffer()`/`text()`/`json()` they need and no stream at all.
93
+ */
94
+ function hasStreamBody(response: Response): boolean {
95
+ const body = response.body;
96
+ return !!body && typeof body.getReader === 'function';
97
+ }
98
+
99
+ /**
100
+ * Capped replacement for `response.text()`.
101
+ *
102
+ * Decodes UTF-8 non-fatally, matching `Response.text()` — a malformed byte becomes
103
+ * U+FFFD rather than throwing, so swapping this in cannot turn a body that used to
104
+ * parse into an exception.
105
+ */
106
+ export async function readTextCapped(
107
+ response: Response,
108
+ maxBytes: number = MAX_JSON_BODY_BYTES,
109
+ ): Promise<string> {
110
+ if (hasStreamBody(response)) {
111
+ return new TextDecoder('utf-8').decode(await readBodyCapped(response, maxBytes));
112
+ }
113
+ // Double/null-body ladder — see hasStreamBody for why this is unreachable for a
114
+ // real body-bearing response.
115
+ if (typeof response.arrayBuffer === 'function') {
116
+ return new TextDecoder('utf-8').decode(new Uint8Array(await response.arrayBuffer()));
117
+ }
118
+ return response.text();
119
+ }
120
+
121
+ /**
122
+ * Capped replacement for `response.json()`.
123
+ *
124
+ * Throws {@link BodyTooLargeError} for an over-cap body and a `SyntaxError` for a
125
+ * malformed one, so callers that already distinguish "malformed JSON" from other
126
+ * failures keep working: the size failure is a distinct, catchable type rather
127
+ * than something that masquerades as a parse error.
128
+ */
129
+ export async function readJsonCapped<T = unknown>(
130
+ response: Response,
131
+ maxBytes: number = MAX_JSON_BODY_BYTES,
132
+ ): Promise<T> {
133
+ if (hasStreamBody(response) || typeof response.arrayBuffer === 'function' || typeof response.text === 'function') {
134
+ return JSON.parse(await readTextCapped(response, maxBytes)) as T;
135
+ }
136
+ // A double that implements json() alone.
137
+ return response.json() as Promise<T>;
138
+ }
139
+
140
+ /**
141
+ * Accumulator for a Node `IncomingMessage` body that decodes UTF-8 exactly ONCE,
142
+ * at the end.
143
+ *
144
+ * The obvious `body += chunk` is silently lossy. `+=` stringifies each Buffer on
145
+ * its own, so a multi-byte character straddling a chunk boundary is decoded as two
146
+ * independent fragments and both halves become U+FFFD. Chunk boundaries fall
147
+ * wherever the socket happens to split, so the damage scales with payload size and
148
+ * lands only on non-ASCII text — measured at ~15 mangled characters in a 560KB CJK
149
+ * body. Nothing throws: `JSON.parse` still succeeds, so the corruption reaches the
150
+ * report and the knowledge store unannounced.
151
+ *
152
+ * Buffering the chunks and concatenating before a single decode is what makes a
153
+ * split sequence whole again.
154
+ *
155
+ * `byteLength` counts real bytes, which is also the honest unit for a payload cap —
156
+ * the old `body.length` measured UTF-16 code units of an already-corrupted string.
157
+ */
158
+ export class Utf8Body {
159
+ private readonly chunks: Buffer[] = [];
160
+ private bytes = 0;
161
+
162
+ /** Total bytes accumulated so far. */
163
+ get byteLength(): number {
164
+ return this.bytes;
165
+ }
166
+
167
+ push(chunk: Buffer): void {
168
+ this.chunks.push(chunk);
169
+ this.bytes += chunk.byteLength;
170
+ }
171
+
172
+ /** Concatenate and decode. Safe to call once the stream has ended. */
173
+ toString(): string {
174
+ return Buffer.concat(this.chunks).toString('utf8');
175
+ }
176
+ }
@@ -13,8 +13,44 @@
13
13
 
14
14
  import type { Model } from '@earendil-works/pi-ai';
15
15
  import { extractUsage } from '../types/llm.ts';
16
+ import { logger } from '../logger.ts';
16
17
  import { metrics } from './metrics.ts';
17
18
 
19
+ /** Models already warned about, so the notice fires at most once per model per process. */
20
+ const unpricedModelsWarned = new Set<string>();
21
+
22
+ /**
23
+ * Warn once when a model bills tokens but carries an all-zero price table.
24
+ *
25
+ * Cost is never read off the wire — pi computes it as tokens × the model's local
26
+ * price table — so an unpriced table yields $0.00 silently and indefinitely. That
27
+ * is legitimate for flat-rate plans and local servers, but it is also exactly what
28
+ * a misconfiguration looks like: a hand-written `models[]` entry in models.json
29
+ * REPLACES pi's catalog entry for the same id, discarding its pricing. This
30
+ * happened in practice and went unnoticed for weeks because every display site
31
+ * suppressed a zero cost. The message deliberately does not assert a fault — it
32
+ * states the condition and names the usual cause.
33
+ */
34
+ function warnIfUnpriced(model: Model<any>, tokens: number, cost: number): void {
35
+ if (tokens <= 0 || cost > 0) return;
36
+ const m = model as unknown as { provider?: string; id?: string; cost?: Record<string, number> };
37
+ const key = `${m.provider ?? 'unknown'}/${m.id ?? 'unknown'}`;
38
+ if (unpricedModelsWarned.has(key)) return;
39
+
40
+ const table = m.cost;
41
+ const allZero =
42
+ !table || Object.values(table).every((v) => typeof v !== 'number' || v === 0);
43
+ if (!allZero) return;
44
+
45
+ unpricedModelsWarned.add(key);
46
+ logger.warn(
47
+ `[LlmUsage] Model "${key}" consumed ${tokens} tokens but reports $0.00 — its price table is all zeros. ` +
48
+ `Expected for flat-rate plans and local servers. Otherwise the price data is missing: a hand-written ` +
49
+ `"models" entry in ~/.pi/agent/models.json replaces pi's catalog entry (and its pricing) for the same ` +
50
+ `model id — remove it and let pi's catalog supply the model, or use "modelOverrides", which merges.`
51
+ );
52
+ }
53
+
18
54
  /** Structural subset of the run observer — just the token sink. Kept structural (rather
19
55
  * than importing ResearchObserver from core/) so this foundation-layer helper does not
20
56
  * depend on an upper layer; the full ResearchObserver satisfies it. */
@@ -43,6 +79,7 @@ export function recordLlmUsage(
43
79
  ): { tokens: number; cost: number } {
44
80
  if (!rawUsage) return { tokens: 0, cost: 0 };
45
81
  const { tokens, cost } = extractUsage(model, rawUsage);
82
+ warnIfUnpriced(model, tokens, cost);
46
83
  if (tokens > 0 || cost > 0) {
47
84
  const labels: Record<string, string> = { component: opts.component };
48
85
  if (opts.complexity !== undefined) labels['complexity'] = String(opts.complexity);
@@ -283,8 +283,15 @@ export function buildResearchSummary(stats: ResearchStats): string {
283
283
  if (stats.tokens > 0) {
284
284
  resourceParts.push(`**${formatTokens(stats.tokens)}** tokens`);
285
285
  }
286
- if (stats.cost > 0) {
287
- const costStr = stats.cost < 0.01 ? '<$0.01' : `$${stats.cost.toFixed(2)}`;
286
+ // Render the cost whenever tokens were actually billed-through, INCLUDING when it
287
+ // is exactly zero. Gating on `cost > 0` hid a real defect for weeks: a model whose
288
+ // price table is all zeros (an unpriced or catalog-overwritten entry) produced no
289
+ // cost line at all, so a misconfiguration was indistinguishable from a run that
290
+ // simply had not reported yet. "$0.00" beside a large token count is conspicuous;
291
+ // silence is not. Zero is also the honest figure for flat-rate and local providers.
292
+ if (stats.tokens > 0 || stats.cost > 0) {
293
+ const costStr =
294
+ stats.cost === 0 ? '$0.00' : stats.cost < 0.01 ? '<$0.01' : `$${stats.cost.toFixed(2)}`;
288
295
  resourceParts.push(`**${costStr}**`);
289
296
  }
290
297
  if (stats.durationMs > 0) {
@@ -317,6 +324,9 @@ export interface SessionStats {
317
324
  totalUrlsDiscovered: number;
318
325
  totalSearchQueries: number;
319
326
  totalTokens: number;
327
+ /** Summed LLM cost across the session's runs. The Session Metrics UI advertises
328
+ * "API cost estimates"; without this field it had no cost to show. */
329
+ totalCost: number;
320
330
  totalToolUsage: {
321
331
  searches: number;
322
332
  scrapes: number;
@@ -339,6 +349,7 @@ export function aggregateSessionStats(
339
349
  let totalUrlsDiscovered = 0;
340
350
  let totalSearchQueries = 0;
341
351
  let totalTokens = 0;
352
+ let totalCost = 0;
342
353
  const totalToolUsage = {
343
354
  searches: 0,
344
355
  scrapes: 0,
@@ -359,6 +370,7 @@ export function aggregateSessionStats(
359
370
  totalUrlsDiscovered += stats.urlsDiscovered;
360
371
  totalSearchQueries += stats.searchQueries;
361
372
  totalTokens += stats.tokens;
373
+ totalCost += stats.cost;
362
374
  totalToolUsage.searches += stats.toolUsage.searches;
363
375
  totalToolUsage.scrapes += stats.toolUsage.scrapes;
364
376
  totalToolUsage.securitySearches += stats.toolUsage.securitySearches;
@@ -377,6 +389,7 @@ export function aggregateSessionStats(
377
389
  totalUrlsDiscovered,
378
390
  totalSearchQueries,
379
391
  totalTokens,
392
+ totalCost,
380
393
  totalToolUsage,
381
394
  };
382
395
  }
@@ -403,6 +416,16 @@ export function buildSessionOverview(stats: SessionStats): string {
403
416
  }
404
417
  if (stats.totalTokens > 0) {
405
418
  overviewParts.push(`**${formatTokens(stats.totalTokens)}** tokens`);
419
+ // Shown alongside tokens even at exactly $0.00 — the Session Metrics view
420
+ // advertises cost estimates, and a silent omission is what let an unpriced
421
+ // model table go unnoticed. Zero is correct for flat-rate and local providers.
422
+ const costStr =
423
+ stats.totalCost === 0
424
+ ? '$0.00'
425
+ : stats.totalCost < 0.01
426
+ ? '<$0.01'
427
+ : `$${stats.totalCost.toFixed(2)}`;
428
+ overviewParts.push(`**${costStr}**`);
406
429
  }
407
430
  if (stats.totalDurationMs > 0) {
408
431
  overviewParts.push(`**${formatDuration(stats.totalDurationMs)}** total research time`);
@@ -207,8 +207,15 @@ export async function exportResearchReport(
207
207
  */
208
208
  export function appendExportMessage(result: string, filepath: string, totalCost?: number): string {
209
209
  let footer = `\n\n---\n\nResearch report saved to: ${filepath}`;
210
- if (totalCost !== undefined && totalCost > 0) {
211
- const costStr = totalCost < 0.01 ? '<$0.01' : `$${totalCost.toFixed(4)}`.replace(/0+$/, '').replace(/\.$/, '');
210
+ // Report a defined cost even when it is exactly zero — suppressing it made an
211
+ // unpriced/misconfigured model table look identical to "no cost reported yet".
212
+ if (totalCost !== undefined) {
213
+ const costStr =
214
+ totalCost === 0
215
+ ? '$0.00'
216
+ : totalCost < 0.01
217
+ ? '<$0.01'
218
+ : `$${totalCost.toFixed(4)}`.replace(/0+$/, '').replace(/\.$/, '');
212
219
  footer += `\nTotal cost: ${costStr}`;
213
220
  }
214
221
  return result + footer;