@lincoln504/pi-research 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +4 -4
  2. package/dist/cli.mjs +1404 -600
  3. package/dist/thread-worker.mjs +40 -11
  4. package/docs/AGENT-SKILL.md +11 -4
  5. package/docs/ARCHITECTURE.md +19 -9
  6. package/docs/CHANGELOG.md +107 -1
  7. package/docs/CONFIGURATION.md +10 -8
  8. package/docs/KNOWLEDGE-STORE.md +5 -4
  9. package/docs/PI-EXTENSION.md +1 -1
  10. package/docs/SDK.md +63 -5
  11. package/docs/THIRD-PARTY-NOTICES.md +1 -1
  12. package/package.json +5 -4
  13. package/scripts/build.cjs +23 -5
  14. package/scripts/cleanup.cjs +50 -10
  15. package/scripts/prepare.cjs +15 -5
  16. package/scripts/setup.cjs +24 -4
  17. package/skills/pi-research/README.md +9 -2
  18. package/skills/pi-research/SKILL.md +8 -5
  19. package/skills/pi-research/references/configuration.md +7 -3
  20. package/skills/pi-research/scripts/run.mjs +24 -2
  21. package/src/cli.ts +161 -21
  22. package/src/config.ts +56 -15
  23. package/src/constants.ts +31 -0
  24. package/src/core/interfaces/knowledge-interfaces.ts +6 -1
  25. package/src/core/interfaces/state-manager-interfaces.ts +1 -1
  26. package/src/core/llm/model-registry-factory.ts +40 -3
  27. package/src/core/llm/research-model-resolver.ts +2 -2
  28. package/src/core/pi-version.ts +136 -0
  29. package/src/core/service-registry.ts +117 -8
  30. package/src/index.ts +11 -18
  31. package/src/infrastructure/browser/browser-client.ts +16 -4
  32. package/src/infrastructure/browser/browser-error-utils.ts +12 -1
  33. package/src/infrastructure/browser/browser-server.ts +17 -5
  34. package/src/infrastructure/browser/browser-task-scheduler.ts +101 -23
  35. package/src/infrastructure/browser/cleanup-utils.ts +45 -7
  36. package/src/infrastructure/browser/client-agent.ts +4 -3
  37. package/src/infrastructure/browser/priority-task-queue.ts +10 -3
  38. package/src/infrastructure/browser/scheduler-factory.ts +26 -1
  39. package/src/infrastructure/browser/task-execution-service.ts +92 -28
  40. package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
  41. package/src/infrastructure/browser/thread-worker.mjs +40 -11
  42. package/src/infrastructure/browser/worker-pool-manager.ts +11 -3
  43. package/src/infrastructure/embedding/embedding-client.ts +22 -8
  44. package/src/infrastructure/embedding/embedding-factory.ts +28 -11
  45. package/src/infrastructure/embedding/embedding-server.ts +65 -4
  46. package/src/infrastructure/file-lock-service.ts +223 -26
  47. package/src/infrastructure/process-lifecycle-service.ts +108 -31
  48. package/src/infrastructure/state/state-backup-manager.ts +5 -7
  49. package/src/infrastructure/state/state-manager-service.ts +1 -1
  50. package/src/infrastructure/state/state-manager.ts +19 -20
  51. package/src/infrastructure/types/state-types.ts +9 -0
  52. package/src/knowledge/embedder-init.ts +1 -1
  53. package/src/knowledge/embedder.ts +52 -3
  54. package/src/knowledge/index.ts +1 -1
  55. package/src/knowledge/store.ts +35 -9
  56. package/src/knowledge/webgpu-viability.ts +9 -4
  57. package/src/knowledge/writer-queue.ts +88 -46
  58. package/src/orchestration/deep-research-orchestrator.ts +13 -2
  59. package/src/orchestration/quick-research-orchestrator.ts +2 -2
  60. package/src/orchestration/research-orchestration-service.ts +2 -4
  61. package/src/orchestration/research-synthesis-service.ts +34 -92
  62. package/src/security/cisa-kev.ts +10 -3
  63. package/src/security/github-advisories.ts +18 -7
  64. package/src/security/nvd-client.ts +17 -6
  65. package/src/security/osv-client.ts +14 -4
  66. package/src/skill-install/skill-installer.ts +135 -9
  67. package/src/stackexchange/rest-client.ts +6 -2
  68. package/src/tools/research-knowledge-search.ts +23 -2
  69. package/src/tools/search.ts +6 -1
  70. package/src/utils/atomic-replace.ts +101 -0
  71. package/src/utils/circuit-breaker.ts +20 -0
  72. package/src/utils/citation-utils.ts +106 -8
  73. package/src/utils/http-body.ts +176 -0
  74. package/src/utils/llm-usage.ts +37 -0
  75. package/src/utils/metrics-summary.ts +25 -2
  76. package/src/utils/research-export.ts +9 -2
  77. package/src/web-research/browser-search.ts +50 -4
  78. package/src/web-research/retry-utils.ts +13 -1
  79. package/src/web-research/search.ts +20 -5
  80. package/src/web-research/types.ts +17 -4
  81. package/src/web-research/web-scraper.ts +7 -50
  82. package/src/youtube/transcript-client.ts +5 -1
package/src/cli.ts CHANGED
@@ -22,6 +22,7 @@
22
22
  */
23
23
 
24
24
  import * as fs from 'node:fs';
25
+ import * as os from 'node:os';
25
26
  import * as path from 'node:path';
26
27
  import { fileURLToPath } from 'node:url';
27
28
 
@@ -84,8 +85,47 @@ export const EXIT = {
84
85
  * re-running a broken command.
85
86
  */
86
87
  TEMPFAIL: 75,
88
+ /**
89
+ * The run was CANCELLED via a programmatic abort (the caller's AbortSignal),
90
+ * with no OS signal involved. 130 — 128+SIGINT — is the conventional
91
+ * "interrupted" code and the natural default when there is no signal number to
92
+ * derive one from.
93
+ *
94
+ * A cancellation caused by an actual SIGNAL does NOT use this constant: see
95
+ * exitCodeForSignal below, which reports the true 128+N.
96
+ *
97
+ * Distinct from SOFTWARE for the same reason TEMPFAIL is: a deliberate stop is
98
+ * not a fault. Reported as SOFTWARE (70), a cancel matched the skill contract's
99
+ * "runtime error — suggest one retry for transient failures", so an agent
100
+ * relaying the result would offer to re-run the very research the user had just
101
+ * interrupted, and a --json consumer could not tell a cancel from a crash.
102
+ */
103
+ CANCELLED: 130,
87
104
  } as const;
88
105
 
106
+ /**
107
+ * Exit code for a run stopped by `sig`, following the POSIX 128+N convention that
108
+ * a shell uses when reporting a signal-terminated child (SIGINT → 130,
109
+ * SIGTERM → 143, SIGHUP → 129, SIGQUIT → 131).
110
+ *
111
+ * Why derive it per-signal instead of returning a single "cancelled" code: this
112
+ * process INSTALLS handlers for those signals, so it is never actually killed by
113
+ * them — it catches them and exits deliberately. If it exited a flat 130, the code
114
+ * a wrapper observes would depend on whether our handler won the race against a
115
+ * force-kill: handler wins → 130, handler loses → the shell's 143. Deriving 128+N
116
+ * makes the handler transparent, so the same user action reports the same number
117
+ * either way.
118
+ *
119
+ * Every value lands in 129..143, safely inside the 0-255 range a POSIX wait status
120
+ * preserves — nothing here can be truncated to a false 0. Windows does not
121
+ * implement the 128+N convention at all (its exit codes are 32-bit and carry no
122
+ * signal semantics), so there these are simply our documented contract values.
123
+ */
124
+ export function exitCodeForSignal(sig: NodeJS.Signals): number {
125
+ const signum = (os.constants.signals as Record<string, number | undefined>)[sig];
126
+ return typeof signum === 'number' ? 128 + signum : EXIT.CANCELLED;
127
+ }
128
+
89
129
  // ---------------------------------------------------------------------------
90
130
  // Output helpers
91
131
  // ---------------------------------------------------------------------------
@@ -137,9 +177,19 @@ function makeProgressObserver(): HeadlessObserverOptions {
137
177
  case 'evaluation_decision':
138
178
  toStderr(` • evaluator: ${data?.action}\n`);
139
179
  break;
140
- case 'tokens_consumed':
141
- if (data?.cost) toStderr(` • tokens: ${data?.tokens ?? 0} ($${Number(data.cost).toFixed(4)})\n`);
180
+ case 'tokens_consumed': {
181
+ // A truthiness check on cost used to gate this whole line, so a model with
182
+ // an all-zero price table suppressed the TOKEN count as well — the run
183
+ // looked like it consumed nothing. Tokens stand on their own; cost is
184
+ // appended whenever it is a number, including 0.
185
+ const consumed = Number(data?.tokens ?? 0);
186
+ const costVal = typeof data?.cost === 'number' ? data.cost : undefined;
187
+ if (consumed > 0 || costVal !== undefined) {
188
+ const costPart = costVal === undefined ? '' : ` ($${costVal.toFixed(4)})`;
189
+ toStderr(` • tokens: ${consumed}${costPart}\n`);
190
+ }
142
191
  break;
192
+ }
143
193
  case 'complete':
144
194
  toStderr(` • complete (${data?.result?.length ?? 0} chars)\n`);
145
195
  break;
@@ -180,20 +230,33 @@ export function bridgeConfigEnv(explicitConfigPath?: string): { path: string; lo
180
230
  // iteration and the "don't clobber the real environment" guard would then
181
231
  // block the overlay's value — cli.env could never override config.env.
182
232
  const merged: Record<string, string> = {};
233
+ // Keys whose winning value came from an EXPLICIT --config file (not the
234
+ // default base, not the cli.env overlay). Project-scoped keys are only
235
+ // bridged for these — see below.
236
+ const fromExplicitBase = new Set<string>();
183
237
  for (const filePath of [basePath, cliOverlayPath]) {
184
238
  try {
185
239
  if (!fs.existsSync(filePath)) continue;
186
- Object.assign(merged, parseDotEnv(fs.readFileSync(filePath, 'utf-8')));
240
+ const parsed = parseDotEnv(fs.readFileSync(filePath, 'utf-8'));
241
+ Object.assign(merged, parsed);
242
+ for (const k of Object.keys(parsed)) {
243
+ if (explicitConfigPath !== undefined && filePath === basePath) fromExplicitBase.add(k);
244
+ else fromExplicitBase.delete(k);
245
+ }
187
246
  loaded = true;
188
247
  } catch {
189
248
  // ignore unreadable file; fall through to the next
190
249
  }
191
250
  }
192
251
  for (const [k, v] of Object.entries(merged)) {
193
- // Skip per-directory (project-scoped) keys: promoting them into process.env would make
194
- // config.env out-rank the per-directory registry, defeating a per-cwd override. They are
195
- // still applied as a lower-precedence file layer by getConfig()/loadEnvFiles().
196
- if (isProjectScopedKey(k)) continue;
252
+ // Skip per-directory (project-scoped) keys from the AMBIENT files: promoting them into
253
+ // process.env would make config.env out-rank the per-directory registry, defeating a
254
+ // per-cwd override. They are still applied as a lower-precedence file layer by
255
+ // getConfig()/loadEnvFiles(). An EXPLICIT --config file is different: getConfig never
256
+ // reads that path, so skipping its keys silently dropped them (a depth or
257
+ // knowledge-mode set in `--config ci.env` simply did not apply) — and a file the user
258
+ // named on this exact invocation outranking the per-directory registry is the intent.
259
+ if (isProjectScopedKey(k) && !fromExplicitBase.has(k)) continue;
197
260
  if (process.env[k] === undefined && (k.startsWith('PI_RESEARCH_') || k === 'STACKEXCHANGE_API_KEY' || k === 'GITHUB_TOKEN' || k === 'NVD_API_KEY')) {
198
261
  process.env[k] = v;
199
262
  }
@@ -853,9 +916,25 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
853
916
  // machine as a setup problem would send the user to reconfigure a working install.
854
917
  const isCapacity = err instanceof Error && err.name === 'ResearchRunCapacityError';
855
918
 
919
+ // Cancellation. Primary signal is the latch set by the SIGINT/SIGTERM handler —
920
+ // an exact fact, unlike the message, which varies by whichever subsystem noticed
921
+ // the abort first ('Aborted' from the search layer, 'Research aborted' /
922
+ // 'Research cancelled' from the orchestrator). The anchored message signatures
923
+ // only cover a programmatic abort where no signal arrived. Deliberately NOT
924
+ // matched: a bare `err.name === 'AbortError'` — internal timeout aborts
925
+ // (AbortSignal-driven retry/scrape deadlines) throw that name too, and an
926
+ // escaped one would report "cancelled — do not re-run" for a run nobody
927
+ // stopped. In the CLI every real cancellation sets the latch or carries these
928
+ // exact signatures. Checked before everything below: an abort can unwind
929
+ // through a provider call and surface wording that the config/rate-limit
930
+ // heuristics would otherwise claim.
931
+ const isCancellation =
932
+ cancellationRequested ||
933
+ (err instanceof Error && /^(aborted|research (aborted|cancelled))$/i.test(msg));
934
+
856
935
  // Auth / model / config-shaped errors → CONFIG exit + config block.
857
936
  const isConfigError =
858
- !isTaggedResearchStop && !isCapacity && (
937
+ !isTaggedResearchStop && !isCapacity && !isCancellation && (
859
938
  lower.includes('no llm model available') ||
860
939
  lower.includes('not found in pi') ||
861
940
  lower.includes('no api key') ||
@@ -869,20 +948,27 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
869
948
  // "api key" in its body (e.g. "rate limit exceeded for this api key"), and
870
949
  // classifying that as a setup problem sends the user to reconfigure a
871
950
  // perfectly working key instead of waiting out the limit.
872
- const isRateLimit = lower.includes('429') || lower.includes('rate limit') || lower.includes('too many requests');
873
- const exitCode = isCapacity
874
- ? EXIT.TEMPFAIL
875
- : isConfigError && !isRateLimit
876
- ? EXIT.CONFIG
877
- : EXIT.SOFTWARE;
951
+ const isRateLimit = !isCancellation && (lower.includes('429') || lower.includes('rate limit') || lower.includes('too many requests'));
952
+ const exitCode = isCancellation
953
+ ? (cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED)
954
+ : isCapacity
955
+ ? EXIT.TEMPFAIL
956
+ : isConfigError && !isRateLimit
957
+ ? EXIT.CONFIG
958
+ : EXIT.SOFTWARE;
878
959
 
879
960
  // --json: emit a structured error on stdout so a machine consumer never has to
880
961
  // special-case plain-text stderr. Mirrors the { ok: true, ... } success shape.
881
962
  if (json) {
882
963
  const payload: Record<string, unknown> = { ok: false, error: msg, exitCode };
883
964
  // Machine-readable "this will succeed later" flag, so a consumer can back off
884
- // and retry without string-matching the human message.
885
- if (isCapacity || isRateLimit) payload['retryable'] = true;
965
+ // and retry without string-matching the human message. Gated on cancellation:
966
+ // a Ctrl-C landing in the same window a queue-wait expires must never emit
967
+ // `cancelled: true` alongside `retryable: true` — a consumer honouring the
968
+ // latter would re-run the research the user just stopped.
969
+ if ((isCapacity || isRateLimit) && !isCancellation) payload['retryable'] = true;
970
+ // A cancel is emphatically NOT retryable-by-default: the user asked to stop.
971
+ if (isCancellation) payload['cancelled'] = true;
886
972
  if (process.env['PI_RESEARCH_DEBUG'] === 'true' && err instanceof Error && err.stack) {
887
973
  payload['stack'] = err.stack;
888
974
  }
@@ -890,6 +976,13 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
890
976
  return exitCode;
891
977
  }
892
978
 
979
+ // Cancellation is a completed intention, not a failure — never a stack trace,
980
+ // and never the word "failed".
981
+ if (isCancellation) {
982
+ toStderr(`\npi-research ${what} cancelled.\n`);
983
+ return exitCode;
984
+ }
985
+
893
986
  // Capacity exhaustion is a healthy-machine "busy" state — report it as such and
894
987
  // never with a stack trace, which would read as a crash for a routine condition.
895
988
  if (isCapacity) {
@@ -1014,10 +1107,14 @@ CONFIGURE
1014
1107
  Run \`${BINARY_NAME} status\` to see exactly what is detected.
1015
1108
 
1016
1109
  EXIT CODES
1017
- 0 success
1018
- 64 bad arguments
1019
- 78 not configured — message prints the exact locations to fix
1020
- 70 runtime error (network, provider, internal)
1110
+ 0 success
1111
+ 64 bad arguments
1112
+ 75 at capacity — every concurrent run slot is busy; nothing is broken, retry later
1113
+ 78 not configured — message prints the exact locations to fix
1114
+ (on \`knowledge\`, also returned when the store is deliberately disabled)
1115
+ 130 cancelled — Ctrl-C; nothing failed, do not auto-retry
1116
+ (other stop signals report POSIX 128+N: 143 SIGTERM, 129 SIGHUP, 131 SIGQUIT)
1117
+ 70 runtime error (network, provider, internal)
1021
1118
  `;
1022
1119
  }
1023
1120
 
@@ -1257,6 +1354,32 @@ const flushAndExit = (code: number): void => {
1257
1354
  // outlives that one run.
1258
1355
  let activeResearchAbortController: AbortController | null = null;
1259
1356
 
1357
+ /**
1358
+ * Set once SIGINT/SIGTERM has been handled. Read by reportError so a cancellation
1359
+ * is classified by the fact that we were signalled, not by pattern-matching the
1360
+ * message an aborted subsystem happened to throw.
1361
+ */
1362
+ let cancellationRequested = false;
1363
+
1364
+ /**
1365
+ * WHICH signal fired — so a cancellation that unwinds through reportError
1366
+ * reports the same 128+N the signal path itself would (SIGTERM → 143, not a
1367
+ * flat 130). reportError races onSignal's own flushAndExit; latching the name
1368
+ * makes both racers agree on the number, and keeps the `exitCode` field in
1369
+ * `--json` from contradicting the actual exit status.
1370
+ */
1371
+ let cancellationSignal: NodeJS.Signals | null = null;
1372
+
1373
+ /** Test seams for the module-level cancellation latch (set only by onSignal). */
1374
+ export function _resetCancellationLatchForTests(): void {
1375
+ cancellationRequested = false;
1376
+ cancellationSignal = null;
1377
+ }
1378
+ export function _markCancellationForTests(sig?: NodeJS.Signals): void {
1379
+ cancellationRequested = true;
1380
+ cancellationSignal = sig ?? null;
1381
+ }
1382
+
1260
1383
  async function main(argv: string[]): Promise<number> {
1261
1384
  // Register signal handlers so SIGINT/SIGTERM always trigger SDK shutdown
1262
1385
  // before the process exits. safeShutdown() is idempotent — safe to call
@@ -1265,6 +1388,8 @@ async function main(argv: string[]): Promise<number> {
1265
1388
  const onSignal = (sig: string) => {
1266
1389
  if (_signalCleanupDone) return;
1267
1390
  _signalCleanupDone = true;
1391
+ cancellationRequested = true;
1392
+ cancellationSignal = sig as NodeJS.Signals;
1268
1393
  toStderr(`\n[pi-research] ${sig} — cleaning up…\n`);
1269
1394
  // Abort the in-flight research run FIRST (synchronously ahead of the shutdown
1270
1395
  // teardown below) so the orchestrator's existing signal-aware cancellation path
@@ -1274,10 +1399,25 @@ async function main(argv: string[]): Promise<number> {
1274
1399
  // Fire-and-forget: we can't await here, but safeShutdown swallows errors
1275
1400
  // and the finally blocks in each command handler also call it. Drain via
1276
1401
  // flushAndExit (not a bare process.exit) so buffered stdout isn't truncated.
1277
- void safeShutdown().finally(() => flushAndExit(EXIT.SOFTWARE));
1402
+ // 128+N, not a flat CANCELLED: see exitCodeForSignal. Because we handle these
1403
+ // signals rather than dying from them, a fixed code would make the observed
1404
+ // exit status depend on whether this handler beat a force-kill.
1405
+ void safeShutdown().finally(() => flushAndExit(exitCodeForSignal(sig as NodeJS.Signals)));
1278
1406
  };
1279
1407
  process.once('SIGINT', () => onSignal('SIGINT'));
1280
1408
  process.once('SIGTERM', () => onSignal('SIGTERM'));
1409
+ // SIGHUP (terminal closed) and, on Windows, SIGBREAK (Ctrl+Break). Without these
1410
+ // the run dies without cancelling: no abort of the in-flight research, no browser
1411
+ // or embedding-server teardown, and no 128+N exit code. On Windows it matters
1412
+ // most — SIGTERM is never delivered there (process.kill maps to TerminateProcess),
1413
+ // so Ctrl-C and Ctrl+Break are the only signals that reach this handler at all.
1414
+ // The SDK already registers both (see _addSignalHandlers); the CLI did not, which
1415
+ // left the SDK's handler to tear the container down underneath a live orchestrator
1416
+ // — exactly the race the CLI installs its own handlers to avoid.
1417
+ process.once('SIGHUP', () => onSignal('SIGHUP'));
1418
+ if (process.platform === 'win32') {
1419
+ process.once('SIGBREAK', () => onSignal('SIGBREAK'));
1420
+ }
1281
1421
 
1282
1422
  let parsed: ParsedArgs;
1283
1423
  try {
package/src/config.ts CHANGED
@@ -14,6 +14,7 @@ import { getConfigDirName } from './utils/host-config.ts';
14
14
  import { Type, type Static } from 'typebox';
15
15
  import { Value } from 'typebox/value';
16
16
  import { normalizeWorkspacePath } from './utils/text-utils.ts';
17
+ import { replaceFileSync } from './utils/atomic-replace.ts';
17
18
 
18
19
  /**
19
20
  * Validates configuration schema using TypeBox.
@@ -501,7 +502,7 @@ export function describeKnowledgeStoreMode(
501
502
  let registryHasKey = false;
502
503
  try {
503
504
  const registry = loadProjectSettingsRegistry();
504
- registryHasKey = registry[normalizeWorkspacePath(cwd)]?.[KEY] !== undefined;
505
+ registryHasKey = findRegistryEntry(registry, normalizeWorkspacePath(cwd))?.[KEY] !== undefined;
505
506
  } catch { /* unreadable registry → not the origin */ }
506
507
  if (registryHasKey) origin = 'this directory (project settings)';
507
508
  else if (iface && fileHasKey(getInterfaceEnvFilePath(iface))) origin = `${iface}.env overlay`;
@@ -546,12 +547,48 @@ export function parseDotEnv(content: string): Record<string, string> {
546
547
  * Load environment variables from global and local files.
547
548
  * Order: Global File < Interface overlay < Legacy .env < Centralized Registry (WINS)
548
549
  */
550
+ /**
551
+ * Look up a directory's entry in the project-settings registry, tolerating case.
552
+ *
553
+ * On Windows and on macOS's default (case-insensitive) filesystem, `C:\\Users\\Bob\\Proj`
554
+ * and `c:\\users\\bob\\proj` are the SAME directory but different object keys, so a shell
555
+ * that reports a different casing than the one the settings were saved under made those
556
+ * settings silently vanish. normalizeWorkspacePath only resolves and strips a trailing
557
+ * separator; it does not fold case, and it must not start doing so — the key it produces
558
+ * is persisted, so folding it would orphan every entry already on disk.
559
+ *
560
+ * Resolving at READ time instead fixes the lookup without touching stored data: exact
561
+ * match first (the only path taken on Linux, and the fast path everywhere), then a
562
+ * case-insensitive scan as a fallback.
563
+ */
564
+ function findRegistryKey(
565
+ registry: Record<string, Record<string, string>>,
566
+ normalizedCwd: string,
567
+ ): string | undefined {
568
+ if (registry[normalizedCwd] !== undefined) return normalizedCwd;
569
+ // Only case-insensitive filesystems can produce this mismatch; skip the scan elsewhere.
570
+ if (process.platform !== 'win32' && process.platform !== 'darwin') return undefined;
571
+ const lowered = normalizedCwd.toLowerCase();
572
+ for (const key of Object.keys(registry)) {
573
+ if (key.toLowerCase() === lowered) return key;
574
+ }
575
+ return undefined;
576
+ }
577
+
578
+ function findRegistryEntry(
579
+ registry: Record<string, Record<string, string>>,
580
+ normalizedCwd: string,
581
+ ): Record<string, string> | undefined {
582
+ const key = findRegistryKey(registry, normalizedCwd);
583
+ return key === undefined ? undefined : registry[key];
584
+ }
585
+
549
586
  function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, string> {
550
587
  const merged: Record<string, string> = {};
551
588
  const globalPath = getGlobalEnvFilePath();
552
589
  const registry = loadProjectSettingsRegistry();
553
590
  const normalizedCwd = normalizeWorkspacePath(cwd);
554
- const homeRegistry = registry[normalizeWorkspacePath(os.homedir())];
591
+ const homeRegistry = findRegistryEntry(registry, normalizeWorkspacePath(os.homedir()));
555
592
 
556
593
  // 1. ONE-TIME MIGRATION: Populate config.env from HOME registry if missing
557
594
  try {
@@ -613,15 +650,17 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
613
650
  Object.entries(legacyEnv).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
614
651
  );
615
652
  const existingLocal = Object.fromEntries(
616
- Object.entries(registry[normalizedCwd] ?? {}).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
653
+ Object.entries(findRegistryEntry(registry, normalizedCwd) ?? {}).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
617
654
  );
618
655
  if (Object.keys(legacyLocal).length > 0 && JSON.stringify(existingLocal) !== JSON.stringify(legacyLocal)) {
619
656
  logger.info(`[config] Migrating legacy .pi-research.env settings from ${cwd} to central registry...`);
620
657
  // Persist under the registry lock, reading the latest on-disk state so a concurrent
621
658
  // write isn't lost; also reflect the merge in the in-memory `registry` used just below.
622
- registry[normalizedCwd] = { ...registry[normalizedCwd], ...legacyLocal };
659
+ const mergeKey = findRegistryKey(registry, normalizedCwd) ?? normalizedCwd;
660
+ registry[mergeKey] = { ...registry[mergeKey], ...legacyLocal };
623
661
  updateProjectSettingsRegistry((disk) => {
624
- disk[normalizedCwd] = { ...disk[normalizedCwd], ...legacyLocal };
662
+ const diskKey = findRegistryKey(disk, normalizedCwd) ?? normalizedCwd;
663
+ disk[diskKey] = { ...disk[diskKey], ...legacyLocal };
625
664
  });
626
665
  }
627
666
  } catch (err) {
@@ -633,8 +672,9 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
633
672
  // LOCAL_SCOPE_KEYS are applied (and may override), never user-scoped keys — those belong to
634
673
  // config.env. Any non-local key present here is stale pollution (e.g. a pre-fix legacy
635
674
  // migration) and is ignored rather than allowed to override the user's global settings.
636
- if (registry[normalizedCwd]) {
637
- for (const [key, val] of Object.entries(registry[normalizedCwd])) {
675
+ const cwdEntry = findRegistryEntry(registry, normalizedCwd);
676
+ if (cwdEntry) {
677
+ for (const [key, val] of Object.entries(cwdEntry)) {
638
678
  if (LOCAL_SCOPE_KEYS.has(key)) {
639
679
  // Don't interpolate values in warnings — a config key could carry a secret and
640
680
  // redactSecrets won't catch short/non-standard tokens. Key + winner is enough.
@@ -763,7 +803,11 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
763
803
  // Read-merge-write under one lock so a concurrent local write to another directory
764
804
  // isn't lost (the merge onto the existing entry happens against the latest on-disk state).
765
805
  updateProjectSettingsRegistry((registry) => {
766
- registry[normalizedCwd] = { ...(registry[normalizedCwd] ?? {}), ...newValues };
806
+ // Merge into the entry that already represents THIS directory, even if it was
807
+ // written with different casing on a case-insensitive filesystem. Writing a
808
+ // second key would leave the earlier settings stranded and unreachable.
809
+ const key = findRegistryKey(registry, normalizedCwd) ?? normalizedCwd;
810
+ registry[key] = { ...(registry[key] ?? {}), ...newValues };
767
811
  });
768
812
 
769
813
  logger.debug(`[config] Saved project settings for ${normalizedCwd} to central registry.`);
@@ -870,15 +914,12 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
870
914
  const tmpPath = `${p}.tmp.${process.pid}.${Date.now()}`;
871
915
  fs.writeFileSync(tmpPath, outLines.join('\n'), { encoding: 'utf-8', mode: 0o600 });
872
916
  try {
873
- fs.renameSync(tmpPath, p);
874
- } catch (renameErr) {
875
- if (process.platform === 'win32') {
876
- fs.copyFileSync(tmpPath, p);
917
+ if (replaceFileSync(tmpPath, p) === 'copied') {
877
918
  try { fs.unlinkSync(tmpPath); } catch { /* best effort */ }
878
- } else {
879
- try { fs.unlinkSync(tmpPath); } catch { /* best effort */ }
880
- throw renameErr;
881
919
  }
920
+ } catch (renameErr) {
921
+ try { fs.unlinkSync(tmpPath); } catch { /* best effort */ }
922
+ throw renameErr;
882
923
  }
883
924
  // Tighten regardless of how the file came to exist (fresh rename, Windows copy over
884
925
  // an existing file, or a pre-hardening 0644 file that the rename replaced-in-place).
package/src/constants.ts CHANGED
@@ -177,3 +177,34 @@ export const RESEARCHER_LAUNCH_DELAY_MS = 1500;
177
177
  export const MAX_QUERIES_PER_RESEARCHER_LEVEL_1 = 10;
178
178
  export const MAX_QUERIES_PER_RESEARCHER_LEVEL_2 = 15;
179
179
  export const MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;
180
+
181
+ /**
182
+ * Tools excluded from every researcher unless the caller says otherwise.
183
+ *
184
+ * `grep` is developer-only: during WEB research a high-capability model otherwise
185
+ * spends turns searching the local filesystem instead of the internet.
186
+ */
187
+ export const DEFAULT_EXCLUDED_TOOLS: readonly string[] = ['grep'];
188
+
189
+ /**
190
+ * Resolve the effective researcher exclusion list.
191
+ *
192
+ * An EMPTY array is "no preference", not "exclude nothing". The two front-ends
193
+ * express no-preference differently — the pi extension always builds an array
194
+ * (`[...new Set([...])]`, often empty) while the CLI omits the key entirely — so a
195
+ * plain truthiness test made the same user input mean opposite things: grep was
196
+ * available in the extension and excluded on the CLI, regardless of depth.
197
+ *
198
+ * `configDisabled` (PI_RESEARCH_DISABLED_TOOLS) is strictly ADDITIVE. Folding it in
199
+ * as a replacement meant disabling any unrelated tool silently ENABLED grep, so a
200
+ * subtract-only setting could grant a capability.
201
+ */
202
+ export function resolveExcludedTools(
203
+ callerExclusions: readonly string[] | undefined,
204
+ configDisabled: readonly string[] = [],
205
+ ): string[] {
206
+ const base = callerExclusions && callerExclusions.length > 0
207
+ ? callerExclusions
208
+ : DEFAULT_EXCLUDED_TOOLS;
209
+ return [...new Set([...base, ...configDisabled])];
210
+ }
@@ -120,7 +120,12 @@ export interface IKnowledgeStore extends IService {
120
120
  /** Extended operations */
121
121
  addDocuments(docs: StoreDocument[]): Promise<void>;
122
122
  deleteByUrl(url: string): Promise<void>;
123
- deleteByUrlAndType(url: string, ingestionType: string): Promise<void>;
123
+ /**
124
+ * @param olderThan When set, only rows with `timestamp < olderThan` are removed —
125
+ * lets a refresh insert its replacement first and prune the superseded rows
126
+ * afterwards, so a failed insert can never leave the URL with nothing.
127
+ */
128
+ deleteByUrlAndType(url: string, ingestionType: string, olderThan?: number): Promise<void>;
124
129
  isStoreClosed?(): boolean;
125
130
 
126
131
  /**
@@ -18,7 +18,7 @@ export interface IStateManager extends IService {
18
18
  getBrowserServer(): Promise<{ port: number; pid: number; schedulerId?: string; authSecret?: string } | null>;
19
19
  setBrowserServer(port: number, pid: number, schedulerId?: string, authSecret?: string): Promise<void>;
20
20
  clearBrowserServer(): Promise<void>;
21
- getEmbeddingServer(): Promise<{ port: number; pid: number; startTime?: number; serverId: string; model?: string } | null>;
21
+ getEmbeddingServer(): Promise<{ port: number; pid: number; startTime?: number; serverId: string; model?: string; authSecret?: string } | null>;
22
22
  clearEmbeddingServer(): Promise<void>;
23
23
  isPidAlive(pid: number, expectedSchedulerId?: string, skipLock?: boolean): Promise<boolean>;
24
24
  acquireGpuLock(sessionId?: string, timeoutMs?: number): Promise<boolean>;
@@ -219,6 +219,13 @@ export function constructMinimalModel(provider: string, modelId: string, _apiKey
219
219
  baseUrl: '', // Provider-specific base URLs are handled by pi-ai internal registry
220
220
  reasoning: false,
221
221
  input: ['text'],
222
+ // No catalog is available on this path, so there is no price data and no real
223
+ // context/output ceiling to read. The zeros below are "unknown", NOT "free":
224
+ // cost reporting for this model will read $0.00 (see warnIfUnpriced in
225
+ // utils/llm-usage.ts), and the window figures are conservative guesses, not the
226
+ // model's true limits — so docs/CONFIGURATION.md's "clamped to the model's real
227
+ // ceiling" guarantee does not hold here. Anyone relying on accurate cost or the
228
+ // full context window should give pi a credential so it can fetch the real catalog.
222
229
  cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
223
230
  contextWindow: 128_000,
224
231
  maxTokens: 32_768,
@@ -253,7 +260,7 @@ export function resolveModel(registry: ModelRegistry, modelSpec?: string, provid
253
260
  return constructMinimalModel(prov, modelId, apiKey);
254
261
  }
255
262
 
256
- throw new Error(`Model "${modelSpec}" not found in pi's configured model registry. Check ${path.join(getAgentDir(), 'models.json')}.`);
263
+ throw new Error(buildModelNotFoundMessage(modelSpec));
257
264
  }
258
265
 
259
266
  // Validate format: must contain a slash OR be found as a bare model ID.
@@ -306,11 +313,41 @@ export function resolveModel(registry: ModelRegistry, modelSpec?: string, provid
306
313
  return constructMinimalModel(provider, modelSpec, apiKey);
307
314
  }
308
315
 
309
- throw new Error(
310
- `No LLM model available. Please configure your model registry (${path.join(getAgentDir(), 'models.json')}) or provide an explicit apiKey.`,
316
+ throw new Error(buildNoModelAvailableMessage());
317
+ }
318
+
319
+ /**
320
+ * Credential-first guidance for model-resolution failures.
321
+ *
322
+ * These messages used to say only "Check ~/.pi/agent/models.json", which is the
323
+ * WRONG remedy for a provider pi ships built in: the fix there is a credential,
324
+ * after which pi fetches the provider's catalog (with real pricing and context
325
+ * limits) on its own. Following the old advice, users hand-wrote `models[]`
326
+ * entries — and because such an entry REPLACES pi's catalog entry wholesale
327
+ * rather than merging into it, that silently zeroed out pricing and capped
328
+ * context/output windows. Lead with the credential path; scope models.json to
329
+ * genuinely custom providers; name `modelOverrides` for single-field tweaks.
330
+ */
331
+ function credentialGuidance(): string {
332
+ const agentDir = getAgentDir();
333
+ return (
334
+ ` • Provider pi supports built-in (openrouter, anthropic, openai, …): add a credential — run \`pi\` and use /login, ` +
335
+ `set the provider's API-key environment variable, or add it to ${path.join(agentDir, 'auth.json')}. ` +
336
+ `pi then fetches that provider's model catalog automatically.\n` +
337
+ ` • Custom or self-hosted provider (Ollama, vLLM, a proxy): declare it in ${path.join(agentDir, 'models.json')}.\n` +
338
+ ` Note: a models.json entry REPLACES pi's catalog entry for the same model id, discarding its pricing and\n` +
339
+ ` context/output limits. To adjust a single field of a built-in model, use "modelOverrides" instead — it merges.`
311
340
  );
312
341
  }
313
342
 
343
+ export function buildModelNotFoundMessage(modelSpec: string): string {
344
+ return `Model "${modelSpec}" not found in pi's model registry.\n${credentialGuidance()}`;
345
+ }
346
+
347
+ export function buildNoModelAvailableMessage(): string {
348
+ return `No LLM model available.\n${credentialGuidance()}\n • Or pass an explicit apiKey.`;
349
+ }
350
+
314
351
  /**
315
352
  * Read the user's provider order from ~/.pi/agent/models.json.
316
353
  * Returns provider keys in document (insertion) order, or [] when the file is
@@ -9,7 +9,7 @@ import { type Model } from '@earendil-works/pi-ai';
9
9
  import { type ModelRegistry } from '@earendil-works/pi-coding-agent';
10
10
  import { getConfig } from '../../config.ts';
11
11
  import { logger } from '../../logger.ts';
12
- import { pickPreferredAvailable, readModelsJsonProviderOrder, safeGetAll, safeGetAvailable } from './model-registry-factory.ts';
12
+ import { buildNoModelAvailableMessage, pickPreferredAvailable, readModelsJsonProviderOrder, safeGetAll, safeGetAvailable } from './model-registry-factory.ts';
13
13
 
14
14
  /**
15
15
  * Resolve the research model with standardized priority:
@@ -87,5 +87,5 @@ export function resolveResearchModel(options: {
87
87
  return pickPreferredAvailable(anyModel, readModelsJsonProviderOrder()) as Model<any>;
88
88
  }
89
89
 
90
- throw new Error('No LLM model available for research. Please configure your model registry (~/.pi/agent/models.json).');
90
+ throw new Error(buildNoModelAvailableMessage());
91
91
  }