@lincoln504/pi-research 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/cli.mjs +1404 -600
- package/dist/thread-worker.mjs +40 -11
- package/docs/AGENT-SKILL.md +11 -4
- package/docs/ARCHITECTURE.md +19 -9
- package/docs/CHANGELOG.md +107 -1
- package/docs/CONFIGURATION.md +10 -8
- package/docs/KNOWLEDGE-STORE.md +5 -4
- package/docs/PI-EXTENSION.md +1 -1
- package/docs/SDK.md +63 -5
- package/docs/THIRD-PARTY-NOTICES.md +1 -1
- package/package.json +5 -4
- package/scripts/build.cjs +23 -5
- package/scripts/cleanup.cjs +50 -10
- package/scripts/prepare.cjs +15 -5
- package/scripts/setup.cjs +24 -4
- package/skills/pi-research/README.md +9 -2
- package/skills/pi-research/SKILL.md +8 -5
- package/skills/pi-research/references/configuration.md +7 -3
- package/skills/pi-research/scripts/run.mjs +24 -2
- package/src/cli.ts +161 -21
- package/src/config.ts +56 -15
- package/src/constants.ts +31 -0
- package/src/core/interfaces/knowledge-interfaces.ts +6 -1
- package/src/core/interfaces/state-manager-interfaces.ts +1 -1
- package/src/core/llm/model-registry-factory.ts +40 -3
- package/src/core/llm/research-model-resolver.ts +2 -2
- package/src/core/pi-version.ts +136 -0
- package/src/core/service-registry.ts +117 -8
- package/src/index.ts +11 -18
- package/src/infrastructure/browser/browser-client.ts +16 -4
- package/src/infrastructure/browser/browser-error-utils.ts +12 -1
- package/src/infrastructure/browser/browser-server.ts +17 -5
- package/src/infrastructure/browser/browser-task-scheduler.ts +101 -23
- package/src/infrastructure/browser/cleanup-utils.ts +45 -7
- package/src/infrastructure/browser/client-agent.ts +4 -3
- package/src/infrastructure/browser/priority-task-queue.ts +10 -3
- package/src/infrastructure/browser/scheduler-factory.ts +26 -1
- package/src/infrastructure/browser/task-execution-service.ts +92 -28
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
- package/src/infrastructure/browser/thread-worker.mjs +40 -11
- package/src/infrastructure/browser/worker-pool-manager.ts +11 -3
- package/src/infrastructure/embedding/embedding-client.ts +22 -8
- package/src/infrastructure/embedding/embedding-factory.ts +28 -11
- package/src/infrastructure/embedding/embedding-server.ts +65 -4
- package/src/infrastructure/file-lock-service.ts +223 -26
- package/src/infrastructure/process-lifecycle-service.ts +108 -31
- package/src/infrastructure/state/state-backup-manager.ts +5 -7
- package/src/infrastructure/state/state-manager-service.ts +1 -1
- package/src/infrastructure/state/state-manager.ts +19 -20
- package/src/infrastructure/types/state-types.ts +9 -0
- package/src/knowledge/embedder-init.ts +1 -1
- package/src/knowledge/embedder.ts +52 -3
- package/src/knowledge/index.ts +1 -1
- package/src/knowledge/store.ts +35 -9
- package/src/knowledge/webgpu-viability.ts +9 -4
- package/src/knowledge/writer-queue.ts +88 -46
- package/src/orchestration/deep-research-orchestrator.ts +13 -2
- package/src/orchestration/quick-research-orchestrator.ts +2 -2
- package/src/orchestration/research-orchestration-service.ts +2 -4
- package/src/orchestration/research-synthesis-service.ts +34 -92
- package/src/security/cisa-kev.ts +10 -3
- package/src/security/github-advisories.ts +18 -7
- package/src/security/nvd-client.ts +17 -6
- package/src/security/osv-client.ts +14 -4
- package/src/skill-install/skill-installer.ts +135 -9
- package/src/stackexchange/rest-client.ts +6 -2
- package/src/tools/research-knowledge-search.ts +23 -2
- package/src/tools/search.ts +6 -1
- package/src/utils/atomic-replace.ts +101 -0
- package/src/utils/circuit-breaker.ts +20 -0
- package/src/utils/citation-utils.ts +106 -8
- package/src/utils/http-body.ts +176 -0
- package/src/utils/llm-usage.ts +37 -0
- package/src/utils/metrics-summary.ts +25 -2
- package/src/utils/research-export.ts +9 -2
- package/src/web-research/browser-search.ts +50 -4
- package/src/web-research/retry-utils.ts +13 -1
- package/src/web-research/search.ts +20 -5
- package/src/web-research/types.ts +17 -4
- package/src/web-research/web-scraper.ts +7 -50
- package/src/youtube/transcript-client.ts +5 -1
package/src/cli.ts
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
import * as fs from 'node:fs';
|
|
25
|
+
import * as os from 'node:os';
|
|
25
26
|
import * as path from 'node:path';
|
|
26
27
|
import { fileURLToPath } from 'node:url';
|
|
27
28
|
|
|
@@ -84,8 +85,47 @@ export const EXIT = {
|
|
|
84
85
|
* re-running a broken command.
|
|
85
86
|
*/
|
|
86
87
|
TEMPFAIL: 75,
|
|
88
|
+
/**
|
|
89
|
+
* The run was CANCELLED via a programmatic abort (the caller's AbortSignal),
|
|
90
|
+
* with no OS signal involved. 130 — 128+SIGINT — is the conventional
|
|
91
|
+
* "interrupted" code and the natural default when there is no signal number to
|
|
92
|
+
* derive one from.
|
|
93
|
+
*
|
|
94
|
+
* A cancellation caused by an actual SIGNAL does NOT use this constant: see
|
|
95
|
+
* exitCodeForSignal below, which reports the true 128+N.
|
|
96
|
+
*
|
|
97
|
+
* Distinct from SOFTWARE for the same reason TEMPFAIL is: a deliberate stop is
|
|
98
|
+
* not a fault. Reported as SOFTWARE (70), a cancel matched the skill contract's
|
|
99
|
+
* "runtime error — suggest one retry for transient failures", so an agent
|
|
100
|
+
* relaying the result would offer to re-run the very research the user had just
|
|
101
|
+
* interrupted, and a --json consumer could not tell a cancel from a crash.
|
|
102
|
+
*/
|
|
103
|
+
CANCELLED: 130,
|
|
87
104
|
} as const;
|
|
88
105
|
|
|
106
|
+
/**
|
|
107
|
+
* Exit code for a run stopped by `sig`, following the POSIX 128+N convention that
|
|
108
|
+
* a shell uses when reporting a signal-terminated child (SIGINT → 130,
|
|
109
|
+
* SIGTERM → 143, SIGHUP → 129, SIGQUIT → 131).
|
|
110
|
+
*
|
|
111
|
+
* Why derive it per-signal instead of returning a single "cancelled" code: this
|
|
112
|
+
* process INSTALLS handlers for those signals, so it is never actually killed by
|
|
113
|
+
* them — it catches them and exits deliberately. If it exited a flat 130, the code
|
|
114
|
+
* a wrapper observes would depend on whether our handler won the race against a
|
|
115
|
+
* force-kill: handler wins → 130, handler loses → the shell's 143. Deriving 128+N
|
|
116
|
+
* makes the handler transparent, so the same user action reports the same number
|
|
117
|
+
* either way.
|
|
118
|
+
*
|
|
119
|
+
* Every value lands in 129..143, safely inside the 0-255 range a POSIX wait status
|
|
120
|
+
* preserves — nothing here can be truncated to a false 0. Windows does not
|
|
121
|
+
* implement the 128+N convention at all (its exit codes are 32-bit and carry no
|
|
122
|
+
* signal semantics), so there these are simply our documented contract values.
|
|
123
|
+
*/
|
|
124
|
+
export function exitCodeForSignal(sig: NodeJS.Signals): number {
|
|
125
|
+
const signum = (os.constants.signals as Record<string, number | undefined>)[sig];
|
|
126
|
+
return typeof signum === 'number' ? 128 + signum : EXIT.CANCELLED;
|
|
127
|
+
}
|
|
128
|
+
|
|
89
129
|
// ---------------------------------------------------------------------------
|
|
90
130
|
// Output helpers
|
|
91
131
|
// ---------------------------------------------------------------------------
|
|
@@ -137,9 +177,19 @@ function makeProgressObserver(): HeadlessObserverOptions {
|
|
|
137
177
|
case 'evaluation_decision':
|
|
138
178
|
toStderr(` • evaluator: ${data?.action}\n`);
|
|
139
179
|
break;
|
|
140
|
-
case 'tokens_consumed':
|
|
141
|
-
|
|
180
|
+
case 'tokens_consumed': {
|
|
181
|
+
// A truthiness check on cost used to gate this whole line, so a model with
|
|
182
|
+
// an all-zero price table suppressed the TOKEN count as well — the run
|
|
183
|
+
// looked like it consumed nothing. Tokens stand on their own; cost is
|
|
184
|
+
// appended whenever it is a number, including 0.
|
|
185
|
+
const consumed = Number(data?.tokens ?? 0);
|
|
186
|
+
const costVal = typeof data?.cost === 'number' ? data.cost : undefined;
|
|
187
|
+
if (consumed > 0 || costVal !== undefined) {
|
|
188
|
+
const costPart = costVal === undefined ? '' : ` ($${costVal.toFixed(4)})`;
|
|
189
|
+
toStderr(` • tokens: ${consumed}${costPart}\n`);
|
|
190
|
+
}
|
|
142
191
|
break;
|
|
192
|
+
}
|
|
143
193
|
case 'complete':
|
|
144
194
|
toStderr(` • complete (${data?.result?.length ?? 0} chars)\n`);
|
|
145
195
|
break;
|
|
@@ -180,20 +230,33 @@ export function bridgeConfigEnv(explicitConfigPath?: string): { path: string; lo
|
|
|
180
230
|
// iteration and the "don't clobber the real environment" guard would then
|
|
181
231
|
// block the overlay's value — cli.env could never override config.env.
|
|
182
232
|
const merged: Record<string, string> = {};
|
|
233
|
+
// Keys whose winning value came from an EXPLICIT --config file (not the
|
|
234
|
+
// default base, not the cli.env overlay). Project-scoped keys are only
|
|
235
|
+
// bridged for these — see below.
|
|
236
|
+
const fromExplicitBase = new Set<string>();
|
|
183
237
|
for (const filePath of [basePath, cliOverlayPath]) {
|
|
184
238
|
try {
|
|
185
239
|
if (!fs.existsSync(filePath)) continue;
|
|
186
|
-
|
|
240
|
+
const parsed = parseDotEnv(fs.readFileSync(filePath, 'utf-8'));
|
|
241
|
+
Object.assign(merged, parsed);
|
|
242
|
+
for (const k of Object.keys(parsed)) {
|
|
243
|
+
if (explicitConfigPath !== undefined && filePath === basePath) fromExplicitBase.add(k);
|
|
244
|
+
else fromExplicitBase.delete(k);
|
|
245
|
+
}
|
|
187
246
|
loaded = true;
|
|
188
247
|
} catch {
|
|
189
248
|
// ignore unreadable file; fall through to the next
|
|
190
249
|
}
|
|
191
250
|
}
|
|
192
251
|
for (const [k, v] of Object.entries(merged)) {
|
|
193
|
-
// Skip per-directory (project-scoped) keys: promoting them into
|
|
194
|
-
// config.env out-rank the per-directory registry, defeating a
|
|
195
|
-
// still applied as a lower-precedence file layer by
|
|
196
|
-
|
|
252
|
+
// Skip per-directory (project-scoped) keys from the AMBIENT files: promoting them into
|
|
253
|
+
// process.env would make config.env out-rank the per-directory registry, defeating a
|
|
254
|
+
// per-cwd override. They are still applied as a lower-precedence file layer by
|
|
255
|
+
// getConfig()/loadEnvFiles(). An EXPLICIT --config file is different: getConfig never
|
|
256
|
+
// reads that path, so skipping its keys silently dropped them (a depth or
|
|
257
|
+
// knowledge-mode set in `--config ci.env` simply did not apply) — and a file the user
|
|
258
|
+
// named on this exact invocation outranking the per-directory registry is the intent.
|
|
259
|
+
if (isProjectScopedKey(k) && !fromExplicitBase.has(k)) continue;
|
|
197
260
|
if (process.env[k] === undefined && (k.startsWith('PI_RESEARCH_') || k === 'STACKEXCHANGE_API_KEY' || k === 'GITHUB_TOKEN' || k === 'NVD_API_KEY')) {
|
|
198
261
|
process.env[k] = v;
|
|
199
262
|
}
|
|
@@ -853,9 +916,25 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
|
|
|
853
916
|
// machine as a setup problem would send the user to reconfigure a working install.
|
|
854
917
|
const isCapacity = err instanceof Error && err.name === 'ResearchRunCapacityError';
|
|
855
918
|
|
|
919
|
+
// Cancellation. Primary signal is the latch set by the SIGINT/SIGTERM handler —
|
|
920
|
+
// an exact fact, unlike the message, which varies by whichever subsystem noticed
|
|
921
|
+
// the abort first ('Aborted' from the search layer, 'Research aborted' /
|
|
922
|
+
// 'Research cancelled' from the orchestrator). The anchored message signatures
|
|
923
|
+
// only cover a programmatic abort where no signal arrived. Deliberately NOT
|
|
924
|
+
// matched: a bare `err.name === 'AbortError'` — internal timeout aborts
|
|
925
|
+
// (AbortSignal-driven retry/scrape deadlines) throw that name too, and an
|
|
926
|
+
// escaped one would report "cancelled — do not re-run" for a run nobody
|
|
927
|
+
// stopped. In the CLI every real cancellation sets the latch or carries these
|
|
928
|
+
// exact signatures. Checked before everything below: an abort can unwind
|
|
929
|
+
// through a provider call and surface wording that the config/rate-limit
|
|
930
|
+
// heuristics would otherwise claim.
|
|
931
|
+
const isCancellation =
|
|
932
|
+
cancellationRequested ||
|
|
933
|
+
(err instanceof Error && /^(aborted|research (aborted|cancelled))$/i.test(msg));
|
|
934
|
+
|
|
856
935
|
// Auth / model / config-shaped errors → CONFIG exit + config block.
|
|
857
936
|
const isConfigError =
|
|
858
|
-
!isTaggedResearchStop && !isCapacity && (
|
|
937
|
+
!isTaggedResearchStop && !isCapacity && !isCancellation && (
|
|
859
938
|
lower.includes('no llm model available') ||
|
|
860
939
|
lower.includes('not found in pi') ||
|
|
861
940
|
lower.includes('no api key') ||
|
|
@@ -869,20 +948,27 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
|
|
|
869
948
|
// "api key" in its body (e.g. "rate limit exceeded for this api key"), and
|
|
870
949
|
// classifying that as a setup problem sends the user to reconfigure a
|
|
871
950
|
// perfectly working key instead of waiting out the limit.
|
|
872
|
-
const isRateLimit = lower.includes('429') || lower.includes('rate limit') || lower.includes('too many requests');
|
|
873
|
-
const exitCode =
|
|
874
|
-
? EXIT.
|
|
875
|
-
:
|
|
876
|
-
? EXIT.
|
|
877
|
-
:
|
|
951
|
+
const isRateLimit = !isCancellation && (lower.includes('429') || lower.includes('rate limit') || lower.includes('too many requests'));
|
|
952
|
+
const exitCode = isCancellation
|
|
953
|
+
? (cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED)
|
|
954
|
+
: isCapacity
|
|
955
|
+
? EXIT.TEMPFAIL
|
|
956
|
+
: isConfigError && !isRateLimit
|
|
957
|
+
? EXIT.CONFIG
|
|
958
|
+
: EXIT.SOFTWARE;
|
|
878
959
|
|
|
879
960
|
// --json: emit a structured error on stdout so a machine consumer never has to
|
|
880
961
|
// special-case plain-text stderr. Mirrors the { ok: true, ... } success shape.
|
|
881
962
|
if (json) {
|
|
882
963
|
const payload: Record<string, unknown> = { ok: false, error: msg, exitCode };
|
|
883
964
|
// Machine-readable "this will succeed later" flag, so a consumer can back off
|
|
884
|
-
// and retry without string-matching the human message.
|
|
885
|
-
|
|
965
|
+
// and retry without string-matching the human message. Gated on cancellation:
|
|
966
|
+
// a Ctrl-C landing in the same window a queue-wait expires must never emit
|
|
967
|
+
// `cancelled: true` alongside `retryable: true` — a consumer honouring the
|
|
968
|
+
// latter would re-run the research the user just stopped.
|
|
969
|
+
if ((isCapacity || isRateLimit) && !isCancellation) payload['retryable'] = true;
|
|
970
|
+
// A cancel is emphatically NOT retryable-by-default: the user asked to stop.
|
|
971
|
+
if (isCancellation) payload['cancelled'] = true;
|
|
886
972
|
if (process.env['PI_RESEARCH_DEBUG'] === 'true' && err instanceof Error && err.stack) {
|
|
887
973
|
payload['stack'] = err.stack;
|
|
888
974
|
}
|
|
@@ -890,6 +976,13 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
|
|
|
890
976
|
return exitCode;
|
|
891
977
|
}
|
|
892
978
|
|
|
979
|
+
// Cancellation is a completed intention, not a failure — never a stack trace,
|
|
980
|
+
// and never the word "failed".
|
|
981
|
+
if (isCancellation) {
|
|
982
|
+
toStderr(`\npi-research ${what} cancelled.\n`);
|
|
983
|
+
return exitCode;
|
|
984
|
+
}
|
|
985
|
+
|
|
893
986
|
// Capacity exhaustion is a healthy-machine "busy" state — report it as such and
|
|
894
987
|
// never with a stack trace, which would read as a crash for a routine condition.
|
|
895
988
|
if (isCapacity) {
|
|
@@ -1014,10 +1107,14 @@ CONFIGURE
|
|
|
1014
1107
|
Run \`${BINARY_NAME} status\` to see exactly what is detected.
|
|
1015
1108
|
|
|
1016
1109
|
EXIT CODES
|
|
1017
|
-
0
|
|
1018
|
-
64
|
|
1019
|
-
|
|
1020
|
-
|
|
1110
|
+
0 success
|
|
1111
|
+
64 bad arguments
|
|
1112
|
+
75 at capacity — every concurrent run slot is busy; nothing is broken, retry later
|
|
1113
|
+
78 not configured — message prints the exact locations to fix
|
|
1114
|
+
(on \`knowledge\`, also returned when the store is deliberately disabled)
|
|
1115
|
+
130 cancelled — Ctrl-C; nothing failed, do not auto-retry
|
|
1116
|
+
(other stop signals report POSIX 128+N: 143 SIGTERM, 129 SIGHUP, 131 SIGQUIT)
|
|
1117
|
+
70 runtime error (network, provider, internal)
|
|
1021
1118
|
`;
|
|
1022
1119
|
}
|
|
1023
1120
|
|
|
@@ -1257,6 +1354,32 @@ const flushAndExit = (code: number): void => {
|
|
|
1257
1354
|
// outlives that one run.
|
|
1258
1355
|
let activeResearchAbortController: AbortController | null = null;
|
|
1259
1356
|
|
|
1357
|
+
/**
|
|
1358
|
+
* Set once SIGINT/SIGTERM has been handled. Read by reportError so a cancellation
|
|
1359
|
+
* is classified by the fact that we were signalled, not by pattern-matching the
|
|
1360
|
+
* message an aborted subsystem happened to throw.
|
|
1361
|
+
*/
|
|
1362
|
+
let cancellationRequested = false;
|
|
1363
|
+
|
|
1364
|
+
/**
|
|
1365
|
+
* WHICH signal fired — so a cancellation that unwinds through reportError
|
|
1366
|
+
* reports the same 128+N the signal path itself would (SIGTERM → 143, not a
|
|
1367
|
+
* flat 130). reportError races onSignal's own flushAndExit; latching the name
|
|
1368
|
+
* makes both racers agree on the number, and keeps the `exitCode` field in
|
|
1369
|
+
* `--json` from contradicting the actual exit status.
|
|
1370
|
+
*/
|
|
1371
|
+
let cancellationSignal: NodeJS.Signals | null = null;
|
|
1372
|
+
|
|
1373
|
+
/** Test seams for the module-level cancellation latch (set only by onSignal). */
|
|
1374
|
+
export function _resetCancellationLatchForTests(): void {
|
|
1375
|
+
cancellationRequested = false;
|
|
1376
|
+
cancellationSignal = null;
|
|
1377
|
+
}
|
|
1378
|
+
export function _markCancellationForTests(sig?: NodeJS.Signals): void {
|
|
1379
|
+
cancellationRequested = true;
|
|
1380
|
+
cancellationSignal = sig ?? null;
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1260
1383
|
async function main(argv: string[]): Promise<number> {
|
|
1261
1384
|
// Register signal handlers so SIGINT/SIGTERM always trigger SDK shutdown
|
|
1262
1385
|
// before the process exits. safeShutdown() is idempotent — safe to call
|
|
@@ -1265,6 +1388,8 @@ async function main(argv: string[]): Promise<number> {
|
|
|
1265
1388
|
const onSignal = (sig: string) => {
|
|
1266
1389
|
if (_signalCleanupDone) return;
|
|
1267
1390
|
_signalCleanupDone = true;
|
|
1391
|
+
cancellationRequested = true;
|
|
1392
|
+
cancellationSignal = sig as NodeJS.Signals;
|
|
1268
1393
|
toStderr(`\n[pi-research] ${sig} — cleaning up…\n`);
|
|
1269
1394
|
// Abort the in-flight research run FIRST (synchronously ahead of the shutdown
|
|
1270
1395
|
// teardown below) so the orchestrator's existing signal-aware cancellation path
|
|
@@ -1274,10 +1399,25 @@ async function main(argv: string[]): Promise<number> {
|
|
|
1274
1399
|
// Fire-and-forget: we can't await here, but safeShutdown swallows errors
|
|
1275
1400
|
// and the finally blocks in each command handler also call it. Drain via
|
|
1276
1401
|
// flushAndExit (not a bare process.exit) so buffered stdout isn't truncated.
|
|
1277
|
-
|
|
1402
|
+
// 128+N, not a flat CANCELLED: see exitCodeForSignal. Because we handle these
|
|
1403
|
+
// signals rather than dying from them, a fixed code would make the observed
|
|
1404
|
+
// exit status depend on whether this handler beat a force-kill.
|
|
1405
|
+
void safeShutdown().finally(() => flushAndExit(exitCodeForSignal(sig as NodeJS.Signals)));
|
|
1278
1406
|
};
|
|
1279
1407
|
process.once('SIGINT', () => onSignal('SIGINT'));
|
|
1280
1408
|
process.once('SIGTERM', () => onSignal('SIGTERM'));
|
|
1409
|
+
// SIGHUP (terminal closed) and, on Windows, SIGBREAK (Ctrl+Break). Without these
|
|
1410
|
+
// the run dies without cancelling: no abort of the in-flight research, no browser
|
|
1411
|
+
// or embedding-server teardown, and no 128+N exit code. On Windows it matters
|
|
1412
|
+
// most — SIGTERM is never delivered there (process.kill maps to TerminateProcess),
|
|
1413
|
+
// so Ctrl-C and Ctrl+Break are the only signals that reach this handler at all.
|
|
1414
|
+
// The SDK already registers both (see _addSignalHandlers); the CLI did not, which
|
|
1415
|
+
// left the SDK's handler to tear the container down underneath a live orchestrator
|
|
1416
|
+
// — exactly the race the CLI installs its own handlers to avoid.
|
|
1417
|
+
process.once('SIGHUP', () => onSignal('SIGHUP'));
|
|
1418
|
+
if (process.platform === 'win32') {
|
|
1419
|
+
process.once('SIGBREAK', () => onSignal('SIGBREAK'));
|
|
1420
|
+
}
|
|
1281
1421
|
|
|
1282
1422
|
let parsed: ParsedArgs;
|
|
1283
1423
|
try {
|
package/src/config.ts
CHANGED
|
@@ -14,6 +14,7 @@ import { getConfigDirName } from './utils/host-config.ts';
|
|
|
14
14
|
import { Type, type Static } from 'typebox';
|
|
15
15
|
import { Value } from 'typebox/value';
|
|
16
16
|
import { normalizeWorkspacePath } from './utils/text-utils.ts';
|
|
17
|
+
import { replaceFileSync } from './utils/atomic-replace.ts';
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Validates configuration schema using TypeBox.
|
|
@@ -501,7 +502,7 @@ export function describeKnowledgeStoreMode(
|
|
|
501
502
|
let registryHasKey = false;
|
|
502
503
|
try {
|
|
503
504
|
const registry = loadProjectSettingsRegistry();
|
|
504
|
-
registryHasKey = registry
|
|
505
|
+
registryHasKey = findRegistryEntry(registry, normalizeWorkspacePath(cwd))?.[KEY] !== undefined;
|
|
505
506
|
} catch { /* unreadable registry → not the origin */ }
|
|
506
507
|
if (registryHasKey) origin = 'this directory (project settings)';
|
|
507
508
|
else if (iface && fileHasKey(getInterfaceEnvFilePath(iface))) origin = `${iface}.env overlay`;
|
|
@@ -546,12 +547,48 @@ export function parseDotEnv(content: string): Record<string, string> {
|
|
|
546
547
|
* Load environment variables from global and local files.
|
|
547
548
|
* Order: Global File < Interface overlay < Legacy .env < Centralized Registry (WINS)
|
|
548
549
|
*/
|
|
550
|
+
/**
|
|
551
|
+
* Look up a directory's entry in the project-settings registry, tolerating case.
|
|
552
|
+
*
|
|
553
|
+
* On Windows and on macOS's default (case-insensitive) filesystem, `C:\\Users\\Bob\\Proj`
|
|
554
|
+
* and `c:\\users\\bob\\proj` are the SAME directory but different object keys, so a shell
|
|
555
|
+
* that reports a different casing than the one the settings were saved under made those
|
|
556
|
+
* settings silently vanish. normalizeWorkspacePath only resolves and strips a trailing
|
|
557
|
+
* separator; it does not fold case, and it must not start doing so — the key it produces
|
|
558
|
+
* is persisted, so folding it would orphan every entry already on disk.
|
|
559
|
+
*
|
|
560
|
+
* Resolving at READ time instead fixes the lookup without touching stored data: exact
|
|
561
|
+
* match first (the only path taken on Linux, and the fast path everywhere), then a
|
|
562
|
+
* case-insensitive scan as a fallback.
|
|
563
|
+
*/
|
|
564
|
+
function findRegistryKey(
|
|
565
|
+
registry: Record<string, Record<string, string>>,
|
|
566
|
+
normalizedCwd: string,
|
|
567
|
+
): string | undefined {
|
|
568
|
+
if (registry[normalizedCwd] !== undefined) return normalizedCwd;
|
|
569
|
+
// Only case-insensitive filesystems can produce this mismatch; skip the scan elsewhere.
|
|
570
|
+
if (process.platform !== 'win32' && process.platform !== 'darwin') return undefined;
|
|
571
|
+
const lowered = normalizedCwd.toLowerCase();
|
|
572
|
+
for (const key of Object.keys(registry)) {
|
|
573
|
+
if (key.toLowerCase() === lowered) return key;
|
|
574
|
+
}
|
|
575
|
+
return undefined;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
function findRegistryEntry(
|
|
579
|
+
registry: Record<string, Record<string, string>>,
|
|
580
|
+
normalizedCwd: string,
|
|
581
|
+
): Record<string, string> | undefined {
|
|
582
|
+
const key = findRegistryKey(registry, normalizedCwd);
|
|
583
|
+
return key === undefined ? undefined : registry[key];
|
|
584
|
+
}
|
|
585
|
+
|
|
549
586
|
function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, string> {
|
|
550
587
|
const merged: Record<string, string> = {};
|
|
551
588
|
const globalPath = getGlobalEnvFilePath();
|
|
552
589
|
const registry = loadProjectSettingsRegistry();
|
|
553
590
|
const normalizedCwd = normalizeWorkspacePath(cwd);
|
|
554
|
-
const homeRegistry = registry
|
|
591
|
+
const homeRegistry = findRegistryEntry(registry, normalizeWorkspacePath(os.homedir()));
|
|
555
592
|
|
|
556
593
|
// 1. ONE-TIME MIGRATION: Populate config.env from HOME registry if missing
|
|
557
594
|
try {
|
|
@@ -613,15 +650,17 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
|
|
|
613
650
|
Object.entries(legacyEnv).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
|
|
614
651
|
);
|
|
615
652
|
const existingLocal = Object.fromEntries(
|
|
616
|
-
Object.entries(registry
|
|
653
|
+
Object.entries(findRegistryEntry(registry, normalizedCwd) ?? {}).filter(([k]) => LOCAL_SCOPE_KEYS.has(k))
|
|
617
654
|
);
|
|
618
655
|
if (Object.keys(legacyLocal).length > 0 && JSON.stringify(existingLocal) !== JSON.stringify(legacyLocal)) {
|
|
619
656
|
logger.info(`[config] Migrating legacy .pi-research.env settings from ${cwd} to central registry...`);
|
|
620
657
|
// Persist under the registry lock, reading the latest on-disk state so a concurrent
|
|
621
658
|
// write isn't lost; also reflect the merge in the in-memory `registry` used just below.
|
|
622
|
-
|
|
659
|
+
const mergeKey = findRegistryKey(registry, normalizedCwd) ?? normalizedCwd;
|
|
660
|
+
registry[mergeKey] = { ...registry[mergeKey], ...legacyLocal };
|
|
623
661
|
updateProjectSettingsRegistry((disk) => {
|
|
624
|
-
|
|
662
|
+
const diskKey = findRegistryKey(disk, normalizedCwd) ?? normalizedCwd;
|
|
663
|
+
disk[diskKey] = { ...disk[diskKey], ...legacyLocal };
|
|
625
664
|
});
|
|
626
665
|
}
|
|
627
666
|
} catch (err) {
|
|
@@ -633,8 +672,9 @@ function loadEnvFiles(cwd: string, iface?: ConfigInterface): Record<string, stri
|
|
|
633
672
|
// LOCAL_SCOPE_KEYS are applied (and may override), never user-scoped keys — those belong to
|
|
634
673
|
// config.env. Any non-local key present here is stale pollution (e.g. a pre-fix legacy
|
|
635
674
|
// migration) and is ignored rather than allowed to override the user's global settings.
|
|
636
|
-
|
|
637
|
-
|
|
675
|
+
const cwdEntry = findRegistryEntry(registry, normalizedCwd);
|
|
676
|
+
if (cwdEntry) {
|
|
677
|
+
for (const [key, val] of Object.entries(cwdEntry)) {
|
|
638
678
|
if (LOCAL_SCOPE_KEYS.has(key)) {
|
|
639
679
|
// Don't interpolate values in warnings — a config key could carry a secret and
|
|
640
680
|
// redactSecrets won't catch short/non-standard tokens. Key + winner is enough.
|
|
@@ -763,7 +803,11 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
|
|
|
763
803
|
// Read-merge-write under one lock so a concurrent local write to another directory
|
|
764
804
|
// isn't lost (the merge onto the existing entry happens against the latest on-disk state).
|
|
765
805
|
updateProjectSettingsRegistry((registry) => {
|
|
766
|
-
|
|
806
|
+
// Merge into the entry that already represents THIS directory, even if it was
|
|
807
|
+
// written with different casing on a case-insensitive filesystem. Writing a
|
|
808
|
+
// second key would leave the earlier settings stranded and unreachable.
|
|
809
|
+
const key = findRegistryKey(registry, normalizedCwd) ?? normalizedCwd;
|
|
810
|
+
registry[key] = { ...(registry[key] ?? {}), ...newValues };
|
|
767
811
|
});
|
|
768
812
|
|
|
769
813
|
logger.debug(`[config] Saved project settings for ${normalizedCwd} to central registry.`);
|
|
@@ -870,15 +914,12 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
|
|
|
870
914
|
const tmpPath = `${p}.tmp.${process.pid}.${Date.now()}`;
|
|
871
915
|
fs.writeFileSync(tmpPath, outLines.join('\n'), { encoding: 'utf-8', mode: 0o600 });
|
|
872
916
|
try {
|
|
873
|
-
|
|
874
|
-
} catch (renameErr) {
|
|
875
|
-
if (process.platform === 'win32') {
|
|
876
|
-
fs.copyFileSync(tmpPath, p);
|
|
917
|
+
if (replaceFileSync(tmpPath, p) === 'copied') {
|
|
877
918
|
try { fs.unlinkSync(tmpPath); } catch { /* best effort */ }
|
|
878
|
-
} else {
|
|
879
|
-
try { fs.unlinkSync(tmpPath); } catch { /* best effort */ }
|
|
880
|
-
throw renameErr;
|
|
881
919
|
}
|
|
920
|
+
} catch (renameErr) {
|
|
921
|
+
try { fs.unlinkSync(tmpPath); } catch { /* best effort */ }
|
|
922
|
+
throw renameErr;
|
|
882
923
|
}
|
|
883
924
|
// Tighten regardless of how the file came to exist (fresh rename, Windows copy over
|
|
884
925
|
// an existing file, or a pre-hardening 0644 file that the rename replaced-in-place).
|
package/src/constants.ts
CHANGED
|
@@ -177,3 +177,34 @@ export const RESEARCHER_LAUNCH_DELAY_MS = 1500;
|
|
|
177
177
|
export const MAX_QUERIES_PER_RESEARCHER_LEVEL_1 = 10;
|
|
178
178
|
export const MAX_QUERIES_PER_RESEARCHER_LEVEL_2 = 15;
|
|
179
179
|
export const MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Tools excluded from every researcher unless the caller says otherwise.
|
|
183
|
+
*
|
|
184
|
+
* `grep` is developer-only: during WEB research a high-capability model otherwise
|
|
185
|
+
* spends turns searching the local filesystem instead of the internet.
|
|
186
|
+
*/
|
|
187
|
+
export const DEFAULT_EXCLUDED_TOOLS: readonly string[] = ['grep'];
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Resolve the effective researcher exclusion list.
|
|
191
|
+
*
|
|
192
|
+
* An EMPTY array is "no preference", not "exclude nothing". The two front-ends
|
|
193
|
+
* express no-preference differently — the pi extension always builds an array
|
|
194
|
+
* (`[...new Set([...])]`, often empty) while the CLI omits the key entirely — so a
|
|
195
|
+
* plain truthiness test made the same user input mean opposite things: grep was
|
|
196
|
+
* available in the extension and excluded on the CLI, regardless of depth.
|
|
197
|
+
*
|
|
198
|
+
* `configDisabled` (PI_RESEARCH_DISABLED_TOOLS) is strictly ADDITIVE. Folding it in
|
|
199
|
+
* as a replacement meant disabling any unrelated tool silently ENABLED grep, so a
|
|
200
|
+
* subtract-only setting could grant a capability.
|
|
201
|
+
*/
|
|
202
|
+
export function resolveExcludedTools(
|
|
203
|
+
callerExclusions: readonly string[] | undefined,
|
|
204
|
+
configDisabled: readonly string[] = [],
|
|
205
|
+
): string[] {
|
|
206
|
+
const base = callerExclusions && callerExclusions.length > 0
|
|
207
|
+
? callerExclusions
|
|
208
|
+
: DEFAULT_EXCLUDED_TOOLS;
|
|
209
|
+
return [...new Set([...base, ...configDisabled])];
|
|
210
|
+
}
|
|
@@ -120,7 +120,12 @@ export interface IKnowledgeStore extends IService {
|
|
|
120
120
|
/** Extended operations */
|
|
121
121
|
addDocuments(docs: StoreDocument[]): Promise<void>;
|
|
122
122
|
deleteByUrl(url: string): Promise<void>;
|
|
123
|
-
|
|
123
|
+
/**
|
|
124
|
+
* @param olderThan When set, only rows with `timestamp < olderThan` are removed —
|
|
125
|
+
* lets a refresh insert its replacement first and prune the superseded rows
|
|
126
|
+
* afterwards, so a failed insert can never leave the URL with nothing.
|
|
127
|
+
*/
|
|
128
|
+
deleteByUrlAndType(url: string, ingestionType: string, olderThan?: number): Promise<void>;
|
|
124
129
|
isStoreClosed?(): boolean;
|
|
125
130
|
|
|
126
131
|
/**
|
|
@@ -18,7 +18,7 @@ export interface IStateManager extends IService {
|
|
|
18
18
|
getBrowserServer(): Promise<{ port: number; pid: number; schedulerId?: string; authSecret?: string } | null>;
|
|
19
19
|
setBrowserServer(port: number, pid: number, schedulerId?: string, authSecret?: string): Promise<void>;
|
|
20
20
|
clearBrowserServer(): Promise<void>;
|
|
21
|
-
getEmbeddingServer(): Promise<{ port: number; pid: number; startTime?: number; serverId: string; model?: string } | null>;
|
|
21
|
+
getEmbeddingServer(): Promise<{ port: number; pid: number; startTime?: number; serverId: string; model?: string; authSecret?: string } | null>;
|
|
22
22
|
clearEmbeddingServer(): Promise<void>;
|
|
23
23
|
isPidAlive(pid: number, expectedSchedulerId?: string, skipLock?: boolean): Promise<boolean>;
|
|
24
24
|
acquireGpuLock(sessionId?: string, timeoutMs?: number): Promise<boolean>;
|
|
@@ -219,6 +219,13 @@ export function constructMinimalModel(provider: string, modelId: string, _apiKey
|
|
|
219
219
|
baseUrl: '', // Provider-specific base URLs are handled by pi-ai internal registry
|
|
220
220
|
reasoning: false,
|
|
221
221
|
input: ['text'],
|
|
222
|
+
// No catalog is available on this path, so there is no price data and no real
|
|
223
|
+
// context/output ceiling to read. The zeros below are "unknown", NOT "free":
|
|
224
|
+
// cost reporting for this model will read $0.00 (see warnIfUnpriced in
|
|
225
|
+
// utils/llm-usage.ts), and the window figures are conservative guesses, not the
|
|
226
|
+
// model's true limits — so docs/CONFIGURATION.md's "clamped to the model's real
|
|
227
|
+
// ceiling" guarantee does not hold here. Anyone relying on accurate cost or the
|
|
228
|
+
// full context window should give pi a credential so it can fetch the real catalog.
|
|
222
229
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
223
230
|
contextWindow: 128_000,
|
|
224
231
|
maxTokens: 32_768,
|
|
@@ -253,7 +260,7 @@ export function resolveModel(registry: ModelRegistry, modelSpec?: string, provid
|
|
|
253
260
|
return constructMinimalModel(prov, modelId, apiKey);
|
|
254
261
|
}
|
|
255
262
|
|
|
256
|
-
throw new Error(
|
|
263
|
+
throw new Error(buildModelNotFoundMessage(modelSpec));
|
|
257
264
|
}
|
|
258
265
|
|
|
259
266
|
// Validate format: must contain a slash OR be found as a bare model ID.
|
|
@@ -306,11 +313,41 @@ export function resolveModel(registry: ModelRegistry, modelSpec?: string, provid
|
|
|
306
313
|
return constructMinimalModel(provider, modelSpec, apiKey);
|
|
307
314
|
}
|
|
308
315
|
|
|
309
|
-
throw new Error(
|
|
310
|
-
|
|
316
|
+
throw new Error(buildNoModelAvailableMessage());
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Credential-first guidance for model-resolution failures.
|
|
321
|
+
*
|
|
322
|
+
* These messages used to say only "Check ~/.pi/agent/models.json", which is the
|
|
323
|
+
* WRONG remedy for a provider pi ships built in: the fix there is a credential,
|
|
324
|
+
* after which pi fetches the provider's catalog (with real pricing and context
|
|
325
|
+
* limits) on its own. Following the old advice, users hand-wrote `models[]`
|
|
326
|
+
* entries — and because such an entry REPLACES pi's catalog entry wholesale
|
|
327
|
+
* rather than merging into it, that silently zeroed out pricing and capped
|
|
328
|
+
* context/output windows. Lead with the credential path; scope models.json to
|
|
329
|
+
* genuinely custom providers; name `modelOverrides` for single-field tweaks.
|
|
330
|
+
*/
|
|
331
|
+
function credentialGuidance(): string {
|
|
332
|
+
const agentDir = getAgentDir();
|
|
333
|
+
return (
|
|
334
|
+
` • Provider pi supports built-in (openrouter, anthropic, openai, …): add a credential — run \`pi\` and use /login, ` +
|
|
335
|
+
`set the provider's API-key environment variable, or add it to ${path.join(agentDir, 'auth.json')}. ` +
|
|
336
|
+
`pi then fetches that provider's model catalog automatically.\n` +
|
|
337
|
+
` • Custom or self-hosted provider (Ollama, vLLM, a proxy): declare it in ${path.join(agentDir, 'models.json')}.\n` +
|
|
338
|
+
` Note: a models.json entry REPLACES pi's catalog entry for the same model id, discarding its pricing and\n` +
|
|
339
|
+
` context/output limits. To adjust a single field of a built-in model, use "modelOverrides" instead — it merges.`
|
|
311
340
|
);
|
|
312
341
|
}
|
|
313
342
|
|
|
343
|
+
export function buildModelNotFoundMessage(modelSpec: string): string {
|
|
344
|
+
return `Model "${modelSpec}" not found in pi's model registry.\n${credentialGuidance()}`;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
export function buildNoModelAvailableMessage(): string {
|
|
348
|
+
return `No LLM model available.\n${credentialGuidance()}\n • Or pass an explicit apiKey.`;
|
|
349
|
+
}
|
|
350
|
+
|
|
314
351
|
/**
|
|
315
352
|
* Read the user's provider order from ~/.pi/agent/models.json.
|
|
316
353
|
* Returns provider keys in document (insertion) order, or [] when the file is
|
|
@@ -9,7 +9,7 @@ import { type Model } from '@earendil-works/pi-ai';
|
|
|
9
9
|
import { type ModelRegistry } from '@earendil-works/pi-coding-agent';
|
|
10
10
|
import { getConfig } from '../../config.ts';
|
|
11
11
|
import { logger } from '../../logger.ts';
|
|
12
|
-
import { pickPreferredAvailable, readModelsJsonProviderOrder, safeGetAll, safeGetAvailable } from './model-registry-factory.ts';
|
|
12
|
+
import { buildNoModelAvailableMessage, pickPreferredAvailable, readModelsJsonProviderOrder, safeGetAll, safeGetAvailable } from './model-registry-factory.ts';
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
15
|
* Resolve the research model with standardized priority:
|
|
@@ -87,5 +87,5 @@ export function resolveResearchModel(options: {
|
|
|
87
87
|
return pickPreferredAvailable(anyModel, readModelsJsonProviderOrder()) as Model<any>;
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
throw new Error(
|
|
90
|
+
throw new Error(buildNoModelAvailableMessage());
|
|
91
91
|
}
|