@lincoln504/pi-research 1.3.10 → 1.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cli.mjs +264 -61
- package/dist/prompts/system-coordinator.md +1 -1
- package/dist/prompts/system-lead-evaluator.md +1 -1
- package/dist/thread-worker.mjs +31 -21
- package/docs/ARCHITECTURE.md +60 -22
- package/docs/CHANGELOG.md +103 -1
- package/docs/CONFIGURATION.md +6 -5
- package/package.json +18 -10
- package/scripts/cleanup.cjs +26 -5
- package/scripts/setup.cjs +25 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cli.ts +94 -13
- package/src/config.ts +6 -15
- package/src/constants.ts +13 -0
- package/src/core/service-registry.ts +19 -3
- package/src/infrastructure/browser/config.ts +36 -5
- package/src/infrastructure/browser/task-execution-service.ts +12 -2
- package/src/infrastructure/browser/thread-worker-browser.ts +34 -18
- package/src/infrastructure/browser/thread-worker.mjs +31 -21
- package/src/infrastructure/gpu-resource-service.ts +15 -1
- package/src/knowledge/embedder-init.ts +34 -2
- package/src/knowledge/store.ts +75 -6
- package/src/knowledge/writer-queue.ts +28 -0
- package/src/orchestration/research-orchestration-service.ts +35 -1
- package/src/orchestration/research-synthesis-service.ts +4 -2
- package/src/orchestration/researcher.ts +2 -13
- package/src/prompts/system-coordinator.md +1 -1
- package/src/prompts/system-lead-evaluator.md +1 -1
- package/src/sdk.ts +10 -1
- package/src/security/nvd-client.ts +55 -4
- package/src/security/osv-client.ts +13 -1
- package/src/tools/research-knowledge-search.ts +18 -1
- package/src/tools/research-tool-definition.ts +8 -1
- package/src/utils/citation-utils.ts +4 -2
- package/src/utils/log-utils.ts +16 -1
- package/src/utils/text-utils.ts +17 -3
- package/src/web-research/scraper-utils.ts +55 -14
package/src/cli.ts
CHANGED
|
@@ -44,6 +44,8 @@ import { exportResearchReport, appendExportMessage } from './utils/research-expo
|
|
|
44
44
|
import { getConfig, getGlobalConfigDir, getGlobalEnvFilePath, getInterfaceEnvFilePath, getStateDir, saveConfig, resetConfig, describeKnowledgeStoreMode, isProjectScopedKey, parseDotEnv } from './config.ts';
|
|
45
45
|
import { getAgentDir } from '@earendil-works/pi-coding-agent';
|
|
46
46
|
import { buildModelRegistry, safeGetAvailable } from './core/llm/model-registry-factory.ts';
|
|
47
|
+
import { RESEARCH_TOOL_NAMES } from './constants.ts';
|
|
48
|
+
import { isBrowserAvailable } from './infrastructure/browser/config.ts';
|
|
47
49
|
|
|
48
50
|
// ---------------------------------------------------------------------------
|
|
49
51
|
// Constants
|
|
@@ -234,7 +236,8 @@ export class ConfigFileError extends Error {}
|
|
|
234
236
|
* the real environment, so real shell exports always win.
|
|
235
237
|
*/
|
|
236
238
|
export function bridgeConfigEnv(explicitConfigPath?: string): { path: string; loaded: boolean } {
|
|
237
|
-
const
|
|
239
|
+
const globalBasePath = getGlobalEnvFilePath();
|
|
240
|
+
const basePath = explicitConfigPath ?? globalBasePath;
|
|
238
241
|
const cliOverlayPath = getInterfaceEnvFilePath('cli');
|
|
239
242
|
let loaded = false;
|
|
240
243
|
// Merge base then overlay into one map FIRST (overlay wins), and only then
|
|
@@ -252,13 +255,18 @@ export function bridgeConfigEnv(explicitConfigPath?: string): { path: string; lo
|
|
|
252
255
|
// the user passed on this exact invocation is the most specific intent, and the
|
|
253
256
|
// ambient cli.env overlay silently overriding it re-created the very failure
|
|
254
257
|
// this bridge exists to fix (a key set in `--config ci.env` simply not applying).
|
|
258
|
+
// The GLOBAL base always stays underneath: this bridge exists for env-only keys
|
|
259
|
+
// (PI_RESEARCH_API_KEY & co.) that getConfig never carries, so dropping the base
|
|
260
|
+
// here silently dropped credentials stored only in config.env — `--config extra.env`
|
|
261
|
+
// exit-78'd on a correctly configured machine while the help text promised
|
|
262
|
+
// "base config.env keys it does not set still apply".
|
|
255
263
|
const layers = explicitConfigPath !== undefined
|
|
256
|
-
? [cliOverlayPath,
|
|
257
|
-
: [
|
|
264
|
+
? [globalBasePath, cliOverlayPath, explicitConfigPath]
|
|
265
|
+
: [globalBasePath, cliOverlayPath];
|
|
258
266
|
for (const filePath of layers) {
|
|
259
267
|
// Only the EXPLICIT --config file fails loudly (see ConfigFileError above);
|
|
260
268
|
// the ambient layers keep their missing/unreadable tolerance unchanged.
|
|
261
|
-
const isExplicit = explicitConfigPath !== undefined && filePath ===
|
|
269
|
+
const isExplicit = explicitConfigPath !== undefined && filePath === explicitConfigPath;
|
|
262
270
|
try {
|
|
263
271
|
if (!fs.existsSync(filePath)) {
|
|
264
272
|
if (isExplicit) throw new ConfigFileError(`--config file not found: ${filePath}`);
|
|
@@ -495,11 +503,20 @@ async function detectCredentials(explicitModel?: string): Promise<CredentialDete
|
|
|
495
503
|
modelFrom,
|
|
496
504
|
piAuthPresent,
|
|
497
505
|
piModelsPresent,
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
506
|
+
// A model configured with no usable key is a different (and misleading to
|
|
507
|
+
// conflate) gap than nothing configured at all: SKILL.md tells agents to relay
|
|
508
|
+
// this text verbatim, so "No model … is configured" pointed users who HAD set
|
|
509
|
+
// PI_RESEARCH_MODEL at the wrong knob.
|
|
510
|
+
problem: model
|
|
511
|
+
? `A model is configured (${model}) but no API key is available for ` +
|
|
512
|
+
`${provider ? `provider "${provider}"` : 'its provider'}. Provide one via the ` +
|
|
513
|
+
'PI_RESEARCH_API_KEY environment variable (with PI_RESEARCH_PROVIDER), or, if you use pi, ' +
|
|
514
|
+
"via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider " +
|
|
515
|
+
'apiKey in ~/.pi/agent/models.json).'
|
|
516
|
+
: 'No model or API key is configured for pi-research. Provide credentials via environment ' +
|
|
517
|
+
'variables (PI_RESEARCH_API_KEY + PI_RESEARCH_PROVIDER + PI_RESEARCH_MODEL) or, if you use ' +
|
|
518
|
+
"pi, via pi's configuration (a provider entry in ~/.pi/agent/auth.json, or a per-provider " +
|
|
519
|
+
'apiKey in ~/.pi/agent/models.json).',
|
|
503
520
|
};
|
|
504
521
|
}
|
|
505
522
|
|
|
@@ -604,6 +621,14 @@ export async function cmdResearch(args: ResearchArgs): Promise<number> {
|
|
|
604
621
|
...(args.initialLinks ? { initialLinks: args.initialLinks } : {}),
|
|
605
622
|
}, abortController.signal);
|
|
606
623
|
|
|
624
|
+
// Snapshot the run diagnostics NOW, before any further await: on a cancelled
|
|
625
|
+
// run, onSignal's fire-and-forget safeShutdown runs concurrently with this
|
|
626
|
+
// tail and resets these SDK globals — reading them at JSON-emit time made
|
|
627
|
+
// `researchers`/`trackedErrors` silently vanish from a cancelled-but-
|
|
628
|
+
// successful payload depending on who won the race.
|
|
629
|
+
const outcome = getLastResearcherOutcome();
|
|
630
|
+
const errorReport = getLastErrorReport();
|
|
631
|
+
|
|
607
632
|
// Optionally persist the report to a file (opt-in via
|
|
608
633
|
// PI_RESEARCH_REPORT_EXPORT_ENABLED). The SDK is a pure library and does not
|
|
609
634
|
// write files; the CLI front-end does, mirroring the pi tool path.
|
|
@@ -636,8 +661,7 @@ export async function cmdResearch(args: ResearchArgs): Promise<number> {
|
|
|
636
661
|
// (ISSUES-ANALYSIS-2026-07-09.md, Symptom 4): a caller reading only stdout/JSON
|
|
637
662
|
// had no way to distinguish "thin report, little material out there" from "thin
|
|
638
663
|
// report because most researchers died mid-run" without opening the log file.
|
|
639
|
-
|
|
640
|
-
const errorReport = getLastErrorReport();
|
|
664
|
+
// (outcome/errorReport were snapshotted right after the run resolved — see above.)
|
|
641
665
|
toStdout(pretty({
|
|
642
666
|
ok: true,
|
|
643
667
|
depth,
|
|
@@ -719,11 +743,20 @@ export async function cmdKnowledge(queries: string[], json?: boolean): Promise<n
|
|
|
719
743
|
await safeShutdown();
|
|
720
744
|
}
|
|
721
745
|
|
|
746
|
+
// Same latch as cmdResearch's post-run check: the abort signal's checks inside
|
|
747
|
+
// searchKnowledge are point-in-time, so a Ctrl-C landing after the last one lets
|
|
748
|
+
// the search resolve normally — without this, an interrupted run raced onSignal's
|
|
749
|
+
// flushAndExit and could report ok:true / exit 0. The rationale comment above
|
|
750
|
+
// named exactly this failure; only the throwing half was previously handled.
|
|
751
|
+
if (cancellationRequested) {
|
|
752
|
+
exit = cancellationSignal ? exitCodeForSignal(cancellationSignal) : EXIT.CANCELLED;
|
|
753
|
+
}
|
|
754
|
+
|
|
722
755
|
if (json) {
|
|
723
756
|
// `ok: true` makes `ok` a reliable success/failure discriminant across every
|
|
724
757
|
// knowledge JSON variant (disabled-store above, reportError's ok:false below,
|
|
725
758
|
// and this success shape) — additive only, every existing field is unchanged.
|
|
726
|
-
toStdout(pretty({ ok: true, ...result }));
|
|
759
|
+
toStdout(pretty({ ok: true, ...(cancellationRequested ? { cancelled: true } : {}), ...result }));
|
|
727
760
|
} else {
|
|
728
761
|
toStdout((result.text.endsWith('\n') ? result.text : result.text + '\n'));
|
|
729
762
|
}
|
|
@@ -738,6 +771,18 @@ async function cmdStatus(json?: boolean): Promise<number> {
|
|
|
738
771
|
const det = await detectCredentials();
|
|
739
772
|
const paths = resolvedConfigPaths();
|
|
740
773
|
const cfg = getConfig(process.cwd(), 'cli');
|
|
774
|
+
// The stealth browser is installed by this package's `postinstall`
|
|
775
|
+
// (scripts/setup.cjs), which npm 11.19+ BLOCKS BY DEFAULT — it now warns
|
|
776
|
+
// "packages have install scripts not yet covered by allowScripts" and carries
|
|
777
|
+
// on, so the browser is simply absent on an otherwise successful install.
|
|
778
|
+
// Every other native artifact survives that (onnxruntime, sharp, esbuild and
|
|
779
|
+
// webgpu all ship prebuilt platform packages), which is exactly what makes
|
|
780
|
+
// this one worth reporting: nothing else fails, and without this the first
|
|
781
|
+
// symptom is a scrape falling back mid-run. A missing browser degrades
|
|
782
|
+
// capability rather than breaking the tool — the fetch path still serves most
|
|
783
|
+
// pages — so it is reported alongside `ready` rather than folded into it.
|
|
784
|
+
// isBrowserAvailable() is a resolve + existsSync; it never launches anything.
|
|
785
|
+
const browserInstalled = isBrowserAvailable();
|
|
741
786
|
const summary = {
|
|
742
787
|
package: '@lincoln504/pi-research',
|
|
743
788
|
version: PKG_VERSION,
|
|
@@ -751,6 +796,14 @@ async function cmdStatus(json?: boolean): Promise<number> {
|
|
|
751
796
|
modelFrom: det.modelFrom,
|
|
752
797
|
problem: det.problem ?? null,
|
|
753
798
|
},
|
|
799
|
+
browser: {
|
|
800
|
+
installed: browserInstalled,
|
|
801
|
+
// Same remedy the health check prints, so both surfaces name one command.
|
|
802
|
+
fix: browserInstalled ? null : 'npx camoufox-js fetch',
|
|
803
|
+
note: browserInstalled
|
|
804
|
+
? null
|
|
805
|
+
: 'Stealth browser not installed — scraping falls back to plain fetch, which some sites block. npm 11.19+ blocks install scripts by default, which skips this package\'s postinstall.',
|
|
806
|
+
},
|
|
754
807
|
knowledgeStoreMode: cfg.KNOWLEDGE_STORE_MODE,
|
|
755
808
|
defaultDepth: cfg.DEFAULT_RESEARCH_DEPTH,
|
|
756
809
|
paths,
|
|
@@ -764,6 +817,10 @@ async function cmdStatus(json?: boolean): Promise<number> {
|
|
|
764
817
|
toStdout(`pi-research ${PKG_VERSION}\n`);
|
|
765
818
|
toStdout(`ready: ${summary.ready ? 'yes' : 'no'}\n\n`);
|
|
766
819
|
toStdout(configBlock(det, summary.ready ? undefined : '— fix the problem above, then re-run.') + '\n');
|
|
820
|
+
if (!summary.browser.installed) {
|
|
821
|
+
toStdout(`\nstealth browser: NOT INSTALLED — run: ${summary.browser.fix}\n`);
|
|
822
|
+
toStdout(` ${summary.browser.note}\n`);
|
|
823
|
+
}
|
|
767
824
|
toStdout(`\nknowledge store mode: ${cfg.KNOWLEDGE_STORE_MODE} (default depth: ${cfg.DEFAULT_RESEARCH_DEPTH})\n`);
|
|
768
825
|
return EXIT.OK;
|
|
769
826
|
}
|
|
@@ -1032,7 +1089,17 @@ export async function reportError(err: unknown, what: string, json?: boolean): P
|
|
|
1032
1089
|
// --json: emit a structured error on stdout so a machine consumer never has to
|
|
1033
1090
|
// special-case plain-text stderr. Mirrors the { ok: true, ... } success shape.
|
|
1034
1091
|
if (json) {
|
|
1035
|
-
|
|
1092
|
+
// A cancellation reports itself as a cancellation, not as whatever component
|
|
1093
|
+
// happened to notice the abort first. The internal message is genuinely
|
|
1094
|
+
// misleading here — a Ctrl-C unwinding through the planner surfaces as
|
|
1095
|
+
// "Coordinator failed: Request was aborted", so a consumer relaying `error`
|
|
1096
|
+
// verbatim (which SKILL.md instructs agents to do) tells the user a component
|
|
1097
|
+
// broke when they simply stopped the run. The plain-text branch below already
|
|
1098
|
+
// refuses to say "failed" for a cancel; JSON said it anyway. The original
|
|
1099
|
+
// wording is preserved under `cause` so nothing diagnostic is lost.
|
|
1100
|
+
const payload: Record<string, unknown> = isCancellation
|
|
1101
|
+
? { ok: false, error: `pi-research ${what} cancelled.`, cause: msg, exitCode }
|
|
1102
|
+
: { ok: false, error: msg, exitCode };
|
|
1036
1103
|
// Machine-readable "this will succeed later" flag, so a consumer can back off
|
|
1037
1104
|
// and retry without string-matching the human message. Gated on cancellation:
|
|
1038
1105
|
// a Ctrl-C landing in the same window a queue-wait expires must never emit
|
|
@@ -1158,6 +1225,7 @@ COMMANDS
|
|
|
1158
1225
|
--dry-run | --json Plan only; JSON output.
|
|
1159
1226
|
|
|
1160
1227
|
help, --help, -h Show this help.
|
|
1228
|
+
--version, -v Print the pi-research version.
|
|
1161
1229
|
|
|
1162
1230
|
CONFIGURE
|
|
1163
1231
|
Credentials are resolved in this order (first match wins):
|
|
@@ -1372,6 +1440,15 @@ export function parseArgs(argv: string[]): ParsedArgs {
|
|
|
1372
1440
|
} else if (a === '--exclude-tools') {
|
|
1373
1441
|
const v = takeOptionValue(rest, ++i, '--exclude-tools');
|
|
1374
1442
|
excludeTools = v.split(',').map((s) => s.trim()).filter(Boolean);
|
|
1443
|
+
// Unknown names fail fast: the merge downstream is a blind Set union, so a
|
|
1444
|
+
// typo ("securty_search") silently excluded nothing while the user believed
|
|
1445
|
+
// the tool was off for the whole (billed) run.
|
|
1446
|
+
const unknown = excludeTools.filter((t) => !RESEARCH_TOOL_NAMES.includes(t));
|
|
1447
|
+
if (unknown.length > 0) {
|
|
1448
|
+
throw new UsageError(
|
|
1449
|
+
`--exclude-tools: unknown tool name${unknown.length === 1 ? '' : 's'} ` +
|
|
1450
|
+
`${unknown.map((t) => `"${t}"`).join(', ')}. Valid names: ${RESEARCH_TOOL_NAMES.join(', ')}.`);
|
|
1451
|
+
}
|
|
1375
1452
|
if (excludeTools.length === 0) excludeTools = undefined;
|
|
1376
1453
|
} else if (a === '--initial-links') {
|
|
1377
1454
|
// Consume until the next flag or end. Validate each entry: these are templated
|
|
@@ -1392,6 +1469,10 @@ export function parseArgs(argv: string[]): ParsedArgs {
|
|
|
1392
1469
|
}
|
|
1393
1470
|
j++;
|
|
1394
1471
|
}
|
|
1472
|
+
// Every other value-taking flag fails fast on a missing value; a bare
|
|
1473
|
+
// `--initial-links` (immediately followed by another flag or end of args)
|
|
1474
|
+
// silently proceeding with zero links was the one exception.
|
|
1475
|
+
if (j === i + 1) throw new UsageError('--initial-links requires at least one URL.');
|
|
1395
1476
|
i = j - 1;
|
|
1396
1477
|
} else if (a === '--config') {
|
|
1397
1478
|
configPath = takeOptionValue(rest, ++i, '--config');
|
package/src/config.ts
CHANGED
|
@@ -122,15 +122,6 @@ export const ConfigSchema = Type.Object({
|
|
|
122
122
|
* the model's real maxTokens at call time.
|
|
123
123
|
* Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
|
|
124
124
|
SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
|
|
125
|
-
/** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
|
|
126
|
-
* Without a cap the session inherits the model's full catalog ceiling as the
|
|
127
|
-
* per-request max_tokens — ~230k for large-context models — which providers that
|
|
128
|
-
* budget-check max_tokens up front (OpenRouter 402s against remaining credits)
|
|
129
|
-
* reject outright on low balances, even though the actual response would cost
|
|
130
|
-
* cents. Default 16384: researcher turns are tool calls plus a cited report, far
|
|
131
|
-
* below this. Clamped to the model's real ceiling at session build time.
|
|
132
|
-
* Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
|
|
133
|
-
RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
|
|
134
125
|
/** LLM Model override for researcher sub-agents and knowledge synthesis.
|
|
135
126
|
* Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
|
|
136
127
|
* When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
|
|
@@ -167,7 +158,8 @@ export const ConfigSchema = Type.Object({
|
|
|
167
158
|
/**
|
|
168
159
|
* Comma-separated list of research tools to DISABLE for every researcher in a run
|
|
169
160
|
* (e.g. "scrape", or "youtube_transcript,stackexchange"). Valid tool names:
|
|
170
|
-
* search, scrape, security_search, stackexchange, youtube_transcript, grep
|
|
161
|
+
* search, scrape, security_search, stackexchange, youtube_transcript, grep, read
|
|
162
|
+
* (RESEARCH_TOOL_NAMES in constants.ts is the canonical list).
|
|
171
163
|
* Disable-only: the list is merged into the per-run excludeTools stream, so disabled
|
|
172
164
|
* tools are both removed from each researcher's toolset AND named in the
|
|
173
165
|
* coordinator/evaluator "DISABLED TOOLS" prompt section. Env-only
|
|
@@ -242,7 +234,6 @@ const USER_MIGRATION_KEYS = [
|
|
|
242
234
|
'PI_RESEARCH_LLM_THINKING_LEVEL',
|
|
243
235
|
'PI_RESEARCH_PLANNING_MAX_TOKENS',
|
|
244
236
|
'PI_RESEARCH_SYNTHESIS_MAX_TOKENS',
|
|
245
|
-
'PI_RESEARCH_RESEARCHER_MAX_TOKENS',
|
|
246
237
|
'PI_RESEARCH_MIGRATION_STRATEGY',
|
|
247
238
|
'PI_RESEARCH_CONSOLE_LOG',
|
|
248
239
|
'PI_RESEARCH_MODEL',
|
|
@@ -875,7 +866,6 @@ export function saveConfig(config: Config, scope: 'local' | 'user' = 'local', cw
|
|
|
875
866
|
PI_RESEARCH_LLM_THINKING_LEVEL: config.LLM_THINKING_LEVEL,
|
|
876
867
|
PI_RESEARCH_PLANNING_MAX_TOKENS: String(config.PLANNING_MAX_TOKENS),
|
|
877
868
|
PI_RESEARCH_SYNTHESIS_MAX_TOKENS: String(config.SYNTHESIS_MAX_TOKENS),
|
|
878
|
-
PI_RESEARCH_RESEARCHER_MAX_TOKENS: String(config.RESEARCHER_MAX_TOKENS),
|
|
879
869
|
PI_RESEARCH_MIGRATION_STRATEGY: config.MIGRATION_STRATEGY,
|
|
880
870
|
PI_RESEARCH_CONSOLE_LOG: String(config.CONSOLE_LOG),
|
|
881
871
|
...(config.RESEARCH_MODEL ? { PI_RESEARCH_MODEL: config.RESEARCH_MODEL } : {}),
|
|
@@ -1106,7 +1096,6 @@ export function createConfig(env: Record<string, string | undefined>, processEnv
|
|
|
1106
1096
|
LLM_THINKING_LEVEL: parseEnvEnum(e, 'PI_RESEARCH_LLM_THINKING_LEVEL', ['off', 'minimal', 'low', 'medium', 'high'] as const, DEFAULTS.LLM_THINKING_LEVEL),
|
|
1107
1097
|
PLANNING_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_PLANNING_MAX_TOKENS', DEFAULTS.PLANNING_MAX_TOKENS, 1024, 131072),
|
|
1108
1098
|
SYNTHESIS_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_SYNTHESIS_MAX_TOKENS', DEFAULTS.SYNTHESIS_MAX_TOKENS, 1024, 131072),
|
|
1109
|
-
RESEARCHER_MAX_TOKENS: parseEnvNumber(e, 'PI_RESEARCH_RESEARCHER_MAX_TOKENS', DEFAULTS.RESEARCHER_MAX_TOKENS, 1024, 131072),
|
|
1110
1099
|
MIGRATION_STRATEGY: parseEnvEnum(e, 'PI_RESEARCH_MIGRATION_STRATEGY', ['drop', 're-embed', 'backup'] as const, DEFAULTS.MIGRATION_STRATEGY),
|
|
1111
1100
|
CONSOLE_LOG: parseEnvBool(e, 'PI_RESEARCH_CONSOLE_LOG', DEFAULTS.CONSOLE_LOG),
|
|
1112
1101
|
RESEARCH_MODEL: parseEnvString(e, 'PI_RESEARCH_MODEL', DEFAULTS.RESEARCH_MODEL),
|
|
@@ -1226,7 +1215,9 @@ function parseEnvNumber(env: Record<string, string | undefined>, key: string, de
|
|
|
1226
1215
|
}
|
|
1227
1216
|
|
|
1228
1217
|
function parseEnvBool(env: Record<string, string | undefined>, key: string, def: boolean): boolean {
|
|
1229
|
-
|
|
1218
|
+
// Trim like parseEnvNumber does: file values arrive trimmed via parseDotEnv, but a
|
|
1219
|
+
// real env var `PI_RESEARCH_X='true '` otherwise silently read false.
|
|
1220
|
+
const v = env[key]?.trim();
|
|
1230
1221
|
if (v === undefined || v === '') return def;
|
|
1231
1222
|
// Strict by design: only the literal 'true' (any case) is truthy; every other value is false.
|
|
1232
1223
|
// This is intentional and asserted in config.test.ts ('1'/'yes' → false) — do not loosen it.
|
|
@@ -1249,7 +1240,7 @@ function parseEnvEnum<T extends string>(
|
|
|
1249
1240
|
allowed: readonly T[],
|
|
1250
1241
|
def: T,
|
|
1251
1242
|
): T {
|
|
1252
|
-
const raw = env[key];
|
|
1243
|
+
const raw = env[key]?.trim();
|
|
1253
1244
|
if (raw === undefined || raw === '') return def;
|
|
1254
1245
|
if ((allowed as readonly string[]).includes(raw)) return raw as T;
|
|
1255
1246
|
logger.warn(`[config] Invalid ${key}="${raw}" (expected one of: ${allowed.join(', ')}). Using default "${def}".`);
|
package/src/constants.ts
CHANGED
|
@@ -182,6 +182,19 @@ export const MAX_QUERIES_PER_RESEARCHER_LEVEL_3 = 20;
|
|
|
182
182
|
*/
|
|
183
183
|
export const DEFAULT_EXCLUDED_TOOLS: readonly string[] = ['grep'];
|
|
184
184
|
|
|
185
|
+
/**
|
|
186
|
+
* The complete set of excludable research tool names — the vocabulary
|
|
187
|
+
* `--exclude-tools` / `excludeTools` / PI_RESEARCH_DISABLED_TOOLS accept.
|
|
188
|
+
* Kept here (next to DEFAULT_EXCLUDED_TOOLS) so validation and the exclusion
|
|
189
|
+
* machinery cannot drift; the DISABLED_TOOLS doc in config.ts names the same set.
|
|
190
|
+
* `read` is a researcher tool too (local file reads) — excluding it is a
|
|
191
|
+
* legitimate hardening move, and omitting it from this list broke exactly that
|
|
192
|
+
* on the CLI when validation was introduced.
|
|
193
|
+
*/
|
|
194
|
+
export const RESEARCH_TOOL_NAMES: readonly string[] = [
|
|
195
|
+
'search', 'scrape', 'security_search', 'stackexchange', 'youtube_transcript', 'grep', 'read',
|
|
196
|
+
];
|
|
197
|
+
|
|
185
198
|
/**
|
|
186
199
|
* Resolve the effective researcher exclusion list.
|
|
187
200
|
*
|
|
@@ -178,10 +178,26 @@ export class ServiceContainer {
|
|
|
178
178
|
options: ServiceContainerOptions = {}
|
|
179
179
|
): Promise<void> {
|
|
180
180
|
const mergedOptions = { ...this.defaultOptions, ...options, allowOverwrite: true };
|
|
181
|
-
|
|
181
|
+
|
|
182
182
|
// Dispose existing service if present
|
|
183
|
-
|
|
184
|
-
|
|
183
|
+
const registration = this.services.get(name);
|
|
184
|
+
if (registration) {
|
|
185
|
+
// Join an already-in-flight clear()/replace()/disposeAll() on this
|
|
186
|
+
// registration before touching its instance — the same guard those paths
|
|
187
|
+
// use on each other. Without it, a concurrent teardown and this call both
|
|
188
|
+
// dispose the SAME instance, and register() below then swaps the whole
|
|
189
|
+
// registration object out from under the in-flight one, orphaning any
|
|
190
|
+
// instance it constructs.
|
|
191
|
+
// while(), not if(): a newer disposal can begin while we are awaiting this
|
|
192
|
+
// one, exactly as get() documents for the same field.
|
|
193
|
+
while (registration.disposalPromise) {
|
|
194
|
+
await registration.disposalPromise.catch(() => { /* disposal errors are logged by clear()/replace() */ });
|
|
195
|
+
}
|
|
196
|
+
// Settle an in-flight initialization too, so we never discard a service
|
|
197
|
+
// that is still being built — mirrors replace().
|
|
198
|
+
if (registration.initializationPromise) {
|
|
199
|
+
await registration.initializationPromise.catch(() => { /* failed init cleaned up after itself */ });
|
|
200
|
+
}
|
|
185
201
|
if (registration.instance && registration.instance.dispose) {
|
|
186
202
|
try {
|
|
187
203
|
await registration.instance.dispose();
|
|
@@ -31,15 +31,40 @@ function getWindowsCamoufoxDir(): string {
|
|
|
31
31
|
return join(homedir(), 'AppData', 'Local', 'camoufox', 'camoufox', 'Cache');
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* The custom camoufox location, if the user set one.
|
|
36
|
+
*
|
|
37
|
+
* CAMOUFOX_INSTALL_DIR takes precedence because it is camoufox-js's OWN
|
|
38
|
+
* variable and the only one that can ever move the install: camoufox-js ≥0.12.0
|
|
39
|
+
* resolves INSTALL_DIR from it. PLAYWRIGHT_BROWSERS_PATH is kept as a documented
|
|
40
|
+
* alias, but on its own it only ever moved where we LOOK — camoufox went on
|
|
41
|
+
* installing to the default cache, so setting it made the browser permanently
|
|
42
|
+
* "not found" right after a successful download. Both are exported onward to
|
|
43
|
+
* camoufox-js (getBrowserEnv, scripts/setup.cjs) so that install and lookup
|
|
44
|
+
* cannot disagree.
|
|
45
|
+
*
|
|
46
|
+
* CAVEAT, deliberate: the pinned camoufox-js (0.10.x — see
|
|
47
|
+
* thread-worker-browser.ts for why it cannot move yet) hardcodes
|
|
48
|
+
* `userCacheDir("camoufox")` and honours NEITHER variable, so on the shipped
|
|
49
|
+
* pin a custom location still only relocates lookup and the underlying bug
|
|
50
|
+
* remains. This plumbing is correct and inert rather than wrong: it starts
|
|
51
|
+
* working the moment the pin can advance, and costs nothing until then. Until
|
|
52
|
+
* that happens, treat a custom browser directory as unsupported.
|
|
53
|
+
*/
|
|
54
|
+
function getCustomCamoufoxDir(): string | undefined {
|
|
55
|
+
return process.env['CAMOUFOX_INSTALL_DIR'] || process.env['PLAYWRIGHT_BROWSERS_PATH'] || undefined;
|
|
56
|
+
}
|
|
57
|
+
|
|
34
58
|
/**
|
|
35
59
|
* Get the camoufox binary cache directory.
|
|
36
|
-
* Uses
|
|
60
|
+
* Uses the custom location above if set, otherwise the standard user cache location
|
|
37
61
|
* that camoufox uses by default (~/.cache/camoufox on Linux).
|
|
38
62
|
* We do NOT override HOME — that trick was unreliable and caused install/runtime mismatches.
|
|
39
63
|
*/
|
|
40
64
|
export function getBrowserCacheDir(): string {
|
|
41
|
-
|
|
42
|
-
|
|
65
|
+
const custom = getCustomCamoufoxDir();
|
|
66
|
+
if (custom) {
|
|
67
|
+
return custom;
|
|
43
68
|
}
|
|
44
69
|
// Mirror getCamoufoxBinaryPath() so both functions agree on the cache location
|
|
45
70
|
const osPlatform = platform();
|
|
@@ -109,11 +134,17 @@ export function getBrowserProfileDir(config?: Config): string {
|
|
|
109
134
|
*/
|
|
110
135
|
export function getBrowserEnv(config?: Config): NodeJS.ProcessEnv {
|
|
111
136
|
const env: NodeJS.ProcessEnv = { ...process.env };
|
|
112
|
-
const customPath =
|
|
137
|
+
const customPath = getCustomCamoufoxDir();
|
|
113
138
|
if (customPath) {
|
|
114
139
|
env['PLAYWRIGHT_BROWSERS_PATH'] = customPath;
|
|
140
|
+
// The one that camoufox-js actually reads. Without it the worker's
|
|
141
|
+
// camoufox launches from the DEFAULT cache while everything on this side
|
|
142
|
+
// resolves the custom path — the install/runtime mismatch this file's
|
|
143
|
+
// header warns about, just arriving through a different door.
|
|
144
|
+
env['CAMOUFOX_INSTALL_DIR'] = customPath;
|
|
115
145
|
} else {
|
|
116
146
|
delete env['PLAYWRIGHT_BROWSERS_PATH'];
|
|
147
|
+
delete env['CAMOUFOX_INSTALL_DIR'];
|
|
117
148
|
}
|
|
118
149
|
// Tell thread-workers where to write their lifecycle/error log. Precedence:
|
|
119
150
|
// 1. A user-set PI_RESEARCH_LOG_FILE (already copied from process.env above) wins —
|
|
@@ -164,7 +195,7 @@ export function ensureBrowserCacheDir(): string {
|
|
|
164
195
|
* Matches camoufox-js's own resolution logic.
|
|
165
196
|
*/
|
|
166
197
|
export function getCamoufoxBinaryPath(): string {
|
|
167
|
-
const customPath =
|
|
198
|
+
const customPath = getCustomCamoufoxDir();
|
|
168
199
|
if (customPath) {
|
|
169
200
|
return customPath;
|
|
170
201
|
}
|
|
@@ -136,9 +136,19 @@ async function isRegisteredLeaderAlive(container: ServiceContainer): Promise<boo
|
|
|
136
136
|
return result;
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
-
/**
|
|
140
|
-
|
|
139
|
+
/**
|
|
140
|
+
* Test seam: drop BOTH module-level globals that survive between tests — the
|
|
141
|
+
* coalesced liveness probe (1s window) and the restart cooldown (10s window).
|
|
142
|
+
*
|
|
143
|
+
* Both are far longer than a test file's runtime, so a test that reaches
|
|
144
|
+
* restartSchedulerWithHerdGuard leaves state that silently short-circuits the
|
|
145
|
+
* next one: isRegisteredLeaderAliveUncached / forceSchedulerRestart simply are
|
|
146
|
+
* not re-invoked. Today nothing observes that only because the one test that
|
|
147
|
+
* reaches this path happens to run last — ordering, not a guarantee.
|
|
148
|
+
*/
|
|
149
|
+
export function _resetTaskExecutionGlobalsForTests(): void {
|
|
141
150
|
leaderLivenessProbe = null;
|
|
151
|
+
lastRestartTime = 0;
|
|
142
152
|
}
|
|
143
153
|
|
|
144
154
|
/**
|
|
@@ -134,29 +134,45 @@ export async function initBrowser(): Promise<void> {
|
|
|
134
134
|
// (consumers install without our lockfile, so a floating range resolves
|
|
135
135
|
// to a newer, broken version):
|
|
136
136
|
//
|
|
137
|
-
// • camoufox-js ^0.10.2 →
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
//
|
|
137
|
+
// • camoufox-js ^0.10.2 → the two ORIGINAL reasons for this pin have
|
|
138
|
+
// both lapsed (camoufox restored Windows assets in v152.0.4-beta.26,
|
|
139
|
+
// 2026-07-16; and impit's `only-allow pnpm` guard existed only in
|
|
140
|
+
// 0.13.1/0.14.0), but a THIRD, worse one replaced them. 0.12.0
|
|
141
|
+
// requires `better-sqlite3 ^13.0.1`, and better-sqlite3 13 DROPPED
|
|
142
|
+
// PREBUILT BINARIES: 12.x installs via `prebuild-install || node-gyp
|
|
143
|
+
// rebuild`, 13.x has no install script at all, so npm falls back to
|
|
144
|
+
// auto-running `node-gyp rebuild` for its binding.gyp and every
|
|
145
|
+
// install needs a full C++ toolchain. Measured: `npm ci` fails
|
|
146
|
+
// outright on the Windows CI runner ("could not find a version of
|
|
147
|
+
// Visual Studio 2017 or newer"), which is most Windows consumers.
|
|
148
|
+
// Upgrading again means checking better-sqlite3 first, not camoufox.
|
|
141
149
|
// • playwright-core 1.60.0 (exact) → playwright-core and the camoufox
|
|
142
150
|
// binary are one matched Juggler-protocol pair. 1.61 added a
|
|
143
|
-
// viewport.isMobile field the
|
|
151
|
+
// viewport.isMobile field the build REJECTS ("property
|
|
144
152
|
// viewport.isMobile not described in this scheme"), failing every
|
|
145
|
-
// launch below.
|
|
146
|
-
//
|
|
153
|
+
// launch below. Corroborated upstream: camoufox-js 0.12.0 declares
|
|
154
|
+
// `peerDependencies: playwright-core "<1.61.0"`, the same bound this
|
|
155
|
+
// pin has held by hand.
|
|
147
156
|
// • impit 0.13.0 (exact, a direct dep though only camoufox-js uses it)
|
|
148
|
-
// →
|
|
149
|
-
//
|
|
150
|
-
// `
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
// a published package
|
|
157
|
+
// → camoufox-js 0.10.2 requires `impit ^0.13.0`, and within that range
|
|
158
|
+
// 0.13.0 is the only version free of the `preinstall: npx only-allow
|
|
159
|
+
// pnpm` guard (an upstream mistake in 0.13.1/0.14.0, dropped again in
|
|
160
|
+
// 0.14.1) that hard-fails `npm install -g`. The exact pin is also the
|
|
161
|
+
// ONLY way to force a version for consumers, since npm `overrides`
|
|
162
|
+
// don't propagate to installers of a published package.
|
|
154
163
|
//
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
//
|
|
164
|
+
// What is NOT pinned: the browser BINARY. `camoufox-js fetch` takes no
|
|
165
|
+
// version argument — it walks the GitHub releases newest-first and takes
|
|
166
|
+
// the first non-prerelease one with an asset for this OS/arch. A consumer
|
|
167
|
+
// therefore gets whatever camoufox published most recently, whichever
|
|
168
|
+
// camoufox-js they have, and a future release could break launches with no
|
|
169
|
+
// change on our side. Both the current newest (v152.0.4-beta.28, Firefox
|
|
170
|
+
// 152) and the older v135.0.1-beta.24 an existing cache may hold were
|
|
171
|
+
// verified launching and driving under playwright-core 1.60.0.
|
|
172
|
+
//
|
|
173
|
+
// Always re-verify a REAL headless run after touching any of this: the
|
|
174
|
+
// unit and integration suites mock the browser, so a Juggler mismatch
|
|
175
|
+
// reaches production with a fully green suite.
|
|
160
176
|
let CamoufoxModule: any;
|
|
161
177
|
try {
|
|
162
178
|
CamoufoxModule = await import('camoufox-js');
|
|
@@ -99,7 +99,10 @@ var init_log_utils = __esm({
|
|
|
99
99
|
`${B_LEFT}(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})${B_RIGHT}`,
|
|
100
100
|
"g"
|
|
101
101
|
);
|
|
102
|
-
JWT_PATTERN = new RegExp(
|
|
102
|
+
JWT_PATTERN = new RegExp(
|
|
103
|
+
`${B_LEFT}eyJ[A-Za-z0-9_-]{1,1024}\\.[A-Za-z0-9_-]{1,8192}\\.[A-Za-z0-9_-]{1,4096}`,
|
|
104
|
+
"g"
|
|
105
|
+
);
|
|
103
106
|
BASIC_AUTH_PATTERN = new RegExp(`${B_LEFT}Basic\\s+[A-Za-z0-9+/]{16,}={0,2}`, "gi");
|
|
104
107
|
BEARER_AUTH_PATTERN = new RegExp(`${B_LEFT}Bearer\\s+[A-Za-z0-9._~+/-]+=*`, "gi");
|
|
105
108
|
PROVIDER_TOKEN_PATTERN = new RegExp(
|
|
@@ -1509,6 +1512,7 @@ var scraper_utils_exports = {};
|
|
|
1509
1512
|
__export(scraper_utils_exports, {
|
|
1510
1513
|
createJsMarkdownConverter: () => createJsMarkdownConverter,
|
|
1511
1514
|
createNativeMarkdownConverter: () => createNativeMarkdownConverter,
|
|
1515
|
+
disposeSsrfSafeFetcher: () => disposeSsrfSafeFetcher,
|
|
1512
1516
|
extractDomain: () => extractDomain,
|
|
1513
1517
|
formatErrorWithCause: () => formatErrorWithCause,
|
|
1514
1518
|
getRandomUserAgent: () => getRandomUserAgent,
|
|
@@ -1644,19 +1648,33 @@ function isSsrfBlockError(error) {
|
|
|
1644
1648
|
}
|
|
1645
1649
|
return false;
|
|
1646
1650
|
}
|
|
1647
|
-
|
|
1648
|
-
|
|
1651
|
+
function getSsrfSafeFetcher() {
|
|
1652
|
+
cachedSsrfFetcher ??= (async () => {
|
|
1653
|
+
try {
|
|
1654
|
+
const { Agent, fetch: undiciFetch } = await import("undici");
|
|
1655
|
+
return {
|
|
1656
|
+
fetch: undiciFetch,
|
|
1657
|
+
dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
|
|
1658
|
+
};
|
|
1659
|
+
} catch (e) {
|
|
1660
|
+
logger.warn(`[Scrapers] undici unavailable; connect-time SSRF pinning disabled (request-time validation still active): ${e instanceof Error ? e.message : String(e)}`);
|
|
1661
|
+
return null;
|
|
1662
|
+
}
|
|
1663
|
+
})();
|
|
1664
|
+
return cachedSsrfFetcher;
|
|
1665
|
+
}
|
|
1666
|
+
async function disposeSsrfSafeFetcher() {
|
|
1667
|
+
const pending = cachedSsrfFetcher;
|
|
1668
|
+
cachedSsrfFetcher = void 0;
|
|
1669
|
+
if (pending === void 0) return;
|
|
1670
|
+
const cached = await pending.catch(() => null);
|
|
1671
|
+
const closable = cached?.dispatcher;
|
|
1672
|
+
if (!closable?.close) return;
|
|
1649
1673
|
try {
|
|
1650
|
-
|
|
1651
|
-
cachedSsrfFetcher = {
|
|
1652
|
-
fetch: undiciFetch,
|
|
1653
|
-
dispatcher: new Agent({ connect: { lookup: ssrfSafeLookup } })
|
|
1654
|
-
};
|
|
1674
|
+
await closable.close();
|
|
1655
1675
|
} catch (e) {
|
|
1656
|
-
logger.
|
|
1657
|
-
cachedSsrfFetcher = null;
|
|
1676
|
+
logger.debug(`[Scrapers] Error closing SSRF dispatcher: ${e instanceof Error ? e.message : String(e)}`);
|
|
1658
1677
|
}
|
|
1659
|
-
return cachedSsrfFetcher;
|
|
1660
1678
|
}
|
|
1661
1679
|
function formatErrorWithCause(err) {
|
|
1662
1680
|
try {
|
|
@@ -2544,15 +2562,6 @@ var ConfigSchema = Type.Object({
|
|
|
2544
2562
|
* the model's real maxTokens at call time.
|
|
2545
2563
|
* Not exposed in TUI — controlled via PI_RESEARCH_SYNTHESIS_MAX_TOKENS env var. */
|
|
2546
2564
|
SYNTHESIS_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 32768 }),
|
|
2547
|
-
/** Max OUTPUT tokens per researcher LLM call (deep and quick sub-agent sessions).
|
|
2548
|
-
* Without a cap the session inherits the model's full catalog ceiling as the
|
|
2549
|
-
* per-request max_tokens — ~230k for large-context models — which providers that
|
|
2550
|
-
* budget-check max_tokens up front (OpenRouter 402s against remaining credits)
|
|
2551
|
-
* reject outright on low balances, even though the actual response would cost
|
|
2552
|
-
* cents. Default 16384: researcher turns are tool calls plus a cited report, far
|
|
2553
|
-
* below this. Clamped to the model's real ceiling at session build time.
|
|
2554
|
-
* Not exposed in TUI — controlled via PI_RESEARCH_RESEARCHER_MAX_TOKENS env var. */
|
|
2555
|
-
RESEARCHER_MAX_TOKENS: Type.Number({ minimum: 1024, maximum: 131072, default: 16384 }),
|
|
2556
2565
|
/** LLM Model override for researcher sub-agents and knowledge synthesis.
|
|
2557
2566
|
* Format: provider/model-id (e.g. google/gemini-2.0-flash-001) or just model-id.
|
|
2558
2567
|
* When set, this overrides ctx.model for researcher sub-agents (both deep and quick)
|
|
@@ -2589,7 +2598,8 @@ var ConfigSchema = Type.Object({
|
|
|
2589
2598
|
/**
|
|
2590
2599
|
* Comma-separated list of research tools to DISABLE for every researcher in a run
|
|
2591
2600
|
* (e.g. "scrape", or "youtube_transcript,stackexchange"). Valid tool names:
|
|
2592
|
-
* search, scrape, security_search, stackexchange, youtube_transcript, grep
|
|
2601
|
+
* search, scrape, security_search, stackexchange, youtube_transcript, grep, read
|
|
2602
|
+
* (RESEARCH_TOOL_NAMES in constants.ts is the canonical list).
|
|
2593
2603
|
* Disable-only: the list is merged into the per-run excludeTools stream, so disabled
|
|
2594
2604
|
* tools are both removed from each researcher's toolset AND named in the
|
|
2595
2605
|
* coordinator/evaluator "DISABLED TOOLS" prompt section. Env-only
|
|
@@ -27,7 +27,21 @@ export class GPUResourceService implements IGPUResourceService {
|
|
|
27
27
|
lifecycle = ServiceLifecycle.UNINITIALIZED;
|
|
28
28
|
|
|
29
29
|
private processLifecycle: IProcessLifecycle | null = null;
|
|
30
|
-
|
|
30
|
+
/**
|
|
31
|
+
* Backstop for a holder that is alive but wedged. Step 3 already reclaims from
|
|
32
|
+
* *dead* owners via PID+startTime liveness, so this only ever fires against a
|
|
33
|
+
* process we just proved is running — which makes it a lock STEAL, and the
|
|
34
|
+
* threshold must therefore exceed the longest legitimate hold.
|
|
35
|
+
*
|
|
36
|
+
* It previously did not: at 5 minutes it exactly equalled the default
|
|
37
|
+
* EMBEDDING_MODEL_INIT_TIMEOUT_MS (300000, schema max 600000), and the
|
|
38
|
+
* embedder holds this lock across the whole model download + pipeline load.
|
|
39
|
+
* A first-time HuggingFace fetch running near its own permitted budget would
|
|
40
|
+
* have the GPU lock taken out from under it, admitting exactly the concurrent
|
|
41
|
+
* WebGPU/Dawn init this lock exists to prevent. Keep this strictly above that
|
|
42
|
+
* schema maximum.
|
|
43
|
+
*/
|
|
44
|
+
private gpuLockStaleThresholdMs: number = 1000 * 60 * 15; // 15 min > EMBEDDING_MODEL_INIT_TIMEOUT_MS max (10 min)
|
|
31
45
|
|
|
32
46
|
constructor(options?: GPUResourceServiceOptions) {
|
|
33
47
|
if (options) {
|