@lincoln504/pi-research 0.1.12 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +64 -115
- package/dist/cli.mjs +23211 -0
- package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
- package/dist/prompts/research-knowledge-search-extractor.md +41 -0
- package/dist/prompts/research-tool-usage.md +92 -0
- package/dist/prompts/researcher.md +89 -0
- package/dist/prompts/system-coordinator.md +75 -0
- package/dist/prompts/system-lead-evaluator.md +118 -0
- package/dist/thread-worker.mjs +2611 -0
- package/dist/webgpu-probe.mjs +80 -0
- package/docs/AGENT-SKILL.md +102 -0
- package/docs/ARCHITECTURE.md +306 -80
- package/docs/CONFIGURATION.md +249 -0
- package/docs/KNOWLEDGE-STORE.md +213 -0
- package/docs/PI-EXTENSION.md +91 -0
- package/docs/SDK.md +113 -0
- package/docs/THIRD-PARTY-NOTICES.md +64 -0
- package/package.json +102 -36
- package/scripts/build.cjs +140 -0
- package/scripts/cleanup.cjs +149 -0
- package/scripts/prepare.cjs +28 -0
- package/scripts/setup.cjs +134 -0
- package/skills/pi-research/README.md +90 -0
- package/skills/pi-research/SKILL.md +102 -0
- package/skills/pi-research/references/configuration.md +77 -0
- package/skills/pi-research/scripts/run.mjs +237 -0
- package/src/cleanup/research-cleanup.ts +140 -0
- package/src/cli.ts +1044 -0
- package/src/config.ts +1036 -147
- package/src/constants.ts +115 -84
- package/src/core/health-check-service.ts +265 -0
- package/src/core/interfaces/health-check-interfaces.ts +50 -0
- package/src/core/interfaces/knowledge-interfaces.ts +190 -0
- package/src/core/interfaces/observer-interfaces.ts +67 -0
- package/src/core/interfaces/orchestration-interfaces.ts +86 -0
- package/src/core/interfaces/planning-interfaces.ts +100 -0
- package/src/core/interfaces/process-interfaces.ts +37 -0
- package/src/core/interfaces/research-plan-types.ts +52 -0
- package/src/core/interfaces/scheduler-interfaces.ts +91 -0
- package/src/core/interfaces/service-names.ts +39 -0
- package/src/core/interfaces/state-manager-interfaces.ts +29 -0
- package/src/core/interfaces/steering-interfaces.ts +31 -0
- package/src/core/llm/agentic-repair.ts +169 -0
- package/src/core/llm/inject-date.ts +52 -0
- package/src/core/llm/llm-timeout.ts +64 -0
- package/src/core/llm/llm-utils.ts +86 -0
- package/src/core/llm/model-registry-factory.ts +287 -0
- package/src/core/llm/pi-ai-completion.ts +29 -0
- package/src/core/llm/prompts.ts +50 -0
- package/src/core/llm/research-model-resolver.ts +91 -0
- package/src/core/planning-constants.ts +13 -0
- package/src/core/planning-service.ts +649 -0
- package/src/core/planning-utils.ts +299 -0
- package/src/core/scheduler-factory.ts +48 -0
- package/src/core/scheduler-service.ts +123 -0
- package/src/core/service-initialization.ts +194 -0
- package/src/core/service-interfaces.ts +48 -0
- package/src/core/service-registry.ts +659 -0
- package/src/healthcheck/index.ts +271 -265
- package/src/healthcheck/registry.ts +130 -0
- package/src/index.ts +639 -50
- package/src/infrastructure/browser/browser-cleanup.ts +309 -0
- package/src/infrastructure/browser/browser-client.ts +199 -0
- package/src/infrastructure/browser/browser-error-utils.ts +160 -0
- package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
- package/src/infrastructure/browser/browser-server.ts +174 -0
- package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
- package/src/infrastructure/browser/cleanup-utils.ts +155 -0
- package/src/infrastructure/browser/client-agent.ts +31 -0
- package/src/infrastructure/browser/config.ts +288 -0
- package/src/infrastructure/browser/ensure-browser.ts +205 -0
- package/src/infrastructure/browser/index.ts +37 -0
- package/src/infrastructure/browser/priority-task-queue.ts +226 -0
- package/src/infrastructure/browser/scheduler-factory.ts +354 -0
- package/src/infrastructure/browser/task-execution-service.ts +209 -0
- package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
- package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
- package/src/infrastructure/browser/thread-worker-types.ts +25 -0
- package/src/infrastructure/browser/thread-worker.mjs +2611 -0
- package/src/infrastructure/browser/thread-worker.ts +168 -0
- package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
- package/src/infrastructure/embedding/embedding-client.ts +205 -0
- package/src/infrastructure/embedding/embedding-factory.ts +324 -0
- package/src/infrastructure/embedding/embedding-server.ts +535 -0
- package/src/infrastructure/file-lock-service.ts +592 -0
- package/src/infrastructure/gpu-resource-service.ts +191 -0
- package/src/infrastructure/knowledge-store-service.ts +475 -0
- package/src/infrastructure/metrics-service.ts +129 -0
- package/src/infrastructure/process-lifecycle-service.ts +253 -0
- package/src/infrastructure/scheduler-factory-service.ts +58 -0
- package/src/infrastructure/service-initialization.ts +292 -0
- package/src/infrastructure/state/state-backup-manager.ts +290 -0
- package/src/infrastructure/state/state-browser-api.ts +62 -0
- package/src/infrastructure/state/state-browser-manager.ts +70 -0
- package/src/infrastructure/state/state-manager-service.ts +264 -0
- package/src/infrastructure/state/state-manager.ts +547 -0
- package/src/infrastructure/state/state-metrics.ts +92 -0
- package/src/infrastructure/state/state-migration.ts +103 -0
- package/src/infrastructure/state/state-path-configuration.ts +119 -0
- package/src/infrastructure/state/state-session-api.ts +94 -0
- package/src/infrastructure/state/state-session-manager.ts +127 -0
- package/src/infrastructure/state/state-validator.ts +93 -0
- package/src/infrastructure/types/state-types.ts +85 -0
- package/src/knowledge/chunker.ts +157 -0
- package/src/knowledge/embedder-init.ts +413 -0
- package/src/knowledge/embedder-utils.ts +296 -0
- package/src/knowledge/embedder.ts +592 -0
- package/src/knowledge/index.ts +126 -0
- package/src/knowledge/lancedb-loader.ts +35 -0
- package/src/knowledge/migration.ts +17 -0
- package/src/knowledge/model-config.ts +132 -0
- package/src/knowledge/onnx-env.ts +48 -0
- package/src/knowledge/store-operations.ts +291 -0
- package/src/knowledge/store-schema.ts +79 -0
- package/src/knowledge/store.ts +1251 -0
- package/src/knowledge/transformers-loader.ts +31 -0
- package/src/knowledge/webgpu-probe.mjs +80 -0
- package/src/knowledge/webgpu-viability.ts +255 -0
- package/src/knowledge/writer-queue.ts +252 -0
- package/src/logger.ts +316 -122
- package/src/observers/research-observer-impl.ts +459 -0
- package/src/orchestration/deep-research-orchestrator.ts +524 -773
- package/src/orchestration/deep-research-types.ts +47 -35
- package/src/orchestration/headless-observer.ts +114 -0
- package/src/orchestration/orchestration-types.ts +33 -0
- package/src/orchestration/quick-research-orchestrator.ts +463 -0
- package/src/orchestration/research-orchestration-service.ts +549 -0
- package/src/orchestration/research-session-service.ts +178 -0
- package/src/orchestration/research-synthesis-service.ts +348 -0
- package/src/orchestration/researcher-executor.ts +394 -0
- package/src/orchestration/researcher.ts +115 -42
- package/src/orchestration/service-initialization.ts +109 -0
- package/src/orchestration/session-context.ts +7 -3
- package/src/orchestration/session-state.ts +672 -0
- package/src/prompts/research-knowledge-relevance-triage.md +35 -0
- package/src/prompts/research-knowledge-search-extractor.md +41 -0
- package/src/prompts/research-tool-usage.md +63 -16
- package/src/prompts/researcher.md +59 -136
- package/src/prompts/system-coordinator.md +64 -13
- package/src/prompts/system-lead-evaluator.md +82 -189
- package/src/research-config.ts +872 -0
- package/src/sdk.ts +847 -0
- package/src/security/cisa-kev.ts +72 -9
- package/src/security/cvss.ts +92 -0
- package/src/security/ecosystem.ts +45 -0
- package/src/security/github-advisories.ts +168 -428
- package/src/security/github-advisory-types.ts +282 -0
- package/src/security/index.ts +63 -177
- package/src/security/nvd-client.ts +550 -0
- package/src/security/nvd-types.ts +179 -0
- package/src/security/osv-client.ts +246 -0
- package/src/security/osv-types.ts +285 -0
- package/src/security/types.ts +67 -53
- package/src/skill-install/skill-installer.ts +519 -0
- package/src/stackexchange/index.ts +81 -27
- package/src/stackexchange/output/compact.ts +7 -6
- package/src/stackexchange/output/table.ts +11 -7
- package/src/stackexchange/queries.ts +0 -10
- package/src/stackexchange/rest-client.ts +141 -66
- package/src/stackexchange/types.ts +0 -17
- package/src/tool.ts +15 -423
- package/src/tools/grep.ts +32 -310
- package/src/tools/health-tool-definition.ts +101 -0
- package/src/tools/index.ts +38 -9
- package/src/tools/research-knowledge-search.ts +974 -0
- package/src/tools/research-knowledge-types.ts +77 -0
- package/src/tools/research-tool-definition.ts +494 -0
- package/src/tools/scrape.ts +286 -388
- package/src/tools/search.ts +98 -153
- package/src/tools/security.ts +107 -80
- package/src/tools/stackexchange.ts +58 -35
- package/src/tools/youtube-transcript.ts +163 -0
- package/src/tui/knowledge-search-panel.ts +86 -0
- package/src/tui/pulse.ts +86 -0
- package/src/tui/research-health.ts +124 -0
- package/src/tui/research-panel-color-utils.ts +340 -0
- package/src/tui/research-panel-format.ts +54 -0
- package/src/tui/research-panel-state.ts +404 -0
- package/src/tui/research-panel-wave.ts +280 -0
- package/src/tui/research-panel.ts +339 -435
- package/src/tui/research-tui-manager.ts +233 -0
- package/src/tui/tui-controller.ts +137 -0
- package/src/types/extension-context.ts +18 -49
- package/src/types/index.ts +61 -0
- package/src/types/llm.ts +68 -29
- package/src/types/research-panel-types.ts +61 -0
- package/src/utils/circuit-breaker.ts +146 -0
- package/src/utils/citation-utils.ts +97 -0
- package/src/utils/disk-space-checker.ts +55 -0
- package/src/utils/error-tracker.ts +265 -0
- package/src/utils/host-config.ts +45 -0
- package/src/utils/input-validation.ts +8 -10
- package/src/utils/json-utils.ts +150 -16
- package/src/utils/llm-usage.ts +54 -0
- package/src/utils/log-rotation.ts +125 -0
- package/src/utils/log-utils.ts +159 -0
- package/src/utils/make-resource-loader.ts +3 -1
- package/src/utils/metrics-summary.ts +459 -0
- package/src/utils/metrics.ts +302 -0
- package/src/utils/research-export.ts +28 -12
- package/src/utils/safe-unref.ts +31 -0
- package/src/utils/shared-links.ts +195 -90
- package/src/utils/shutdown-manager.ts +65 -0
- package/src/utils/stdio-capture.ts +407 -0
- package/src/utils/text-utils.ts +264 -7
- package/src/utils/tool-usage-tracker.ts +83 -16
- package/src/utils/url-utils.ts +255 -0
- package/src/utils/user-agent.ts +28 -0
- package/src/web-research/browser-search.ts +179 -0
- package/src/web-research/retry-utils.ts +173 -77
- package/src/web-research/scraper-types.ts +72 -0
- package/src/web-research/scraper-utils.ts +434 -0
- package/src/web-research/search.ts +78 -218
- package/src/web-research/types.ts +9 -24
- package/src/web-research/utils.ts +2 -143
- package/src/web-research/web-scraper.ts +477 -0
- package/src/youtube/index.ts +154 -0
- package/src/youtube/potoken.ts +259 -0
- package/src/youtube/transcript-client.ts +390 -0
- package/src/youtube/video-id.ts +118 -0
- package/config/searxng/default-settings.yml +0 -189
- package/config/searxng/limiter.toml +0 -34
- package/config/tooling/eslint.config.js +0 -73
- package/config/tooling/vitest.config.integration.ts +0 -17
- package/config/tooling/vitest.config.ts +0 -40
- package/config/tooling/vitest.config.unit.ts +0 -12
- package/docs/assets/README-banner.jpg +0 -0
- package/scripts/setup.js +0 -174
- package/src/infrastructure/searxng-lifecycle.ts +0 -577
- package/src/infrastructure/searxng-manager.ts +0 -1183
- package/src/infrastructure/state-manager.ts +0 -918
- package/src/orchestration/deep-research-reducer.ts +0 -129
- package/src/orchestration/id-utils.ts +0 -63
- package/src/security/nvd.ts +0 -532
- package/src/security/osv.ts +0 -487
- package/src/stackexchange/cache.ts +0 -111
- package/src/stackexchange/output/index.ts +0 -7
- package/src/stackexchange/output/json.ts +0 -21
- package/src/tui/searxng-status.ts +0 -80
- package/src/utils/inject-date.ts +0 -29
- package/src/utils/research-error.ts +0 -205
- package/src/utils/searxng-config.ts +0 -94
- package/src/utils/session-state.ts +0 -295
- package/src/web-research/dependencies.ts +0 -20
- package/src/web-research/scrapers.ts +0 -935
- package/tsconfig.json +0 -40
|
@@ -5,12 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { logger } from '../logger.ts';
|
|
8
|
-
|
|
9
|
-
/** Maximum query length */
|
|
10
|
-
export const MAX_QUERY_LENGTH = 500;
|
|
11
|
-
|
|
12
|
-
/** Minimum query length */
|
|
13
|
-
export const MIN_QUERY_LENGTH = 3;
|
|
8
|
+
import { MAX_QUERY_LENGTH, MIN_QUERY_LENGTH } from '../constants.ts';
|
|
14
9
|
|
|
15
10
|
/**
|
|
16
11
|
* Validate research query
|
|
@@ -36,16 +31,19 @@ export function validateQuery(query: string): { isValid: boolean; error?: string
|
|
|
36
31
|
return { isValid: false, error: `Query must be at most ${MAX_QUERY_LENGTH} characters` };
|
|
37
32
|
}
|
|
38
33
|
|
|
39
|
-
// Check for
|
|
40
|
-
if (
|
|
41
|
-
return { isValid: false, error: 'Query cannot be
|
|
34
|
+
// Check for whitespace-only queries
|
|
35
|
+
if (/^\s+$/.test(query)) {
|
|
36
|
+
return { isValid: false, error: 'Query cannot be whitespace-only' };
|
|
42
37
|
}
|
|
43
38
|
|
|
44
39
|
// Check for potentially dangerous patterns
|
|
45
40
|
const dangerousPatterns = [
|
|
46
41
|
/<script/i,
|
|
47
42
|
/javascript:/i,
|
|
48
|
-
/on\w+\s*=/i, // Event handlers like onclick=
|
|
43
|
+
/(?<!\w)on\w+\s*=/i, // Event handlers like onclick= — a non-word boundary before `on` rejects
|
|
44
|
+
// false positives ("pokemon2=", "season2=") while still catching tag-internal
|
|
45
|
+
// slash-delimited handlers ("<svg/onload=", "<img/onerror=") the old [\s"'<]
|
|
46
|
+
// class missed (it omitted `/`).
|
|
49
47
|
/<iframe/i,
|
|
50
48
|
/<object/i,
|
|
51
49
|
/<embed/i,
|
package/src/utils/json-utils.ts
CHANGED
|
@@ -54,11 +54,69 @@ export function extractJsonFromCodeBlocks<T = unknown>(
|
|
|
54
54
|
};
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Walk forward from `start` tracking brace depth, respecting JSON string literals.
|
|
59
|
+
* Returns the index of the matching closing `}`, or -1 if not found.
|
|
60
|
+
* Also returns whether the walk ended while inside a string literal.
|
|
61
|
+
*/
|
|
62
|
+
function findMatchingBracket(text: string, start: number): { index: number; inString: boolean } {
|
|
63
|
+
let depth = 0;
|
|
64
|
+
let inString = false;
|
|
65
|
+
let escaped = false;
|
|
66
|
+
const open = text[start]!;
|
|
67
|
+
const close = open === '{' ? '}' : ']';
|
|
68
|
+
|
|
69
|
+
for (let i = start; i < text.length; i++) {
|
|
70
|
+
const ch = text[i]!;
|
|
71
|
+
if (escaped) { escaped = false; continue; }
|
|
72
|
+
if (ch === '\\' && inString) { escaped = true; continue; }
|
|
73
|
+
if (ch === '"') { inString = !inString; continue; }
|
|
74
|
+
if (inString) continue;
|
|
75
|
+
if (ch === open) depth++;
|
|
76
|
+
else if (ch === close) {
|
|
77
|
+
depth--;
|
|
78
|
+
if (depth === 0) return { index: i, inString: false };
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return { index: -1, inString };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Deterministically repair common JSON formation errors:
|
|
86
|
+
* 1. Replace smart quotes (“, ”, ‘, ’) with standard quotes.
|
|
87
|
+
* 2. Remove trailing commas before closing braces/brackets.
|
|
88
|
+
*/
|
|
89
|
+
function preRepairJson(jsonStr: string): string {
|
|
90
|
+
return jsonStr
|
|
91
|
+
// 1. Replace smart quotes
|
|
92
|
+
.replace(/[\u201C\u201D]/g, '"')
|
|
93
|
+
.replace(/[\u2018\u2019]/g, "'")
|
|
94
|
+
// 2. Remove trailing commas (only if they are followed by } or ] while NOT in a string)
|
|
95
|
+
// This is a bit tricky with regex but works for most common cases
|
|
96
|
+
.replace(/,\s*([}\]])/g, '$1');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Parse a JSON slice, trying it verbatim first and only running preRepairJson on
|
|
101
|
+
* failure. preRepairJson rewrites smart quotes and trailing commas even inside
|
|
102
|
+
* string values, so applying it unconditionally would corrupt otherwise-valid JSON
|
|
103
|
+
* whose content legitimately contains curly quotes (common when a summary quotes
|
|
104
|
+
* web text). Raw-first keeps the happy path lossless; repair is the fallback.
|
|
105
|
+
*/
|
|
106
|
+
function parseRawThenRepair(jsonStr: string): unknown {
|
|
107
|
+
try {
|
|
108
|
+
return JSON.parse(jsonStr);
|
|
109
|
+
} catch {
|
|
110
|
+
return JSON.parse(preRepairJson(jsonStr));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
57
114
|
/**
|
|
58
115
|
* Extract JSON object from raw text
|
|
59
116
|
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
117
|
+
* Finds the first `{` then walks forward with depth-tracking (respecting string
|
|
118
|
+
* literals) to locate its matching `}`. This correctly handles text that contains
|
|
119
|
+
* multiple JSON objects or has trailing content after the object.
|
|
62
120
|
*
|
|
63
121
|
* @param text - Text to search for JSON object
|
|
64
122
|
* @returns Extraction result with parsed value or error
|
|
@@ -67,9 +125,8 @@ export function extractJsonObject<T = unknown>(
|
|
|
67
125
|
text: string
|
|
68
126
|
): JsonExtractionResult<T> {
|
|
69
127
|
const objStart = text.indexOf('{');
|
|
70
|
-
const objEnd = text.lastIndexOf('}');
|
|
71
128
|
|
|
72
|
-
if (objStart === -1
|
|
129
|
+
if (objStart === -1) {
|
|
73
130
|
return {
|
|
74
131
|
success: false,
|
|
75
132
|
value: undefined,
|
|
@@ -77,9 +134,38 @@ export function extractJsonObject<T = unknown>(
|
|
|
77
134
|
};
|
|
78
135
|
}
|
|
79
136
|
|
|
137
|
+
const { index: objEnd, inString } = findMatchingBracket(text, objStart);
|
|
138
|
+
|
|
139
|
+
if (objEnd === -1) {
|
|
140
|
+
// Attempt local repair for truncated JSON
|
|
141
|
+
logger.debug('[json-utils] JSON object truncated; attempting local repair');
|
|
142
|
+
let partialText = text.slice(objStart);
|
|
143
|
+
|
|
144
|
+
// If we were inside a string, we MUST close it first
|
|
145
|
+
if (inString) {
|
|
146
|
+
partialText += '"';
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
for (let i = 1; i <= 15; i++) {
|
|
150
|
+
try {
|
|
151
|
+
const candidate = preRepairJson(partialText + '}'.repeat(i));
|
|
152
|
+
const parsed = JSON.parse(candidate);
|
|
153
|
+
logger.debug(`[json-utils] JSON object truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing braces`);
|
|
154
|
+
return { success: true, value: parsed as T, method: 'raw-object' };
|
|
155
|
+
} catch {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
return {
|
|
161
|
+
success: false,
|
|
162
|
+
value: undefined,
|
|
163
|
+
error: 'No matching closing brace found and local repair failed',
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
80
167
|
try {
|
|
81
|
-
const
|
|
82
|
-
const parsed = JSON.parse(jsonStr);
|
|
168
|
+
const parsed = parseRawThenRepair(text.slice(objStart, objEnd + 1));
|
|
83
169
|
return { success: true, value: parsed as T, method: 'raw-object' };
|
|
84
170
|
} catch (err) {
|
|
85
171
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -103,19 +189,48 @@ export function extractJsonObject<T = unknown>(
|
|
|
103
189
|
export function extractJsonArray<T = unknown>(
|
|
104
190
|
text: string
|
|
105
191
|
): JsonExtractionResult<T[]> {
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
192
|
+
const arrStart = text.indexOf('[');
|
|
193
|
+
|
|
194
|
+
if (arrStart === -1) {
|
|
195
|
+
return {
|
|
196
|
+
success: false,
|
|
197
|
+
value: undefined,
|
|
198
|
+
error: 'No JSON array boundaries found',
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const { index: arrEnd, inString } = findMatchingBracket(text, arrStart);
|
|
203
|
+
if (arrEnd === -1) {
|
|
204
|
+
// Attempt local repair for truncated JSON
|
|
205
|
+
logger.debug('[json-utils] JSON array truncated; attempting local repair');
|
|
206
|
+
let partialText = text.slice(arrStart);
|
|
207
|
+
|
|
208
|
+
if (inString) {
|
|
209
|
+
partialText += '"';
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
for (let i = 1; i <= 15; i++) {
|
|
213
|
+
try {
|
|
214
|
+
const candidate = preRepairJson(partialText + ']'.repeat(i));
|
|
215
|
+
const parsed = JSON.parse(candidate);
|
|
216
|
+
if (Array.isArray(parsed)) {
|
|
217
|
+
logger.debug(`[json-utils] JSON array truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing brackets`);
|
|
218
|
+
return { success: true, value: parsed as T[], method: 'raw-array' };
|
|
219
|
+
}
|
|
220
|
+
} catch {
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
110
225
|
return {
|
|
111
226
|
success: false,
|
|
112
227
|
value: undefined,
|
|
113
|
-
error: '
|
|
228
|
+
error: 'No matching closing bracket found and local repair failed',
|
|
114
229
|
};
|
|
115
230
|
}
|
|
116
231
|
|
|
117
232
|
try {
|
|
118
|
-
const parsed =
|
|
233
|
+
const parsed = parseRawThenRepair(text.slice(arrStart, arrEnd + 1));
|
|
119
234
|
if (!Array.isArray(parsed)) {
|
|
120
235
|
return {
|
|
121
236
|
success: false,
|
|
@@ -138,9 +253,10 @@ export function extractJsonArray<T = unknown>(
|
|
|
138
253
|
* Extract JSON with fallback chain
|
|
139
254
|
*
|
|
140
255
|
* Tries multiple extraction methods in order:
|
|
141
|
-
* 1.
|
|
142
|
-
* 2.
|
|
143
|
-
* 3. Raw JSON
|
|
256
|
+
* 1. Direct parse of the whole trimmed string (the payload IS the JSON)
|
|
257
|
+
* 2. Code blocks
|
|
258
|
+
* 3. Raw JSON object (for objects)
|
|
259
|
+
* 4. Raw JSON array (for arrays)
|
|
144
260
|
*
|
|
145
261
|
* @param text - Text to extract JSON from
|
|
146
262
|
* @param targetType - Whether to expect 'object', 'array', or 'any'
|
|
@@ -150,7 +266,25 @@ export function extractJson<T = unknown>(
|
|
|
150
266
|
text: string,
|
|
151
267
|
targetType: 'object' | 'array' | 'any' = 'any'
|
|
152
268
|
): JsonExtractionResult<T> {
|
|
153
|
-
//
|
|
269
|
+
// Direct parse first: when the entire response is the JSON payload, a fenced
|
|
270
|
+
// ```json example embedded INSIDE one of its string values must not win —
|
|
271
|
+
// code-block-first extraction would latch onto that example and return it
|
|
272
|
+
// instead of the real payload.
|
|
273
|
+
try {
|
|
274
|
+
const direct = JSON.parse(text.trim());
|
|
275
|
+
const isArray = Array.isArray(direct);
|
|
276
|
+
const typeMatches =
|
|
277
|
+
targetType === 'any' ||
|
|
278
|
+
(targetType === 'array' ? isArray : !isArray && direct !== null && typeof direct === 'object');
|
|
279
|
+
if (typeMatches && direct !== null && typeof direct === 'object') {
|
|
280
|
+
logger.debug('[json-utils] Extracted JSON via direct parse of full text');
|
|
281
|
+
return { success: true, value: direct as T, method: isArray ? 'raw-array' : 'raw-object' };
|
|
282
|
+
}
|
|
283
|
+
} catch {
|
|
284
|
+
// Not a pure-JSON payload — fall through to the extraction chain.
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Try code blocks next (most reliable for mixed prose+JSON responses)
|
|
154
288
|
const codeBlockResult = extractJsonFromCodeBlocks<T>(text);
|
|
155
289
|
if (codeBlockResult.success) {
|
|
156
290
|
logger.debug('[json-utils] Extracted JSON from code block');
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified LLM usage accounting.
|
|
3
|
+
*
|
|
4
|
+
* Every billed LLM call in a research run must record its token/cost the same way,
|
|
5
|
+
* to the same run-scoped `metrics` counters (`llm_tokens_total` / `llm_cost_total`)
|
|
6
|
+
* and — when a run observer is present — to the live `onTokensConsumed` event. Those
|
|
7
|
+
* counters are the single source of truth the tool layer reports (see
|
|
8
|
+
* research-tool-definition.ts → extractRunStats). Prior to this helper the same
|
|
9
|
+
* extract→increment→emit block was hand-copied at each call site, and drift (a call
|
|
10
|
+
* site that counted metrics but forgot the observer, or an agentic-repair pass that
|
|
11
|
+
* counted nothing at all) silently under-reported cost. Route every call site here.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { Model } from '@earendil-works/pi-ai';
|
|
15
|
+
import { extractUsage } from '../types/llm.ts';
|
|
16
|
+
import { metrics } from './metrics.ts';
|
|
17
|
+
|
|
18
|
+
/** Structural subset of the run observer — just the token sink. Kept structural (rather
|
|
19
|
+
* than importing ResearchObserver from core/) so this foundation-layer helper does not
|
|
20
|
+
* depend on an upper layer; the full ResearchObserver satisfies it. */
|
|
21
|
+
export interface TokenSink {
|
|
22
|
+
onTokensConsumed?: (tokens: number, cost: number) => void;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface RecordUsageOptions {
|
|
26
|
+
/** Metrics `component` label (e.g. 'coordinator', 'evaluator', 'researcher'). */
|
|
27
|
+
component: string;
|
|
28
|
+
/** Optional complexity label, stringified for the metric labels. */
|
|
29
|
+
complexity?: number | string;
|
|
30
|
+
/** Live run observer; receives onTokensConsumed for the interactive cost meter. */
|
|
31
|
+
observer?: TokenSink | null;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Extract usage from a raw LLM response usage object and record it to the active
|
|
36
|
+
* run-scoped metrics registry (+ the observer, if any). Zero/absent usage is a no-op.
|
|
37
|
+
* Returns the extracted { tokens, cost } for callers that also need the value.
|
|
38
|
+
*/
|
|
39
|
+
export function recordLlmUsage(
|
|
40
|
+
model: Model<any>,
|
|
41
|
+
rawUsage: unknown,
|
|
42
|
+
opts: RecordUsageOptions
|
|
43
|
+
): { tokens: number; cost: number } {
|
|
44
|
+
if (!rawUsage) return { tokens: 0, cost: 0 };
|
|
45
|
+
const { tokens, cost } = extractUsage(model, rawUsage);
|
|
46
|
+
if (tokens > 0 || cost > 0) {
|
|
47
|
+
const labels: Record<string, string> = { component: opts.component };
|
|
48
|
+
if (opts.complexity !== undefined) labels['complexity'] = String(opts.complexity);
|
|
49
|
+
metrics.increment('llm_tokens_total', tokens, labels);
|
|
50
|
+
metrics.increment('llm_cost_total', cost, labels);
|
|
51
|
+
opts.observer?.onTokensConsumed?.(tokens, cost);
|
|
52
|
+
}
|
|
53
|
+
return { tokens, cost };
|
|
54
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Log Rotation
|
|
3
|
+
*
|
|
4
|
+
* Handles log file rotation and cleanup.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import * as fs from 'node:fs';
|
|
8
|
+
import * as path from 'node:path';
|
|
9
|
+
export interface RotationLogger {
|
|
10
|
+
log(...args: unknown[]): void;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export class LogRotation {
|
|
14
|
+
private readonly MAX_LOG_SIZE = 10 * 1024 * 1024; // 10MB max file size
|
|
15
|
+
private readonly MAX_LOG_FILES = 10; // Keep last 10 archived logs
|
|
16
|
+
private lastRotationCheck: number = 0;
|
|
17
|
+
private readonly logger: RotationLogger;
|
|
18
|
+
|
|
19
|
+
constructor(logger: RotationLogger) {
|
|
20
|
+
this.logger = logger;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Clear all research logs including archives.
|
|
25
|
+
*/
|
|
26
|
+
clearLogs(logFile: string, logDir: string): void {
|
|
27
|
+
try {
|
|
28
|
+
if (fs.existsSync(logFile)) {
|
|
29
|
+
fs.unlinkSync(logFile);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const files = fs.readdirSync(logDir);
|
|
33
|
+
const baseName = path.basename(logFile);
|
|
34
|
+
const archives = files.filter(f => f.startsWith(baseName) && f !== baseName);
|
|
35
|
+
|
|
36
|
+
for (const archive of archives) {
|
|
37
|
+
try {
|
|
38
|
+
fs.unlinkSync(path.join(logDir, archive));
|
|
39
|
+
} catch { /* ignore */ }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
this.logger.log('[Logger] All logs and archives cleared.');
|
|
43
|
+
} catch (err) {
|
|
44
|
+
this.logger.log('[Logger] Failed to clear logs:', err);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Rotate log files when they exceed MAX_LOG_SIZE.
|
|
50
|
+
* Archives are created with ISO timestamp suffix.
|
|
51
|
+
* Old archives beyond MAX_LOG_FILES are cleaned up.
|
|
52
|
+
*/
|
|
53
|
+
rotateLogFile(logFile: string, logDir: string): void {
|
|
54
|
+
try {
|
|
55
|
+
const stats = fs.statSync(logFile);
|
|
56
|
+
const fileSize = stats.size;
|
|
57
|
+
|
|
58
|
+
if (fileSize <= this.MAX_LOG_SIZE) {
|
|
59
|
+
return; // No rotation needed
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Create archive filename with timestamp
|
|
63
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
64
|
+
const archivePath = `${logFile}.${timestamp}`;
|
|
65
|
+
|
|
66
|
+
// Rename current log file to archive
|
|
67
|
+
try {
|
|
68
|
+
fs.renameSync(logFile, archivePath);
|
|
69
|
+
} catch (renameErr) {
|
|
70
|
+
// fs.renameSync fails on Windows if target exists (NTFS). Fall back to copy+delete.
|
|
71
|
+
if (process.platform === 'win32') {
|
|
72
|
+
fs.copyFileSync(logFile, archivePath);
|
|
73
|
+
fs.unlinkSync(logFile);
|
|
74
|
+
} else {
|
|
75
|
+
throw renameErr;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Clean up old archives
|
|
80
|
+
try {
|
|
81
|
+
const files = fs.readdirSync(logDir);
|
|
82
|
+
const logFiles = files
|
|
83
|
+
.filter(f => f.startsWith(path.basename(logFile)) && f !== path.basename(logFile))
|
|
84
|
+
.sort(); // Sort by timestamp (oldest first)
|
|
85
|
+
|
|
86
|
+
// Remove excess archives
|
|
87
|
+
const toDelete = logFiles.slice(0, -this.MAX_LOG_FILES);
|
|
88
|
+
for (const file of toDelete) {
|
|
89
|
+
try {
|
|
90
|
+
fs.unlinkSync(path.join(logDir, file));
|
|
91
|
+
} catch {
|
|
92
|
+
// Ignore cleanup errors
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
} catch {
|
|
96
|
+
// Ignore cleanup errors
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
this.logger.log('[Logger] Rotated log file to:', archivePath);
|
|
100
|
+
} catch (_error) {
|
|
101
|
+
// Ignore rotation errors (file might not exist yet, etc.)
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Check if rotation is needed and perform if necessary.
|
|
107
|
+
*
|
|
108
|
+
* Throttled to one filesystem stat per 60s. We deliberately do NOT offer a
|
|
109
|
+
* `force` bypass: a previous version forced a check on every WARN/ERROR, which
|
|
110
|
+
* turned into a synchronous fs.statSync per line (thousands per heavy run) and
|
|
111
|
+
* also reset the throttle so the timer path never fired. The 60s timer
|
|
112
|
+
* preserves all log data on rotation regardless of when it lands — a file may
|
|
113
|
+
* transiently exceed MAX_LOG_SIZE by at most one check interval, which is fine.
|
|
114
|
+
* @returns true if a rotation check was performed
|
|
115
|
+
*/
|
|
116
|
+
checkAndRotate(logFile: string, logDir: string): boolean {
|
|
117
|
+
const now = Date.now();
|
|
118
|
+
if (now - this.lastRotationCheck > 60_000) {
|
|
119
|
+
this.rotateLogFile(logFile, logDir);
|
|
120
|
+
this.lastRotationCheck = now;
|
|
121
|
+
return true;
|
|
122
|
+
}
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Log Utilities
|
|
3
|
+
*
|
|
4
|
+
* Utilities for log configuration, context management, and formatting.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
8
|
+
import { randomBytes } from 'node:crypto';
|
|
9
|
+
import * as os from 'node:os';
|
|
10
|
+
import * as path from 'node:path';
|
|
11
|
+
|
|
12
|
+
export interface LogContext {
|
|
13
|
+
sessionId?: string;
|
|
14
|
+
sessionFile?: string;
|
|
15
|
+
cwd?: string;
|
|
16
|
+
researchRunId?: string;
|
|
17
|
+
toolName?: string;
|
|
18
|
+
phase?: string;
|
|
19
|
+
eventName?: string;
|
|
20
|
+
[key: string]: unknown;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const logContextStorage = new AsyncLocalStorage<LogContext>();
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Resolve the consolidated diagnostic log path.
|
|
27
|
+
* Defaults to {os.tmpdir()}/pi-research.log; overridable via PI_RESEARCH_LOG_PATH.
|
|
28
|
+
* All scopes (main process, researchers, embedding server, browser worker) write
|
|
29
|
+
* to this single file — logging is intentionally consolidated, not per-run-id.
|
|
30
|
+
*/
|
|
31
|
+
export function buildDefaultDebugLogPath(): string {
|
|
32
|
+
const override = process.env['PI_RESEARCH_LOG_PATH'];
|
|
33
|
+
if (override) return override;
|
|
34
|
+
return path.join(os.tmpdir(), 'pi-research.log');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Check whether debug/verbose logging is enabled.
|
|
39
|
+
*
|
|
40
|
+
* Reads PI_RESEARCH_DEBUG env var. This env var is kept in sync with
|
|
41
|
+
* config.DEBUG whenever the config is loaded or saved (see config.ts).
|
|
42
|
+
* Set to "true" to enable INFO+DEBUG logs.
|
|
43
|
+
*/
|
|
44
|
+
export function isVerboseFromEnv(): boolean {
|
|
45
|
+
return process.env['PI_RESEARCH_DEBUG'] === 'true';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function createResearchRunId(): string {
|
|
49
|
+
return `run-${randomBytes(4).toString('hex')}`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function getLogContext(): LogContext {
|
|
53
|
+
return logContextStorage.getStore() ?? {};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function runWithLogContext<T>(context: LogContext, callback: () => T): T {
|
|
57
|
+
const parent = logContextStorage.getStore() ?? {};
|
|
58
|
+
return logContextStorage.run({ ...parent, ...context }, callback);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function safeJsonStringify(value: unknown): string {
|
|
62
|
+
try {
|
|
63
|
+
return JSON.stringify(value, null, 0);
|
|
64
|
+
} catch {
|
|
65
|
+
return '[unserializable]';
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function formatArg(arg: unknown): string {
|
|
70
|
+
if (arg instanceof Error) {
|
|
71
|
+
return arg.stack ?? arg.message;
|
|
72
|
+
}
|
|
73
|
+
if (typeof arg === 'object' && arg !== null) {
|
|
74
|
+
return safeJsonStringify(arg);
|
|
75
|
+
}
|
|
76
|
+
return String(arg);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Upper bound on a single log message; defends the log file against being
|
|
80
|
+
* flooded by large network/scraped payloads passed to the logger. */
|
|
81
|
+
const MAX_LOG_MESSAGE_LENGTH = 10_000;
|
|
82
|
+
|
|
83
|
+
// ANSI escape sequences embedded in untrusted content (we add our own colours
|
|
84
|
+
// separately for the console). Matching the ESC control byte is the point here.
|
|
85
|
+
// eslint-disable-next-line no-control-regex
|
|
86
|
+
const ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
|
|
87
|
+
// Credentials embedded in a URL userinfo component: scheme://user:pass@host
|
|
88
|
+
const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
|
|
89
|
+
// key=value / key: "value" pairs whose key names a secret.
|
|
90
|
+
const SENSITIVE_KV_PATTERN =
|
|
91
|
+
/\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
|
|
92
|
+
// Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
|
|
93
|
+
const KNOWN_TOKEN_PATTERN =
|
|
94
|
+
/\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
|
|
95
|
+
// JSON Web Tokens — header.payload.signature, each a base64url segment.
|
|
96
|
+
const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
|
|
97
|
+
// HTTP Basic credentials: "Basic <base64>".
|
|
98
|
+
const BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
|
|
99
|
+
// HTTP Bearer credentials: "Bearer <token>". The token is often opaque — many
|
|
100
|
+
// providers (e.g. GLM/zhipuai) use dotted keys that are NOT JWTs and carry no
|
|
101
|
+
// known prefix, so KNOWN_TOKEN_PATTERN/JWT_PATTERN miss them and the KV pass
|
|
102
|
+
// only masks the word "Bearer" (its value class stops at the space). Match the
|
|
103
|
+
// scheme and redact the whole RFC 6750 b64token that follows, whatever its shape.
|
|
104
|
+
const BEARER_AUTH_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
|
|
105
|
+
// Provider keys not covered by KNOWN_TOKEN_PATTERN: Google API/OAuth, Stripe.
|
|
106
|
+
const PROVIDER_TOKEN_PATTERN =
|
|
107
|
+
/\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
|
|
108
|
+
// Long opaque hex secrets (>=32 hex chars), e.g. the 64-hex browser auth secret
|
|
109
|
+
// or MD5/SHA-style tokens. Diagnostic logs may over-redact content hashes of
|
|
110
|
+
// this length; that trade-off favors not persisting credentials clear-text.
|
|
111
|
+
const LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
|
|
112
|
+
// All C0 control characters plus DEL — newlines/carriage returns are the log/
|
|
113
|
+
// terminal-injection vector when written to a raw console line. Matching these
|
|
114
|
+
// control bytes is exactly the intent.
|
|
115
|
+
// eslint-disable-next-line no-control-regex
|
|
116
|
+
const CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Redact secrets and bound the size of a log message before it is persisted.
|
|
120
|
+
*
|
|
121
|
+
* Removes embedded ANSI sequences, masks credentials in URLs, sensitive
|
|
122
|
+
* key/value pairs, and known token formats, and truncates oversized payloads.
|
|
123
|
+
* Applied to every message before it reaches the log file or the console, so
|
|
124
|
+
* neither sink receives clear-text secrets or unbounded network data.
|
|
125
|
+
*/
|
|
126
|
+
export function redactSecrets(message: string): string {
|
|
127
|
+
let out = message.length > MAX_LOG_MESSAGE_LENGTH
|
|
128
|
+
? `${message.slice(0, MAX_LOG_MESSAGE_LENGTH)}…[truncated ${message.length - MAX_LOG_MESSAGE_LENGTH} chars]`
|
|
129
|
+
: message;
|
|
130
|
+
out = out.replace(ANSI_PATTERN, '');
|
|
131
|
+
out = out.replace(URL_CREDENTIALS_PATTERN, '$1[REDACTED]@');
|
|
132
|
+
// Redact whole-token formats BEFORE the key/value pass: the KV value class
|
|
133
|
+
// stops at whitespace, so "Authorization: Basic <b64>" would otherwise mask
|
|
134
|
+
// only the word "Basic" and leave the credential blob exposed.
|
|
135
|
+
out = out.replace(JWT_PATTERN, '[REDACTED]');
|
|
136
|
+
out = out.replace(BASIC_AUTH_PATTERN, 'Basic [REDACTED]');
|
|
137
|
+
// Before the KV pass, for the same reason as Basic above: the KV value class
|
|
138
|
+
// stops at the space after "Bearer", leaving an opaque token exposed.
|
|
139
|
+
out = out.replace(BEARER_AUTH_PATTERN, 'Bearer [REDACTED]');
|
|
140
|
+
out = out.replace(SENSITIVE_KV_PATTERN, (_m, key: string, sep: string) => `${key}${sep}[REDACTED]`);
|
|
141
|
+
out = out.replace(KNOWN_TOKEN_PATTERN, '[REDACTED]');
|
|
142
|
+
out = out.replace(PROVIDER_TOKEN_PATTERN, '[REDACTED]');
|
|
143
|
+
out = out.replace(LONG_HEX_SECRET_PATTERN, '[REDACTED]');
|
|
144
|
+
return out;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Neutralize control characters (newlines, carriage returns, etc.) so that
|
|
149
|
+
* untrusted content cannot forge or corrupt a raw console log line
|
|
150
|
+
* (log-injection defense). Use for sinks that are NOT already structurally
|
|
151
|
+
* escaped — the JSON log file escapes control chars on its own.
|
|
152
|
+
*/
|
|
153
|
+
export function neutralizeControlChars(message: string): string {
|
|
154
|
+
// Strip CR/LF explicitly first. These are the actual log-injection vector
|
|
155
|
+
// (forged/corrupted log lines) and an explicit line-break replacement is the
|
|
156
|
+
// form static analysis recognizes as a log-injection sanitizer. The second
|
|
157
|
+
// pass removes any remaining C0 control chars + DEL.
|
|
158
|
+
return message.replace(/\r\n|\r|\n/g, ' ').replace(CONTROL_CHARS_PATTERN, ' ');
|
|
159
|
+
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Both files use identical ResourceLoader implementations.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import type { ResourceLoader, ExtensionRuntime } from '@
|
|
8
|
+
import type { ResourceLoader, ExtensionRuntime } from '@earendil-works/pi-coding-agent';
|
|
9
9
|
|
|
10
10
|
export function makeResourceLoader(systemPromptText: string): ResourceLoader {
|
|
11
11
|
const mockRuntime: ExtensionRuntime = {
|
|
@@ -27,6 +27,8 @@ export function makeResourceLoader(systemPromptText: string): ResourceLoader {
|
|
|
27
27
|
setModel: async () => true,
|
|
28
28
|
getThinkingLevel: () => 'off',
|
|
29
29
|
setThinkingLevel: () => {},
|
|
30
|
+
assertActive: () => {},
|
|
31
|
+
invalidate: () => {},
|
|
30
32
|
};
|
|
31
33
|
|
|
32
34
|
return {
|