@lincoln504/pi-research 0.1.12 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +64 -115
  3. package/dist/cli.mjs +23211 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +102 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +213 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +113 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +64 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1044 -0
  30. package/src/config.ts +1036 -147
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +264 -7
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -174
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -577
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
@@ -5,12 +5,7 @@
5
5
  */
6
6
 
7
7
  import { logger } from '../logger.ts';
8
-
9
- /** Maximum query length */
10
- export const MAX_QUERY_LENGTH = 500;
11
-
12
- /** Minimum query length */
13
- export const MIN_QUERY_LENGTH = 3;
8
+ import { MAX_QUERY_LENGTH, MIN_QUERY_LENGTH } from '../constants.ts';
14
9
 
15
10
  /**
16
11
  * Validate research query
@@ -36,16 +31,19 @@ export function validateQuery(query: string): { isValid: boolean; error?: string
36
31
  return { isValid: false, error: `Query must be at most ${MAX_QUERY_LENGTH} characters` };
37
32
  }
38
33
 
39
- // Check for empty or whitespace-only queries
40
- if (trimmed === '') {
41
- return { isValid: false, error: 'Query cannot be empty or whitespace-only' };
34
+ // Check for whitespace-only queries
35
+ if (/^\s+$/.test(query)) {
36
+ return { isValid: false, error: 'Query cannot be whitespace-only' };
42
37
  }
43
38
 
44
39
  // Check for potentially dangerous patterns
45
40
  const dangerousPatterns = [
46
41
  /<script/i,
47
42
  /javascript:/i,
48
- /on\w+\s*=/i, // Event handlers like onclick=
43
+ /(?<!\w)on\w+\s*=/i, // Event handlers like onclick= — a non-word boundary before `on` rejects
44
+ // false positives ("pokemon2=", "season2=") while still catching tag-internal
45
+ // slash-delimited handlers ("<svg/onload=", "<img/onerror=") the old [\s"'<]
46
+ // class missed (it omitted `/`).
49
47
  /<iframe/i,
50
48
  /<object/i,
51
49
  /<embed/i,
@@ -54,11 +54,69 @@ export function extractJsonFromCodeBlocks<T = unknown>(
54
54
  };
55
55
  }
56
56
 
57
+ /**
58
+ * Walk forward from `start` tracking brace depth, respecting JSON string literals.
59
+ * Returns the index of the matching closing `}`, or -1 if not found.
60
+ * Also returns whether the walk ended while inside a string literal.
61
+ */
62
+ function findMatchingBracket(text: string, start: number): { index: number; inString: boolean } {
63
+ let depth = 0;
64
+ let inString = false;
65
+ let escaped = false;
66
+ const open = text[start]!;
67
+ const close = open === '{' ? '}' : ']';
68
+
69
+ for (let i = start; i < text.length; i++) {
70
+ const ch = text[i]!;
71
+ if (escaped) { escaped = false; continue; }
72
+ if (ch === '\\' && inString) { escaped = true; continue; }
73
+ if (ch === '"') { inString = !inString; continue; }
74
+ if (inString) continue;
75
+ if (ch === open) depth++;
76
+ else if (ch === close) {
77
+ depth--;
78
+ if (depth === 0) return { index: i, inString: false };
79
+ }
80
+ }
81
+ return { index: -1, inString };
82
+ }
83
+
84
+ /**
85
+ * Deterministically repair common JSON formation errors:
86
+ * 1. Replace smart quotes (“, ”, ‘, ’) with standard quotes.
87
+ * 2. Remove trailing commas before closing braces/brackets.
88
+ */
89
+ function preRepairJson(jsonStr: string): string {
90
+ return jsonStr
91
+ // 1. Replace smart quotes
92
+ .replace(/[\u201C\u201D]/g, '"')
93
+ .replace(/[\u2018\u2019]/g, "'")
94
+ // 2. Remove trailing commas (only if they are followed by } or ] while NOT in a string)
95
+ // This is a bit tricky with regex but works for most common cases
96
+ .replace(/,\s*([}\]])/g, '$1');
97
+ }
98
+
99
+ /**
100
+ * Parse a JSON slice, trying it verbatim first and only running preRepairJson on
101
+ * failure. preRepairJson rewrites smart quotes and trailing commas even inside
102
+ * string values, so applying it unconditionally would corrupt otherwise-valid JSON
103
+ * whose content legitimately contains curly quotes (common when a summary quotes
104
+ * web text). Raw-first keeps the happy path lossless; repair is the fallback.
105
+ */
106
+ function parseRawThenRepair(jsonStr: string): unknown {
107
+ try {
108
+ return JSON.parse(jsonStr);
109
+ } catch {
110
+ return JSON.parse(preRepairJson(jsonStr));
111
+ }
112
+ }
113
+
57
114
  /**
58
115
  * Extract JSON object from raw text
59
116
  *
60
- * Looks for a JSON object spanning from the first `{` to the last `}`.
61
- * Useful when JSON is embedded in prose without code blocks.
117
+ * Finds the first `{` then walks forward with depth-tracking (respecting string
118
+ * literals) to locate its matching `}`. This correctly handles text that contains
119
+ * multiple JSON objects or has trailing content after the object.
62
120
  *
63
121
  * @param text - Text to search for JSON object
64
122
  * @returns Extraction result with parsed value or error
@@ -67,9 +125,8 @@ export function extractJsonObject<T = unknown>(
67
125
  text: string
68
126
  ): JsonExtractionResult<T> {
69
127
  const objStart = text.indexOf('{');
70
- const objEnd = text.lastIndexOf('}');
71
128
 
72
- if (objStart === -1 || objEnd <= objStart) {
129
+ if (objStart === -1) {
73
130
  return {
74
131
  success: false,
75
132
  value: undefined,
@@ -77,9 +134,38 @@ export function extractJsonObject<T = unknown>(
77
134
  };
78
135
  }
79
136
 
137
+ const { index: objEnd, inString } = findMatchingBracket(text, objStart);
138
+
139
+ if (objEnd === -1) {
140
+ // Attempt local repair for truncated JSON
141
+ logger.debug('[json-utils] JSON object truncated; attempting local repair');
142
+ let partialText = text.slice(objStart);
143
+
144
+ // If we were inside a string, we MUST close it first
145
+ if (inString) {
146
+ partialText += '"';
147
+ }
148
+
149
+ for (let i = 1; i <= 15; i++) {
150
+ try {
151
+ const candidate = preRepairJson(partialText + '}'.repeat(i));
152
+ const parsed = JSON.parse(candidate);
153
+ logger.debug(`[json-utils] JSON object truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing braces`);
154
+ return { success: true, value: parsed as T, method: 'raw-object' };
155
+ } catch {
156
+ continue;
157
+ }
158
+ }
159
+
160
+ return {
161
+ success: false,
162
+ value: undefined,
163
+ error: 'No matching closing brace found and local repair failed',
164
+ };
165
+ }
166
+
80
167
  try {
81
- const jsonStr = text.slice(objStart, objEnd + 1);
82
- const parsed = JSON.parse(jsonStr);
168
+ const parsed = parseRawThenRepair(text.slice(objStart, objEnd + 1));
83
169
  return { success: true, value: parsed as T, method: 'raw-object' };
84
170
  } catch (err) {
85
171
  const errorMsg = err instanceof Error ? err.message : String(err);
@@ -103,19 +189,48 @@ export function extractJsonObject<T = unknown>(
103
189
  export function extractJsonArray<T = unknown>(
104
190
  text: string
105
191
  ): JsonExtractionResult<T[]> {
106
- const stripped = text.trim();
107
-
108
- // Quick check if it's a JSON array
109
- if (!stripped.startsWith('[') || !stripped.endsWith(']')) {
192
+ const arrStart = text.indexOf('[');
193
+
194
+ if (arrStart === -1) {
195
+ return {
196
+ success: false,
197
+ value: undefined,
198
+ error: 'No JSON array boundaries found',
199
+ };
200
+ }
201
+
202
+ const { index: arrEnd, inString } = findMatchingBracket(text, arrStart);
203
+ if (arrEnd === -1) {
204
+ // Attempt local repair for truncated JSON
205
+ logger.debug('[json-utils] JSON array truncated; attempting local repair');
206
+ let partialText = text.slice(arrStart);
207
+
208
+ if (inString) {
209
+ partialText += '"';
210
+ }
211
+
212
+ for (let i = 1; i <= 15; i++) {
213
+ try {
214
+ const candidate = preRepairJson(partialText + ']'.repeat(i));
215
+ const parsed = JSON.parse(candidate);
216
+ if (Array.isArray(parsed)) {
217
+ logger.debug(`[json-utils] JSON array truncated; salvaged by appending ${inString ? 'quote and ' : ''}${i} closing brackets`);
218
+ return { success: true, value: parsed as T[], method: 'raw-array' };
219
+ }
220
+ } catch {
221
+ continue;
222
+ }
223
+ }
224
+
110
225
  return {
111
226
  success: false,
112
227
  value: undefined,
113
- error: 'Text is not a JSON array',
228
+ error: 'No matching closing bracket found and local repair failed',
114
229
  };
115
230
  }
116
231
 
117
232
  try {
118
- const parsed = JSON.parse(stripped);
233
+ const parsed = parseRawThenRepair(text.slice(arrStart, arrEnd + 1));
119
234
  if (!Array.isArray(parsed)) {
120
235
  return {
121
236
  success: false,
@@ -138,9 +253,10 @@ export function extractJsonArray<T = unknown>(
138
253
  * Extract JSON with fallback chain
139
254
  *
140
255
  * Tries multiple extraction methods in order:
141
- * 1. Code blocks (most reliable)
142
- * 2. Raw JSON object (for objects)
143
- * 3. Raw JSON array (for arrays)
256
+ * 1. Direct parse of the whole trimmed string (the payload IS the JSON)
257
+ * 2. Code blocks
258
+ * 3. Raw JSON object (for objects)
259
+ * 4. Raw JSON array (for arrays)
144
260
  *
145
261
  * @param text - Text to extract JSON from
146
262
  * @param targetType - Whether to expect 'object', 'array', or 'any'
@@ -150,7 +266,25 @@ export function extractJson<T = unknown>(
150
266
  text: string,
151
267
  targetType: 'object' | 'array' | 'any' = 'any'
152
268
  ): JsonExtractionResult<T> {
153
- // Try code blocks first (most reliable)
269
+ // Direct parse first: when the entire response is the JSON payload, a fenced
270
+ // ```json example embedded INSIDE one of its string values must not win —
271
+ // code-block-first extraction would latch onto that example and return it
272
+ // instead of the real payload.
273
+ try {
274
+ const direct = JSON.parse(text.trim());
275
+ const isArray = Array.isArray(direct);
276
+ const typeMatches =
277
+ targetType === 'any' ||
278
+ (targetType === 'array' ? isArray : !isArray && direct !== null && typeof direct === 'object');
279
+ if (typeMatches && direct !== null && typeof direct === 'object') {
280
+ logger.debug('[json-utils] Extracted JSON via direct parse of full text');
281
+ return { success: true, value: direct as T, method: isArray ? 'raw-array' : 'raw-object' };
282
+ }
283
+ } catch {
284
+ // Not a pure-JSON payload — fall through to the extraction chain.
285
+ }
286
+
287
+ // Try code blocks next (most reliable for mixed prose+JSON responses)
154
288
  const codeBlockResult = extractJsonFromCodeBlocks<T>(text);
155
289
  if (codeBlockResult.success) {
156
290
  logger.debug('[json-utils] Extracted JSON from code block');
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Unified LLM usage accounting.
3
+ *
4
+ * Every billed LLM call in a research run must record its token/cost the same way,
5
+ * to the same run-scoped `metrics` counters (`llm_tokens_total` / `llm_cost_total`)
6
+ * and — when a run observer is present — to the live `onTokensConsumed` event. Those
7
+ * counters are the single source of truth the tool layer reports (see
8
+ * research-tool-definition.ts → extractRunStats). Prior to this helper the same
9
+ * extract→increment→emit block was hand-copied at each call site, and drift (a call
10
+ * site that counted metrics but forgot the observer, or an agentic-repair pass that
11
+ * counted nothing at all) silently under-reported cost. Route every call site here.
12
+ */
13
+
14
+ import type { Model } from '@earendil-works/pi-ai';
15
+ import { extractUsage } from '../types/llm.ts';
16
+ import { metrics } from './metrics.ts';
17
+
18
+ /** Structural subset of the run observer — just the token sink. Kept structural (rather
19
+ * than importing ResearchObserver from core/) so this foundation-layer helper does not
20
+ * depend on an upper layer; the full ResearchObserver satisfies it. */
21
+ export interface TokenSink {
22
+ onTokensConsumed?: (tokens: number, cost: number) => void;
23
+ }
24
+
25
+ export interface RecordUsageOptions {
26
+ /** Metrics `component` label (e.g. 'coordinator', 'evaluator', 'researcher'). */
27
+ component: string;
28
+ /** Optional complexity label, stringified for the metric labels. */
29
+ complexity?: number | string;
30
+ /** Live run observer; receives onTokensConsumed for the interactive cost meter. */
31
+ observer?: TokenSink | null;
32
+ }
33
+
34
+ /**
35
+ * Extract usage from a raw LLM response usage object and record it to the active
36
+ * run-scoped metrics registry (+ the observer, if any). Zero/absent usage is a no-op.
37
+ * Returns the extracted { tokens, cost } for callers that also need the value.
38
+ */
39
+ export function recordLlmUsage(
40
+ model: Model<any>,
41
+ rawUsage: unknown,
42
+ opts: RecordUsageOptions
43
+ ): { tokens: number; cost: number } {
44
+ if (!rawUsage) return { tokens: 0, cost: 0 };
45
+ const { tokens, cost } = extractUsage(model, rawUsage);
46
+ if (tokens > 0 || cost > 0) {
47
+ const labels: Record<string, string> = { component: opts.component };
48
+ if (opts.complexity !== undefined) labels['complexity'] = String(opts.complexity);
49
+ metrics.increment('llm_tokens_total', tokens, labels);
50
+ metrics.increment('llm_cost_total', cost, labels);
51
+ opts.observer?.onTokensConsumed?.(tokens, cost);
52
+ }
53
+ return { tokens, cost };
54
+ }
@@ -0,0 +1,125 @@
1
+ /**
2
+ * Log Rotation
3
+ *
4
+ * Handles log file rotation and cleanup.
5
+ */
6
+
7
+ import * as fs from 'node:fs';
8
+ import * as path from 'node:path';
9
+ export interface RotationLogger {
10
+ log(...args: unknown[]): void;
11
+ }
12
+
13
+ export class LogRotation {
14
+ private readonly MAX_LOG_SIZE = 10 * 1024 * 1024; // 10MB max file size
15
+ private readonly MAX_LOG_FILES = 10; // Keep last 10 archived logs
16
+ private lastRotationCheck: number = 0;
17
+ private readonly logger: RotationLogger;
18
+
19
+ constructor(logger: RotationLogger) {
20
+ this.logger = logger;
21
+ }
22
+
23
+ /**
24
+ * Clear all research logs including archives.
25
+ */
26
+ clearLogs(logFile: string, logDir: string): void {
27
+ try {
28
+ if (fs.existsSync(logFile)) {
29
+ fs.unlinkSync(logFile);
30
+ }
31
+
32
+ const files = fs.readdirSync(logDir);
33
+ const baseName = path.basename(logFile);
34
+ const archives = files.filter(f => f.startsWith(baseName) && f !== baseName);
35
+
36
+ for (const archive of archives) {
37
+ try {
38
+ fs.unlinkSync(path.join(logDir, archive));
39
+ } catch { /* ignore */ }
40
+ }
41
+
42
+ this.logger.log('[Logger] All logs and archives cleared.');
43
+ } catch (err) {
44
+ this.logger.log('[Logger] Failed to clear logs:', err);
45
+ }
46
+ }
47
+
48
+ /**
49
+ * Rotate log files when they exceed MAX_LOG_SIZE.
50
+ * Archives are created with ISO timestamp suffix.
51
+ * Old archives beyond MAX_LOG_FILES are cleaned up.
52
+ */
53
+ rotateLogFile(logFile: string, logDir: string): void {
54
+ try {
55
+ const stats = fs.statSync(logFile);
56
+ const fileSize = stats.size;
57
+
58
+ if (fileSize <= this.MAX_LOG_SIZE) {
59
+ return; // No rotation needed
60
+ }
61
+
62
+ // Create archive filename with timestamp
63
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
64
+ const archivePath = `${logFile}.${timestamp}`;
65
+
66
+ // Rename current log file to archive
67
+ try {
68
+ fs.renameSync(logFile, archivePath);
69
+ } catch (renameErr) {
70
+ // fs.renameSync fails on Windows if target exists (NTFS). Fall back to copy+delete.
71
+ if (process.platform === 'win32') {
72
+ fs.copyFileSync(logFile, archivePath);
73
+ fs.unlinkSync(logFile);
74
+ } else {
75
+ throw renameErr;
76
+ }
77
+ }
78
+
79
+ // Clean up old archives
80
+ try {
81
+ const files = fs.readdirSync(logDir);
82
+ const logFiles = files
83
+ .filter(f => f.startsWith(path.basename(logFile)) && f !== path.basename(logFile))
84
+ .sort(); // Sort by timestamp (oldest first)
85
+
86
+ // Remove excess archives
87
+ const toDelete = logFiles.slice(0, -this.MAX_LOG_FILES);
88
+ for (const file of toDelete) {
89
+ try {
90
+ fs.unlinkSync(path.join(logDir, file));
91
+ } catch {
92
+ // Ignore cleanup errors
93
+ }
94
+ }
95
+ } catch {
96
+ // Ignore cleanup errors
97
+ }
98
+
99
+ this.logger.log('[Logger] Rotated log file to:', archivePath);
100
+ } catch (_error) {
101
+ // Ignore rotation errors (file might not exist yet, etc.)
102
+ }
103
+ }
104
+
105
+ /**
106
+ * Check if rotation is needed and perform if necessary.
107
+ *
108
+ * Throttled to one filesystem stat per 60s. We deliberately do NOT offer a
109
+ * `force` bypass: a previous version forced a check on every WARN/ERROR, which
110
+ * turned into a synchronous fs.statSync per line (thousands per heavy run) and
111
+ * also reset the throttle so the timer path never fired. The 60s timer
112
+ * preserves all log data on rotation regardless of when it lands — a file may
113
+ * transiently exceed MAX_LOG_SIZE by at most one check interval, which is fine.
114
+ * @returns true if a rotation check was performed
115
+ */
116
+ checkAndRotate(logFile: string, logDir: string): boolean {
117
+ const now = Date.now();
118
+ if (now - this.lastRotationCheck > 60_000) {
119
+ this.rotateLogFile(logFile, logDir);
120
+ this.lastRotationCheck = now;
121
+ return true;
122
+ }
123
+ return false;
124
+ }
125
+ }
@@ -0,0 +1,159 @@
1
+ /**
2
+ * Log Utilities
3
+ *
4
+ * Utilities for log configuration, context management, and formatting.
5
+ */
6
+
7
+ import { AsyncLocalStorage } from 'node:async_hooks';
8
+ import { randomBytes } from 'node:crypto';
9
+ import * as os from 'node:os';
10
+ import * as path from 'node:path';
11
+
12
+ export interface LogContext {
13
+ sessionId?: string;
14
+ sessionFile?: string;
15
+ cwd?: string;
16
+ researchRunId?: string;
17
+ toolName?: string;
18
+ phase?: string;
19
+ eventName?: string;
20
+ [key: string]: unknown;
21
+ }
22
+
23
+ const logContextStorage = new AsyncLocalStorage<LogContext>();
24
+
25
+ /**
26
+ * Resolve the consolidated diagnostic log path.
27
+ * Defaults to {os.tmpdir()}/pi-research.log; overridable via PI_RESEARCH_LOG_PATH.
28
+ * All scopes (main process, researchers, embedding server, browser worker) write
29
+ * to this single file — logging is intentionally consolidated, not per-run-id.
30
+ */
31
+ export function buildDefaultDebugLogPath(): string {
32
+ const override = process.env['PI_RESEARCH_LOG_PATH'];
33
+ if (override) return override;
34
+ return path.join(os.tmpdir(), 'pi-research.log');
35
+ }
36
+
37
+ /**
38
+ * Check whether debug/verbose logging is enabled.
39
+ *
40
+ * Reads PI_RESEARCH_DEBUG env var. This env var is kept in sync with
41
+ * config.DEBUG whenever the config is loaded or saved (see config.ts).
42
+ * Set to "true" to enable INFO+DEBUG logs.
43
+ */
44
+ export function isVerboseFromEnv(): boolean {
45
+ return process.env['PI_RESEARCH_DEBUG'] === 'true';
46
+ }
47
+
48
+ export function createResearchRunId(): string {
49
+ return `run-${randomBytes(4).toString('hex')}`;
50
+ }
51
+
52
+ export function getLogContext(): LogContext {
53
+ return logContextStorage.getStore() ?? {};
54
+ }
55
+
56
+ export function runWithLogContext<T>(context: LogContext, callback: () => T): T {
57
+ const parent = logContextStorage.getStore() ?? {};
58
+ return logContextStorage.run({ ...parent, ...context }, callback);
59
+ }
60
+
61
+ export function safeJsonStringify(value: unknown): string {
62
+ try {
63
+ return JSON.stringify(value, null, 0);
64
+ } catch {
65
+ return '[unserializable]';
66
+ }
67
+ }
68
+
69
+ export function formatArg(arg: unknown): string {
70
+ if (arg instanceof Error) {
71
+ return arg.stack ?? arg.message;
72
+ }
73
+ if (typeof arg === 'object' && arg !== null) {
74
+ return safeJsonStringify(arg);
75
+ }
76
+ return String(arg);
77
+ }
78
+
79
+ /** Upper bound on a single log message; defends the log file against being
80
+ * flooded by large network/scraped payloads passed to the logger. */
81
+ const MAX_LOG_MESSAGE_LENGTH = 10_000;
82
+
83
+ // ANSI escape sequences embedded in untrusted content (we add our own colours
84
+ // separately for the console). Matching the ESC control byte is the point here.
85
+ // eslint-disable-next-line no-control-regex
86
+ const ANSI_PATTERN = /\x1b\[[0-9;]*[A-Za-z]/g;
87
+ // Credentials embedded in a URL userinfo component: scheme://user:pass@host
88
+ const URL_CREDENTIALS_PATTERN = /([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s:@]+@/gi;
89
+ // key=value / key: "value" pairs whose key names a secret.
90
+ const SENSITIVE_KV_PATTERN =
91
+ /\b(api[_-]?key|apikey|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|client[_-]?secret|password|passwd|pwd|authorization|bearer|set[_-]?cookie|cookie|session[_-]?id|session|csrf[_-]?token|xsrf[_-]?token|private[_-]?key)\b(["']?\s*[:=]\s*["']?)([^\s"',&)]+)/gi;
92
+ // Well-known opaque credential formats (OpenAI, GitHub, AWS, Slack, Anthropic).
93
+ const KNOWN_TOKEN_PATTERN =
94
+ /\b(sk-[A-Za-z0-9_-]{16,}|gh[posru]_[A-Za-z0-9]{16,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
95
+ // JSON Web Tokens — header.payload.signature, each a base64url segment.
96
+ const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g;
97
+ // HTTP Basic credentials: "Basic <base64>".
98
+ const BASIC_AUTH_PATTERN = /\bBasic\s+[A-Za-z0-9+/]{16,}={0,2}/gi;
99
+ // HTTP Bearer credentials: "Bearer <token>". The token is often opaque — many
100
+ // providers (e.g. GLM/zhipuai) use dotted keys that are NOT JWTs and carry no
101
+ // known prefix, so KNOWN_TOKEN_PATTERN/JWT_PATTERN miss them and the KV pass
102
+ // only masks the word "Bearer" (its value class stops at the space). Match the
103
+ // scheme and redact the whole RFC 6750 b64token that follows, whatever its shape.
104
+ const BEARER_AUTH_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi;
105
+ // Provider keys not covered by KNOWN_TOKEN_PATTERN: Google API/OAuth, Stripe.
106
+ const PROVIDER_TOKEN_PATTERN =
107
+ /\b(AIza[0-9A-Za-z_-]{35}|ya29\.[0-9A-Za-z_-]{20,}|[sp]k_(?:live|test)_[0-9A-Za-z]{16,})\b/g;
108
+ // Long opaque hex secrets (>=32 hex chars), e.g. the 64-hex browser auth secret
109
+ // or MD5/SHA-style tokens. Diagnostic logs may over-redact content hashes of
110
+ // this length; that trade-off favors not persisting credentials clear-text.
111
+ const LONG_HEX_SECRET_PATTERN = /\b[0-9a-fA-F]{32,}\b/g;
112
+ // All C0 control characters plus DEL — newlines/carriage returns are the log/
113
+ // terminal-injection vector when written to a raw console line. Matching these
114
+ // control bytes is exactly the intent.
115
+ // eslint-disable-next-line no-control-regex
116
+ const CONTROL_CHARS_PATTERN = /[\x00-\x1f\x7f]/g;
117
+
118
+ /**
119
+ * Redact secrets and bound the size of a log message before it is persisted.
120
+ *
121
+ * Removes embedded ANSI sequences, masks credentials in URLs, sensitive
122
+ * key/value pairs, and known token formats, and truncates oversized payloads.
123
+ * Applied to every message before it reaches the log file or the console, so
124
+ * neither sink receives clear-text secrets or unbounded network data.
125
+ */
126
+ export function redactSecrets(message: string): string {
127
+ let out = message.length > MAX_LOG_MESSAGE_LENGTH
128
+ ? `${message.slice(0, MAX_LOG_MESSAGE_LENGTH)}…[truncated ${message.length - MAX_LOG_MESSAGE_LENGTH} chars]`
129
+ : message;
130
+ out = out.replace(ANSI_PATTERN, '');
131
+ out = out.replace(URL_CREDENTIALS_PATTERN, '$1[REDACTED]@');
132
+ // Redact whole-token formats BEFORE the key/value pass: the KV value class
133
+ // stops at whitespace, so "Authorization: Basic <b64>" would otherwise mask
134
+ // only the word "Basic" and leave the credential blob exposed.
135
+ out = out.replace(JWT_PATTERN, '[REDACTED]');
136
+ out = out.replace(BASIC_AUTH_PATTERN, 'Basic [REDACTED]');
137
+ // Before the KV pass, for the same reason as Basic above: the KV value class
138
+ // stops at the space after "Bearer", leaving an opaque token exposed.
139
+ out = out.replace(BEARER_AUTH_PATTERN, 'Bearer [REDACTED]');
140
+ out = out.replace(SENSITIVE_KV_PATTERN, (_m, key: string, sep: string) => `${key}${sep}[REDACTED]`);
141
+ out = out.replace(KNOWN_TOKEN_PATTERN, '[REDACTED]');
142
+ out = out.replace(PROVIDER_TOKEN_PATTERN, '[REDACTED]');
143
+ out = out.replace(LONG_HEX_SECRET_PATTERN, '[REDACTED]');
144
+ return out;
145
+ }
146
+
147
+ /**
148
+ * Neutralize control characters (newlines, carriage returns, etc.) so that
149
+ * untrusted content cannot forge or corrupt a raw console log line
150
+ * (log-injection defense). Use for sinks that are NOT already structurally
151
+ * escaped — the JSON log file escapes control chars on its own.
152
+ */
153
+ export function neutralizeControlChars(message: string): string {
154
+ // Strip CR/LF explicitly first. These are the actual log-injection vector
155
+ // (forged/corrupted log lines) and an explicit line-break replacement is the
156
+ // form static analysis recognizes as a log-injection sanitizer. The second
157
+ // pass removes any remaining C0 control chars + DEL.
158
+ return message.replace(/\r\n|\r|\n/g, ' ').replace(CONTROL_CHARS_PATTERN, ' ');
159
+ }
@@ -5,7 +5,7 @@
5
5
  * Both files use identical ResourceLoader implementations.
6
6
  */
7
7
 
8
- import type { ResourceLoader, ExtensionRuntime } from '@mariozechner/pi-coding-agent';
8
+ import type { ResourceLoader, ExtensionRuntime } from '@earendil-works/pi-coding-agent';
9
9
 
10
10
  export function makeResourceLoader(systemPromptText: string): ResourceLoader {
11
11
  const mockRuntime: ExtensionRuntime = {
@@ -27,6 +27,8 @@ export function makeResourceLoader(systemPromptText: string): ResourceLoader {
27
27
  setModel: async () => true,
28
28
  getThinkingLevel: () => 'off',
29
29
  setThinkingLevel: () => {},
30
+ assertActive: () => {},
31
+ invalidate: () => {},
30
32
  };
31
33
 
32
34
  return {