@lincoln504/pi-research 0.1.13 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +55 -138
  3. package/dist/cli.mjs +23375 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +130 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +211 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +122 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +66 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1195 -0
  30. package/src/config.ts +1032 -155
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +258 -11
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -212
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -644
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
@@ -0,0 +1,61 @@
1
+ /**
2
+ * Research Panel Types
3
+ *
4
+ * Type definitions for the research TUI panel.
5
+ */
6
+
7
+ /**
8
+ * Local Theme interface — mirrors pi-tui Theme.fg()
9
+ */
10
+ export interface Theme {
11
+ fg: (color: any, text: string) => string;
12
+ }
13
+
14
+ export interface SliceState {
15
+ id: string;
16
+ label: string; // Displayed label: "1", "2", "3", etc.
17
+ completed: boolean;
18
+ queued: boolean;
19
+ tokens?: number; // Tokens for this specific agent
20
+ cost?: number; // Cost for this specific agent
21
+ status?: string; // Optional status message like "Searching..."
22
+ /** Active flash color — set by flashSlice, cleared by timeout. Always present. */
23
+ flash: 'green' | 'red' | null;
24
+ /** Terminal outcome: true if this researcher failed (retries exhausted / no links).
25
+ * Rendered distinctly (error color + "failed" marker) so a failed researcher is not
26
+ * visually identical to a successful one — a completed slice is otherwise drawn muted. */
27
+ failed?: boolean;
28
+ }
29
+
30
+ export interface ResearchProgress {
31
+ /** Total tool calls expected across all researchers in the initial plan. */
32
+ expected: number;
33
+ /** Actual tool calls completed so far. */
34
+ made: number;
35
+ }
36
+
37
+ export interface ResearchPanelState {
38
+ sessionId: string;
39
+ /** Alias for sessionId — used by the orchestrator */
40
+ researchId: string;
41
+ query: string;
42
+ slices: Map<string, SliceState>;
43
+ modelName: string;
44
+ /** True when the research is actively searching (queueing/executing search tool) */
45
+ isSearching?: boolean;
46
+ /** Optional progress bar data. Set after planning completes. */
47
+ progress?: ResearchProgress;
48
+ /** Animation frame counter for traveling wave effect during search */
49
+ waveFrame?: number;
50
+ /** Last-visit frame per position for wave animation decay */
51
+ waveColors?: number[];
52
+ /** Timer for wave animation */
53
+ waveTimer?: NodeJS.Timeout | null;
54
+ /** Flag indicating whether completed researchers should be cleared */
55
+ needsClear?: boolean;
56
+ /** 1-2 word topic title derived from the query after the coordinator LLM responds */
57
+ title?: string;
58
+ /** Whether the current research phase can accept steering messages.
59
+ * True during search/researcher phases. False during coordinator/evaluator LLM calls and after completion. */
60
+ steeringAcceptable?: boolean;
61
+ }
@@ -0,0 +1,146 @@
1
+ import { logger } from '../logger.ts';
2
+ import { metrics } from './metrics.ts';
3
+
4
+ export type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
5
+
6
+ export interface CircuitBreakerOptions {
7
+ /** How many consecutive failures to open the circuit (default: 5) */
8
+ failureThreshold?: number;
9
+ /** How long to wait before testing recovery in HALF_OPEN state (default: 30000ms) */
10
+ resetTimeoutMs?: number;
11
+ /** How many successful calls needed in HALF_OPEN to close the circuit (default: 1) */
12
+ halfOpenMaxCalls?: number;
13
+ /** Optional name for logging */
14
+ name?: string;
15
+ /**
16
+ * Function to determine if an error should count towards failure limit.
17
+ * If it returns false, the error is passed through without tripping the breaker.
18
+ * (default: counts all errors)
19
+ */
20
+ isTransientError?: (error: unknown) => boolean;
21
+ }
22
+
23
+ export class CircuitBreaker {
24
+ private state: CircuitState = 'CLOSED';
25
+ private failureCount = 0;
26
+ private successCount = 0;
27
+ private nextAttemptTime = 0;
28
+ // Number of trial calls currently executing in HALF_OPEN. Caps concurrent probes so a burst of
29
+ // parallel callers doesn't all hit a still-recovering dependency the instant the breaker half-opens.
30
+ private halfOpenInFlight = 0;
31
+ private options: Required<CircuitBreakerOptions>;
32
+
33
+ constructor(options: CircuitBreakerOptions = {}) {
34
+ this.options = {
35
+ failureThreshold: options.failureThreshold ?? 5,
36
+ resetTimeoutMs: options.resetTimeoutMs ?? 30000,
37
+ halfOpenMaxCalls: options.halfOpenMaxCalls ?? 1,
38
+ name: options.name ?? 'CircuitBreaker',
39
+ isTransientError: options.isTransientError ?? (() => true),
40
+ };
41
+ }
42
+
43
+ async execute<T>(action: () => Promise<T>): Promise<T> {
44
+ const startTime = Date.now();
45
+ metrics.increment('circuit_breaker_calls_total', 1, { breaker: this.options.name, state: this.state });
46
+
47
+ if (this.state === 'OPEN') {
48
+ if (Date.now() >= this.nextAttemptTime) {
49
+ this.transitionTo('HALF_OPEN');
50
+ } else {
51
+ metrics.increment('circuit_breaker_rejected_total', 1, { breaker: this.options.name, reason: 'open' });
52
+ throw new Error(`CircuitBreaker '${this.options.name}' is OPEN. Fast-failing to prevent cascading failure.`);
53
+ }
54
+ }
55
+
56
+ // HALF_OPEN admission control: allow only halfOpenMaxCalls trial calls to execute at once and
57
+ // fast-fail the rest. Without this, once the first caller flips OPEN→HALF_OPEN every other
58
+ // concurrent caller sees state !== 'OPEN', skips the gate, and hits the still-down dependency
59
+ // simultaneously — defeating the single-trial probe and immediately re-opening the breaker.
60
+ let admittedAsProbe = false;
61
+ if (this.state === 'HALF_OPEN') {
62
+ if (this.halfOpenInFlight >= this.options.halfOpenMaxCalls) {
63
+ metrics.increment('circuit_breaker_rejected_total', 1, { breaker: this.options.name, reason: 'half_open_probe_limit' });
64
+ throw new Error(`CircuitBreaker '${this.options.name}' is HALF_OPEN (trial in progress). Fast-failing to limit probes.`);
65
+ }
66
+ this.halfOpenInFlight++;
67
+ admittedAsProbe = true;
68
+ }
69
+
70
+ try {
71
+ const result = await action();
72
+ const duration = Date.now() - startTime;
73
+ metrics.observe('circuit_breaker_call_duration_ms', duration, { breaker: this.options.name, status: 'success' });
74
+ this.onSuccess();
75
+ return result;
76
+ } catch (error) {
77
+ const duration = Date.now() - startTime;
78
+ metrics.observe('circuit_breaker_call_duration_ms', duration, { breaker: this.options.name, status: 'error' });
79
+ if (this.options.isTransientError(error)) {
80
+ this.onFailure(error);
81
+ }
82
+ throw error;
83
+ } finally {
84
+ if (admittedAsProbe) this.halfOpenInFlight = Math.max(0, this.halfOpenInFlight - 1);
85
+ }
86
+ }
87
+
88
+ private onSuccess() {
89
+ metrics.increment('circuit_breaker_success_total', 1, { breaker: this.options.name, state: this.state });
90
+ if (this.state === 'HALF_OPEN') {
91
+ this.successCount++;
92
+ if (this.successCount >= this.options.halfOpenMaxCalls) {
93
+ this.transitionTo('CLOSED');
94
+ }
95
+ } else if (this.state === 'CLOSED') {
96
+ // Reset consecutive failures on success
97
+ this.failureCount = 0;
98
+ }
99
+ }
100
+
101
+ private onFailure(error: unknown) {
102
+ this.failureCount++;
103
+ const errMsg = error instanceof Error ? error.message : String(error);
104
+ metrics.increment('circuit_breaker_failures_total', 1, { breaker: this.options.name, state: this.state });
105
+
106
+ if (this.state === 'HALF_OPEN') {
107
+ logger.warn(`[CircuitBreaker] '${this.options.name}' failed during HALF_OPEN state: ${errMsg}`);
108
+ this.transitionTo('OPEN');
109
+ } else if (this.state === 'CLOSED') {
110
+ if (this.failureCount >= this.options.failureThreshold) {
111
+ logger.error(`[CircuitBreaker] '${this.options.name}' reached failure threshold (${this.failureCount}): ${errMsg}`);
112
+ this.transitionTo('OPEN');
113
+ }
114
+ }
115
+ }
116
+
117
+ private transitionTo(newState: CircuitState) {
118
+ logger.warn(`[CircuitBreaker] '${this.options.name}' transitioning from ${this.state} to ${newState}`);
119
+ metrics.increment('circuit_breaker_state_transitions_total', 1, { breaker: this.options.name, from: this.state, to: newState });
120
+ metrics.setGauge('circuit_breaker_state', newState === 'CLOSED' ? 0 : newState === 'OPEN' ? 1 : 2, { breaker: this.options.name });
121
+ this.state = newState;
122
+ if (newState === 'OPEN') {
123
+ this.nextAttemptTime = Date.now() + this.options.resetTimeoutMs;
124
+ } else if (newState === 'HALF_OPEN') {
125
+ this.successCount = 0;
126
+ this.halfOpenInFlight = 0;
127
+ } else if (newState === 'CLOSED') {
128
+ this.failureCount = 0;
129
+ this.successCount = 0;
130
+ }
131
+ }
132
+
133
+ public getState(): CircuitState {
134
+ return this.state;
135
+ }
136
+
137
+ public reset(): void {
138
+ metrics.increment('circuit_breaker_resets_total', 1, { breaker: this.options.name });
139
+ this.state = 'CLOSED';
140
+ this.failureCount = 0;
141
+ this.successCount = 0;
142
+ this.halfOpenInFlight = 0;
143
+ this.nextAttemptTime = 0;
144
+ metrics.setGauge('circuit_breaker_state', 0, { breaker: this.options.name });
145
+ }
146
+ }
@@ -0,0 +1,97 @@
1
+ import { parseCitations, lastCitedLinksHeaderIndex, type Citation } from './text-utils.ts';
2
+ import { normalizeUrl } from './shared-links.ts';
3
+
4
+ export interface GlobalCitation extends Citation {
5
+ id: number;
6
+ }
7
+
8
+ /**
9
+ * Normalizes citations across multiple reports.
10
+ *
11
+ * 1. Extracts all citations from all reports.
12
+ * 2. Deduplicates URLs and assigns global IDs.
13
+ * 3. Replaces local citations [N] in each report with global IDs [G].
14
+ * 4. Removes the local CITED LINKS section from each report.
15
+ *
16
+ * @param reports - Map of researcher ID to report content
17
+ * @returns Object containing normalized reports and the global citation list
18
+ */
19
+ export function normalizeCitations(reports: Map<string, string>): {
20
+ normalizedReports: Map<string, string>;
21
+ globalCitations: GlobalCitation[];
22
+ } {
23
+ const globalCitations: GlobalCitation[] = [];
24
+ const urlToGlobalId = new Map<string, number>();
25
+ const normalizedReports = new Map<string, string>();
26
+
27
+ // First pass: Collect all unique URLs and assign global IDs
28
+ for (const report of reports.values()) {
29
+ const citations = parseCitations(report);
30
+ for (const cit of citations) {
31
+ const normUrl = normalizeUrl(cit.url);
32
+ if (!urlToGlobalId.has(normUrl)) {
33
+ const id = globalCitations.length + 1;
34
+ urlToGlobalId.set(normUrl, id);
35
+ globalCitations.push({
36
+ id,
37
+ url: cit.url,
38
+ description: cit.description,
39
+ source: cit.source,
40
+ });
41
+ }
42
+ }
43
+ }
44
+
45
+ // Second pass: Normalize each report
46
+ for (const [id, report] of reports.entries()) {
47
+ const localCitations = parseCitations(report);
48
+ // Map of local index (1-based) to global ID
49
+ const localToGlobal = new Map<number, number>();
50
+
51
+ localCitations.forEach((cit, index) => {
52
+ const globalId = urlToGlobalId.get(normalizeUrl(cit.url));
53
+ if (globalId !== undefined) {
54
+ // Key by the number the report actually WROTE for this entry (e.g. `[3]`), not its list
55
+ // position. A report that numbers CITED LINKS non-sequentially (or has an entry dropped as
56
+ // an implausible URL) would otherwise remap inline [N] to the wrong global id or leave it
57
+ // dangling. Fall back to position when a written number is unavailable.
58
+ const localKey = cit.number ?? (index + 1);
59
+ localToGlobal.set(localKey, globalId);
60
+ }
61
+ });
62
+
63
+ // Remove the trailing CITED LINKS section before renumbering — but only a real
64
+ // line-leading, all-caps header, never an incidental "cited links" mention in
65
+ // prose (an unanchored split there would drop real findings before the
66
+ // evaluator ever sees them).
67
+ const headerIdx = lastCitedLinksHeaderIndex(report);
68
+ let content = headerIdx >= 0 ? report.slice(0, headerIdx) : report;
69
+
70
+ // Replace [N] with [GlobalID]
71
+ // Also handle [N][M] combinations
72
+ content = content.replace(/\[(\d+)\]/g, (match, p1) => {
73
+ const localId = parseInt(p1, 10);
74
+ const globalId = localToGlobal.get(localId);
75
+ return globalId !== undefined ? `[${globalId}]` : match;
76
+ });
77
+
78
+ normalizedReports.set(id, content.trim());
79
+ }
80
+
81
+ return { normalizedReports, globalCitations };
82
+ }
83
+
84
+ /**
85
+ * Formats the global citation list into a CITED LINKS section string.
86
+ */
87
+ export function formatCitedLinks(citations: GlobalCitation[]): string {
88
+ if (citations.length === 0) return '';
89
+
90
+ const links = citations.map(cit => {
91
+ const sourcePart = cit.source ? ` [Source: ${cit.source}]` : '';
92
+ const descPart = cit.description ? ` — ${cit.description}` : '';
93
+ return `[${cit.id}] ${cit.url}${sourcePart}${descPart}`;
94
+ });
95
+
96
+ return `CITED LINKS\n${links.join('\n')}`;
97
+ }
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Disk Space Checker
3
+ *
4
+ * Checks available disk space for logging.
5
+ */
6
+
7
+ import * as fs from 'node:fs';
8
+
9
+ export class DiskSpaceChecker {
10
+ private readonly MIN_DISK_SPACE_BYTES = 1_048_576; // 1MB minimum
11
+ private readonly DISK_SPACE_CHECK_INTERVAL_MS = 60_000; // Check every 60 seconds
12
+ private lastDiskSpaceCheck: number = 0;
13
+ private hasDiskSpace: boolean = true;
14
+
15
+ /**
16
+ * Check if there is sufficient disk space for logging.
17
+ * Checks are throttled to avoid excessive filesystem operations.
18
+ */
19
+ checkDiskSpace(logDir: string): boolean {
20
+ const now = Date.now();
21
+
22
+ // Only check periodically (every 60 seconds)
23
+ if (now - this.lastDiskSpaceCheck < this.DISK_SPACE_CHECK_INTERVAL_MS) {
24
+ return this.hasDiskSpace;
25
+ }
26
+
27
+ this.lastDiskSpaceCheck = now;
28
+
29
+ try {
30
+ // fs.statfsSync is cross-platform on Node >=18.15 (incl. Windows), and the
31
+ // package floor is Node >=22.19, so it is always available — no wmic/exec
32
+ // fallback is needed (wmic is removed on Windows 11 24H2 anyway).
33
+ const stats = fs.statfsSync(logDir);
34
+ const availableBytes = stats.bavail * stats.bsize;
35
+
36
+ if (availableBytes < this.MIN_DISK_SPACE_BYTES) {
37
+ this.hasDiskSpace = false;
38
+ // Write directly to stderr — this check runs inside the logger itself,
39
+ // so using the logger here would create a circular dependency.
40
+ process.stderr.write(
41
+ `[pi-research] Insufficient disk space for logging: ` +
42
+ `${Math.round(availableBytes / 1024 / 1024)}MB available, ` +
43
+ `minimum ${this.MIN_DISK_SPACE_BYTES / 1024 / 1024}MB required\n`
44
+ );
45
+ } else {
46
+ this.hasDiskSpace = true;
47
+ }
48
+ } catch (_error) {
49
+ // On error, assume OK to avoid blocking logging
50
+ this.hasDiskSpace = true;
51
+ }
52
+
53
+ return this.hasDiskSpace;
54
+ }
55
+ }
@@ -0,0 +1,265 @@
1
+ /**
2
+ * Error Tracker - Session-Scoped Error Pattern Tracking
3
+ *
4
+ * Provides automatic error pattern detection and grouping for research operations.
5
+ * Session-scoped (in-memory) storage that's cleared between research runs.
6
+ */
7
+
8
+ import { AsyncLocalStorage } from 'node:async_hooks';
9
+
10
+ /**
11
+ * Error context information for tracking where errors occur
12
+ */
13
+ export interface ErrorContext {
14
+ researchId?: string;
15
+ mode?: string;
16
+ component?: string;
17
+ operation?: string;
18
+ toolName?: string;
19
+ phase?: string;
20
+ sessionId?: string;
21
+ url?: string;
22
+ domain?: string;
23
+ [key: string]: unknown;
24
+ }
25
+
26
+ /**
27
+ * Error pattern with aggregated information
28
+ */
29
+ export interface ErrorPattern {
30
+ signature: string;
31
+ message: string;
32
+ count: number;
33
+ firstSeen: string;
34
+ lastSeen: string;
35
+ contexts: ErrorContext[];
36
+ domainCounts: Map<string, number>;
37
+ }
38
+
39
+ /**
40
+ * Complete error report with aggregated statistics
41
+ */
42
+ export interface ErrorReport {
43
+ totalErrors: number;
44
+ uniquePatterns: number;
45
+ patterns: ErrorPattern[];
46
+ byDomain: Map<string, number>;
47
+ byType: Map<string, number>;
48
+ }
49
+
50
+ // Storage for isolation between concurrent research runs
51
+ const trackerStorage = new AsyncLocalStorage<ErrorTracker>();
52
+
53
+ /**
54
+ * Run a function with a specific ErrorTracker bound to the async context.
55
+ */
56
+ export function runWithTracker<T>(tracker: ErrorTracker, fn: () => T): T {
57
+ return trackerStorage.run(tracker, fn);
58
+ }
59
+
60
+ /**
61
+ * Session-scoped error tracker with pattern detection
62
+ */
63
+ export class ErrorTracker {
64
+ private patterns = new Map<string, ErrorPattern>();
65
+ private readonly MAX_CONTEXTS_PER_PATTERN = 10;
66
+ // Bound total memory. The global singleton is never clear()'d, so out-of-run errors (logger,
67
+ // scrapers, healthcheck) would otherwise grow these maps without limit over a long-lived host.
68
+ private readonly MAX_PATTERNS = 500;
69
+ private readonly MAX_DOMAINS_PER_PATTERN = 100;
70
+
71
+ /**
72
+ * Normalize an error message to extract a stable signature.
73
+ * Removes UUIDs, numbers (except HTTP status codes), URLs, and normalizes whitespace.
74
+ *
75
+ * @param message - The error message to normalize
76
+ * @returns A normalized signature for grouping similar errors
77
+ */
78
+ public extractSignature(message: string): string {
79
+ return message
80
+ // Remove UUIDs (8-4-4-4-12 hex format)
81
+ .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '<UUID>')
82
+ // Remove numbers (except HTTP status codes like 200, 301, 302, 403, 404, 500, 503)
83
+ .replace(/\b(?![12]\d{2}\b|[3-5]\d{2}\b)\d+\b/g, '<NUM>')
84
+ // Remove URLs
85
+ .replace(/https?:\/\/[^\s]+/g, '<URL>')
86
+ // Normalize whitespace to single spaces
87
+ .replace(/\s+/g, ' ')
88
+ .trim();
89
+ }
90
+
91
+ /**
92
+ * Track an error with optional context.
93
+ * Groups similar errors by normalized signature and maintains a rolling buffer of contexts.
94
+ *
95
+ * @param error - The error to track (Error object, string message, or unknown)
96
+ * @param context - Optional context information about where the error occurred
97
+ */
98
+ public trackError(error: Error | string | unknown, context: ErrorContext = {}): void {
99
+ let message: string;
100
+
101
+ if (error instanceof Error) {
102
+ message = error.message;
103
+ } else if (typeof error === 'string') {
104
+ message = error;
105
+ } else {
106
+ // Convert unknown error to string
107
+ message = String(error);
108
+ }
109
+
110
+ const signature = this.extractSignature(message);
111
+ const now = new Date().toISOString();
112
+
113
+ let pattern = this.patterns.get(signature);
114
+ if (!pattern) {
115
+ // Evict the least-frequent pattern before adding a new one at the cap, so the map stays
116
+ // bounded while retaining the highest-signal (most frequent) patterns.
117
+ if (this.patterns.size >= this.MAX_PATTERNS) {
118
+ this.evictLeastFrequentPattern();
119
+ }
120
+ pattern = {
121
+ signature,
122
+ message,
123
+ count: 0,
124
+ firstSeen: now,
125
+ lastSeen: now,
126
+ contexts: [],
127
+ domainCounts: new Map<string, number>(),
128
+ };
129
+ this.patterns.set(signature, pattern);
130
+ }
131
+
132
+ pattern.count++;
133
+ pattern.lastSeen = now;
134
+
135
+ // Track domain count if available. Cap distinct domains per pattern: existing domains keep
136
+ // incrementing, but once at the cap new distinct domains are dropped to bound memory.
137
+ if (context.domain) {
138
+ const existing = pattern.domainCounts.get(context.domain);
139
+ if (existing !== undefined) {
140
+ pattern.domainCounts.set(context.domain, existing + 1);
141
+ } else if (pattern.domainCounts.size < this.MAX_DOMAINS_PER_PATTERN) {
142
+ pattern.domainCounts.set(context.domain, 1);
143
+ }
144
+ }
145
+
146
+ // Add context to rolling buffer (keep only last 10)
147
+ pattern.contexts.push(context);
148
+ if (pattern.contexts.length > this.MAX_CONTEXTS_PER_PATTERN) {
149
+ pattern.contexts.shift();
150
+ }
151
+
152
+ }
153
+
154
+ /**
155
+ * Generate a complete error report with sorted patterns and domain/type breakdowns.
156
+ *
157
+ * @returns An ErrorReport with aggregated error information
158
+ */
159
+ public getReport(): ErrorReport {
160
+ const patterns = Array.from(this.patterns.values());
161
+ let totalErrors = 0;
162
+
163
+ for (const pattern of patterns) {
164
+ totalErrors += pattern.count;
165
+ }
166
+
167
+ // Sort patterns by frequency (most frequent first)
168
+ patterns.sort((a, b) => b.count - a.count);
169
+
170
+ const byDomain = this.getErrorsByDomain();
171
+ const byType = this.getErrorsByType();
172
+
173
+ return {
174
+ totalErrors,
175
+ uniquePatterns: patterns.length,
176
+ patterns,
177
+ byDomain,
178
+ byType,
179
+ };
180
+ }
181
+
182
+ /**
183
+ * Group errors by domain from context information.
184
+ *
185
+ * @returns A Map of domain names to error counts
186
+ */
187
+ public getErrorsByDomain(): Map<string, number> {
188
+ const totalDomainCounts = new Map<string, number>();
189
+
190
+ for (const pattern of this.patterns.values()) {
191
+ for (const [domain, count] of pattern.domainCounts.entries()) {
192
+ totalDomainCounts.set(domain, (totalDomainCounts.get(domain) ?? 0) + count);
193
+ }
194
+ }
195
+
196
+ return totalDomainCounts;
197
+ }
198
+
199
+ /**
200
+ * Group errors by type (first part of signature before colon or first few words).
201
+ *
202
+ * @returns A Map of error types to error counts
203
+ */
204
+ public getErrorsByType(): Map<string, number> {
205
+ const typeCounts = new Map<string, number>();
206
+
207
+ for (const pattern of this.patterns.values()) {
208
+ // Extract type: first part of signature before colon, or first 3 words
209
+ const parts = pattern.signature.split(':');
210
+ let type = parts[0]?.trim() ?? 'Unknown';
211
+
212
+ if (type === pattern.signature) {
213
+ // No colon found, use first 3 words
214
+ const words = pattern.signature.split(' ');
215
+ const firstWords = words.slice(0, 3).filter(w => w.length > 0);
216
+ type = firstWords.length > 0 ? firstWords.join(' ') : 'Unknown';
217
+ }
218
+
219
+ typeCounts.set(type, (typeCounts.get(type) ?? 0) + pattern.count);
220
+ }
221
+
222
+ return typeCounts;
223
+ }
224
+
225
+ /**
226
+ * Evict the least-frequent pattern to keep the map bounded (see MAX_PATTERNS). O(n) but only
227
+ * runs when a new signature is added while already at the cap.
228
+ */
229
+ private evictLeastFrequentPattern(): void {
230
+ let victimKey: string | undefined;
231
+ let victimCount = Infinity;
232
+ for (const [key, p] of this.patterns) {
233
+ if (p.count < victimCount) {
234
+ victimCount = p.count;
235
+ victimKey = key;
236
+ }
237
+ }
238
+ if (victimKey !== undefined) {
239
+ this.patterns.delete(victimKey);
240
+ }
241
+ }
242
+
243
+ /**
244
+ * Clear all tracked errors (called between research runs).
245
+ */
246
+ public clear(): void {
247
+ this.patterns.clear();
248
+ }
249
+ }
250
+
251
+ /**
252
+ * Global singleton instance that proxies to the context-bound instance if available,
253
+ * enabling session-scoped error isolation while falling back to a shared global tracker.
254
+ */
255
+ const globalInstance = new ErrorTracker();
256
+ export const errorTracker: ErrorTracker = new Proxy(globalInstance, {
257
+ get(target, prop, receiver) {
258
+ const currentTracker = trackerStorage.getStore() ?? target;
259
+ const value = Reflect.get(currentTracker, prop, receiver);
260
+ if (typeof value === 'function') {
261
+ return value.bind(currentTracker);
262
+ }
263
+ return value;
264
+ }
265
+ });
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Host pi configuration access — resolved LAZILY.
3
+ *
4
+ * `@earendil-works/pi-coding-agent` is provided by the host pi process and is
5
+ * NOT reliably present in this extension's own module-resolution scope when pi
6
+ * runs the extension (though declared as a regular dependency, it resolves
7
+ * against the host, not the extension's own `node_modules`). A
8
+ * top-level `import { CONFIG_DIR_NAME } from '@earendil-works/pi-coding-agent'`
9
+ * is hoisted and executed on module load, so any *standalone* bundle that
10
+ * transitively imports such a module — most importantly the poolifier-spawned
11
+ * `thread-worker.mjs`, which resolves modules relative to its own file and
12
+ * cannot see pi's global install — crashes with ERR_MODULE_NOT_FOUND before a
13
+ * single line of its own code runs.
14
+ *
15
+ * Resolving the value through this helper instead keeps the host package out of
16
+ * every module's load-time graph: the `require()` happens on first call, inside
17
+ * a try/catch, and degrades to pi's own stable default where the host package
18
+ * is not resolvable (i.e. inside the worker thread).
19
+ */
20
+ import { createRequire } from 'node:module';
21
+
22
+ let _configDirName: string | undefined;
23
+
24
+ /**
25
+ * The host pi config directory name (e.g. `.pi`).
26
+ *
27
+ * Resolution order: explicit env override (`PI_RESEARCH_CONFIG_DIR_NAME`) →
28
+ * the host `@earendil-works/pi-coding-agent` package (only where it is actually
29
+ * resolvable, i.e. the main pi process) → the stable default `.pi`.
30
+ */
31
+ export function getConfigDirName(): string {
32
+ if (_configDirName !== undefined) return _configDirName;
33
+ const override = process.env['PI_RESEARCH_CONFIG_DIR_NAME'];
34
+ if (override) return (_configDirName = override);
35
+ try {
36
+ const require = createRequire(import.meta.url);
37
+ const host = require('@earendil-works/pi-coding-agent') as { CONFIG_DIR_NAME?: string };
38
+ _configDirName = host.CONFIG_DIR_NAME || '.pi';
39
+ } catch {
40
+ // Host package not resolvable in this context (e.g. the browser worker
41
+ // thread) — fall back to pi's own stable default.
42
+ _configDirName = '.pi';
43
+ }
44
+ return _configDirName;
45
+ }