@lincoln504/pi-research 1.1.2 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/README.md +3 -3
  2. package/dist/cli.mjs +2599 -1268
  3. package/dist/thread-worker.mjs +224 -147
  4. package/docs/AGENT-SKILL.md +11 -4
  5. package/docs/ARCHITECTURE.md +20 -10
  6. package/docs/CHANGELOG.md +195 -17
  7. package/docs/CONFIGURATION.md +9 -7
  8. package/docs/KNOWLEDGE-STORE.md +4 -3
  9. package/docs/PI-EXTENSION.md +1 -1
  10. package/docs/SDK.md +55 -0
  11. package/docs/THIRD-PARTY-NOTICES.md +1 -1
  12. package/package.json +5 -4
  13. package/scripts/cleanup.cjs +114 -12
  14. package/scripts/prepare.cjs +15 -5
  15. package/scripts/setup.cjs +24 -4
  16. package/skills/pi-research/README.md +9 -2
  17. package/skills/pi-research/SKILL.md +6 -3
  18. package/skills/pi-research/references/configuration.md +6 -2
  19. package/skills/pi-research/scripts/run.mjs +24 -2
  20. package/src/cleanup/research-cleanup.ts +12 -4
  21. package/src/cli.ts +372 -86
  22. package/src/config.ts +134 -17
  23. package/src/constants.ts +31 -0
  24. package/src/core/interfaces/health-check-interfaces.ts +3 -39
  25. package/src/core/interfaces/knowledge-interfaces.ts +6 -1
  26. package/src/core/interfaces/planning-interfaces.ts +8 -0
  27. package/src/core/interfaces/service-names.ts +0 -1
  28. package/src/core/interfaces/state-manager-interfaces.ts +11 -3
  29. package/src/core/llm/agentic-repair.ts +14 -0
  30. package/src/core/pi-version.ts +136 -0
  31. package/src/core/planning-service.ts +43 -1
  32. package/src/core/service-initialization.ts +6 -14
  33. package/src/core/service-interfaces.ts +1 -1
  34. package/src/core/service-registry.ts +117 -8
  35. package/src/healthcheck/index.ts +7 -2
  36. package/src/index.ts +11 -18
  37. package/src/infrastructure/browser/browser-client.ts +80 -13
  38. package/src/infrastructure/browser/browser-error-utils.ts +6 -1
  39. package/src/infrastructure/browser/browser-server.ts +41 -12
  40. package/src/infrastructure/browser/browser-task-scheduler.ts +159 -35
  41. package/src/infrastructure/browser/cleanup-utils.ts +95 -9
  42. package/src/infrastructure/browser/client-agent.ts +6 -5
  43. package/src/infrastructure/browser/config.ts +14 -3
  44. package/src/infrastructure/browser/priority-task-queue.ts +10 -3
  45. package/src/infrastructure/browser/scheduler-factory.ts +44 -4
  46. package/src/infrastructure/browser/task-execution-service.ts +92 -28
  47. package/src/infrastructure/browser/thread-worker-lifecycle.ts +50 -13
  48. package/src/infrastructure/browser/thread-worker-messaging.ts +54 -2
  49. package/src/infrastructure/browser/thread-worker-types.ts +3 -1
  50. package/src/infrastructure/browser/thread-worker.mjs +224 -147
  51. package/src/infrastructure/browser/worker-pool-manager.ts +18 -5
  52. package/src/infrastructure/embedding/embedding-client.ts +22 -8
  53. package/src/infrastructure/embedding/embedding-factory.ts +61 -15
  54. package/src/infrastructure/embedding/embedding-server.ts +105 -11
  55. package/src/infrastructure/file-lock-service.ts +295 -28
  56. package/src/infrastructure/knowledge-store-service.ts +48 -3
  57. package/src/infrastructure/process-lifecycle-service.ts +134 -34
  58. package/src/infrastructure/research-run-semaphore.ts +37 -3
  59. package/src/infrastructure/state/state-backup-manager.ts +5 -7
  60. package/src/infrastructure/state/state-browser-api.ts +5 -2
  61. package/src/infrastructure/state/state-browser-manager.ts +15 -1
  62. package/src/infrastructure/state/state-manager-service.ts +5 -5
  63. package/src/infrastructure/state/state-manager.ts +34 -23
  64. package/src/infrastructure/types/state-types.ts +9 -0
  65. package/src/knowledge/chunker.ts +88 -1
  66. package/src/knowledge/embedder-init.ts +1 -1
  67. package/src/knowledge/embedder.ts +188 -46
  68. package/src/knowledge/index.ts +1 -1
  69. package/src/knowledge/store.ts +101 -21
  70. package/src/knowledge/webgpu-viability.ts +9 -4
  71. package/src/knowledge/writer-queue.ts +88 -46
  72. package/src/logger.ts +5 -1
  73. package/src/observers/research-observer-impl.ts +31 -0
  74. package/src/orchestration/abort-utils.ts +60 -0
  75. package/src/orchestration/deep-research-orchestrator.ts +25 -4
  76. package/src/orchestration/headless-observer.ts +8 -0
  77. package/src/orchestration/quick-research-orchestrator.ts +2 -2
  78. package/src/orchestration/research-orchestration-service.ts +65 -15
  79. package/src/orchestration/research-session-service.ts +37 -11
  80. package/src/orchestration/research-synthesis-service.ts +33 -11
  81. package/src/orchestration/researcher-executor.ts +100 -20
  82. package/src/orchestration/researcher.ts +39 -20
  83. package/src/orchestration/session-state.ts +11 -0
  84. package/src/sdk.ts +125 -19
  85. package/src/security/cisa-kev.ts +10 -3
  86. package/src/security/github-advisories.ts +18 -7
  87. package/src/security/index.ts +51 -30
  88. package/src/security/nvd-client.ts +24 -8
  89. package/src/security/osv-client.ts +14 -4
  90. package/src/skill-install/skill-installer.ts +153 -11
  91. package/src/stackexchange/index.ts +20 -5
  92. package/src/stackexchange/output/table.ts +7 -2
  93. package/src/stackexchange/rest-client.ts +41 -3
  94. package/src/tools/research-knowledge-search.ts +15 -0
  95. package/src/tools/research-tool-definition.ts +8 -2
  96. package/src/tools/scrape.ts +48 -25
  97. package/src/tools/search.ts +16 -2
  98. package/src/tools/security.ts +68 -7
  99. package/src/tui/research-health.ts +18 -2
  100. package/src/utils/atomic-replace.ts +101 -0
  101. package/src/utils/circuit-breaker.ts +9 -2
  102. package/src/utils/citation-utils.ts +3 -1
  103. package/src/utils/disk-space-checker.ts +11 -6
  104. package/src/utils/http-body.ts +176 -0
  105. package/src/utils/json-utils.ts +116 -76
  106. package/src/utils/llm-usage.ts +17 -1
  107. package/src/utils/log-rotation.ts +42 -10
  108. package/src/utils/log-utils.ts +40 -0
  109. package/src/utils/metrics-summary.ts +27 -9
  110. package/src/utils/stdio-capture.ts +63 -22
  111. package/src/utils/text-utils.ts +8 -1
  112. package/src/utils/tool-usage-tracker.ts +9 -2
  113. package/src/web-research/browser-search.ts +70 -5
  114. package/src/web-research/scraper-utils.ts +13 -0
  115. package/src/web-research/search.ts +25 -6
  116. package/src/web-research/types.ts +19 -4
  117. package/src/web-research/web-scraper.ts +72 -54
  118. package/src/youtube/transcript-client.ts +5 -1
  119. package/src/core/health-check-service.ts +0 -265
@@ -1,265 +0,0 @@
1
- /**
2
- * Health Check Service
3
- *
4
- * Manages health check cache state and backoff logic.
5
- * Provides thread-safe access to health check state and prevents
6
- * thundering herd problems during health checks.
7
- */
8
-
9
- import type { IHealthCheckService, HealthCheckResult } from './service-interfaces.ts';
10
- import { ServiceNames } from './service-interfaces.ts';
11
- import { ServiceLifecycle } from './service-registry.ts';
12
- import { logger } from '../logger.ts';
13
-
14
- /**
15
- * Default backoff configuration
16
- */
17
- const DEFAULT_BACKOFF_BASE_MS = 1000; // 1 second
18
- const DEFAULT_BACKOFF_MAX_MS = 60000; // 1 minute
19
- const DEFAULT_BACKOFF_MULTIPLIER = 2;
20
-
21
- /**
22
- * Health Check Service Implementation
23
- */
24
- export class HealthCheckService implements IHealthCheckService {
25
- readonly name = ServiceNames.HEALTH_CHECK_CACHE;
26
- lifecycle = ServiceLifecycle.UNINITIALIZED;
27
-
28
- // Thread-safe state management using atomic operations
29
- private _pendingCheck: Promise<HealthCheckResult> | null = null;
30
- private _failureCount: number = 0;
31
- private _backoffUntil: number = 0;
32
-
33
- // Backoff configuration
34
- private readonly _backoffBaseMs: number;
35
- private readonly _backoffMaxMs: number;
36
- private readonly _backoffMultiplier: number;
37
-
38
- constructor(options?: {
39
- backoffBaseMs?: number;
40
- backoffMaxMs?: number;
41
- backoffMultiplier?: number;
42
- }) {
43
- this._backoffBaseMs = options?.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
44
- this._backoffMaxMs = options?.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
45
- this._backoffMultiplier = options?.backoffMultiplier ?? DEFAULT_BACKOFF_MULTIPLIER;
46
- }
47
-
48
- async initialize(): Promise<void> {
49
- if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
50
- return;
51
- }
52
-
53
- this.lifecycle = ServiceLifecycle.INITIALIZING;
54
- logger.debug('[HealthCheckService] Initializing...');
55
-
56
- // Initialize with default state
57
- this._pendingCheck = null;
58
- this._failureCount = 0;
59
- this._backoffUntil = 0;
60
-
61
- this.lifecycle = ServiceLifecycle.INITIALIZED;
62
- logger.debug('[HealthCheckService] Initialized');
63
- }
64
-
65
- async dispose(): Promise<void> {
66
- if (this.lifecycle === ServiceLifecycle.DISPOSED) {
67
- return;
68
- }
69
-
70
- this.lifecycle = ServiceLifecycle.DISPOSING;
71
- logger.debug('[HealthCheckService] Disposing...');
72
-
73
- // Clear state
74
- this.clear();
75
-
76
- this.lifecycle = ServiceLifecycle.DISPOSED;
77
- logger.debug('[HealthCheckService] Disposed');
78
- }
79
-
80
- /**
81
- * Get the cached pending health check promise
82
- */
83
- getPendingCheck(): Promise<HealthCheckResult> | null {
84
- return this._pendingCheck;
85
- }
86
-
87
- /**
88
- * Set a pending health check promise
89
- * This should only be called from orchestration code that coordinates health checks
90
- */
91
- setPendingCheck(promise: Promise<HealthCheckResult> | null): void {
92
- this._pendingCheck = promise;
93
-
94
- // Clean up the promise when it settles to avoid memory leaks
95
- if (promise) {
96
- promise.finally(() => {
97
- // Only clear if this is still the same promise
98
- if (this._pendingCheck === promise) {
99
- this._pendingCheck = null;
100
- }
101
- });
102
- }
103
- }
104
-
105
- /**
106
- * Get the failure count
107
- */
108
- getFailureCount(): number {
109
- return this._failureCount;
110
- }
111
-
112
- /**
113
- * Increment the failure count
114
- */
115
- incrementFailureCount(): void {
116
- this._failureCount++;
117
- logger.debug(`[HealthCheckService] Failure count incremented to ${this._failureCount}`);
118
- }
119
-
120
- /**
121
- * Reset the failure count
122
- */
123
- resetFailureCount(): void {
124
- if (this._failureCount > 0) {
125
- logger.debug(`[HealthCheckService] Failure count reset from ${this._failureCount} to 0`);
126
- }
127
- this._failureCount = 0;
128
- }
129
-
130
- /**
131
- * Get the backoff timestamp
132
- */
133
- getBackoffUntil(): number {
134
- return this._backoffUntil;
135
- }
136
-
137
- /**
138
- * Set the backoff timestamp
139
- */
140
- setBackoffUntil(timestamp: number): void {
141
- this._backoffUntil = timestamp;
142
- const remainingMs = Math.max(0, timestamp - Date.now());
143
- if (remainingMs > 0) {
144
- logger.debug(`[HealthCheckService] Backoff set: ${remainingMs}ms remaining`);
145
- }
146
- }
147
-
148
- /**
149
- * Clear all cache state
150
- */
151
- clear(): void {
152
- this._pendingCheck = null;
153
- this._failureCount = 0;
154
- this._backoffUntil = 0;
155
- logger.debug('[HealthCheckService] Cache cleared');
156
- }
157
-
158
- /**
159
- * Check if a backoff is currently active
160
- */
161
- isBackoffActive(): boolean {
162
- return Date.now() < this._backoffUntil;
163
- }
164
-
165
- /**
166
- * Get remaining backoff time in milliseconds
167
- */
168
- getBackoffRemainingMs(): number {
169
- return Math.max(0, this._backoffUntil - Date.now());
170
- }
171
-
172
- /**
173
- * Calculate the next backoff duration based on failure count
174
- * Uses exponential backoff with jitter
175
- */
176
- calculateNextBackoffMs(): number {
177
- // Exponential backoff: base * 2^failures, capped at max
178
- const backoffMs = Math.min(
179
- this._backoffBaseMs * Math.pow(this._backoffMultiplier, this._failureCount),
180
- this._backoffMaxMs
181
- );
182
-
183
- // Add jitter (±25%) to prevent thundering herd
184
- const jitter = backoffMs * 0.25 * (Math.random() * 2 - 1);
185
-
186
- return Math.floor(backoffMs + jitter);
187
- }
188
-
189
- /**
190
- * Set backoff based on current failure count
191
- */
192
- setBackoff(): void {
193
- const backoffMs = this.calculateNextBackoffMs();
194
- this._backoffUntil = Date.now() + backoffMs;
195
- logger.debug(`[HealthCheckService] Backoff set: ${backoffMs}ms (${this._failureCount} failures)`);
196
- }
197
-
198
- /**
199
- * Record a health check failure
200
- * Increments failure count and sets backoff
201
- */
202
- recordFailure(): void {
203
- this.incrementFailureCount();
204
- this.setBackoff();
205
- }
206
-
207
- /**
208
- * Record a health check success
209
- * Resets failure count and clears backoff
210
- */
211
- recordSuccess(): void {
212
- this.resetFailureCount();
213
- this._backoffUntil = 0;
214
- logger.debug('[HealthCheckService] Health check success recorded');
215
- }
216
-
217
- /**
218
- * Get the current backoff state as an object
219
- */
220
- getBackoffState(): {
221
- isActive: boolean;
222
- remainingMs: number;
223
- failureCount: number;
224
- nextBackoffMs: number;
225
- } {
226
- return {
227
- isActive: this.isBackoffActive(),
228
- remainingMs: this.getBackoffRemainingMs(),
229
- failureCount: this._failureCount,
230
- nextBackoffMs: this.calculateNextBackoffMs(),
231
- };
232
- }
233
-
234
- /**
235
- * Wait for backoff to complete if active
236
- * Returns immediately if no backoff is active
237
- */
238
- async waitForBackoff(): Promise<void> {
239
- const remainingMs = this.getBackoffRemainingMs();
240
- if (remainingMs > 0) {
241
- logger.debug(`[HealthCheckService] Waiting for backoff: ${remainingMs}ms`);
242
- await new Promise(resolve => setTimeout(resolve, remainingMs));
243
- logger.debug('[HealthCheckService] Backoff complete');
244
- }
245
- }
246
-
247
- /**
248
- * Check if a new health check should be allowed
249
- * Returns false if backoff is active or a check is already pending
250
- */
251
- shouldAllowCheck(): boolean {
252
- if (this._pendingCheck !== null) {
253
- logger.debug('[HealthCheckService] Check not allowed: already pending');
254
- return false;
255
- }
256
-
257
- if (this.isBackoffActive()) {
258
- const remainingMs = this.getBackoffRemainingMs();
259
- logger.debug(`[HealthCheckService] Check not allowed: backoff active (${remainingMs}ms remaining)`);
260
- return false;
261
- }
262
-
263
- return true;
264
- }
265
- }