@lincoln504/pi-research 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +1334 -750
- package/dist/thread-worker.mjs +194 -136
- package/docs/ARCHITECTURE.md +1 -1
- package/docs/CHANGELOG.md +114 -16
- package/package.json +1 -1
- package/scripts/cleanup.cjs +64 -2
- package/skills/pi-research/SKILL.md +1 -1
- package/src/cleanup/research-cleanup.ts +12 -4
- package/src/cli.ts +228 -70
- package/src/config.ts +78 -2
- package/src/core/interfaces/health-check-interfaces.ts +3 -39
- package/src/core/interfaces/planning-interfaces.ts +8 -0
- package/src/core/interfaces/service-names.ts +0 -1
- package/src/core/interfaces/state-manager-interfaces.ts +10 -2
- package/src/core/llm/agentic-repair.ts +14 -0
- package/src/core/planning-service.ts +43 -1
- package/src/core/service-initialization.ts +6 -14
- package/src/core/service-interfaces.ts +1 -1
- package/src/healthcheck/index.ts +7 -2
- package/src/infrastructure/browser/browser-client.ts +64 -9
- package/src/infrastructure/browser/browser-server.ts +24 -7
- package/src/infrastructure/browser/browser-task-scheduler.ts +58 -12
- package/src/infrastructure/browser/cleanup-utils.ts +50 -2
- package/src/infrastructure/browser/client-agent.ts +6 -6
- package/src/infrastructure/browser/config.ts +14 -3
- package/src/infrastructure/browser/scheduler-factory.ts +18 -3
- package/src/infrastructure/browser/thread-worker-messaging.ts +54 -2
- package/src/infrastructure/browser/thread-worker-types.ts +3 -1
- package/src/infrastructure/browser/thread-worker.mjs +194 -136
- package/src/infrastructure/browser/worker-pool-manager.ts +7 -2
- package/src/infrastructure/embedding/embedding-factory.ts +35 -6
- package/src/infrastructure/embedding/embedding-server.ts +40 -7
- package/src/infrastructure/file-lock-service.ts +72 -2
- package/src/infrastructure/knowledge-store-service.ts +48 -3
- package/src/infrastructure/process-lifecycle-service.ts +28 -5
- package/src/infrastructure/research-run-semaphore.ts +37 -3
- package/src/infrastructure/state/state-browser-api.ts +5 -2
- package/src/infrastructure/state/state-browser-manager.ts +15 -1
- package/src/infrastructure/state/state-manager-service.ts +4 -4
- package/src/infrastructure/state/state-manager.ts +15 -3
- package/src/knowledge/chunker.ts +88 -1
- package/src/knowledge/embedder.ts +136 -43
- package/src/knowledge/store.ts +66 -12
- package/src/logger.ts +5 -1
- package/src/observers/research-observer-impl.ts +31 -0
- package/src/orchestration/abort-utils.ts +60 -0
- package/src/orchestration/deep-research-orchestrator.ts +12 -2
- package/src/orchestration/headless-observer.ts +8 -0
- package/src/orchestration/research-orchestration-service.ts +63 -11
- package/src/orchestration/research-session-service.ts +37 -11
- package/src/orchestration/researcher-executor.ts +100 -20
- package/src/orchestration/researcher.ts +39 -20
- package/src/orchestration/session-state.ts +11 -0
- package/src/sdk.ts +125 -19
- package/src/security/index.ts +51 -30
- package/src/security/nvd-client.ts +7 -2
- package/src/skill-install/skill-installer.ts +18 -2
- package/src/stackexchange/index.ts +20 -5
- package/src/stackexchange/output/table.ts +7 -2
- package/src/stackexchange/rest-client.ts +35 -1
- package/src/tools/research-tool-definition.ts +8 -2
- package/src/tools/scrape.ts +48 -25
- package/src/tools/search.ts +10 -1
- package/src/tools/security.ts +68 -7
- package/src/tui/research-health.ts +18 -2
- package/src/utils/circuit-breaker.ts +9 -2
- package/src/utils/disk-space-checker.ts +11 -6
- package/src/utils/json-utils.ts +116 -76
- package/src/utils/llm-usage.ts +17 -1
- package/src/utils/log-rotation.ts +42 -10
- package/src/utils/log-utils.ts +40 -0
- package/src/utils/metrics-summary.ts +27 -9
- package/src/utils/stdio-capture.ts +63 -22
- package/src/utils/text-utils.ts +8 -1
- package/src/utils/tool-usage-tracker.ts +9 -2
- package/src/web-research/browser-search.ts +26 -7
- package/src/web-research/scraper-utils.ts +13 -0
- package/src/web-research/search.ts +6 -2
- package/src/web-research/types.ts +4 -2
- package/src/web-research/web-scraper.ts +65 -4
- package/src/core/health-check-service.ts +0 -265
|
@@ -1,265 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Health Check Service
|
|
3
|
-
*
|
|
4
|
-
* Manages health check cache state and backoff logic.
|
|
5
|
-
* Provides thread-safe access to health check state and prevents
|
|
6
|
-
* thundering herd problems during health checks.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import type { IHealthCheckService, HealthCheckResult } from './service-interfaces.ts';
|
|
10
|
-
import { ServiceNames } from './service-interfaces.ts';
|
|
11
|
-
import { ServiceLifecycle } from './service-registry.ts';
|
|
12
|
-
import { logger } from '../logger.ts';
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Default backoff configuration
|
|
16
|
-
*/
|
|
17
|
-
const DEFAULT_BACKOFF_BASE_MS = 1000; // 1 second
|
|
18
|
-
const DEFAULT_BACKOFF_MAX_MS = 60000; // 1 minute
|
|
19
|
-
const DEFAULT_BACKOFF_MULTIPLIER = 2;
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Health Check Service Implementation
|
|
23
|
-
*/
|
|
24
|
-
export class HealthCheckService implements IHealthCheckService {
|
|
25
|
-
readonly name = ServiceNames.HEALTH_CHECK_CACHE;
|
|
26
|
-
lifecycle = ServiceLifecycle.UNINITIALIZED;
|
|
27
|
-
|
|
28
|
-
// Thread-safe state management using atomic operations
|
|
29
|
-
private _pendingCheck: Promise<HealthCheckResult> | null = null;
|
|
30
|
-
private _failureCount: number = 0;
|
|
31
|
-
private _backoffUntil: number = 0;
|
|
32
|
-
|
|
33
|
-
// Backoff configuration
|
|
34
|
-
private readonly _backoffBaseMs: number;
|
|
35
|
-
private readonly _backoffMaxMs: number;
|
|
36
|
-
private readonly _backoffMultiplier: number;
|
|
37
|
-
|
|
38
|
-
constructor(options?: {
|
|
39
|
-
backoffBaseMs?: number;
|
|
40
|
-
backoffMaxMs?: number;
|
|
41
|
-
backoffMultiplier?: number;
|
|
42
|
-
}) {
|
|
43
|
-
this._backoffBaseMs = options?.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
|
|
44
|
-
this._backoffMaxMs = options?.backoffMaxMs ?? DEFAULT_BACKOFF_MAX_MS;
|
|
45
|
-
this._backoffMultiplier = options?.backoffMultiplier ?? DEFAULT_BACKOFF_MULTIPLIER;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
async initialize(): Promise<void> {
|
|
49
|
-
if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
|
|
50
|
-
return;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
this.lifecycle = ServiceLifecycle.INITIALIZING;
|
|
54
|
-
logger.debug('[HealthCheckService] Initializing...');
|
|
55
|
-
|
|
56
|
-
// Initialize with default state
|
|
57
|
-
this._pendingCheck = null;
|
|
58
|
-
this._failureCount = 0;
|
|
59
|
-
this._backoffUntil = 0;
|
|
60
|
-
|
|
61
|
-
this.lifecycle = ServiceLifecycle.INITIALIZED;
|
|
62
|
-
logger.debug('[HealthCheckService] Initialized');
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
async dispose(): Promise<void> {
|
|
66
|
-
if (this.lifecycle === ServiceLifecycle.DISPOSED) {
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
this.lifecycle = ServiceLifecycle.DISPOSING;
|
|
71
|
-
logger.debug('[HealthCheckService] Disposing...');
|
|
72
|
-
|
|
73
|
-
// Clear state
|
|
74
|
-
this.clear();
|
|
75
|
-
|
|
76
|
-
this.lifecycle = ServiceLifecycle.DISPOSED;
|
|
77
|
-
logger.debug('[HealthCheckService] Disposed');
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Get the cached pending health check promise
|
|
82
|
-
*/
|
|
83
|
-
getPendingCheck(): Promise<HealthCheckResult> | null {
|
|
84
|
-
return this._pendingCheck;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Set a pending health check promise
|
|
89
|
-
* This should only be called from orchestration code that coordinates health checks
|
|
90
|
-
*/
|
|
91
|
-
setPendingCheck(promise: Promise<HealthCheckResult> | null): void {
|
|
92
|
-
this._pendingCheck = promise;
|
|
93
|
-
|
|
94
|
-
// Clean up the promise when it settles to avoid memory leaks
|
|
95
|
-
if (promise) {
|
|
96
|
-
promise.finally(() => {
|
|
97
|
-
// Only clear if this is still the same promise
|
|
98
|
-
if (this._pendingCheck === promise) {
|
|
99
|
-
this._pendingCheck = null;
|
|
100
|
-
}
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Get the failure count
|
|
107
|
-
*/
|
|
108
|
-
getFailureCount(): number {
|
|
109
|
-
return this._failureCount;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
/**
|
|
113
|
-
* Increment the failure count
|
|
114
|
-
*/
|
|
115
|
-
incrementFailureCount(): void {
|
|
116
|
-
this._failureCount++;
|
|
117
|
-
logger.debug(`[HealthCheckService] Failure count incremented to ${this._failureCount}`);
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* Reset the failure count
|
|
122
|
-
*/
|
|
123
|
-
resetFailureCount(): void {
|
|
124
|
-
if (this._failureCount > 0) {
|
|
125
|
-
logger.debug(`[HealthCheckService] Failure count reset from ${this._failureCount} to 0`);
|
|
126
|
-
}
|
|
127
|
-
this._failureCount = 0;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* Get the backoff timestamp
|
|
132
|
-
*/
|
|
133
|
-
getBackoffUntil(): number {
|
|
134
|
-
return this._backoffUntil;
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
/**
|
|
138
|
-
* Set the backoff timestamp
|
|
139
|
-
*/
|
|
140
|
-
setBackoffUntil(timestamp: number): void {
|
|
141
|
-
this._backoffUntil = timestamp;
|
|
142
|
-
const remainingMs = Math.max(0, timestamp - Date.now());
|
|
143
|
-
if (remainingMs > 0) {
|
|
144
|
-
logger.debug(`[HealthCheckService] Backoff set: ${remainingMs}ms remaining`);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/**
|
|
149
|
-
* Clear all cache state
|
|
150
|
-
*/
|
|
151
|
-
clear(): void {
|
|
152
|
-
this._pendingCheck = null;
|
|
153
|
-
this._failureCount = 0;
|
|
154
|
-
this._backoffUntil = 0;
|
|
155
|
-
logger.debug('[HealthCheckService] Cache cleared');
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
/**
|
|
159
|
-
* Check if a backoff is currently active
|
|
160
|
-
*/
|
|
161
|
-
isBackoffActive(): boolean {
|
|
162
|
-
return Date.now() < this._backoffUntil;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
/**
|
|
166
|
-
* Get remaining backoff time in milliseconds
|
|
167
|
-
*/
|
|
168
|
-
getBackoffRemainingMs(): number {
|
|
169
|
-
return Math.max(0, this._backoffUntil - Date.now());
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
/**
|
|
173
|
-
* Calculate the next backoff duration based on failure count
|
|
174
|
-
* Uses exponential backoff with jitter
|
|
175
|
-
*/
|
|
176
|
-
calculateNextBackoffMs(): number {
|
|
177
|
-
// Exponential backoff: base * 2^failures, capped at max
|
|
178
|
-
const backoffMs = Math.min(
|
|
179
|
-
this._backoffBaseMs * Math.pow(this._backoffMultiplier, this._failureCount),
|
|
180
|
-
this._backoffMaxMs
|
|
181
|
-
);
|
|
182
|
-
|
|
183
|
-
// Add jitter (±25%) to prevent thundering herd
|
|
184
|
-
const jitter = backoffMs * 0.25 * (Math.random() * 2 - 1);
|
|
185
|
-
|
|
186
|
-
return Math.floor(backoffMs + jitter);
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/**
|
|
190
|
-
* Set backoff based on current failure count
|
|
191
|
-
*/
|
|
192
|
-
setBackoff(): void {
|
|
193
|
-
const backoffMs = this.calculateNextBackoffMs();
|
|
194
|
-
this._backoffUntil = Date.now() + backoffMs;
|
|
195
|
-
logger.debug(`[HealthCheckService] Backoff set: ${backoffMs}ms (${this._failureCount} failures)`);
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
/**
|
|
199
|
-
* Record a health check failure
|
|
200
|
-
* Increments failure count and sets backoff
|
|
201
|
-
*/
|
|
202
|
-
recordFailure(): void {
|
|
203
|
-
this.incrementFailureCount();
|
|
204
|
-
this.setBackoff();
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/**
|
|
208
|
-
* Record a health check success
|
|
209
|
-
* Resets failure count and clears backoff
|
|
210
|
-
*/
|
|
211
|
-
recordSuccess(): void {
|
|
212
|
-
this.resetFailureCount();
|
|
213
|
-
this._backoffUntil = 0;
|
|
214
|
-
logger.debug('[HealthCheckService] Health check success recorded');
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
/**
|
|
218
|
-
* Get the current backoff state as an object
|
|
219
|
-
*/
|
|
220
|
-
getBackoffState(): {
|
|
221
|
-
isActive: boolean;
|
|
222
|
-
remainingMs: number;
|
|
223
|
-
failureCount: number;
|
|
224
|
-
nextBackoffMs: number;
|
|
225
|
-
} {
|
|
226
|
-
return {
|
|
227
|
-
isActive: this.isBackoffActive(),
|
|
228
|
-
remainingMs: this.getBackoffRemainingMs(),
|
|
229
|
-
failureCount: this._failureCount,
|
|
230
|
-
nextBackoffMs: this.calculateNextBackoffMs(),
|
|
231
|
-
};
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/**
|
|
235
|
-
* Wait for backoff to complete if active
|
|
236
|
-
* Returns immediately if no backoff is active
|
|
237
|
-
*/
|
|
238
|
-
async waitForBackoff(): Promise<void> {
|
|
239
|
-
const remainingMs = this.getBackoffRemainingMs();
|
|
240
|
-
if (remainingMs > 0) {
|
|
241
|
-
logger.debug(`[HealthCheckService] Waiting for backoff: ${remainingMs}ms`);
|
|
242
|
-
await new Promise(resolve => setTimeout(resolve, remainingMs));
|
|
243
|
-
logger.debug('[HealthCheckService] Backoff complete');
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
/**
|
|
248
|
-
* Check if a new health check should be allowed
|
|
249
|
-
* Returns false if backoff is active or a check is already pending
|
|
250
|
-
*/
|
|
251
|
-
shouldAllowCheck(): boolean {
|
|
252
|
-
if (this._pendingCheck !== null) {
|
|
253
|
-
logger.debug('[HealthCheckService] Check not allowed: already pending');
|
|
254
|
-
return false;
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
if (this.isBackoffActive()) {
|
|
258
|
-
const remainingMs = this.getBackoffRemainingMs();
|
|
259
|
-
logger.debug(`[HealthCheckService] Check not allowed: backoff active (${remainingMs}ms remaining)`);
|
|
260
|
-
return false;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
return true;
|
|
264
|
-
}
|
|
265
|
-
}
|