@lincoln504/pi-research 0.1.12 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +64 -115
- package/dist/cli.mjs +23211 -0
- package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
- package/dist/prompts/research-knowledge-search-extractor.md +41 -0
- package/dist/prompts/research-tool-usage.md +92 -0
- package/dist/prompts/researcher.md +89 -0
- package/dist/prompts/system-coordinator.md +75 -0
- package/dist/prompts/system-lead-evaluator.md +118 -0
- package/dist/thread-worker.mjs +2611 -0
- package/dist/webgpu-probe.mjs +80 -0
- package/docs/AGENT-SKILL.md +102 -0
- package/docs/ARCHITECTURE.md +306 -80
- package/docs/CONFIGURATION.md +249 -0
- package/docs/KNOWLEDGE-STORE.md +213 -0
- package/docs/PI-EXTENSION.md +91 -0
- package/docs/SDK.md +113 -0
- package/docs/THIRD-PARTY-NOTICES.md +64 -0
- package/package.json +102 -36
- package/scripts/build.cjs +140 -0
- package/scripts/cleanup.cjs +149 -0
- package/scripts/prepare.cjs +28 -0
- package/scripts/setup.cjs +134 -0
- package/skills/pi-research/README.md +90 -0
- package/skills/pi-research/SKILL.md +102 -0
- package/skills/pi-research/references/configuration.md +77 -0
- package/skills/pi-research/scripts/run.mjs +237 -0
- package/src/cleanup/research-cleanup.ts +140 -0
- package/src/cli.ts +1044 -0
- package/src/config.ts +1036 -147
- package/src/constants.ts +115 -84
- package/src/core/health-check-service.ts +265 -0
- package/src/core/interfaces/health-check-interfaces.ts +50 -0
- package/src/core/interfaces/knowledge-interfaces.ts +190 -0
- package/src/core/interfaces/observer-interfaces.ts +67 -0
- package/src/core/interfaces/orchestration-interfaces.ts +86 -0
- package/src/core/interfaces/planning-interfaces.ts +100 -0
- package/src/core/interfaces/process-interfaces.ts +37 -0
- package/src/core/interfaces/research-plan-types.ts +52 -0
- package/src/core/interfaces/scheduler-interfaces.ts +91 -0
- package/src/core/interfaces/service-names.ts +39 -0
- package/src/core/interfaces/state-manager-interfaces.ts +29 -0
- package/src/core/interfaces/steering-interfaces.ts +31 -0
- package/src/core/llm/agentic-repair.ts +169 -0
- package/src/core/llm/inject-date.ts +52 -0
- package/src/core/llm/llm-timeout.ts +64 -0
- package/src/core/llm/llm-utils.ts +86 -0
- package/src/core/llm/model-registry-factory.ts +287 -0
- package/src/core/llm/pi-ai-completion.ts +29 -0
- package/src/core/llm/prompts.ts +50 -0
- package/src/core/llm/research-model-resolver.ts +91 -0
- package/src/core/planning-constants.ts +13 -0
- package/src/core/planning-service.ts +649 -0
- package/src/core/planning-utils.ts +299 -0
- package/src/core/scheduler-factory.ts +48 -0
- package/src/core/scheduler-service.ts +123 -0
- package/src/core/service-initialization.ts +194 -0
- package/src/core/service-interfaces.ts +48 -0
- package/src/core/service-registry.ts +659 -0
- package/src/healthcheck/index.ts +271 -265
- package/src/healthcheck/registry.ts +130 -0
- package/src/index.ts +639 -50
- package/src/infrastructure/browser/browser-cleanup.ts +309 -0
- package/src/infrastructure/browser/browser-client.ts +199 -0
- package/src/infrastructure/browser/browser-error-utils.ts +160 -0
- package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
- package/src/infrastructure/browser/browser-server.ts +174 -0
- package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
- package/src/infrastructure/browser/cleanup-utils.ts +155 -0
- package/src/infrastructure/browser/client-agent.ts +31 -0
- package/src/infrastructure/browser/config.ts +288 -0
- package/src/infrastructure/browser/ensure-browser.ts +205 -0
- package/src/infrastructure/browser/index.ts +37 -0
- package/src/infrastructure/browser/priority-task-queue.ts +226 -0
- package/src/infrastructure/browser/scheduler-factory.ts +354 -0
- package/src/infrastructure/browser/task-execution-service.ts +209 -0
- package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
- package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
- package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
- package/src/infrastructure/browser/thread-worker-types.ts +25 -0
- package/src/infrastructure/browser/thread-worker.mjs +2611 -0
- package/src/infrastructure/browser/thread-worker.ts +168 -0
- package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
- package/src/infrastructure/embedding/embedding-client.ts +205 -0
- package/src/infrastructure/embedding/embedding-factory.ts +324 -0
- package/src/infrastructure/embedding/embedding-server.ts +535 -0
- package/src/infrastructure/file-lock-service.ts +592 -0
- package/src/infrastructure/gpu-resource-service.ts +191 -0
- package/src/infrastructure/knowledge-store-service.ts +475 -0
- package/src/infrastructure/metrics-service.ts +129 -0
- package/src/infrastructure/process-lifecycle-service.ts +253 -0
- package/src/infrastructure/scheduler-factory-service.ts +58 -0
- package/src/infrastructure/service-initialization.ts +292 -0
- package/src/infrastructure/state/state-backup-manager.ts +290 -0
- package/src/infrastructure/state/state-browser-api.ts +62 -0
- package/src/infrastructure/state/state-browser-manager.ts +70 -0
- package/src/infrastructure/state/state-manager-service.ts +264 -0
- package/src/infrastructure/state/state-manager.ts +547 -0
- package/src/infrastructure/state/state-metrics.ts +92 -0
- package/src/infrastructure/state/state-migration.ts +103 -0
- package/src/infrastructure/state/state-path-configuration.ts +119 -0
- package/src/infrastructure/state/state-session-api.ts +94 -0
- package/src/infrastructure/state/state-session-manager.ts +127 -0
- package/src/infrastructure/state/state-validator.ts +93 -0
- package/src/infrastructure/types/state-types.ts +85 -0
- package/src/knowledge/chunker.ts +157 -0
- package/src/knowledge/embedder-init.ts +413 -0
- package/src/knowledge/embedder-utils.ts +296 -0
- package/src/knowledge/embedder.ts +592 -0
- package/src/knowledge/index.ts +126 -0
- package/src/knowledge/lancedb-loader.ts +35 -0
- package/src/knowledge/migration.ts +17 -0
- package/src/knowledge/model-config.ts +132 -0
- package/src/knowledge/onnx-env.ts +48 -0
- package/src/knowledge/store-operations.ts +291 -0
- package/src/knowledge/store-schema.ts +79 -0
- package/src/knowledge/store.ts +1251 -0
- package/src/knowledge/transformers-loader.ts +31 -0
- package/src/knowledge/webgpu-probe.mjs +80 -0
- package/src/knowledge/webgpu-viability.ts +255 -0
- package/src/knowledge/writer-queue.ts +252 -0
- package/src/logger.ts +316 -122
- package/src/observers/research-observer-impl.ts +459 -0
- package/src/orchestration/deep-research-orchestrator.ts +524 -773
- package/src/orchestration/deep-research-types.ts +47 -35
- package/src/orchestration/headless-observer.ts +114 -0
- package/src/orchestration/orchestration-types.ts +33 -0
- package/src/orchestration/quick-research-orchestrator.ts +463 -0
- package/src/orchestration/research-orchestration-service.ts +549 -0
- package/src/orchestration/research-session-service.ts +178 -0
- package/src/orchestration/research-synthesis-service.ts +348 -0
- package/src/orchestration/researcher-executor.ts +394 -0
- package/src/orchestration/researcher.ts +115 -42
- package/src/orchestration/service-initialization.ts +109 -0
- package/src/orchestration/session-context.ts +7 -3
- package/src/orchestration/session-state.ts +672 -0
- package/src/prompts/research-knowledge-relevance-triage.md +35 -0
- package/src/prompts/research-knowledge-search-extractor.md +41 -0
- package/src/prompts/research-tool-usage.md +63 -16
- package/src/prompts/researcher.md +59 -136
- package/src/prompts/system-coordinator.md +64 -13
- package/src/prompts/system-lead-evaluator.md +82 -189
- package/src/research-config.ts +872 -0
- package/src/sdk.ts +847 -0
- package/src/security/cisa-kev.ts +72 -9
- package/src/security/cvss.ts +92 -0
- package/src/security/ecosystem.ts +45 -0
- package/src/security/github-advisories.ts +168 -428
- package/src/security/github-advisory-types.ts +282 -0
- package/src/security/index.ts +63 -177
- package/src/security/nvd-client.ts +550 -0
- package/src/security/nvd-types.ts +179 -0
- package/src/security/osv-client.ts +246 -0
- package/src/security/osv-types.ts +285 -0
- package/src/security/types.ts +67 -53
- package/src/skill-install/skill-installer.ts +519 -0
- package/src/stackexchange/index.ts +81 -27
- package/src/stackexchange/output/compact.ts +7 -6
- package/src/stackexchange/output/table.ts +11 -7
- package/src/stackexchange/queries.ts +0 -10
- package/src/stackexchange/rest-client.ts +141 -66
- package/src/stackexchange/types.ts +0 -17
- package/src/tool.ts +15 -423
- package/src/tools/grep.ts +32 -310
- package/src/tools/health-tool-definition.ts +101 -0
- package/src/tools/index.ts +38 -9
- package/src/tools/research-knowledge-search.ts +974 -0
- package/src/tools/research-knowledge-types.ts +77 -0
- package/src/tools/research-tool-definition.ts +494 -0
- package/src/tools/scrape.ts +286 -388
- package/src/tools/search.ts +98 -153
- package/src/tools/security.ts +107 -80
- package/src/tools/stackexchange.ts +58 -35
- package/src/tools/youtube-transcript.ts +163 -0
- package/src/tui/knowledge-search-panel.ts +86 -0
- package/src/tui/pulse.ts +86 -0
- package/src/tui/research-health.ts +124 -0
- package/src/tui/research-panel-color-utils.ts +340 -0
- package/src/tui/research-panel-format.ts +54 -0
- package/src/tui/research-panel-state.ts +404 -0
- package/src/tui/research-panel-wave.ts +280 -0
- package/src/tui/research-panel.ts +339 -435
- package/src/tui/research-tui-manager.ts +233 -0
- package/src/tui/tui-controller.ts +137 -0
- package/src/types/extension-context.ts +18 -49
- package/src/types/index.ts +61 -0
- package/src/types/llm.ts +68 -29
- package/src/types/research-panel-types.ts +61 -0
- package/src/utils/circuit-breaker.ts +146 -0
- package/src/utils/citation-utils.ts +97 -0
- package/src/utils/disk-space-checker.ts +55 -0
- package/src/utils/error-tracker.ts +265 -0
- package/src/utils/host-config.ts +45 -0
- package/src/utils/input-validation.ts +8 -10
- package/src/utils/json-utils.ts +150 -16
- package/src/utils/llm-usage.ts +54 -0
- package/src/utils/log-rotation.ts +125 -0
- package/src/utils/log-utils.ts +159 -0
- package/src/utils/make-resource-loader.ts +3 -1
- package/src/utils/metrics-summary.ts +459 -0
- package/src/utils/metrics.ts +302 -0
- package/src/utils/research-export.ts +28 -12
- package/src/utils/safe-unref.ts +31 -0
- package/src/utils/shared-links.ts +195 -90
- package/src/utils/shutdown-manager.ts +65 -0
- package/src/utils/stdio-capture.ts +407 -0
- package/src/utils/text-utils.ts +264 -7
- package/src/utils/tool-usage-tracker.ts +83 -16
- package/src/utils/url-utils.ts +255 -0
- package/src/utils/user-agent.ts +28 -0
- package/src/web-research/browser-search.ts +179 -0
- package/src/web-research/retry-utils.ts +173 -77
- package/src/web-research/scraper-types.ts +72 -0
- package/src/web-research/scraper-utils.ts +434 -0
- package/src/web-research/search.ts +78 -218
- package/src/web-research/types.ts +9 -24
- package/src/web-research/utils.ts +2 -143
- package/src/web-research/web-scraper.ts +477 -0
- package/src/youtube/index.ts +154 -0
- package/src/youtube/potoken.ts +259 -0
- package/src/youtube/transcript-client.ts +390 -0
- package/src/youtube/video-id.ts +118 -0
- package/config/searxng/default-settings.yml +0 -189
- package/config/searxng/limiter.toml +0 -34
- package/config/tooling/eslint.config.js +0 -73
- package/config/tooling/vitest.config.integration.ts +0 -17
- package/config/tooling/vitest.config.ts +0 -40
- package/config/tooling/vitest.config.unit.ts +0 -12
- package/docs/assets/README-banner.jpg +0 -0
- package/scripts/setup.js +0 -174
- package/src/infrastructure/searxng-lifecycle.ts +0 -577
- package/src/infrastructure/searxng-manager.ts +0 -1183
- package/src/infrastructure/state-manager.ts +0 -918
- package/src/orchestration/deep-research-reducer.ts +0 -129
- package/src/orchestration/id-utils.ts +0 -63
- package/src/security/nvd.ts +0 -532
- package/src/security/osv.ts +0 -487
- package/src/stackexchange/cache.ts +0 -111
- package/src/stackexchange/output/index.ts +0 -7
- package/src/stackexchange/output/json.ts +0 -21
- package/src/tui/searxng-status.ts +0 -80
- package/src/utils/inject-date.ts +0 -29
- package/src/utils/research-error.ts +0 -205
- package/src/utils/searxng-config.ts +0 -94
- package/src/utils/session-state.ts +0 -295
- package/src/web-research/dependencies.ts +0 -20
- package/src/web-research/scrapers.ts +0 -935
- package/tsconfig.json +0 -40
package/src/healthcheck/index.ts
CHANGED
|
@@ -1,309 +1,315 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Health Check Module
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* Fast timeouts, simple checks (just verify tools respond with content).
|
|
4
|
+
* Validates system capabilities using the central health registry.
|
|
5
|
+
* This module is stateless and does not cache results.
|
|
7
6
|
*/
|
|
8
7
|
|
|
9
|
-
import {
|
|
8
|
+
import { execFileSync } from 'node:child_process';
|
|
10
9
|
import { getConfig } from '../config.ts';
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import
|
|
18
|
-
|
|
19
|
-
// Health check cache to avoid running multiple health checks concurrently
|
|
20
|
-
interface CachedHealthCheck {
|
|
21
|
-
result: HealthCheckResult;
|
|
22
|
-
timestamp: number;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
let cachedHealthCheck: CachedHealthCheck | null = null;
|
|
26
|
-
const HEALTH_CHECK_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
27
|
-
let healthCheckPromise: Promise<HealthCheckResult> | null = null;
|
|
28
|
-
|
|
29
|
-
// Get timeout from config or use defaults
|
|
30
|
-
const config = getConfig();
|
|
31
|
-
const SEARCH_TIMEOUT_MS = config.HEALTH_CHECK_TIMEOUT_MS || 15000; // 15s max for search
|
|
32
|
-
const SCRAPE_TIMEOUT_MS = config.HEALTH_CHECK_TIMEOUT_MS ? config.HEALTH_CHECK_TIMEOUT_MS + 5000 : 20000; // 5s longer than search
|
|
33
|
-
|
|
34
|
-
export interface HealthCheckResult {
|
|
35
|
-
success: boolean;
|
|
36
|
-
searchOk: boolean;
|
|
37
|
-
scrapeOk: boolean;
|
|
38
|
-
error?: string;
|
|
39
|
-
details: {
|
|
40
|
-
searchQuery?: string;
|
|
41
|
-
searchResultCount?: number;
|
|
42
|
-
searchDurationMs?: number;
|
|
43
|
-
workingEngines?: string[];
|
|
44
|
-
workingEngineCount?: number;
|
|
45
|
-
scrapedUrl?: string;
|
|
46
|
-
scrapedContentLength?: number;
|
|
47
|
-
scrapedDurationMs?: number;
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* QUICK scrape validation: just check content exists and is readable
|
|
53
|
-
*/
|
|
54
|
-
function validateScrapeOutput(markdown: string): { valid: boolean; error?: string } {
|
|
55
|
-
if (typeof markdown !== 'string') {
|
|
56
|
-
return { valid: false, error: 'Content not a string' };
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
const trimmed = markdown.trim();
|
|
60
|
-
if (trimmed.length === 0) {
|
|
61
|
-
return { valid: false, error: 'Content is empty' };
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// Just check it has some readable text (not garbage)
|
|
65
|
-
if (!/[a-zA-Z0-9]{10,}/.test(markdown)) {
|
|
66
|
-
return { valid: false, error: 'Content not readable' };
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
return { valid: true };
|
|
70
|
-
}
|
|
10
|
+
import { isBrowserAvailable, resolveHeadlessMode } from '../infrastructure/browser/config.ts';
|
|
11
|
+
import { runBrowserHealthCheck } from '../infrastructure/browser/task-execution-service.ts';
|
|
12
|
+
import { getService, tryGetServiceContainerFromCtx, getServiceContainer, ServiceLifecycle } from '../core/service-registry.ts';
|
|
13
|
+
import type { ServiceContainer } from '../core/service-registry.ts';
|
|
14
|
+
import { ServiceNames } from '../core/service-interfaces.ts';
|
|
15
|
+
import type { IStateManager, IHealthRegistryService, IKnowledgeStoreService, ISchedulerService } from '../core/service-interfaces.ts';
|
|
16
|
+
import { healthRegistry as globalHealthRegistry } from './registry.ts';
|
|
17
|
+
import { isNativeStackUnavailableError } from '../knowledge/embedder-utils.ts';
|
|
71
18
|
|
|
72
19
|
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
* an external caller to have registered it beforehand.
|
|
20
|
+
* Standalone BrowserCapability check, exported so tests can call it directly
|
|
21
|
+
* without going through the full registry.
|
|
76
22
|
*/
|
|
77
|
-
function
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
23
|
+
export async function checkBrowserCapability(): Promise<{ healthy: boolean; error?: string; diagnostic?: Record<string, any> }> {
|
|
24
|
+
const mockMode = process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' &&
|
|
25
|
+
process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
|
|
26
|
+
if (isBrowserAvailable() || mockMode) {
|
|
27
|
+
// resolveHeadlessMode() selects 'virtual' (Xvfb) only under the explicit
|
|
28
|
+
// PI_RESEARCH_USE_XVFB=true opt-in on a bare Linux TTY. In that case camoufox
|
|
29
|
+
// spawns Xvfb internally, so fail early (before research starts) if it is not
|
|
30
|
+
// installed. The default path is headless:true, which needs no Xvfb.
|
|
31
|
+
if (!mockMode && resolveHeadlessMode() === 'virtual') {
|
|
32
|
+
try {
|
|
33
|
+
execFileSync('which', ['Xvfb'], { stdio: 'ignore' });
|
|
34
|
+
} catch {
|
|
35
|
+
return {
|
|
36
|
+
healthy: false,
|
|
37
|
+
error: 'No display server found on Linux (DISPLAY not set) and Xvfb is not installed. Run: sudo apt install xvfb',
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return { healthy: true, diagnostic: { status: mockMode ? 'mocked' : 'available' } };
|
|
42
|
+
} else {
|
|
43
|
+
return { healthy: false, error: 'Camoufox (browser) not found. Run "npx camoufox-js fetch" to install the browser.' };
|
|
81
44
|
}
|
|
82
|
-
|
|
83
|
-
setSearxngManager(manager);
|
|
84
45
|
}
|
|
85
46
|
|
|
86
47
|
/**
|
|
87
|
-
*
|
|
88
|
-
* Runs the actual health check tests
|
|
48
|
+
* Register all health checks with a registry
|
|
89
49
|
*/
|
|
90
|
-
|
|
91
|
-
|
|
50
|
+
export function registerHealthChecks(registry: IHealthRegistryService, container: ServiceContainer = getServiceContainer()): void {
|
|
51
|
+
const config = getConfig();
|
|
52
|
+
const healthTimeoutMs = config.HEALTH_CHECK_TIMEOUT_MS;
|
|
92
53
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
searchOk: false,
|
|
96
|
-
scrapeOk: false,
|
|
97
|
-
error: undefined,
|
|
98
|
-
details: {},
|
|
99
|
-
};
|
|
54
|
+
// Register Browser Capability Check
|
|
55
|
+
registry.register('BrowserCapability', checkBrowserCapability, { timeoutMs: healthTimeoutMs, critical: true });
|
|
100
56
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
57
|
+
// Register Browser Runtime Check
|
|
58
|
+
registry.register('BrowserRuntime', async (options) => {
|
|
59
|
+
try {
|
|
60
|
+
const scheduler = await getService<ISchedulerService>(ServiceNames.SCHEDULER, { container }, container);
|
|
61
|
+
|
|
62
|
+
// Check if initialized but idle (unless forced)
|
|
63
|
+
if (!scheduler.isReady() && !options?.force) {
|
|
64
|
+
return { healthy: true, diagnostic: { status: 'ready (idle)' } };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Perform a real test
|
|
68
|
+
const searchResult = await runBrowserHealthCheck(undefined, 1, undefined, container);
|
|
69
|
+
if (searchResult.success) {
|
|
70
|
+
return { healthy: true, diagnostic: { status: options?.force && !scheduler.isReady() ? 'initialized & active' : 'active' } };
|
|
71
|
+
} else {
|
|
72
|
+
return { healthy: false, error: 'Browser healthcheck failed: worker reported failure or page failed to load.' };
|
|
73
|
+
}
|
|
74
|
+
} catch (e) {
|
|
75
|
+
return { healthy: false, error: `Browser healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
|
|
76
|
+
}
|
|
77
|
+
// Honor the configured health timeout but keep a 150s floor: a cold browser
|
|
78
|
+
// launch (camoufox download/spawn) can legitimately take minutes, so we never
|
|
79
|
+
// go below that, but a user who raises HEALTH_CHECK_TIMEOUT_MS past it is
|
|
80
|
+
// respected (same pattern as the KnowledgeStore check's Math.max floor).
|
|
81
|
+
}, { timeoutMs: Math.max(healthTimeoutMs, 150000), critical: true });
|
|
82
|
+
|
|
83
|
+
// Register Knowledge Store Check
|
|
84
|
+
registry.register('KnowledgeStore', async (options) => {
|
|
85
|
+
// Idle fast-path — mirrors the BrowserRuntime check above. Read the ALREADY
|
|
86
|
+
// resolved instance via tryGet(), which does NOT initialize the service;
|
|
87
|
+
// getService() (below) would run the full LanceDB open + ONNX model load +
|
|
88
|
+
// WebGPU probe, so a non-forced health check must never take that path on a
|
|
89
|
+
// cold store. Only inspect for real when the store is already initialized OR an
|
|
90
|
+
// explicit probe is requested (force).
|
|
91
|
+
const existing = container.tryGet<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE);
|
|
92
|
+
const alreadyInitialized = existing?.lifecycle === ServiceLifecycle.INITIALIZED;
|
|
93
|
+
if (!alreadyInitialized && !options?.force) {
|
|
94
|
+
const idleCwd = existing?.getCwd() ?? process.cwd();
|
|
95
|
+
if (getConfig(idleCwd).KNOWLEDGE_STORE_MODE === 'none') {
|
|
96
|
+
return { healthy: true, diagnostic: { status: 'disabled in config' } };
|
|
97
|
+
}
|
|
98
|
+
// A store that resolved-then-disabled reports WHY, accurately: 'native' is a
|
|
99
|
+
// permanent platform capability gap; 'mode' is a revivable Knowledge-Mode=none
|
|
100
|
+
// (which may have since been re-enabled — it re-initializes on next real use).
|
|
101
|
+
// Never claim "native unavailable" for a mode-disable on a perfectly capable host.
|
|
102
|
+
if (existing?.lifecycle === ServiceLifecycle.DISABLED) {
|
|
103
|
+
const reason = existing.getDisabledReason();
|
|
104
|
+
return { healthy: true, diagnostic: {
|
|
105
|
+
status: reason === 'native'
|
|
106
|
+
? 'disabled (native embedding/vector stack unavailable on this platform)'
|
|
107
|
+
: 'disabled',
|
|
108
|
+
} };
|
|
109
|
+
}
|
|
110
|
+
// A DISPOSED/DISPOSING service is not "ready" — report it accurately without
|
|
111
|
+
// initializing anything (it re-resolves lazily on next real use, so still healthy).
|
|
112
|
+
const status = existing?.lifecycle === ServiceLifecycle.INITIALIZING
|
|
113
|
+
? 'initializing'
|
|
114
|
+
: (existing?.lifecycle === ServiceLifecycle.DISPOSED || existing?.lifecycle === ServiceLifecycle.DISPOSING)
|
|
115
|
+
? 'disposed (not running)'
|
|
116
|
+
: 'ready (idle)';
|
|
117
|
+
return { healthy: true, diagnostic: { status } };
|
|
109
118
|
}
|
|
110
119
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
//
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
let queryResults;
|
|
120
|
+
// Full inspection (already-initialized OR forced): resolve via getService —
|
|
121
|
+
// cheap when warm (early-returns), full init only on a forced cold probe.
|
|
122
|
+
// Resolve cwd from the live service (it captured ctx.cwd at init) so the
|
|
123
|
+
// health check reads the SAME config the store actually uses. Reading
|
|
124
|
+
// process.cwd() here silently ignored SDK/CLI cwd overrides.
|
|
125
|
+
let service: IKnowledgeStoreService | null = null;
|
|
126
|
+
let initError: unknown = null;
|
|
119
127
|
try {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
);
|
|
125
|
-
} catch (err) {
|
|
126
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
127
|
-
result.error = `Search request failed: ${msg}. Network may be unreachable or SearXNG not responding.`;
|
|
128
|
-
logger.error('[healthcheck] Search error:', result.error);
|
|
129
|
-
return result;
|
|
128
|
+
service = await getService<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE, { container }, container);
|
|
129
|
+
} catch (e) {
|
|
130
|
+
// service stays null; classify the failure below (native-unavailable vs real).
|
|
131
|
+
initError = e;
|
|
130
132
|
}
|
|
131
|
-
|
|
132
|
-
const
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
result.error = 'Search returned empty result object (SearXNG not initialized?)';
|
|
136
|
-
logger.error('[healthcheck]', result.error);
|
|
137
|
-
return result;
|
|
133
|
+
const cwd = service?.getCwd() ?? process.cwd();
|
|
134
|
+
const config = getConfig(cwd);
|
|
135
|
+
if (config.KNOWLEDGE_STORE_MODE === 'none') {
|
|
136
|
+
return { healthy: true, diagnostic: { status: 'disabled in config' } };
|
|
138
137
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
138
|
+
// The native ML/vector stack ships no prebuilt for some platforms (Intel macOS
|
|
139
|
+
// / darwin-x64 has no onnxruntime-node binary nor @lancedb native binding). That
|
|
140
|
+
// is a permanent capability gap, not a fault — report the store as DISABLED
|
|
141
|
+
// (healthy) so research still runs without the optional cache. Otherwise this
|
|
142
|
+
// critical component drags overall health to 'unhealthy' and quick (depth-0)
|
|
143
|
+
// research aborts with "Research cannot start" for any fresh user on such a host.
|
|
144
|
+
if (isNativeStackUnavailableError(initError)) {
|
|
145
|
+
return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
|
|
146
146
|
}
|
|
147
|
-
|
|
148
|
-
//
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
|
|
152
|
-
if (
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
147
|
+
// getService can SUCCEED on a native-unavailable platform: the service memoizes
|
|
148
|
+
// the failure as DISABLED instead of throwing, so initError stays null and the
|
|
149
|
+
// getStore()/getEmbedder() null checks below would report a false UNHEALTHY
|
|
150
|
+
// ("Embedder service not available") on a forced check — contradicting the
|
|
151
|
+
// non-forced path's correct "disabled" verdict above. Report the same verdict.
|
|
152
|
+
if (service?.lifecycle === ServiceLifecycle.DISABLED) {
|
|
153
|
+
const reason = service.getDisabledReason();
|
|
154
|
+
return { healthy: true, diagnostic: {
|
|
155
|
+
status: reason === 'native'
|
|
156
|
+
? 'disabled (native embedding/vector stack unavailable on this platform)'
|
|
157
|
+
: 'disabled',
|
|
158
|
+
} };
|
|
156
159
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
generalEngines.includes((r.engine || '').toLowerCase())
|
|
161
|
-
);
|
|
162
|
-
|
|
163
|
-
// Count how many engines returned at least one result
|
|
164
|
-
const engineCounts = new Map<string, number>();
|
|
165
|
-
generalEngines.forEach(e => engineCounts.set(e, 0));
|
|
166
|
-
generalResults.forEach((r: SearXNGResult) => {
|
|
167
|
-
const engine = (r.engine || '').toLowerCase();
|
|
168
|
-
if (engineCounts.has(engine)) {
|
|
169
|
-
engineCounts.set(engine, (engineCounts.get(engine) || 0) + 1);
|
|
160
|
+
try {
|
|
161
|
+
if (!service) {
|
|
162
|
+
return { healthy: false, error: 'Knowledge store service not available' };
|
|
170
163
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
const workingEngineCount = workingEngines.length;
|
|
164
|
+
const store = await service.getStore();
|
|
165
|
+
const counts = store ? await store.countScoped() : { local: 0, global: 0, projects: 0 };
|
|
166
|
+
const embedder = await service.getEmbedder();
|
|
175
167
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
.map(([e, c]) => `${e}: ${c}`)
|
|
180
|
-
.join(', ');
|
|
181
|
-
result.error = `No working engines found: 0/${generalEngines.length} returned results. Need at least 1 engine. Engine results: ${engineReport}. Your network or search engines may be blocked.`;
|
|
182
|
-
logger.error('[healthcheck] Search validation failed:', result.error);
|
|
183
|
-
return result;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
result.searchOk = true;
|
|
187
|
-
result.details.searchQuery = searchQuery;
|
|
188
|
-
result.details.searchResultCount = generalResults.length;
|
|
189
|
-
result.details.searchDurationMs = searchDurationMs;
|
|
190
|
-
result.details.workingEngines = workingEngines.map(([e, _]) => e);
|
|
191
|
-
result.details.workingEngineCount = workingEngineCount;
|
|
192
|
-
const workingEnginesList = workingEngines.map(([e, _]) => e).join(', ');
|
|
193
|
-
logger.log(`[healthcheck] ✓ Search OK: ${generalResults.length} results from ${workingEngineCount} engines [${workingEnginesList}] in ${searchDurationMs}ms`);
|
|
194
|
-
|
|
195
|
-
// PHASE 2: Test scrape with timeout enforcement
|
|
196
|
-
logger.log('[healthcheck] Phase 2: Testing scrape tool (timeout: ' + SCRAPE_TIMEOUT_MS + 'ms)...');
|
|
197
|
-
const scrapeUrl = 'https://en.wikipedia.org/wiki/Python_(programming_language)';
|
|
198
|
-
const scrapeStartTime = Date.now();
|
|
168
|
+
if (!embedder) {
|
|
169
|
+
return { healthy: false, error: 'Embedder service not available' };
|
|
170
|
+
}
|
|
199
171
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
)
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
172
|
+
// A remote embedding-server client reports isInitialized()===true
|
|
173
|
+
// unconditionally (the server, not the client, owns the model), so trusting
|
|
174
|
+
// it would mask a dead server. When the embedder exposes a liveness probe
|
|
175
|
+
// (fetchHealth on the HTTP client), ping it so a down server is reported
|
|
176
|
+
// unhealthy instead of a false "ready". This does NOT warm a local GPU model.
|
|
177
|
+
const probe = (embedder as { fetchHealth?: () => Promise<void> }).fetchHealth;
|
|
178
|
+
if (typeof probe === 'function') {
|
|
179
|
+
try {
|
|
180
|
+
await probe.call(embedder);
|
|
181
|
+
} catch (e) {
|
|
182
|
+
return { healthy: false, error: `Embedding server unreachable: ${e instanceof Error ? e.message : String(e)}` };
|
|
183
|
+
}
|
|
184
|
+
}
|
|
213
185
|
|
|
214
|
-
|
|
186
|
+
// Lazy-aware health check: if not initialized, we check if the store is open
|
|
187
|
+
// but don't force a model load to GPU just for the health check (unless forced).
|
|
188
|
+
if (!embedder.isInitialized() && !options?.force) {
|
|
189
|
+
return {
|
|
190
|
+
healthy: true,
|
|
191
|
+
diagnostic: {
|
|
192
|
+
status: 'ready (idle)',
|
|
193
|
+
device: embedder.getOriginalDevice(),
|
|
194
|
+
localEntries: counts.local,
|
|
195
|
+
globalEntries: counts.global
|
|
196
|
+
}
|
|
197
|
+
};
|
|
198
|
+
}
|
|
215
199
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
}
|
|
200
|
+
// Force initialization if requested
|
|
201
|
+
if (options?.force && !embedder.isInitialized()) {
|
|
202
|
+
await embedder.embed(' ');
|
|
203
|
+
}
|
|
221
204
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
205
|
+
const device = embedder.getDevice();
|
|
206
|
+
return {
|
|
207
|
+
healthy: true,
|
|
208
|
+
diagnostic: {
|
|
209
|
+
status: 'initialized',
|
|
210
|
+
device,
|
|
211
|
+
model: config.EMBEDDING_MODEL,
|
|
212
|
+
localEntries: counts.local,
|
|
213
|
+
globalEntries: counts.global,
|
|
214
|
+
totalProjects: counts.projects
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
} catch (e) {
|
|
218
|
+
// Same platform-capability gap as above, but surfacing only when the store
|
|
219
|
+
// is touched lazily (getStore/getEmbedder load the native binding here).
|
|
220
|
+
if (isNativeStackUnavailableError(e)) {
|
|
221
|
+
return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
|
|
222
|
+
}
|
|
223
|
+
return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
|
|
226
224
|
}
|
|
225
|
+
}, { timeoutMs: Math.max(healthTimeoutMs, 45000) });
|
|
227
226
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
227
|
+
// Register State Manager Check
|
|
228
|
+
registry.register('StateManager', async () => {
|
|
229
|
+
try {
|
|
230
|
+
const stateManager = await getService<IStateManager>(ServiceNames.STATE_MANAGER, { container }, container);
|
|
231
|
+
const stats = await stateManager.getMetrics();
|
|
232
|
+
const gpuOwner = await stateManager.getGpuOwner();
|
|
233
|
+
|
|
234
|
+
return {
|
|
235
|
+
healthy: true,
|
|
236
|
+
diagnostic: {
|
|
237
|
+
status: 'operational',
|
|
238
|
+
sessions: stats.activeSessions,
|
|
239
|
+
gpuLocked: !!gpuOwner,
|
|
240
|
+
gpuOwner: gpuOwner?.sessionId || 'none'
|
|
241
|
+
}
|
|
242
|
+
};
|
|
243
|
+
} catch (e) {
|
|
244
|
+
return { healthy: false, error: `State manager healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
|
|
235
245
|
}
|
|
246
|
+
}, { timeoutMs: healthTimeoutMs });
|
|
247
|
+
}
|
|
236
248
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
result.details.scrapedContentLength = markdown.length;
|
|
240
|
-
result.details.scrapedDurationMs = scrapeDurationMs;
|
|
241
|
-
logger.log(`[healthcheck] ✓ Scrape OK: ${markdown.length} bytes in ${scrapeDurationMs}ms`);
|
|
249
|
+
// Ensure global registry is populated
|
|
250
|
+
registerHealthChecks(globalHealthRegistry);
|
|
242
251
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
252
|
+
/**
|
|
253
|
+
* Decide whether a FAILED health check is a transient "pool busy" condition (research
|
|
254
|
+
* should proceed) rather than a genuine failure (research must abort).
|
|
255
|
+
*
|
|
256
|
+
* Returns true when the browser is installed/constructible — BrowserCapability is healthy
|
|
257
|
+
* (it never touches the worker pool) — AND every unhealthy component failed with a TIMEOUT,
|
|
258
|
+
* i.e. its probe queued out behind in-flight scrapes on the shared fixed-size worker pool
|
|
259
|
+
* under load. Under sustained concurrency the runtime probe and the StateManager/KnowledgeStore
|
|
260
|
+
* probes (which share the same congested pool/GPU) frequently time out together, so any
|
|
261
|
+
* combination of timed-out probes is treated as "busy", not "dead". The pool is operational
|
|
262
|
+
* and the per-task search/scrape timeouts already bound the real work, so aborting the whole
|
|
263
|
+
* session on this signal needlessly discards a research category. A non-timeout failure
|
|
264
|
+
* (browser not installed, "connection refused", embedder missing) returns false → caller
|
|
265
|
+
* must abort.
|
|
266
|
+
*/
|
|
267
|
+
export function isBusyPoolHealthFailure(
|
|
268
|
+
health: { components?: Array<{ component?: string; healthy?: boolean; error?: string }> },
|
|
269
|
+
): boolean {
|
|
270
|
+
const components = health.components || [];
|
|
271
|
+
const capability = components.find(c => c.component === 'BrowserCapability');
|
|
272
|
+
// BrowserCapability is the liveness signal: it only checks the browser is installed
|
|
273
|
+
// and constructible, never touching the worker pool. If it is healthy (or absent),
|
|
274
|
+
// the browser CAN run — so any failures elsewhere are about throughput, not death.
|
|
275
|
+
const capabilityOk = !capability || capability.healthy !== false;
|
|
276
|
+
const unhealthy = components.filter(c => c.healthy === false);
|
|
277
|
+
// Proceed only when EVERY unhealthy component failed with a TIMEOUT. Under sustained
|
|
278
|
+
// concurrent load the BrowserRuntime probe — and often the StateManager/KnowledgeStore
|
|
279
|
+
// probes sharing the same congested pool/GPU — all queue out together; requiring the
|
|
280
|
+
// runtime probe to be the SOLE failure was too strict and let a second co-timed-out
|
|
281
|
+
// probe abort an otherwise-operational session. A non-timeout error (browser not
|
|
282
|
+
// installed, "connection refused", embedder missing) is a real fault → returns false.
|
|
283
|
+
const isTimeout = (c: { error?: string }) => /tim(e|ed)\s*out|timeout/i.test(String(c.error || ''));
|
|
284
|
+
return capabilityOk && unhealthy.length > 0 && unhealthy.every(isTimeout);
|
|
253
285
|
}
|
|
254
286
|
|
|
255
287
|
/**
|
|
256
|
-
*
|
|
257
|
-
* If a health check is already in progress, returns the same promise
|
|
258
|
-
* If a recent successful health check is cached, returns the cached result
|
|
288
|
+
* Perform a full system health check
|
|
259
289
|
*/
|
|
260
|
-
export async function runHealthCheck()
|
|
261
|
-
const
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
} else {
|
|
270
|
-
logger.log(`[healthcheck] Cached health check result expired (${Math.round(age / 1000)}s old)`);
|
|
271
|
-
cachedHealthCheck = null;
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
// If a health check is already running, return the existing promise
|
|
276
|
-
if (healthCheckPromise) {
|
|
277
|
-
logger.log('[healthcheck] Health check already in progress, reusing existing promise');
|
|
278
|
-
return healthCheckPromise;
|
|
290
|
+
export async function runHealthCheck(options?: { force?: boolean; ctx?: any }) {
|
|
291
|
+
const container = tryGetServiceContainerFromCtx(options?.ctx);
|
|
292
|
+
|
|
293
|
+
let registry: IHealthRegistryService;
|
|
294
|
+
try {
|
|
295
|
+
registry = await getService<IHealthRegistryService>(ServiceNames.HEALTH_REGISTRY, options?.ctx, container);
|
|
296
|
+
} catch {
|
|
297
|
+
// Fallback to global registry if service not found (CLI/Legacy)
|
|
298
|
+
registry = globalHealthRegistry;
|
|
279
299
|
}
|
|
280
300
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
|
-
return result;
|
|
293
|
-
} finally {
|
|
294
|
-
// Clear the promise so a new health check can be started
|
|
295
|
-
healthCheckPromise = null;
|
|
296
|
-
}
|
|
297
|
-
})();
|
|
298
|
-
|
|
299
|
-
return healthCheckPromise;
|
|
301
|
+
const result = await registry.runAll(options);
|
|
302
|
+
return {
|
|
303
|
+
success: result.status !== 'unhealthy',
|
|
304
|
+
status: result.status,
|
|
305
|
+
components: result.components,
|
|
306
|
+
error: result.status === 'unhealthy'
|
|
307
|
+
? result.components.find(c => !c.healthy && registry.isCritical(c.component))?.error
|
|
308
|
+
: undefined
|
|
309
|
+
};
|
|
300
310
|
}
|
|
301
311
|
|
|
302
312
|
/**
|
|
303
|
-
*
|
|
304
|
-
* Useful for testing or forcing a recheck
|
|
313
|
+
* Export health registry for external use
|
|
305
314
|
*/
|
|
306
|
-
export
|
|
307
|
-
cachedHealthCheck = null;
|
|
308
|
-
logger.log('[healthcheck] Health check cache cleared');
|
|
309
|
-
}
|
|
315
|
+
export { globalHealthRegistry as healthRegistry };
|