@lincoln504/pi-research 0.1.12 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +64 -115
  3. package/dist/cli.mjs +23211 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +102 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +213 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +113 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +64 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1044 -0
  30. package/src/config.ts +1036 -147
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +264 -7
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -174
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -577
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
@@ -1,309 +1,315 @@
1
1
  /**
2
2
  * Health Check Module
3
3
  *
4
- * QUICK connectivity validation for search and scrape tools.
5
- * Makes real requests to verify network is alive.
6
- * Fast timeouts, simple checks (just verify tools respond with content).
4
+ * Validates system capabilities using the central health registry.
5
+ * This module is stateless and does not cache results.
7
6
  */
8
7
 
9
- import { logger } from '../logger.ts';
8
+ import { execFileSync } from 'node:child_process';
10
9
  import { getConfig } from '../config.ts';
11
- import { getManager } from '../infrastructure/searxng-lifecycle.ts';
12
- import { search } from '../web-research/search.ts';
13
- import { scrapeSingle } from '../web-research/scrapers.ts';
14
- import { setSearxngManager } from '../web-research/utils.ts';
15
- import { getActiveSearxngEngines } from '../utils/searxng-config.ts';
16
- import { withTimeout } from '../web-research/retry-utils.ts';
17
- import type { SearXNGResult } from '../web-research/types.ts';
18
-
19
- // Health check cache to avoid running multiple health checks concurrently
20
- interface CachedHealthCheck {
21
- result: HealthCheckResult;
22
- timestamp: number;
23
- }
24
-
25
- let cachedHealthCheck: CachedHealthCheck | null = null;
26
- const HEALTH_CHECK_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
27
- let healthCheckPromise: Promise<HealthCheckResult> | null = null;
28
-
29
- // Get timeout from config or use defaults
30
- const config = getConfig();
31
- const SEARCH_TIMEOUT_MS = config.HEALTH_CHECK_TIMEOUT_MS || 15000; // 15s max for search
32
- const SCRAPE_TIMEOUT_MS = config.HEALTH_CHECK_TIMEOUT_MS ? config.HEALTH_CHECK_TIMEOUT_MS + 5000 : 20000; // 5s longer than search
33
-
34
- export interface HealthCheckResult {
35
- success: boolean;
36
- searchOk: boolean;
37
- scrapeOk: boolean;
38
- error?: string;
39
- details: {
40
- searchQuery?: string;
41
- searchResultCount?: number;
42
- searchDurationMs?: number;
43
- workingEngines?: string[];
44
- workingEngineCount?: number;
45
- scrapedUrl?: string;
46
- scrapedContentLength?: number;
47
- scrapedDurationMs?: number;
48
- };
49
- }
50
-
51
- /**
52
- * QUICK scrape validation: just check content exists and is readable
53
- */
54
- function validateScrapeOutput(markdown: string): { valid: boolean; error?: string } {
55
- if (typeof markdown !== 'string') {
56
- return { valid: false, error: 'Content not a string' };
57
- }
58
-
59
- const trimmed = markdown.trim();
60
- if (trimmed.length === 0) {
61
- return { valid: false, error: 'Content is empty' };
62
- }
63
-
64
- // Just check it has some readable text (not garbage)
65
- if (!/[a-zA-Z0-9]{10,}/.test(markdown)) {
66
- return { valid: false, error: 'Content not readable' };
67
- }
68
-
69
- return { valid: true };
70
- }
10
+ import { isBrowserAvailable, resolveHeadlessMode } from '../infrastructure/browser/config.ts';
11
+ import { runBrowserHealthCheck } from '../infrastructure/browser/task-execution-service.ts';
12
+ import { getService, tryGetServiceContainerFromCtx, getServiceContainer, ServiceLifecycle } from '../core/service-registry.ts';
13
+ import type { ServiceContainer } from '../core/service-registry.ts';
14
+ import { ServiceNames } from '../core/service-interfaces.ts';
15
+ import type { IStateManager, IHealthRegistryService, IKnowledgeStoreService, ISchedulerService } from '../core/service-interfaces.ts';
16
+ import { healthRegistry as globalHealthRegistry } from './registry.ts';
17
+ import { isNativeStackUnavailableError } from '../knowledge/embedder-utils.ts';
71
18
 
72
19
  /**
73
- * Healthcheck uses the same web-research search/scrape helpers as the main tool.
74
- * Ensure they are wired to the current lifecycle manager instead of relying on
75
- * an external caller to have registered it beforehand.
20
+ * Standalone BrowserCapability check, exported so tests can call it directly
21
+ * without going through the full registry.
76
22
  */
77
- function ensureHealthcheckManager(): void {
78
- const manager = getManager();
79
- if (!manager) {
80
- throw new Error('SearXNG manager not initialized. Call initLifecycle() before runHealthCheck().');
23
+ export async function checkBrowserCapability(): Promise<{ healthy: boolean; error?: string; diagnostic?: Record<string, any> }> {
24
+ const mockMode = process.env['PI_RESEARCH_MOCK_SEARCH'] === 'true' &&
25
+ process.env['PI_RESEARCH_MOCK_SCRAPE'] === 'true';
26
+ if (isBrowserAvailable() || mockMode) {
27
+ // resolveHeadlessMode() selects 'virtual' (Xvfb) only under the explicit
28
+ // PI_RESEARCH_USE_XVFB=true opt-in on a bare Linux TTY. In that case camoufox
29
+ // spawns Xvfb internally, so fail early (before research starts) if it is not
30
+ // installed. The default path is headless:true, which needs no Xvfb.
31
+ if (!mockMode && resolveHeadlessMode() === 'virtual') {
32
+ try {
33
+ execFileSync('which', ['Xvfb'], { stdio: 'ignore' });
34
+ } catch {
35
+ return {
36
+ healthy: false,
37
+ error: 'No display server found on Linux (DISPLAY not set) and Xvfb is not installed. Run: sudo apt install xvfb',
38
+ };
39
+ }
40
+ }
41
+ return { healthy: true, diagnostic: { status: mockMode ? 'mocked' : 'available' } };
42
+ } else {
43
+ return { healthy: false, error: 'Camoufox (browser) not found. Run "npx camoufox-js fetch" to install the browser.' };
81
44
  }
82
-
83
- setSearxngManager(manager);
84
45
  }
85
46
 
86
47
  /**
87
- * Internal health check implementation
88
- * Runs the actual health check tests
48
+ * Register all health checks with a registry
89
49
  */
90
- async function performHealthCheck(): Promise<HealthCheckResult> {
91
- logger.log('[healthcheck] Starting network connectivity health check...');
50
+ export function registerHealthChecks(registry: IHealthRegistryService, container: ServiceContainer = getServiceContainer()): void {
51
+ const config = getConfig();
52
+ const healthTimeoutMs = config.HEALTH_CHECK_TIMEOUT_MS;
92
53
 
93
- const result: HealthCheckResult = {
94
- success: false,
95
- searchOk: false,
96
- scrapeOk: false,
97
- error: undefined,
98
- details: {},
99
- };
54
+ // Register Browser Capability Check
55
+ registry.register('BrowserCapability', checkBrowserCapability, { timeoutMs: healthTimeoutMs, critical: true });
100
56
 
101
- try {
102
- // Skip health check if explicitly requested via env
103
- if (process.env['PI_RESEARCH_SKIP_HEALTHCHECK'] === '1') {
104
- logger.log('[healthcheck] Skipping health check (PI_RESEARCH_SKIP_HEALTHCHECK=1)');
105
- result.success = true;
106
- result.searchOk = true;
107
- result.scrapeOk = true;
108
- return result;
57
+ // Register Browser Runtime Check
58
+ registry.register('BrowserRuntime', async (options) => {
59
+ try {
60
+ const scheduler = await getService<ISchedulerService>(ServiceNames.SCHEDULER, { container }, container);
61
+
62
+ // Check if initialized but idle (unless forced)
63
+ if (!scheduler.isReady() && !options?.force) {
64
+ return { healthy: true, diagnostic: { status: 'ready (idle)' } };
65
+ }
66
+
67
+ // Perform a real test
68
+ const searchResult = await runBrowserHealthCheck(undefined, 1, undefined, container);
69
+ if (searchResult.success) {
70
+ return { healthy: true, diagnostic: { status: options?.force && !scheduler.isReady() ? 'initialized & active' : 'active' } };
71
+ } else {
72
+ return { healthy: false, error: 'Browser healthcheck failed: worker reported failure or page failed to load.' };
73
+ }
74
+ } catch (e) {
75
+ return { healthy: false, error: `Browser healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
76
+ }
77
+ // Honor the configured health timeout but keep a 150s floor: a cold browser
78
+ // launch (camoufox download/spawn) can legitimately take minutes, so we never
79
+ // go below that, but a user who raises HEALTH_CHECK_TIMEOUT_MS past it is
80
+ // respected (same pattern as the KnowledgeStore check's Math.max floor).
81
+ }, { timeoutMs: Math.max(healthTimeoutMs, 150000), critical: true });
82
+
83
+ // Register Knowledge Store Check
84
+ registry.register('KnowledgeStore', async (options) => {
85
+ // Idle fast-path — mirrors the BrowserRuntime check above. Read the ALREADY
86
+ // resolved instance via tryGet(), which does NOT initialize the service;
87
+ // getService() (below) would run the full LanceDB open + ONNX model load +
88
+ // WebGPU probe, so a non-forced health check must never take that path on a
89
+ // cold store. Only inspect for real when the store is already initialized OR an
90
+ // explicit probe is requested (force).
91
+ const existing = container.tryGet<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE);
92
+ const alreadyInitialized = existing?.lifecycle === ServiceLifecycle.INITIALIZED;
93
+ if (!alreadyInitialized && !options?.force) {
94
+ const idleCwd = existing?.getCwd() ?? process.cwd();
95
+ if (getConfig(idleCwd).KNOWLEDGE_STORE_MODE === 'none') {
96
+ return { healthy: true, diagnostic: { status: 'disabled in config' } };
97
+ }
98
+ // A store that resolved-then-disabled reports WHY, accurately: 'native' is a
99
+ // permanent platform capability gap; 'mode' is a revivable Knowledge-Mode=none
100
+ // (which may have since been re-enabled — it re-initializes on next real use).
101
+ // Never claim "native unavailable" for a mode-disable on a perfectly capable host.
102
+ if (existing?.lifecycle === ServiceLifecycle.DISABLED) {
103
+ const reason = existing.getDisabledReason();
104
+ return { healthy: true, diagnostic: {
105
+ status: reason === 'native'
106
+ ? 'disabled (native embedding/vector stack unavailable on this platform)'
107
+ : 'disabled',
108
+ } };
109
+ }
110
+ // A DISPOSED/DISPOSING service is not "ready" — report it accurately without
111
+ // initializing anything (it re-resolves lazily on next real use, so still healthy).
112
+ const status = existing?.lifecycle === ServiceLifecycle.INITIALIZING
113
+ ? 'initializing'
114
+ : (existing?.lifecycle === ServiceLifecycle.DISPOSED || existing?.lifecycle === ServiceLifecycle.DISPOSING)
115
+ ? 'disposed (not running)'
116
+ : 'ready (idle)';
117
+ return { healthy: true, diagnostic: { status } };
109
118
  }
110
119
 
111
- ensureHealthcheckManager();
112
-
113
- // PHASE 1: Test general web search engines (Bing, DuckDuckGo, Brave — NOT Wikipedia)
114
- logger.log('[healthcheck] Phase 1: Testing general web search engines (timeout: ' + SEARCH_TIMEOUT_MS + 'ms)...');
115
- const searchQuery = 'open source software';
116
- const searchStartTime = Date.now();
117
-
118
- let queryResults;
120
+ // Full inspection (already-initialized OR forced): resolve via getService —
121
+ // cheap when warm (early-returns), full init only on a forced cold probe.
122
+ // Resolve cwd from the live service (it captured ctx.cwd at init) so the
123
+ // health check reads the SAME config the store actually uses. Reading
124
+ // process.cwd() here silently ignored SDK/CLI cwd overrides.
125
+ let service: IKnowledgeStoreService | null = null;
126
+ let initError: unknown = null;
119
127
  try {
120
- queryResults = await withTimeout(
121
- search([searchQuery]),
122
- SEARCH_TIMEOUT_MS,
123
- 'Search'
124
- );
125
- } catch (err) {
126
- const msg = err instanceof Error ? err.message : String(err);
127
- result.error = `Search request failed: ${msg}. Network may be unreachable or SearXNG not responding.`;
128
- logger.error('[healthcheck] Search error:', result.error);
129
- return result;
128
+ service = await getService<IKnowledgeStoreService>(ServiceNames.KNOWLEDGE_STORE, { container }, container);
129
+ } catch (e) {
130
+ // service stays null; classify the failure below (native-unavailable vs real).
131
+ initError = e;
130
132
  }
131
-
132
- const searchDurationMs = Date.now() - searchStartTime;
133
-
134
- if (queryResults.length === 0 || !queryResults[0]) {
135
- result.error = 'Search returned empty result object (SearXNG not initialized?)';
136
- logger.error('[healthcheck]', result.error);
137
- return result;
133
+ const cwd = service?.getCwd() ?? process.cwd();
134
+ const config = getConfig(cwd);
135
+ if (config.KNOWLEDGE_STORE_MODE === 'none') {
136
+ return { healthy: true, diagnostic: { status: 'disabled in config' } };
138
137
  }
139
-
140
- const qr = queryResults[0];
141
- if (qr.error) {
142
- const errMsg = qr.error.message || String(qr.error);
143
- result.error = `Search request failed: ${errMsg}. SearXNG or engines are not responding.`;
144
- logger.error('[healthcheck]', result.error);
145
- return result;
138
+ // The native ML/vector stack ships no prebuilt for some platforms (Intel macOS
139
+ // / darwin-x64 has no onnxruntime-node binary nor @lancedb native binding). That
140
+ // is a permanent capability gap, not a fault — report the store as DISABLED
141
+ // (healthy) so research still runs without the optional cache. Otherwise this
142
+ // critical component drags overall health to 'unhealthy' and quick (depth-0)
143
+ // research aborts with "Research cannot start" for any fresh user on such a host.
144
+ if (isNativeStackUnavailableError(initError)) {
145
+ return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
146
146
  }
147
-
148
- // Check for at least 2 working general search engines (not Wikipedia)
149
- // Wikipedia is an encyclopedic engine, not suitable for general web research
150
- // Load the list of active engines from the actual SearXNG config (not hardcoded)
151
- const generalEngines = getActiveSearxngEngines();
152
- if (generalEngines.length === 0) {
153
- result.error = 'No active general search engines found in SearXNG configuration';
154
- logger.error('[healthcheck]', result.error);
155
- return result;
147
+ // getService can SUCCEED on a native-unavailable platform: the service memoizes
148
+ // the failure as DISABLED instead of throwing, so initError stays null and the
149
+ // getStore()/getEmbedder() null checks below would report a false UNHEALTHY
150
+ // ("Embedder service not available") on a forced check — contradicting the
151
+ // non-forced path's correct "disabled" verdict above. Report the same verdict.
152
+ if (service?.lifecycle === ServiceLifecycle.DISABLED) {
153
+ const reason = service.getDisabledReason();
154
+ return { healthy: true, diagnostic: {
155
+ status: reason === 'native'
156
+ ? 'disabled (native embedding/vector stack unavailable on this platform)'
157
+ : 'disabled',
158
+ } };
156
159
  }
157
- logger.log(`[healthcheck] Checking for results from engines: [${generalEngines.join(', ')}]`);
158
-
159
- const generalResults = qr.results.filter((r: SearXNGResult) =>
160
- generalEngines.includes((r.engine || '').toLowerCase())
161
- );
162
-
163
- // Count how many engines returned at least one result
164
- const engineCounts = new Map<string, number>();
165
- generalEngines.forEach(e => engineCounts.set(e, 0));
166
- generalResults.forEach((r: SearXNGResult) => {
167
- const engine = (r.engine || '').toLowerCase();
168
- if (engineCounts.has(engine)) {
169
- engineCounts.set(engine, (engineCounts.get(engine) || 0) + 1);
160
+ try {
161
+ if (!service) {
162
+ return { healthy: false, error: 'Knowledge store service not available' };
170
163
  }
171
- });
172
-
173
- const workingEngines = Array.from(engineCounts.entries()).filter(([_, count]) => count > 0);
174
- const workingEngineCount = workingEngines.length;
164
+ const store = await service.getStore();
165
+ const counts = store ? await store.countScoped() : { local: 0, global: 0, projects: 0 };
166
+ const embedder = await service.getEmbedder();
175
167
 
176
- // Require at least 1 engine to return results (local use may have rate limits)
177
- if (workingEngineCount < 1) {
178
- const engineReport = Array.from(engineCounts.entries())
179
- .map(([e, c]) => `${e}: ${c}`)
180
- .join(', ');
181
- result.error = `No working engines found: 0/${generalEngines.length} returned results. Need at least 1 engine. Engine results: ${engineReport}. Your network or search engines may be blocked.`;
182
- logger.error('[healthcheck] Search validation failed:', result.error);
183
- return result;
184
- }
185
-
186
- result.searchOk = true;
187
- result.details.searchQuery = searchQuery;
188
- result.details.searchResultCount = generalResults.length;
189
- result.details.searchDurationMs = searchDurationMs;
190
- result.details.workingEngines = workingEngines.map(([e, _]) => e);
191
- result.details.workingEngineCount = workingEngineCount;
192
- const workingEnginesList = workingEngines.map(([e, _]) => e).join(', ');
193
- logger.log(`[healthcheck] ✓ Search OK: ${generalResults.length} results from ${workingEngineCount} engines [${workingEnginesList}] in ${searchDurationMs}ms`);
194
-
195
- // PHASE 2: Test scrape with timeout enforcement
196
- logger.log('[healthcheck] Phase 2: Testing scrape tool (timeout: ' + SCRAPE_TIMEOUT_MS + 'ms)...');
197
- const scrapeUrl = 'https://en.wikipedia.org/wiki/Python_(programming_language)';
198
- const scrapeStartTime = Date.now();
168
+ if (!embedder) {
169
+ return { healthy: false, error: 'Embedder service not available' };
170
+ }
199
171
 
200
- let scrapeResult;
201
- try {
202
- scrapeResult = await withTimeout(
203
- scrapeSingle(scrapeUrl),
204
- SCRAPE_TIMEOUT_MS,
205
- 'Scrape'
206
- );
207
- } catch (err) {
208
- const msg = err instanceof Error ? err.message : String(err);
209
- result.error = `Scrape request failed: ${msg}. Network timeout or target unreachable.`;
210
- logger.error('[healthcheck]', result.error);
211
- return result;
212
- }
172
+ // A remote embedding-server client reports isInitialized()===true
173
+ // unconditionally (the server, not the client, owns the model), so trusting
174
+ // it would mask a dead server. When the embedder exposes a liveness probe
175
+ // (fetchHealth on the HTTP client), ping it so a down server is reported
176
+ // unhealthy instead of a false "ready". This does NOT warm a local GPU model.
177
+ const probe = (embedder as { fetchHealth?: () => Promise<void> }).fetchHealth;
178
+ if (typeof probe === 'function') {
179
+ try {
180
+ await probe.call(embedder);
181
+ } catch (e) {
182
+ return { healthy: false, error: `Embedding server unreachable: ${e instanceof Error ? e.message : String(e)}` };
183
+ }
184
+ }
213
185
 
214
- const scrapeDurationMs = Date.now() - scrapeStartTime;
186
+ // Lazy-aware health check: if not initialized, we check if the store is open
187
+ // but don't force a model load to GPU just for the health check (unless forced).
188
+ if (!embedder.isInitialized() && !options?.force) {
189
+ return {
190
+ healthy: true,
191
+ diagnostic: {
192
+ status: 'ready (idle)',
193
+ device: embedder.getOriginalDevice(),
194
+ localEntries: counts.local,
195
+ globalEntries: counts.global
196
+ }
197
+ };
198
+ }
215
199
 
216
- if (!scrapeResult) {
217
- result.error = 'Scrape returned null result (scraper crashed?)';
218
- logger.error('[healthcheck]', result.error);
219
- return result;
220
- }
200
+ // Force initialization if requested
201
+ if (options?.force && !embedder.isInitialized()) {
202
+ await embedder.embed(' ');
203
+ }
221
204
 
222
- if (scrapeResult.source === 'failed') {
223
- result.error = `Scrape failed: ${scrapeResult.error || 'unknown error'}. Check if target URL is accessible.`;
224
- logger.error('[healthcheck]', result.error);
225
- return result;
205
+ const device = embedder.getDevice();
206
+ return {
207
+ healthy: true,
208
+ diagnostic: {
209
+ status: 'initialized',
210
+ device,
211
+ model: config.EMBEDDING_MODEL,
212
+ localEntries: counts.local,
213
+ globalEntries: counts.global,
214
+ totalProjects: counts.projects
215
+ }
216
+ };
217
+ } catch (e) {
218
+ // Same platform-capability gap as above, but surfacing only when the store
219
+ // is touched lazily (getStore/getEmbedder load the native binding here).
220
+ if (isNativeStackUnavailableError(e)) {
221
+ return { healthy: true, diagnostic: { status: 'disabled (native embedding/vector stack unavailable on this platform)' } };
222
+ }
223
+ return { healthy: false, error: `Knowledge store healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
226
224
  }
225
+ }, { timeoutMs: Math.max(healthTimeoutMs, 45000) });
227
226
 
228
- // ROBUST validation of scraped content
229
- const markdown = scrapeResult.markdown || '';
230
- const scrapeValidation = validateScrapeOutput(markdown);
231
- if (!scrapeValidation.valid) {
232
- result.error = `Scrape validation failed: ${scrapeValidation.error}`;
233
- logger.error('[healthcheck]', result.error);
234
- return result;
227
+ // Register State Manager Check
228
+ registry.register('StateManager', async () => {
229
+ try {
230
+ const stateManager = await getService<IStateManager>(ServiceNames.STATE_MANAGER, { container }, container);
231
+ const stats = await stateManager.getMetrics();
232
+ const gpuOwner = await stateManager.getGpuOwner();
233
+
234
+ return {
235
+ healthy: true,
236
+ diagnostic: {
237
+ status: 'operational',
238
+ sessions: stats.activeSessions,
239
+ gpuLocked: !!gpuOwner,
240
+ gpuOwner: gpuOwner?.sessionId || 'none'
241
+ }
242
+ };
243
+ } catch (e) {
244
+ return { healthy: false, error: `State manager healthcheck failed: ${e instanceof Error ? e.message : String(e)}` };
235
245
  }
246
+ }, { timeoutMs: healthTimeoutMs });
247
+ }
236
248
 
237
- result.scrapeOk = true;
238
- result.details.scrapedUrl = scrapeUrl;
239
- result.details.scrapedContentLength = markdown.length;
240
- result.details.scrapedDurationMs = scrapeDurationMs;
241
- logger.log(`[healthcheck] ✓ Scrape OK: ${markdown.length} bytes in ${scrapeDurationMs}ms`);
249
+ // Ensure global registry is populated
250
+ registerHealthChecks(globalHealthRegistry);
242
251
 
243
- // All checks passed
244
- result.success = true;
245
- const totalMs = searchDurationMs + scrapeDurationMs;
246
- logger.log(`[healthcheck] ✓ HEALTH CHECK PASSED in ${totalMs}ms - all systems ready`);
247
- return result;
248
- } catch (err) {
249
- result.error = `Unexpected error during health check: ${err instanceof Error ? err.message : String(err)}`;
250
- logger.error('[healthcheck] Unexpected error:', err);
251
- return result;
252
- }
252
+ /**
253
+ * Decide whether a FAILED health check is a transient "pool busy" condition (research
254
+ * should proceed) rather than a genuine failure (research must abort).
255
+ *
256
+ * Returns true when the browser is installed/constructible — BrowserCapability is healthy
257
+ * (it never touches the worker pool) — AND every unhealthy component failed with a TIMEOUT,
258
+ * i.e. its probe queued out behind in-flight scrapes on the shared fixed-size worker pool
259
+ * under load. Under sustained concurrency the runtime probe and the StateManager/KnowledgeStore
260
+ * probes (which share the same congested pool/GPU) frequently time out together, so any
261
+ * combination of timed-out probes is treated as "busy", not "dead". The pool is operational
262
+ * and the per-task search/scrape timeouts already bound the real work, so aborting the whole
263
+ * session on this signal needlessly discards a research category. A non-timeout failure
264
+ * (browser not installed, "connection refused", embedder missing) returns false → caller
265
+ * must abort.
266
+ */
267
+ export function isBusyPoolHealthFailure(
268
+ health: { components?: Array<{ component?: string; healthy?: boolean; error?: string }> },
269
+ ): boolean {
270
+ const components = health.components || [];
271
+ const capability = components.find(c => c.component === 'BrowserCapability');
272
+ // BrowserCapability is the liveness signal: it only checks the browser is installed
273
+ // and constructible, never touching the worker pool. If it is healthy (or absent),
274
+ // the browser CAN run — so any failures elsewhere are about throughput, not death.
275
+ const capabilityOk = !capability || capability.healthy !== false;
276
+ const unhealthy = components.filter(c => c.healthy === false);
277
+ // Proceed only when EVERY unhealthy component failed with a TIMEOUT. Under sustained
278
+ // concurrent load the BrowserRuntime probe — and often the StateManager/KnowledgeStore
279
+ // probes sharing the same congested pool/GPU — all queue out together; requiring the
280
+ // runtime probe to be the SOLE failure was too strict and let a second co-timed-out
281
+ // probe abort an otherwise-operational session. A non-timeout error (browser not
282
+ // installed, "connection refused", embedder missing) is a real fault → returns false.
283
+ const isTimeout = (c: { error?: string }) => /tim(e|ed)\s*out|timeout/i.test(String(c.error || ''));
284
+ return capabilityOk && unhealthy.length > 0 && unhealthy.every(isTimeout);
253
285
  }
254
286
 
255
287
  /**
256
- * Run health check with caching to avoid concurrent executions
257
- * If a health check is already in progress, returns the same promise
258
- * If a recent successful health check is cached, returns the cached result
288
+ * Perform a full system health check
259
289
  */
260
- export async function runHealthCheck(): Promise<HealthCheckResult> {
261
- const now = Date.now();
262
-
263
- // Check if we have a recent cached result
264
- if (cachedHealthCheck) {
265
- const age = now - cachedHealthCheck.timestamp;
266
- if (age < HEALTH_CHECK_CACHE_TTL_MS) {
267
- logger.log(`[healthcheck] Using cached health check result (${Math.round(age / 1000)}s old)`);
268
- return cachedHealthCheck.result;
269
- } else {
270
- logger.log(`[healthcheck] Cached health check result expired (${Math.round(age / 1000)}s old)`);
271
- cachedHealthCheck = null;
272
- }
273
- }
274
-
275
- // If a health check is already running, return the existing promise
276
- if (healthCheckPromise) {
277
- logger.log('[healthcheck] Health check already in progress, reusing existing promise');
278
- return healthCheckPromise;
290
+ export async function runHealthCheck(options?: { force?: boolean; ctx?: any }) {
291
+ const container = tryGetServiceContainerFromCtx(options?.ctx);
292
+
293
+ let registry: IHealthRegistryService;
294
+ try {
295
+ registry = await getService<IHealthRegistryService>(ServiceNames.HEALTH_REGISTRY, options?.ctx, container);
296
+ } catch {
297
+ // Fallback to global registry if service not found (CLI/Legacy)
298
+ registry = globalHealthRegistry;
279
299
  }
280
300
 
281
- // Start a new health check
282
- healthCheckPromise = (async () => {
283
- try {
284
- const result = await performHealthCheck();
285
- // Cache successful results
286
- if (result.success) {
287
- cachedHealthCheck = {
288
- result,
289
- timestamp: now,
290
- };
291
- }
292
- return result;
293
- } finally {
294
- // Clear the promise so a new health check can be started
295
- healthCheckPromise = null;
296
- }
297
- })();
298
-
299
- return healthCheckPromise;
301
+ const result = await registry.runAll(options);
302
+ return {
303
+ success: result.status !== 'unhealthy',
304
+ status: result.status,
305
+ components: result.components,
306
+ error: result.status === 'unhealthy'
307
+ ? result.components.find(c => !c.healthy && registry.isCritical(c.component))?.error
308
+ : undefined
309
+ };
300
310
  }
301
311
 
302
312
  /**
303
- * Clear the health check cache
304
- * Useful for testing or forcing a recheck
313
+ * Export health registry for external use
305
314
  */
306
- export function clearHealthCheckCache(): void {
307
- cachedHealthCheck = null;
308
- logger.log('[healthcheck] Health check cache cleared');
309
- }
315
+ export { globalHealthRegistry as healthRegistry };