@lincoln504/pi-research 0.1.13 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +55 -138
  3. package/dist/cli.mjs +23375 -0
  4. package/dist/prompts/research-knowledge-relevance-triage.md +35 -0
  5. package/dist/prompts/research-knowledge-search-extractor.md +41 -0
  6. package/dist/prompts/research-tool-usage.md +92 -0
  7. package/dist/prompts/researcher.md +89 -0
  8. package/dist/prompts/system-coordinator.md +75 -0
  9. package/dist/prompts/system-lead-evaluator.md +118 -0
  10. package/dist/thread-worker.mjs +2611 -0
  11. package/dist/webgpu-probe.mjs +80 -0
  12. package/docs/AGENT-SKILL.md +130 -0
  13. package/docs/ARCHITECTURE.md +306 -80
  14. package/docs/CONFIGURATION.md +249 -0
  15. package/docs/KNOWLEDGE-STORE.md +211 -0
  16. package/docs/PI-EXTENSION.md +91 -0
  17. package/docs/SDK.md +122 -0
  18. package/docs/THIRD-PARTY-NOTICES.md +66 -0
  19. package/package.json +102 -36
  20. package/scripts/build.cjs +140 -0
  21. package/scripts/cleanup.cjs +149 -0
  22. package/scripts/prepare.cjs +28 -0
  23. package/scripts/setup.cjs +134 -0
  24. package/skills/pi-research/README.md +90 -0
  25. package/skills/pi-research/SKILL.md +102 -0
  26. package/skills/pi-research/references/configuration.md +77 -0
  27. package/skills/pi-research/scripts/run.mjs +237 -0
  28. package/src/cleanup/research-cleanup.ts +140 -0
  29. package/src/cli.ts +1195 -0
  30. package/src/config.ts +1032 -155
  31. package/src/constants.ts +115 -84
  32. package/src/core/health-check-service.ts +265 -0
  33. package/src/core/interfaces/health-check-interfaces.ts +50 -0
  34. package/src/core/interfaces/knowledge-interfaces.ts +190 -0
  35. package/src/core/interfaces/observer-interfaces.ts +67 -0
  36. package/src/core/interfaces/orchestration-interfaces.ts +86 -0
  37. package/src/core/interfaces/planning-interfaces.ts +100 -0
  38. package/src/core/interfaces/process-interfaces.ts +37 -0
  39. package/src/core/interfaces/research-plan-types.ts +52 -0
  40. package/src/core/interfaces/scheduler-interfaces.ts +91 -0
  41. package/src/core/interfaces/service-names.ts +39 -0
  42. package/src/core/interfaces/state-manager-interfaces.ts +29 -0
  43. package/src/core/interfaces/steering-interfaces.ts +31 -0
  44. package/src/core/llm/agentic-repair.ts +169 -0
  45. package/src/core/llm/inject-date.ts +52 -0
  46. package/src/core/llm/llm-timeout.ts +64 -0
  47. package/src/core/llm/llm-utils.ts +86 -0
  48. package/src/core/llm/model-registry-factory.ts +287 -0
  49. package/src/core/llm/pi-ai-completion.ts +29 -0
  50. package/src/core/llm/prompts.ts +50 -0
  51. package/src/core/llm/research-model-resolver.ts +91 -0
  52. package/src/core/planning-constants.ts +13 -0
  53. package/src/core/planning-service.ts +649 -0
  54. package/src/core/planning-utils.ts +299 -0
  55. package/src/core/scheduler-factory.ts +48 -0
  56. package/src/core/scheduler-service.ts +123 -0
  57. package/src/core/service-initialization.ts +194 -0
  58. package/src/core/service-interfaces.ts +48 -0
  59. package/src/core/service-registry.ts +659 -0
  60. package/src/healthcheck/index.ts +271 -265
  61. package/src/healthcheck/registry.ts +130 -0
  62. package/src/index.ts +639 -50
  63. package/src/infrastructure/browser/browser-cleanup.ts +309 -0
  64. package/src/infrastructure/browser/browser-client.ts +199 -0
  65. package/src/infrastructure/browser/browser-error-utils.ts +160 -0
  66. package/src/infrastructure/browser/browser-lifecycle.ts +120 -0
  67. package/src/infrastructure/browser/browser-server.ts +174 -0
  68. package/src/infrastructure/browser/browser-task-scheduler.ts +461 -0
  69. package/src/infrastructure/browser/cleanup-utils.ts +155 -0
  70. package/src/infrastructure/browser/client-agent.ts +31 -0
  71. package/src/infrastructure/browser/config.ts +288 -0
  72. package/src/infrastructure/browser/ensure-browser.ts +205 -0
  73. package/src/infrastructure/browser/index.ts +37 -0
  74. package/src/infrastructure/browser/priority-task-queue.ts +226 -0
  75. package/src/infrastructure/browser/scheduler-factory.ts +354 -0
  76. package/src/infrastructure/browser/task-execution-service.ts +209 -0
  77. package/src/infrastructure/browser/thread-worker-browser.ts +282 -0
  78. package/src/infrastructure/browser/thread-worker-lifecycle.ts +323 -0
  79. package/src/infrastructure/browser/thread-worker-messaging.ts +662 -0
  80. package/src/infrastructure/browser/thread-worker-types.ts +25 -0
  81. package/src/infrastructure/browser/thread-worker.mjs +2611 -0
  82. package/src/infrastructure/browser/thread-worker.ts +168 -0
  83. package/src/infrastructure/browser/worker-pool-manager.ts +449 -0
  84. package/src/infrastructure/embedding/embedding-client.ts +205 -0
  85. package/src/infrastructure/embedding/embedding-factory.ts +324 -0
  86. package/src/infrastructure/embedding/embedding-server.ts +535 -0
  87. package/src/infrastructure/file-lock-service.ts +592 -0
  88. package/src/infrastructure/gpu-resource-service.ts +191 -0
  89. package/src/infrastructure/knowledge-store-service.ts +475 -0
  90. package/src/infrastructure/metrics-service.ts +129 -0
  91. package/src/infrastructure/process-lifecycle-service.ts +253 -0
  92. package/src/infrastructure/scheduler-factory-service.ts +58 -0
  93. package/src/infrastructure/service-initialization.ts +292 -0
  94. package/src/infrastructure/state/state-backup-manager.ts +290 -0
  95. package/src/infrastructure/state/state-browser-api.ts +62 -0
  96. package/src/infrastructure/state/state-browser-manager.ts +70 -0
  97. package/src/infrastructure/state/state-manager-service.ts +264 -0
  98. package/src/infrastructure/state/state-manager.ts +547 -0
  99. package/src/infrastructure/state/state-metrics.ts +92 -0
  100. package/src/infrastructure/state/state-migration.ts +103 -0
  101. package/src/infrastructure/state/state-path-configuration.ts +119 -0
  102. package/src/infrastructure/state/state-session-api.ts +94 -0
  103. package/src/infrastructure/state/state-session-manager.ts +127 -0
  104. package/src/infrastructure/state/state-validator.ts +93 -0
  105. package/src/infrastructure/types/state-types.ts +85 -0
  106. package/src/knowledge/chunker.ts +157 -0
  107. package/src/knowledge/embedder-init.ts +413 -0
  108. package/src/knowledge/embedder-utils.ts +296 -0
  109. package/src/knowledge/embedder.ts +592 -0
  110. package/src/knowledge/index.ts +126 -0
  111. package/src/knowledge/lancedb-loader.ts +35 -0
  112. package/src/knowledge/migration.ts +17 -0
  113. package/src/knowledge/model-config.ts +132 -0
  114. package/src/knowledge/onnx-env.ts +48 -0
  115. package/src/knowledge/store-operations.ts +291 -0
  116. package/src/knowledge/store-schema.ts +79 -0
  117. package/src/knowledge/store.ts +1251 -0
  118. package/src/knowledge/transformers-loader.ts +31 -0
  119. package/src/knowledge/webgpu-probe.mjs +80 -0
  120. package/src/knowledge/webgpu-viability.ts +255 -0
  121. package/src/knowledge/writer-queue.ts +252 -0
  122. package/src/logger.ts +316 -122
  123. package/src/observers/research-observer-impl.ts +459 -0
  124. package/src/orchestration/deep-research-orchestrator.ts +524 -773
  125. package/src/orchestration/deep-research-types.ts +47 -35
  126. package/src/orchestration/headless-observer.ts +114 -0
  127. package/src/orchestration/orchestration-types.ts +33 -0
  128. package/src/orchestration/quick-research-orchestrator.ts +463 -0
  129. package/src/orchestration/research-orchestration-service.ts +549 -0
  130. package/src/orchestration/research-session-service.ts +178 -0
  131. package/src/orchestration/research-synthesis-service.ts +348 -0
  132. package/src/orchestration/researcher-executor.ts +394 -0
  133. package/src/orchestration/researcher.ts +115 -42
  134. package/src/orchestration/service-initialization.ts +109 -0
  135. package/src/orchestration/session-context.ts +7 -3
  136. package/src/orchestration/session-state.ts +672 -0
  137. package/src/prompts/research-knowledge-relevance-triage.md +35 -0
  138. package/src/prompts/research-knowledge-search-extractor.md +41 -0
  139. package/src/prompts/research-tool-usage.md +63 -16
  140. package/src/prompts/researcher.md +59 -136
  141. package/src/prompts/system-coordinator.md +64 -13
  142. package/src/prompts/system-lead-evaluator.md +82 -189
  143. package/src/research-config.ts +872 -0
  144. package/src/sdk.ts +847 -0
  145. package/src/security/cisa-kev.ts +72 -9
  146. package/src/security/cvss.ts +92 -0
  147. package/src/security/ecosystem.ts +45 -0
  148. package/src/security/github-advisories.ts +168 -428
  149. package/src/security/github-advisory-types.ts +282 -0
  150. package/src/security/index.ts +63 -177
  151. package/src/security/nvd-client.ts +550 -0
  152. package/src/security/nvd-types.ts +179 -0
  153. package/src/security/osv-client.ts +246 -0
  154. package/src/security/osv-types.ts +285 -0
  155. package/src/security/types.ts +67 -53
  156. package/src/skill-install/skill-installer.ts +519 -0
  157. package/src/stackexchange/index.ts +81 -27
  158. package/src/stackexchange/output/compact.ts +7 -6
  159. package/src/stackexchange/output/table.ts +11 -7
  160. package/src/stackexchange/queries.ts +0 -10
  161. package/src/stackexchange/rest-client.ts +141 -66
  162. package/src/stackexchange/types.ts +0 -17
  163. package/src/tool.ts +15 -423
  164. package/src/tools/grep.ts +32 -310
  165. package/src/tools/health-tool-definition.ts +101 -0
  166. package/src/tools/index.ts +38 -9
  167. package/src/tools/research-knowledge-search.ts +974 -0
  168. package/src/tools/research-knowledge-types.ts +77 -0
  169. package/src/tools/research-tool-definition.ts +494 -0
  170. package/src/tools/scrape.ts +286 -388
  171. package/src/tools/search.ts +98 -153
  172. package/src/tools/security.ts +107 -80
  173. package/src/tools/stackexchange.ts +58 -35
  174. package/src/tools/youtube-transcript.ts +163 -0
  175. package/src/tui/knowledge-search-panel.ts +86 -0
  176. package/src/tui/pulse.ts +86 -0
  177. package/src/tui/research-health.ts +124 -0
  178. package/src/tui/research-panel-color-utils.ts +340 -0
  179. package/src/tui/research-panel-format.ts +54 -0
  180. package/src/tui/research-panel-state.ts +404 -0
  181. package/src/tui/research-panel-wave.ts +280 -0
  182. package/src/tui/research-panel.ts +339 -435
  183. package/src/tui/research-tui-manager.ts +233 -0
  184. package/src/tui/tui-controller.ts +137 -0
  185. package/src/types/extension-context.ts +18 -49
  186. package/src/types/index.ts +61 -0
  187. package/src/types/llm.ts +68 -29
  188. package/src/types/research-panel-types.ts +61 -0
  189. package/src/utils/circuit-breaker.ts +146 -0
  190. package/src/utils/citation-utils.ts +97 -0
  191. package/src/utils/disk-space-checker.ts +55 -0
  192. package/src/utils/error-tracker.ts +265 -0
  193. package/src/utils/host-config.ts +45 -0
  194. package/src/utils/input-validation.ts +8 -10
  195. package/src/utils/json-utils.ts +150 -16
  196. package/src/utils/llm-usage.ts +54 -0
  197. package/src/utils/log-rotation.ts +125 -0
  198. package/src/utils/log-utils.ts +159 -0
  199. package/src/utils/make-resource-loader.ts +3 -1
  200. package/src/utils/metrics-summary.ts +459 -0
  201. package/src/utils/metrics.ts +302 -0
  202. package/src/utils/research-export.ts +28 -12
  203. package/src/utils/safe-unref.ts +31 -0
  204. package/src/utils/shared-links.ts +195 -90
  205. package/src/utils/shutdown-manager.ts +65 -0
  206. package/src/utils/stdio-capture.ts +407 -0
  207. package/src/utils/text-utils.ts +258 -11
  208. package/src/utils/tool-usage-tracker.ts +83 -16
  209. package/src/utils/url-utils.ts +255 -0
  210. package/src/utils/user-agent.ts +28 -0
  211. package/src/web-research/browser-search.ts +179 -0
  212. package/src/web-research/retry-utils.ts +173 -77
  213. package/src/web-research/scraper-types.ts +72 -0
  214. package/src/web-research/scraper-utils.ts +434 -0
  215. package/src/web-research/search.ts +78 -218
  216. package/src/web-research/types.ts +9 -24
  217. package/src/web-research/utils.ts +2 -143
  218. package/src/web-research/web-scraper.ts +477 -0
  219. package/src/youtube/index.ts +154 -0
  220. package/src/youtube/potoken.ts +259 -0
  221. package/src/youtube/transcript-client.ts +390 -0
  222. package/src/youtube/video-id.ts +118 -0
  223. package/config/searxng/default-settings.yml +0 -189
  224. package/config/searxng/limiter.toml +0 -34
  225. package/config/tooling/eslint.config.js +0 -73
  226. package/config/tooling/vitest.config.integration.ts +0 -17
  227. package/config/tooling/vitest.config.ts +0 -40
  228. package/config/tooling/vitest.config.unit.ts +0 -12
  229. package/docs/assets/README-banner.jpg +0 -0
  230. package/scripts/setup.js +0 -212
  231. package/src/infrastructure/searxng-lifecycle.ts +0 -644
  232. package/src/infrastructure/searxng-manager.ts +0 -1183
  233. package/src/infrastructure/state-manager.ts +0 -918
  234. package/src/orchestration/deep-research-reducer.ts +0 -129
  235. package/src/orchestration/id-utils.ts +0 -63
  236. package/src/security/nvd.ts +0 -532
  237. package/src/security/osv.ts +0 -487
  238. package/src/stackexchange/cache.ts +0 -111
  239. package/src/stackexchange/output/index.ts +0 -7
  240. package/src/stackexchange/output/json.ts +0 -21
  241. package/src/tui/searxng-status.ts +0 -80
  242. package/src/utils/inject-date.ts +0 -29
  243. package/src/utils/research-error.ts +0 -205
  244. package/src/utils/searxng-config.ts +0 -94
  245. package/src/utils/session-state.ts +0 -295
  246. package/src/web-research/dependencies.ts +0 -20
  247. package/src/web-research/scrapers.ts +0 -935
  248. package/tsconfig.json +0 -40
@@ -0,0 +1,191 @@
1
+ /**
2
+ * GPU Resource Service
3
+ *
4
+ * Service wrapper for GPU resource management.
5
+ * Provides clean interface for acquiring and releasing the global GPU lock.
6
+ */
7
+
8
+ import { ServiceLifecycle, getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
9
+ import { logger } from '../logger.ts';
10
+ import { ServiceNames } from '../core/service-interfaces.ts';
11
+ import type { IGPUResourceService, IProcessLifecycle, IStateManager } from '../core/service-interfaces.ts';
12
+ import type { SingletonState } from './types/state-types.ts';
13
+
14
+ /**
15
+ * GPU Resource Service Options
16
+ */
17
+ export interface GPUResourceServiceOptions {
18
+ processLifecycle: IProcessLifecycle;
19
+ gpuLockStaleThresholdMs?: number;
20
+ }
21
+
22
+ /**
23
+ * GPU Resource Service Implementation
24
+ */
25
+ export class GPUResourceService implements IGPUResourceService {
26
+ readonly name = ServiceNames.GPU_RESOURCE_SERVICE;
27
+ lifecycle = ServiceLifecycle.UNINITIALIZED;
28
+
29
+ private processLifecycle: IProcessLifecycle | null = null;
30
+ private gpuLockStaleThresholdMs: number = 1000 * 60 * 5; // 5 minutes default
31
+
32
+ constructor(options?: GPUResourceServiceOptions) {
33
+ if (options) {
34
+ this.processLifecycle = options.processLifecycle;
35
+ if (options.gpuLockStaleThresholdMs) {
36
+ this.gpuLockStaleThresholdMs = options.gpuLockStaleThresholdMs;
37
+ }
38
+ }
39
+ }
40
+
41
+ async initialize(ctx?: any): Promise<void> {
42
+ if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
43
+ return;
44
+ }
45
+
46
+ this.lifecycle = ServiceLifecycle.INITIALIZING;
47
+ logger.debug('[GPUResourceService] Initializing...');
48
+
49
+ const container = tryGetServiceContainerFromCtx(ctx);
50
+
51
+ if (!this.processLifecycle) {
52
+ this.processLifecycle = await getService<IProcessLifecycle>(ServiceNames.PROCESS_LIFECYCLE, ctx, container);
53
+ }
54
+
55
+ this.lifecycle = ServiceLifecycle.INITIALIZED;
56
+ logger.debug('[GPUResourceService] Initialized');
57
+ }
58
+
59
+ async dispose(): Promise<void> {
60
+ this.lifecycle = ServiceLifecycle.DISPOSED;
61
+ }
62
+
63
+ /**
64
+ * Internal logic to acquire the GPU lock
65
+ */
66
+ private async performAcquireGpuLock(
67
+ updateState: (updater: (state: SingletonState) => SingletonState | Promise<SingletonState>) => Promise<void>,
68
+ sessionId?: string,
69
+ timeoutMs?: number
70
+ ): Promise<boolean> {
71
+ const start = Date.now();
72
+ const pid = this.processLifecycle!.getCurrentPid();
73
+ const startTime = await this.processLifecycle!.getCurrentProcessStartTime();
74
+
75
+ while (true) {
76
+ let acquired = false;
77
+ let shouldRetry = false;
78
+
79
+ await updateState(async (state) => {
80
+ const currentOwner = state.gpuOwner;
81
+
82
+ // 1. Check if no one owns it
83
+ if (!currentOwner) {
84
+ state.gpuOwner = { pid, startTime: startTime ?? undefined, startedAt: Date.now(), sessionId, holds: 1 };
85
+ acquired = true;
86
+ return state;
87
+ }
88
+
89
+ // 2. Check for re-entrancy (same PID and same process start time).
90
+ // The lock is shared by N concurrent in-process embedding calls, so a
91
+ // re-entrant acquire must increment the hold count rather than re-take
92
+ // a single-owner record — otherwise the first release frees the lock
93
+ // while sibling calls are still on the GPU.
94
+ if (currentOwner.pid === pid && currentOwner.startTime === (startTime ?? undefined)) {
95
+ state.gpuOwner = {
96
+ ...currentOwner,
97
+ startedAt: Date.now(),
98
+ sessionId: sessionId || currentOwner.sessionId,
99
+ holds: (currentOwner.holds ?? 1) + 1,
100
+ };
101
+ acquired = true;
102
+ return state;
103
+ }
104
+
105
+ // 3. Check if owner is dead
106
+ const isAlive = await this.processLifecycle!.isProcessAlive(currentOwner.pid, currentOwner.startTime);
107
+ if (!isAlive) {
108
+ logger.warn(`[GPUResourceService] Reclaiming GPU lock from dead process ${currentOwner.pid}`);
109
+ state.gpuOwner = { pid, startTime: startTime ?? undefined, startedAt: Date.now(), sessionId, holds: 1 };
110
+ acquired = true;
111
+ return state;
112
+ }
113
+
114
+ // 4. Check if lock is stale
115
+ const age = Date.now() - currentOwner.startedAt;
116
+ if (age > this.gpuLockStaleThresholdMs) {
117
+ logger.warn(`[GPUResourceService] Reclaiming stale GPU lock from process ${currentOwner.pid} (age: ${age}ms)`);
118
+ state.gpuOwner = { pid, startTime: startTime ?? undefined, startedAt: Date.now(), sessionId, holds: 1 };
119
+ acquired = true;
120
+ return state;
121
+ }
122
+
123
+ // 5. Still held by another live process
124
+ if (timeoutMs && Date.now() - start < timeoutMs) {
125
+ shouldRetry = true;
126
+ }
127
+ return state;
128
+ });
129
+
130
+ if (acquired) return true;
131
+ if (!shouldRetry) return false;
132
+
133
+ // Wait a bit before retrying
134
+ await new Promise(resolve => setTimeout(resolve, 50));
135
+ }
136
+ }
137
+
138
+ /**
139
+ * Acquire the global GPU lock
140
+ */
141
+ async acquireGpuLock(sessionId?: string | ((updater: any) => Promise<any>), timeoutMs?: number | string, ctx?: any): Promise<boolean> {
142
+ // Check if we are being called by StateManager (sessionId is actually updateState function)
143
+ if (typeof sessionId === 'function') {
144
+ const updateState = sessionId as any;
145
+ const actualSessionId = timeoutMs as any as string;
146
+ const actualTimeoutMs = ctx as any as number;
147
+ return this.performAcquireGpuLock(updateState, actualSessionId, actualTimeoutMs);
148
+ }
149
+
150
+ // Public service interface: delegate to StateManager
151
+ const container = tryGetServiceContainerFromCtx(ctx);
152
+ const stateManager = await getService<IStateManager>(ServiceNames.STATE_MANAGER, ctx, container);
153
+ return stateManager.acquireGpuLock(sessionId as string | undefined, timeoutMs as number | undefined);
154
+ }
155
+
156
+ /**
157
+ * Release the global GPU lock
158
+ */
159
+ async releaseGpuLock(pid?: number | ((updater: any) => Promise<any>), ctx?: any): Promise<void> {
160
+ // Check if we are being called by StateManager (pid is actually updateState function)
161
+ if (typeof pid === 'function') {
162
+ const updateState = pid as any;
163
+ const actualPid = ctx as any as number;
164
+ const currentPid = this.processLifecycle!.getCurrentPid();
165
+ const targetPid = actualPid || currentPid;
166
+
167
+ await updateState(async (state: any) => {
168
+ if (state.gpuOwner && state.gpuOwner.pid === targetPid) {
169
+ // Releasing our own lock decrements the re-entrant hold count and only
170
+ // clears the owner record when the last in-process hold is released.
171
+ // An explicit foreign pid is a forced takedown (e.g. a dead owner) and
172
+ // releases fully regardless of hold count.
173
+ const isOwnLock = targetPid === currentPid && !actualPid;
174
+ const remaining = (state.gpuOwner.holds ?? 1) - 1;
175
+ if (!isOwnLock || remaining <= 0) {
176
+ delete state.gpuOwner;
177
+ } else {
178
+ state.gpuOwner = { ...state.gpuOwner, holds: remaining };
179
+ }
180
+ }
181
+ return state;
182
+ });
183
+ return;
184
+ }
185
+
186
+ // Public service interface: delegate to StateManager
187
+ const container = tryGetServiceContainerFromCtx(ctx);
188
+ const stateManager = await getService<IStateManager>(ServiceNames.STATE_MANAGER, ctx, container);
189
+ return stateManager.releaseGpuLock(pid as number | undefined);
190
+ }
191
+ }
@@ -0,0 +1,475 @@
1
+ /**
2
+ * Knowledge Store Service
3
+ *
4
+ * Service wrapper for the knowledge store functionality.
5
+ * Provides clean interface for embedding and storage operations.
6
+ */
7
+
8
+ import { ServiceLifecycle, getService, tryGetServiceContainerFromCtx } from '../core/service-registry.ts';
9
+ import { ServiceNames } from '../core/interfaces/service-names.ts';
10
+ import { logger } from '../logger.ts';
11
+ import type { IEmbedder, IKnowledgeStore, IKnowledgeStoreService, IWriterQueue, IProcessLifecycle } from '../core/service-interfaces.ts';
12
+ import { FileLockService } from './file-lock-service.ts';
13
+ import { StatePathConfiguration } from './state/state-path-configuration.ts';
14
+ import * as path from 'node:path';
15
+ import { normalizeWorkspacePath } from '../utils/text-utils.ts';
16
+
17
+ // Static imports from knowledge module
18
+ import {
19
+ createKnowledgeStoreComponents,
20
+ forceDeleteKnowledgeStore,
21
+ SUPPORTED_MODELS,
22
+ getModelEmbedderConfig as getKnowledgeModelEmbedderConfig,
23
+ getModelChunkConfig as getKnowledgeModelChunkConfig,
24
+ } from '../knowledge/index.ts';
25
+ import { getEmbedder, clearEmbeddingInstance } from './embedding/embedding-factory.ts';
26
+ import { getConfig } from '../config.ts';
27
+ import { isNativeStackUnavailableError } from '../knowledge/embedder-utils.ts';
28
+
29
+ /**
30
+ * Knowledge Store Service Implementation
31
+ */
32
+ export class KnowledgeStoreService implements IKnowledgeStoreService {
33
+ readonly name = ServiceNames.KNOWLEDGE_STORE;
34
+ lifecycle = ServiceLifecycle.UNINITIALIZED;
35
+
36
+ // Knowledge store components
37
+ private _embedder: IEmbedder | null = null;
38
+ private _store: IKnowledgeStore | null = null;
39
+ private _writerQueue: IWriterQueue | null = null;
40
+ private _initLock: FileLockService | null = null;
41
+ private _cwd: string = process.cwd();
42
+
43
+ // Why the service last went DISABLED. 'mode' (Knowledge Mode = none) is revivable — a
44
+ // later /research-config change re-enables it without a Pi restart. 'native' (missing
45
+ // onnxruntime/lancedb binding) is permanent on this platform, so it stays memoized.
46
+ private _disabledReason: 'mode' | 'native' | null = null;
47
+
48
+ // The KNOWLEDGE_STORE_MODE the live store was last built with. Lets initialize() detect a runtime
49
+ // mode change (project↔global re-scope, or enabled→none) and re-initialize so a /research-config
50
+ // change applies without a Pi restart — not just the DISABLED→enabled revival.
51
+ private _initializedMode: string | undefined = undefined;
52
+
53
+ // Initialization promise to prevent concurrent initialization
54
+ private _initializationPromise: Promise<void> | null = null;
55
+
56
+ async initialize(ctx?: any): Promise<void> {
57
+ // Only an EXPLICIT ctx.cwd may re-scope the store. The lazy callers
58
+ // (getStore/getEmbedder/getWriterQueue) call initialize() with no ctx — they
59
+ // must NOT flip the scope. Resolving the missing cwd to process.cwd() here was
60
+ // a scoping bug: when the pi host's process.cwd() differs from the session's
61
+ // ctx.cwd (e.g. pi launched from ~ but working in ~/projA), every getStore()
62
+ // disposed the correctly-scoped store and rebuilt it against process.cwd(),
63
+ // so reads/writes silently scoped to the wrong workspace. Falling back to the
64
+ // already-resolved this._cwd keeps the scope stable across lazy calls.
65
+ const newCwd = ctx?.cwd || this._cwd;
66
+
67
+ // If already settled for this cwd, return early — UNLESS the live Knowledge Mode has changed
68
+ // since we initialized (via /research-config), in which case re-initialize so the change
69
+ // applies without a Pi restart:
70
+ // - DISABLED('mode') and mode is now != 'none' : was off, now enabled → revive.
71
+ // - INITIALIZED and mode != the built mode : project↔global re-scope, or enabled→'none'.
72
+ // A native-unavailable DISABLED ('native') stays memoized — retrying is futile on that platform.
73
+ if ((this.lifecycle === ServiceLifecycle.INITIALIZED || this.lifecycle === ServiceLifecycle.DISABLED) && this._cwd === newCwd) {
74
+ const liveMode: string = (ctx?.config || getConfig(this._cwd)).KNOWLEDGE_STORE_MODE;
75
+ // Reviving a DISABLED store creates handles (no live state to tear down), so it is safe even
76
+ // from a lazy call and uses the live config. Re-scoping/disabling a LIVE (INITIALIZED) store,
77
+ // however, disposes handles — restrict that to an EXPLICIT ctx.config (mirrors the ctx.cwd
78
+ // invariant): a lazy getStore() reads the base getConfig, which can differ from the
79
+ // iface-resolved config the store was built with, and must never dispose out from under a run.
80
+ const reviveFromMode =
81
+ this.lifecycle === ServiceLifecycle.DISABLED && this._disabledReason === 'mode' && liveMode !== 'none';
82
+ const modeChanged =
83
+ this.lifecycle === ServiceLifecycle.INITIALIZED &&
84
+ ctx?.config !== undefined &&
85
+ this._initializedMode !== undefined &&
86
+ this._initializedMode !== ctx.config.KNOWLEDGE_STORE_MODE;
87
+ if (!reviveFromMode && !modeChanged) {
88
+ return;
89
+ }
90
+ logger.log(`[KnowledgeStoreService] Knowledge Mode changed (${this._initializedMode ?? this._disabledReason} → ${liveMode}); re-initializing store (no restart needed).`);
91
+ // An INITIALIZED store holds live handles (embedder/LanceDB/writer) — dispose before rebuilding.
92
+ if (this.lifecycle === ServiceLifecycle.INITIALIZED) {
93
+ await this.dispose();
94
+ }
95
+ }
96
+
97
+ // If CWD changed, we must dispose the old store and re-initialize.
98
+ // This applies to both INITIALIZED and DISABLED states — if the service
99
+ // is disabled in one project but the new project has knowledge mode enabled,
100
+ // we need to re-initialize.
101
+ if ((this.lifecycle === ServiceLifecycle.INITIALIZED || this.lifecycle === ServiceLifecycle.DISABLED) && this._cwd !== newCwd) {
102
+ logger.log(`[KnowledgeStoreService] CWD changed from ${this._cwd} to ${newCwd}. Re-initializing store...`);
103
+ await this.dispose();
104
+ }
105
+
106
+ // Return existing initialization promise if in progress for the same CWD
107
+ if (this._initializationPromise && this._cwd === newCwd) {
108
+ return this._initializationPromise;
109
+ }
110
+
111
+ this.lifecycle = ServiceLifecycle.INITIALIZING;
112
+ logger.debug('[KnowledgeStoreService] Initializing...');
113
+
114
+ this._initializationPromise = (async () => {
115
+ try {
116
+ // Create the knowledge store components.
117
+ // Prefer an explicit ctx.cwd; otherwise keep the cwd already resolved on this
118
+ // service (defaults to process.cwd() on a never-initialized instance). This
119
+ // mirrors the newCwd resolution above so a lazy getStore() never re-scopes.
120
+ this._cwd = ctx?.cwd || this._cwd;
121
+ // Prefer a caller-resolved config (the SDK/CLI/tool seed ctx.config with the
122
+ // interface-resolved overlay); else load this directory's config.
123
+ const config = ctx?.config || getConfig(this._cwd);
124
+ // Record the mode this build reflects so a later runtime mode change is detected above.
125
+ this._initializedMode = config.KNOWLEDGE_STORE_MODE;
126
+ const embedderFactory = () => getEmbedder(config);
127
+ const reconnectFactory = async () => {
128
+ // MUST await: clearEmbeddingInstance() only nulls the cached instance AFTER its
129
+ // internal `await instance.dispose()` resolves. Without awaiting, getEmbedder(config)
130
+ // runs while the old instance is still cached and (for unchanged config) returns that
131
+ // same still-disposing instance — so the reconnect hands back the very embedder it
132
+ // was trying to replace on a leader handoff.
133
+ await clearEmbeddingInstance();
134
+ return getEmbedder(config);
135
+ };
136
+
137
+ // Acquire lock for initialization/migration
138
+ const container = tryGetServiceContainerFromCtx(ctx);
139
+ const pathConfig = await getService<StatePathConfiguration>(ServiceNames.STATE_PATH_CONFIGURATION, undefined, container);
140
+ const lockPath = path.join(pathConfig.getLockDirPath(), 'knowledge-store-init.lock');
141
+
142
+ // Re-use or create the init lock
143
+ if (!this._initLock) {
144
+ // Increase threshold to 60s because createKnowledgeStoreComponents retries for ~15-20s total
145
+ // and we want to avoid lock theft during this critical initialization phase.
146
+ const processLifecycle = await getService<IProcessLifecycle>(ServiceNames.PROCESS_LIFECYCLE, undefined, container);
147
+ this._initLock = new FileLockService({
148
+ lockFilePath: lockPath,
149
+ lockStaleThreshold: 60000,
150
+ processLifecycle,
151
+ });
152
+ await this._initLock.initialize();
153
+ }
154
+
155
+ const initLock = this._initLock;
156
+ let components;
157
+ try {
158
+ components = await initLock.withLock(async () => {
159
+ return createKnowledgeStoreComponents(
160
+ embedderFactory,
161
+ reconnectFactory,
162
+ (fn) => initLock.withLock(fn),
163
+ config,
164
+ this._cwd
165
+ );
166
+ });
167
+ } catch (err) {
168
+ const errorMsg = err instanceof Error ? err.message : String(err);
169
+ // Detect LanceDB corruption (often due to 0-byte manifest/txn files after a crash)
170
+ if (errorMsg.includes('Generic memory error') && errorMsg.includes('Invalid range 0..0')) {
171
+ logger.warn('[KnowledgeStoreService] Detected corrupted Knowledge Store. Clearing and retrying initialization...');
172
+ try {
173
+ await forceDeleteKnowledgeStore(config, this._cwd);
174
+ components = await initLock.withLock(async () => {
175
+ return createKnowledgeStoreComponents(
176
+ embedderFactory,
177
+ reconnectFactory,
178
+ (fn) => initLock.withLock(fn),
179
+ config,
180
+ this._cwd
181
+ );
182
+ });
183
+ } catch (retryErr) {
184
+ logger.error('[KnowledgeStoreService] Retry after clearing store failed:', retryErr);
185
+ throw retryErr;
186
+ }
187
+ } else {
188
+ throw err;
189
+ }
190
+ }
191
+
192
+ if (!components) {
193
+ logger.debug('[KnowledgeStoreService] Knowledge store is disabled. Setting lifecycle to DISABLED.');
194
+ this.lifecycle = ServiceLifecycle.DISABLED;
195
+ this._disabledReason = 'mode';
196
+ this._embedder = null;
197
+ this._store = null;
198
+ this._writerQueue = null;
199
+ return;
200
+ }
201
+
202
+ this._embedder = components.embedder;
203
+ this._store = components.store;
204
+ this._writerQueue = components.writerQueue;
205
+
206
+ const originalDevice = this._embedder?.getOriginalDevice() ?? '(unknown)';
207
+ const actualDevice = this._embedder?.isInitialized() ? (this._embedder.getDevice() ?? '(deferred)') : '(deferred)';
208
+
209
+ logger.debug(`[KnowledgeStoreService] Initialized. Device: ${actualDevice} (original: ${originalDevice})`);
210
+
211
+ this.lifecycle = ServiceLifecycle.INITIALIZED;
212
+ this._disabledReason = null;
213
+ } catch (err) {
214
+ // A genuinely missing native binding (e.g. Intel macOS / darwin-x64, which
215
+ // has no onnxruntime-node nor @lancedb prebuilt) will never succeed on a
216
+ // retry. Memoize it as DISABLED so every later getStore()/embed() degrades
217
+ // gracefully instead of re-running the full init + retry/backoff storm.
218
+ if (isNativeStackUnavailableError(err)) {
219
+ logger.warn('[KnowledgeStoreService] Native ML/vector stack unavailable on this platform; disabling the knowledge store.');
220
+ this.lifecycle = ServiceLifecycle.DISABLED;
221
+ this._disabledReason = 'native';
222
+ this._embedder = null;
223
+ this._store = null;
224
+ this._writerQueue = null;
225
+ return;
226
+ }
227
+ logger.error('[KnowledgeStoreService] Initialization failed:', err);
228
+ this.lifecycle = ServiceLifecycle.UNINITIALIZED;
229
+ throw err;
230
+ } finally {
231
+ this._initializationPromise = null;
232
+ }
233
+ })();
234
+
235
+ return this._initializationPromise;
236
+ }
237
+
238
+ async dispose(): Promise<void> {
239
+ if (this.lifecycle === ServiceLifecycle.DISPOSED || this.lifecycle === ServiceLifecycle.UNINITIALIZED) {
240
+ return;
241
+ }
242
+
243
+ // If an initialization is still in flight, let it finish before we tear down.
244
+ // Otherwise the init closure would re-assign this._embedder/_store/_writerQueue
245
+ // and flip lifecycle back to INITIALIZED *after* dispose() nulled everything —
246
+ // resurrecting a disposed service with live, never-disposed components (leaked
247
+ // ONNX sessions + LanceDB handles) that isReady() then reports as ready.
248
+ if (this._initializationPromise) {
249
+ try {
250
+ await this._initializationPromise;
251
+ } catch {
252
+ // init failed; its own catch already reset state — nothing to wait on.
253
+ }
254
+ }
255
+
256
+ this.lifecycle = ServiceLifecycle.DISPOSING;
257
+ logger.debug('[KnowledgeStoreService] Disposing...');
258
+
259
+ try {
260
+ if (this._writerQueue) {
261
+ await this._writerQueue.dispose?.();
262
+ }
263
+
264
+ if (this._store) {
265
+ await this._store.close();
266
+ }
267
+
268
+ if (this._embedder) {
269
+ await this._embedder.dispose?.();
270
+ }
271
+
272
+ if (this._initLock) {
273
+ await this._initLock.dispose();
274
+ }
275
+
276
+ this._embedder = null;
277
+ this._store = null;
278
+ this._writerQueue = null;
279
+ this._initLock = null;
280
+
281
+ logger.debug('[KnowledgeStoreService] Disposed');
282
+ } catch (err) {
283
+ logger.error('[KnowledgeStoreService] Error during disposal:', err);
284
+ } finally {
285
+ this.lifecycle = ServiceLifecycle.DISPOSED;
286
+ }
287
+ }
288
+
289
+ /**
290
+ * Check if the knowledge store is ready
291
+ */
292
+ isReady(): boolean {
293
+ return this._embedder !== null && this._store !== null && this._writerQueue !== null;
294
+ }
295
+
296
+ /** The working directory this service initialized against (from ctx.cwd). */
297
+ getCwd(): string {
298
+ return this._cwd;
299
+ }
300
+
301
+ getDisabledReason(): 'mode' | 'native' | null {
302
+ return this._disabledReason;
303
+ }
304
+
305
+ /**
306
+ * Check if the embedder is initialized
307
+ */
308
+ isEmbedderInitialized(): boolean {
309
+ return this._embedder !== null && this._embedder.isInitialized();
310
+ }
311
+
312
+ /**
313
+ * Get the embedder instance
314
+ */
315
+ async getEmbedder(): Promise<IEmbedder | null> {
316
+ await this.initialize();
317
+ if (this.lifecycle === ServiceLifecycle.DISABLED) {
318
+ return null;
319
+ }
320
+ if (!this._embedder) {
321
+ throw new Error('[KnowledgeStoreService] Embedder not initialized');
322
+ }
323
+ return this._embedder;
324
+ }
325
+
326
+ /**
327
+ * Get the knowledge store instance
328
+ */
329
+ async getStore(): Promise<IKnowledgeStore | null> {
330
+ await this.initialize();
331
+ if (this.lifecycle === ServiceLifecycle.DISABLED) {
332
+ return null;
333
+ }
334
+ if (!this._store) {
335
+ throw new Error('[KnowledgeStoreService] Store not initialized');
336
+ }
337
+ return this._store;
338
+ }
339
+
340
+ /**
341
+ * Get the writer queue instance
342
+ */
343
+ async getWriterQueue(): Promise<IWriterQueue | null> {
344
+ await this.initialize();
345
+ if (this.lifecycle === ServiceLifecycle.DISABLED) {
346
+ return null;
347
+ }
348
+ if (!this._writerQueue) {
349
+ throw new Error('[KnowledgeStoreService] Writer queue not initialized');
350
+ }
351
+ return this._writerQueue;
352
+ }
353
+
354
+ /**
355
+ * Get the embedder device
356
+ */
357
+ getDevice(): string | null {
358
+ return this._embedder?.getDevice() ?? null;
359
+ }
360
+
361
+ /**
362
+ * Get the original device preference
363
+ */
364
+ getOriginalDevice(): string | null {
365
+ return this._embedder?.getOriginalDevice() ?? null;
366
+ }
367
+
368
+ /**
369
+ * Embed a text string
370
+ */
371
+ async embed(text: string): Promise<number[]> {
372
+ const embedder = await this.getEmbedder();
373
+ if (!embedder) {
374
+ throw new Error('[KnowledgeStoreService] Embedder not available (store disabled)');
375
+ }
376
+ const result = await embedder.embed(text);
377
+ return Array.from(result);
378
+ }
379
+
380
+ /**
381
+ * Embed multiple text strings
382
+ */
383
+ async embedMany(texts: string[]): Promise<number[][]> {
384
+ const embedder = await this.getEmbedder();
385
+ if (!embedder) {
386
+ throw new Error('[KnowledgeStoreService] Embedder not available (store disabled)');
387
+ }
388
+ const results = await embedder.embedMany(texts);
389
+ return results.map(r => Array.from(r));
390
+ }
391
+
392
+ /**
393
+ * Clear the knowledge store entries for the current scope.
394
+ *
395
+ * In 'global' mode, clears only global entries (is_global = true).
396
+ * In 'project' mode, clears only this workspace's entries.
397
+ * In 'none' mode, no-op (store is disabled).
398
+ */
399
+ async clear(): Promise<void> {
400
+ const config = getConfig(this._cwd);
401
+ if (config.KNOWLEDGE_STORE_MODE === 'global') {
402
+ await this.clearGlobal();
403
+ } else if (config.KNOWLEDGE_STORE_MODE === 'project') {
404
+ await this.clearLocal();
405
+ }
406
+ // 'none' mode: no-op (store is disabled, nothing to clear)
407
+ }
408
+
409
+ /**
410
+ * Clear only local project entries (workspace-scoped, NOT global).
411
+ * Entries that are both local AND global are left alone — use clearGlobal
412
+ * for those.
413
+ */
414
+ async clearLocal(): Promise<void> {
415
+ const store = await this.getStore();
416
+ if (!store) return;
417
+
418
+ // Normalize workspace path to match how entries are stored (see KnowledgeStore.getWorkspace)
419
+ const workspace = normalizeWorkspacePath(this._cwd);
420
+ const escaped = workspace.replace(/'/g, "''");
421
+
422
+ // Delete all entries that were contributed from this project workspace.
423
+ // If they were shared globally, they will also be removed from the global view.
424
+ await store.clear(`workspace = '${escaped}'`);
425
+ }
426
+
427
+ /**
428
+ * Clear only global entries (cross-project, visible to all workspaces).
429
+ */
430
+ async clearGlobal(): Promise<void> {
431
+ const store = await this.getStore();
432
+ if (store) {
433
+ await store.clear('is_global = true');
434
+ }
435
+ }
436
+
437
+ /**
438
+ * Export the knowledge store for web use.
439
+ */
440
+ async exportForWeb(outputPath: string): Promise<void> {
441
+ const store = await this.getStore();
442
+ if (store) {
443
+ await store.exportForWeb(outputPath);
444
+ }
445
+ }
446
+
447
+ /**
448
+ * Get supported models
449
+ */
450
+ getSupportedModels(): ReadonlyArray<{ id: string; multilingual: boolean }> {
451
+ return SUPPORTED_MODELS;
452
+ }
453
+
454
+ /**
455
+ * Get model embedder configuration
456
+ */
457
+ getModelEmbedderConfig(modelId: string): {
458
+ pooling: 'mean' | 'cls' | 'last_token';
459
+ queryPrefix?: string;
460
+ documentPrefix?: string;
461
+ maxTokens?: number;
462
+ batchSize?: number;
463
+ charsPerToken?: number;
464
+ useCache?: boolean;
465
+ } {
466
+ return getKnowledgeModelEmbedderConfig(modelId);
467
+ }
468
+
469
+ /**
470
+ * Get model chunk configuration
471
+ */
472
+ getModelChunkConfig(modelId: string): { chunkSize: number; overlapPct: number } {
473
+ return getKnowledgeModelChunkConfig(modelId);
474
+ }
475
+ }