@octopi-agent/engine 0.56.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +43 -1
  4. package/dist/config-schema/substrate.d.ts +42 -0
  5. package/dist/config-schema/substrate.js +58 -0
  6. package/dist/config.d.ts +39 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/agent/config-bridge.js +3 -0
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  23. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  24. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  25. package/dist/harness/capabilities/document/factory.js +37 -0
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  37. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  39. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  40. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  41. package/dist/harness/capabilities/document/worker-port.js +78 -0
  42. package/dist/harness/context/capabilities/index.d.ts +2 -2
  43. package/dist/harness/context/capabilities/index.js +2 -2
  44. package/dist/harness/context/constitution/default-agents.md +1 -0
  45. package/dist/harness/context/index.d.ts +2 -2
  46. package/dist/harness/context/index.js +1 -1
  47. package/dist/harness/context/layer-types.d.ts +26 -2
  48. package/dist/harness/context/layer-types.js +59 -11
  49. package/dist/harness/context/layers.d.ts +3 -0
  50. package/dist/harness/context/layers.js +38 -12
  51. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  52. package/dist/harness/context/system-prompt-assembler.js +2 -0
  53. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  54. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
  55. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  56. package/dist/harness/index.d.ts +5 -3
  57. package/dist/harness/index.js +3 -2
  58. package/dist/harness/knowledge/adapters.d.ts +8 -5
  59. package/dist/harness/knowledge/adapters.js +53 -13
  60. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  61. package/dist/harness/knowledge/client.d.ts +197 -0
  62. package/dist/harness/knowledge/client.js +237 -0
  63. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  64. package/dist/harness/knowledge/connectors.d.ts +3 -3
  65. package/dist/harness/knowledge/connectors.js +14 -8
  66. package/dist/harness/knowledge/db.d.ts +42 -3
  67. package/dist/harness/knowledge/db.js +466 -83
  68. package/dist/harness/knowledge/describe.d.ts +2 -0
  69. package/dist/harness/knowledge/describe.js +5 -1
  70. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  71. package/dist/harness/knowledge/document-extract-worker.js +68 -0
  72. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  73. package/dist/harness/knowledge/embed-runner.js +171 -0
  74. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  75. package/dist/harness/knowledge/engine-thread.js +302 -0
  76. package/dist/harness/knowledge/extract-document.d.ts +33 -0
  77. package/dist/harness/knowledge/extract-document.js +86 -0
  78. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  79. package/dist/harness/knowledge/fetchers.js +52 -16
  80. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  81. package/dist/harness/knowledge/file-identity.js +172 -0
  82. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  83. package/dist/harness/knowledge/file-index-store.js +132 -0
  84. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  85. package/dist/harness/knowledge/file-limits.js +175 -0
  86. package/dist/harness/knowledge/fts.d.ts +60 -0
  87. package/dist/harness/knowledge/fts.js +218 -0
  88. package/dist/harness/knowledge/grounding.d.ts +3 -2
  89. package/dist/harness/knowledge/http-app.d.ts +50 -0
  90. package/dist/harness/knowledge/http-app.js +652 -0
  91. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  92. package/dist/harness/knowledge/http-bridge.js +17 -0
  93. package/dist/harness/knowledge/index-store.d.ts +166 -63
  94. package/dist/harness/knowledge/index-store.js +1025 -282
  95. package/dist/harness/knowledge/index.d.ts +47 -5
  96. package/dist/harness/knowledge/index.js +24 -2
  97. package/dist/harness/knowledge/ingest.d.ts +322 -49
  98. package/dist/harness/knowledge/ingest.js +1383 -267
  99. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  100. package/dist/harness/knowledge/job-control-state.js +44 -0
  101. package/dist/harness/knowledge/job-control.d.ts +35 -0
  102. package/dist/harness/knowledge/job-control.js +86 -0
  103. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  104. package/dist/harness/knowledge/job-queue.js +173 -0
  105. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  106. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  107. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  108. package/dist/harness/knowledge/local-stack.js +63 -0
  109. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  110. package/dist/harness/knowledge/membership-store.js +198 -0
  111. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  112. package/dist/harness/knowledge/network-guard.js +130 -85
  113. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  114. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  115. package/dist/harness/knowledge/query-service.d.ts +272 -0
  116. package/dist/harness/knowledge/query-service.js +326 -0
  117. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  118. package/dist/harness/knowledge/query-worker-client.js +207 -0
  119. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  120. package/dist/harness/knowledge/query-worker.js +186 -0
  121. package/dist/harness/knowledge/read-http.d.ts +35 -0
  122. package/dist/harness/knowledge/read-http.js +373 -0
  123. package/dist/harness/knowledge/retriever.d.ts +30 -5
  124. package/dist/harness/knowledge/retriever.js +98 -35
  125. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  126. package/dist/harness/knowledge/secret-scan.js +25 -2
  127. package/dist/harness/knowledge/serve.d.ts +48 -0
  128. package/dist/harness/knowledge/serve.js +336 -0
  129. package/dist/harness/knowledge/source-store.d.ts +83 -16
  130. package/dist/harness/knowledge/source-store.js +205 -78
  131. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  132. package/dist/harness/knowledge/start-service-process.js +158 -0
  133. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  134. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  135. package/dist/harness/knowledge/topics.d.ts +13 -0
  136. package/dist/harness/knowledge/topics.js +48 -0
  137. package/dist/harness/knowledge/types.d.ts +1 -1
  138. package/dist/harness/knowledge/types.js +1 -1
  139. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  140. package/dist/harness/knowledge/vector-ann.js +68 -0
  141. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  142. package/dist/harness/knowledge/writer-lock.js +71 -0
  143. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  144. package/dist/harness/knowledge/writer-service.js +355 -0
  145. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  146. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  147. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  148. package/dist/harness/knowledge/writer-worker.js +189 -0
  149. package/dist/harness/memory/index.d.ts +3 -2
  150. package/dist/harness/memory/index.js +3 -1
  151. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  152. package/dist/harness/memory/retrieval-rank.js +60 -0
  153. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  154. package/dist/harness/memory/sqlite/embedding.js +109 -16
  155. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  156. package/dist/harness/memory/sqlite/index.js +1 -1
  157. package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
  158. package/dist/harness/memory/sqlite/keyword-search.js +104 -17
  159. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  160. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  161. package/dist/harness/memory/store.js +39 -21
  162. package/dist/harness/memory/types.d.ts +7 -0
  163. package/dist/harness/run/runner.d.ts +1 -0
  164. package/dist/harness/run/runner.js +4 -2
  165. package/dist/harness/session/attachments/service.d.ts +1 -1
  166. package/dist/harness/session/attachments/service.js +1 -1
  167. package/dist/harness/session/tasks/index.d.ts +1 -1
  168. package/dist/harness/session/tasks/index.js +1 -1
  169. package/dist/harness/session/tasks/render.d.ts +10 -0
  170. package/dist/harness/session/tasks/render.js +27 -0
  171. package/dist/index.d.ts +4 -2
  172. package/dist/index.js +2 -1
  173. package/dist/worker-path.d.ts +8 -0
  174. package/dist/worker-path.js +15 -0
  175. package/package.json +2 -2
@@ -0,0 +1,158 @@
1
+ /**
2
+ * startKnowledgeServiceProcess — fork 独立子进程托管 Knowledge Service
3
+ *
4
+ * 为什么是进程而不是 Worker:知识索引重且稳定不可控,不得与 Gateway 共命运。
5
+ * Worker 仍共享堆/OOM 与进程生命周期,只是把 event loop 挪开,不是故障隔离。
6
+ *
7
+ * fork 根因约束(与 `octopi serve` daemon 同款):
8
+ * - `execArgv: []`:禁止继承父进程 loader/inspect/NODE_OPTIONS 钩子
9
+ * - 剥离 `NODE_CHANNEL_*`:父进程若是 IPC 子进程(vitest / daemon),继承会错绑通道
10
+ * - `stdio` 不用 `inherit`:Windows 管道句柄继承会干扰 IPC
11
+ * - Electron 宿主:`ELECTRON_RUN_AS_NODE=1` + **保持** `process.execPath`。
12
+ * 换成捆绑 node.exe 当 execPath 会切断 Windows 上的 fork IPC。
13
+ */
14
+ import { fork } from 'node:child_process';
15
+ import { fileURLToPath } from 'node:url';
16
+ import { mkdtemp, writeFile, rm } from 'node:fs/promises';
17
+ import { tmpdir } from 'node:os';
18
+ import { join } from 'node:path';
19
+ import { resolveWorkerUrl } from '../../worker-path.js';
20
+ function isElectronHost() {
21
+ return (Boolean(process.versions.electron) ||
22
+ /electron|MiMo\.exe/i.test(process.execPath));
23
+ }
24
+ function childNodeEnv() {
25
+ const env = { ...process.env };
26
+ // 父进程可能是 IPC 子进程(vitest worker / octopi serve daemon):不得把通道句柄带进孙进程
27
+ delete env.NODE_CHANNEL_FD;
28
+ delete env.NODE_CHANNEL_SERIALIZATION;
29
+ delete env.NODE_UNIQUE_ID;
30
+ if (isElectronHost()) {
31
+ env.ELECTRON_RUN_AS_NODE = '1';
32
+ }
33
+ return env;
34
+ }
35
+ /**
36
+ * 在独立子进程启动 Knowledge HTTP Service。
37
+ *
38
+ * @param opts - 与 startKnowledgeService 同构 + embeddingModels(子进程内重建 provider)
39
+ * @returns child + 端口 + stop
40
+ */
41
+ export async function startKnowledgeServiceProcess(opts) {
42
+ const entry = fileURLToPath(resolveWorkerUrl('./knowledge-serve-child.js', import.meta.url));
43
+ const bootDir = await mkdtemp(join(tmpdir(), 'octopi-kn-boot-'));
44
+ const bootPath = join(bootDir, 'boot.json');
45
+ await writeFile(bootPath, JSON.stringify({
46
+ dbPath: opts.dbPath,
47
+ host: opts.host,
48
+ port: opts.port ?? 18280,
49
+ tokens: opts.tokens,
50
+ documentConfig: opts.documentConfig ?? null,
51
+ embed: opts.embed,
52
+ sqliteVecExtensionPath: opts.sqliteVecExtensionPath,
53
+ embeddingModels: opts.embeddingModels ?? null,
54
+ }), 'utf8');
55
+ // @types/node 的 ForkOptions 未声明 windowsHide 等字段,运行时有效
56
+ // execPath 必须保持 process.execPath:Electron 下换捆绑 node 会切断 IPC
57
+ const forkOpts = {
58
+ stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
59
+ windowsHide: true,
60
+ execArgv: [],
61
+ env: childNodeEnv(),
62
+ };
63
+ const child = fork(entry, [bootPath], forkOpts);
64
+ const startTimeoutMs = opts.startTimeoutMs ?? 30_000;
65
+ try {
66
+ const port = await new Promise((resolve, reject) => {
67
+ const timer = setTimeout(() => reject(new Error('knowledge_process_start_timeout')), startTimeoutMs);
68
+ const settle = (fn) => {
69
+ clearTimeout(timer);
70
+ child.off('message', onMsg);
71
+ fn();
72
+ };
73
+ const onMsg = (msg) => {
74
+ if (msg?.type === 'ready' && typeof msg.port === 'number') {
75
+ const port = msg.port;
76
+ settle(() => resolve(port));
77
+ }
78
+ else if (msg?.type === 'error') {
79
+ settle(() => reject(new Error(msg.error ?? 'knowledge_process_start_failed')));
80
+ }
81
+ };
82
+ child.on('message', onMsg);
83
+ child.once('error', (err) => {
84
+ settle(() => reject(err));
85
+ });
86
+ child.once('exit', (code, signal) => {
87
+ settle(() => reject(new Error(signal
88
+ ? `knowledge_process_signal_${signal}`
89
+ : `knowledge_process_exit_${code ?? 'unknown'}`)));
90
+ });
91
+ });
92
+ return {
93
+ child,
94
+ port,
95
+ stop: async () => {
96
+ await stopChild(child);
97
+ },
98
+ };
99
+ }
100
+ finally {
101
+ await rm(bootDir, { recursive: true, force: true }).catch(() => undefined);
102
+ }
103
+ }
104
+ async function stopChild(child) {
105
+ if (child.exitCode != null || child.signalCode != null)
106
+ return;
107
+ await new Promise((resolve) => {
108
+ const timer = setTimeout(() => {
109
+ try {
110
+ child.kill('SIGTERM');
111
+ }
112
+ catch {
113
+ /* already gone */
114
+ }
115
+ const hard = setTimeout(() => {
116
+ try {
117
+ child.kill('SIGKILL');
118
+ }
119
+ catch {
120
+ /* ignore */
121
+ }
122
+ resolve();
123
+ }, 2_000);
124
+ hard.unref?.();
125
+ }, 3_000);
126
+ child.once('exit', () => {
127
+ clearTimeout(timer);
128
+ resolve();
129
+ });
130
+ try {
131
+ if (child.connected) {
132
+ child.send({ type: 'close' }, () => {
133
+ /* send callback only observes channel errors; exit path resolves */
134
+ });
135
+ }
136
+ else {
137
+ clearTimeout(timer);
138
+ try {
139
+ child.kill('SIGTERM');
140
+ }
141
+ catch {
142
+ /* ignore */
143
+ }
144
+ resolve();
145
+ }
146
+ }
147
+ catch {
148
+ clearTimeout(timer);
149
+ try {
150
+ child.kill('SIGTERM');
151
+ }
152
+ catch {
153
+ /* ignore */
154
+ }
155
+ resolve();
156
+ }
157
+ });
158
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Text parse worker — 读盘 + hash + 切块 + FTS token 全在独立线程
3
+ *
4
+ * 主线程/Service 事件循环不得做多 MB 文本的 CPU 切块与 CJK 二元组。
5
+ */
6
+ import { parentPort, workerData } from 'node:worker_threads';
7
+ import { readFile } from 'node:fs/promises';
8
+ import { createHash } from 'node:crypto';
9
+ import { FormatAdapterRegistry, htmlAdapter, markdownAdapter, textAdapter, } from './adapters.js';
10
+ import { buildFtsTokens } from './fts.js';
11
+ function adapterById(id) {
12
+ if (id === 'html')
13
+ return htmlAdapter;
14
+ if (id === 'markdown')
15
+ return markdownAdapter;
16
+ if (id === 'text')
17
+ return textAdapter;
18
+ return null;
19
+ }
20
+ async function main() {
21
+ const job = workerData;
22
+ const content = job.inlineText != null ? job.inlineText : await readFile(job.path, 'utf8');
23
+ const contentHash = createHash('sha256').update(content).digest('hex');
24
+ const registry = new FormatAdapterRegistry();
25
+ let adapter = registry.match(job.path) ?? adapterById(job.adapterId);
26
+ if (!adapter) {
27
+ // 虚拟文档常无扩展名:退 text,而不是整源 no_adapter
28
+ adapter = textAdapter;
29
+ }
30
+ const maxTextChars = job.maxTextChars ?? 2_000_000;
31
+ const text = job.sizeDecision === 'partial' && content.length > maxTextChars
32
+ ? `${content.slice(0, maxTextChars)}\n\ntruncated: ${content.length - maxTextChars} chars omitted]`
33
+ : content;
34
+ const chunks = adapter.chunk(text, job.path);
35
+ for (const c of chunks) {
36
+ c.ftsToks = buildFtsTokens(c.text, job.path);
37
+ }
38
+ parentPort?.postMessage({
39
+ ok: true,
40
+ contentHash,
41
+ contentLength: content.length,
42
+ adapterId: adapter.id,
43
+ chunks,
44
+ });
45
+ }
46
+ main().catch((err) => {
47
+ parentPort?.postMessage({
48
+ ok: false,
49
+ error: {
50
+ name: err instanceof Error ? err.name : 'Error',
51
+ message: err instanceof Error ? err.message : String(err),
52
+ },
53
+ });
54
+ });
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Catalog topics — 从索引路径派生主题线索(无 LLM)
3
+ *
4
+ * 供 Tier 0 catalog 的 `topics` 字段;不依赖人工 description / auto-describe。
5
+ */
6
+ /**
7
+ * 路径词频 → 话题短词(≤ maxTopics)
8
+ *
9
+ * @param paths 已入库相对/绝对路径
10
+ * @param maxTopics 上限(默认 6)
11
+ * @returns 稳定排序的短词列表
12
+ */
13
+ export declare function deriveTopicsFromPaths(paths: string[], maxTopics?: number): string[];
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Catalog topics — 从索引路径派生主题线索(无 LLM)
3
+ *
4
+ * 供 Tier 0 catalog 的 `topics` 字段;不依赖人工 description / auto-describe。
5
+ */
6
+ const STOP_TOKENS = new Set([
7
+ 'src', 'lib', 'dist', 'build', 'out', 'bin', 'pkg', 'packages', 'node_modules',
8
+ 'test', 'tests', 'spec', 'specs', 'assets', 'public', 'static', 'vendor',
9
+ 'the', 'and', 'for', 'with', 'from', 'and', 'app', 'utils', 'common', 'helpers',
10
+ ]);
11
+ /** 从单条路径抽出候选词(目录名 + 文件名词干) */
12
+ function tokensFromPath(path) {
13
+ const parts = path.split(/[\\/]+/).filter(Boolean);
14
+ const out = [];
15
+ for (const part of parts) {
16
+ const stem = part.replace(/\.[^.]+$/, '');
17
+ for (const t of stem.split(/[-_\s.]+/)) {
18
+ if (!t)
19
+ continue;
20
+ const lower = t.toLowerCase();
21
+ if (lower.length < 2 || STOP_TOKENS.has(lower) || /^\d+$/.test(lower))
22
+ continue;
23
+ out.push(lower);
24
+ }
25
+ }
26
+ return out;
27
+ }
28
+ /**
29
+ * 路径词频 → 话题短词(≤ maxTopics)
30
+ *
31
+ * @param paths 已入库相对/绝对路径
32
+ * @param maxTopics 上限(默认 6)
33
+ * @returns 稳定排序的短词列表
34
+ */
35
+ export function deriveTopicsFromPaths(paths, maxTopics = 6) {
36
+ if (paths.length === 0)
37
+ return [];
38
+ const counts = new Map();
39
+ for (const p of paths) {
40
+ for (const t of tokensFromPath(p)) {
41
+ counts.set(t, (counts.get(t) ?? 0) + 1);
42
+ }
43
+ }
44
+ return [...counts.entries()]
45
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
46
+ .slice(0, maxTopics)
47
+ .map(([t]) => t);
48
+ }
@@ -2,7 +2,7 @@
2
2
  * Knowledge 源模型 — 外生语料登记(arch/knowledge-layer.md)
3
3
  *
4
4
  * Scope:Global(默认可见可屏蔽)/ Project(显式挂载)/ Session(临时)。
5
- * 无 Agent 级源。Index 不在本模块(P2+);此处是权威注册表与可见性。
5
+ * 无 Agent 级源。此处是源类型与可见性契约;Index 投影在 index-store/db。
6
6
  */
7
7
  /** 不透明 branded id(跨边界防混用;AGENTS:Brand opaque IDs) */
8
8
  export type Branded<T, B extends string> = T & {
@@ -2,7 +2,7 @@
2
2
  * Knowledge 源模型 — 外生语料登记(arch/knowledge-layer.md)
3
3
  *
4
4
  * Scope:Global(默认可见可屏蔽)/ Project(显式挂载)/ Session(临时)。
5
- * 无 Agent 级源。Index 不在本模块(P2+);此处是权威注册表与可见性。
5
+ * 无 Agent 级源。此处是源类型与可见性契约;Index 投影在 index-store/db。
6
6
  */
7
7
  /** 从外部输入收窄为 SourceId(仅边界处使用) */
8
8
  export function asSourceId(id) {
@@ -0,0 +1,42 @@
1
+ /**
2
+ * VectorAnn — 无 sqlite-vec 时的近似候选裁剪(IVF-lite 桶分区)
3
+ *
4
+ * 目标:把 JS 余弦从 O(全库) 降到 O(候选桶)。桶 = 向量粗量化哈希;
5
+ * 查询先扫同桶,不足再扩桶,最后才全扫。sqlite-vec 可用时仍优先 KNN。
6
+ * Index 非权威:bucket 可由 embedding 重算重建。
7
+ */
8
+ /** 桶数(256:粗量化后碰撞可控,SQL IN 列表也短) */
9
+ export declare const VECTOR_BUCKETS = 256;
10
+ /**
11
+ * 向量 → 桶 id(粗量化:取前若干维 × 8 档,混合成 0..255)
12
+ *
13
+ * @param embedding - 浮点向量
14
+ */
15
+ export declare function vectorBucket(embedding: number[] | Float32Array): number;
16
+ /** 查询侧:主桶 + 邻桶(汉明距离 1 的 8 个变体) */
17
+ export declare function queryBuckets(embedding: number[]): number[];
18
+ export interface AnnCandidateRow {
19
+ id: string;
20
+ embedding: number[] | null;
21
+ }
22
+ export interface AnnScoredHit {
23
+ id: string;
24
+ score: number;
25
+ }
26
+ /**
27
+ * 桶裁剪 + 余弦打分(不加载 text)
28
+ *
29
+ * @param queryEmbedding - 查询向量
30
+ * @param rows - 候选行(调用方已按 sourceIds 过滤;可先按 bucket 预取)
31
+ * @param limit - top-k
32
+ * @returns 按分数降序的 id+score
33
+ */
34
+ export declare function scoreAnnCandidates(queryEmbedding: number[], rows: AnnCandidateRow[], limit: number): AnnScoredHit[];
35
+ /**
36
+ * 由桶分组排序候选:主桶优先,再邻桶,最后其余(供分阶段扫)
37
+ *
38
+ * @param queryEmbedding - 查询向量
39
+ * @param bucketOf - chunkId → bucket
40
+ * @returns 按优先级排序的 bucket id 列表
41
+ */
42
+ export declare function bucketScanOrder(queryEmbedding: number[]): number[];
@@ -0,0 +1,68 @@
1
+ /**
2
+ * VectorAnn — 无 sqlite-vec 时的近似候选裁剪(IVF-lite 桶分区)
3
+ *
4
+ * 目标:把 JS 余弦从 O(全库) 降到 O(候选桶)。桶 = 向量粗量化哈希;
5
+ * 查询先扫同桶,不足再扩桶,最后才全扫。sqlite-vec 可用时仍优先 KNN。
6
+ * Index 非权威:bucket 可由 embedding 重算重建。
7
+ */
8
+ import { cosineSimilarity } from '../memory/sqlite/vector-search.js';
9
+ /** 桶数(256:粗量化后碰撞可控,SQL IN 列表也短) */
10
+ export const VECTOR_BUCKETS = 256;
11
+ /**
12
+ * 向量 → 桶 id(粗量化:取前若干维 × 8 档,混合成 0..255)
13
+ *
14
+ * @param embedding - 浮点向量
15
+ */
16
+ export function vectorBucket(embedding) {
17
+ if (!embedding?.length)
18
+ return 0;
19
+ const n = Math.min(8, embedding.length);
20
+ let h = 0;
21
+ for (let i = 0; i < n; i++) {
22
+ // 量化到 [-4, 3] 共 8 档,降低浮点噪声
23
+ const q = Math.max(-4, Math.min(3, Math.floor(Number(embedding[i]) * 2)));
24
+ h = (h * 8 + (q + 4)) & 0xff;
25
+ }
26
+ return h;
27
+ }
28
+ /** 查询侧:主桶 + 邻桶(汉明距离 1 的 8 个变体) */
29
+ export function queryBuckets(embedding) {
30
+ const primary = vectorBucket(embedding);
31
+ const out = new Set([primary]);
32
+ for (let bit = 0; bit < 8; bit++) {
33
+ out.add(primary ^ (1 << bit));
34
+ }
35
+ return [...out];
36
+ }
37
+ /**
38
+ * 桶裁剪 + 余弦打分(不加载 text)
39
+ *
40
+ * @param queryEmbedding - 查询向量
41
+ * @param rows - 候选行(调用方已按 sourceIds 过滤;可先按 bucket 预取)
42
+ * @param limit - top-k
43
+ * @returns 按分数降序的 id+score
44
+ */
45
+ export function scoreAnnCandidates(queryEmbedding, rows, limit) {
46
+ const hits = [];
47
+ for (const row of rows) {
48
+ const emb = row.embedding;
49
+ if (!emb?.length || emb.length !== queryEmbedding.length)
50
+ continue;
51
+ const sim = cosineSimilarity(queryEmbedding, emb);
52
+ if (sim <= 0)
53
+ continue;
54
+ hits.push({ id: row.id, score: sim });
55
+ }
56
+ hits.sort((a, b) => b.score - a.score);
57
+ return hits.slice(0, limit);
58
+ }
59
+ /**
60
+ * 由桶分组排序候选:主桶优先,再邻桶,最后其余(供分阶段扫)
61
+ *
62
+ * @param queryEmbedding - 查询向量
63
+ * @param bucketOf - chunkId → bucket
64
+ * @returns 按优先级排序的 bucket id 列表
65
+ */
66
+ export function bucketScanOrder(queryEmbedding) {
67
+ return queryBuckets(queryEmbedding);
68
+ }
@@ -0,0 +1,13 @@
1
+ export interface WriterLockHolder {
2
+ lockPath: string;
3
+ /** 释放锁文件(仅仍属本 PID 时) */
4
+ release(): void;
5
+ }
6
+ /**
7
+ * 独占创建 writer.lock;持有者死亡则回收。
8
+ *
9
+ * @param dbPath - knowledge.db 绝对路径
10
+ * @returns lock 路径与 release
11
+ * @throws 已有存活持有者时抛错
12
+ */
13
+ export declare function acquireKnowledgeWriterLock(dbPath: string): Promise<WriterLockHolder>;
@@ -0,0 +1,71 @@
1
+ /**
2
+ * knowledge.db writer lock — sole writer process (Knowledge Service child)
3
+ *
4
+ * The lock MUST be held by the process that opens knowledge.db, not by Gateway.
5
+ */
6
+ import { readFileSync, unlinkSync, writeFileSync } from 'node:fs';
7
+ async function isPidAlive(pid) {
8
+ try {
9
+ process.kill(pid, 0);
10
+ return true;
11
+ }
12
+ catch {
13
+ return false;
14
+ }
15
+ }
16
+ /**
17
+ * 独占创建 writer.lock;持有者死亡则回收。
18
+ *
19
+ * @param dbPath - knowledge.db 绝对路径
20
+ * @returns lock 路径与 release
21
+ * @throws 已有存活持有者时抛错
22
+ */
23
+ export async function acquireKnowledgeWriterLock(dbPath) {
24
+ const lockPath = `${dbPath}.writer.lock`;
25
+ const tryWrite = () => {
26
+ writeFileSync(lockPath, JSON.stringify({ pid: process.pid, at: Date.now() }), { encoding: 'utf8', flag: 'wx' });
27
+ };
28
+ try {
29
+ tryWrite();
30
+ }
31
+ catch {
32
+ let holderPid;
33
+ try {
34
+ const lock = JSON.parse(readFileSync(lockPath, 'utf8'));
35
+ holderPid = lock.pid;
36
+ }
37
+ catch {
38
+ holderPid = undefined;
39
+ }
40
+ const holderAlive = holderPid != null && holderPid !== process.pid && (await isPidAlive(holderPid));
41
+ if (holderAlive) {
42
+ throw new Error(`knowledge.db already locked by PID ${holderPid} (${lockPath}); stop the other Knowledge Service or use remote baseUrl`);
43
+ }
44
+ try {
45
+ unlinkSync(lockPath);
46
+ }
47
+ catch {
48
+ /* ignore */
49
+ }
50
+ tryWrite();
51
+ }
52
+ return {
53
+ lockPath,
54
+ release() {
55
+ try {
56
+ const lock = JSON.parse(readFileSync(lockPath, 'utf8'));
57
+ if (lock.pid !== process.pid)
58
+ return;
59
+ }
60
+ catch {
61
+ return;
62
+ }
63
+ try {
64
+ unlinkSync(lockPath);
65
+ }
66
+ catch {
67
+ /* ignore */
68
+ }
69
+ },
70
+ };
71
+ }