@octopi-agent/engine 0.58.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +38 -1
- package/dist/config-schema/substrate.d.ts +37 -0
- package/dist/config-schema/substrate.js +47 -0
- package/dist/config.d.ts +30 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/index.d.ts +4 -2
- package/dist/harness/index.js +2 -1
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +80 -17
- package/dist/harness/knowledge/source-store.js +160 -89
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WorkerWriteService — API 线程侧 Writer RPC 客户端
|
|
3
|
+
*
|
|
4
|
+
* 不持有 SQLite;一切变更进 Writer Worker(唯一写者)。
|
|
5
|
+
*/
|
|
6
|
+
import { Worker } from 'node:worker_threads';
|
|
7
|
+
import { resolveWorkerUrl } from '../../worker-path.js';
|
|
8
|
+
export class WorkerWriteService {
|
|
9
|
+
worker;
|
|
10
|
+
nextId = 1;
|
|
11
|
+
pending = new Map();
|
|
12
|
+
progressCbs = new Set();
|
|
13
|
+
disposed = false;
|
|
14
|
+
bootSettled = false;
|
|
15
|
+
bootResolve = null;
|
|
16
|
+
bootReject = null;
|
|
17
|
+
bootPromise;
|
|
18
|
+
timeoutMs;
|
|
19
|
+
constructor(worker, timeoutMs) {
|
|
20
|
+
this.worker = worker;
|
|
21
|
+
this.timeoutMs = timeoutMs;
|
|
22
|
+
this.bootPromise = new Promise((resolve, reject) => {
|
|
23
|
+
this.bootResolve = resolve;
|
|
24
|
+
this.bootReject = reject;
|
|
25
|
+
});
|
|
26
|
+
worker.on('message', (msg) => {
|
|
27
|
+
if (!msg || typeof msg !== 'object')
|
|
28
|
+
return;
|
|
29
|
+
if (msg.type === 'ready') {
|
|
30
|
+
if (!this.bootSettled) {
|
|
31
|
+
this.bootSettled = true;
|
|
32
|
+
this.bootResolve?.();
|
|
33
|
+
}
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
if (msg.type === 'error') {
|
|
37
|
+
const e = new Error(msg.message ?? 'knowledge_writer_boot_failed');
|
|
38
|
+
if (!this.bootSettled) {
|
|
39
|
+
this.bootSettled = true;
|
|
40
|
+
this.bootReject?.(e);
|
|
41
|
+
}
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
if (msg.type === 'progress' && msg.evt) {
|
|
45
|
+
for (const cb of this.progressCbs) {
|
|
46
|
+
try {
|
|
47
|
+
cb(msg.evt);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
/* SSE 订阅方异常不得打断 */
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
if (msg.id == null)
|
|
56
|
+
return;
|
|
57
|
+
const slot = this.pending.get(msg.id);
|
|
58
|
+
if (!slot)
|
|
59
|
+
return;
|
|
60
|
+
this.pending.delete(msg.id);
|
|
61
|
+
clearTimeout(slot.timer);
|
|
62
|
+
if (msg.ok)
|
|
63
|
+
slot.resolve(msg.data);
|
|
64
|
+
else {
|
|
65
|
+
const err = new Error(msg.error ?? 'knowledge_write_failed');
|
|
66
|
+
if (msg.code)
|
|
67
|
+
err.code = msg.code;
|
|
68
|
+
slot.reject(err);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
worker.on('error', (err) => {
|
|
72
|
+
const e = err instanceof Error ? err : new Error(String(err));
|
|
73
|
+
if (!this.bootSettled) {
|
|
74
|
+
this.bootSettled = true;
|
|
75
|
+
this.bootReject?.(e);
|
|
76
|
+
}
|
|
77
|
+
this.failAll(e);
|
|
78
|
+
});
|
|
79
|
+
worker.on('exit', () => {
|
|
80
|
+
const e = new Error('knowledge_writer_worker_exit');
|
|
81
|
+
if (!this.bootSettled) {
|
|
82
|
+
this.bootSettled = true;
|
|
83
|
+
this.bootReject?.(e);
|
|
84
|
+
}
|
|
85
|
+
this.failAll(e);
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
static async start(opts) {
|
|
89
|
+
const worker = new Worker(resolveWorkerUrl('./writer-worker.js', import.meta.url), {
|
|
90
|
+
workerData: {
|
|
91
|
+
dbPath: opts.dbPath,
|
|
92
|
+
documentConfig: opts.documentConfig ?? null,
|
|
93
|
+
sqliteVecExtensionPath: opts.sqliteVecExtensionPath,
|
|
94
|
+
embeddingModels: opts.embeddingModels ?? null,
|
|
95
|
+
embed: opts.embed ?? null,
|
|
96
|
+
testEmbeddingStub: opts.testEmbeddingStub ?? false,
|
|
97
|
+
},
|
|
98
|
+
env: process.env,
|
|
99
|
+
});
|
|
100
|
+
const client = new WorkerWriteService(worker, opts.timeoutMs ?? 60_000);
|
|
101
|
+
const bootTimer = setTimeout(() => {
|
|
102
|
+
if (!client.bootSettled) {
|
|
103
|
+
client.bootSettled = true;
|
|
104
|
+
client.bootReject?.(new Error('knowledge_writer_boot_timeout'));
|
|
105
|
+
}
|
|
106
|
+
}, 60_000);
|
|
107
|
+
try {
|
|
108
|
+
await client.bootPromise;
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
clearTimeout(bootTimer);
|
|
112
|
+
}
|
|
113
|
+
return client;
|
|
114
|
+
}
|
|
115
|
+
failAll(err) {
|
|
116
|
+
for (const [, slot] of this.pending) {
|
|
117
|
+
clearTimeout(slot.timer);
|
|
118
|
+
slot.reject(err);
|
|
119
|
+
}
|
|
120
|
+
this.pending.clear();
|
|
121
|
+
}
|
|
122
|
+
call(method, args, timeoutMs) {
|
|
123
|
+
if (this.disposed) {
|
|
124
|
+
return Promise.reject(new Error('knowledge_write_disposed'));
|
|
125
|
+
}
|
|
126
|
+
const id = this.nextId++;
|
|
127
|
+
const ms = timeoutMs ?? this.timeoutMs;
|
|
128
|
+
return new Promise((resolve, reject) => {
|
|
129
|
+
const timer = setTimeout(() => {
|
|
130
|
+
this.pending.delete(id);
|
|
131
|
+
reject(new Error(`knowledge_write_timeout after ${ms}ms`));
|
|
132
|
+
}, ms);
|
|
133
|
+
this.pending.set(id, { resolve, reject, timer });
|
|
134
|
+
this.worker.postMessage({ id, method, args });
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
static err(code, message) {
|
|
138
|
+
return Object.assign(new Error(message), { code });
|
|
139
|
+
}
|
|
140
|
+
rethrow(err) {
|
|
141
|
+
if (err instanceof Error && err.code)
|
|
142
|
+
throw err;
|
|
143
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
144
|
+
if (/not resource owner|not source owner|not project owner/i.test(msg)) {
|
|
145
|
+
throw WorkerWriteService.err('not_resource_owner', msg);
|
|
146
|
+
}
|
|
147
|
+
if (/principal not registered/i.test(msg)) {
|
|
148
|
+
throw WorkerWriteService.err('principal_not_registered', msg);
|
|
149
|
+
}
|
|
150
|
+
if (/not principal owner/i.test(msg)) {
|
|
151
|
+
throw WorkerWriteService.err('not_principal_owner', msg);
|
|
152
|
+
}
|
|
153
|
+
if (/source not found/i.test(msg)) {
|
|
154
|
+
throw WorkerWriteService.err('source_not_found', msg);
|
|
155
|
+
}
|
|
156
|
+
if (/not empty|non-empty|仍有/i.test(msg)) {
|
|
157
|
+
throw WorkerWriteService.err('project_not_empty', msg);
|
|
158
|
+
}
|
|
159
|
+
if (/items\[\] invalid/i.test(msg)) {
|
|
160
|
+
throw WorkerWriteService.err('bad_request', msg);
|
|
161
|
+
}
|
|
162
|
+
throw err instanceof Error ? err : new Error(msg);
|
|
163
|
+
}
|
|
164
|
+
async upsertPrincipal(identity, agentId, body) {
|
|
165
|
+
try {
|
|
166
|
+
await this.call('upsertPrincipal', { identity, agentId, body });
|
|
167
|
+
}
|
|
168
|
+
catch (e) {
|
|
169
|
+
this.rethrow(e);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
async ensurePrincipal(identity, agentId, autoRegister) {
|
|
173
|
+
try {
|
|
174
|
+
await this.call('ensurePrincipal', { identity, agentId, autoRegister });
|
|
175
|
+
}
|
|
176
|
+
catch (e) {
|
|
177
|
+
this.rethrow(e);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
async assertOwnPrincipal(identity, agentId) {
|
|
181
|
+
try {
|
|
182
|
+
await this.call('assertOwnPrincipal', { identity, agentId });
|
|
183
|
+
}
|
|
184
|
+
catch (e) {
|
|
185
|
+
this.rethrow(e);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
async createProject(identity, input) {
|
|
189
|
+
try {
|
|
190
|
+
return (await this.call('createProject', { identity, input }));
|
|
191
|
+
}
|
|
192
|
+
catch (e) {
|
|
193
|
+
this.rethrow(e);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
async removeProject(identity, projectKey) {
|
|
197
|
+
try {
|
|
198
|
+
return (await this.call('removeProject', { identity, projectKey }));
|
|
199
|
+
}
|
|
200
|
+
catch (e) {
|
|
201
|
+
this.rethrow(e);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
async registerSource(identity, input) {
|
|
205
|
+
try {
|
|
206
|
+
return (await this.call('registerSource', { identity, input }, 120_000));
|
|
207
|
+
}
|
|
208
|
+
catch (e) {
|
|
209
|
+
this.rethrow(e);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
async updateSource(identity, sourceId, patch) {
|
|
213
|
+
try {
|
|
214
|
+
return (await this.call('updateSource', { identity, sourceId, patch }));
|
|
215
|
+
}
|
|
216
|
+
catch (e) {
|
|
217
|
+
this.rethrow(e);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
async removeSource(identity, sourceId) {
|
|
221
|
+
try {
|
|
222
|
+
return (await this.call('removeSource', { identity, sourceId }, 120_000));
|
|
223
|
+
}
|
|
224
|
+
catch (e) {
|
|
225
|
+
this.rethrow(e);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
async describeSource(identity, sourceId) {
|
|
229
|
+
try {
|
|
230
|
+
return (await this.call('describeSource', { identity, sourceId }));
|
|
231
|
+
}
|
|
232
|
+
catch (e) {
|
|
233
|
+
this.rethrow(e);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
async reindexSource(sourceId) {
|
|
237
|
+
try {
|
|
238
|
+
await this.call('reindexSource', { sourceId });
|
|
239
|
+
}
|
|
240
|
+
catch (e) {
|
|
241
|
+
this.rethrow(e);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
async reprocessFiles(sourceId, paths) {
|
|
245
|
+
try {
|
|
246
|
+
return (await this.call('reprocessFiles', { sourceId, paths }, 120_000));
|
|
247
|
+
}
|
|
248
|
+
catch (e) {
|
|
249
|
+
this.rethrow(e);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
async reprocessByFilter(sourceId, filter) {
|
|
253
|
+
try {
|
|
254
|
+
return (await this.call('reprocessByFilter', { sourceId, filter }, 120_000));
|
|
255
|
+
}
|
|
256
|
+
catch (e) {
|
|
257
|
+
this.rethrow(e);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
async abortSource(sourceId) {
|
|
261
|
+
try {
|
|
262
|
+
return (await this.call('abortSource', { sourceId }, 15_000));
|
|
263
|
+
}
|
|
264
|
+
catch (e) {
|
|
265
|
+
this.rethrow(e);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
async resumeSource(sourceId) {
|
|
269
|
+
try {
|
|
270
|
+
return (await this.call('resumeSource', { sourceId }, 15_000));
|
|
271
|
+
}
|
|
272
|
+
catch (e) {
|
|
273
|
+
this.rethrow(e);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
async abortAllOwned(identity) {
|
|
277
|
+
try {
|
|
278
|
+
return (await this.call('abortAllOwned', { identity }, 30_000));
|
|
279
|
+
}
|
|
280
|
+
catch (e) {
|
|
281
|
+
this.rethrow(e);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
async resumeAllOwned(identity) {
|
|
285
|
+
try {
|
|
286
|
+
return (await this.call('resumeAllOwned', { identity }, 30_000));
|
|
287
|
+
}
|
|
288
|
+
catch (e) {
|
|
289
|
+
this.rethrow(e);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
async assignProject(identity, agentId, projectKey) {
|
|
293
|
+
try {
|
|
294
|
+
await this.call('assignProject', { identity, agentId, projectKey });
|
|
295
|
+
}
|
|
296
|
+
catch (e) {
|
|
297
|
+
this.rethrow(e);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
async unassignProject(identity, agentId, projectKey) {
|
|
301
|
+
try {
|
|
302
|
+
await this.call('unassignProject', { identity, agentId, projectKey });
|
|
303
|
+
}
|
|
304
|
+
catch (e) {
|
|
305
|
+
this.rethrow(e);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
async hideSource(identity, agentId, sourceId) {
|
|
309
|
+
try {
|
|
310
|
+
await this.call('hideSource', { identity, agentId, sourceId });
|
|
311
|
+
}
|
|
312
|
+
catch (e) {
|
|
313
|
+
this.rethrow(e);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
async unhideSource(identity, agentId, sourceId) {
|
|
317
|
+
try {
|
|
318
|
+
await this.call('unhideSource', { identity, agentId, sourceId });
|
|
319
|
+
}
|
|
320
|
+
catch (e) {
|
|
321
|
+
this.rethrow(e);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
async replaceSessionVisibility(identity, agentId, sessionId, items) {
|
|
325
|
+
try {
|
|
326
|
+
return (await this.call('replaceSessionVisibility', {
|
|
327
|
+
identity,
|
|
328
|
+
agentId,
|
|
329
|
+
sessionId,
|
|
330
|
+
items,
|
|
331
|
+
}));
|
|
332
|
+
}
|
|
333
|
+
catch (e) {
|
|
334
|
+
this.rethrow(e);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
async setSessionVisibilityItem(identity, sessionId, item) {
|
|
338
|
+
try {
|
|
339
|
+
await this.call('setSessionVisibilityItem', { identity, sessionId, item });
|
|
340
|
+
}
|
|
341
|
+
catch (e) {
|
|
342
|
+
this.rethrow(e);
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
async clearSessionVisibility(identity, sessionId, target) {
|
|
346
|
+
try {
|
|
347
|
+
await this.call('clearSessionVisibility', { identity, sessionId, target });
|
|
348
|
+
}
|
|
349
|
+
catch (e) {
|
|
350
|
+
this.rethrow(e);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
async isSourceOwner(sourceId, gatewayId) {
|
|
354
|
+
return (await this.call('isSourceOwner', { sourceId, gatewayId }, 10_000));
|
|
355
|
+
}
|
|
356
|
+
async sourceVisibleToGateway(sourceId, gatewayId) {
|
|
357
|
+
return (await this.call('sourceVisibleToGateway', { sourceId, gatewayId }, 10_000));
|
|
358
|
+
}
|
|
359
|
+
async startIngestRuntime() {
|
|
360
|
+
await this.call('startIngestRuntime', {}, 30_000);
|
|
361
|
+
}
|
|
362
|
+
onProgress(cb) {
|
|
363
|
+
this.progressCbs.add(cb);
|
|
364
|
+
return () => {
|
|
365
|
+
this.progressCbs.delete(cb);
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
async dispose() {
|
|
369
|
+
if (this.disposed)
|
|
370
|
+
return;
|
|
371
|
+
try {
|
|
372
|
+
await this.call('shutdown', {}, 10_000);
|
|
373
|
+
}
|
|
374
|
+
catch {
|
|
375
|
+
/* worker 可能已退 */
|
|
376
|
+
}
|
|
377
|
+
this.disposed = true;
|
|
378
|
+
this.failAll(new Error('knowledge_write_disposed'));
|
|
379
|
+
await this.worker.terminate();
|
|
380
|
+
}
|
|
381
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writer Worker — knowledge.db 唯一写者 + ingest
|
|
3
|
+
*
|
|
4
|
+
* API 线程(engine-thread)只做写路由编排;本线程独占写连接与 KnowledgeIngest。
|
|
5
|
+
* 长同步段必须让出(见 index-store / ingest),否则控制 RPC(abort)会排队。
|
|
6
|
+
*/
|
|
7
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
8
|
+
import { KnowledgeDatabase } from './db.js';
|
|
9
|
+
import { LocalKnowledgeWriteService, } from './writer-service.js';
|
|
10
|
+
function stubEmbeddingProvider() {
|
|
11
|
+
let n = 0;
|
|
12
|
+
return {
|
|
13
|
+
name: 'test-stub',
|
|
14
|
+
dimensions: 4,
|
|
15
|
+
async embed() {
|
|
16
|
+
n += 1;
|
|
17
|
+
return [n, 0.1, 0.2, 0.3];
|
|
18
|
+
},
|
|
19
|
+
async embedBatch(texts) {
|
|
20
|
+
return texts.map((_, i) => [i + n, 0.1, 0.2, 0.3]);
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
async function main() {
|
|
25
|
+
const boot = workerData;
|
|
26
|
+
const port = parentPort;
|
|
27
|
+
if (!port)
|
|
28
|
+
throw new Error('knowledge writer worker requires parentPort');
|
|
29
|
+
const db = await KnowledgeDatabase.create({
|
|
30
|
+
dbPath: boot.dbPath,
|
|
31
|
+
sqliteVec: boot.sqliteVecExtensionPath
|
|
32
|
+
? { extensionPath: boot.sqliteVecExtensionPath }
|
|
33
|
+
: true,
|
|
34
|
+
});
|
|
35
|
+
let embeddingProvider = null;
|
|
36
|
+
if (boot.testEmbeddingStub) {
|
|
37
|
+
embeddingProvider = stubEmbeddingProvider();
|
|
38
|
+
}
|
|
39
|
+
else if (boot.embeddingModels && boot.embed?.enabled !== false) {
|
|
40
|
+
const { resolveEmbeddingRuntime } = await import('../memory/sqlite/embedding-from-models.js');
|
|
41
|
+
const runtime = resolveEmbeddingRuntime(boot.embeddingModels);
|
|
42
|
+
embeddingProvider = runtime?.provider ?? null;
|
|
43
|
+
}
|
|
44
|
+
const write = new LocalKnowledgeWriteService({
|
|
45
|
+
db,
|
|
46
|
+
documentConfig: boot.documentConfig ?? null,
|
|
47
|
+
embeddingProvider: embeddingProvider,
|
|
48
|
+
embed: boot.embed ?? null,
|
|
49
|
+
});
|
|
50
|
+
write.onProgress((evt) => {
|
|
51
|
+
port.postMessage({ type: 'progress', evt });
|
|
52
|
+
});
|
|
53
|
+
port.on('message', (msg) => {
|
|
54
|
+
void (async () => {
|
|
55
|
+
const id = msg?.id;
|
|
56
|
+
if (id == null)
|
|
57
|
+
return;
|
|
58
|
+
try {
|
|
59
|
+
if (msg.method === 'shutdown') {
|
|
60
|
+
try {
|
|
61
|
+
await write.dispose();
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
/* dispose 幂等 */
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
db.close();
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
/* already closed */
|
|
71
|
+
}
|
|
72
|
+
port.postMessage({ id, ok: true, data: null });
|
|
73
|
+
port.close();
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
const args = msg.args ?? {};
|
|
77
|
+
const identity = args.identity;
|
|
78
|
+
let data;
|
|
79
|
+
switch (msg.method) {
|
|
80
|
+
case 'upsertPrincipal':
|
|
81
|
+
await write.upsertPrincipal(identity, String(args.agentId), args.body ?? {});
|
|
82
|
+
data = null;
|
|
83
|
+
break;
|
|
84
|
+
case 'ensurePrincipal':
|
|
85
|
+
await write.ensurePrincipal(identity, String(args.agentId), Boolean(args.autoRegister));
|
|
86
|
+
data = null;
|
|
87
|
+
break;
|
|
88
|
+
case 'assertOwnPrincipal':
|
|
89
|
+
await write.assertOwnPrincipal(identity, String(args.agentId));
|
|
90
|
+
data = null;
|
|
91
|
+
break;
|
|
92
|
+
case 'createProject':
|
|
93
|
+
data = await write.createProject(identity, args.input);
|
|
94
|
+
break;
|
|
95
|
+
case 'removeProject':
|
|
96
|
+
data = await write.removeProject(identity, String(args.projectKey));
|
|
97
|
+
break;
|
|
98
|
+
case 'registerSource':
|
|
99
|
+
data = await write.registerSource(identity, args.input);
|
|
100
|
+
break;
|
|
101
|
+
case 'updateSource':
|
|
102
|
+
data = await write.updateSource(identity, String(args.sourceId), args.patch ?? {});
|
|
103
|
+
break;
|
|
104
|
+
case 'removeSource':
|
|
105
|
+
data = await write.removeSource(identity, String(args.sourceId));
|
|
106
|
+
break;
|
|
107
|
+
case 'describeSource':
|
|
108
|
+
data = await write.describeSource(identity, String(args.sourceId));
|
|
109
|
+
break;
|
|
110
|
+
case 'reindexSource':
|
|
111
|
+
await write.reindexSource(String(args.sourceId));
|
|
112
|
+
data = null;
|
|
113
|
+
break;
|
|
114
|
+
case 'reprocessFiles':
|
|
115
|
+
data = await write.reprocessFiles(String(args.sourceId), args.paths ?? []);
|
|
116
|
+
break;
|
|
117
|
+
case 'reprocessByFilter':
|
|
118
|
+
data = await write.reprocessByFilter(String(args.sourceId), args.filter);
|
|
119
|
+
break;
|
|
120
|
+
case 'abortSource':
|
|
121
|
+
data = await write.abortSource(String(args.sourceId));
|
|
122
|
+
break;
|
|
123
|
+
case 'resumeSource':
|
|
124
|
+
data = await write.resumeSource(String(args.sourceId));
|
|
125
|
+
break;
|
|
126
|
+
case 'abortAllOwned':
|
|
127
|
+
data = await write.abortAllOwned(identity);
|
|
128
|
+
break;
|
|
129
|
+
case 'resumeAllOwned':
|
|
130
|
+
data = await write.resumeAllOwned(identity);
|
|
131
|
+
break;
|
|
132
|
+
case 'assignProject':
|
|
133
|
+
await write.assignProject(identity, String(args.agentId), String(args.projectKey));
|
|
134
|
+
data = null;
|
|
135
|
+
break;
|
|
136
|
+
case 'unassignProject':
|
|
137
|
+
await write.unassignProject(identity, String(args.agentId), String(args.projectKey));
|
|
138
|
+
data = null;
|
|
139
|
+
break;
|
|
140
|
+
case 'hideSource':
|
|
141
|
+
await write.hideSource(identity, String(args.agentId), String(args.sourceId));
|
|
142
|
+
data = null;
|
|
143
|
+
break;
|
|
144
|
+
case 'unhideSource':
|
|
145
|
+
await write.unhideSource(identity, String(args.agentId), String(args.sourceId));
|
|
146
|
+
data = null;
|
|
147
|
+
break;
|
|
148
|
+
case 'replaceSessionVisibility':
|
|
149
|
+
data = await write.replaceSessionVisibility(identity, String(args.agentId), String(args.sessionId), args.items ?? []);
|
|
150
|
+
break;
|
|
151
|
+
case 'setSessionVisibilityItem':
|
|
152
|
+
await write.setSessionVisibilityItem(identity, String(args.sessionId), args.item);
|
|
153
|
+
data = null;
|
|
154
|
+
break;
|
|
155
|
+
case 'clearSessionVisibility':
|
|
156
|
+
await write.clearSessionVisibility(identity, String(args.sessionId), args.target);
|
|
157
|
+
data = null;
|
|
158
|
+
break;
|
|
159
|
+
case 'isSourceOwner':
|
|
160
|
+
data = await write.isSourceOwner(String(args.sourceId), String(args.gatewayId));
|
|
161
|
+
break;
|
|
162
|
+
case 'sourceVisibleToGateway':
|
|
163
|
+
data = await write.sourceVisibleToGateway(String(args.sourceId), String(args.gatewayId));
|
|
164
|
+
break;
|
|
165
|
+
case 'startIngestRuntime':
|
|
166
|
+
await write.startIngestRuntime();
|
|
167
|
+
data = null;
|
|
168
|
+
break;
|
|
169
|
+
default:
|
|
170
|
+
throw new Error(`unknown_write_method:${msg.method}`);
|
|
171
|
+
}
|
|
172
|
+
port.postMessage({ id, ok: true, data });
|
|
173
|
+
}
|
|
174
|
+
catch (err) {
|
|
175
|
+
port.postMessage({
|
|
176
|
+
id,
|
|
177
|
+
ok: false,
|
|
178
|
+
error: err instanceof Error ? err.message : String(err),
|
|
179
|
+
code: err.code,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
})();
|
|
183
|
+
});
|
|
184
|
+
port.postMessage({ type: 'ready' });
|
|
185
|
+
}
|
|
186
|
+
main().catch((err) => {
|
|
187
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
188
|
+
parentPort?.postMessage({ type: 'error', message });
|
|
189
|
+
});
|
|
@@ -17,9 +17,12 @@ export interface EmbeddingProvider {
|
|
|
17
17
|
readonly name: string;
|
|
18
18
|
/** 向量维度 */
|
|
19
19
|
readonly dimensions: number;
|
|
20
|
-
/** 生成 embedding */
|
|
20
|
+
/** 生成 embedding;输入超模型上下文时由实现做截断回退 */
|
|
21
21
|
embed(text: string): Promise<number[]>;
|
|
22
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* 批量生成 embedding。
|
|
24
|
+
* 与入参等长;失败条目为空数组,由调用方按条续试(禁止静默丢条)。
|
|
25
|
+
*/
|
|
23
26
|
embedBatch(texts: string[]): Promise<number[][]>;
|
|
24
27
|
}
|
|
25
28
|
/** HTTP 请求/响应字段映射(type=http 或覆盖 openai/ollama 默认) */
|