@octopi-agent/engine 0.58.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +38 -1
- package/dist/config-schema/substrate.d.ts +37 -0
- package/dist/config-schema/substrate.js +47 -0
- package/dist/config.d.ts +30 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/index.d.ts +4 -2
- package/dist/harness/index.js +2 -1
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +80 -17
- package/dist/harness/knowledge/source-store.js +160 -89
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KnowledgeWriteService — 唯一写者端口(API 线程不得直接摸 SQLite)
|
|
3
|
+
*
|
|
4
|
+
* 生产:Writer Worker 持有 knowledge.db + ingest;API 经 RPC 调用。
|
|
5
|
+
* 测试 / :memory::LocalKnowledgeWriteService 同进程实现(同一接口)。
|
|
6
|
+
*/
|
|
7
|
+
import { KnowledgeSourceStore } from './source-store.js';
|
|
8
|
+
import { KnowledgeIndexStore } from './index-store.js';
|
|
9
|
+
import { MembershipStore } from './membership-store.js';
|
|
10
|
+
import { KnowledgeIngest } from './ingest.js';
|
|
11
|
+
import { generateKnowledgeDescription } from './describe.js';
|
|
12
|
+
/** 进程内唯一写者(测试 / :memory:)。 */
|
|
13
|
+
export class LocalKnowledgeWriteService {
|
|
14
|
+
deps;
|
|
15
|
+
sources;
|
|
16
|
+
index;
|
|
17
|
+
memberships;
|
|
18
|
+
ingest;
|
|
19
|
+
progressCbs = new Set();
|
|
20
|
+
constructor(deps) {
|
|
21
|
+
this.deps = deps;
|
|
22
|
+
this.sources = new KnowledgeSourceStore(deps.db);
|
|
23
|
+
this.index = new KnowledgeIndexStore(deps.db);
|
|
24
|
+
this.memberships = new MembershipStore(deps.db);
|
|
25
|
+
const embed = deps.embed ?? null;
|
|
26
|
+
this.ingest = new KnowledgeIngest({
|
|
27
|
+
sourceStore: this.sources,
|
|
28
|
+
indexStore: this.index,
|
|
29
|
+
documentConfig: deps.documentConfig ?? null,
|
|
30
|
+
embeddingProvider: embed?.enabled === false ? null : (deps.embeddingProvider ?? null),
|
|
31
|
+
...(embed?.embedBatch != null ? { embedBatch: embed.embedBatch } : {}),
|
|
32
|
+
...(embed?.embedMinIntervalMs != null ? { embedMinIntervalMs: embed.embedMinIntervalMs } : {}),
|
|
33
|
+
...(embed?.embedConcurrency != null ? { embedConcurrency: embed.embedConcurrency } : {}),
|
|
34
|
+
...(embed?.embedSecretPolicy != null ? { embedSecretPolicy: embed.embedSecretPolicy } : {}),
|
|
35
|
+
});
|
|
36
|
+
this.ingest.on('knowledge.index.progress', (evt) => {
|
|
37
|
+
for (const cb of this.progressCbs) {
|
|
38
|
+
try {
|
|
39
|
+
cb(evt);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
/* SSE 订阅方异常不得打断 ingest */
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
get db() {
|
|
48
|
+
return this.deps.db;
|
|
49
|
+
}
|
|
50
|
+
async upsertPrincipal(identity, agentId, body) {
|
|
51
|
+
const now = Date.now();
|
|
52
|
+
this.db.raw
|
|
53
|
+
.prepare(`INSERT INTO knowledge_principals
|
|
54
|
+
(tenant_id, gateway_id, local_agent_id, display_name, status, last_seen_at, created_at, updated_at)
|
|
55
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
56
|
+
ON CONFLICT(tenant_id, gateway_id, local_agent_id) DO UPDATE SET
|
|
57
|
+
display_name = COALESCE(excluded.display_name, knowledge_principals.display_name),
|
|
58
|
+
status = excluded.status,
|
|
59
|
+
last_seen_at = excluded.last_seen_at,
|
|
60
|
+
updated_at = excluded.updated_at`)
|
|
61
|
+
.run(identity.tenantId, identity.gatewayId, agentId, body.displayName ?? null, body.status ?? 'active', now, now, now);
|
|
62
|
+
}
|
|
63
|
+
async ensurePrincipal(identity, agentId, autoRegister) {
|
|
64
|
+
const row = this.db.raw
|
|
65
|
+
.prepare(`SELECT status FROM knowledge_principals
|
|
66
|
+
WHERE tenant_id = ? AND gateway_id = ? AND local_agent_id = ?`)
|
|
67
|
+
.get(identity.tenantId, identity.gatewayId, agentId);
|
|
68
|
+
if (row)
|
|
69
|
+
return;
|
|
70
|
+
if (autoRegister) {
|
|
71
|
+
await this.upsertPrincipal(identity, agentId, {});
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
throw Object.assign(new Error('principal not registered'), {
|
|
75
|
+
code: 'principal_not_registered',
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
async assertOwnPrincipal(identity, agentId) {
|
|
79
|
+
// 不得按 gateway 预过滤后再比较——那是永不失败的空检查。
|
|
80
|
+
// 同名 principal 可挂多 gateway(复合 PK);仅当**另一 gateway 已登记**时拒绝。
|
|
81
|
+
const rows = this.db.raw
|
|
82
|
+
.prepare(`SELECT gateway_id FROM knowledge_principals
|
|
83
|
+
WHERE tenant_id = ? AND local_agent_id = ?`)
|
|
84
|
+
.all(identity.tenantId, agentId);
|
|
85
|
+
const foreign = rows.find((r) => r.gateway_id !== identity.gatewayId);
|
|
86
|
+
if (foreign) {
|
|
87
|
+
throw Object.assign(new Error('not principal owner'), { code: 'not_principal_owner' });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
async createProject(identity, input) {
|
|
91
|
+
const existing = this.db.raw
|
|
92
|
+
.prepare(`SELECT registered_by FROM knowledge_projects
|
|
93
|
+
WHERE tenant_id = ? AND project_key = ?`)
|
|
94
|
+
.get(identity.tenantId, input.projectKey);
|
|
95
|
+
if (existing?.registered_by && existing.registered_by !== identity.gatewayId) {
|
|
96
|
+
throw Object.assign(new Error('project owned by another gateway'), {
|
|
97
|
+
code: 'not_resource_owner',
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
this.sources.createProject(input.projectKey, input.displayName, {
|
|
101
|
+
tenantId: identity.tenantId,
|
|
102
|
+
registeredBy: identity.gatewayId,
|
|
103
|
+
});
|
|
104
|
+
const visibility = input.visibility === 'public' ? 'public' : 'private';
|
|
105
|
+
this.db.raw
|
|
106
|
+
.prepare(`UPDATE knowledge_projects SET registered_by = ?, visibility = ?
|
|
107
|
+
WHERE tenant_id = ? AND project_key = ?`)
|
|
108
|
+
.run(identity.gatewayId, visibility, identity.tenantId, input.projectKey);
|
|
109
|
+
return {
|
|
110
|
+
projectKey: input.projectKey,
|
|
111
|
+
displayName: input.displayName ?? null,
|
|
112
|
+
registeredBy: identity.gatewayId,
|
|
113
|
+
visibility,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
async removeProject(identity, projectKey) {
|
|
117
|
+
const row = this.db.raw
|
|
118
|
+
.prepare(`SELECT registered_by FROM knowledge_projects
|
|
119
|
+
WHERE tenant_id = ? AND project_key = ?`)
|
|
120
|
+
.get(identity.tenantId, projectKey);
|
|
121
|
+
if (row?.registered_by && row.registered_by !== identity.gatewayId) {
|
|
122
|
+
throw Object.assign(new Error('not project owner'), { code: 'not_resource_owner' });
|
|
123
|
+
}
|
|
124
|
+
try {
|
|
125
|
+
const removed = this.sources.removeProject(projectKey, { tenantId: identity.tenantId });
|
|
126
|
+
return { projectKey, removed };
|
|
127
|
+
}
|
|
128
|
+
catch (e) {
|
|
129
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
130
|
+
if (/not empty|non-empty|仍有/i.test(msg)) {
|
|
131
|
+
throw Object.assign(new Error(msg), { code: 'project_not_empty' });
|
|
132
|
+
}
|
|
133
|
+
throw e;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
async registerSource(identity, rawInput) {
|
|
137
|
+
const input = { ...rawInput };
|
|
138
|
+
delete input.id;
|
|
139
|
+
delete input.registeredBy;
|
|
140
|
+
delete input.registered_by;
|
|
141
|
+
delete input.tenantId;
|
|
142
|
+
delete input.tenant_id;
|
|
143
|
+
const src = this.sources.register(input);
|
|
144
|
+
this.db.raw
|
|
145
|
+
.prepare(`UPDATE knowledge_sources SET registered_by = ?, visibility = ?, tenant_id = ?
|
|
146
|
+
WHERE id = ?`)
|
|
147
|
+
.run(identity.gatewayId, typeof input.visibility === 'string' ? input.visibility : 'private', identity.tenantId, src.id);
|
|
148
|
+
void this.ingest.ingestSource(src.id, { full: true }).catch((e) => {
|
|
149
|
+
console.warn(`[Knowledge] auto-ingest after register failed (${src.id}): ${e instanceof Error ? e.message : String(e)}`);
|
|
150
|
+
});
|
|
151
|
+
return { ...src, registeredBy: identity.gatewayId };
|
|
152
|
+
}
|
|
153
|
+
async updateSource(identity, sourceId, patch) {
|
|
154
|
+
const row = this.db.raw
|
|
155
|
+
.prepare('SELECT registered_by FROM knowledge_sources WHERE id = ?')
|
|
156
|
+
.get(sourceId);
|
|
157
|
+
if (row?.registered_by && row.registered_by !== identity.gatewayId) {
|
|
158
|
+
throw Object.assign(new Error('not source owner'), { code: 'not_resource_owner' });
|
|
159
|
+
}
|
|
160
|
+
const updated = this.sources.update(sourceId, patch);
|
|
161
|
+
if (!updated)
|
|
162
|
+
return null;
|
|
163
|
+
if (typeof patch.visibility === 'string') {
|
|
164
|
+
this.db.raw
|
|
165
|
+
.prepare('UPDATE knowledge_sources SET visibility = ? WHERE id = ?')
|
|
166
|
+
.run(patch.visibility, sourceId);
|
|
167
|
+
}
|
|
168
|
+
return updated;
|
|
169
|
+
}
|
|
170
|
+
async removeSource(identity, sourceId) {
|
|
171
|
+
const src = this.sources.get(sourceId);
|
|
172
|
+
if (!src) {
|
|
173
|
+
throw Object.assign(new Error('source not found'), { code: 'source_not_found' });
|
|
174
|
+
}
|
|
175
|
+
const row = this.db.raw
|
|
176
|
+
.prepare('SELECT registered_by FROM knowledge_sources WHERE id = ?')
|
|
177
|
+
.get(sourceId);
|
|
178
|
+
if (row?.registered_by && row.registered_by !== identity.gatewayId) {
|
|
179
|
+
throw Object.assign(new Error('not source owner'), { code: 'not_resource_owner' });
|
|
180
|
+
}
|
|
181
|
+
// 先停在途 parse/embed:否则 purge 让出窗口内 upsert 会把 File/Chunk 再写回来
|
|
182
|
+
this.ingest.abortJobs({ sourceId });
|
|
183
|
+
const purged = await this.memberships.unclaimAllForSourceAsync(sourceId, (fileId) => this.index.purgeFileAsync(fileId));
|
|
184
|
+
this.sources.remove(sourceId);
|
|
185
|
+
return { id: sourceId, purgedFiles: purged.length };
|
|
186
|
+
}
|
|
187
|
+
async describeSource(identity, sourceId) {
|
|
188
|
+
await this.assertSourceOwner(identity, sourceId);
|
|
189
|
+
const src = this.sources.get(sourceId);
|
|
190
|
+
if (!src)
|
|
191
|
+
return null;
|
|
192
|
+
const paths = this.db.raw
|
|
193
|
+
.prepare('SELECT logical_path FROM knowledge_memberships WHERE source_id = ? ORDER BY logical_path LIMIT 20')
|
|
194
|
+
.all(sourceId);
|
|
195
|
+
const sample = paths.map((p) => p.logical_path).join('\n') || src.location;
|
|
196
|
+
const r = await generateKnowledgeDescription(src, sample, { enabled: true });
|
|
197
|
+
this.sources.update(sourceId, { generatedDescription: r.description });
|
|
198
|
+
return { generatedDescription: r.description, source: r.source };
|
|
199
|
+
}
|
|
200
|
+
async assertSourceOwner(identity, sourceId) {
|
|
201
|
+
const row = this.db.raw
|
|
202
|
+
.prepare('SELECT registered_by FROM knowledge_sources WHERE id = ?')
|
|
203
|
+
.get(sourceId);
|
|
204
|
+
if (!row) {
|
|
205
|
+
throw Object.assign(new Error('source not found'), { code: 'source_not_found' });
|
|
206
|
+
}
|
|
207
|
+
if (row.registered_by && row.registered_by !== identity.gatewayId) {
|
|
208
|
+
throw Object.assign(new Error('not source owner'), { code: 'not_resource_owner' });
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
async reindexSource(sourceId) {
|
|
212
|
+
void this.ingest.ingestSource(sourceId, { full: true }).catch((e) => {
|
|
213
|
+
console.warn(`[Knowledge] reindex kick failed (${sourceId}): ${e instanceof Error ? e.message : String(e)}`);
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
async reprocessFiles(sourceId, paths) {
|
|
217
|
+
return this.ingest.reprocessFiles(sourceId, paths);
|
|
218
|
+
}
|
|
219
|
+
async reprocessByFilter(sourceId, filter) {
|
|
220
|
+
return this.ingest.reprocessByFilter(sourceId, filter);
|
|
221
|
+
}
|
|
222
|
+
async abortSource(sourceId) {
|
|
223
|
+
const stats = this.ingest.abortJobs({ sourceId });
|
|
224
|
+
return { ...stats, ...this.ingest.jobControlState(sourceId) };
|
|
225
|
+
}
|
|
226
|
+
async resumeSource(sourceId) {
|
|
227
|
+
this.ingest.resumeJobs({ sourceId });
|
|
228
|
+
return this.ingest.jobControlState(sourceId);
|
|
229
|
+
}
|
|
230
|
+
async abortAllOwned(identity) {
|
|
231
|
+
const owned = this.sources.list().filter((s) => this.isOwnerRow(s.id, identity.gatewayId));
|
|
232
|
+
let cancelledQueued = 0;
|
|
233
|
+
let abortedRunning = 0;
|
|
234
|
+
let runningJobs = 0;
|
|
235
|
+
for (const s of owned) {
|
|
236
|
+
const st = this.ingest.abortJobs({ sourceId: s.id });
|
|
237
|
+
cancelledQueued += st.cancelledQueued;
|
|
238
|
+
abortedRunning += st.abortedRunning;
|
|
239
|
+
runningJobs += st.runningJobs;
|
|
240
|
+
}
|
|
241
|
+
return { cancelledQueued, abortedRunning, runningJobs };
|
|
242
|
+
}
|
|
243
|
+
async resumeAllOwned(identity) {
|
|
244
|
+
const owned = this.sources.list().filter((s) => this.isOwnerRow(s.id, identity.gatewayId));
|
|
245
|
+
let restoredCancelled = 0;
|
|
246
|
+
let embedQueued = 0;
|
|
247
|
+
for (const s of owned) {
|
|
248
|
+
const st = this.ingest.resumeJobs({ sourceId: s.id });
|
|
249
|
+
restoredCancelled += st.restoredCancelled;
|
|
250
|
+
embedQueued += st.embedQueued;
|
|
251
|
+
}
|
|
252
|
+
return { restoredCancelled, embedQueued };
|
|
253
|
+
}
|
|
254
|
+
isOwnerRow(sourceId, gatewayId) {
|
|
255
|
+
const row = this.db.raw
|
|
256
|
+
.prepare('SELECT registered_by FROM knowledge_sources WHERE id = ?')
|
|
257
|
+
.get(sourceId);
|
|
258
|
+
if (!row)
|
|
259
|
+
return false;
|
|
260
|
+
if (row.registered_by && row.registered_by !== gatewayId)
|
|
261
|
+
return false;
|
|
262
|
+
return true;
|
|
263
|
+
}
|
|
264
|
+
async assignProject(identity, agentId, projectKey) {
|
|
265
|
+
this.sources.assignProject(projectKey, agentId, identity);
|
|
266
|
+
}
|
|
267
|
+
async unassignProject(identity, agentId, projectKey) {
|
|
268
|
+
this.sources.unassignProject(projectKey, agentId, identity);
|
|
269
|
+
}
|
|
270
|
+
async hideSource(identity, agentId, sourceId) {
|
|
271
|
+
this.sources.hideSource(agentId, sourceId, identity);
|
|
272
|
+
}
|
|
273
|
+
async unhideSource(identity, agentId, sourceId) {
|
|
274
|
+
this.sources.unhideSource(agentId, sourceId, identity);
|
|
275
|
+
}
|
|
276
|
+
async replaceSessionVisibility(identity, _agentId, sessionId, items) {
|
|
277
|
+
const typed = items.map((raw) => {
|
|
278
|
+
const targetType = raw.targetType;
|
|
279
|
+
const targetId = typeof raw.targetId === 'string' ? raw.targetId : '';
|
|
280
|
+
const op = raw.op;
|
|
281
|
+
if (!targetId ||
|
|
282
|
+
(targetType !== 'project' && targetType !== 'source') ||
|
|
283
|
+
(op !== 'include' && op !== 'exclude')) {
|
|
284
|
+
throw Object.assign(new Error('items[] invalid'), { code: 'bad_request' });
|
|
285
|
+
}
|
|
286
|
+
return { targetType, targetId, op };
|
|
287
|
+
});
|
|
288
|
+
this.sources.replaceSessionVisibility(sessionId, typed, identity);
|
|
289
|
+
return typed.length;
|
|
290
|
+
}
|
|
291
|
+
async setSessionVisibilityItem(identity, sessionId, item) {
|
|
292
|
+
this.sources.setSessionVisibility(sessionId, item, identity);
|
|
293
|
+
}
|
|
294
|
+
async clearSessionVisibility(identity, sessionId, target) {
|
|
295
|
+
this.sources.clearSessionVisibility(sessionId, target, identity);
|
|
296
|
+
}
|
|
297
|
+
async isSourceOwner(sourceId, gatewayId) {
|
|
298
|
+
return this.isOwnerRow(sourceId, gatewayId);
|
|
299
|
+
}
|
|
300
|
+
async sourceVisibleToGateway(sourceId, gatewayId) {
|
|
301
|
+
const row = this.db.raw
|
|
302
|
+
.prepare('SELECT registered_by, visibility FROM knowledge_sources WHERE id = ?')
|
|
303
|
+
.get(sourceId);
|
|
304
|
+
if (!row)
|
|
305
|
+
return true;
|
|
306
|
+
if (row.registered_by === gatewayId)
|
|
307
|
+
return true;
|
|
308
|
+
return row.visibility === 'public';
|
|
309
|
+
}
|
|
310
|
+
async startIngestRuntime() {
|
|
311
|
+
const { sources, ingest } = this;
|
|
312
|
+
ingest.onSourceSettled = (sourceId) => {
|
|
313
|
+
void (async () => {
|
|
314
|
+
try {
|
|
315
|
+
const src = sources.get(sourceId);
|
|
316
|
+
if (!src || src.description?.trim())
|
|
317
|
+
return;
|
|
318
|
+
const paths = this.db.raw
|
|
319
|
+
.prepare('SELECT logical_path FROM knowledge_memberships WHERE source_id = ? ORDER BY logical_path LIMIT 20')
|
|
320
|
+
.all(sourceId);
|
|
321
|
+
const sample = paths.map((p) => p.logical_path).join('\n') || src.location;
|
|
322
|
+
const r = await generateKnowledgeDescription(src, sample, { enabled: true });
|
|
323
|
+
sources.update(sourceId, { generatedDescription: r.description });
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
/* describe 失败不影响索引 */
|
|
327
|
+
}
|
|
328
|
+
})();
|
|
329
|
+
};
|
|
330
|
+
for (const s of sources.list()) {
|
|
331
|
+
if (s.status === 'removed' || s.status === 'disabled')
|
|
332
|
+
continue;
|
|
333
|
+
if (s.sync.enabled !== false && s.sync.strategy === 'watch') {
|
|
334
|
+
ingest.startWatch(s.id);
|
|
335
|
+
}
|
|
336
|
+
if (s.status === 'pending') {
|
|
337
|
+
void ingest.ingestSource(s.id, { full: true }).catch((e) => {
|
|
338
|
+
console.warn(`[Knowledge] recover pending failed (${s.id}): ${e instanceof Error ? e.message : String(e)}`);
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
ingest.startReconciler();
|
|
343
|
+
ingest.startPolling();
|
|
344
|
+
}
|
|
345
|
+
onProgress(cb) {
|
|
346
|
+
this.progressCbs.add(cb);
|
|
347
|
+
return () => {
|
|
348
|
+
this.progressCbs.delete(cb);
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
async dispose() {
|
|
352
|
+
this.progressCbs.clear();
|
|
353
|
+
this.ingest.dispose();
|
|
354
|
+
}
|
|
355
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import type { KnowledgeWriteService, WriteIdentity, WriteAbortStats, WriteResumeStats, WriteRegisterResult, WriteDescribeResult, WriteReprocessResult } from './writer-service.js';
|
|
2
|
+
import type { KnowledgeSource } from './types.js';
|
|
3
|
+
import type { KnowledgeJobControlState } from './job-control-state.js';
|
|
4
|
+
import type { IngestProgressEvent } from './ingest.js';
|
|
5
|
+
import type { DocumentCapabilityConfig } from '../capabilities/document/factory.js';
|
|
6
|
+
export interface WorkerWriteStartOptions {
|
|
7
|
+
dbPath: string;
|
|
8
|
+
documentConfig?: DocumentCapabilityConfig | null;
|
|
9
|
+
sqliteVecExtensionPath?: string;
|
|
10
|
+
embeddingModels?: {
|
|
11
|
+
providers?: Record<string, unknown>;
|
|
12
|
+
embedding?: unknown;
|
|
13
|
+
} | null;
|
|
14
|
+
embed?: {
|
|
15
|
+
enabled?: boolean;
|
|
16
|
+
embedBatch?: number;
|
|
17
|
+
embedMinIntervalMs?: number;
|
|
18
|
+
embedConcurrency?: number;
|
|
19
|
+
embedSecretPolicy?: 'allow' | 'redact' | 'skip';
|
|
20
|
+
} | null;
|
|
21
|
+
testEmbeddingStub?: boolean;
|
|
22
|
+
timeoutMs?: number;
|
|
23
|
+
}
|
|
24
|
+
export declare class WorkerWriteService implements KnowledgeWriteService {
|
|
25
|
+
private readonly worker;
|
|
26
|
+
private nextId;
|
|
27
|
+
private readonly pending;
|
|
28
|
+
private readonly progressCbs;
|
|
29
|
+
private disposed;
|
|
30
|
+
private bootSettled;
|
|
31
|
+
private bootResolve;
|
|
32
|
+
private bootReject;
|
|
33
|
+
private readonly bootPromise;
|
|
34
|
+
private readonly timeoutMs;
|
|
35
|
+
private constructor();
|
|
36
|
+
static start(opts: WorkerWriteStartOptions): Promise<WorkerWriteService>;
|
|
37
|
+
private failAll;
|
|
38
|
+
private call;
|
|
39
|
+
private static err;
|
|
40
|
+
private rethrow;
|
|
41
|
+
upsertPrincipal(identity: WriteIdentity, agentId: string, body: {
|
|
42
|
+
displayName?: string;
|
|
43
|
+
status?: string;
|
|
44
|
+
}): Promise<void>;
|
|
45
|
+
ensurePrincipal(identity: WriteIdentity, agentId: string, autoRegister: boolean): Promise<void>;
|
|
46
|
+
assertOwnPrincipal(identity: WriteIdentity, agentId: string): Promise<void>;
|
|
47
|
+
createProject(identity: WriteIdentity, input: {
|
|
48
|
+
projectKey: string;
|
|
49
|
+
displayName?: string;
|
|
50
|
+
visibility?: string;
|
|
51
|
+
}): Promise<{
|
|
52
|
+
projectKey: string;
|
|
53
|
+
displayName: string | null;
|
|
54
|
+
registeredBy: string;
|
|
55
|
+
visibility: 'public' | 'private';
|
|
56
|
+
}>;
|
|
57
|
+
removeProject(identity: WriteIdentity, projectKey: string): Promise<{
|
|
58
|
+
projectKey: string;
|
|
59
|
+
removed: boolean;
|
|
60
|
+
}>;
|
|
61
|
+
registerSource(identity: WriteIdentity, input: Record<string, unknown>): Promise<WriteRegisterResult>;
|
|
62
|
+
updateSource(identity: WriteIdentity, sourceId: string, patch: Record<string, unknown>): Promise<KnowledgeSource | null>;
|
|
63
|
+
removeSource(identity: WriteIdentity, sourceId: string): Promise<{
|
|
64
|
+
id: string;
|
|
65
|
+
purgedFiles: number;
|
|
66
|
+
}>;
|
|
67
|
+
describeSource(identity: WriteIdentity, sourceId: string): Promise<WriteDescribeResult | null>;
|
|
68
|
+
reindexSource(sourceId: string): Promise<void>;
|
|
69
|
+
reprocessFiles(sourceId: string, paths: string[]): Promise<WriteReprocessResult>;
|
|
70
|
+
reprocessByFilter(sourceId: string, filter?: {
|
|
71
|
+
status?: string;
|
|
72
|
+
ext?: string;
|
|
73
|
+
q?: string;
|
|
74
|
+
}): Promise<WriteReprocessResult>;
|
|
75
|
+
abortSource(sourceId: string): Promise<WriteAbortStats & KnowledgeJobControlState>;
|
|
76
|
+
resumeSource(sourceId: string): Promise<KnowledgeJobControlState>;
|
|
77
|
+
abortAllOwned(identity: WriteIdentity): Promise<WriteAbortStats>;
|
|
78
|
+
resumeAllOwned(identity: WriteIdentity): Promise<WriteResumeStats>;
|
|
79
|
+
assignProject(identity: WriteIdentity, agentId: string, projectKey: string): Promise<void>;
|
|
80
|
+
unassignProject(identity: WriteIdentity, agentId: string, projectKey: string): Promise<void>;
|
|
81
|
+
hideSource(identity: WriteIdentity, agentId: string, sourceId: string): Promise<void>;
|
|
82
|
+
unhideSource(identity: WriteIdentity, agentId: string, sourceId: string): Promise<void>;
|
|
83
|
+
replaceSessionVisibility(identity: WriteIdentity, agentId: string, sessionId: string, items: Array<{
|
|
84
|
+
targetType: string;
|
|
85
|
+
targetId: string;
|
|
86
|
+
op: string;
|
|
87
|
+
}>): Promise<number>;
|
|
88
|
+
setSessionVisibilityItem(identity: WriteIdentity, sessionId: string, item: {
|
|
89
|
+
targetType: 'project' | 'source';
|
|
90
|
+
targetId: string;
|
|
91
|
+
op: 'include' | 'exclude';
|
|
92
|
+
}): Promise<void>;
|
|
93
|
+
clearSessionVisibility(identity: WriteIdentity, sessionId: string, target?: {
|
|
94
|
+
targetType: 'project' | 'source';
|
|
95
|
+
targetId: string;
|
|
96
|
+
}): Promise<void>;
|
|
97
|
+
isSourceOwner(sourceId: string, gatewayId: string): Promise<boolean>;
|
|
98
|
+
sourceVisibleToGateway(sourceId: string, gatewayId: string): Promise<boolean>;
|
|
99
|
+
startIngestRuntime(): Promise<void>;
|
|
100
|
+
onProgress(cb: (evt: IngestProgressEvent) => void): () => void;
|
|
101
|
+
dispose(): Promise<void>;
|
|
102
|
+
}
|