@octopi-agent/engine 0.56.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +43 -1
- package/dist/config-schema/substrate.d.ts +42 -0
- package/dist/config-schema/substrate.js +58 -0
- package/dist/config.d.ts +39 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/index.d.ts +5 -3
- package/dist/harness/index.js +3 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +42 -3
- package/dist/harness/knowledge/db.js +466 -83
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +68 -0
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +33 -0
- package/dist/harness/knowledge/extract-document.js +86 -0
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +166 -63
- package/dist/harness/knowledge/index-store.js +1025 -282
- package/dist/harness/knowledge/index.d.ts +47 -5
- package/dist/harness/knowledge/index.js +24 -2
- package/dist/harness/knowledge/ingest.d.ts +322 -49
- package/dist/harness/knowledge/ingest.js +1383 -267
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +30 -5
- package/dist/harness/knowledge/retriever.js +98 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +83 -16
- package/dist/harness/knowledge/source-store.js +205 -78
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
- package/dist/harness/memory/sqlite/keyword-search.js +104 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KnowledgeQueryService — 只读查询面(search / list / stats / catalog…)
|
|
3
|
+
*
|
|
4
|
+
* 生产:Query Worker + 独立只读连接(WAL 一写多读),与 ingest 写路径线程分离。
|
|
5
|
+
* 测试 / :memory::LocalKnowledgeQueryService(同连接,小库可接受)。
|
|
6
|
+
*
|
|
7
|
+
* **禁止**在此接口上实现任何写路径(唯一写者是 Writer Worker / LocalKnowledgeWriteService)。
|
|
8
|
+
*/
|
|
9
|
+
import { readJobControlState, } from './job-control-state.js';
|
|
10
|
+
import { KnowledgeHitLog } from './hit-log.js';
|
|
11
|
+
/**
|
|
12
|
+
* 进程内只读实现(测试 / 共享 :memory:)。
|
|
13
|
+
* 与 Worker 实现同接口;生产 Engine 线程应使用 Worker 以隔离写阻塞。
|
|
14
|
+
*/
|
|
15
|
+
export class LocalKnowledgeQueryService {
|
|
16
|
+
deps;
|
|
17
|
+
constructor(deps) {
|
|
18
|
+
this.deps = deps;
|
|
19
|
+
}
|
|
20
|
+
async search(query) {
|
|
21
|
+
return this.deps.retriever.search(query.q, {
|
|
22
|
+
agentId: query.agentId,
|
|
23
|
+
sessionId: query.sessionId,
|
|
24
|
+
limit: query.limit,
|
|
25
|
+
keywordOnly: query.keywordOnly,
|
|
26
|
+
tenantId: query.identity?.tenantId,
|
|
27
|
+
gatewayId: query.identity?.gatewayId,
|
|
28
|
+
sourceIds: query.sourceIds,
|
|
29
|
+
source: query.source,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
async autoGround(query) {
|
|
33
|
+
return this.deps.retriever.autoGround(query.q, {
|
|
34
|
+
agentId: query.agentId,
|
|
35
|
+
sessionId: query.sessionId,
|
|
36
|
+
limit: query.limit,
|
|
37
|
+
tenantId: query.identity?.tenantId,
|
|
38
|
+
gatewayId: query.identity?.gatewayId,
|
|
39
|
+
...(query.recall ? { recall: query.recall } : {}),
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
async listSources(query) {
|
|
43
|
+
const { sources } = this.deps;
|
|
44
|
+
let list = sources.list();
|
|
45
|
+
if (query.identity) {
|
|
46
|
+
list = list.filter((s) => this.gatewayVisible(s, query.identity));
|
|
47
|
+
}
|
|
48
|
+
if (query.scopeLevel === 'global' || query.scopeLevel === 'project' || query.scopeLevel === 'session') {
|
|
49
|
+
list = list.filter((s) => s.scopeRef.level === query.scopeLevel);
|
|
50
|
+
}
|
|
51
|
+
if (query.projectKey != null && query.projectKey !== '') {
|
|
52
|
+
list = list.filter((s) => s.scopeRef.key === query.projectKey);
|
|
53
|
+
}
|
|
54
|
+
if (query.sessionId != null && query.sessionId !== '') {
|
|
55
|
+
list = list.filter((s) => {
|
|
56
|
+
if (s.scopeRef.level === 'session')
|
|
57
|
+
return s.scopeRef.key === query.sessionId;
|
|
58
|
+
return true;
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return list;
|
|
62
|
+
}
|
|
63
|
+
async listProjects(identity) {
|
|
64
|
+
return this.deps.sources.listProjects({
|
|
65
|
+
tenantId: identity?.tenantId,
|
|
66
|
+
gatewayId: identity?.gatewayId,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
async getSource(sourceId, identity) {
|
|
70
|
+
const src = this.deps.sources.get(sourceId);
|
|
71
|
+
if (!src)
|
|
72
|
+
return null;
|
|
73
|
+
if (identity && !this.gatewayVisible(src, identity))
|
|
74
|
+
return null;
|
|
75
|
+
return { source: src, stats: this.deps.index.sourceStats(sourceId) };
|
|
76
|
+
}
|
|
77
|
+
async listFiles(sourceId, identity) {
|
|
78
|
+
if (identity) {
|
|
79
|
+
const src = this.deps.sources.get(sourceId);
|
|
80
|
+
if (!src || !this.gatewayVisible(src, identity))
|
|
81
|
+
return [];
|
|
82
|
+
}
|
|
83
|
+
return this.deps.index.listFiles(sourceId);
|
|
84
|
+
}
|
|
85
|
+
async listFilesPaged(sourceId, opts, identity) {
|
|
86
|
+
if (identity) {
|
|
87
|
+
const src = this.deps.sources.get(sourceId);
|
|
88
|
+
if (!src || !this.gatewayVisible(src, identity)) {
|
|
89
|
+
return {
|
|
90
|
+
items: [],
|
|
91
|
+
total: 0,
|
|
92
|
+
page: opts?.page ?? 1,
|
|
93
|
+
pageSize: opts?.pageSize ?? 50,
|
|
94
|
+
statusCounts: { indexed: 0, skipped: 0, error: 0 },
|
|
95
|
+
extCounts: [],
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return this.deps.index.listFilesPaged(sourceId, opts);
|
|
100
|
+
}
|
|
101
|
+
async principalStats(agentId, identity) {
|
|
102
|
+
const { sources, db } = this.deps;
|
|
103
|
+
const visible = sources.listVisible(agentId).filter((s) => {
|
|
104
|
+
return !identity || this.gatewayVisible(s, identity);
|
|
105
|
+
});
|
|
106
|
+
const stats = db.stats();
|
|
107
|
+
return {
|
|
108
|
+
stats,
|
|
109
|
+
visibility: {
|
|
110
|
+
assignedProjects: sources
|
|
111
|
+
.list()
|
|
112
|
+
.filter((s) => s.scopeRef.level === 'project' &&
|
|
113
|
+
sources.isVisible(s, agentId) &&
|
|
114
|
+
(!identity || this.gatewayVisible(s, identity)))
|
|
115
|
+
.map((s) => s.scopeRef.key),
|
|
116
|
+
hiddenSourceIds: sources.listHidden(agentId, identity ?? {}),
|
|
117
|
+
},
|
|
118
|
+
visibleSources: visible.length,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
async catalog(agentId, identity) {
|
|
122
|
+
const items = this.deps.sources.catalogFor(agentId);
|
|
123
|
+
if (!identity)
|
|
124
|
+
return items;
|
|
125
|
+
return items.filter((item) => {
|
|
126
|
+
const src = this.deps.sources.get(item.id);
|
|
127
|
+
return !src || this.gatewayVisible(src, identity);
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
async listChunks(agentId, sourceId, path, identity, sessionId) {
|
|
131
|
+
const { sources, index } = this.deps;
|
|
132
|
+
const src = sources.get(sourceId);
|
|
133
|
+
if (!src)
|
|
134
|
+
return null;
|
|
135
|
+
if (identity && !this.gatewayVisible(src, identity))
|
|
136
|
+
return null;
|
|
137
|
+
if (!sources.isVisible(src, agentId, sessionId))
|
|
138
|
+
return null;
|
|
139
|
+
return index.listChunksByPath(sourceId, path);
|
|
140
|
+
}
|
|
141
|
+
async read(agentId, body, identity, sessionId) {
|
|
142
|
+
const { sources, index } = this.deps;
|
|
143
|
+
const visible = (src) => {
|
|
144
|
+
if (!src)
|
|
145
|
+
return false;
|
|
146
|
+
if (identity && !this.gatewayVisible(src, identity))
|
|
147
|
+
return false;
|
|
148
|
+
return sources.isVisible(src, agentId, sessionId);
|
|
149
|
+
};
|
|
150
|
+
if (body.chunkId) {
|
|
151
|
+
const row = index.getChunk(body.chunkId);
|
|
152
|
+
if (!row)
|
|
153
|
+
return { found: false };
|
|
154
|
+
const src = sources.get(row.sourceId);
|
|
155
|
+
if (!visible(src))
|
|
156
|
+
return { found: false };
|
|
157
|
+
return {
|
|
158
|
+
found: true,
|
|
159
|
+
path: row.path,
|
|
160
|
+
startLine: row.startLine,
|
|
161
|
+
endLine: row.endLine,
|
|
162
|
+
text: row.text,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
if (!body.sourceId || !body.path)
|
|
166
|
+
return { found: false };
|
|
167
|
+
const src = sources.get(body.sourceId);
|
|
168
|
+
if (!visible(src))
|
|
169
|
+
return { found: false };
|
|
170
|
+
const chunks = index.listChunksByPath(body.sourceId, body.path);
|
|
171
|
+
return {
|
|
172
|
+
found: chunks.length > 0,
|
|
173
|
+
path: body.path,
|
|
174
|
+
chunkCount: chunks.length,
|
|
175
|
+
text: chunks.map((c) => c.text).join('\n\n'),
|
|
176
|
+
chunks: chunks.map((c) => ({
|
|
177
|
+
id: c.id,
|
|
178
|
+
text: c.text,
|
|
179
|
+
startLine: c.startLine,
|
|
180
|
+
endLine: c.endLine,
|
|
181
|
+
ordinal: c.ordinal,
|
|
182
|
+
path: c.path,
|
|
183
|
+
})),
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
async visibility(agentId, identity) {
|
|
187
|
+
const { sources } = this.deps;
|
|
188
|
+
const assigned = sources
|
|
189
|
+
.list()
|
|
190
|
+
.filter((s) => s.scopeRef.level === 'project' &&
|
|
191
|
+
sources.isVisible(s, agentId, undefined, identity) &&
|
|
192
|
+
(!identity || this.gatewayVisible(s, identity)))
|
|
193
|
+
.map((s) => s.scopeRef.key);
|
|
194
|
+
return {
|
|
195
|
+
assignedProjects: [...new Set(assigned)],
|
|
196
|
+
hiddenSourceIds: sources.listHidden(agentId, identity ?? {}),
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
async sessionVisibility(agentId, sessionId, identity) {
|
|
200
|
+
void agentId;
|
|
201
|
+
const list = this.deps.sources.listSessionVisibility(sessionId, identity);
|
|
202
|
+
if (!identity)
|
|
203
|
+
return list;
|
|
204
|
+
return list.filter((item) => {
|
|
205
|
+
if (item.targetType !== 'source')
|
|
206
|
+
return true;
|
|
207
|
+
const src = this.deps.sources.get(item.targetId);
|
|
208
|
+
return !src || this.gatewayVisible(src, identity);
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
async dbStats() {
|
|
212
|
+
return this.deps.db.stats();
|
|
213
|
+
}
|
|
214
|
+
async getSourceDetail(sourceId, identity) {
|
|
215
|
+
const { sources, index, db } = this.deps;
|
|
216
|
+
const src = sources.get(sourceId);
|
|
217
|
+
if (!src)
|
|
218
|
+
return null;
|
|
219
|
+
if (identity && !this.gatewayVisible(src, identity))
|
|
220
|
+
return null;
|
|
221
|
+
const jobControl = readJobControlState(db, index, sourceId, {
|
|
222
|
+
embeddingEnabled: this.deps.embeddingEnabled ?? true,
|
|
223
|
+
});
|
|
224
|
+
return {
|
|
225
|
+
source: src,
|
|
226
|
+
stats: index.sourceStats(sourceId),
|
|
227
|
+
jobControl,
|
|
228
|
+
assignedAgentIds: src.scopeRef.level === 'project'
|
|
229
|
+
? sources.listProjectAgents(src.scopeRef.key, identity)
|
|
230
|
+
: [],
|
|
231
|
+
hiddenForAgentIds: sources.listAgentsHidingSource(sourceId, identity),
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
async getPrincipal(identity, agentId) {
|
|
235
|
+
const row = this.deps.db.raw
|
|
236
|
+
.prepare(`SELECT * FROM knowledge_principals
|
|
237
|
+
WHERE tenant_id = ? AND gateway_id = ? AND local_agent_id = ?`)
|
|
238
|
+
.get(identity.tenantId ?? 'default', identity.gatewayId ?? 'default', agentId);
|
|
239
|
+
return row ?? null;
|
|
240
|
+
}
|
|
241
|
+
async isPrincipalForeign(identity, agentId) {
|
|
242
|
+
const row = this.deps.db.raw
|
|
243
|
+
.prepare(`SELECT 1 AS ok FROM knowledge_principals
|
|
244
|
+
WHERE tenant_id = ? AND local_agent_id = ? AND gateway_id != ?
|
|
245
|
+
LIMIT 1`)
|
|
246
|
+
.get(identity.tenantId ?? 'default', agentId, identity.gatewayId ?? 'default');
|
|
247
|
+
return Boolean(row);
|
|
248
|
+
}
|
|
249
|
+
async isSourceOwner(sourceId, gatewayId) {
|
|
250
|
+
const row = this.deps.db.raw
|
|
251
|
+
.prepare('SELECT registered_by FROM knowledge_sources WHERE id = ?')
|
|
252
|
+
.get(sourceId);
|
|
253
|
+
if (!row)
|
|
254
|
+
return false;
|
|
255
|
+
if (row.registered_by && row.registered_by !== gatewayId)
|
|
256
|
+
return false;
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
async ready() {
|
|
260
|
+
const { db, index } = this.deps;
|
|
261
|
+
const stats = db.stats();
|
|
262
|
+
const chunks = Number(stats.chunks ?? 0);
|
|
263
|
+
const ftsRows = Number(stats.ftsChunks ?? 0);
|
|
264
|
+
return {
|
|
265
|
+
ready: true,
|
|
266
|
+
sqliteVec: db.sqliteVecEnabled,
|
|
267
|
+
fts: index.ftsAvailable,
|
|
268
|
+
ftsBackfill: {
|
|
269
|
+
running: index.ftsBackfillRunning,
|
|
270
|
+
pendingChunks: Math.max(0, chunks - ftsRows),
|
|
271
|
+
},
|
|
272
|
+
stats,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
async listJobs(query) {
|
|
276
|
+
const where = [
|
|
277
|
+
`source_id IN (SELECT id FROM knowledge_sources WHERE tenant_id = ? AND (registered_by = ? OR visibility = 'public'))`,
|
|
278
|
+
];
|
|
279
|
+
const args = [query.identity.tenantId ?? 'default', query.identity.gatewayId ?? 'default'];
|
|
280
|
+
if (query.sourceId) {
|
|
281
|
+
where.push('source_id = ?');
|
|
282
|
+
args.push(query.sourceId);
|
|
283
|
+
}
|
|
284
|
+
if (query.status) {
|
|
285
|
+
where.push('status = ?');
|
|
286
|
+
args.push(query.status);
|
|
287
|
+
}
|
|
288
|
+
const limit = Math.max(1, Math.min(query.limit ?? 200, 1000));
|
|
289
|
+
return this.deps.db.raw
|
|
290
|
+
.prepare(`SELECT * FROM knowledge_jobs WHERE ${where.join(' AND ')} ORDER BY created_at DESC LIMIT ?`)
|
|
291
|
+
.all(...args, limit);
|
|
292
|
+
}
|
|
293
|
+
async promotionCandidates() {
|
|
294
|
+
return new KnowledgeHitLog(this.deps.db).promotionCandidates();
|
|
295
|
+
}
|
|
296
|
+
dispose() {
|
|
297
|
+
/* 只读;连接由调用方持有 */
|
|
298
|
+
}
|
|
299
|
+
gatewayVisible(src, identity) {
|
|
300
|
+
const row = this.deps.db.raw
|
|
301
|
+
.prepare('SELECT registered_by, visibility FROM knowledge_sources WHERE id = ?')
|
|
302
|
+
.get(src.id);
|
|
303
|
+
if (!row)
|
|
304
|
+
return true;
|
|
305
|
+
if (!identity.gatewayId)
|
|
306
|
+
return true;
|
|
307
|
+
if (row.registered_by === identity.gatewayId)
|
|
308
|
+
return true;
|
|
309
|
+
return row.visibility === 'public';
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
export async function createKnowledgeQueryService(opts) {
|
|
313
|
+
if (opts.mode === 'local' || opts.dbPath === ':memory:') {
|
|
314
|
+
if (!opts.local)
|
|
315
|
+
throw new Error('local query service requires local deps');
|
|
316
|
+
return new LocalKnowledgeQueryService(opts.local);
|
|
317
|
+
}
|
|
318
|
+
const { WorkerQueryService } = await import('./query-worker-client.js');
|
|
319
|
+
return WorkerQueryService.start({
|
|
320
|
+
dbPath: opts.dbPath,
|
|
321
|
+
embeddingModels: opts.embeddingModels ?? null,
|
|
322
|
+
sqliteVecExtensionPath: opts.sqliteVecExtensionPath,
|
|
323
|
+
role: opts.role ?? 'all',
|
|
324
|
+
queryTimeoutMs: opts.queryTimeoutMs,
|
|
325
|
+
});
|
|
326
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { KnowledgeQueryService, SearchQuery, AutoGroundQuery, ListSourcesQuery, QueryIdentity, PrincipalStats, ReadChunkResult, SourceStatsBundle, SourceDetail, ListJobsQuery, ReadySnapshot, QueryWorkerRole } from './query-service.js';
|
|
2
|
+
import type { PromotionCandidate } from './hit-log.js';
|
|
3
|
+
import type { HybridSearchResult, AutoGroundDecision } from './retriever.js';
|
|
4
|
+
import type { KnowledgeSource } from './types.js';
|
|
5
|
+
import type { KnowledgeCatalogItem } from './catalog-types.js';
|
|
6
|
+
import type { IndexedFileRecord } from './index-store.js';
|
|
7
|
+
export declare class WorkerQueryService implements KnowledgeQueryService {
|
|
8
|
+
private readonly worker;
|
|
9
|
+
private readonly queryTimeoutMs;
|
|
10
|
+
private nextId;
|
|
11
|
+
private readonly pending;
|
|
12
|
+
private disposed;
|
|
13
|
+
private bootSettled;
|
|
14
|
+
private bootResolve;
|
|
15
|
+
private bootReject;
|
|
16
|
+
private readonly bootPromise;
|
|
17
|
+
private constructor();
|
|
18
|
+
/**
|
|
19
|
+
* 启动 Query Worker。
|
|
20
|
+
*
|
|
21
|
+
* @param opts - dbPath / embeddingModels(provider 闭包不跨线程,Worker 内重建)
|
|
22
|
+
*/
|
|
23
|
+
static start(opts: {
|
|
24
|
+
dbPath: string;
|
|
25
|
+
sqliteVecExtensionPath?: string;
|
|
26
|
+
queryTimeoutMs?: number;
|
|
27
|
+
role?: QueryWorkerRole;
|
|
28
|
+
embeddingModels?: {
|
|
29
|
+
providers?: Record<string, unknown>;
|
|
30
|
+
embedding?: unknown;
|
|
31
|
+
} | null;
|
|
32
|
+
}): Promise<WorkerQueryService>;
|
|
33
|
+
private failAll;
|
|
34
|
+
private call;
|
|
35
|
+
search(query: SearchQuery): Promise<HybridSearchResult>;
|
|
36
|
+
autoGround(query: AutoGroundQuery): Promise<AutoGroundDecision>;
|
|
37
|
+
listSources(query: ListSourcesQuery): Promise<KnowledgeSource[]>;
|
|
38
|
+
listProjects(identity?: QueryIdentity): Promise<Array<Record<string, unknown>>>;
|
|
39
|
+
getSource(sourceId: string, identity?: QueryIdentity): Promise<{
|
|
40
|
+
source: KnowledgeSource;
|
|
41
|
+
stats: SourceStatsBundle;
|
|
42
|
+
} | null>;
|
|
43
|
+
listFiles(sourceId: string, identity?: QueryIdentity): Promise<IndexedFileRecord[]>;
|
|
44
|
+
listFilesPaged(sourceId: string, opts?: {
|
|
45
|
+
status?: 'indexed' | 'skipped' | 'error' | 'all';
|
|
46
|
+
ext?: string;
|
|
47
|
+
q?: string;
|
|
48
|
+
page?: number;
|
|
49
|
+
pageSize?: number;
|
|
50
|
+
}, identity?: QueryIdentity): Promise<{
|
|
51
|
+
items: Array<IndexedFileRecord & {
|
|
52
|
+
ext: string;
|
|
53
|
+
}>;
|
|
54
|
+
total: number;
|
|
55
|
+
page: number;
|
|
56
|
+
pageSize: number;
|
|
57
|
+
statusCounts: {
|
|
58
|
+
indexed: number;
|
|
59
|
+
skipped: number;
|
|
60
|
+
error: number;
|
|
61
|
+
};
|
|
62
|
+
extCounts: Array<{
|
|
63
|
+
ext: string;
|
|
64
|
+
n: number;
|
|
65
|
+
}>;
|
|
66
|
+
}>;
|
|
67
|
+
principalStats(agentId: string, identity?: QueryIdentity): Promise<PrincipalStats>;
|
|
68
|
+
catalog(agentId: string, identity?: QueryIdentity): Promise<KnowledgeCatalogItem[]>;
|
|
69
|
+
listChunks(agentId: string, sourceId: string, path: string, identity?: QueryIdentity, sessionId?: string): Promise<Array<{
|
|
70
|
+
id: string;
|
|
71
|
+
text: string;
|
|
72
|
+
startLine: number;
|
|
73
|
+
endLine: number;
|
|
74
|
+
ordinal: number;
|
|
75
|
+
path: string;
|
|
76
|
+
}> | null>;
|
|
77
|
+
read(agentId: string, body: {
|
|
78
|
+
chunkId?: string;
|
|
79
|
+
sourceId?: string;
|
|
80
|
+
path?: string;
|
|
81
|
+
}, identity?: QueryIdentity, sessionId?: string): Promise<ReadChunkResult>;
|
|
82
|
+
visibility(agentId: string, identity?: QueryIdentity): Promise<{
|
|
83
|
+
assignedProjects: string[];
|
|
84
|
+
hiddenSourceIds: string[];
|
|
85
|
+
}>;
|
|
86
|
+
sessionVisibility(agentId: string, sessionId: string, identity?: QueryIdentity): Promise<Array<Record<string, unknown>>>;
|
|
87
|
+
dbStats(): Promise<Record<string, number>>;
|
|
88
|
+
getSourceDetail(sourceId: string, identity?: QueryIdentity): Promise<SourceDetail | null>;
|
|
89
|
+
getPrincipal(identity: QueryIdentity, agentId: string): Promise<Record<string, unknown> | null>;
|
|
90
|
+
isPrincipalForeign(identity: QueryIdentity, agentId: string): Promise<boolean>;
|
|
91
|
+
isSourceOwner(sourceId: string, gatewayId: string): Promise<boolean>;
|
|
92
|
+
ready(): Promise<ReadySnapshot>;
|
|
93
|
+
listJobs(query: ListJobsQuery): Promise<Array<Record<string, unknown>>>;
|
|
94
|
+
promotionCandidates(): Promise<PromotionCandidate[]>;
|
|
95
|
+
dispose(): Promise<void>;
|
|
96
|
+
}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WorkerQueryService — Query Worker RPC 客户端(主线程 Meta/Search 或 Engine 残留读)
|
|
3
|
+
*
|
|
4
|
+
* 只读操作转发到独立线程;写路径永不经过这里。
|
|
5
|
+
*/
|
|
6
|
+
import { Worker } from 'node:worker_threads';
|
|
7
|
+
import { resolveWorkerUrl } from '../../worker-path.js';
|
|
8
|
+
export class WorkerQueryService {
|
|
9
|
+
worker;
|
|
10
|
+
queryTimeoutMs;
|
|
11
|
+
nextId = 1;
|
|
12
|
+
pending = new Map();
|
|
13
|
+
disposed = false;
|
|
14
|
+
bootSettled = false;
|
|
15
|
+
bootResolve = null;
|
|
16
|
+
bootReject = null;
|
|
17
|
+
bootPromise;
|
|
18
|
+
constructor(worker, queryTimeoutMs) {
|
|
19
|
+
this.worker = worker;
|
|
20
|
+
this.queryTimeoutMs = queryTimeoutMs;
|
|
21
|
+
this.bootPromise = new Promise((resolve, reject) => {
|
|
22
|
+
this.bootResolve = resolve;
|
|
23
|
+
this.bootReject = reject;
|
|
24
|
+
});
|
|
25
|
+
worker.on('message', (msg) => {
|
|
26
|
+
if (!msg || typeof msg !== 'object' || msg.id == null)
|
|
27
|
+
return;
|
|
28
|
+
// id:0 = worker 握手(ready / boot error);勿等业务 RPC 超时才发现
|
|
29
|
+
if (msg.id === 0) {
|
|
30
|
+
if (this.bootSettled)
|
|
31
|
+
return;
|
|
32
|
+
this.bootSettled = true;
|
|
33
|
+
if (msg.ok)
|
|
34
|
+
this.bootResolve?.();
|
|
35
|
+
else
|
|
36
|
+
this.bootReject?.(new Error(msg.error ?? 'knowledge_query_worker_boot_failed'));
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const slot = this.pending.get(msg.id);
|
|
40
|
+
if (!slot)
|
|
41
|
+
return;
|
|
42
|
+
this.pending.delete(msg.id);
|
|
43
|
+
clearTimeout(slot.timer);
|
|
44
|
+
if (msg.ok)
|
|
45
|
+
slot.resolve(msg.data);
|
|
46
|
+
else
|
|
47
|
+
slot.reject(new Error(msg.error ?? 'knowledge_query_failed'));
|
|
48
|
+
});
|
|
49
|
+
worker.on('error', (err) => {
|
|
50
|
+
const e = err instanceof Error ? err : new Error(String(err));
|
|
51
|
+
if (!this.bootSettled) {
|
|
52
|
+
this.bootSettled = true;
|
|
53
|
+
this.bootReject?.(e);
|
|
54
|
+
}
|
|
55
|
+
this.failAll(e);
|
|
56
|
+
});
|
|
57
|
+
worker.on('exit', () => {
|
|
58
|
+
const e = new Error('knowledge_query_worker_exit');
|
|
59
|
+
if (!this.bootSettled) {
|
|
60
|
+
this.bootSettled = true;
|
|
61
|
+
this.bootReject?.(e);
|
|
62
|
+
}
|
|
63
|
+
this.failAll(e);
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* 启动 Query Worker。
|
|
68
|
+
*
|
|
69
|
+
* @param opts - dbPath / embeddingModels(provider 闭包不跨线程,Worker 内重建)
|
|
70
|
+
*/
|
|
71
|
+
static async start(opts) {
|
|
72
|
+
const worker = new Worker(resolveWorkerUrl('./query-worker.js', import.meta.url), {
|
|
73
|
+
workerData: {
|
|
74
|
+
dbPath: opts.dbPath,
|
|
75
|
+
sqliteVecExtensionPath: opts.sqliteVecExtensionPath,
|
|
76
|
+
embeddingModels: opts.embeddingModels ?? null,
|
|
77
|
+
role: opts.role ?? 'all',
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
const client = new WorkerQueryService(worker, opts.queryTimeoutMs ?? 15_000);
|
|
81
|
+
const bootTimer = setTimeout(() => {
|
|
82
|
+
if (!client.bootSettled) {
|
|
83
|
+
client.bootSettled = true;
|
|
84
|
+
client.bootReject?.(new Error('knowledge_query_worker_boot_timeout'));
|
|
85
|
+
}
|
|
86
|
+
}, 30_000);
|
|
87
|
+
try {
|
|
88
|
+
await client.bootPromise;
|
|
89
|
+
await client.call({ method: 'dbStats' }, 15_000);
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
clearTimeout(bootTimer);
|
|
93
|
+
}
|
|
94
|
+
return client;
|
|
95
|
+
}
|
|
96
|
+
failAll(err) {
|
|
97
|
+
for (const [, slot] of this.pending) {
|
|
98
|
+
clearTimeout(slot.timer);
|
|
99
|
+
slot.reject(err);
|
|
100
|
+
}
|
|
101
|
+
this.pending.clear();
|
|
102
|
+
}
|
|
103
|
+
call(payload, timeoutMs) {
|
|
104
|
+
if (this.disposed) {
|
|
105
|
+
return Promise.reject(new Error('knowledge_query_disposed'));
|
|
106
|
+
}
|
|
107
|
+
const id = this.nextId++;
|
|
108
|
+
const ms = timeoutMs ?? this.queryTimeoutMs;
|
|
109
|
+
return new Promise((resolve, reject) => {
|
|
110
|
+
const timer = setTimeout(() => {
|
|
111
|
+
this.pending.delete(id);
|
|
112
|
+
reject(new Error(`knowledge_query_timeout after ${ms}ms`));
|
|
113
|
+
}, ms);
|
|
114
|
+
this.pending.set(id, { resolve, reject, timer });
|
|
115
|
+
this.worker.postMessage({ id, ...payload });
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
async search(query) {
|
|
119
|
+
return (await this.call({ method: 'search', query }));
|
|
120
|
+
}
|
|
121
|
+
async autoGround(query) {
|
|
122
|
+
return (await this.call({ method: 'autoGround', query }, 20_000));
|
|
123
|
+
}
|
|
124
|
+
async listSources(query) {
|
|
125
|
+
return (await this.call({ method: 'listSources', query }));
|
|
126
|
+
}
|
|
127
|
+
async listProjects(identity) {
|
|
128
|
+
return (await this.call({ method: 'listProjects', identity }));
|
|
129
|
+
}
|
|
130
|
+
async getSource(sourceId, identity) {
|
|
131
|
+
return (await this.call({ method: 'getSource', sourceId, identity }));
|
|
132
|
+
}
|
|
133
|
+
async listFiles(sourceId, identity) {
|
|
134
|
+
return (await this.call({ method: 'listFiles', sourceId, identity }));
|
|
135
|
+
}
|
|
136
|
+
async listFilesPaged(sourceId, opts, identity) {
|
|
137
|
+
return (await this.call({ method: 'listFilesPaged', sourceId, opts, identity }));
|
|
138
|
+
}
|
|
139
|
+
async principalStats(agentId, identity) {
|
|
140
|
+
return (await this.call({ method: 'principalStats', agentId, identity }));
|
|
141
|
+
}
|
|
142
|
+
async catalog(agentId, identity) {
|
|
143
|
+
return (await this.call({ method: 'catalog', agentId, identity }));
|
|
144
|
+
}
|
|
145
|
+
async listChunks(agentId, sourceId, path, identity, sessionId) {
|
|
146
|
+
return (await this.call({
|
|
147
|
+
method: 'listChunks',
|
|
148
|
+
agentId,
|
|
149
|
+
sourceId,
|
|
150
|
+
path,
|
|
151
|
+
identity,
|
|
152
|
+
sessionId,
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
async read(agentId, body, identity, sessionId) {
|
|
156
|
+
return (await this.call({ method: 'read', agentId, body, identity, sessionId }));
|
|
157
|
+
}
|
|
158
|
+
async visibility(agentId, identity) {
|
|
159
|
+
return (await this.call({ method: 'visibility', agentId, identity }));
|
|
160
|
+
}
|
|
161
|
+
async sessionVisibility(agentId, sessionId, identity) {
|
|
162
|
+
return (await this.call({
|
|
163
|
+
method: 'sessionVisibility',
|
|
164
|
+
agentId,
|
|
165
|
+
sessionId,
|
|
166
|
+
identity,
|
|
167
|
+
}));
|
|
168
|
+
}
|
|
169
|
+
async dbStats() {
|
|
170
|
+
return (await this.call({ method: 'dbStats' }));
|
|
171
|
+
}
|
|
172
|
+
async getSourceDetail(sourceId, identity) {
|
|
173
|
+
return (await this.call({ method: 'getSourceDetail', sourceId, identity }));
|
|
174
|
+
}
|
|
175
|
+
async getPrincipal(identity, agentId) {
|
|
176
|
+
return (await this.call({ method: 'getPrincipal', identity, agentId }));
|
|
177
|
+
}
|
|
178
|
+
async isPrincipalForeign(identity, agentId) {
|
|
179
|
+
return (await this.call({ method: 'isPrincipalForeign', identity, agentId }));
|
|
180
|
+
}
|
|
181
|
+
async isSourceOwner(sourceId, gatewayId) {
|
|
182
|
+
return (await this.call({ method: 'isSourceOwner', sourceId, gatewayId }));
|
|
183
|
+
}
|
|
184
|
+
async ready() {
|
|
185
|
+
return (await this.call({ method: 'ready' }));
|
|
186
|
+
}
|
|
187
|
+
async listJobs(query) {
|
|
188
|
+
return (await this.call({ method: 'listJobs', query }));
|
|
189
|
+
}
|
|
190
|
+
async promotionCandidates() {
|
|
191
|
+
return (await this.call({ method: 'promotionCandidates' }));
|
|
192
|
+
}
|
|
193
|
+
async dispose() {
|
|
194
|
+
if (this.disposed)
|
|
195
|
+
return;
|
|
196
|
+
// 必须先发 shutdown 再置 disposed:call() 在 disposed 后会立刻 reject
|
|
197
|
+
try {
|
|
198
|
+
await this.call({ method: 'shutdown' }, 5_000);
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
/* worker 可能已退 */
|
|
202
|
+
}
|
|
203
|
+
this.disposed = true;
|
|
204
|
+
this.failAll(new Error('knowledge_query_disposed'));
|
|
205
|
+
await this.worker.terminate();
|
|
206
|
+
}
|
|
207
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|