@octopi-agent/engine 0.58.0 → 0.70.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +122 -1
- package/dist/config-schema/engine.js +4 -1
- package/dist/config-schema/substrate.d.ts +123 -0
- package/dist/config-schema/substrate.js +161 -0
- package/dist/config.d.ts +101 -1
- package/dist/harness/agent/builder.d.ts +15 -1
- package/dist/harness/agent/builder.js +41 -4
- package/dist/harness/agent/config-bridge.js +17 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/capabilities/summary/gate.js +2 -2
- package/dist/harness/context/capabilities/summary/policies.js +5 -5
- package/dist/harness/context/capabilities/summary/registry.js +29 -24
- package/dist/harness/context/layer-health.js +3 -2
- package/dist/harness/context/layers.d.ts +27 -13
- package/dist/harness/context/layers.js +76 -66
- package/dist/harness/context/system-prompt-assembler.d.ts +3 -0
- package/dist/harness/context/system-prompt-assembler.js +11 -1
- package/dist/harness/extension/plugin-ecosystem/client-tools/index.d.ts +4 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/index.js +3 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.d.ts +56 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.js +281 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/registry.d.ts +98 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/registry.js +250 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/stream.d.ts +124 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/stream.js +372 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/types.d.ts +109 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/types.js +8 -0
- package/dist/harness/extension/plugin-ecosystem/index.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/index.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +1 -1
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.d.ts +11 -1
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +14 -0
- package/dist/harness/index.d.ts +8 -5
- package/dist/harness/index.js +4 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +97 -19
- package/dist/harness/knowledge/source-store.js +166 -94
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/cognition-decay.d.ts +36 -0
- package/dist/harness/memory/cognition-decay.js +55 -0
- package/dist/harness/memory/cognition-gates.d.ts +60 -0
- package/dist/harness/memory/cognition-gates.js +191 -0
- package/dist/harness/memory/cognition-trigger.d.ts +30 -0
- package/dist/harness/memory/cognition-trigger.js +41 -0
- package/dist/harness/memory/cognition-types.d.ts +1 -1
- package/dist/harness/memory/cognition.d.ts +40 -8
- package/dist/harness/memory/cognition.js +432 -79
- package/dist/harness/memory/conceptualizer.d.ts +66 -0
- package/dist/harness/memory/conceptualizer.js +246 -0
- package/dist/harness/memory/index.d.ts +16 -2
- package/dist/harness/memory/index.js +11 -1
- package/dist/harness/memory/sqlite/agent-db.d.ts +9 -0
- package/dist/harness/memory/sqlite/agent-db.js +118 -15
- package/dist/harness/memory/sqlite/cognition-store.d.ts +55 -16
- package/dist/harness/memory/sqlite/cognition-store.js +591 -144
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/memory/sqlite/wisdom-store.d.ts +33 -14
- package/dist/harness/memory/sqlite/wisdom-store.js +257 -44
- package/dist/harness/memory/types.d.ts +331 -16
- package/dist/harness/memory/types.js +49 -0
- package/dist/harness/memory/wisdom-formation.d.ts +101 -0
- package/dist/harness/memory/wisdom-formation.js +304 -0
- package/dist/harness/memory/wisdom-gates.d.ts +50 -0
- package/dist/harness/memory/wisdom-gates.js +204 -0
- package/dist/harness/memory/wisdom-policy.d.ts +107 -0
- package/dist/harness/memory/wisdom-policy.js +220 -0
- package/dist/harness/memory/wisdom-types.d.ts +3 -2
- package/dist/harness/memory/wisdom-types.js +2 -2
- package/dist/harness/memory/wisdom.d.ts +27 -0
- package/dist/harness/memory/wisdom.js +181 -0
- package/dist/harness/run/agent/agent.d.ts +9 -0
- package/dist/harness/run/agent/agent.js +19 -0
- package/dist/harness/run/reliability/run-agent.js +6 -2
- package/dist/harness/run/runner.js +2 -1
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/index.d.ts +1 -0
- package/dist/harness/session/index.js +1 -0
- package/dist/harness/session/title.d.ts +86 -0
- package/dist/harness/session/title.js +268 -0
- package/dist/harness/shared/events/agent-event-map.d.ts +8 -0
- package/dist/harness/shared/events/agent-event-map.js +1 -0
- package/dist/index.d.ts +7 -3
- package/dist/index.js +3 -1
- package/dist/subsystems/cognition-steward/conceptualize/SUBSYSTEM.md +43 -0
- package/dist/subsystems/cognition-steward/conceptualize/config.yaml +50 -0
- package/dist/subsystems/cognition-steward/conceptualize/handler.d.ts +9 -0
- package/dist/subsystems/cognition-steward/conceptualize/handler.js +125 -0
- package/dist/subsystems/cognition-steward/consolidate/SUBSYSTEM.md +6 -0
- package/dist/subsystems/cognition-steward/consolidate/config.yaml +54 -0
- package/dist/subsystems/cognition-steward/consolidate/handler.d.ts +12 -0
- package/dist/subsystems/cognition-steward/consolidate/handler.js +93 -0
- package/dist/subsystems/wisdom-steward/evaluate/config.yaml +50 -0
- package/dist/subsystems/wisdom-steward/evaluate/handler.d.ts +18 -0
- package/dist/subsystems/wisdom-steward/evaluate/handler.js +91 -0
- package/dist/subsystems/wisdom-steward/formation/SUBSYSTEM.md +42 -0
- package/dist/subsystems/wisdom-steward/formation/config.yaml +54 -0
- package/dist/subsystems/wisdom-steward/formation/handler.d.ts +18 -0
- package/dist/subsystems/wisdom-steward/formation/handler.js +212 -0
- package/dist/subsystems/wisdom-steward/govern/config.yaml +55 -0
- package/dist/subsystems/wisdom-steward/govern/handler.d.ts +17 -0
- package/dist/subsystems/wisdom-steward/govern/handler.js +68 -0
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MembershipStore — Source↔File 认领 + 零认领 purge(契约 §3.5 / §6.2)
|
|
3
|
+
*/
|
|
4
|
+
import type { KnowledgeDatabase } from './db.js';
|
|
5
|
+
import type { FileIdentity } from './file-identity.js';
|
|
6
|
+
export interface MembershipRow {
|
|
7
|
+
sourceId: string;
|
|
8
|
+
fileId: string;
|
|
9
|
+
logicalPath: string;
|
|
10
|
+
createdAt: number;
|
|
11
|
+
}
|
|
12
|
+
export interface FileRow {
|
|
13
|
+
id: string;
|
|
14
|
+
identityKey: string;
|
|
15
|
+
size: number;
|
|
16
|
+
mtime: number;
|
|
17
|
+
contentHash: string | null;
|
|
18
|
+
status: string;
|
|
19
|
+
chunkCount: number;
|
|
20
|
+
}
|
|
21
|
+
export interface ReconcileResult {
|
|
22
|
+
added: number;
|
|
23
|
+
removed: number;
|
|
24
|
+
purgedFiles: number;
|
|
25
|
+
}
|
|
26
|
+
export declare class MembershipStore {
|
|
27
|
+
private readonly db;
|
|
28
|
+
constructor(db: KnowledgeDatabase);
|
|
29
|
+
/**
|
|
30
|
+
* 解析/登记物理 File;版本变化时更新 version 并标记 needsParse。
|
|
31
|
+
*/
|
|
32
|
+
upsertByIdentity(identity: FileIdentity, opts?: {
|
|
33
|
+
tenantId?: string;
|
|
34
|
+
}): {
|
|
35
|
+
fileId: string;
|
|
36
|
+
created: boolean;
|
|
37
|
+
versionChanged: boolean;
|
|
38
|
+
};
|
|
39
|
+
claim(sourceId: string, fileId: string, logicalPath: string): void;
|
|
40
|
+
unclaim(sourceId: string, logicalPath: string): string | null;
|
|
41
|
+
claimCount(fileId: string): number;
|
|
42
|
+
listBySource(sourceId: string): MembershipRow[];
|
|
43
|
+
listByFile(fileId: string): MembershipRow[];
|
|
44
|
+
/**
|
|
45
|
+
* 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
|
|
46
|
+
* 大源删除分批让出,避免同步连环 purge 堵死 Engine。
|
|
47
|
+
*
|
|
48
|
+
* @returns purge 的 fileId 列表
|
|
49
|
+
*/
|
|
50
|
+
unclaimAllForSourceAsync(sourceId: string, purge: (fileId: string) => void | Promise<void>, opts?: {
|
|
51
|
+
yieldEvery?: number;
|
|
52
|
+
}): Promise<string[]>;
|
|
53
|
+
/**
|
|
54
|
+
* 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
|
|
55
|
+
* 返回 purge 的 fileId 列表。
|
|
56
|
+
*/
|
|
57
|
+
unclaimAllForSource(sourceId: string, purge: (fileId: string) => void): string[];
|
|
58
|
+
/**
|
|
59
|
+
* Walk 对账(§6.2):discovered 为 logical_path → fileId。
|
|
60
|
+
* 删掉本 Source 不在 discovered 中的 Membership;零认领则 purge。
|
|
61
|
+
*/
|
|
62
|
+
reconcileSource(sourceId: string, discovered: Map<string, string>, purge: (fileId: string) => void): ReconcileResult;
|
|
63
|
+
getFile(fileId: string): FileRow | null;
|
|
64
|
+
findFileByIdentity(identityKey: string, tenantId?: string): FileRow | null;
|
|
65
|
+
}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MembershipStore — Source↔File 认领 + 零认领 purge(契约 §3.5 / §6.2)
|
|
3
|
+
*/
|
|
4
|
+
function newId(prefix) {
|
|
5
|
+
return `${prefix}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`;
|
|
6
|
+
}
|
|
7
|
+
export class MembershipStore {
|
|
8
|
+
db;
|
|
9
|
+
constructor(db) {
|
|
10
|
+
this.db = db;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* 解析/登记物理 File;版本变化时更新 version 并标记 needsParse。
|
|
14
|
+
*/
|
|
15
|
+
upsertByIdentity(identity, opts) {
|
|
16
|
+
const tenant = opts?.tenantId ?? 'default';
|
|
17
|
+
const now = Date.now();
|
|
18
|
+
const row = this.db.raw
|
|
19
|
+
.prepare(`SELECT id, size, mtime FROM knowledge_files
|
|
20
|
+
WHERE tenant_id = ? AND identity_key = ?`)
|
|
21
|
+
.get(tenant, identity.key);
|
|
22
|
+
if (row) {
|
|
23
|
+
const versionChanged = row.size !== identity.size || row.mtime !== identity.mtime;
|
|
24
|
+
if (versionChanged) {
|
|
25
|
+
this.db.raw
|
|
26
|
+
.prepare(`UPDATE knowledge_files SET size = ?, mtime = ?, indexed_at = ? WHERE id = ?`)
|
|
27
|
+
.run(identity.size, identity.mtime, now, row.id);
|
|
28
|
+
}
|
|
29
|
+
return { fileId: row.id, created: false, versionChanged };
|
|
30
|
+
}
|
|
31
|
+
const fileId = newId('file');
|
|
32
|
+
this.db.raw
|
|
33
|
+
.prepare(`INSERT INTO knowledge_files
|
|
34
|
+
(id, tenant_id, identity_key, size, mtime, content_hash, status, chunk_count, indexed_at)
|
|
35
|
+
VALUES (?, ?, ?, ?, ?, NULL, 'pending', 0, ?)`)
|
|
36
|
+
.run(fileId, tenant, identity.key, identity.size, identity.mtime, now);
|
|
37
|
+
return { fileId, created: true, versionChanged: true };
|
|
38
|
+
}
|
|
39
|
+
claim(sourceId, fileId, logicalPath) {
|
|
40
|
+
const prev = this.db.raw
|
|
41
|
+
.prepare(`SELECT file_id FROM knowledge_memberships
|
|
42
|
+
WHERE source_id = ? AND logical_path = ?`)
|
|
43
|
+
.get(sourceId, logicalPath);
|
|
44
|
+
this.db.raw
|
|
45
|
+
.prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
|
|
46
|
+
VALUES (?, ?, ?, ?)
|
|
47
|
+
ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
|
|
48
|
+
.run(sourceId, fileId, logicalPath, Date.now());
|
|
49
|
+
// 换绑后旧 File 若零认领,由调用方 purge(此处只记孤儿风险)
|
|
50
|
+
if (prev && prev.file_id !== fileId && this.claimCount(prev.file_id) === 0) {
|
|
51
|
+
// 留给 reconcile 补 purge;避免 claim 路径上嵌套删除事务
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
unclaim(sourceId, logicalPath) {
|
|
55
|
+
const row = this.db.raw
|
|
56
|
+
.prepare(`SELECT file_id FROM knowledge_memberships
|
|
57
|
+
WHERE source_id = ? AND logical_path = ?`)
|
|
58
|
+
.get(sourceId, logicalPath);
|
|
59
|
+
this.db.raw
|
|
60
|
+
.prepare(`DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
61
|
+
.run(sourceId, logicalPath);
|
|
62
|
+
return row?.file_id ?? null;
|
|
63
|
+
}
|
|
64
|
+
claimCount(fileId) {
|
|
65
|
+
const r = this.db.raw
|
|
66
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?`)
|
|
67
|
+
.get(fileId);
|
|
68
|
+
return r.n;
|
|
69
|
+
}
|
|
70
|
+
listBySource(sourceId) {
|
|
71
|
+
const rows = this.db.raw
|
|
72
|
+
.prepare(`SELECT source_id, file_id, logical_path, created_at
|
|
73
|
+
FROM knowledge_memberships WHERE source_id = ? ORDER BY logical_path`)
|
|
74
|
+
.all(sourceId);
|
|
75
|
+
return rows.map((r) => ({
|
|
76
|
+
sourceId: r.source_id,
|
|
77
|
+
fileId: r.file_id,
|
|
78
|
+
logicalPath: r.logical_path,
|
|
79
|
+
createdAt: r.created_at,
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
listByFile(fileId) {
|
|
83
|
+
const rows = this.db.raw
|
|
84
|
+
.prepare(`SELECT source_id, file_id, logical_path, created_at
|
|
85
|
+
FROM knowledge_memberships WHERE file_id = ?`)
|
|
86
|
+
.all(fileId);
|
|
87
|
+
return rows.map((r) => ({
|
|
88
|
+
sourceId: r.source_id,
|
|
89
|
+
fileId: r.file_id,
|
|
90
|
+
logicalPath: r.logical_path,
|
|
91
|
+
createdAt: r.created_at,
|
|
92
|
+
}));
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
|
|
96
|
+
* 大源删除分批让出,避免同步连环 purge 堵死 Engine。
|
|
97
|
+
*
|
|
98
|
+
* @returns purge 的 fileId 列表
|
|
99
|
+
*/
|
|
100
|
+
async unclaimAllForSourceAsync(sourceId, purge, opts) {
|
|
101
|
+
const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
|
|
102
|
+
const files = this.db.raw
|
|
103
|
+
.prepare(`SELECT DISTINCT file_id FROM knowledge_memberships WHERE source_id = ?`)
|
|
104
|
+
.all(sourceId);
|
|
105
|
+
this.db.raw.prepare(`DELETE FROM knowledge_memberships WHERE source_id = ?`).run(sourceId);
|
|
106
|
+
const purged = [];
|
|
107
|
+
for (let i = 0; i < files.length; i++) {
|
|
108
|
+
const f = files[i];
|
|
109
|
+
if (this.claimCount(f.file_id) === 0) {
|
|
110
|
+
await purge(f.file_id);
|
|
111
|
+
purged.push(f.file_id);
|
|
112
|
+
}
|
|
113
|
+
if ((i + 1) % yieldEvery === 0) {
|
|
114
|
+
await new Promise((r) => setImmediate(r));
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return purged;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
|
|
121
|
+
* 返回 purge 的 fileId 列表。
|
|
122
|
+
*/
|
|
123
|
+
unclaimAllForSource(sourceId, purge) {
|
|
124
|
+
const files = this.db.raw
|
|
125
|
+
.prepare(`SELECT DISTINCT file_id FROM knowledge_memberships WHERE source_id = ?`)
|
|
126
|
+
.all(sourceId);
|
|
127
|
+
this.db.raw.prepare(`DELETE FROM knowledge_memberships WHERE source_id = ?`).run(sourceId);
|
|
128
|
+
const purged = [];
|
|
129
|
+
for (const f of files) {
|
|
130
|
+
if (this.claimCount(f.file_id) === 0) {
|
|
131
|
+
purge(f.file_id);
|
|
132
|
+
purged.push(f.file_id);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return purged;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Walk 对账(§6.2):discovered 为 logical_path → fileId。
|
|
139
|
+
* 删掉本 Source 不在 discovered 中的 Membership;零认领则 purge。
|
|
140
|
+
*/
|
|
141
|
+
reconcileSource(sourceId, discovered, purge) {
|
|
142
|
+
const now = Date.now();
|
|
143
|
+
let added = 0;
|
|
144
|
+
for (const [logicalPath, fileId] of discovered) {
|
|
145
|
+
this.claim(sourceId, fileId, logicalPath);
|
|
146
|
+
added += 1;
|
|
147
|
+
}
|
|
148
|
+
const existing = this.listBySource(sourceId);
|
|
149
|
+
let removed = 0;
|
|
150
|
+
let purgedFiles = 0;
|
|
151
|
+
for (const m of existing) {
|
|
152
|
+
if (discovered.has(m.logicalPath))
|
|
153
|
+
continue;
|
|
154
|
+
this.unclaim(sourceId, m.logicalPath);
|
|
155
|
+
removed += 1;
|
|
156
|
+
if (this.claimCount(m.fileId) === 0) {
|
|
157
|
+
purge(m.fileId);
|
|
158
|
+
purgedFiles += 1;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
void now;
|
|
162
|
+
return { added, removed, purgedFiles };
|
|
163
|
+
}
|
|
164
|
+
getFile(fileId) {
|
|
165
|
+
const r = this.db.raw
|
|
166
|
+
.prepare(`SELECT id, identity_key, size, mtime, content_hash, status, chunk_count
|
|
167
|
+
FROM knowledge_files WHERE id = ?`)
|
|
168
|
+
.get(fileId);
|
|
169
|
+
if (!r)
|
|
170
|
+
return null;
|
|
171
|
+
return {
|
|
172
|
+
id: r.id,
|
|
173
|
+
identityKey: r.identity_key,
|
|
174
|
+
size: r.size,
|
|
175
|
+
mtime: r.mtime,
|
|
176
|
+
contentHash: r.content_hash,
|
|
177
|
+
status: r.status,
|
|
178
|
+
chunkCount: r.chunk_count,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
findFileByIdentity(identityKey, tenantId = 'default') {
|
|
182
|
+
const r = this.db.raw
|
|
183
|
+
.prepare(`SELECT id, identity_key, size, mtime, content_hash, status, chunk_count
|
|
184
|
+
FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?`)
|
|
185
|
+
.get(tenantId, identityKey);
|
|
186
|
+
if (!r)
|
|
187
|
+
return null;
|
|
188
|
+
return {
|
|
189
|
+
id: r.id,
|
|
190
|
+
identityKey: r.identity_key,
|
|
191
|
+
size: r.size,
|
|
192
|
+
mtime: r.mtime,
|
|
193
|
+
contentHash: r.content_hash,
|
|
194
|
+
status: r.status,
|
|
195
|
+
chunkCount: r.chunk_count,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* 出站网络门禁 — Knowledge 外源 ingest SSRF 防护
|
|
3
3
|
*
|
|
4
4
|
* 默认拒绝私网/环回/link-local/云 metadata;源级 allowPrivateNetwork 显式放行。
|
|
5
|
-
*
|
|
5
|
+
* 重定向上每跳复验地址;**不**做 IP 钉扎——DNS TOCTOU 见 docs/KNOWN-ISSUES。
|
|
6
6
|
*/
|
|
7
7
|
export interface NetworkGuardOptions {
|
|
8
8
|
/** 放行私网/环回(源级显式开启) */
|
|
@@ -28,7 +28,16 @@ export interface FetchResult {
|
|
|
28
28
|
notModified: boolean;
|
|
29
29
|
}
|
|
30
30
|
/**
|
|
31
|
-
*
|
|
31
|
+
* 凭证只属于「源入口 origin」。远端内容指向的任何其他 origin 一律不带鉴权头。
|
|
32
|
+
*
|
|
33
|
+
* @param trustedOrigin - 源入口 URL 的 origin(listUrl / source.location)
|
|
34
|
+
* @param targetUrl - 本次实际请求的 URL
|
|
35
|
+
* @param headers - 候选请求头(含凭证)
|
|
36
|
+
* @returns 同 origin 时原样返回;跨 origin 时只保留安全白名单
|
|
37
|
+
*/
|
|
38
|
+
export declare function authHeadersForUrl(trustedOrigin: string, targetUrl: string, headers?: Record<string, string>): Record<string, string>;
|
|
39
|
+
/**
|
|
40
|
+
* 判断 IP 是否属于受限地址(私网/环回/metadata/未指定)
|
|
32
41
|
*
|
|
33
42
|
* @param ip - IPv4/IPv6 字面量
|
|
34
43
|
*/
|
|
@@ -2,13 +2,48 @@
|
|
|
2
2
|
* 出站网络门禁 — Knowledge 外源 ingest SSRF 防护
|
|
3
3
|
*
|
|
4
4
|
* 默认拒绝私网/环回/link-local/云 metadata;源级 allowPrivateNetwork 显式放行。
|
|
5
|
-
*
|
|
5
|
+
* 重定向上每跳复验地址;**不**做 IP 钉扎——DNS TOCTOU 见 docs/KNOWN-ISSUES。
|
|
6
6
|
*/
|
|
7
7
|
import { lookup } from 'node:dns/promises';
|
|
8
8
|
import { isIP } from 'node:net';
|
|
9
9
|
const DEFAULT_MAX_REDIRECTS = 5;
|
|
10
10
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
11
11
|
const DEFAULT_MAX_BYTES = 5_000_000;
|
|
12
|
+
/** 跨 origin 时唯一允许携带的请求头(白名单;其余含凭证一律剥离) */
|
|
13
|
+
const SAFE_CROSS_ORIGIN_HEADERS = new Set([
|
|
14
|
+
'accept',
|
|
15
|
+
'accept-language',
|
|
16
|
+
'user-agent',
|
|
17
|
+
'if-none-match',
|
|
18
|
+
'if-modified-since',
|
|
19
|
+
]);
|
|
20
|
+
/**
|
|
21
|
+
* 凭证只属于「源入口 origin」。远端内容指向的任何其他 origin 一律不带鉴权头。
|
|
22
|
+
*
|
|
23
|
+
* @param trustedOrigin - 源入口 URL 的 origin(listUrl / source.location)
|
|
24
|
+
* @param targetUrl - 本次实际请求的 URL
|
|
25
|
+
* @param headers - 候选请求头(含凭证)
|
|
26
|
+
* @returns 同 origin 时原样返回;跨 origin 时只保留安全白名单
|
|
27
|
+
*/
|
|
28
|
+
export function authHeadersForUrl(trustedOrigin, targetUrl, headers) {
|
|
29
|
+
if (!headers)
|
|
30
|
+
return {};
|
|
31
|
+
let sameOrigin = false;
|
|
32
|
+
try {
|
|
33
|
+
sameOrigin = new URL(targetUrl).origin === trustedOrigin;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
sameOrigin = false;
|
|
37
|
+
}
|
|
38
|
+
if (sameOrigin)
|
|
39
|
+
return { ...headers };
|
|
40
|
+
const out = {};
|
|
41
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
42
|
+
if (SAFE_CROSS_ORIGIN_HEADERS.has(k.toLowerCase()))
|
|
43
|
+
out[k] = v;
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
12
47
|
/** 云 metadata / 链路本地 */
|
|
13
48
|
function isLinkLocalOrMetadata(ip) {
|
|
14
49
|
const v = ip.toLowerCase();
|
|
@@ -24,11 +59,19 @@ function isLoopback(ip) {
|
|
|
24
59
|
const v = ip.toLowerCase();
|
|
25
60
|
return v === '::1' || v === '0:0:0:0:0:0:0:1' || v.startsWith('127.');
|
|
26
61
|
}
|
|
62
|
+
/** 未指定地址(:: / 0.0.0.0):连出去通常等价本机 */
|
|
63
|
+
function isUnspecified(ip) {
|
|
64
|
+
const v = ip.toLowerCase().replace(/^\[|\]$/g, '');
|
|
65
|
+
return v === '0.0.0.0' || v === '::' || v === '0:0:0:0:0:0:0:0';
|
|
66
|
+
}
|
|
27
67
|
function isPrivateIPv4(ip) {
|
|
28
68
|
const parts = ip.split('.').map(Number);
|
|
29
69
|
if (parts.length !== 4 || parts.some((n) => Number.isNaN(n)))
|
|
30
70
|
return false;
|
|
31
71
|
const [a, b] = parts;
|
|
72
|
+
// 0.0.0.0/8 「this network」;0.0.0.0 连出常落本机
|
|
73
|
+
if (a === 0)
|
|
74
|
+
return true;
|
|
32
75
|
if (a === 10)
|
|
33
76
|
return true;
|
|
34
77
|
if (a === 172 && b >= 16 && b <= 31)
|
|
@@ -40,18 +83,37 @@ function isPrivateIPv4(ip) {
|
|
|
40
83
|
return true;
|
|
41
84
|
return false;
|
|
42
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* IPv4-mapped IPv6 → 点分 IPv4(含 `::ffff:7f00:1` 十六进制形态)。
|
|
88
|
+
* 解不出则 null。
|
|
89
|
+
*/
|
|
90
|
+
function extractIPv4Mapped(ip) {
|
|
91
|
+
const v = ip.toLowerCase().replace(/^\[|\]$/g, '');
|
|
92
|
+
if (!v.startsWith('::ffff:'))
|
|
93
|
+
return null;
|
|
94
|
+
const rest = v.slice(7);
|
|
95
|
+
if (rest.includes('.'))
|
|
96
|
+
return rest;
|
|
97
|
+
const parts = rest.split(':');
|
|
98
|
+
if (parts.length !== 2)
|
|
99
|
+
return null;
|
|
100
|
+
const hi = Number.parseInt(parts[0], 16);
|
|
101
|
+
const lo = Number.parseInt(parts[1], 16);
|
|
102
|
+
if (!Number.isFinite(hi) || !Number.isFinite(lo))
|
|
103
|
+
return null;
|
|
104
|
+
return `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
|
105
|
+
}
|
|
43
106
|
function isPrivateIPv6(ip) {
|
|
44
107
|
const v = ip.toLowerCase().replace(/^\[|\]$/g, '');
|
|
45
108
|
if (v.startsWith('fc') || v.startsWith('fd'))
|
|
46
109
|
return true; // ULA
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
return isPrivateIPv4(mapped) || isLoopback(mapped);
|
|
50
|
-
}
|
|
110
|
+
const mapped = extractIPv4Mapped(v);
|
|
111
|
+
if (mapped)
|
|
112
|
+
return isPrivateIPv4(mapped) || isLoopback(mapped) || isUnspecified(mapped);
|
|
51
113
|
return false;
|
|
52
114
|
}
|
|
53
115
|
/**
|
|
54
|
-
* 判断 IP 是否属于受限地址(私网/环回/metadata
|
|
116
|
+
* 判断 IP 是否属于受限地址(私网/环回/metadata/未指定)
|
|
55
117
|
*
|
|
56
118
|
* @param ip - IPv4/IPv6 字面量
|
|
57
119
|
*/
|
|
@@ -59,7 +121,7 @@ export function isRestrictedIp(ip) {
|
|
|
59
121
|
const v = ip.trim().toLowerCase().replace(/^\[|\]$/g, '');
|
|
60
122
|
if (!v)
|
|
61
123
|
return true;
|
|
62
|
-
if (isLoopback(v) || isLinkLocalOrMetadata(v))
|
|
124
|
+
if (isUnspecified(v) || isLoopback(v) || isLinkLocalOrMetadata(v))
|
|
63
125
|
return true;
|
|
64
126
|
if (isIP(v) === 4)
|
|
65
127
|
return isPrivateIPv4(v);
|
|
@@ -128,28 +190,11 @@ export async function guardedFetch(rawUrl, options, init) {
|
|
|
128
190
|
const maxBytes = options?.maxResponseBytes ?? DEFAULT_MAX_BYTES;
|
|
129
191
|
let current = rawUrl;
|
|
130
192
|
let redirects = 0;
|
|
131
|
-
/** 凭证仅允许与首个请求同 origin
|
|
193
|
+
/** 凭证仅允许与首个请求同 origin 的后续跳;跨域重定向只保留安全白名单 */
|
|
132
194
|
const trustedOrigin = new URL(rawUrl).origin;
|
|
133
|
-
const SENSITIVE_HEADERS = [
|
|
134
|
-
'authorization',
|
|
135
|
-
'proxy-authorization',
|
|
136
|
-
'cookie',
|
|
137
|
-
'x-api-key',
|
|
138
|
-
'api-key',
|
|
139
|
-
'x-auth-token',
|
|
140
|
-
'x-access-token',
|
|
141
|
-
];
|
|
142
195
|
for (;;) {
|
|
143
196
|
await assertUrlAllowed(current, options);
|
|
144
|
-
const
|
|
145
|
-
const userHeaders = { ...(init?.headers ?? {}) };
|
|
146
|
-
if (!sameOrigin) {
|
|
147
|
-
for (const key of Object.keys(userHeaders)) {
|
|
148
|
-
if (SENSITIVE_HEADERS.includes(key.toLowerCase())) {
|
|
149
|
-
delete userHeaders[key];
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
}
|
|
197
|
+
const userHeaders = authHeadersForUrl(trustedOrigin, current, init?.headers);
|
|
153
198
|
const headers = {
|
|
154
199
|
'user-agent': 'octopi-knowledge-ingest/1.0',
|
|
155
200
|
accept: 'text/html,application/xhtml+xml,text/plain,text/markdown,application/json;q=0.9,*/*;q=0.1',
|
|
@@ -160,6 +205,7 @@ export async function guardedFetch(rawUrl, options, init) {
|
|
|
160
205
|
if (options?.lastModified)
|
|
161
206
|
headers['if-modified-since'] = options.lastModified;
|
|
162
207
|
const controller = new AbortController();
|
|
208
|
+
// 超时覆盖「响应头 + 响应体」全过程;只掐 headers 会让 drip body 永久占槽
|
|
163
209
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
164
210
|
let response;
|
|
165
211
|
try {
|
|
@@ -169,74 +215,73 @@ export async function guardedFetch(rawUrl, options, init) {
|
|
|
169
215
|
redirect: 'manual',
|
|
170
216
|
signal: controller.signal,
|
|
171
217
|
});
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
218
|
+
// 304 Not Modified:条件 GET 成功短路(勿进 redirect 分支)
|
|
219
|
+
if (response.status === 304) {
|
|
220
|
+
const responseHeaders = {};
|
|
221
|
+
response.headers.forEach((value, key) => {
|
|
222
|
+
responseHeaders[key.toLowerCase()] = value;
|
|
223
|
+
});
|
|
224
|
+
return {
|
|
225
|
+
url: current,
|
|
226
|
+
status: 304,
|
|
227
|
+
headers: responseHeaders,
|
|
228
|
+
body: '',
|
|
229
|
+
notModified: true,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
// 手动跟随,便于每跳 SSRF 复验
|
|
233
|
+
if (response.status >= 300 && response.status < 400) {
|
|
234
|
+
const loc = response.headers.get('location');
|
|
235
|
+
if (!loc) {
|
|
236
|
+
throw new Error(`redirect without location: ${response.status}`);
|
|
237
|
+
}
|
|
238
|
+
redirects += 1;
|
|
239
|
+
if (redirects > maxRedirects) {
|
|
240
|
+
throw new Error(`too many redirects (max ${maxRedirects})`);
|
|
241
|
+
}
|
|
242
|
+
current = new URL(loc, current).toString();
|
|
243
|
+
clearTimeout(timer);
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
180
246
|
const responseHeaders = {};
|
|
181
247
|
response.headers.forEach((value, key) => {
|
|
182
248
|
responseHeaders[key.toLowerCase()] = value;
|
|
183
249
|
});
|
|
250
|
+
const contentType = responseHeaders['content-type'];
|
|
251
|
+
if (!response.ok) {
|
|
252
|
+
throw new Error(`HTTP ${response.status} for ${current}`);
|
|
253
|
+
}
|
|
254
|
+
const reader = response.body?.getReader();
|
|
255
|
+
let body = '';
|
|
256
|
+
if (reader) {
|
|
257
|
+
const decoder = new TextDecoder();
|
|
258
|
+
let total = 0;
|
|
259
|
+
while (true) {
|
|
260
|
+
const { done, value } = await reader.read();
|
|
261
|
+
if (done)
|
|
262
|
+
break;
|
|
263
|
+
total += value.byteLength;
|
|
264
|
+
if (total > maxBytes) {
|
|
265
|
+
await reader.cancel();
|
|
266
|
+
throw new Error(`response too large (> ${maxBytes} bytes): ${current}`);
|
|
267
|
+
}
|
|
268
|
+
body += decoder.decode(value, { stream: true });
|
|
269
|
+
}
|
|
270
|
+
body += decoder.decode();
|
|
271
|
+
}
|
|
184
272
|
return {
|
|
185
273
|
url: current,
|
|
186
|
-
status:
|
|
274
|
+
status: response.status,
|
|
187
275
|
headers: responseHeaders,
|
|
188
|
-
body
|
|
189
|
-
|
|
276
|
+
body,
|
|
277
|
+
contentType,
|
|
278
|
+
etag: responseHeaders['etag'],
|
|
279
|
+
lastModified: responseHeaders['last-modified'],
|
|
280
|
+
notModified: false,
|
|
190
281
|
};
|
|
191
282
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
const loc = response.headers.get('location');
|
|
195
|
-
if (!loc) {
|
|
196
|
-
throw new Error(`redirect without location: ${response.status}`);
|
|
197
|
-
}
|
|
198
|
-
redirects += 1;
|
|
199
|
-
if (redirects > maxRedirects) {
|
|
200
|
-
throw new Error(`too many redirects (max ${maxRedirects})`);
|
|
201
|
-
}
|
|
202
|
-
current = new URL(loc, current).toString();
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
const responseHeaders = {};
|
|
206
|
-
response.headers.forEach((value, key) => {
|
|
207
|
-
responseHeaders[key.toLowerCase()] = value;
|
|
208
|
-
});
|
|
209
|
-
const contentType = responseHeaders['content-type'];
|
|
210
|
-
if (!response.ok) {
|
|
211
|
-
throw new Error(`HTTP ${response.status} for ${current}`);
|
|
212
|
-
}
|
|
213
|
-
const reader = response.body?.getReader();
|
|
214
|
-
let body = '';
|
|
215
|
-
if (reader) {
|
|
216
|
-
const decoder = new TextDecoder();
|
|
217
|
-
let total = 0;
|
|
218
|
-
while (true) {
|
|
219
|
-
const { done, value } = await reader.read();
|
|
220
|
-
if (done)
|
|
221
|
-
break;
|
|
222
|
-
total += value.byteLength;
|
|
223
|
-
if (total > maxBytes) {
|
|
224
|
-
await reader.cancel();
|
|
225
|
-
throw new Error(`response too large (> ${maxBytes} bytes): ${current}`);
|
|
226
|
-
}
|
|
227
|
-
body += decoder.decode(value, { stream: true });
|
|
228
|
-
}
|
|
229
|
-
body += decoder.decode();
|
|
283
|
+
finally {
|
|
284
|
+
clearTimeout(timer);
|
|
230
285
|
}
|
|
231
|
-
return {
|
|
232
|
-
url: current,
|
|
233
|
-
status: response.status,
|
|
234
|
-
headers: responseHeaders,
|
|
235
|
-
body,
|
|
236
|
-
contentType,
|
|
237
|
-
etag: responseHeaders['etag'],
|
|
238
|
-
lastModified: responseHeaders['last-modified'],
|
|
239
|
-
notModified: false,
|
|
240
|
-
};
|
|
241
286
|
}
|
|
242
287
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { KnowledgeChunkDraft } from './adapters.js';
|
|
2
|
+
export interface TextParseWorkerOptions {
|
|
3
|
+
/** 部分抽取时的截断阈值 */
|
|
4
|
+
maxTextChars?: number;
|
|
5
|
+
sizeDecision?: 'ok' | 'partial';
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
signal?: AbortSignal;
|
|
8
|
+
/** 文本 worker 并发上限(默认 4;轻于 SheetJS) */
|
|
9
|
+
concurrency?: number;
|
|
10
|
+
}
|
|
11
|
+
export interface TextParseResult {
|
|
12
|
+
contentHash: string;
|
|
13
|
+
contentLength: number;
|
|
14
|
+
adapterId: string;
|
|
15
|
+
chunks: KnowledgeChunkDraft[];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* 在 worker 中完成文本文件的 hash + 切块 + FTS token。
|
|
19
|
+
*
|
|
20
|
+
* @param filePath - 本地文件路径
|
|
21
|
+
* @param options - 截断策略 / 超时 / 中止 / 并发
|
|
22
|
+
* @returns contentHash + chunks(含 ftsToks)
|
|
23
|
+
*/
|
|
24
|
+
/**
|
|
25
|
+
* 已有文本内容的切块 + ftsToks(虚拟文档 / DocumentPort 回退)。
|
|
26
|
+
*
|
|
27
|
+
* @param text - UTF-8 文本
|
|
28
|
+
* @param path - 用于 adapter 启发式与 FTS 路径 token
|
|
29
|
+
* @returns chunks(含 ftsToks)
|
|
30
|
+
*/
|
|
31
|
+
export declare function chunkTextWithFtsToks(text: string, path: string, signal?: AbortSignal, adapterId?: string, timeoutMs?: number): Promise<KnowledgeChunkDraft[]>;
|
|
32
|
+
export declare function parseTextInWorker(filePath: string, options?: TextParseWorkerOptions): Promise<TextParseResult>;
|