@octopi-agent/engine 0.58.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +38 -1
  4. package/dist/config-schema/substrate.d.ts +37 -0
  5. package/dist/config-schema/substrate.js +47 -0
  6. package/dist/config.d.ts +30 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  23. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  24. package/dist/harness/capabilities/document/factory.js +37 -0
  25. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  37. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  39. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  40. package/dist/harness/capabilities/document/worker-port.js +78 -0
  41. package/dist/harness/context/capabilities/index.d.ts +2 -2
  42. package/dist/harness/context/capabilities/index.js +2 -2
  43. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  44. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
  45. package/dist/harness/index.d.ts +4 -2
  46. package/dist/harness/index.js +2 -1
  47. package/dist/harness/knowledge/adapters.d.ts +8 -5
  48. package/dist/harness/knowledge/adapters.js +53 -13
  49. package/dist/harness/knowledge/client.d.ts +197 -0
  50. package/dist/harness/knowledge/client.js +237 -0
  51. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  52. package/dist/harness/knowledge/connectors.d.ts +3 -3
  53. package/dist/harness/knowledge/connectors.js +14 -8
  54. package/dist/harness/knowledge/db.d.ts +40 -5
  55. package/dist/harness/knowledge/db.js +441 -113
  56. package/dist/harness/knowledge/document-extract-worker.js +55 -12
  57. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  58. package/dist/harness/knowledge/embed-runner.js +171 -0
  59. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  60. package/dist/harness/knowledge/engine-thread.js +302 -0
  61. package/dist/harness/knowledge/extract-document.d.ts +21 -5
  62. package/dist/harness/knowledge/extract-document.js +32 -8
  63. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  64. package/dist/harness/knowledge/fetchers.js +52 -16
  65. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  66. package/dist/harness/knowledge/file-identity.js +172 -0
  67. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  68. package/dist/harness/knowledge/file-index-store.js +132 -0
  69. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  70. package/dist/harness/knowledge/file-limits.js +175 -0
  71. package/dist/harness/knowledge/fts.d.ts +60 -0
  72. package/dist/harness/knowledge/fts.js +218 -0
  73. package/dist/harness/knowledge/grounding.d.ts +3 -2
  74. package/dist/harness/knowledge/http-app.d.ts +50 -0
  75. package/dist/harness/knowledge/http-app.js +652 -0
  76. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  77. package/dist/harness/knowledge/http-bridge.js +17 -0
  78. package/dist/harness/knowledge/index-store.d.ts +158 -93
  79. package/dist/harness/knowledge/index-store.js +982 -441
  80. package/dist/harness/knowledge/index.d.ts +45 -5
  81. package/dist/harness/knowledge/index.js +23 -2
  82. package/dist/harness/knowledge/ingest.d.ts +262 -87
  83. package/dist/harness/knowledge/ingest.js +1072 -480
  84. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  85. package/dist/harness/knowledge/job-control-state.js +44 -0
  86. package/dist/harness/knowledge/job-control.d.ts +35 -0
  87. package/dist/harness/knowledge/job-control.js +86 -0
  88. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  89. package/dist/harness/knowledge/job-queue.js +173 -0
  90. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  91. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  92. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  93. package/dist/harness/knowledge/local-stack.js +63 -0
  94. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  95. package/dist/harness/knowledge/membership-store.js +198 -0
  96. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  97. package/dist/harness/knowledge/network-guard.js +130 -85
  98. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  99. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  100. package/dist/harness/knowledge/query-service.d.ts +272 -0
  101. package/dist/harness/knowledge/query-service.js +326 -0
  102. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  103. package/dist/harness/knowledge/query-worker-client.js +207 -0
  104. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  105. package/dist/harness/knowledge/query-worker.js +186 -0
  106. package/dist/harness/knowledge/read-http.d.ts +35 -0
  107. package/dist/harness/knowledge/read-http.js +373 -0
  108. package/dist/harness/knowledge/retriever.d.ts +22 -5
  109. package/dist/harness/knowledge/retriever.js +66 -35
  110. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  111. package/dist/harness/knowledge/secret-scan.js +25 -2
  112. package/dist/harness/knowledge/serve.d.ts +48 -0
  113. package/dist/harness/knowledge/serve.js +336 -0
  114. package/dist/harness/knowledge/source-store.d.ts +80 -17
  115. package/dist/harness/knowledge/source-store.js +160 -89
  116. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  117. package/dist/harness/knowledge/start-service-process.js +158 -0
  118. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  119. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  120. package/dist/harness/knowledge/types.d.ts +1 -1
  121. package/dist/harness/knowledge/types.js +1 -1
  122. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  123. package/dist/harness/knowledge/vector-ann.js +68 -0
  124. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  125. package/dist/harness/knowledge/writer-lock.js +71 -0
  126. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  127. package/dist/harness/knowledge/writer-service.js +355 -0
  128. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  129. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  130. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  131. package/dist/harness/knowledge/writer-worker.js +189 -0
  132. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  133. package/dist/harness/memory/sqlite/embedding.js +109 -16
  134. package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
  135. package/dist/harness/memory/sqlite/keyword-search.js +64 -0
  136. package/dist/harness/session/attachments/service.d.ts +1 -1
  137. package/dist/harness/session/attachments/service.js +1 -1
  138. package/dist/index.d.ts +4 -2
  139. package/dist/index.js +2 -1
  140. package/dist/worker-path.d.ts +8 -0
  141. package/dist/worker-path.js +15 -0
  142. package/package.json +2 -2
@@ -0,0 +1,65 @@
1
+ /**
2
+ * MembershipStore — Source↔File 认领 + 零认领 purge(契约 §3.5 / §6.2)
3
+ */
4
+ import type { KnowledgeDatabase } from './db.js';
5
+ import type { FileIdentity } from './file-identity.js';
6
+ export interface MembershipRow {
7
+ sourceId: string;
8
+ fileId: string;
9
+ logicalPath: string;
10
+ createdAt: number;
11
+ }
12
+ export interface FileRow {
13
+ id: string;
14
+ identityKey: string;
15
+ size: number;
16
+ mtime: number;
17
+ contentHash: string | null;
18
+ status: string;
19
+ chunkCount: number;
20
+ }
21
+ export interface ReconcileResult {
22
+ added: number;
23
+ removed: number;
24
+ purgedFiles: number;
25
+ }
26
+ export declare class MembershipStore {
27
+ private readonly db;
28
+ constructor(db: KnowledgeDatabase);
29
+ /**
30
+ * 解析/登记物理 File;版本变化时更新 version 并标记 needsParse。
31
+ */
32
+ upsertByIdentity(identity: FileIdentity, opts?: {
33
+ tenantId?: string;
34
+ }): {
35
+ fileId: string;
36
+ created: boolean;
37
+ versionChanged: boolean;
38
+ };
39
+ claim(sourceId: string, fileId: string, logicalPath: string): void;
40
+ unclaim(sourceId: string, logicalPath: string): string | null;
41
+ claimCount(fileId: string): number;
42
+ listBySource(sourceId: string): MembershipRow[];
43
+ listByFile(fileId: string): MembershipRow[];
44
+ /**
45
+ * 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
46
+ * 大源删除分批让出,避免同步连环 purge 堵死 Engine。
47
+ *
48
+ * @returns purge 的 fileId 列表
49
+ */
50
+ unclaimAllForSourceAsync(sourceId: string, purge: (fileId: string) => void | Promise<void>, opts?: {
51
+ yieldEvery?: number;
52
+ }): Promise<string[]>;
53
+ /**
54
+ * 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
55
+ * 返回 purge 的 fileId 列表。
56
+ */
57
+ unclaimAllForSource(sourceId: string, purge: (fileId: string) => void): string[];
58
+ /**
59
+ * Walk 对账(§6.2):discovered 为 logical_path → fileId。
60
+ * 删掉本 Source 不在 discovered 中的 Membership;零认领则 purge。
61
+ */
62
+ reconcileSource(sourceId: string, discovered: Map<string, string>, purge: (fileId: string) => void): ReconcileResult;
63
+ getFile(fileId: string): FileRow | null;
64
+ findFileByIdentity(identityKey: string, tenantId?: string): FileRow | null;
65
+ }
@@ -0,0 +1,198 @@
1
+ /**
2
+ * MembershipStore — Source↔File 认领 + 零认领 purge(契约 §3.5 / §6.2)
3
+ */
4
+ function newId(prefix) {
5
+ return `${prefix}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`;
6
+ }
7
+ export class MembershipStore {
8
+ db;
9
+ constructor(db) {
10
+ this.db = db;
11
+ }
12
+ /**
13
+ * 解析/登记物理 File;版本变化时更新 version 并标记 needsParse。
14
+ */
15
+ upsertByIdentity(identity, opts) {
16
+ const tenant = opts?.tenantId ?? 'default';
17
+ const now = Date.now();
18
+ const row = this.db.raw
19
+ .prepare(`SELECT id, size, mtime FROM knowledge_files
20
+ WHERE tenant_id = ? AND identity_key = ?`)
21
+ .get(tenant, identity.key);
22
+ if (row) {
23
+ const versionChanged = row.size !== identity.size || row.mtime !== identity.mtime;
24
+ if (versionChanged) {
25
+ this.db.raw
26
+ .prepare(`UPDATE knowledge_files SET size = ?, mtime = ?, indexed_at = ? WHERE id = ?`)
27
+ .run(identity.size, identity.mtime, now, row.id);
28
+ }
29
+ return { fileId: row.id, created: false, versionChanged };
30
+ }
31
+ const fileId = newId('file');
32
+ this.db.raw
33
+ .prepare(`INSERT INTO knowledge_files
34
+ (id, tenant_id, identity_key, size, mtime, content_hash, status, chunk_count, indexed_at)
35
+ VALUES (?, ?, ?, ?, ?, NULL, 'pending', 0, ?)`)
36
+ .run(fileId, tenant, identity.key, identity.size, identity.mtime, now);
37
+ return { fileId, created: true, versionChanged: true };
38
+ }
39
+ claim(sourceId, fileId, logicalPath) {
40
+ const prev = this.db.raw
41
+ .prepare(`SELECT file_id FROM knowledge_memberships
42
+ WHERE source_id = ? AND logical_path = ?`)
43
+ .get(sourceId, logicalPath);
44
+ this.db.raw
45
+ .prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
46
+ VALUES (?, ?, ?, ?)
47
+ ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
48
+ .run(sourceId, fileId, logicalPath, Date.now());
49
+ // 换绑后旧 File 若零认领,由调用方 purge(此处只记孤儿风险)
50
+ if (prev && prev.file_id !== fileId && this.claimCount(prev.file_id) === 0) {
51
+ // 留给 reconcile 补 purge;避免 claim 路径上嵌套删除事务
52
+ }
53
+ }
54
+ unclaim(sourceId, logicalPath) {
55
+ const row = this.db.raw
56
+ .prepare(`SELECT file_id FROM knowledge_memberships
57
+ WHERE source_id = ? AND logical_path = ?`)
58
+ .get(sourceId, logicalPath);
59
+ this.db.raw
60
+ .prepare(`DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
61
+ .run(sourceId, logicalPath);
62
+ return row?.file_id ?? null;
63
+ }
64
+ claimCount(fileId) {
65
+ const r = this.db.raw
66
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?`)
67
+ .get(fileId);
68
+ return r.n;
69
+ }
70
+ listBySource(sourceId) {
71
+ const rows = this.db.raw
72
+ .prepare(`SELECT source_id, file_id, logical_path, created_at
73
+ FROM knowledge_memberships WHERE source_id = ? ORDER BY logical_path`)
74
+ .all(sourceId);
75
+ return rows.map((r) => ({
76
+ sourceId: r.source_id,
77
+ fileId: r.file_id,
78
+ logicalPath: r.logical_path,
79
+ createdAt: r.created_at,
80
+ }));
81
+ }
82
+ listByFile(fileId) {
83
+ const rows = this.db.raw
84
+ .prepare(`SELECT source_id, file_id, logical_path, created_at
85
+ FROM knowledge_memberships WHERE file_id = ?`)
86
+ .all(fileId);
87
+ return rows.map((r) => ({
88
+ sourceId: r.source_id,
89
+ fileId: r.file_id,
90
+ logicalPath: r.logical_path,
91
+ createdAt: r.created_at,
92
+ }));
93
+ }
94
+ /**
95
+ * 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
96
+ * 大源删除分批让出,避免同步连环 purge 堵死 Engine。
97
+ *
98
+ * @returns purge 的 fileId 列表
99
+ */
100
+ async unclaimAllForSourceAsync(sourceId, purge, opts) {
101
+ const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
102
+ const files = this.db.raw
103
+ .prepare(`SELECT DISTINCT file_id FROM knowledge_memberships WHERE source_id = ?`)
104
+ .all(sourceId);
105
+ this.db.raw.prepare(`DELETE FROM knowledge_memberships WHERE source_id = ?`).run(sourceId);
106
+ const purged = [];
107
+ for (let i = 0; i < files.length; i++) {
108
+ const f = files[i];
109
+ if (this.claimCount(f.file_id) === 0) {
110
+ await purge(f.file_id);
111
+ purged.push(f.file_id);
112
+ }
113
+ if ((i + 1) % yieldEvery === 0) {
114
+ await new Promise((r) => setImmediate(r));
115
+ }
116
+ }
117
+ return purged;
118
+ }
119
+ /**
120
+ * 解绑 Source:删其全部 Membership,并对失去认领的 File 做 purge。
121
+ * 返回 purge 的 fileId 列表。
122
+ */
123
+ unclaimAllForSource(sourceId, purge) {
124
+ const files = this.db.raw
125
+ .prepare(`SELECT DISTINCT file_id FROM knowledge_memberships WHERE source_id = ?`)
126
+ .all(sourceId);
127
+ this.db.raw.prepare(`DELETE FROM knowledge_memberships WHERE source_id = ?`).run(sourceId);
128
+ const purged = [];
129
+ for (const f of files) {
130
+ if (this.claimCount(f.file_id) === 0) {
131
+ purge(f.file_id);
132
+ purged.push(f.file_id);
133
+ }
134
+ }
135
+ return purged;
136
+ }
137
+ /**
138
+ * Walk 对账(§6.2):discovered 为 logical_path → fileId。
139
+ * 删掉本 Source 不在 discovered 中的 Membership;零认领则 purge。
140
+ */
141
+ reconcileSource(sourceId, discovered, purge) {
142
+ const now = Date.now();
143
+ let added = 0;
144
+ for (const [logicalPath, fileId] of discovered) {
145
+ this.claim(sourceId, fileId, logicalPath);
146
+ added += 1;
147
+ }
148
+ const existing = this.listBySource(sourceId);
149
+ let removed = 0;
150
+ let purgedFiles = 0;
151
+ for (const m of existing) {
152
+ if (discovered.has(m.logicalPath))
153
+ continue;
154
+ this.unclaim(sourceId, m.logicalPath);
155
+ removed += 1;
156
+ if (this.claimCount(m.fileId) === 0) {
157
+ purge(m.fileId);
158
+ purgedFiles += 1;
159
+ }
160
+ }
161
+ void now;
162
+ return { added, removed, purgedFiles };
163
+ }
164
+ getFile(fileId) {
165
+ const r = this.db.raw
166
+ .prepare(`SELECT id, identity_key, size, mtime, content_hash, status, chunk_count
167
+ FROM knowledge_files WHERE id = ?`)
168
+ .get(fileId);
169
+ if (!r)
170
+ return null;
171
+ return {
172
+ id: r.id,
173
+ identityKey: r.identity_key,
174
+ size: r.size,
175
+ mtime: r.mtime,
176
+ contentHash: r.content_hash,
177
+ status: r.status,
178
+ chunkCount: r.chunk_count,
179
+ };
180
+ }
181
+ findFileByIdentity(identityKey, tenantId = 'default') {
182
+ const r = this.db.raw
183
+ .prepare(`SELECT id, identity_key, size, mtime, content_hash, status, chunk_count
184
+ FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?`)
185
+ .get(tenantId, identityKey);
186
+ if (!r)
187
+ return null;
188
+ return {
189
+ id: r.id,
190
+ identityKey: r.identity_key,
191
+ size: r.size,
192
+ mtime: r.mtime,
193
+ contentHash: r.content_hash,
194
+ status: r.status,
195
+ chunkCount: r.chunk_count,
196
+ };
197
+ }
198
+ }
@@ -2,7 +2,7 @@
2
2
  * 出站网络门禁 — Knowledge 外源 ingest SSRF 防护
3
3
  *
4
4
  * 默认拒绝私网/环回/link-local/云 metadata;源级 allowPrivateNetwork 显式放行。
5
- * 重定向每跳复验;DNS 解析后校验目标 IP(防 rebinding)。
5
+ * 重定向上每跳复验地址;**不**做 IP 钉扎——DNS TOCTOU 见 docs/KNOWN-ISSUES。
6
6
  */
7
7
  export interface NetworkGuardOptions {
8
8
  /** 放行私网/环回(源级显式开启) */
@@ -28,7 +28,16 @@ export interface FetchResult {
28
28
  notModified: boolean;
29
29
  }
30
30
  /**
31
- * 判断 IP 是否属于受限地址(私网/环回/metadata)
31
+ * 凭证只属于「源入口 origin」。远端内容指向的任何其他 origin 一律不带鉴权头。
32
+ *
33
+ * @param trustedOrigin - 源入口 URL 的 origin(listUrl / source.location)
34
+ * @param targetUrl - 本次实际请求的 URL
35
+ * @param headers - 候选请求头(含凭证)
36
+ * @returns 同 origin 时原样返回;跨 origin 时只保留安全白名单
37
+ */
38
+ export declare function authHeadersForUrl(trustedOrigin: string, targetUrl: string, headers?: Record<string, string>): Record<string, string>;
39
+ /**
40
+ * 判断 IP 是否属于受限地址(私网/环回/metadata/未指定)
32
41
  *
33
42
  * @param ip - IPv4/IPv6 字面量
34
43
  */
@@ -2,13 +2,48 @@
2
2
  * 出站网络门禁 — Knowledge 外源 ingest SSRF 防护
3
3
  *
4
4
  * 默认拒绝私网/环回/link-local/云 metadata;源级 allowPrivateNetwork 显式放行。
5
- * 重定向每跳复验;DNS 解析后校验目标 IP(防 rebinding)。
5
+ * 重定向上每跳复验地址;**不**做 IP 钉扎——DNS TOCTOU 见 docs/KNOWN-ISSUES。
6
6
  */
7
7
  import { lookup } from 'node:dns/promises';
8
8
  import { isIP } from 'node:net';
9
9
  const DEFAULT_MAX_REDIRECTS = 5;
10
10
  const DEFAULT_TIMEOUT_MS = 30_000;
11
11
  const DEFAULT_MAX_BYTES = 5_000_000;
12
+ /** 跨 origin 时唯一允许携带的请求头(白名单;其余含凭证一律剥离) */
13
+ const SAFE_CROSS_ORIGIN_HEADERS = new Set([
14
+ 'accept',
15
+ 'accept-language',
16
+ 'user-agent',
17
+ 'if-none-match',
18
+ 'if-modified-since',
19
+ ]);
20
+ /**
21
+ * 凭证只属于「源入口 origin」。远端内容指向的任何其他 origin 一律不带鉴权头。
22
+ *
23
+ * @param trustedOrigin - 源入口 URL 的 origin(listUrl / source.location)
24
+ * @param targetUrl - 本次实际请求的 URL
25
+ * @param headers - 候选请求头(含凭证)
26
+ * @returns 同 origin 时原样返回;跨 origin 时只保留安全白名单
27
+ */
28
+ export function authHeadersForUrl(trustedOrigin, targetUrl, headers) {
29
+ if (!headers)
30
+ return {};
31
+ let sameOrigin = false;
32
+ try {
33
+ sameOrigin = new URL(targetUrl).origin === trustedOrigin;
34
+ }
35
+ catch {
36
+ sameOrigin = false;
37
+ }
38
+ if (sameOrigin)
39
+ return { ...headers };
40
+ const out = {};
41
+ for (const [k, v] of Object.entries(headers)) {
42
+ if (SAFE_CROSS_ORIGIN_HEADERS.has(k.toLowerCase()))
43
+ out[k] = v;
44
+ }
45
+ return out;
46
+ }
12
47
  /** 云 metadata / 链路本地 */
13
48
  function isLinkLocalOrMetadata(ip) {
14
49
  const v = ip.toLowerCase();
@@ -24,11 +59,19 @@ function isLoopback(ip) {
24
59
  const v = ip.toLowerCase();
25
60
  return v === '::1' || v === '0:0:0:0:0:0:0:1' || v.startsWith('127.');
26
61
  }
62
+ /** 未指定地址(:: / 0.0.0.0):连出去通常等价本机 */
63
+ function isUnspecified(ip) {
64
+ const v = ip.toLowerCase().replace(/^\[|\]$/g, '');
65
+ return v === '0.0.0.0' || v === '::' || v === '0:0:0:0:0:0:0:0';
66
+ }
27
67
  function isPrivateIPv4(ip) {
28
68
  const parts = ip.split('.').map(Number);
29
69
  if (parts.length !== 4 || parts.some((n) => Number.isNaN(n)))
30
70
  return false;
31
71
  const [a, b] = parts;
72
+ // 0.0.0.0/8 「this network」;0.0.0.0 连出常落本机
73
+ if (a === 0)
74
+ return true;
32
75
  if (a === 10)
33
76
  return true;
34
77
  if (a === 172 && b >= 16 && b <= 31)
@@ -40,18 +83,37 @@ function isPrivateIPv4(ip) {
40
83
  return true;
41
84
  return false;
42
85
  }
86
+ /**
87
+ * IPv4-mapped IPv6 → 点分 IPv4(含 `::ffff:7f00:1` 十六进制形态)。
88
+ * 解不出则 null。
89
+ */
90
+ function extractIPv4Mapped(ip) {
91
+ const v = ip.toLowerCase().replace(/^\[|\]$/g, '');
92
+ if (!v.startsWith('::ffff:'))
93
+ return null;
94
+ const rest = v.slice(7);
95
+ if (rest.includes('.'))
96
+ return rest;
97
+ const parts = rest.split(':');
98
+ if (parts.length !== 2)
99
+ return null;
100
+ const hi = Number.parseInt(parts[0], 16);
101
+ const lo = Number.parseInt(parts[1], 16);
102
+ if (!Number.isFinite(hi) || !Number.isFinite(lo))
103
+ return null;
104
+ return `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
105
+ }
43
106
  function isPrivateIPv6(ip) {
44
107
  const v = ip.toLowerCase().replace(/^\[|\]$/g, '');
45
108
  if (v.startsWith('fc') || v.startsWith('fd'))
46
109
  return true; // ULA
47
- if (v.startsWith('::ffff:')) {
48
- const mapped = v.slice(7);
49
- return isPrivateIPv4(mapped) || isLoopback(mapped);
50
- }
110
+ const mapped = extractIPv4Mapped(v);
111
+ if (mapped)
112
+ return isPrivateIPv4(mapped) || isLoopback(mapped) || isUnspecified(mapped);
51
113
  return false;
52
114
  }
53
115
  /**
54
- * 判断 IP 是否属于受限地址(私网/环回/metadata)
116
+ * 判断 IP 是否属于受限地址(私网/环回/metadata/未指定)
55
117
  *
56
118
  * @param ip - IPv4/IPv6 字面量
57
119
  */
@@ -59,7 +121,7 @@ export function isRestrictedIp(ip) {
59
121
  const v = ip.trim().toLowerCase().replace(/^\[|\]$/g, '');
60
122
  if (!v)
61
123
  return true;
62
- if (isLoopback(v) || isLinkLocalOrMetadata(v))
124
+ if (isUnspecified(v) || isLoopback(v) || isLinkLocalOrMetadata(v))
63
125
  return true;
64
126
  if (isIP(v) === 4)
65
127
  return isPrivateIPv4(v);
@@ -128,28 +190,11 @@ export async function guardedFetch(rawUrl, options, init) {
128
190
  const maxBytes = options?.maxResponseBytes ?? DEFAULT_MAX_BYTES;
129
191
  let current = rawUrl;
130
192
  let redirects = 0;
131
- /** 凭证仅允许与首个请求同 origin 的后续跳;跨域重定向剥离敏感头 */
193
+ /** 凭证仅允许与首个请求同 origin 的后续跳;跨域重定向只保留安全白名单 */
132
194
  const trustedOrigin = new URL(rawUrl).origin;
133
- const SENSITIVE_HEADERS = [
134
- 'authorization',
135
- 'proxy-authorization',
136
- 'cookie',
137
- 'x-api-key',
138
- 'api-key',
139
- 'x-auth-token',
140
- 'x-access-token',
141
- ];
142
195
  for (;;) {
143
196
  await assertUrlAllowed(current, options);
144
- const sameOrigin = new URL(current).origin === trustedOrigin;
145
- const userHeaders = { ...(init?.headers ?? {}) };
146
- if (!sameOrigin) {
147
- for (const key of Object.keys(userHeaders)) {
148
- if (SENSITIVE_HEADERS.includes(key.toLowerCase())) {
149
- delete userHeaders[key];
150
- }
151
- }
152
- }
197
+ const userHeaders = authHeadersForUrl(trustedOrigin, current, init?.headers);
153
198
  const headers = {
154
199
  'user-agent': 'octopi-knowledge-ingest/1.0',
155
200
  accept: 'text/html,application/xhtml+xml,text/plain,text/markdown,application/json;q=0.9,*/*;q=0.1',
@@ -160,6 +205,7 @@ export async function guardedFetch(rawUrl, options, init) {
160
205
  if (options?.lastModified)
161
206
  headers['if-modified-since'] = options.lastModified;
162
207
  const controller = new AbortController();
208
+ // 超时覆盖「响应头 + 响应体」全过程;只掐 headers 会让 drip body 永久占槽
163
209
  const timer = setTimeout(() => controller.abort(), timeoutMs);
164
210
  let response;
165
211
  try {
@@ -169,74 +215,73 @@ export async function guardedFetch(rawUrl, options, init) {
169
215
  redirect: 'manual',
170
216
  signal: controller.signal,
171
217
  });
172
- }
173
- catch (err) {
174
- clearTimeout(timer);
175
- throw err;
176
- }
177
- clearTimeout(timer);
178
- // 304 Not Modified:条件 GET 成功短路(勿进 redirect 分支)
179
- if (response.status === 304) {
218
+ // 304 Not Modified:条件 GET 成功短路(勿进 redirect 分支)
219
+ if (response.status === 304) {
220
+ const responseHeaders = {};
221
+ response.headers.forEach((value, key) => {
222
+ responseHeaders[key.toLowerCase()] = value;
223
+ });
224
+ return {
225
+ url: current,
226
+ status: 304,
227
+ headers: responseHeaders,
228
+ body: '',
229
+ notModified: true,
230
+ };
231
+ }
232
+ // 手动跟随,便于每跳 SSRF 复验
233
+ if (response.status >= 300 && response.status < 400) {
234
+ const loc = response.headers.get('location');
235
+ if (!loc) {
236
+ throw new Error(`redirect without location: ${response.status}`);
237
+ }
238
+ redirects += 1;
239
+ if (redirects > maxRedirects) {
240
+ throw new Error(`too many redirects (max ${maxRedirects})`);
241
+ }
242
+ current = new URL(loc, current).toString();
243
+ clearTimeout(timer);
244
+ continue;
245
+ }
180
246
  const responseHeaders = {};
181
247
  response.headers.forEach((value, key) => {
182
248
  responseHeaders[key.toLowerCase()] = value;
183
249
  });
250
+ const contentType = responseHeaders['content-type'];
251
+ if (!response.ok) {
252
+ throw new Error(`HTTP ${response.status} for ${current}`);
253
+ }
254
+ const reader = response.body?.getReader();
255
+ let body = '';
256
+ if (reader) {
257
+ const decoder = new TextDecoder();
258
+ let total = 0;
259
+ while (true) {
260
+ const { done, value } = await reader.read();
261
+ if (done)
262
+ break;
263
+ total += value.byteLength;
264
+ if (total > maxBytes) {
265
+ await reader.cancel();
266
+ throw new Error(`response too large (> ${maxBytes} bytes): ${current}`);
267
+ }
268
+ body += decoder.decode(value, { stream: true });
269
+ }
270
+ body += decoder.decode();
271
+ }
184
272
  return {
185
273
  url: current,
186
- status: 304,
274
+ status: response.status,
187
275
  headers: responseHeaders,
188
- body: '',
189
- notModified: true,
276
+ body,
277
+ contentType,
278
+ etag: responseHeaders['etag'],
279
+ lastModified: responseHeaders['last-modified'],
280
+ notModified: false,
190
281
  };
191
282
  }
192
- // 手动跟随,便于每跳 SSRF 复验
193
- if (response.status >= 300 && response.status < 400) {
194
- const loc = response.headers.get('location');
195
- if (!loc) {
196
- throw new Error(`redirect without location: ${response.status}`);
197
- }
198
- redirects += 1;
199
- if (redirects > maxRedirects) {
200
- throw new Error(`too many redirects (max ${maxRedirects})`);
201
- }
202
- current = new URL(loc, current).toString();
203
- continue;
204
- }
205
- const responseHeaders = {};
206
- response.headers.forEach((value, key) => {
207
- responseHeaders[key.toLowerCase()] = value;
208
- });
209
- const contentType = responseHeaders['content-type'];
210
- if (!response.ok) {
211
- throw new Error(`HTTP ${response.status} for ${current}`);
212
- }
213
- const reader = response.body?.getReader();
214
- let body = '';
215
- if (reader) {
216
- const decoder = new TextDecoder();
217
- let total = 0;
218
- while (true) {
219
- const { done, value } = await reader.read();
220
- if (done)
221
- break;
222
- total += value.byteLength;
223
- if (total > maxBytes) {
224
- await reader.cancel();
225
- throw new Error(`response too large (> ${maxBytes} bytes): ${current}`);
226
- }
227
- body += decoder.decode(value, { stream: true });
228
- }
229
- body += decoder.decode();
283
+ finally {
284
+ clearTimeout(timer);
230
285
  }
231
- return {
232
- url: current,
233
- status: response.status,
234
- headers: responseHeaders,
235
- body,
236
- contentType,
237
- etag: responseHeaders['etag'],
238
- lastModified: responseHeaders['last-modified'],
239
- notModified: false,
240
- };
241
286
  }
242
287
  }
@@ -0,0 +1,32 @@
1
+ import type { KnowledgeChunkDraft } from './adapters.js';
2
+ export interface TextParseWorkerOptions {
3
+ /** 部分抽取时的截断阈值 */
4
+ maxTextChars?: number;
5
+ sizeDecision?: 'ok' | 'partial';
6
+ timeoutMs?: number;
7
+ signal?: AbortSignal;
8
+ /** 文本 worker 并发上限(默认 4;轻于 SheetJS) */
9
+ concurrency?: number;
10
+ }
11
+ export interface TextParseResult {
12
+ contentHash: string;
13
+ contentLength: number;
14
+ adapterId: string;
15
+ chunks: KnowledgeChunkDraft[];
16
+ }
17
+ /**
18
+ * 在 worker 中完成文本文件的 hash + 切块 + FTS token。
19
+ *
20
+ * @param filePath - 本地文件路径
21
+ * @param options - 截断策略 / 超时 / 中止 / 并发
22
+ * @returns contentHash + chunks(含 ftsToks)
23
+ */
24
+ /**
25
+ * 已有文本内容的切块 + ftsToks(虚拟文档 / DocumentPort 回退)。
26
+ *
27
+ * @param text - UTF-8 文本
28
+ * @param path - 用于 adapter 启发式与 FTS 路径 token
29
+ * @returns chunks(含 ftsToks)
30
+ */
31
+ export declare function chunkTextWithFtsToks(text: string, path: string, signal?: AbortSignal, adapterId?: string, timeoutMs?: number): Promise<KnowledgeChunkDraft[]>;
32
+ export declare function parseTextInWorker(filePath: string, options?: TextParseWorkerOptions): Promise<TextParseResult>;