@octopi-agent/engine 0.56.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +43 -1
  4. package/dist/config-schema/substrate.d.ts +42 -0
  5. package/dist/config-schema/substrate.js +58 -0
  6. package/dist/config.d.ts +39 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/agent/config-bridge.js +3 -0
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  23. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  24. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  25. package/dist/harness/capabilities/document/factory.js +37 -0
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  37. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  39. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  40. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  41. package/dist/harness/capabilities/document/worker-port.js +78 -0
  42. package/dist/harness/context/capabilities/index.d.ts +2 -2
  43. package/dist/harness/context/capabilities/index.js +2 -2
  44. package/dist/harness/context/constitution/default-agents.md +1 -0
  45. package/dist/harness/context/index.d.ts +2 -2
  46. package/dist/harness/context/index.js +1 -1
  47. package/dist/harness/context/layer-types.d.ts +26 -2
  48. package/dist/harness/context/layer-types.js +59 -11
  49. package/dist/harness/context/layers.d.ts +3 -0
  50. package/dist/harness/context/layers.js +38 -12
  51. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  52. package/dist/harness/context/system-prompt-assembler.js +2 -0
  53. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  54. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
  55. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  56. package/dist/harness/index.d.ts +5 -3
  57. package/dist/harness/index.js +3 -2
  58. package/dist/harness/knowledge/adapters.d.ts +8 -5
  59. package/dist/harness/knowledge/adapters.js +53 -13
  60. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  61. package/dist/harness/knowledge/client.d.ts +197 -0
  62. package/dist/harness/knowledge/client.js +237 -0
  63. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  64. package/dist/harness/knowledge/connectors.d.ts +3 -3
  65. package/dist/harness/knowledge/connectors.js +14 -8
  66. package/dist/harness/knowledge/db.d.ts +42 -3
  67. package/dist/harness/knowledge/db.js +466 -83
  68. package/dist/harness/knowledge/describe.d.ts +2 -0
  69. package/dist/harness/knowledge/describe.js +5 -1
  70. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  71. package/dist/harness/knowledge/document-extract-worker.js +68 -0
  72. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  73. package/dist/harness/knowledge/embed-runner.js +171 -0
  74. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  75. package/dist/harness/knowledge/engine-thread.js +302 -0
  76. package/dist/harness/knowledge/extract-document.d.ts +33 -0
  77. package/dist/harness/knowledge/extract-document.js +86 -0
  78. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  79. package/dist/harness/knowledge/fetchers.js +52 -16
  80. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  81. package/dist/harness/knowledge/file-identity.js +172 -0
  82. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  83. package/dist/harness/knowledge/file-index-store.js +132 -0
  84. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  85. package/dist/harness/knowledge/file-limits.js +175 -0
  86. package/dist/harness/knowledge/fts.d.ts +60 -0
  87. package/dist/harness/knowledge/fts.js +218 -0
  88. package/dist/harness/knowledge/grounding.d.ts +3 -2
  89. package/dist/harness/knowledge/http-app.d.ts +50 -0
  90. package/dist/harness/knowledge/http-app.js +652 -0
  91. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  92. package/dist/harness/knowledge/http-bridge.js +17 -0
  93. package/dist/harness/knowledge/index-store.d.ts +166 -63
  94. package/dist/harness/knowledge/index-store.js +1025 -282
  95. package/dist/harness/knowledge/index.d.ts +47 -5
  96. package/dist/harness/knowledge/index.js +24 -2
  97. package/dist/harness/knowledge/ingest.d.ts +322 -49
  98. package/dist/harness/knowledge/ingest.js +1383 -267
  99. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  100. package/dist/harness/knowledge/job-control-state.js +44 -0
  101. package/dist/harness/knowledge/job-control.d.ts +35 -0
  102. package/dist/harness/knowledge/job-control.js +86 -0
  103. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  104. package/dist/harness/knowledge/job-queue.js +173 -0
  105. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  106. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  107. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  108. package/dist/harness/knowledge/local-stack.js +63 -0
  109. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  110. package/dist/harness/knowledge/membership-store.js +198 -0
  111. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  112. package/dist/harness/knowledge/network-guard.js +130 -85
  113. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  114. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  115. package/dist/harness/knowledge/query-service.d.ts +272 -0
  116. package/dist/harness/knowledge/query-service.js +326 -0
  117. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  118. package/dist/harness/knowledge/query-worker-client.js +207 -0
  119. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  120. package/dist/harness/knowledge/query-worker.js +186 -0
  121. package/dist/harness/knowledge/read-http.d.ts +35 -0
  122. package/dist/harness/knowledge/read-http.js +373 -0
  123. package/dist/harness/knowledge/retriever.d.ts +30 -5
  124. package/dist/harness/knowledge/retriever.js +98 -35
  125. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  126. package/dist/harness/knowledge/secret-scan.js +25 -2
  127. package/dist/harness/knowledge/serve.d.ts +48 -0
  128. package/dist/harness/knowledge/serve.js +336 -0
  129. package/dist/harness/knowledge/source-store.d.ts +83 -16
  130. package/dist/harness/knowledge/source-store.js +205 -78
  131. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  132. package/dist/harness/knowledge/start-service-process.js +158 -0
  133. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  134. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  135. package/dist/harness/knowledge/topics.d.ts +13 -0
  136. package/dist/harness/knowledge/topics.js +48 -0
  137. package/dist/harness/knowledge/types.d.ts +1 -1
  138. package/dist/harness/knowledge/types.js +1 -1
  139. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  140. package/dist/harness/knowledge/vector-ann.js +68 -0
  141. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  142. package/dist/harness/knowledge/writer-lock.js +71 -0
  143. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  144. package/dist/harness/knowledge/writer-service.js +355 -0
  145. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  146. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  147. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  148. package/dist/harness/knowledge/writer-worker.js +189 -0
  149. package/dist/harness/memory/index.d.ts +3 -2
  150. package/dist/harness/memory/index.js +3 -1
  151. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  152. package/dist/harness/memory/retrieval-rank.js +60 -0
  153. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  154. package/dist/harness/memory/sqlite/embedding.js +109 -16
  155. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  156. package/dist/harness/memory/sqlite/index.js +1 -1
  157. package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
  158. package/dist/harness/memory/sqlite/keyword-search.js +104 -17
  159. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  160. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  161. package/dist/harness/memory/store.js +39 -21
  162. package/dist/harness/memory/types.d.ts +7 -0
  163. package/dist/harness/run/runner.d.ts +1 -0
  164. package/dist/harness/run/runner.js +4 -2
  165. package/dist/harness/session/attachments/service.d.ts +1 -1
  166. package/dist/harness/session/attachments/service.js +1 -1
  167. package/dist/harness/session/tasks/index.d.ts +1 -1
  168. package/dist/harness/session/tasks/index.js +1 -1
  169. package/dist/harness/session/tasks/render.d.ts +10 -0
  170. package/dist/harness/session/tasks/render.js +27 -0
  171. package/dist/index.d.ts +4 -2
  172. package/dist/index.js +2 -1
  173. package/dist/worker-path.d.ts +8 -0
  174. package/dist/worker-path.js +15 -0
  175. package/package.json +2 -2
@@ -0,0 +1,86 @@
1
+ /**
2
+ * extractDocumentInWorker — 线程内文档抽取 + 切块 + FTS token
3
+ */
4
+ import { Worker } from 'node:worker_threads';
5
+ import { DocumentExtractError } from '../capabilities/document/errors.js';
6
+ import { resolveWorkerUrl } from '../../worker-path.js';
7
+ /** postMessage 不保留 class 原型;按 name/code 还原 DocumentExtractError */
8
+ function rehydrateWorkerError(payload) {
9
+ if (typeof payload === 'string')
10
+ return new Error(payload);
11
+ if (payload && typeof payload === 'object' && 'message' in payload) {
12
+ const e = payload;
13
+ if (e.name === 'DocumentExtractError' && e.code) {
14
+ return new DocumentExtractError(e.code, e.message, e.requires);
15
+ }
16
+ return new Error(e.message);
17
+ }
18
+ return new Error('extract_worker_failed');
19
+ }
20
+ /**
21
+ * 在 worker 线程抽取文档为 Markdown,并完成切块 + FTS token。
22
+ *
23
+ * @param filePath - 本地文件路径
24
+ * @param options - 超时 / 大小 / 中止 / 部分抽取限额
25
+ * @returns ExtractResult + chunks
26
+ * @throws Error 超时 / abort / worker 失败 / 抽取错误
27
+ */
28
+ export function extractDocumentInWorker(filePath, options = {}) {
29
+ const timeoutMs = options.timeoutMs ?? 60_000;
30
+ return new Promise((resolve, reject) => {
31
+ if (options.signal?.aborted) {
32
+ reject(new Error('extract_aborted'));
33
+ return;
34
+ }
35
+ let settled = false;
36
+ const worker = new Worker(resolveWorkerUrl('./document-extract-worker.js', import.meta.url), {
37
+ workerData: {
38
+ path: filePath,
39
+ name: filePath,
40
+ timeoutMs,
41
+ maxFileBytes: options.maxFileBytes,
42
+ maxSheets: options.maxSheets,
43
+ maxRowsPerSheet: options.maxRowsPerSheet,
44
+ maxPages: options.maxPages,
45
+ maxTextChars: options.maxTextChars,
46
+ documentConfig: options.documentConfig ?? null,
47
+ },
48
+ });
49
+ const finish = (fn) => {
50
+ if (settled)
51
+ return;
52
+ settled = true;
53
+ clearTimeout(timer);
54
+ options.signal?.removeEventListener('abort', onAbort);
55
+ void worker.terminate();
56
+ fn();
57
+ };
58
+ const onAbort = () => {
59
+ finish(() => reject(new Error('extract_aborted')));
60
+ };
61
+ options.signal?.addEventListener('abort', onAbort, { once: true });
62
+ const timer = setTimeout(() => {
63
+ finish(() => reject(new Error(`extract_worker_timeout after ${timeoutMs}ms`)));
64
+ }, timeoutMs);
65
+ worker.on('message', (msg) => {
66
+ if (msg?.ok && msg.result && msg.contentHash) {
67
+ finish(() => resolve({
68
+ result: msg.result,
69
+ chunks: msg.chunks ?? [],
70
+ contentHash: msg.contentHash,
71
+ }));
72
+ }
73
+ else {
74
+ finish(() => reject(rehydrateWorkerError(msg?.error)));
75
+ }
76
+ });
77
+ worker.on('error', (err) => {
78
+ finish(() => reject(err));
79
+ });
80
+ worker.on('exit', (code) => {
81
+ if (code !== 0) {
82
+ finish(() => reject(new Error(`extract_worker_exit ${code}`)));
83
+ }
84
+ });
85
+ });
86
+ }
@@ -13,6 +13,17 @@ export interface DiscoveredDocRef {
13
13
  externalUrl?: string;
14
14
  etag?: string;
15
15
  lastModified?: string;
16
+ /** 本地文件版本(入队层廉价变更检测用;与 knowledge_files.size/mtime 对齐) */
17
+ size?: number;
18
+ mtime?: number;
19
+ }
20
+ export interface DiscoverResult {
21
+ docs: DiscoveredDocRef[];
22
+ /**
23
+ * 是否为全量结果。false = 被 maxPages/maxBytes 截断或 walk 出错——
24
+ * 此时**禁止** prune,否则会把仍存在的文档误删。
25
+ */
26
+ complete: boolean;
16
27
  }
17
28
  export interface VirtualDocument {
18
29
  path: string;
@@ -25,7 +36,7 @@ export interface VirtualDocument {
25
36
  size: number;
26
37
  }
27
38
  export interface SourceFetcher {
28
- discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<DiscoveredDocRef[]>;
39
+ discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<DiscoverResult>;
29
40
  fetch(source: KnowledgeSource, ref: DiscoveredDocRef, cred?: ResolvedCredential | null): Promise<VirtualDocument | null>;
30
41
  }
31
42
  /** 本地目录/文件 — 收拢原 walk 语义 */
@@ -35,7 +46,7 @@ export declare class LocalFsFetcher implements SourceFetcher {
35
46
  * @param shouldSkipPath - 噪音/二进制路径过滤
36
47
  */
37
48
  constructor(shouldSkipPath: (path: string) => boolean);
38
- discover(source: KnowledgeSource): Promise<DiscoveredDocRef[]>;
49
+ discover(source: KnowledgeSource): Promise<DiscoverResult>;
39
50
  fetch(source: KnowledgeSource, ref: DiscoveredDocRef): Promise<VirtualDocument | null>;
40
51
  }
41
52
  /** URL 单页 / sitemap / 同域有限 crawl */
@@ -59,7 +70,7 @@ export declare class UrlFetcher implements SourceFetcher {
59
70
  /** 从 HTML 抽取同源 a[href] */
60
71
  static extractLinks(html: string, baseUrl: string): string[];
61
72
  private networkOpts;
62
- discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<DiscoveredDocRef[]>;
73
+ discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<DiscoverResult>;
63
74
  private discoverSitemap;
64
75
  private discoverCrawl;
65
76
  fetch(source: KnowledgeSource, ref: DiscoveredDocRef, cred?: ResolvedCredential | null): Promise<VirtualDocument | null>;
@@ -8,7 +8,7 @@ import { createHash } from 'node:crypto';
8
8
  import { readFile, readdir, stat } from 'node:fs/promises';
9
9
  import { basename, join } from 'node:path';
10
10
  import { htmlToStructuredText, looksLikeHtml } from './html.js';
11
- import { guardedFetch } from './network-guard.js';
11
+ import { authHeadersForUrl, guardedFetch } from './network-guard.js';
12
12
  function mergeHeaders(cred, extra) {
13
13
  return { ...(cred?.headers ?? {}), ...(extra ?? {}) };
14
14
  }
@@ -24,12 +24,15 @@ export class LocalFsFetcher {
24
24
  async discover(source) {
25
25
  const root = source.location;
26
26
  const out = [];
27
+ let complete = true;
27
28
  const walkDir = async (dir) => {
28
29
  let entries;
29
30
  try {
30
31
  entries = await readdir(dir, { withFileTypes: true });
31
32
  }
32
33
  catch {
34
+ // 子目录读失败 → keep 集不完整,禁止 prune
35
+ complete = false;
33
36
  return;
34
37
  }
35
38
  for (const ent of entries) {
@@ -42,18 +45,33 @@ export class LocalFsFetcher {
42
45
  else if (ent.isFile()) {
43
46
  if (ent.name === 'knowledge.db' || ent.name.endsWith('.db'))
44
47
  continue;
45
- out.push({ path: abs });
48
+ try {
49
+ const fst = await stat(abs);
50
+ if (!fst.isFile())
51
+ continue;
52
+ out.push({
53
+ path: abs,
54
+ size: fst.size,
55
+ mtime: Math.floor(fst.mtimeMs),
56
+ });
57
+ }
58
+ catch {
59
+ // 竞态删除:当作未发现,避免入队后又 drop
60
+ }
46
61
  }
47
62
  }
48
63
  };
49
64
  const st = await stat(root).catch(() => null);
50
65
  if (!st)
51
- return out;
66
+ return { docs: out, complete: false };
52
67
  if (st.isFile()) {
53
- return [{ path: root }];
68
+ return {
69
+ docs: [{ path: root, size: st.size, mtime: Math.floor(st.mtimeMs) }],
70
+ complete: true,
71
+ };
54
72
  }
55
73
  await walkDir(root);
56
- return out;
74
+ return { docs: out, complete };
57
75
  }
58
76
  async fetch(source, ref) {
59
77
  if (this.shouldSkipPath(ref.path))
@@ -183,12 +201,15 @@ export class UrlFetcher {
183
201
  const sitemapMaxDepth = source.discover?.sitemapMaxDepth ?? 2;
184
202
  this.docCache.clear();
185
203
  if (mode === 'single') {
186
- return [
187
- {
188
- path: UrlFetcher.pathFromUrl(location, location),
189
- externalUrl: location,
190
- },
191
- ];
204
+ return {
205
+ docs: [
206
+ {
207
+ path: UrlFetcher.pathFromUrl(location, location),
208
+ externalUrl: location,
209
+ },
210
+ ],
211
+ complete: true,
212
+ };
192
213
  }
193
214
  if (mode === 'sitemap') {
194
215
  return this.discoverSitemap(location, source, cred, maxPages, maxBytes, sitemapMaxDepth);
@@ -202,6 +223,7 @@ export class UrlFetcher {
202
223
  const docs = [];
203
224
  const seenDoc = new Set();
204
225
  let downloaded = 0;
226
+ let truncated = false;
205
227
  while (queue.length && docs.length < maxPages) {
206
228
  const item = queue.shift();
207
229
  if (seenSm.has(item.url) || item.depth > sitemapMaxDepth)
@@ -209,8 +231,10 @@ export class UrlFetcher {
209
231
  seenSm.add(item.url);
210
232
  const res = await guardedFetch(item.url, { ...opts, maxResponseBytes: Math.min(opts.maxResponseBytes ?? 5_000_000, maxBytes - downloaded) }, { headers: mergeHeaders(cred) });
211
233
  downloaded += res.body.length;
212
- if (downloaded > maxBytes)
234
+ if (downloaded > maxBytes) {
235
+ truncated = true;
213
236
  break;
237
+ }
214
238
  const locs = UrlFetcher.parseSitemapLocs(res.body);
215
239
  const isIndex = /<sitemapindex/i.test(res.body);
216
240
  const smOrigin = new URL(sitemapUrl).origin;
@@ -228,9 +252,14 @@ export class UrlFetcher {
228
252
  externalUrl: loc,
229
253
  });
230
254
  }
255
+ else if (!seenDoc.has(loc) && docs.length >= maxPages) {
256
+ truncated = true;
257
+ }
231
258
  }
232
259
  }
233
- return docs;
260
+ if (queue.length > 0)
261
+ truncated = true;
262
+ return { docs, complete: !truncated };
234
263
  }
235
264
  async discoverCrawl(startUrl, source, cred, maxPages, maxDepth, maxBytes) {
236
265
  const opts = this.networkOpts(source);
@@ -239,6 +268,7 @@ export class UrlFetcher {
239
268
  const queue = [{ url: startUrl, depth: 0 }];
240
269
  const docs = [];
241
270
  let downloaded = 0;
271
+ let truncated = false;
242
272
  while (queue.length && docs.length < maxPages) {
243
273
  const item = queue.shift();
244
274
  if (seen.has(item.url) || item.depth > maxDepth)
@@ -255,8 +285,10 @@ export class UrlFetcher {
255
285
  continue;
256
286
  }
257
287
  downloaded += res.body.length;
258
- if (downloaded > maxBytes)
288
+ if (downloaded > maxBytes) {
289
+ truncated = true;
259
290
  break;
291
+ }
260
292
  const contentType = res.contentType ?? '';
261
293
  const isHtml = looksLikeHtml(res.body, contentType);
262
294
  let content = res.body;
@@ -284,7 +316,9 @@ export class UrlFetcher {
284
316
  }
285
317
  }
286
318
  }
287
- return docs.slice(0, maxPages);
319
+ if (queue.length > 0)
320
+ truncated = true;
321
+ return { docs: docs.slice(0, maxPages), complete: !truncated };
288
322
  }
289
323
  async fetch(source, ref, cred) {
290
324
  const url = ref.externalUrl ?? source.location;
@@ -295,12 +329,14 @@ export class UrlFetcher {
295
329
  return { ...cached, path: ref.path };
296
330
  }
297
331
  const maxDoc = this.network?.maxDocumentBytes ?? this.network?.maxResponseBytes ?? 5_000_000;
332
+ // 凭证只属于 source.location 的 origin;ref 外域不带鉴权
333
+ const trustedOrigin = new URL(source.location).origin;
298
334
  const res = await guardedFetch(url, {
299
335
  ...this.networkOpts(source),
300
336
  maxResponseBytes: maxDoc,
301
337
  etag: ref.etag,
302
338
  lastModified: ref.lastModified,
303
- }, { headers: mergeHeaders(cred) });
339
+ }, { headers: authHeadersForUrl(trustedOrigin, url, mergeHeaders(cred)) });
304
340
  if (res.notModified)
305
341
  return null;
306
342
  const contentType = res.contentType;
@@ -0,0 +1,53 @@
1
+ /**
2
+ * File identity — 物理文件稳定主键(arch/knowledge-service-http.md §6.1)
3
+ *
4
+ * identity_key 不含 size/mtime;版本字段只用于触发 re-parse。
5
+ * 不同 authRef 的外源内容视为不同 File。
6
+ */
7
+ export type FileIdentityKind = 'unix' | 'win' | 'path' | 'url' | 'connector';
8
+ export interface FileIdentity {
9
+ kind: FileIdentityKind;
10
+ /** 稳定键,存 knowledge_files.identity_key */
11
+ key: string;
12
+ /** 版本:原地更新时比较 */
13
+ size: number;
14
+ mtime: number;
15
+ /** 本地规范化绝对路径或 canonical URL */
16
+ canonical: string;
17
+ }
18
+ export interface StatLike {
19
+ dev?: number | bigint;
20
+ ino?: number | bigint;
21
+ size: number;
22
+ mtimeMs: number;
23
+ isFile?: boolean;
24
+ }
25
+ /**
26
+ * Windows / POSIX 路径词法归一(§7)。不要求存在。
27
+ */
28
+ export declare function normalizePathLexical(input: string): string;
29
+ /** 有则 realpath,否则词法 */
30
+ export declare function normalizeFsPath(input: string): Promise<string>;
31
+ export declare function foldPathForCompare(p: string, kind: 'win' | 'posix'): string;
32
+ export declare function pathCompareKind(normalized: string): 'win' | 'posix';
33
+ /**
34
+ * path segment 前缀(`/data/corpus` 不是 `/data/corpus2` 的祖先)
35
+ */
36
+ export declare function isPathPrefix(prefix: string, full: string): boolean;
37
+ export declare function canonicalizeUrl(raw: string): string;
38
+ export declare function urlIdentityKey(raw: string, authRef?: string | null): string;
39
+ export declare function connectorIdentityKey(connectorId: string, resourceKey: string, authRef?: string | null): string;
40
+ /**
41
+ * 本地文件 identity + version。失败退化 path:realpath(不含 size/mtime)。
42
+ */
43
+ export declare function identifyLocalFile(absPath: string): Promise<FileIdentity>;
44
+ export declare function identifyUrl(raw: string, authRef?: string | null, version?: {
45
+ size?: number;
46
+ mtime?: number;
47
+ }): Promise<FileIdentity>;
48
+ export declare function identifyConnector(connectorId: string, resourceKey: string, authRef?: string | null, version?: {
49
+ size?: number;
50
+ mtime?: number;
51
+ }): FileIdentity;
52
+ /** Membership logical_path:相对 root,统一 `/` */
53
+ export declare function logicalPathFrom(root: string, abs: string): string;
@@ -0,0 +1,172 @@
1
+ /**
2
+ * File identity — 物理文件稳定主键(arch/knowledge-service-http.md §6.1)
3
+ *
4
+ * identity_key 不含 size/mtime;版本字段只用于触发 re-parse。
5
+ * 不同 authRef 的外源内容视为不同 File。
6
+ */
7
+ import { stat } from 'node:fs/promises';
8
+ import { realpath } from 'node:fs/promises';
9
+ import path from 'node:path';
10
+ /**
11
+ * Windows / POSIX 路径词法归一(§7)。不要求存在。
12
+ */
13
+ export function normalizePathLexical(input) {
14
+ let p = input.replace(/\\/g, '/');
15
+ // \\?\C:\x → C:/x;\\?\UNC\server\share → //server/share(strip 后必须再测盘符)
16
+ if (p.startsWith('//?/')) {
17
+ p = p.slice(4);
18
+ if (/^UNC\//i.test(p)) {
19
+ p = `//${p.slice(4)}`;
20
+ }
21
+ }
22
+ p = p.replace(/^\\\?\//, '/').replace(/^\?\//, '/');
23
+ if (/^[a-z]:\//i.test(p)) {
24
+ p = p[0].toUpperCase() + p.slice(1);
25
+ }
26
+ else if (/^\/[a-z]:\//i.test(p)) {
27
+ // 残留 /C:/x(历史归一或半截 strip)
28
+ p = p[1].toUpperCase() + p.slice(2);
29
+ }
30
+ else if (p.startsWith('//')) {
31
+ // UNC: //server/share/...
32
+ p = '//' + p.slice(2);
33
+ }
34
+ const abs = p.startsWith('/') || /^[a-z]:\//i.test(p);
35
+ if (!abs) {
36
+ throw new Error(`location_must_be_absolute: ${input}`);
37
+ }
38
+ const isWinDrive = /^[a-z]:\//i.test(p);
39
+ const isUnc = p.startsWith('//');
40
+ const parts = p.split('/');
41
+ const out = [];
42
+ for (const seg of parts) {
43
+ if (seg === '' || seg === '.')
44
+ continue;
45
+ if (seg === '..') {
46
+ if (out.length > (isWinDrive || isUnc ? 1 : 0))
47
+ out.pop();
48
+ continue;
49
+ }
50
+ out.push(seg);
51
+ }
52
+ if (isWinDrive) {
53
+ return out.length ? `${out[0]}/${out.slice(1).join('/')}` : `${out[0] ?? 'C:'}/`;
54
+ }
55
+ if (isUnc) {
56
+ return `//${out.join('/')}`;
57
+ }
58
+ return `/${out.join('/')}`;
59
+ }
60
+ /** 有则 realpath,否则词法 */
61
+ export async function normalizeFsPath(input) {
62
+ const lexical = normalizePathLexical(input);
63
+ try {
64
+ const real = await realpath(input);
65
+ return normalizePathLexical(real);
66
+ }
67
+ catch {
68
+ return lexical;
69
+ }
70
+ }
71
+ export function foldPathForCompare(p, kind) {
72
+ return kind === 'win' ? p.toLowerCase() : p;
73
+ }
74
+ export function pathCompareKind(normalized) {
75
+ return /^[a-z]:\//i.test(normalized) || normalized.startsWith('//') ? 'win' : 'posix';
76
+ }
77
+ /**
78
+ * path segment 前缀(`/data/corpus` 不是 `/data/corpus2` 的祖先)
79
+ */
80
+ export function isPathPrefix(prefix, full) {
81
+ const k = pathCompareKind(prefix);
82
+ const a = foldPathForCompare(prefix, k).replace(/\/+$/, '');
83
+ const b = foldPathForCompare(full, k).replace(/\/+$/, '');
84
+ if (a === b)
85
+ return true;
86
+ return b.startsWith(a.endsWith('/') ? a : `${a}/`);
87
+ }
88
+ export function canonicalizeUrl(raw) {
89
+ const u = new URL(raw);
90
+ u.hash = '';
91
+ // **保留 query**:?id=1 / ?id=2 大概率是不同文档(契约修订)
92
+ u.hostname = u.hostname.toLowerCase();
93
+ if ((u.protocol === 'http:' && u.port === '80') ||
94
+ (u.protocol === 'https:' && u.port === '443')) {
95
+ u.port = '';
96
+ }
97
+ let s = u.toString();
98
+ if (s.endsWith('/') && !u.search)
99
+ s = s.slice(0, -1);
100
+ return s;
101
+ }
102
+ function authSuffix(authRef) {
103
+ return authRef && authRef.length > 0 ? authRef : 'none';
104
+ }
105
+ export function urlIdentityKey(raw, authRef) {
106
+ return `url:${canonicalizeUrl(raw)}#${authSuffix(authRef)}`;
107
+ }
108
+ export function connectorIdentityKey(connectorId, resourceKey, authRef) {
109
+ return `connector:${connectorId}:${resourceKey}#${authSuffix(authRef)}`;
110
+ }
111
+ function pathFallbackKey(normalized) {
112
+ return `path:${normalized}`;
113
+ }
114
+ /**
115
+ * 本地文件 identity + version。失败退化 path:realpath(不含 size/mtime)。
116
+ */
117
+ export async function identifyLocalFile(absPath) {
118
+ const canonical = await normalizeFsPath(absPath);
119
+ let st;
120
+ try {
121
+ st = (await stat(absPath));
122
+ }
123
+ catch (err) {
124
+ throw err;
125
+ }
126
+ const size = Number(st.size ?? 0);
127
+ const mtime = Math.floor(Number(st.mtimeMs ?? 0));
128
+ const ino = st.ino != null ? BigInt(st.ino) : null;
129
+ const dev = st.dev != null ? BigInt(st.dev) : null;
130
+ const isWin = process.platform === 'win32';
131
+ if (ino != null && dev != null && ino > 0n) {
132
+ const key = isWin ? `win:${dev}:${ino}` : `unix:${dev}:${ino}`;
133
+ return { kind: isWin ? 'win' : 'unix', key, size, mtime, canonical };
134
+ }
135
+ return { kind: 'path', key: pathFallbackKey(canonical), size, mtime, canonical };
136
+ }
137
+ export async function identifyUrl(raw, authRef, version) {
138
+ const canonical = canonicalizeUrl(raw);
139
+ return {
140
+ kind: 'url',
141
+ key: urlIdentityKey(raw, authRef),
142
+ size: Number(version?.size ?? 0),
143
+ mtime: Number(version?.mtime ?? 0),
144
+ canonical,
145
+ };
146
+ }
147
+ export function identifyConnector(connectorId, resourceKey, authRef, version) {
148
+ return {
149
+ kind: 'connector',
150
+ key: connectorIdentityKey(connectorId, resourceKey, authRef),
151
+ size: Number(version?.size ?? 0),
152
+ mtime: Number(version?.mtime ?? 0),
153
+ canonical: `${connectorId}:${resourceKey}`,
154
+ };
155
+ }
156
+ /** Membership logical_path:相对 root,统一 `/` */
157
+ export function logicalPathFrom(root, abs) {
158
+ const k = pathCompareKind(root);
159
+ const r = foldPathForCompare(normalizePathLexical(root), k).replace(/\/+$/, '');
160
+ const f = foldPathForCompare(normalizePathLexical(abs), k);
161
+ if (!isPathPrefix(normalizePathLexical(root), normalizePathLexical(abs))) {
162
+ // 大小写:win fold 后前缀
163
+ const rf = foldPathForCompare(r, k);
164
+ const ff = foldPathForCompare(f, k);
165
+ if (!ff.startsWith(rf.endsWith('/') ? rf : `${rf}/`) && ff !== rf) {
166
+ throw new Error(`path_outside_root: ${abs} not under ${root}`);
167
+ }
168
+ return ff === rf ? path.basename(f) : ff.slice(rf.length).replace(/^\//, '');
169
+ }
170
+ const rel = path.posix.relative(r, f);
171
+ return rel === '' ? path.posix.basename(f) : rel;
172
+ }
@@ -0,0 +1,50 @@
1
+ /**
2
+ * FileIndexStore — File 本位索引投影(chunks 挂 file_id;path 在 Membership)
3
+ */
4
+ import type { KnowledgeDatabase } from './db.js';
5
+ export interface ChunkDraft {
6
+ ordinal: number;
7
+ text: string;
8
+ startLine: number;
9
+ endLine: number;
10
+ }
11
+ export interface UpsertFileResult {
12
+ fileId: string;
13
+ chunkCount: number;
14
+ indexedAt: number;
15
+ }
16
+ export declare class FileIndexStore {
17
+ private readonly db;
18
+ private readonly fts;
19
+ constructor(db: KnowledgeDatabase);
20
+ /**
21
+ * 覆盖写入该 File 的全部 chunk(原地更新 / 首次解析)。
22
+ * 不触碰 memberships。
23
+ */
24
+ upsertFileContent(input: {
25
+ fileId: string;
26
+ adapterId?: string | null;
27
+ contentHash?: string | null;
28
+ size: number;
29
+ mtime: number;
30
+ chunks: ChunkDraft[];
31
+ }): UpsertFileResult;
32
+ markFileStatus(fileId: string, status: 'skipped' | 'error' | 'pending' | 'indexed', detail?: string): void;
33
+ /** purge File + chunks + embeddings + FTS(membership 已清) */
34
+ purgeFile(fileId: string): void;
35
+ private dropChunks;
36
+ private deleteKnowledgeVec;
37
+ /**
38
+ * 按可见 source 集合做关键词候选(chunk 去重;path 取 membership)。
39
+ */
40
+ searchKeywordVisible(query: string, visibleSourceIds: string[], limit: number): Array<{
41
+ chunkId: string;
42
+ fileId: string;
43
+ sourceIds: string[];
44
+ path: string;
45
+ ordinal: number;
46
+ text: string;
47
+ startLine: number;
48
+ endLine: number;
49
+ }>;
50
+ }