@octopi-agent/engine 0.56.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +43 -1
- package/dist/config-schema/substrate.d.ts +42 -0
- package/dist/config-schema/substrate.js +58 -0
- package/dist/config.d.ts +39 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/index.d.ts +5 -3
- package/dist/harness/index.js +3 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +42 -3
- package/dist/harness/knowledge/db.js +466 -83
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +68 -0
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +33 -0
- package/dist/harness/knowledge/extract-document.js +86 -0
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +166 -63
- package/dist/harness/knowledge/index-store.js +1025 -282
- package/dist/harness/knowledge/index.d.ts +47 -5
- package/dist/harness/knowledge/index.js +24 -2
- package/dist/harness/knowledge/ingest.d.ts +322 -49
- package/dist/harness/knowledge/ingest.js +1383 -267
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +30 -5
- package/dist/harness/knowledge/retriever.js +98 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +83 -16
- package/dist/harness/knowledge/source-store.js +205 -78
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
- package/dist/harness/memory/sqlite/keyword-search.js +104 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* extractDocumentInWorker — 线程内文档抽取 + 切块 + FTS token
|
|
3
|
+
*/
|
|
4
|
+
import { Worker } from 'node:worker_threads';
|
|
5
|
+
import { DocumentExtractError } from '../capabilities/document/errors.js';
|
|
6
|
+
import { resolveWorkerUrl } from '../../worker-path.js';
|
|
7
|
+
/** postMessage 不保留 class 原型;按 name/code 还原 DocumentExtractError */
|
|
8
|
+
function rehydrateWorkerError(payload) {
|
|
9
|
+
if (typeof payload === 'string')
|
|
10
|
+
return new Error(payload);
|
|
11
|
+
if (payload && typeof payload === 'object' && 'message' in payload) {
|
|
12
|
+
const e = payload;
|
|
13
|
+
if (e.name === 'DocumentExtractError' && e.code) {
|
|
14
|
+
return new DocumentExtractError(e.code, e.message, e.requires);
|
|
15
|
+
}
|
|
16
|
+
return new Error(e.message);
|
|
17
|
+
}
|
|
18
|
+
return new Error('extract_worker_failed');
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* 在 worker 线程抽取文档为 Markdown,并完成切块 + FTS token。
|
|
22
|
+
*
|
|
23
|
+
* @param filePath - 本地文件路径
|
|
24
|
+
* @param options - 超时 / 大小 / 中止 / 部分抽取限额
|
|
25
|
+
* @returns ExtractResult + chunks
|
|
26
|
+
* @throws Error 超时 / abort / worker 失败 / 抽取错误
|
|
27
|
+
*/
|
|
28
|
+
export function extractDocumentInWorker(filePath, options = {}) {
|
|
29
|
+
const timeoutMs = options.timeoutMs ?? 60_000;
|
|
30
|
+
return new Promise((resolve, reject) => {
|
|
31
|
+
if (options.signal?.aborted) {
|
|
32
|
+
reject(new Error('extract_aborted'));
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
let settled = false;
|
|
36
|
+
const worker = new Worker(resolveWorkerUrl('./document-extract-worker.js', import.meta.url), {
|
|
37
|
+
workerData: {
|
|
38
|
+
path: filePath,
|
|
39
|
+
name: filePath,
|
|
40
|
+
timeoutMs,
|
|
41
|
+
maxFileBytes: options.maxFileBytes,
|
|
42
|
+
maxSheets: options.maxSheets,
|
|
43
|
+
maxRowsPerSheet: options.maxRowsPerSheet,
|
|
44
|
+
maxPages: options.maxPages,
|
|
45
|
+
maxTextChars: options.maxTextChars,
|
|
46
|
+
documentConfig: options.documentConfig ?? null,
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
const finish = (fn) => {
|
|
50
|
+
if (settled)
|
|
51
|
+
return;
|
|
52
|
+
settled = true;
|
|
53
|
+
clearTimeout(timer);
|
|
54
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
55
|
+
void worker.terminate();
|
|
56
|
+
fn();
|
|
57
|
+
};
|
|
58
|
+
const onAbort = () => {
|
|
59
|
+
finish(() => reject(new Error('extract_aborted')));
|
|
60
|
+
};
|
|
61
|
+
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
62
|
+
const timer = setTimeout(() => {
|
|
63
|
+
finish(() => reject(new Error(`extract_worker_timeout after ${timeoutMs}ms`)));
|
|
64
|
+
}, timeoutMs);
|
|
65
|
+
worker.on('message', (msg) => {
|
|
66
|
+
if (msg?.ok && msg.result && msg.contentHash) {
|
|
67
|
+
finish(() => resolve({
|
|
68
|
+
result: msg.result,
|
|
69
|
+
chunks: msg.chunks ?? [],
|
|
70
|
+
contentHash: msg.contentHash,
|
|
71
|
+
}));
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
finish(() => reject(rehydrateWorkerError(msg?.error)));
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
worker.on('error', (err) => {
|
|
78
|
+
finish(() => reject(err));
|
|
79
|
+
});
|
|
80
|
+
worker.on('exit', (code) => {
|
|
81
|
+
if (code !== 0) {
|
|
82
|
+
finish(() => reject(new Error(`extract_worker_exit ${code}`)));
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
});
|
|
86
|
+
}
|
|
@@ -13,6 +13,17 @@ export interface DiscoveredDocRef {
|
|
|
13
13
|
externalUrl?: string;
|
|
14
14
|
etag?: string;
|
|
15
15
|
lastModified?: string;
|
|
16
|
+
/** 本地文件版本(入队层廉价变更检测用;与 knowledge_files.size/mtime 对齐) */
|
|
17
|
+
size?: number;
|
|
18
|
+
mtime?: number;
|
|
19
|
+
}
|
|
20
|
+
export interface DiscoverResult {
|
|
21
|
+
docs: DiscoveredDocRef[];
|
|
22
|
+
/**
|
|
23
|
+
* 是否为全量结果。false = 被 maxPages/maxBytes 截断或 walk 出错——
|
|
24
|
+
* 此时**禁止** prune,否则会把仍存在的文档误删。
|
|
25
|
+
*/
|
|
26
|
+
complete: boolean;
|
|
16
27
|
}
|
|
17
28
|
export interface VirtualDocument {
|
|
18
29
|
path: string;
|
|
@@ -25,7 +36,7 @@ export interface VirtualDocument {
|
|
|
25
36
|
size: number;
|
|
26
37
|
}
|
|
27
38
|
export interface SourceFetcher {
|
|
28
|
-
discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<
|
|
39
|
+
discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<DiscoverResult>;
|
|
29
40
|
fetch(source: KnowledgeSource, ref: DiscoveredDocRef, cred?: ResolvedCredential | null): Promise<VirtualDocument | null>;
|
|
30
41
|
}
|
|
31
42
|
/** 本地目录/文件 — 收拢原 walk 语义 */
|
|
@@ -35,7 +46,7 @@ export declare class LocalFsFetcher implements SourceFetcher {
|
|
|
35
46
|
* @param shouldSkipPath - 噪音/二进制路径过滤
|
|
36
47
|
*/
|
|
37
48
|
constructor(shouldSkipPath: (path: string) => boolean);
|
|
38
|
-
discover(source: KnowledgeSource): Promise<
|
|
49
|
+
discover(source: KnowledgeSource): Promise<DiscoverResult>;
|
|
39
50
|
fetch(source: KnowledgeSource, ref: DiscoveredDocRef): Promise<VirtualDocument | null>;
|
|
40
51
|
}
|
|
41
52
|
/** URL 单页 / sitemap / 同域有限 crawl */
|
|
@@ -59,7 +70,7 @@ export declare class UrlFetcher implements SourceFetcher {
|
|
|
59
70
|
/** 从 HTML 抽取同源 a[href] */
|
|
60
71
|
static extractLinks(html: string, baseUrl: string): string[];
|
|
61
72
|
private networkOpts;
|
|
62
|
-
discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<
|
|
73
|
+
discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<DiscoverResult>;
|
|
63
74
|
private discoverSitemap;
|
|
64
75
|
private discoverCrawl;
|
|
65
76
|
fetch(source: KnowledgeSource, ref: DiscoveredDocRef, cred?: ResolvedCredential | null): Promise<VirtualDocument | null>;
|
|
@@ -8,7 +8,7 @@ import { createHash } from 'node:crypto';
|
|
|
8
8
|
import { readFile, readdir, stat } from 'node:fs/promises';
|
|
9
9
|
import { basename, join } from 'node:path';
|
|
10
10
|
import { htmlToStructuredText, looksLikeHtml } from './html.js';
|
|
11
|
-
import { guardedFetch } from './network-guard.js';
|
|
11
|
+
import { authHeadersForUrl, guardedFetch } from './network-guard.js';
|
|
12
12
|
function mergeHeaders(cred, extra) {
|
|
13
13
|
return { ...(cred?.headers ?? {}), ...(extra ?? {}) };
|
|
14
14
|
}
|
|
@@ -24,12 +24,15 @@ export class LocalFsFetcher {
|
|
|
24
24
|
async discover(source) {
|
|
25
25
|
const root = source.location;
|
|
26
26
|
const out = [];
|
|
27
|
+
let complete = true;
|
|
27
28
|
const walkDir = async (dir) => {
|
|
28
29
|
let entries;
|
|
29
30
|
try {
|
|
30
31
|
entries = await readdir(dir, { withFileTypes: true });
|
|
31
32
|
}
|
|
32
33
|
catch {
|
|
34
|
+
// 子目录读失败 → keep 集不完整,禁止 prune
|
|
35
|
+
complete = false;
|
|
33
36
|
return;
|
|
34
37
|
}
|
|
35
38
|
for (const ent of entries) {
|
|
@@ -42,18 +45,33 @@ export class LocalFsFetcher {
|
|
|
42
45
|
else if (ent.isFile()) {
|
|
43
46
|
if (ent.name === 'knowledge.db' || ent.name.endsWith('.db'))
|
|
44
47
|
continue;
|
|
45
|
-
|
|
48
|
+
try {
|
|
49
|
+
const fst = await stat(abs);
|
|
50
|
+
if (!fst.isFile())
|
|
51
|
+
continue;
|
|
52
|
+
out.push({
|
|
53
|
+
path: abs,
|
|
54
|
+
size: fst.size,
|
|
55
|
+
mtime: Math.floor(fst.mtimeMs),
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
// 竞态删除:当作未发现,避免入队后又 drop
|
|
60
|
+
}
|
|
46
61
|
}
|
|
47
62
|
}
|
|
48
63
|
};
|
|
49
64
|
const st = await stat(root).catch(() => null);
|
|
50
65
|
if (!st)
|
|
51
|
-
return out;
|
|
66
|
+
return { docs: out, complete: false };
|
|
52
67
|
if (st.isFile()) {
|
|
53
|
-
return
|
|
68
|
+
return {
|
|
69
|
+
docs: [{ path: root, size: st.size, mtime: Math.floor(st.mtimeMs) }],
|
|
70
|
+
complete: true,
|
|
71
|
+
};
|
|
54
72
|
}
|
|
55
73
|
await walkDir(root);
|
|
56
|
-
return out;
|
|
74
|
+
return { docs: out, complete };
|
|
57
75
|
}
|
|
58
76
|
async fetch(source, ref) {
|
|
59
77
|
if (this.shouldSkipPath(ref.path))
|
|
@@ -183,12 +201,15 @@ export class UrlFetcher {
|
|
|
183
201
|
const sitemapMaxDepth = source.discover?.sitemapMaxDepth ?? 2;
|
|
184
202
|
this.docCache.clear();
|
|
185
203
|
if (mode === 'single') {
|
|
186
|
-
return
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
204
|
+
return {
|
|
205
|
+
docs: [
|
|
206
|
+
{
|
|
207
|
+
path: UrlFetcher.pathFromUrl(location, location),
|
|
208
|
+
externalUrl: location,
|
|
209
|
+
},
|
|
210
|
+
],
|
|
211
|
+
complete: true,
|
|
212
|
+
};
|
|
192
213
|
}
|
|
193
214
|
if (mode === 'sitemap') {
|
|
194
215
|
return this.discoverSitemap(location, source, cred, maxPages, maxBytes, sitemapMaxDepth);
|
|
@@ -202,6 +223,7 @@ export class UrlFetcher {
|
|
|
202
223
|
const docs = [];
|
|
203
224
|
const seenDoc = new Set();
|
|
204
225
|
let downloaded = 0;
|
|
226
|
+
let truncated = false;
|
|
205
227
|
while (queue.length && docs.length < maxPages) {
|
|
206
228
|
const item = queue.shift();
|
|
207
229
|
if (seenSm.has(item.url) || item.depth > sitemapMaxDepth)
|
|
@@ -209,8 +231,10 @@ export class UrlFetcher {
|
|
|
209
231
|
seenSm.add(item.url);
|
|
210
232
|
const res = await guardedFetch(item.url, { ...opts, maxResponseBytes: Math.min(opts.maxResponseBytes ?? 5_000_000, maxBytes - downloaded) }, { headers: mergeHeaders(cred) });
|
|
211
233
|
downloaded += res.body.length;
|
|
212
|
-
if (downloaded > maxBytes)
|
|
234
|
+
if (downloaded > maxBytes) {
|
|
235
|
+
truncated = true;
|
|
213
236
|
break;
|
|
237
|
+
}
|
|
214
238
|
const locs = UrlFetcher.parseSitemapLocs(res.body);
|
|
215
239
|
const isIndex = /<sitemapindex/i.test(res.body);
|
|
216
240
|
const smOrigin = new URL(sitemapUrl).origin;
|
|
@@ -228,9 +252,14 @@ export class UrlFetcher {
|
|
|
228
252
|
externalUrl: loc,
|
|
229
253
|
});
|
|
230
254
|
}
|
|
255
|
+
else if (!seenDoc.has(loc) && docs.length >= maxPages) {
|
|
256
|
+
truncated = true;
|
|
257
|
+
}
|
|
231
258
|
}
|
|
232
259
|
}
|
|
233
|
-
|
|
260
|
+
if (queue.length > 0)
|
|
261
|
+
truncated = true;
|
|
262
|
+
return { docs, complete: !truncated };
|
|
234
263
|
}
|
|
235
264
|
async discoverCrawl(startUrl, source, cred, maxPages, maxDepth, maxBytes) {
|
|
236
265
|
const opts = this.networkOpts(source);
|
|
@@ -239,6 +268,7 @@ export class UrlFetcher {
|
|
|
239
268
|
const queue = [{ url: startUrl, depth: 0 }];
|
|
240
269
|
const docs = [];
|
|
241
270
|
let downloaded = 0;
|
|
271
|
+
let truncated = false;
|
|
242
272
|
while (queue.length && docs.length < maxPages) {
|
|
243
273
|
const item = queue.shift();
|
|
244
274
|
if (seen.has(item.url) || item.depth > maxDepth)
|
|
@@ -255,8 +285,10 @@ export class UrlFetcher {
|
|
|
255
285
|
continue;
|
|
256
286
|
}
|
|
257
287
|
downloaded += res.body.length;
|
|
258
|
-
if (downloaded > maxBytes)
|
|
288
|
+
if (downloaded > maxBytes) {
|
|
289
|
+
truncated = true;
|
|
259
290
|
break;
|
|
291
|
+
}
|
|
260
292
|
const contentType = res.contentType ?? '';
|
|
261
293
|
const isHtml = looksLikeHtml(res.body, contentType);
|
|
262
294
|
let content = res.body;
|
|
@@ -284,7 +316,9 @@ export class UrlFetcher {
|
|
|
284
316
|
}
|
|
285
317
|
}
|
|
286
318
|
}
|
|
287
|
-
|
|
319
|
+
if (queue.length > 0)
|
|
320
|
+
truncated = true;
|
|
321
|
+
return { docs: docs.slice(0, maxPages), complete: !truncated };
|
|
288
322
|
}
|
|
289
323
|
async fetch(source, ref, cred) {
|
|
290
324
|
const url = ref.externalUrl ?? source.location;
|
|
@@ -295,12 +329,14 @@ export class UrlFetcher {
|
|
|
295
329
|
return { ...cached, path: ref.path };
|
|
296
330
|
}
|
|
297
331
|
const maxDoc = this.network?.maxDocumentBytes ?? this.network?.maxResponseBytes ?? 5_000_000;
|
|
332
|
+
// 凭证只属于 source.location 的 origin;ref 外域不带鉴权
|
|
333
|
+
const trustedOrigin = new URL(source.location).origin;
|
|
298
334
|
const res = await guardedFetch(url, {
|
|
299
335
|
...this.networkOpts(source),
|
|
300
336
|
maxResponseBytes: maxDoc,
|
|
301
337
|
etag: ref.etag,
|
|
302
338
|
lastModified: ref.lastModified,
|
|
303
|
-
}, { headers: mergeHeaders(cred) });
|
|
339
|
+
}, { headers: authHeadersForUrl(trustedOrigin, url, mergeHeaders(cred)) });
|
|
304
340
|
if (res.notModified)
|
|
305
341
|
return null;
|
|
306
342
|
const contentType = res.contentType;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File identity — 物理文件稳定主键(arch/knowledge-service-http.md §6.1)
|
|
3
|
+
*
|
|
4
|
+
* identity_key 不含 size/mtime;版本字段只用于触发 re-parse。
|
|
5
|
+
* 不同 authRef 的外源内容视为不同 File。
|
|
6
|
+
*/
|
|
7
|
+
export type FileIdentityKind = 'unix' | 'win' | 'path' | 'url' | 'connector';
|
|
8
|
+
export interface FileIdentity {
|
|
9
|
+
kind: FileIdentityKind;
|
|
10
|
+
/** 稳定键,存 knowledge_files.identity_key */
|
|
11
|
+
key: string;
|
|
12
|
+
/** 版本:原地更新时比较 */
|
|
13
|
+
size: number;
|
|
14
|
+
mtime: number;
|
|
15
|
+
/** 本地规范化绝对路径或 canonical URL */
|
|
16
|
+
canonical: string;
|
|
17
|
+
}
|
|
18
|
+
export interface StatLike {
|
|
19
|
+
dev?: number | bigint;
|
|
20
|
+
ino?: number | bigint;
|
|
21
|
+
size: number;
|
|
22
|
+
mtimeMs: number;
|
|
23
|
+
isFile?: boolean;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Windows / POSIX 路径词法归一(§7)。不要求存在。
|
|
27
|
+
*/
|
|
28
|
+
export declare function normalizePathLexical(input: string): string;
|
|
29
|
+
/** 有则 realpath,否则词法 */
|
|
30
|
+
export declare function normalizeFsPath(input: string): Promise<string>;
|
|
31
|
+
export declare function foldPathForCompare(p: string, kind: 'win' | 'posix'): string;
|
|
32
|
+
export declare function pathCompareKind(normalized: string): 'win' | 'posix';
|
|
33
|
+
/**
|
|
34
|
+
* path segment 前缀(`/data/corpus` 不是 `/data/corpus2` 的祖先)
|
|
35
|
+
*/
|
|
36
|
+
export declare function isPathPrefix(prefix: string, full: string): boolean;
|
|
37
|
+
export declare function canonicalizeUrl(raw: string): string;
|
|
38
|
+
export declare function urlIdentityKey(raw: string, authRef?: string | null): string;
|
|
39
|
+
export declare function connectorIdentityKey(connectorId: string, resourceKey: string, authRef?: string | null): string;
|
|
40
|
+
/**
|
|
41
|
+
* 本地文件 identity + version。失败退化 path:realpath(不含 size/mtime)。
|
|
42
|
+
*/
|
|
43
|
+
export declare function identifyLocalFile(absPath: string): Promise<FileIdentity>;
|
|
44
|
+
export declare function identifyUrl(raw: string, authRef?: string | null, version?: {
|
|
45
|
+
size?: number;
|
|
46
|
+
mtime?: number;
|
|
47
|
+
}): Promise<FileIdentity>;
|
|
48
|
+
export declare function identifyConnector(connectorId: string, resourceKey: string, authRef?: string | null, version?: {
|
|
49
|
+
size?: number;
|
|
50
|
+
mtime?: number;
|
|
51
|
+
}): FileIdentity;
|
|
52
|
+
/** Membership logical_path:相对 root,统一 `/` */
|
|
53
|
+
export declare function logicalPathFrom(root: string, abs: string): string;
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File identity — 物理文件稳定主键(arch/knowledge-service-http.md §6.1)
|
|
3
|
+
*
|
|
4
|
+
* identity_key 不含 size/mtime;版本字段只用于触发 re-parse。
|
|
5
|
+
* 不同 authRef 的外源内容视为不同 File。
|
|
6
|
+
*/
|
|
7
|
+
import { stat } from 'node:fs/promises';
|
|
8
|
+
import { realpath } from 'node:fs/promises';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
/**
|
|
11
|
+
* Windows / POSIX 路径词法归一(§7)。不要求存在。
|
|
12
|
+
*/
|
|
13
|
+
export function normalizePathLexical(input) {
|
|
14
|
+
let p = input.replace(/\\/g, '/');
|
|
15
|
+
// \\?\C:\x → C:/x;\\?\UNC\server\share → //server/share(strip 后必须再测盘符)
|
|
16
|
+
if (p.startsWith('//?/')) {
|
|
17
|
+
p = p.slice(4);
|
|
18
|
+
if (/^UNC\//i.test(p)) {
|
|
19
|
+
p = `//${p.slice(4)}`;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
p = p.replace(/^\\\?\//, '/').replace(/^\?\//, '/');
|
|
23
|
+
if (/^[a-z]:\//i.test(p)) {
|
|
24
|
+
p = p[0].toUpperCase() + p.slice(1);
|
|
25
|
+
}
|
|
26
|
+
else if (/^\/[a-z]:\//i.test(p)) {
|
|
27
|
+
// 残留 /C:/x(历史归一或半截 strip)
|
|
28
|
+
p = p[1].toUpperCase() + p.slice(2);
|
|
29
|
+
}
|
|
30
|
+
else if (p.startsWith('//')) {
|
|
31
|
+
// UNC: //server/share/...
|
|
32
|
+
p = '//' + p.slice(2);
|
|
33
|
+
}
|
|
34
|
+
const abs = p.startsWith('/') || /^[a-z]:\//i.test(p);
|
|
35
|
+
if (!abs) {
|
|
36
|
+
throw new Error(`location_must_be_absolute: ${input}`);
|
|
37
|
+
}
|
|
38
|
+
const isWinDrive = /^[a-z]:\//i.test(p);
|
|
39
|
+
const isUnc = p.startsWith('//');
|
|
40
|
+
const parts = p.split('/');
|
|
41
|
+
const out = [];
|
|
42
|
+
for (const seg of parts) {
|
|
43
|
+
if (seg === '' || seg === '.')
|
|
44
|
+
continue;
|
|
45
|
+
if (seg === '..') {
|
|
46
|
+
if (out.length > (isWinDrive || isUnc ? 1 : 0))
|
|
47
|
+
out.pop();
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
out.push(seg);
|
|
51
|
+
}
|
|
52
|
+
if (isWinDrive) {
|
|
53
|
+
return out.length ? `${out[0]}/${out.slice(1).join('/')}` : `${out[0] ?? 'C:'}/`;
|
|
54
|
+
}
|
|
55
|
+
if (isUnc) {
|
|
56
|
+
return `//${out.join('/')}`;
|
|
57
|
+
}
|
|
58
|
+
return `/${out.join('/')}`;
|
|
59
|
+
}
|
|
60
|
+
/** 有则 realpath,否则词法 */
|
|
61
|
+
export async function normalizeFsPath(input) {
|
|
62
|
+
const lexical = normalizePathLexical(input);
|
|
63
|
+
try {
|
|
64
|
+
const real = await realpath(input);
|
|
65
|
+
return normalizePathLexical(real);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return lexical;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
export function foldPathForCompare(p, kind) {
|
|
72
|
+
return kind === 'win' ? p.toLowerCase() : p;
|
|
73
|
+
}
|
|
74
|
+
export function pathCompareKind(normalized) {
|
|
75
|
+
return /^[a-z]:\//i.test(normalized) || normalized.startsWith('//') ? 'win' : 'posix';
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* path segment 前缀(`/data/corpus` 不是 `/data/corpus2` 的祖先)
|
|
79
|
+
*/
|
|
80
|
+
export function isPathPrefix(prefix, full) {
|
|
81
|
+
const k = pathCompareKind(prefix);
|
|
82
|
+
const a = foldPathForCompare(prefix, k).replace(/\/+$/, '');
|
|
83
|
+
const b = foldPathForCompare(full, k).replace(/\/+$/, '');
|
|
84
|
+
if (a === b)
|
|
85
|
+
return true;
|
|
86
|
+
return b.startsWith(a.endsWith('/') ? a : `${a}/`);
|
|
87
|
+
}
|
|
88
|
+
export function canonicalizeUrl(raw) {
|
|
89
|
+
const u = new URL(raw);
|
|
90
|
+
u.hash = '';
|
|
91
|
+
// **保留 query**:?id=1 / ?id=2 大概率是不同文档(契约修订)
|
|
92
|
+
u.hostname = u.hostname.toLowerCase();
|
|
93
|
+
if ((u.protocol === 'http:' && u.port === '80') ||
|
|
94
|
+
(u.protocol === 'https:' && u.port === '443')) {
|
|
95
|
+
u.port = '';
|
|
96
|
+
}
|
|
97
|
+
let s = u.toString();
|
|
98
|
+
if (s.endsWith('/') && !u.search)
|
|
99
|
+
s = s.slice(0, -1);
|
|
100
|
+
return s;
|
|
101
|
+
}
|
|
102
|
+
function authSuffix(authRef) {
|
|
103
|
+
return authRef && authRef.length > 0 ? authRef : 'none';
|
|
104
|
+
}
|
|
105
|
+
export function urlIdentityKey(raw, authRef) {
|
|
106
|
+
return `url:${canonicalizeUrl(raw)}#${authSuffix(authRef)}`;
|
|
107
|
+
}
|
|
108
|
+
export function connectorIdentityKey(connectorId, resourceKey, authRef) {
|
|
109
|
+
return `connector:${connectorId}:${resourceKey}#${authSuffix(authRef)}`;
|
|
110
|
+
}
|
|
111
|
+
function pathFallbackKey(normalized) {
|
|
112
|
+
return `path:${normalized}`;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* 本地文件 identity + version。失败退化 path:realpath(不含 size/mtime)。
|
|
116
|
+
*/
|
|
117
|
+
export async function identifyLocalFile(absPath) {
|
|
118
|
+
const canonical = await normalizeFsPath(absPath);
|
|
119
|
+
let st;
|
|
120
|
+
try {
|
|
121
|
+
st = (await stat(absPath));
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
throw err;
|
|
125
|
+
}
|
|
126
|
+
const size = Number(st.size ?? 0);
|
|
127
|
+
const mtime = Math.floor(Number(st.mtimeMs ?? 0));
|
|
128
|
+
const ino = st.ino != null ? BigInt(st.ino) : null;
|
|
129
|
+
const dev = st.dev != null ? BigInt(st.dev) : null;
|
|
130
|
+
const isWin = process.platform === 'win32';
|
|
131
|
+
if (ino != null && dev != null && ino > 0n) {
|
|
132
|
+
const key = isWin ? `win:${dev}:${ino}` : `unix:${dev}:${ino}`;
|
|
133
|
+
return { kind: isWin ? 'win' : 'unix', key, size, mtime, canonical };
|
|
134
|
+
}
|
|
135
|
+
return { kind: 'path', key: pathFallbackKey(canonical), size, mtime, canonical };
|
|
136
|
+
}
|
|
137
|
+
export async function identifyUrl(raw, authRef, version) {
|
|
138
|
+
const canonical = canonicalizeUrl(raw);
|
|
139
|
+
return {
|
|
140
|
+
kind: 'url',
|
|
141
|
+
key: urlIdentityKey(raw, authRef),
|
|
142
|
+
size: Number(version?.size ?? 0),
|
|
143
|
+
mtime: Number(version?.mtime ?? 0),
|
|
144
|
+
canonical,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
export function identifyConnector(connectorId, resourceKey, authRef, version) {
|
|
148
|
+
return {
|
|
149
|
+
kind: 'connector',
|
|
150
|
+
key: connectorIdentityKey(connectorId, resourceKey, authRef),
|
|
151
|
+
size: Number(version?.size ?? 0),
|
|
152
|
+
mtime: Number(version?.mtime ?? 0),
|
|
153
|
+
canonical: `${connectorId}:${resourceKey}`,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
/** Membership logical_path:相对 root,统一 `/` */
|
|
157
|
+
export function logicalPathFrom(root, abs) {
|
|
158
|
+
const k = pathCompareKind(root);
|
|
159
|
+
const r = foldPathForCompare(normalizePathLexical(root), k).replace(/\/+$/, '');
|
|
160
|
+
const f = foldPathForCompare(normalizePathLexical(abs), k);
|
|
161
|
+
if (!isPathPrefix(normalizePathLexical(root), normalizePathLexical(abs))) {
|
|
162
|
+
// 大小写:win fold 后前缀
|
|
163
|
+
const rf = foldPathForCompare(r, k);
|
|
164
|
+
const ff = foldPathForCompare(f, k);
|
|
165
|
+
if (!ff.startsWith(rf.endsWith('/') ? rf : `${rf}/`) && ff !== rf) {
|
|
166
|
+
throw new Error(`path_outside_root: ${abs} not under ${root}`);
|
|
167
|
+
}
|
|
168
|
+
return ff === rf ? path.basename(f) : ff.slice(rf.length).replace(/^\//, '');
|
|
169
|
+
}
|
|
170
|
+
const rel = path.posix.relative(r, f);
|
|
171
|
+
return rel === '' ? path.posix.basename(f) : rel;
|
|
172
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FileIndexStore — File 本位索引投影(chunks 挂 file_id;path 在 Membership)
|
|
3
|
+
*/
|
|
4
|
+
import type { KnowledgeDatabase } from './db.js';
|
|
5
|
+
export interface ChunkDraft {
|
|
6
|
+
ordinal: number;
|
|
7
|
+
text: string;
|
|
8
|
+
startLine: number;
|
|
9
|
+
endLine: number;
|
|
10
|
+
}
|
|
11
|
+
export interface UpsertFileResult {
|
|
12
|
+
fileId: string;
|
|
13
|
+
chunkCount: number;
|
|
14
|
+
indexedAt: number;
|
|
15
|
+
}
|
|
16
|
+
export declare class FileIndexStore {
|
|
17
|
+
private readonly db;
|
|
18
|
+
private readonly fts;
|
|
19
|
+
constructor(db: KnowledgeDatabase);
|
|
20
|
+
/**
|
|
21
|
+
* 覆盖写入该 File 的全部 chunk(原地更新 / 首次解析)。
|
|
22
|
+
* 不触碰 memberships。
|
|
23
|
+
*/
|
|
24
|
+
upsertFileContent(input: {
|
|
25
|
+
fileId: string;
|
|
26
|
+
adapterId?: string | null;
|
|
27
|
+
contentHash?: string | null;
|
|
28
|
+
size: number;
|
|
29
|
+
mtime: number;
|
|
30
|
+
chunks: ChunkDraft[];
|
|
31
|
+
}): UpsertFileResult;
|
|
32
|
+
markFileStatus(fileId: string, status: 'skipped' | 'error' | 'pending' | 'indexed', detail?: string): void;
|
|
33
|
+
/** purge File + chunks + embeddings + FTS(membership 已清) */
|
|
34
|
+
purgeFile(fileId: string): void;
|
|
35
|
+
private dropChunks;
|
|
36
|
+
private deleteKnowledgeVec;
|
|
37
|
+
/**
|
|
38
|
+
* 按可见 source 集合做关键词候选(chunk 去重;path 取 membership)。
|
|
39
|
+
*/
|
|
40
|
+
searchKeywordVisible(query: string, visibleSourceIds: string[], limit: number): Array<{
|
|
41
|
+
chunkId: string;
|
|
42
|
+
fileId: string;
|
|
43
|
+
sourceIds: string[];
|
|
44
|
+
path: string;
|
|
45
|
+
ordinal: number;
|
|
46
|
+
text: string;
|
|
47
|
+
startLine: number;
|
|
48
|
+
endLine: number;
|
|
49
|
+
}>;
|
|
50
|
+
}
|