@octopi-agent/engine 0.58.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +38 -1
- package/dist/config-schema/substrate.d.ts +37 -0
- package/dist/config-schema/substrate.js +47 -0
- package/dist/config.d.ts +30 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/index.d.ts +4 -2
- package/dist/harness/index.js +2 -1
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +80 -17
- package/dist/harness/knowledge/source-store.js +160 -89
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -3,6 +3,14 @@
|
|
|
3
3
|
*
|
|
4
4
|
* 无 adapter 的类型跳过并记 skipped,不挡整库。
|
|
5
5
|
*/
|
|
6
|
+
import { htmlToStructuredText } from './html.js';
|
|
7
|
+
/**
|
|
8
|
+
* Axure 导出侧车(data.js / document.js):压缩伪代码,无检索价值。
|
|
9
|
+
* **只认内容标记**,不按文件名一刀切——业务项目的 data.js 仍须正常索引。
|
|
10
|
+
*/
|
|
11
|
+
export function looksLikeAxureExportJs(head) {
|
|
12
|
+
return head.includes('$axure.loadCurrentPage') || head.includes('$axure.utils.Browser');
|
|
13
|
+
}
|
|
6
14
|
/** 默认噪音目录(对齐 file-search) */
|
|
7
15
|
export const IGNORED_DIRS = new Set([
|
|
8
16
|
'node_modules',
|
|
@@ -36,6 +44,8 @@ function pushParagraphs(content, startLineBase, maxChars) {
|
|
|
36
44
|
const chunks = [];
|
|
37
45
|
const lines = content.split('\n');
|
|
38
46
|
let buf = [];
|
|
47
|
+
/** 当前 buf 的近似长度(避免每行 join 整段) */
|
|
48
|
+
let bufLen = 0;
|
|
39
49
|
let bufStart = startLineBase;
|
|
40
50
|
let ordinal = 0;
|
|
41
51
|
const flush = (endLine) => {
|
|
@@ -43,24 +53,40 @@ function pushParagraphs(content, startLineBase, maxChars) {
|
|
|
43
53
|
if (text) {
|
|
44
54
|
chunks.push({
|
|
45
55
|
ordinal: ordinal++,
|
|
46
|
-
text
|
|
56
|
+
text,
|
|
47
57
|
startLine: bufStart,
|
|
48
58
|
endLine,
|
|
49
59
|
});
|
|
50
60
|
}
|
|
51
61
|
buf = [];
|
|
62
|
+
bufLen = 0;
|
|
52
63
|
};
|
|
53
64
|
for (let i = 0; i < lines.length; i++) {
|
|
54
|
-
|
|
65
|
+
let line = lines[i];
|
|
55
66
|
const absLine = startLineBase + i;
|
|
67
|
+
// 超长单行(压缩 HTML / 导出页)按 maxChars 切开,禁止 slice 丢弃尾部
|
|
68
|
+
while (line.length > maxChars) {
|
|
69
|
+
if (bufLen > 0)
|
|
70
|
+
flush(absLine - 1);
|
|
71
|
+
const head = line.slice(0, maxChars);
|
|
72
|
+
chunks.push({
|
|
73
|
+
ordinal: ordinal++,
|
|
74
|
+
text: head,
|
|
75
|
+
startLine: absLine,
|
|
76
|
+
endLine: absLine,
|
|
77
|
+
});
|
|
78
|
+
line = line.slice(maxChars);
|
|
79
|
+
}
|
|
80
|
+
// 先断再装:禁止「加完一行才 flush」导致 chunk 超 maxChars 一整行
|
|
81
|
+
if (buf.length > 0 && bufLen + line.length + 1 > maxChars) {
|
|
82
|
+
flush(absLine - 1);
|
|
83
|
+
}
|
|
56
84
|
if (buf.length === 0)
|
|
57
85
|
bufStart = absLine;
|
|
58
86
|
buf.push(line);
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
else if (joined.length >= maxChars) {
|
|
87
|
+
bufLen += line.length + 1;
|
|
88
|
+
const isBlank = line.trim() === '';
|
|
89
|
+
if (isBlank && bufLen >= MIN_CHUNK_CHARS) {
|
|
64
90
|
flush(absLine);
|
|
65
91
|
}
|
|
66
92
|
}
|
|
@@ -321,7 +347,8 @@ export const codeAdapter = {
|
|
|
321
347
|
'.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
|
|
322
348
|
'.py', '.go', '.rs', '.java', '.kt', '.c', '.cc', '.cpp', '.h', '.hpp',
|
|
323
349
|
'.rb', '.php', '.cs', '.swift', '.scala', '.sh', '.bash', '.ps1',
|
|
324
|
-
|
|
350
|
+
// .html/.htm/.xhtml 归 htmlAdapter(文档进向量);勿再声明在此
|
|
351
|
+
'.sql', '.css', '.scss', '.vue', '.svelte',
|
|
325
352
|
],
|
|
326
353
|
chunk(content) {
|
|
327
354
|
const bySymbol = chunkCodeBySymbols(content);
|
|
@@ -351,15 +378,18 @@ export const codeAdapter = {
|
|
|
351
378
|
},
|
|
352
379
|
};
|
|
353
380
|
/**
|
|
354
|
-
* HTML
|
|
355
|
-
*
|
|
381
|
+
* HTML:先抽正文再切块。
|
|
382
|
+
* 本地/外源统一走 htmlToStructuredText——Axure 等导出页含大段 script/style,
|
|
383
|
+
* 若 raw HTML 直接进 markdown 切块 + FTS,会产生海量 token 并写爆索引。
|
|
356
384
|
*/
|
|
357
385
|
export const htmlAdapter = {
|
|
358
386
|
id: 'html',
|
|
359
387
|
extensions: ['.html', '.htm', '.xhtml'],
|
|
360
388
|
mimes: ['text/html', 'application/xhtml+xml'],
|
|
361
389
|
chunk(content, path) {
|
|
362
|
-
|
|
390
|
+
const looksRaw = /<\s*[a-z!/?]/i.test(content);
|
|
391
|
+
const text = looksRaw ? htmlToStructuredText(content) : content;
|
|
392
|
+
return markdownAdapter.chunk(text, path);
|
|
363
393
|
},
|
|
364
394
|
};
|
|
365
395
|
export class FormatAdapterRegistry {
|
|
@@ -371,10 +401,20 @@ export class FormatAdapterRegistry {
|
|
|
371
401
|
}
|
|
372
402
|
register(adapter) {
|
|
373
403
|
for (const ext of adapter.extensions) {
|
|
374
|
-
|
|
404
|
+
const key = ext.toLowerCase();
|
|
405
|
+
const prev = this.byExt.get(key);
|
|
406
|
+
if (prev && prev.id !== adapter.id) {
|
|
407
|
+
throw new Error(`format adapter extension conflict: ${key} owned by ${prev.id}, cannot register ${adapter.id}`);
|
|
408
|
+
}
|
|
409
|
+
this.byExt.set(key, adapter);
|
|
375
410
|
}
|
|
376
411
|
for (const mime of adapter.mimes ?? []) {
|
|
377
|
-
|
|
412
|
+
const key = mime.toLowerCase();
|
|
413
|
+
const prev = this.byMime.get(key);
|
|
414
|
+
if (prev && prev.id !== adapter.id) {
|
|
415
|
+
throw new Error(`format adapter mime conflict: ${key} owned by ${prev.id}, cannot register ${adapter.id}`);
|
|
416
|
+
}
|
|
417
|
+
this.byMime.set(key, adapter);
|
|
378
418
|
}
|
|
379
419
|
}
|
|
380
420
|
/**
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KnowledgeClient — Gateway 侧 RPC(不打开 knowledge.db)
|
|
3
|
+
*/
|
|
4
|
+
export interface KnowledgeClientOptions {
|
|
5
|
+
baseUrl: string;
|
|
6
|
+
token: string;
|
|
7
|
+
/**
|
|
8
|
+
* HTTP 超时。默认 30s:索引期 FTS/写库会让 Service 短暂繁忙,
|
|
9
|
+
* 5s 会把正常的「忙」误报成 timeout/503。仍应通过分批让出保证 /health 可达。
|
|
10
|
+
*/
|
|
11
|
+
timeoutMs?: number;
|
|
12
|
+
/**
|
|
13
|
+
* 只读 API 超时(search/list/stats)。默认 12s:
|
|
14
|
+
* 查询走 Query Worker,应快速失败并由 UI 重试,而不是拖到 30s。
|
|
15
|
+
*/
|
|
16
|
+
readTimeoutMs?: number;
|
|
17
|
+
}
|
|
18
|
+
export declare class KnowledgeClient {
|
|
19
|
+
private readonly opts;
|
|
20
|
+
constructor(opts: KnowledgeClientOptions);
|
|
21
|
+
get baseUrl(): string;
|
|
22
|
+
get token(): string;
|
|
23
|
+
private request;
|
|
24
|
+
health(): Promise<{
|
|
25
|
+
ok: boolean;
|
|
26
|
+
service: string;
|
|
27
|
+
version: string;
|
|
28
|
+
}>;
|
|
29
|
+
ready(): Promise<{
|
|
30
|
+
ready: boolean;
|
|
31
|
+
sqliteVec: boolean;
|
|
32
|
+
}>;
|
|
33
|
+
ensurePrincipal(localAgentId: string, displayName?: string): Promise<unknown>;
|
|
34
|
+
listSources(opts?: {
|
|
35
|
+
scopeLevel?: 'global' | 'project' | 'session';
|
|
36
|
+
projectKey?: string;
|
|
37
|
+
sessionId?: string;
|
|
38
|
+
}): Promise<Array<Record<string, unknown>>>;
|
|
39
|
+
createSource(input: Record<string, unknown>): Promise<Record<string, unknown>>;
|
|
40
|
+
patchSource(sourceId: string, patch: Record<string, unknown>): Promise<Record<string, unknown>>;
|
|
41
|
+
deleteSource(sourceId: string): Promise<unknown>;
|
|
42
|
+
reindex(sourceId: string): Promise<unknown>;
|
|
43
|
+
/** 按路径 / 筛选强制重解析(中止态自动 resume) */
|
|
44
|
+
reprocess(sourceId: string, opts: {
|
|
45
|
+
paths?: string[];
|
|
46
|
+
filter?: {
|
|
47
|
+
status?: string;
|
|
48
|
+
ext?: string;
|
|
49
|
+
q?: string;
|
|
50
|
+
};
|
|
51
|
+
}): Promise<{
|
|
52
|
+
queued: number;
|
|
53
|
+
alreadyActive: number;
|
|
54
|
+
cleanedNonFiles: number;
|
|
55
|
+
resumed: boolean;
|
|
56
|
+
rejected: number;
|
|
57
|
+
}>;
|
|
58
|
+
search(agentId: string, q: string, opts?: {
|
|
59
|
+
sessionId?: string;
|
|
60
|
+
limit?: number;
|
|
61
|
+
}): Promise<{
|
|
62
|
+
hits: unknown[];
|
|
63
|
+
keywordHits: number;
|
|
64
|
+
vectorHits: number;
|
|
65
|
+
}>;
|
|
66
|
+
/** turn 级自动召回分档(与 Search Worker `autoGround` 同源) */
|
|
67
|
+
autoGround(agentId: string, q: string, opts?: {
|
|
68
|
+
sessionId?: string;
|
|
69
|
+
limit?: number;
|
|
70
|
+
recall?: 'off' | 'hint' | 'hybrid' | 'inject';
|
|
71
|
+
}): Promise<{
|
|
72
|
+
mode: 'inject' | 'hint' | 'none';
|
|
73
|
+
hits: unknown[];
|
|
74
|
+
hint?: string;
|
|
75
|
+
reason: string;
|
|
76
|
+
coverage: number;
|
|
77
|
+
scoreFloor: number;
|
|
78
|
+
}>;
|
|
79
|
+
stats(agentId: string): Promise<Record<string, number>>;
|
|
80
|
+
visibility(agentId: string, op: 'assignProject' | 'unassignProject' | 'hide' | 'unhide', body: {
|
|
81
|
+
projectKey?: string;
|
|
82
|
+
sourceId?: string;
|
|
83
|
+
}): Promise<unknown>;
|
|
84
|
+
listProjects(): Promise<Array<Record<string, unknown>>>;
|
|
85
|
+
createProject(input: {
|
|
86
|
+
projectKey: string;
|
|
87
|
+
displayName?: string;
|
|
88
|
+
visibility?: string;
|
|
89
|
+
}): Promise<Record<string, unknown>>;
|
|
90
|
+
deleteProject(projectKey: string): Promise<unknown>;
|
|
91
|
+
getSource(sourceId: string): Promise<Record<string, unknown>>;
|
|
92
|
+
/** 中止源任务;省略 sourceId = 中止本 Gateway 全部源 */
|
|
93
|
+
abort(sourceId?: string): Promise<unknown>;
|
|
94
|
+
/** 继续源任务;省略 sourceId = 继续本 Gateway 全部源 */
|
|
95
|
+
resume(sourceId?: string): Promise<unknown>;
|
|
96
|
+
listFiles(sourceId: string): Promise<Array<Record<string, unknown>>>;
|
|
97
|
+
listFilesPaged(sourceId: string, opts?: {
|
|
98
|
+
status?: 'indexed' | 'skipped' | 'error' | 'all';
|
|
99
|
+
ext?: string;
|
|
100
|
+
q?: string;
|
|
101
|
+
page?: number;
|
|
102
|
+
pageSize?: number;
|
|
103
|
+
}): Promise<{
|
|
104
|
+
items: Array<Record<string, unknown>>;
|
|
105
|
+
total: number;
|
|
106
|
+
page: number;
|
|
107
|
+
pageSize: number;
|
|
108
|
+
statusCounts: {
|
|
109
|
+
indexed: number;
|
|
110
|
+
skipped: number;
|
|
111
|
+
error: number;
|
|
112
|
+
};
|
|
113
|
+
extCounts: Array<{
|
|
114
|
+
ext: string;
|
|
115
|
+
n: number;
|
|
116
|
+
}>;
|
|
117
|
+
}>;
|
|
118
|
+
catalog(agentId: string): Promise<unknown[]>;
|
|
119
|
+
promotionCandidates(): Promise<unknown[]>;
|
|
120
|
+
describeSource(sourceId: string): Promise<{
|
|
121
|
+
generatedDescription: string;
|
|
122
|
+
source: string;
|
|
123
|
+
}>;
|
|
124
|
+
listChunks(agentId: string, sourceId: string, path: string): Promise<Array<{
|
|
125
|
+
id: string;
|
|
126
|
+
text: string;
|
|
127
|
+
startLine: number;
|
|
128
|
+
endLine: number;
|
|
129
|
+
ordinal: number;
|
|
130
|
+
path: string;
|
|
131
|
+
}>>;
|
|
132
|
+
read(agentId: string, body: {
|
|
133
|
+
chunkId?: string;
|
|
134
|
+
sourceId?: string;
|
|
135
|
+
path?: string;
|
|
136
|
+
}): Promise<{
|
|
137
|
+
found: boolean;
|
|
138
|
+
path?: string;
|
|
139
|
+
startLine?: number;
|
|
140
|
+
endLine?: number;
|
|
141
|
+
text?: string;
|
|
142
|
+
chunkCount?: number;
|
|
143
|
+
}>;
|
|
144
|
+
getVisibility(agentId: string): Promise<{
|
|
145
|
+
assignedProjects: string[];
|
|
146
|
+
hiddenSourceIds: string[];
|
|
147
|
+
}>;
|
|
148
|
+
sessionVisibility(agentId: string, sessionId: string): Promise<Array<Record<string, unknown>>>;
|
|
149
|
+
setSessionVisibility(agentId: string, item: {
|
|
150
|
+
sessionId: string;
|
|
151
|
+
targetType: 'project' | 'source';
|
|
152
|
+
targetId: string;
|
|
153
|
+
op: 'include' | 'exclude';
|
|
154
|
+
}): Promise<unknown>;
|
|
155
|
+
replaceSessionVisibility(agentId: string, sessionId: string, items: Array<{
|
|
156
|
+
targetType: 'project' | 'source';
|
|
157
|
+
targetId: string;
|
|
158
|
+
op: 'include' | 'exclude';
|
|
159
|
+
}>): Promise<unknown>;
|
|
160
|
+
clearSessionVisibility(agentId: string, sessionId: string, target?: {
|
|
161
|
+
targetType: 'project' | 'source';
|
|
162
|
+
targetId: string;
|
|
163
|
+
}): Promise<unknown>;
|
|
164
|
+
jobControl(sourceId: string): Promise<{
|
|
165
|
+
aborted: boolean;
|
|
166
|
+
jobsQueued: number;
|
|
167
|
+
jobsRunning: number;
|
|
168
|
+
jobsCancelled: number;
|
|
169
|
+
embedMissing: boolean;
|
|
170
|
+
canAbort: boolean;
|
|
171
|
+
canResume: boolean;
|
|
172
|
+
}>;
|
|
173
|
+
jobs(query?: {
|
|
174
|
+
sourceId?: string;
|
|
175
|
+
status?: string;
|
|
176
|
+
}): Promise<Array<Record<string, unknown>>>;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* AutoGroundPort 适配器 — Gateway 侧 turn 级 grounding 走 Knowledge Service。
|
|
180
|
+
* 不打开 knowledge.db;分档逻辑在 Search Worker 的 KnowledgeRetriever.autoGround。
|
|
181
|
+
*/
|
|
182
|
+
export declare class ClientKnowledgeGrounding {
|
|
183
|
+
private readonly client;
|
|
184
|
+
/** 缺省 recall(agents[].knowledge.recall / 全局);请求可覆盖 */
|
|
185
|
+
private readonly defaultRecall?;
|
|
186
|
+
constructor(client: KnowledgeClient,
|
|
187
|
+
/** 缺省 recall(agents[].knowledge.recall / 全局);请求可覆盖 */
|
|
188
|
+
defaultRecall?: "off" | "hint" | "hybrid" | "inject" | undefined);
|
|
189
|
+
autoGround(query: string, opts: {
|
|
190
|
+
agentId: string;
|
|
191
|
+
sessionId?: string;
|
|
192
|
+
limit?: number;
|
|
193
|
+
tenantId?: string;
|
|
194
|
+
gatewayId?: string;
|
|
195
|
+
recall?: 'off' | 'hint' | 'hybrid' | 'inject';
|
|
196
|
+
}): Promise<import('./retriever.js').AutoGroundDecision>;
|
|
197
|
+
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KnowledgeClient — Gateway 侧 RPC(不打开 knowledge.db)
|
|
3
|
+
*/
|
|
4
|
+
export class KnowledgeClient {
|
|
5
|
+
opts;
|
|
6
|
+
constructor(opts) {
|
|
7
|
+
this.opts = opts;
|
|
8
|
+
}
|
|
9
|
+
get baseUrl() {
|
|
10
|
+
return this.opts.baseUrl;
|
|
11
|
+
}
|
|
12
|
+
get token() {
|
|
13
|
+
return this.opts.token;
|
|
14
|
+
}
|
|
15
|
+
async request(method, path, body, headers, opts) {
|
|
16
|
+
const url = `${this.opts.baseUrl.replace(/\/+$/, '')}${path}`;
|
|
17
|
+
// 显式 timeoutMs 约束所有请求;readTimeoutMs 仅在未设 timeoutMs 时给只读更短缺省
|
|
18
|
+
const timeoutMs = opts?.read
|
|
19
|
+
? (this.opts.readTimeoutMs ?? this.opts.timeoutMs ?? 12_000)
|
|
20
|
+
: (this.opts.timeoutMs ?? 30_000);
|
|
21
|
+
const ac = new AbortController();
|
|
22
|
+
const timeoutErr = new Error(`knowledge_http_timeout ${method} ${path} after ${timeoutMs}ms (service busy or blocked)`);
|
|
23
|
+
const t = setTimeout(() => ac.abort(timeoutErr), timeoutMs);
|
|
24
|
+
try {
|
|
25
|
+
const res = await fetch(url, {
|
|
26
|
+
method,
|
|
27
|
+
headers: {
|
|
28
|
+
authorization: `Bearer ${this.opts.token}`,
|
|
29
|
+
...(body != null ? { 'content-type': 'application/json' } : {}),
|
|
30
|
+
...headers,
|
|
31
|
+
},
|
|
32
|
+
body: body != null ? JSON.stringify(body) : undefined,
|
|
33
|
+
signal: ac.signal,
|
|
34
|
+
});
|
|
35
|
+
const json = (await res.json());
|
|
36
|
+
if (!res.ok || json.error) {
|
|
37
|
+
throw new Error(json.error?.message ?? `knowledge_http_${res.status}`);
|
|
38
|
+
}
|
|
39
|
+
return (json.data ?? json);
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
if (ac.signal.aborted)
|
|
43
|
+
throw timeoutErr;
|
|
44
|
+
throw err;
|
|
45
|
+
}
|
|
46
|
+
finally {
|
|
47
|
+
clearTimeout(t);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
health() {
|
|
51
|
+
return this.request('GET', '/health', undefined, undefined, { read: true });
|
|
52
|
+
}
|
|
53
|
+
ready() {
|
|
54
|
+
return this.request('GET', '/v1/ready', undefined, undefined, { read: true });
|
|
55
|
+
}
|
|
56
|
+
ensurePrincipal(localAgentId, displayName) {
|
|
57
|
+
return this.request('PUT', `/v1/principals/${encodeURIComponent(localAgentId)}`, {
|
|
58
|
+
displayName,
|
|
59
|
+
status: 'active',
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
listSources(opts) {
|
|
63
|
+
const qs = new URLSearchParams();
|
|
64
|
+
if (opts?.scopeLevel)
|
|
65
|
+
qs.set('scopeLevel', opts.scopeLevel);
|
|
66
|
+
if (opts?.projectKey)
|
|
67
|
+
qs.set('projectKey', opts.projectKey);
|
|
68
|
+
if (opts?.sessionId)
|
|
69
|
+
qs.set('sessionId', opts.sessionId);
|
|
70
|
+
const q = qs.toString();
|
|
71
|
+
return this.request('GET', `/v1/sources${q ? `?${q}` : ''}`, undefined, undefined, {
|
|
72
|
+
read: true,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
createSource(input) {
|
|
76
|
+
return this.request('POST', '/v1/sources', input);
|
|
77
|
+
}
|
|
78
|
+
patchSource(sourceId, patch) {
|
|
79
|
+
return this.request('PATCH', `/v1/sources/${encodeURIComponent(sourceId)}`, patch);
|
|
80
|
+
}
|
|
81
|
+
deleteSource(sourceId) {
|
|
82
|
+
return this.request('DELETE', `/v1/sources/${encodeURIComponent(sourceId)}`);
|
|
83
|
+
}
|
|
84
|
+
reindex(sourceId) {
|
|
85
|
+
return this.request('POST', `/v1/sources/${encodeURIComponent(sourceId)}/reindex`);
|
|
86
|
+
}
|
|
87
|
+
/** 按路径 / 筛选强制重解析(中止态自动 resume) */
|
|
88
|
+
reprocess(sourceId, opts) {
|
|
89
|
+
return this.request('POST', `/v1/sources/${encodeURIComponent(sourceId)}/reprocess`, opts);
|
|
90
|
+
}
|
|
91
|
+
search(agentId, q, opts) {
|
|
92
|
+
const qs = new URLSearchParams({ q });
|
|
93
|
+
if (opts?.sessionId)
|
|
94
|
+
qs.set('sessionId', opts.sessionId);
|
|
95
|
+
if (opts?.limit != null)
|
|
96
|
+
qs.set('limit', String(opts.limit));
|
|
97
|
+
return this.request('GET', `/v1/principals/${encodeURIComponent(agentId)}/search?${qs}`, undefined, undefined, { read: true });
|
|
98
|
+
}
|
|
99
|
+
/** turn 级自动召回分档(与 Search Worker `autoGround` 同源) */
|
|
100
|
+
autoGround(agentId, q, opts) {
|
|
101
|
+
const qs = new URLSearchParams({ q });
|
|
102
|
+
if (opts?.sessionId)
|
|
103
|
+
qs.set('sessionId', opts.sessionId);
|
|
104
|
+
if (opts?.limit != null)
|
|
105
|
+
qs.set('limit', String(opts.limit));
|
|
106
|
+
if (opts?.recall)
|
|
107
|
+
qs.set('recall', opts.recall);
|
|
108
|
+
return this.request('GET', `/v1/principals/${encodeURIComponent(agentId)}/ground?${qs}`, undefined, undefined, { read: true });
|
|
109
|
+
}
|
|
110
|
+
stats(agentId) {
|
|
111
|
+
return this.request('GET', `/v1/principals/${encodeURIComponent(agentId)}/stats`, undefined, undefined, { read: true });
|
|
112
|
+
}
|
|
113
|
+
visibility(agentId, op, body) {
|
|
114
|
+
return this.request('POST', `/v1/principals/${encodeURIComponent(agentId)}/visibility`, {
|
|
115
|
+
op,
|
|
116
|
+
...body,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
listProjects() {
|
|
120
|
+
return this.request('GET', '/v1/projects');
|
|
121
|
+
}
|
|
122
|
+
createProject(input) {
|
|
123
|
+
return this.request('POST', '/v1/projects', input);
|
|
124
|
+
}
|
|
125
|
+
deleteProject(projectKey) {
|
|
126
|
+
return this.request('DELETE', `/v1/projects/${encodeURIComponent(projectKey)}`);
|
|
127
|
+
}
|
|
128
|
+
getSource(sourceId) {
|
|
129
|
+
return this.request('GET', `/v1/sources/${encodeURIComponent(sourceId)}`, undefined, undefined, { read: true });
|
|
130
|
+
}
|
|
131
|
+
/** 中止源任务;省略 sourceId = 中止本 Gateway 全部源 */
|
|
132
|
+
abort(sourceId) {
|
|
133
|
+
return sourceId
|
|
134
|
+
? this.request('POST', `/v1/sources/${encodeURIComponent(sourceId)}/abort`)
|
|
135
|
+
: this.request('POST', '/v1/jobs/abort');
|
|
136
|
+
}
|
|
137
|
+
/** 继续源任务;省略 sourceId = 继续本 Gateway 全部源 */
|
|
138
|
+
resume(sourceId) {
|
|
139
|
+
return sourceId
|
|
140
|
+
? this.request('POST', `/v1/sources/${encodeURIComponent(sourceId)}/resume`)
|
|
141
|
+
: this.request('POST', '/v1/jobs/resume');
|
|
142
|
+
}
|
|
143
|
+
listFiles(sourceId) {
|
|
144
|
+
return this.request('GET', `/v1/sources/${encodeURIComponent(sourceId)}/files`, undefined, undefined, { read: true });
|
|
145
|
+
}
|
|
146
|
+
listFilesPaged(sourceId, opts) {
|
|
147
|
+
const qs = new URLSearchParams();
|
|
148
|
+
if (opts?.status)
|
|
149
|
+
qs.set('status', opts.status);
|
|
150
|
+
if (opts?.ext)
|
|
151
|
+
qs.set('ext', opts.ext);
|
|
152
|
+
if (opts?.q)
|
|
153
|
+
qs.set('q', opts.q);
|
|
154
|
+
if (opts?.page != null)
|
|
155
|
+
qs.set('page', String(opts.page));
|
|
156
|
+
if (opts?.pageSize != null)
|
|
157
|
+
qs.set('pageSize', String(opts.pageSize));
|
|
158
|
+
const s = qs.toString();
|
|
159
|
+
return this.request('GET', `/v1/sources/${encodeURIComponent(sourceId)}/files${s ? `?${s}` : ''}`, undefined, undefined, { read: true });
|
|
160
|
+
}
|
|
161
|
+
catalog(agentId) {
|
|
162
|
+
return this.request('GET', `/v1/principals/${encodeURIComponent(agentId)}/catalog`, undefined, undefined, { read: true });
|
|
163
|
+
}
|
|
164
|
+
promotionCandidates() {
|
|
165
|
+
return this.request('GET', '/v1/promotion-candidates', undefined, undefined, { read: true });
|
|
166
|
+
}
|
|
167
|
+
describeSource(sourceId) {
|
|
168
|
+
return this.request('POST', `/v1/sources/${encodeURIComponent(sourceId)}/describe`);
|
|
169
|
+
}
|
|
170
|
+
listChunks(agentId, sourceId, path) {
|
|
171
|
+
const qs = new URLSearchParams({ sourceId, path });
|
|
172
|
+
return this.request('GET', `/v1/principals/${encodeURIComponent(agentId)}/chunks?${qs}`, undefined, undefined, { read: true });
|
|
173
|
+
}
|
|
174
|
+
read(agentId, body) {
|
|
175
|
+
return this.request('POST', `/v1/principals/${encodeURIComponent(agentId)}/read`, body);
|
|
176
|
+
}
|
|
177
|
+
getVisibility(agentId) {
|
|
178
|
+
return this.request('GET', `/v1/principals/${encodeURIComponent(agentId)}/visibility`, undefined, undefined, { read: true });
|
|
179
|
+
}
|
|
180
|
+
sessionVisibility(agentId, sessionId) {
|
|
181
|
+
return this.request('GET', `/v1/principals/${encodeURIComponent(agentId)}/session-visibility?sessionId=${encodeURIComponent(sessionId)}`, undefined, undefined, { read: true });
|
|
182
|
+
}
|
|
183
|
+
setSessionVisibility(agentId, item) {
|
|
184
|
+
return this.request('POST', `/v1/principals/${encodeURIComponent(agentId)}/session-visibility`, item);
|
|
185
|
+
}
|
|
186
|
+
replaceSessionVisibility(agentId, sessionId, items) {
|
|
187
|
+
return this.request('PUT', `/v1/principals/${encodeURIComponent(agentId)}/session-visibility`, {
|
|
188
|
+
sessionId,
|
|
189
|
+
items,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
clearSessionVisibility(agentId, sessionId, target) {
|
|
193
|
+
const qs = new URLSearchParams({ sessionId });
|
|
194
|
+
if (target?.targetType && target?.targetId) {
|
|
195
|
+
qs.set('targetType', target.targetType);
|
|
196
|
+
qs.set('targetId', target.targetId);
|
|
197
|
+
}
|
|
198
|
+
return this.request('DELETE', `/v1/principals/${encodeURIComponent(agentId)}/session-visibility?${qs}`);
|
|
199
|
+
}
|
|
200
|
+
jobControl(sourceId) {
|
|
201
|
+
return this.request('GET', `/v1/jobs/control?sourceId=${encodeURIComponent(sourceId)}`, undefined, undefined, { read: true });
|
|
202
|
+
}
|
|
203
|
+
jobs(query) {
|
|
204
|
+
const qs = new URLSearchParams();
|
|
205
|
+
if (query?.sourceId)
|
|
206
|
+
qs.set('sourceId', query.sourceId);
|
|
207
|
+
if (query?.status)
|
|
208
|
+
qs.set('status', query.status);
|
|
209
|
+
const s = qs.toString();
|
|
210
|
+
return this.request('GET', `/v1/jobs${s ? `?${s}` : ''}`, undefined, undefined, {
|
|
211
|
+
read: true,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* AutoGroundPort 适配器 — Gateway 侧 turn 级 grounding 走 Knowledge Service。
|
|
217
|
+
* 不打开 knowledge.db;分档逻辑在 Search Worker 的 KnowledgeRetriever.autoGround。
|
|
218
|
+
*/
|
|
219
|
+
export class ClientKnowledgeGrounding {
|
|
220
|
+
client;
|
|
221
|
+
defaultRecall;
|
|
222
|
+
constructor(client,
|
|
223
|
+
/** 缺省 recall(agents[].knowledge.recall / 全局);请求可覆盖 */
|
|
224
|
+
defaultRecall) {
|
|
225
|
+
this.client = client;
|
|
226
|
+
this.defaultRecall = defaultRecall;
|
|
227
|
+
}
|
|
228
|
+
async autoGround(query, opts) {
|
|
229
|
+
await this.client.ensurePrincipal(opts.agentId).catch(() => undefined);
|
|
230
|
+
const recall = opts.recall ?? this.defaultRecall;
|
|
231
|
+
return (await this.client.autoGround(opts.agentId, query, {
|
|
232
|
+
...(opts.sessionId != null ? { sessionId: opts.sessionId } : {}),
|
|
233
|
+
...(opts.limit != null ? { limit: opts.limit } : {}),
|
|
234
|
+
...(recall ? { recall } : {}),
|
|
235
|
+
}));
|
|
236
|
+
}
|
|
237
|
+
}
|
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
* ConnectorFetcher — 将 KnowledgeConnector 适配为 SourceFetcher
|
|
3
3
|
*/
|
|
4
4
|
import type { ResolvedCredential } from '../governance/credentials/types.js';
|
|
5
|
-
import type { DiscoveredDocRef, SourceFetcher, VirtualDocument } from './fetchers.js';
|
|
5
|
+
import type { DiscoveredDocRef, DiscoverResult, SourceFetcher, VirtualDocument } from './fetchers.js';
|
|
6
6
|
import type { ConnectorRegistry } from './connectors.js';
|
|
7
7
|
import type { KnowledgeSource } from './types.js';
|
|
8
8
|
export declare class ConnectorFetcher implements SourceFetcher {
|
|
9
9
|
private readonly registry;
|
|
10
10
|
constructor(registry: ConnectorRegistry);
|
|
11
|
-
discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<
|
|
11
|
+
discover(source: KnowledgeSource, cred?: ResolvedCredential | null): Promise<DiscoverResult>;
|
|
12
12
|
fetch(source: KnowledgeSource, ref: DiscoveredDocRef, cred?: ResolvedCredential | null): Promise<VirtualDocument | null>;
|
|
13
13
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* 密钥经 CredentialStore(authRef)注入,配置不进 knowledge.db 明文。
|
|
6
6
|
*/
|
|
7
7
|
import type { ResolvedCredential } from '../governance/credentials/types.js';
|
|
8
|
-
import type { DiscoveredDocRef, VirtualDocument } from './fetchers.js';
|
|
8
|
+
import type { DiscoveredDocRef, DiscoverResult, VirtualDocument } from './fetchers.js';
|
|
9
9
|
import type { KnowledgeSource, KnowledgeSourceNetwork } from './types.js';
|
|
10
10
|
export interface ConnectorContext {
|
|
11
11
|
/** 源 location(list 入口或配置引用) */
|
|
@@ -17,7 +17,7 @@ export interface ConnectorContext {
|
|
|
17
17
|
}
|
|
18
18
|
export interface KnowledgeConnector {
|
|
19
19
|
id: string;
|
|
20
|
-
discover(ctx: ConnectorContext, source: KnowledgeSource): Promise<
|
|
20
|
+
discover(ctx: ConnectorContext, source: KnowledgeSource): Promise<DiscoverResult>;
|
|
21
21
|
fetch(ctx: ConnectorContext, source: KnowledgeSource, ref: DiscoveredDocRef): Promise<VirtualDocument | null>;
|
|
22
22
|
}
|
|
23
23
|
/** generic REST list+get 配置(location 可为 JSON 串,或纯 listUrl) */
|
|
@@ -40,7 +40,7 @@ export interface RestConnectorConfig {
|
|
|
40
40
|
/** generic REST list + content/get */
|
|
41
41
|
export declare class RestConnector implements KnowledgeConnector {
|
|
42
42
|
readonly id = "rest";
|
|
43
|
-
discover(ctx: ConnectorContext, source: KnowledgeSource): Promise<
|
|
43
|
+
discover(ctx: ConnectorContext, source: KnowledgeSource): Promise<DiscoverResult>;
|
|
44
44
|
fetch(ctx: ConnectorContext, source: KnowledgeSource, ref: DiscoveredDocRef): Promise<VirtualDocument | null>;
|
|
45
45
|
}
|
|
46
46
|
export declare class ConnectorRegistry {
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { createHash } from 'node:crypto';
|
|
8
8
|
import { htmlToStructuredText, looksLikeHtml } from './html.js';
|
|
9
|
-
import { guardedFetch } from './network-guard.js';
|
|
9
|
+
import { authHeadersForUrl, guardedFetch } from './network-guard.js';
|
|
10
10
|
function dig(obj, path) {
|
|
11
11
|
if (!path)
|
|
12
12
|
return obj;
|
|
@@ -76,19 +76,21 @@ export class RestConnector {
|
|
|
76
76
|
headers: { ...(ctx.auth?.headers ?? {}), accept: 'application/json' },
|
|
77
77
|
});
|
|
78
78
|
const body = JSON.parse(res.body);
|
|
79
|
-
const
|
|
79
|
+
const allItems = pickItems(body, cfg.itemsPath);
|
|
80
|
+
const items = allItems.slice(0, maxPages);
|
|
80
81
|
const out = [];
|
|
81
82
|
items.forEach((raw, index) => {
|
|
82
83
|
if (!raw || typeof raw !== 'object')
|
|
83
84
|
return;
|
|
84
85
|
const item = raw;
|
|
85
86
|
const path = itemPath(item, cfg, index);
|
|
87
|
+
// contentField 条目无独立页面 URL:不得共用 listUrl 当 identity
|
|
86
88
|
const externalUrl = cfg.urlField
|
|
87
|
-
? String(dig(item, cfg.urlField) ??
|
|
88
|
-
:
|
|
89
|
-
out.push({ path, externalUrl });
|
|
89
|
+
? String(dig(item, cfg.urlField) ?? '')
|
|
90
|
+
: undefined;
|
|
91
|
+
out.push({ path, externalUrl: externalUrl || undefined });
|
|
90
92
|
});
|
|
91
|
-
return out;
|
|
93
|
+
return { docs: out, complete: allItems.length <= maxPages };
|
|
92
94
|
}
|
|
93
95
|
async fetch(ctx, source, ref) {
|
|
94
96
|
const cfg = parseRestConfig(ctx.location);
|
|
@@ -111,7 +113,7 @@ export class RestConnector {
|
|
|
111
113
|
return null;
|
|
112
114
|
const item = hit;
|
|
113
115
|
let content = '';
|
|
114
|
-
let externalUrl = ref.externalUrl
|
|
116
|
+
let externalUrl = ref.externalUrl;
|
|
115
117
|
if (cfg.contentField) {
|
|
116
118
|
const raw = dig(item, cfg.contentField);
|
|
117
119
|
content = raw == null ? '' : String(raw);
|
|
@@ -123,8 +125,10 @@ export class RestConnector {
|
|
|
123
125
|
if (!pageUrl)
|
|
124
126
|
return null;
|
|
125
127
|
externalUrl = pageUrl;
|
|
128
|
+
// 凭证只跟 listUrl 同源;被投毒列表指向的外域不得带走 token
|
|
129
|
+
const listOrigin = new URL(cfg.listUrl).origin;
|
|
126
130
|
const page = await guardedFetch(pageUrl, networkFrom(ctx, source), {
|
|
127
|
-
headers: ctx.auth?.headers,
|
|
131
|
+
headers: authHeadersForUrl(listOrigin, pageUrl, ctx.auth?.headers),
|
|
128
132
|
});
|
|
129
133
|
content = looksLikeHtml(page.body, page.contentType)
|
|
130
134
|
? htmlToStructuredText(page.body)
|
|
@@ -147,6 +151,8 @@ function networkFrom(ctx, source) {
|
|
|
147
151
|
return {
|
|
148
152
|
allowPrivateNetwork: source.network?.allowPrivateNetwork === true,
|
|
149
153
|
maxResponseBytes: source.network?.maxResponseBytes,
|
|
154
|
+
timeoutMs: source.network?.timeoutMs,
|
|
155
|
+
maxRedirects: source.network?.maxRedirects,
|
|
150
156
|
};
|
|
151
157
|
}
|
|
152
158
|
export class ConnectorRegistry {
|