@gt-fe/eap-runtime 0.0.20 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/api/handlers/invoke.handler.d.ts +7 -0
- package/dist/api/handlers/invoke.handler.js +52 -10
- package/dist/api/handlers/stream.handler.d.ts +6 -0
- package/dist/api/handlers/stream.handler.js +60 -39
- package/dist/api/routes/agent.routes.d.ts +8 -1
- package/dist/checkpoint/file-checkpoint-saver.d.ts +4 -0
- package/dist/checkpoint/file-checkpoint-saver.js +51 -22
- package/dist/checkpoint/index.d.ts +1 -1
- package/dist/checkpoint/index.js +1 -1
- package/dist/common/types.d.ts +2 -2
- package/dist/common/types.js +1 -1
- package/dist/execution/index.d.ts +2 -0
- package/dist/execution/index.js +2 -0
- package/dist/execution/interceptors.d.ts +40 -0
- package/dist/execution/interceptors.js +23 -0
- package/dist/execution/ports.d.ts +18 -0
- package/dist/execution/ports.js +13 -0
- package/dist/governance/audit-reporter.d.ts +14 -0
- package/dist/governance/audit-reporter.js +18 -0
- package/dist/governance/index.d.ts +2 -0
- package/dist/governance/index.js +1 -0
- package/dist/governance/permission-guard.d.ts +6 -2
- package/dist/governance/permission-guard.js +38 -5
- package/dist/graph/adapters/graph-nesting.d.ts +12 -0
- package/dist/graph/adapters/graph-nesting.js +29 -0
- package/dist/graph/adapters/index.d.ts +2 -0
- package/dist/graph/adapters/index.js +2 -0
- package/dist/graph/adapters/langgraph-checkpointer.d.ts +11 -0
- package/dist/graph/adapters/langgraph-checkpointer.js +48 -9
- package/dist/graph/adapters/langgraph.adapter.d.ts +33 -11
- package/dist/graph/adapters/langgraph.adapter.js +200 -98
- package/dist/graph/adapters/skill-isolate.d.ts +13 -0
- package/dist/graph/adapters/skill-isolate.js +39 -0
- package/dist/graph/adapters/subgraph-compiler.d.ts +7 -2
- package/dist/graph/adapters/subgraph-compiler.js +63 -30
- package/dist/graph/contracts/executable-graph.d.ts +33 -5
- package/dist/graph/hitl/hitl-handler.d.ts +13 -4
- package/dist/graph/hitl/hitl-handler.js +80 -7
- package/dist/graph/nodes/index.d.ts +1 -0
- package/dist/graph/nodes/index.js +1 -0
- package/dist/graph/nodes/llm-node.d.ts +3 -1
- package/dist/graph/nodes/llm-node.js +26 -2
- package/dist/graph/nodes/rag/dedup.d.ts +20 -0
- package/dist/graph/nodes/rag/dedup.js +77 -0
- package/dist/graph/nodes/rag/denoise.d.ts +58 -0
- package/dist/graph/nodes/rag/denoise.js +220 -0
- package/dist/graph/nodes/rag/fusion/fuse.d.ts +11 -0
- package/dist/graph/nodes/rag/fusion/fuse.js +25 -0
- package/dist/graph/nodes/rag/fusion/normalize.d.ts +9 -0
- package/dist/graph/nodes/rag/fusion/normalize.js +135 -0
- package/dist/graph/nodes/rag/index.d.ts +18 -0
- package/dist/graph/nodes/rag/index.js +17 -0
- package/dist/graph/nodes/rag/prompts.d.ts +12 -0
- package/dist/graph/nodes/rag/prompts.js +40 -0
- package/dist/graph/nodes/rag/rag-node.d.ts +21 -0
- package/dist/graph/nodes/rag/rag-node.js +135 -0
- package/dist/graph/nodes/rag/rag-search-tool.d.ts +48 -0
- package/dist/graph/nodes/rag/rag-search-tool.js +230 -0
- package/dist/graph/nodes/rag/react-loop.d.ts +57 -0
- package/dist/graph/nodes/rag/react-loop.js +467 -0
- package/dist/graph/nodes/rag/retrievers/http-embedder.d.ts +29 -0
- package/dist/graph/nodes/rag/retrievers/http-embedder.js +66 -0
- package/dist/graph/nodes/rag/retrievers/in-memory-retriever.d.ts +17 -0
- package/dist/graph/nodes/rag/retrievers/in-memory-retriever.js +18 -0
- package/dist/graph/nodes/rag/retrievers/local-sqlite-retriever.d.ts +46 -0
- package/dist/graph/nodes/rag/retrievers/local-sqlite-retriever.js +85 -0
- package/dist/graph/nodes/rag/tools.d.ts +90 -0
- package/dist/graph/nodes/rag/tools.js +260 -0
- package/dist/graph/nodes/rag/types.d.ts +226 -0
- package/dist/graph/nodes/rag/types.js +9 -0
- package/dist/graph/nodes/tool-node.d.ts +19 -1
- package/dist/graph/nodes/tool-node.js +158 -33
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/memory/semantic-retrieval.service.d.ts +1 -1
- package/dist/memory/semantic-retrieval.service.js +13 -23
- package/dist/providers/index.d.ts +2 -2
- package/dist/providers/index.js +2 -2
- package/dist/providers/local-llm-provider.d.ts +6 -0
- package/dist/providers/local-llm-provider.js +29 -5
- package/dist/providers/routing-llm-provider.d.ts +7 -2
- package/dist/providers/routing-llm-provider.js +71 -16
- package/dist/registry/deployment-snapshot.js +18 -4
- package/dist/registry/graph-loader.d.ts +2 -0
- package/dist/registry/graph-loader.js +39 -2
- package/dist/registry/preload-flags.d.ts +1 -1
- package/dist/registry/preload-flags.js +1 -1
- package/dist/runtime/agent-runtime.d.ts +5 -0
- package/dist/runtime/agent-runtime.js +107 -70
- package/dist/runtime/dev-rag-deps.d.ts +21 -0
- package/dist/runtime/dev-rag-deps.js +186 -0
- package/dist/runtime/dev-runtime.d.ts +18 -2
- package/dist/runtime/dev-runtime.js +68 -8
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/types.d.ts +10 -9
- package/dist/session/apply-session-compression.d.ts +85 -0
- package/dist/session/apply-session-compression.js +281 -0
- package/dist/session/apply-session-memory.d.ts +21 -0
- package/dist/session/apply-session-memory.js +48 -0
- package/dist/session/index.d.ts +2 -0
- package/dist/session/index.js +2 -0
- package/dist/session/persist-turn-messages.d.ts +57 -1
- package/dist/session/persist-turn-messages.js +160 -5
- package/dist/sse/events/index.d.ts +13 -4
- package/dist/sse/events/index.js +9 -1
- package/dist/sse/events/rag-events.d.ts +63 -0
- package/dist/sse/events/rag-events.js +9 -0
- package/dist/sse/events/session-events.d.ts +38 -0
- package/dist/sse/events/session-events.js +4 -0
- package/dist/token/index.d.ts +2 -1
- package/dist/token/index.js +2 -1
- package/dist/token/record-task-token-usage.d.ts +25 -0
- package/dist/token/record-task-token-usage.js +44 -0
- package/dist/token/token-detail-mode.d.ts +4 -4
- package/dist/token/token-detail-mode.js +4 -4
- package/dist/token/token-detail.d.ts +8 -0
- package/dist/token/token-detail.js +22 -0
- package/dist/token/token-stats.service.d.ts +2 -2
- package/dist/token/token-stats.service.js +1 -1
- package/dist/tool/create-tool-executor.d.ts +6 -0
- package/dist/tool/create-tool-executor.js +4 -3
- package/dist/tool/harness-primitives.d.ts +9 -1
- package/dist/tool/harness-primitives.js +43 -12
- package/dist/tool/index.d.ts +2 -1
- package/dist/tool/index.js +2 -1
- package/dist/tool/tool-confirmation.js +8 -5
- package/dist/tool/tool-executor.interface.d.ts +27 -0
- package/dist/tool/tool-orchestrator.d.ts +3 -1
- package/dist/tool/tool-orchestrator.js +15 -6
- package/dist/tool/tool-retry.d.ts +15 -0
- package/dist/tool/tool-retry.js +69 -0
- package/package.json +2 -2
|
@@ -3,18 +3,21 @@
|
|
|
3
3
|
*
|
|
4
4
|
* 将 Skill DSL 编译为子图节点函数,嵌入父 Graph 执行。
|
|
5
5
|
* 子图作为独立 StateGraph 编译,父图通过节点函数调用子图并合并输出。
|
|
6
|
+
* Skill 子图不得再含 type=subgraph(G-3 / B14)。
|
|
6
7
|
*
|
|
7
8
|
* @see design.md §1.4 子图嵌套: Skill作为子图嵌入主Graph
|
|
8
9
|
* @see requirements.md Requirement 1 AC2 — 子图嵌套(Skill作为子图)
|
|
9
10
|
*/
|
|
10
11
|
import { assertSkillPermission } from '../../governance/permission-guard.js';
|
|
11
12
|
import { LangGraphAdapter } from './langgraph.adapter.js';
|
|
13
|
+
import { assertSkillGraphHasNoSubgraphNodes } from './graph-nesting.js';
|
|
12
14
|
import { resolveExecutionTrace, warnTraceSpanFailure, } from '../execution-trace-context.js';
|
|
13
15
|
import { skillSpanFields, skillTraceIdentity } from './skill-entry-span.js';
|
|
14
16
|
import { filterLoadedToolsForSkill, } from '../../tool/system-prompt-builder.js';
|
|
15
17
|
import { unwrapHitlPaused } from '../hitl/hitl-handler.js';
|
|
16
18
|
import { pickContextFields } from '../../token/context-window.js';
|
|
17
19
|
import { assertSkillIoIfEnabled, coerceSkillInputFromMessages, } from '../../registry/skill-io-guard.js';
|
|
20
|
+
import { isolatedSkillMessages, isolatedSkillOutput, } from './skill-isolate.js';
|
|
18
21
|
/**
|
|
19
22
|
* SubGraphCompiler — 子图编译器
|
|
20
23
|
*
|
|
@@ -25,10 +28,12 @@ import { assertSkillIoIfEnabled, coerceSkillInputFromMessages, } from '../../reg
|
|
|
25
28
|
* 4. 执行时,子图内部运行完整的 graph invoke 流程,输出合并回父图 state
|
|
26
29
|
*/
|
|
27
30
|
export class SubGraphCompiler {
|
|
31
|
+
auditReporter;
|
|
28
32
|
adapter;
|
|
29
33
|
governanceClient;
|
|
30
34
|
skillAuditRecorder;
|
|
31
|
-
constructor(adapter, governanceClient, skillAuditRecorder) {
|
|
35
|
+
constructor(adapter, governanceClient, skillAuditRecorder, auditReporter) {
|
|
36
|
+
this.auditReporter = auditReporter;
|
|
32
37
|
this.adapter = adapter ?? new LangGraphAdapter();
|
|
33
38
|
this.governanceClient = governanceClient;
|
|
34
39
|
this.skillAuditRecorder = skillAuditRecorder;
|
|
@@ -39,14 +44,23 @@ export class SubGraphCompiler {
|
|
|
39
44
|
* @param dsl - Skill 的 GraphDSL 定义
|
|
40
45
|
* @param skillId - Skill 资源 ID(用于 checkPermission skill:use)
|
|
41
46
|
*/
|
|
42
|
-
compileSubGraph(dsl, skillId) {
|
|
47
|
+
compileSubGraph(dsl, skillId, options) {
|
|
48
|
+
assertSkillGraphHasNoSubgraphNodes(dsl, skillId ?? dsl.name);
|
|
43
49
|
const adapter = this.adapter;
|
|
50
|
+
const isolateContext = options?.isolateContext === true;
|
|
44
51
|
const resolvedSkillId = skillId ?? dsl.name ?? '';
|
|
45
52
|
const subGraphNodeFn = async (state) => {
|
|
46
|
-
const executionContext = state['executionContext']
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
53
|
+
const executionContext = state['executionContext'] ?? {
|
|
54
|
+
userId: String(state['userId'] ?? ''),
|
|
55
|
+
callerType: 'user',
|
|
56
|
+
roles: [],
|
|
57
|
+
permissions: [],
|
|
58
|
+
delegationToken: String(state['delegationToken'] ?? ''),
|
|
59
|
+
correlationId: String(state['correlationId'] ?? ''),
|
|
60
|
+
};
|
|
61
|
+
await assertSkillPermission(this.governanceClient, executionContext, resolvedSkillId, {
|
|
62
|
+
auditReporter: this.auditReporter,
|
|
63
|
+
});
|
|
50
64
|
const existingSpanId = typeof state['skillSpanId'] === 'string' && state['skillSpanId']
|
|
51
65
|
? state['skillSpanId']
|
|
52
66
|
: '';
|
|
@@ -119,26 +133,32 @@ export class SubGraphCompiler {
|
|
|
119
133
|
if (skillInfo) {
|
|
120
134
|
assertSkillIoIfEnabled(skillInfo, skillInput);
|
|
121
135
|
}
|
|
136
|
+
const parentMessages = messages.map((msg) => ({
|
|
137
|
+
role: msg['role'] ?? 'user',
|
|
138
|
+
content: msg['content'] ?? '',
|
|
139
|
+
toolCalls: msg['tool_calls'] ?? msg['toolCalls'],
|
|
140
|
+
...(typeof msg['toolCallId'] === 'string' ? { toolCallId: msg['toolCallId'] } : {}),
|
|
141
|
+
...(typeof msg['toolName'] === 'string' ? { toolName: msg['toolName'] } : {}),
|
|
142
|
+
...(typeof msg['success'] === 'boolean' ? { success: msg['success'] } : {}),
|
|
143
|
+
...(typeof msg['reasoningContent'] === 'string'
|
|
144
|
+
? { reasoningContent: msg['reasoningContent'] }
|
|
145
|
+
: {}),
|
|
146
|
+
}));
|
|
147
|
+
const subMessagesIn = isolateContext
|
|
148
|
+
? isolatedSkillMessages(parentMessages, skillInput)
|
|
149
|
+
: parentMessages;
|
|
122
150
|
const subGraphState = {
|
|
123
|
-
messages:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
toolCalls: msg['tool_calls'] ?? msg['toolCalls'],
|
|
127
|
-
...(typeof msg['toolCallId'] === 'string' ? { toolCallId: msg['toolCallId'] } : {}),
|
|
128
|
-
...(typeof msg['toolName'] === 'string' ? { toolName: msg['toolName'] } : {}),
|
|
129
|
-
...(typeof msg['success'] === 'boolean' ? { success: msg['success'] } : {}),
|
|
130
|
-
...(typeof msg['reasoningContent'] === 'string'
|
|
131
|
-
? { reasoningContent: msg['reasoningContent'] }
|
|
132
|
-
: {}),
|
|
133
|
-
})),
|
|
134
|
-
// 子图与父图共用 session thread;入口必须 __set,否则 checkpointer 会把父图 messages 再 append 一遍
|
|
135
|
-
replaceCheckpointMessages: true,
|
|
151
|
+
messages: subMessagesIn,
|
|
152
|
+
// 同父 session;checkpointNs=父节点 id → 子图槽 thread_id#父节点id(HITL 同槽,含 isolate)。
|
|
153
|
+
replaceCheckpointMessages: !isolateContext,
|
|
136
154
|
userId,
|
|
137
155
|
delegationToken,
|
|
138
156
|
agentCode,
|
|
139
157
|
taskId: state['taskId'],
|
|
140
158
|
correlationId: state['correlationId'],
|
|
141
159
|
sessionId: state['sessionId'],
|
|
160
|
+
checkpointNs: options?.parentNodeId,
|
|
161
|
+
skillIsolateContext: isolateContext || undefined,
|
|
142
162
|
executionContext: state['executionContext'],
|
|
143
163
|
loadedTools: scopedTools,
|
|
144
164
|
loadedSkills: state['loadedSkills'],
|
|
@@ -154,6 +174,8 @@ export class SubGraphCompiler {
|
|
|
154
174
|
lastSpanId: skillSpanId ?? state['lastSpanId'],
|
|
155
175
|
hitlChoice: state['hitlChoice'],
|
|
156
176
|
hitlRespondedBy: state['hitlRespondedBy'],
|
|
177
|
+
toolSpanId: typeof state['toolSpanId'] === 'string' ? state['toolSpanId'] : undefined,
|
|
178
|
+
hitlToolCallId: typeof state['hitlToolCallId'] === 'string' ? state['hitlToolCallId'] : undefined,
|
|
157
179
|
};
|
|
158
180
|
const result = await adapter.runIsolatedLlmStreamFlags(() => adapter.invoke(subGraph, subGraphState));
|
|
159
181
|
// 进程内调试记录;生产记录是上面的 skill Span
|
|
@@ -192,19 +214,28 @@ export class SubGraphCompiler {
|
|
|
192
214
|
...(reasoningContent ? { reasoningContent } : {}),
|
|
193
215
|
},
|
|
194
216
|
});
|
|
195
|
-
const added = subMessages.slice(
|
|
196
|
-
const outputMessages =
|
|
197
|
-
? added
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
217
|
+
const added = subMessages.slice(subMessagesIn.length);
|
|
218
|
+
const outputMessages = isolateContext
|
|
219
|
+
? isolatedSkillOutput(added.length > 0
|
|
220
|
+
? added
|
|
221
|
+
: result.output
|
|
222
|
+
? [{ role: 'assistant', content: result.output }]
|
|
223
|
+
: [])
|
|
224
|
+
: added.length > 0
|
|
225
|
+
? added
|
|
226
|
+
: result.output
|
|
227
|
+
? [{ role: 'assistant', content: result.output }]
|
|
228
|
+
: [];
|
|
201
229
|
const perNode = result.tokenUsage?.perNode;
|
|
202
230
|
const nestedLastSpanId = (typeof result.finalState?.lastSpanId === 'string' && result.finalState.lastSpanId) ||
|
|
203
231
|
skillSpanId ||
|
|
204
232
|
undefined;
|
|
233
|
+
const parentNodeId = options?.parentNodeId
|
|
234
|
+
|| (dsl.name ? `subgraph:${dsl.name}` : 'subgraph');
|
|
205
235
|
return {
|
|
206
236
|
messages: outputMessages,
|
|
207
|
-
currentNode:
|
|
237
|
+
currentNode: parentNodeId,
|
|
238
|
+
historyPath: [parentNodeId],
|
|
208
239
|
...(nestedLastSpanId ? { lastSpanId: nestedLastSpanId } : {}),
|
|
209
240
|
...(Array.isArray(perNode) && perNode.length > 0
|
|
210
241
|
? {
|
|
@@ -224,15 +255,17 @@ export class SubGraphCompiler {
|
|
|
224
255
|
if (hitlPaused) {
|
|
225
256
|
throw err;
|
|
226
257
|
}
|
|
258
|
+
const failErr = err;
|
|
259
|
+
const failMessage = failErr instanceof Error ? failErr.message : String(failErr);
|
|
227
260
|
await auditCtx?.finish({
|
|
228
261
|
status: 'failed',
|
|
229
|
-
errorMessage:
|
|
262
|
+
errorMessage: failMessage,
|
|
230
263
|
});
|
|
231
264
|
if (traceCollector && skillSpanId) {
|
|
232
265
|
try {
|
|
233
266
|
await traceCollector.completeNode(skillSpanId, {
|
|
234
267
|
status: 'failed',
|
|
235
|
-
error:
|
|
268
|
+
error: failMessage,
|
|
236
269
|
});
|
|
237
270
|
}
|
|
238
271
|
catch (completeErr) {
|
|
@@ -247,10 +280,10 @@ export class SubGraphCompiler {
|
|
|
247
280
|
payload: {
|
|
248
281
|
skillId: resolvedSkillId,
|
|
249
282
|
skillName,
|
|
250
|
-
error:
|
|
283
|
+
error: failMessage,
|
|
251
284
|
},
|
|
252
285
|
});
|
|
253
|
-
throw
|
|
286
|
+
throw failErr;
|
|
254
287
|
}
|
|
255
288
|
};
|
|
256
289
|
return subGraphNodeFn;
|
|
@@ -7,8 +7,14 @@
|
|
|
7
7
|
* @see design.md §1.4 StateGraph Contract
|
|
8
8
|
* @see requirements.md Requirement 1 AC1-AC2
|
|
9
9
|
*/
|
|
10
|
-
import type { GraphDSL as BaseGraphDSL, GraphNode as BaseGraphNode, GraphEdge as BaseGraphEdge, GraphNodeType, GraphConfig } from '@gt-fe/eap-contracts';
|
|
11
|
-
export type {
|
|
10
|
+
import type { GraphDSL as BaseGraphDSL, GraphNode as BaseGraphNode, GraphEdge as BaseGraphEdge, GraphNodeType as ContractsGraphNodeType, GraphConfig } from '@gt-fe/eap-contracts';
|
|
11
|
+
export type { GraphConfig };
|
|
12
|
+
/**
|
|
13
|
+
* GraphNodeType — runtime 侧扩展联合:contracts 官方包的 'rag' 类型尚未合入
|
|
14
|
+
* (客户 MR 跟进中),在此扩展使官方 contracts 零 patch 可用;
|
|
15
|
+
* contracts 合入后此扩展退化为无害超集,可直接删除回归 re-export。
|
|
16
|
+
*/
|
|
17
|
+
export type GraphNodeType = ContractsGraphNodeType | 'rag';
|
|
12
18
|
/**
|
|
13
19
|
* RuntimeGraphConfig — Runtime 层扩展的 Graph 配置
|
|
14
20
|
*
|
|
@@ -26,7 +32,9 @@ export interface RuntimeGraphConfig extends GraphConfig {
|
|
|
26
32
|
* - HITL 标记(interrupt_before / interrupt_after)
|
|
27
33
|
* - 描述信息(调试/Trace 用)
|
|
28
34
|
*/
|
|
29
|
-
export interface GraphNode extends BaseGraphNode {
|
|
35
|
+
export interface GraphNode extends Omit<BaseGraphNode, 'type'> {
|
|
36
|
+
/** 节点类型(contracts 联合 + runtime 扩展 'rag',见 GraphNodeType) */
|
|
37
|
+
type: GraphNodeType;
|
|
30
38
|
/** 节点显示名称(调试/Trace 用) */
|
|
31
39
|
label?: string;
|
|
32
40
|
/** 节点描述 */
|
|
@@ -37,8 +45,6 @@ export interface GraphNode extends BaseGraphNode {
|
|
|
37
45
|
interrupt?: NodeInterruptConfig;
|
|
38
46
|
/** 子图引用 ID(type=subgraph 时必填) */
|
|
39
47
|
subgraphId?: string;
|
|
40
|
-
/** 并行分支定义(type=parallel 时使用) */
|
|
41
|
-
branches?: string[];
|
|
42
48
|
}
|
|
43
49
|
/**
|
|
44
50
|
* GraphEdge — 基础边定义
|
|
@@ -183,6 +189,10 @@ export interface GraphState {
|
|
|
183
189
|
executionContext?: PlatformExecutionContext;
|
|
184
190
|
/** ExecuteConfig.forceToolServer — 单次 invoke 强制走 Gateway 的 toolName */
|
|
185
191
|
forceToolServer?: string[];
|
|
192
|
+
/** 请求级检索范围(前端圈定 fileIds/dirPaths;rag 节点读取,仅本地检索源生效) */
|
|
193
|
+
ragScope?: import('../../graph/nodes/rag/types.js').RagScope;
|
|
194
|
+
/** 会话级数据源勾选(数据源 id 列表;undefined = 全部启用) */
|
|
195
|
+
ragSources?: string[];
|
|
186
196
|
/** GraphLoader 注入:toolName → 运行期 manifest 引用 */
|
|
187
197
|
loadedTools?: import('../../tool/loaded-tool-definition.js').LoadedToolMap;
|
|
188
198
|
/** GraphLoader 注入:已装配 Skill 列表 */
|
|
@@ -212,8 +222,20 @@ export interface GraphState {
|
|
|
212
222
|
* 不入 Checkpoint。
|
|
213
223
|
*/
|
|
214
224
|
traceCollector?: import('../../contracts/trace-collector.js').ITraceCollector;
|
|
225
|
+
/**
|
|
226
|
+
* 当前节点跑在 Skill 隔离子图内。为 true 时禁止 HITL interrupt
|
|
227
|
+
*(隔离不写父 session Checkpoint,确认态会丢)。
|
|
228
|
+
*/
|
|
229
|
+
skillIsolateContext?: boolean;
|
|
215
230
|
/** 当前 Skill 子图 createNode 返回的 span id;子图内 Tool span 的 parentNodeId */
|
|
216
231
|
skillSpanId?: string;
|
|
232
|
+
/**
|
|
233
|
+
* HITL 续跑:暂停时已 create 的 tool Span id。
|
|
234
|
+
* 按 hitlToolCallId 复用,禁止再 createNode。
|
|
235
|
+
*/
|
|
236
|
+
toolSpanId?: string;
|
|
237
|
+
/** 与 toolSpanId 配对的 toolCallId(同一 assistant 多 tool 时只复用这一次调用) */
|
|
238
|
+
hitlToolCallId?: string;
|
|
217
239
|
/** 最近一次 createNode 的 span id;Checkpoint 引用挂在此 Span 上 */
|
|
218
240
|
lastSpanId?: string;
|
|
219
241
|
/**
|
|
@@ -221,6 +243,12 @@ export interface GraphState {
|
|
|
221
243
|
* 仅 stream/invoke 入口使用,不入持久化语义。
|
|
222
244
|
*/
|
|
223
245
|
replaceCheckpointMessages?: boolean;
|
|
246
|
+
/**
|
|
247
|
+
* 子图 invoke 写入 LangGraph `checkpoint_ns`。
|
|
248
|
+
* 父图为空;子图为父节点 id。存储槽与父 thread 分开,不覆盖父 latest。
|
|
249
|
+
* HITL interrupt 用同一槽;resume 返回给父图入口的 GraphState 不含本字段,也不含内层 historyPath / currentNode。
|
|
250
|
+
*/
|
|
251
|
+
checkpointNs?: string;
|
|
224
252
|
[key: string]: unknown;
|
|
225
253
|
}
|
|
226
254
|
/** Graph 内部消息格式 */
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import type { ICheckpointSaver } from '../../checkpoint/contracts/checkpoint-saver.contract.js';
|
|
14
14
|
import type { ITraceCollector } from '../../contracts/trace-collector.js';
|
|
15
15
|
import type { ISSEEmitter } from '../../sse/contracts/sse-emitter.contract.js';
|
|
16
|
-
import type { GraphState } from '../contracts/executable-graph.js';
|
|
16
|
+
import type { GraphState, CheckpointData } from '../contracts/executable-graph.js';
|
|
17
17
|
/** HITL 默认等待时长(毫秒)= 300 秒。SSE `hitl.waited.timeout` 对外用秒。 */
|
|
18
18
|
export declare const DEFAULT_HITL_TIMEOUT_MS = 300000;
|
|
19
19
|
/** 超时收口对外文案(Task / Trace / AppError);枚举走 `hitl_timeout`。 */
|
|
@@ -106,6 +106,8 @@ export declare class HitlPausedError extends Error {
|
|
|
106
106
|
export declare function unwrapHitlPaused(error: unknown): HitlPausedError | null;
|
|
107
107
|
/** Checkpoint state 中持久化的 HITL 等待元数据(进程重启 / 无内存 Map 时冷恢复) */
|
|
108
108
|
export declare const HITL_WAITING_STATE_KEY = "__hitlWaiting";
|
|
109
|
+
/** 父槽上的指针:子图 HITL 的 waiting 在 `sessionId#父节点id`,冷恢复只带 sessionId 时跟随此键。 */
|
|
110
|
+
export declare const HITL_WAITING_SLOT_KEY = "__hitlWaitingSlot";
|
|
109
111
|
export interface PersistedHitlWaiting {
|
|
110
112
|
taskId: string;
|
|
111
113
|
threadId: string;
|
|
@@ -141,6 +143,12 @@ export declare function toHitlTimeoutMs(timeoutMs?: number): number;
|
|
|
141
143
|
export declare function toHitlTimeoutSeconds(timeoutMs?: number): number;
|
|
142
144
|
export declare function isHitlDeadlinePassed(persisted: Pick<PersistedHitlWaiting, 'deadlineAt'> | undefined, now?: number): boolean;
|
|
143
145
|
export declare function readPersistedHitlWaiting(state: unknown): PersistedHitlWaiting | undefined;
|
|
146
|
+
export declare function readHitlWaitingSlot(state: unknown): string | undefined;
|
|
147
|
+
/** 父槽有 waiting 用父槽;否则跟随 `__hitlWaitingSlot` 到子图槽。 */
|
|
148
|
+
export declare function loadHitlCheckpoint(saver: ICheckpointSaver, threadId: string): Promise<{
|
|
149
|
+
threadId: string;
|
|
150
|
+
checkpoint: CheckpointData | null;
|
|
151
|
+
}>;
|
|
144
152
|
/** 门面 stream / HTTP StreamOrchestrator / streamResumeHitl 共用 */
|
|
145
153
|
export declare function buildHitlWaitedPayload(paused: HitlPausedError): {
|
|
146
154
|
question: string;
|
|
@@ -177,9 +185,10 @@ export declare class HitlHandler {
|
|
|
177
185
|
/**
|
|
178
186
|
* 中断 Graph 执行,等待人工确认
|
|
179
187
|
*
|
|
180
|
-
* 写入 `__hitlWaiting`(含 deadlineAt
|
|
181
|
-
*
|
|
182
|
-
*
|
|
188
|
+
* 写入 `__hitlWaiting`(含 deadlineAt)到当前执行图的 Checkpoint 槽。
|
|
189
|
+
* 父图槽=sessionId;子图槽=sessionId#父节点id(与 LangGraph put 同一把钥匙)。
|
|
190
|
+
* 不保留 LangGraph `engineSnapshot`:resume 从父图入口重入
|
|
191
|
+
*(skillSpanId / hitlChoice 防重复 span;子图 resume 去掉 checkpointNs / 内层 historyPath / currentNode)。
|
|
183
192
|
*/
|
|
184
193
|
interrupt(taskId: string, threadId: string, state: GraphState, config: HitlInterruptConfig, extras?: HitlInterruptExtras): Promise<void>;
|
|
185
194
|
private interruptCore;
|
|
@@ -118,6 +118,8 @@ export function unwrapHitlPaused(error) {
|
|
|
118
118
|
}
|
|
119
119
|
/** Checkpoint state 中持久化的 HITL 等待元数据(进程重启 / 无内存 Map 时冷恢复) */
|
|
120
120
|
export const HITL_WAITING_STATE_KEY = '__hitlWaiting';
|
|
121
|
+
/** 父槽上的指针:子图 HITL 的 waiting 在 `sessionId#父节点id`,冷恢复只带 sessionId 时跟随此键。 */
|
|
122
|
+
export const HITL_WAITING_SLOT_KEY = '__hitlWaitingSlot';
|
|
121
123
|
export function toHitlTimeoutMs(timeoutMs) {
|
|
122
124
|
if (typeof timeoutMs !== 'number' || !Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
123
125
|
return DEFAULT_HITL_TIMEOUT_MS;
|
|
@@ -142,6 +144,29 @@ export function readPersistedHitlWaiting(state) {
|
|
|
142
144
|
return undefined;
|
|
143
145
|
return raw;
|
|
144
146
|
}
|
|
147
|
+
export function readHitlWaitingSlot(state) {
|
|
148
|
+
if (!state || typeof state !== 'object')
|
|
149
|
+
return undefined;
|
|
150
|
+
const raw = state[HITL_WAITING_SLOT_KEY];
|
|
151
|
+
if (typeof raw === 'string' && raw.trim())
|
|
152
|
+
return raw.trim();
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
/** 父槽有 waiting 用父槽;否则跟随 `__hitlWaitingSlot` 到子图槽。 */
|
|
156
|
+
export async function loadHitlCheckpoint(saver, threadId) {
|
|
157
|
+
const first = await saver.load(threadId);
|
|
158
|
+
if (readPersistedHitlWaiting(first?.state)) {
|
|
159
|
+
return { threadId, checkpoint: first };
|
|
160
|
+
}
|
|
161
|
+
const slot = readHitlWaitingSlot(first?.state);
|
|
162
|
+
if (slot && slot !== threadId) {
|
|
163
|
+
const nested = await saver.load(slot);
|
|
164
|
+
if (readPersistedHitlWaiting(nested?.state)) {
|
|
165
|
+
return { threadId: slot, checkpoint: nested };
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return { threadId, checkpoint: first };
|
|
169
|
+
}
|
|
145
170
|
/** 门面 stream / HTTP StreamOrchestrator / streamResumeHitl 共用 */
|
|
146
171
|
export function buildHitlWaitedPayload(paused) {
|
|
147
172
|
return {
|
|
@@ -243,9 +268,10 @@ export class HitlHandler {
|
|
|
243
268
|
/**
|
|
244
269
|
* 中断 Graph 执行,等待人工确认
|
|
245
270
|
*
|
|
246
|
-
* 写入 `__hitlWaiting`(含 deadlineAt
|
|
247
|
-
*
|
|
248
|
-
*
|
|
271
|
+
* 写入 `__hitlWaiting`(含 deadlineAt)到当前执行图的 Checkpoint 槽。
|
|
272
|
+
* 父图槽=sessionId;子图槽=sessionId#父节点id(与 LangGraph put 同一把钥匙)。
|
|
273
|
+
* 不保留 LangGraph `engineSnapshot`:resume 从父图入口重入
|
|
274
|
+
*(skillSpanId / hitlChoice 防重复 span;子图 resume 去掉 checkpointNs / 内层 historyPath / currentNode)。
|
|
249
275
|
*/
|
|
250
276
|
async interrupt(taskId, threadId, state, config, extras = {}) {
|
|
251
277
|
const work = () => this.interruptCore(taskId, threadId, state, config, extras);
|
|
@@ -300,6 +326,28 @@ export class HitlHandler {
|
|
|
300
326
|
};
|
|
301
327
|
const persistedCheckpoint = stripCheckpointCollectors(checkpointData);
|
|
302
328
|
await this.checkpointSaver.save(threadId, persistedCheckpoint);
|
|
329
|
+
const parentSessionId = typeof state.sessionId === 'string' && state.sessionId.trim()
|
|
330
|
+
? state.sessionId.trim()
|
|
331
|
+
: '';
|
|
332
|
+
if (parentSessionId && threadId !== parentSessionId) {
|
|
333
|
+
const parent = await this.checkpointSaver.load(parentSessionId);
|
|
334
|
+
const pointerState = {
|
|
335
|
+
...(parent?.state ?? { messages: [], userId: state.userId ?? '', sessionId: parentSessionId }),
|
|
336
|
+
[HITL_WAITING_SLOT_KEY]: threadId,
|
|
337
|
+
};
|
|
338
|
+
await this.checkpointSaver.save(parentSessionId, stripCheckpointCollectors({
|
|
339
|
+
checkpointId: parent?.checkpointId ?? `hitl-slot-${taskId}`,
|
|
340
|
+
threadId: parentSessionId,
|
|
341
|
+
currentNodeId: parent?.currentNodeId ?? '__hitl_waiting__',
|
|
342
|
+
historyPath: parent?.historyPath ?? [],
|
|
343
|
+
state: pointerState,
|
|
344
|
+
createdBy: parent?.createdBy ?? 'system',
|
|
345
|
+
createdAt: parent?.createdAt ?? now,
|
|
346
|
+
updatedBy: 'system',
|
|
347
|
+
updatedAt: now,
|
|
348
|
+
isDeleted: false,
|
|
349
|
+
}));
|
|
350
|
+
}
|
|
303
351
|
const liveCollector = getExecutionTrace()?.collector;
|
|
304
352
|
if (liveCollector) {
|
|
305
353
|
this.waitingCollectors.set(taskId, liveCollector);
|
|
@@ -352,13 +400,14 @@ export class HitlHandler {
|
|
|
352
400
|
async resume(taskId, choice, userId, threadIdHint) {
|
|
353
401
|
return this.withTaskLock(taskId, async () => {
|
|
354
402
|
let waiting = this.waitingTasks.get(taskId);
|
|
355
|
-
const
|
|
356
|
-
if (!
|
|
403
|
+
const hinted = waiting?.threadId ?? threadIdHint;
|
|
404
|
+
if (!hinted) {
|
|
357
405
|
throw new HitlNotWaitingError(taskId);
|
|
358
406
|
}
|
|
359
|
-
const
|
|
407
|
+
const resolved = await loadHitlCheckpoint(this.checkpointSaver, hinted);
|
|
408
|
+
const checkpoint = resolved.checkpoint;
|
|
360
409
|
if (!checkpoint) {
|
|
361
|
-
throw new Error(`Checkpoint not found for thread: ${threadId}`);
|
|
410
|
+
throw new Error(`Checkpoint not found for thread: ${resolved.threadId}`);
|
|
362
411
|
}
|
|
363
412
|
const persisted = readPersistedHitlWaiting(checkpoint.state);
|
|
364
413
|
if (persisted?.timedOut) {
|
|
@@ -380,10 +429,19 @@ export class HitlHandler {
|
|
|
380
429
|
const baseState = prepareHitlCheckpointState(checkpoint.state);
|
|
381
430
|
const cleaned = { ...baseState };
|
|
382
431
|
delete cleaned[HITL_WAITING_STATE_KEY];
|
|
432
|
+
const subgraphNs = typeof cleaned['checkpointNs'] === 'string' ? String(cleaned['checkpointNs']).trim() : '';
|
|
433
|
+
delete cleaned['checkpointNs'];
|
|
434
|
+
if (subgraphNs) {
|
|
435
|
+
// 父图入口重入:内层路径不能带进父图,否则 historyPath 混入子图节点。
|
|
436
|
+
delete cleaned['historyPath'];
|
|
437
|
+
delete cleaned['currentNode'];
|
|
438
|
+
}
|
|
383
439
|
const resumedState = {
|
|
384
440
|
...cleaned,
|
|
385
441
|
hitlChoice: choice,
|
|
386
442
|
hitlRespondedBy: userId,
|
|
443
|
+
...(waiting.toolSpanId ? { toolSpanId: waiting.toolSpanId } : {}),
|
|
444
|
+
...(waiting.toolCallId ? { hitlToolCallId: waiting.toolCallId } : {}),
|
|
387
445
|
};
|
|
388
446
|
const now = new Date().toISOString();
|
|
389
447
|
await this.checkpointSaver.save(waiting.threadId, stripCheckpointCollectors({
|
|
@@ -393,6 +451,21 @@ export class HitlHandler {
|
|
|
393
451
|
updatedBy: userId,
|
|
394
452
|
updatedAt: now,
|
|
395
453
|
}));
|
|
454
|
+
const parentSessionId = typeof resumedState.sessionId === 'string' && resumedState.sessionId.trim()
|
|
455
|
+
? resumedState.sessionId.trim()
|
|
456
|
+
: '';
|
|
457
|
+
if (parentSessionId && waiting.threadId !== parentSessionId) {
|
|
458
|
+
const parent = await this.checkpointSaver.load(parentSessionId);
|
|
459
|
+
if (readHitlWaitingSlot(parent?.state) === waiting.threadId && parent) {
|
|
460
|
+
const parentState = { ...parent.state };
|
|
461
|
+
delete parentState[HITL_WAITING_SLOT_KEY];
|
|
462
|
+
await this.checkpointSaver.save(parentSessionId, stripCheckpointCollectors({
|
|
463
|
+
...parent,
|
|
464
|
+
state: parentState,
|
|
465
|
+
updatedAt: now,
|
|
466
|
+
}));
|
|
467
|
+
}
|
|
468
|
+
}
|
|
396
469
|
const hitlSpanId = waiting.hitlSpanId ?? persisted?.hitlSpanId;
|
|
397
470
|
const resumeTraceId = resolveTraceId(resumedState) ?? resolveTraceId(checkpoint.state);
|
|
398
471
|
const resumeCtx = liveCollector && resumeTraceId
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* @see requirements.md Requirement 1 AC6, Requirement 9
|
|
11
11
|
*/
|
|
12
12
|
import type { NodeRetryConfig, GraphMessage } from '../contracts/executable-graph.js';
|
|
13
|
+
import { type ExecutionInterceptors } from '../../execution/interceptors.js';
|
|
13
14
|
/**
|
|
14
15
|
* LLMNodeConfig — LLM 节点配置
|
|
15
16
|
*/
|
|
@@ -132,6 +133,7 @@ type LangGraphState = Record<string, unknown>;
|
|
|
132
133
|
* 5. 将 LLM 响应追加到 state.messages 并返回
|
|
133
134
|
*/
|
|
134
135
|
export declare class LLMNode {
|
|
136
|
+
private readonly interceptors?;
|
|
135
137
|
private readonly config;
|
|
136
138
|
private readonly provider;
|
|
137
139
|
private readonly retryConfig;
|
|
@@ -145,7 +147,7 @@ export declare class LLMNode {
|
|
|
145
147
|
* @param streaming - 是否启用流式输出(默认 false)
|
|
146
148
|
* @param onChunk - 流式模式下每个文本片段回调(用于 SSE response.chunk)
|
|
147
149
|
*/
|
|
148
|
-
constructor(nodeId: string, config: LLMNodeConfig, provider: ILLMProvider, streaming?: boolean, onChunk?: (chunk: string) => void);
|
|
150
|
+
constructor(nodeId: string, config: LLMNodeConfig, provider: ILLMProvider, streaming?: boolean, onChunk?: (chunk: string) => void, interceptors?: Pick<ExecutionInterceptors, "beforeModel" | "afterModel"> | undefined);
|
|
149
151
|
/**
|
|
150
152
|
* execute — StateGraph 节点函数
|
|
151
153
|
*
|
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
import { formatUnknownError } from '../../utils/format-error.js';
|
|
13
13
|
import { buildContextUsage, turnSectionChars } from '../../token/context-window.js';
|
|
14
14
|
import { isAbortError } from '../../tool/timeout.js';
|
|
15
|
+
import { emitRuntimeLog } from '../../common/logger.js';
|
|
16
|
+
import { decideBeforeModel, } from '../../execution/interceptors.js';
|
|
15
17
|
// ─────────────────────────────────────────────
|
|
16
18
|
// Default retry config
|
|
17
19
|
// ─────────────────────────────────────────────
|
|
@@ -38,6 +40,7 @@ const DEFAULT_RETRY_CONFIG = {
|
|
|
38
40
|
* 5. 将 LLM 响应追加到 state.messages 并返回
|
|
39
41
|
*/
|
|
40
42
|
export class LLMNode {
|
|
43
|
+
interceptors;
|
|
41
44
|
config;
|
|
42
45
|
provider;
|
|
43
46
|
retryConfig;
|
|
@@ -51,7 +54,8 @@ export class LLMNode {
|
|
|
51
54
|
* @param streaming - 是否启用流式输出(默认 false)
|
|
52
55
|
* @param onChunk - 流式模式下每个文本片段回调(用于 SSE response.chunk)
|
|
53
56
|
*/
|
|
54
|
-
constructor(nodeId, config, provider, streaming = false, onChunk) {
|
|
57
|
+
constructor(nodeId, config, provider, streaming = false, onChunk, interceptors) {
|
|
58
|
+
this.interceptors = interceptors;
|
|
55
59
|
this.nodeId = nodeId;
|
|
56
60
|
this.config = config;
|
|
57
61
|
this.provider = provider;
|
|
@@ -71,10 +75,30 @@ export class LLMNode {
|
|
|
71
75
|
const messages = this.extractMessages(state);
|
|
72
76
|
// 注入 system prompt
|
|
73
77
|
const messagesWithSystem = this.prependSystemPrompt(messages);
|
|
78
|
+
const modelInput = {
|
|
79
|
+
messages: messagesWithSystem,
|
|
80
|
+
config: this.config,
|
|
81
|
+
nodeId: this.nodeId,
|
|
82
|
+
};
|
|
83
|
+
const gate = await decideBeforeModel(this.interceptors?.beforeModel, modelInput);
|
|
84
|
+
if (gate.action === 'deny') {
|
|
85
|
+
throw new Error(gate.reason);
|
|
86
|
+
}
|
|
87
|
+
const messagesForModel = gate.messages ?? messagesWithSystem;
|
|
74
88
|
const startedAt = Date.now();
|
|
75
89
|
// 带重试的 LLM 调用
|
|
76
|
-
const response = await this.invokeWithRetry(
|
|
90
|
+
const response = await this.invokeWithRetry(messagesForModel);
|
|
77
91
|
const durationMs = Date.now() - startedAt;
|
|
92
|
+
try {
|
|
93
|
+
await this.interceptors?.afterModel?.({ ...modelInput, messages: messagesForModel }, response);
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
emitRuntimeLog('warn', 'runtime.intercept', 'afterModel failed', {
|
|
97
|
+
event: 'runtime.intercept.after_model_failed',
|
|
98
|
+
nodeId: this.nodeId,
|
|
99
|
+
error: err instanceof Error ? err.message : String(err),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
78
102
|
// 构建 assistant message(保留 reasoningContent 供后续 HTTP 回传)
|
|
79
103
|
const assistantMessage = {
|
|
80
104
|
role: 'assistant',
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dedup — 确定性去重(纯函数,不经 LLM)
|
|
3
|
+
*
|
|
4
|
+
* 精确去重(源内、fuse 前):sourceId:rawId + 归一化 content hash。
|
|
5
|
+
* 约束:入参须先经收集侧(react-loop pushDocs)消除"同 id 不同 content"撞号
|
|
6
|
+
* (重编号 `${rawId}#r${n}`)——本函数 id 命中即弃仅对真重复成立;
|
|
7
|
+
* fuse 的跨源键 sourceId:rawId 同样依赖该唯一性。
|
|
8
|
+
* 近重去重(融合后、降噪判定前,M4):字符 3-gram Jaccard 超 nearDupThreshold
|
|
9
|
+
* 判近重,保留序列靠前者(design-v2 §6.3 ①;跨源候选集几乎不重叠,属低频兜底)。
|
|
10
|
+
*/
|
|
11
|
+
import type { RetrievalDoc } from './types.js';
|
|
12
|
+
export declare function dedupExact(docs: RetrievalDoc[]): RetrievalDoc[];
|
|
13
|
+
/**
|
|
14
|
+
* 归一化内容相等(与 dedupExact 内容去重同口径):"重复"判定单一出处——
|
|
15
|
+
* 收集侧(react-loop pushDocs)跳过同 id 真重复时必须与融合前去重用同一口径,
|
|
16
|
+
* 两处口径分叉会导致同 id 同排版差异的文档一处跳过一处保留。
|
|
17
|
+
*/
|
|
18
|
+
export declare function sameContent(a: string, b: string): boolean;
|
|
19
|
+
/** 近重去重:字符 n-gram Jaccard 超阈值即判近重,保留序列靠前者 */
|
|
20
|
+
export declare function dedupNearDup(docs: RetrievalDoc[], threshold?: number): RetrievalDoc[];
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dedup — 确定性去重(纯函数,不经 LLM)
|
|
3
|
+
*
|
|
4
|
+
* 精确去重(源内、fuse 前):sourceId:rawId + 归一化 content hash。
|
|
5
|
+
* 约束:入参须先经收集侧(react-loop pushDocs)消除"同 id 不同 content"撞号
|
|
6
|
+
* (重编号 `${rawId}#r${n}`)——本函数 id 命中即弃仅对真重复成立;
|
|
7
|
+
* fuse 的跨源键 sourceId:rawId 同样依赖该唯一性。
|
|
8
|
+
* 近重去重(融合后、降噪判定前,M4):字符 3-gram Jaccard 超 nearDupThreshold
|
|
9
|
+
* 判近重,保留序列靠前者(design-v2 §6.3 ①;跨源候选集几乎不重叠,属低频兜底)。
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
export function dedupExact(docs) {
|
|
13
|
+
const seenIds = new Set();
|
|
14
|
+
const seenContent = new Set();
|
|
15
|
+
const out = [];
|
|
16
|
+
for (const d of docs) {
|
|
17
|
+
const idKey = `${d.sourceId}:${d.rawId}`;
|
|
18
|
+
const contentKey = hashContent(d.content);
|
|
19
|
+
if (seenIds.has(idKey) || seenContent.has(contentKey))
|
|
20
|
+
continue;
|
|
21
|
+
seenIds.add(idKey);
|
|
22
|
+
seenContent.add(contentKey);
|
|
23
|
+
out.push(d);
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
}
|
|
27
|
+
function hashContent(content) {
|
|
28
|
+
// 归一化:压空白后取 hash,挡"内容相同仅排版不同"
|
|
29
|
+
const normalized = content.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
30
|
+
return createHash('sha1').update(normalized).digest('hex');
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* 归一化内容相等(与 dedupExact 内容去重同口径):"重复"判定单一出处——
|
|
34
|
+
* 收集侧(react-loop pushDocs)跳过同 id 真重复时必须与融合前去重用同一口径,
|
|
35
|
+
* 两处口径分叉会导致同 id 同排版差异的文档一处跳过一处保留。
|
|
36
|
+
*/
|
|
37
|
+
export function sameContent(a, b) {
|
|
38
|
+
return hashContent(a) === hashContent(b);
|
|
39
|
+
}
|
|
40
|
+
/** 近重去重:字符 n-gram Jaccard 超阈值即判近重,保留序列靠前者 */
|
|
41
|
+
export function dedupNearDup(docs, threshold = 0.8) {
|
|
42
|
+
const kept = [];
|
|
43
|
+
const keptGrams = [];
|
|
44
|
+
for (const d of docs) {
|
|
45
|
+
const grams = charGrams(d.content);
|
|
46
|
+
let near = false;
|
|
47
|
+
for (const g of keptGrams) {
|
|
48
|
+
if (jaccard(grams, g) > threshold) {
|
|
49
|
+
near = true;
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (near)
|
|
54
|
+
continue;
|
|
55
|
+
kept.push(d);
|
|
56
|
+
keptGrams.push(grams);
|
|
57
|
+
}
|
|
58
|
+
return kept;
|
|
59
|
+
}
|
|
60
|
+
const NGRAM_SIZE = 3;
|
|
61
|
+
function charGrams(content) {
|
|
62
|
+
const norm = content.replace(/\s+/g, ' ').trim();
|
|
63
|
+
const grams = new Set();
|
|
64
|
+
for (let i = 0; i + NGRAM_SIZE <= norm.length; i++) {
|
|
65
|
+
grams.add(norm.slice(i, i + NGRAM_SIZE));
|
|
66
|
+
}
|
|
67
|
+
return grams;
|
|
68
|
+
}
|
|
69
|
+
function jaccard(a, b) {
|
|
70
|
+
if (a.size === 0 || b.size === 0)
|
|
71
|
+
return 0;
|
|
72
|
+
let inter = 0;
|
|
73
|
+
for (const x of a)
|
|
74
|
+
if (b.has(x))
|
|
75
|
+
inter++;
|
|
76
|
+
return inter / (a.size + b.size - inter);
|
|
77
|
+
}
|