@devflow-tools/memory-engine 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -26
- package/dist/consolidator.d.ts.map +1 -1
- package/dist/consolidator.js +31 -146
- package/dist/consolidator.js.map +1 -1
- package/dist/embedding-provider.d.ts +21 -1
- package/dist/embedding-provider.d.ts.map +1 -1
- package/dist/embedding-provider.js +157 -24
- package/dist/embedding-provider.js.map +1 -1
- package/dist/event-grouper.d.ts.map +1 -1
- package/dist/event-grouper.js +43 -1
- package/dist/event-grouper.js.map +1 -1
- package/dist/hybrid-search.d.ts +1 -1
- package/dist/hybrid-search.d.ts.map +1 -1
- package/dist/hybrid-search.js +44 -43
- package/dist/hybrid-search.js.map +1 -1
- package/dist/index.d.ts +6 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -5
- package/dist/index.js.map +1 -1
- package/dist/lease-store.d.ts +33 -0
- package/dist/lease-store.d.ts.map +1 -0
- package/dist/lease-store.js +178 -0
- package/dist/lease-store.js.map +1 -0
- package/dist/lease-store.test.d.ts +2 -0
- package/dist/lease-store.test.d.ts.map +1 -0
- package/dist/lease-store.test.js +167 -0
- package/dist/lease-store.test.js.map +1 -0
- package/dist/memory-engine.d.ts +12 -18
- package/dist/memory-engine.d.ts.map +1 -1
- package/dist/memory-engine.js +220 -148
- package/dist/memory-engine.js.map +1 -1
- package/dist/memory-gate.d.ts +64 -39
- package/dist/memory-gate.d.ts.map +1 -1
- package/dist/memory-gate.js +379 -45
- package/dist/memory-gate.js.map +1 -1
- package/dist/memory-store.d.ts +36 -1
- package/dist/memory-store.d.ts.map +1 -1
- package/dist/memory-store.js +142 -21
- package/dist/memory-store.js.map +1 -1
- package/dist/migrations/v2-to-v3.d.ts +3 -0
- package/dist/migrations/v2-to-v3.d.ts.map +1 -0
- package/dist/migrations/v2-to-v3.js +122 -0
- package/dist/migrations/v2-to-v3.js.map +1 -0
- package/dist/migrations/v2-to-v3.test.d.ts +2 -0
- package/dist/migrations/v2-to-v3.test.d.ts.map +1 -0
- package/dist/migrations/v2-to-v3.test.js +88 -0
- package/dist/migrations/v2-to-v3.test.js.map +1 -0
- package/dist/observation-types.d.ts +7 -0
- package/dist/observation-types.d.ts.map +1 -0
- package/dist/observation-types.js +27 -0
- package/dist/observation-types.js.map +1 -0
- package/dist/project-validator.d.ts.map +1 -1
- package/dist/project-validator.js +1 -2
- package/dist/project-validator.js.map +1 -1
- package/dist/rule-extractor.d.ts +14 -0
- package/dist/rule-extractor.d.ts.map +1 -0
- package/dist/rule-extractor.js +109 -0
- package/dist/rule-extractor.js.map +1 -0
- package/dist/session-summarizer.d.ts +16 -8
- package/dist/session-summarizer.d.ts.map +1 -1
- package/dist/session-summarizer.js +171 -160
- package/dist/session-summarizer.js.map +1 -1
- package/dist/{signal-detector.d.ts → signal-analyzer.d.ts} +6 -2
- package/dist/signal-analyzer.d.ts.map +1 -0
- package/dist/{signal-detector.js → signal-analyzer.js} +34 -3
- package/dist/signal-analyzer.js.map +1 -0
- package/dist/signal-buffer.d.ts +3 -0
- package/dist/signal-buffer.d.ts.map +1 -1
- package/dist/signal-buffer.js.map +1 -1
- package/package.json +6 -6
- package/dist/extractor.d.ts +0 -39
- package/dist/extractor.d.ts.map +0 -1
- package/dist/extractor.js +0 -275
- package/dist/extractor.js.map +0 -1
- package/dist/l2-evaluator.d.ts +0 -31
- package/dist/l2-evaluator.d.ts.map +0 -1
- package/dist/l2-evaluator.js +0 -103
- package/dist/l2-evaluator.js.map +0 -1
- package/dist/llm-provider.d.ts +0 -9
- package/dist/llm-provider.d.ts.map +0 -1
- package/dist/llm-provider.js +0 -37
- package/dist/llm-provider.js.map +0 -1
- package/dist/ollama-checker.d.ts +0 -10
- package/dist/ollama-checker.d.ts.map +0 -1
- package/dist/ollama-checker.js +0 -43
- package/dist/ollama-checker.js.map +0 -1
- package/dist/signal-detector.d.ts.map +0 -1
- package/dist/signal-detector.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @devflow-tools/memory-engine
|
|
2
2
|
|
|
3
|
-
DevFlow
|
|
3
|
+
DevFlow 记忆引擎存储 session tool events、结构化 observations 和已审核记忆,并为宿主模型驱动的 MCP 提炼流程提供持久化与搜索能力。
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
@@ -11,47 +11,48 @@ npm install @devflow-tools/memory-engine
|
|
|
11
11
|
## Usage
|
|
12
12
|
|
|
13
13
|
```typescript
|
|
14
|
-
import {
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
14
|
+
import { MemoryGate } from '@devflow-tools/memory-engine';
|
|
15
|
+
|
|
16
|
+
const memory = new MemoryGate('/path/to/project');
|
|
17
|
+
await memory.forceWarmUp();
|
|
18
|
+
|
|
19
|
+
// Hook producers record events for next-session MCP distillation.
|
|
20
|
+
await memory.recordEvent({
|
|
21
|
+
id: 'event-1',
|
|
22
|
+
sessionId: 'session-1',
|
|
23
|
+
tool: 'Bash',
|
|
24
|
+
command: 'npm test',
|
|
25
|
+
exitCode: 1,
|
|
26
|
+
stderr: 'Tests: 1 failed',
|
|
24
27
|
durationMs: 3200,
|
|
28
|
+
createdAt: Date.now(),
|
|
25
29
|
});
|
|
26
30
|
|
|
27
|
-
//
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
// 查看候选记忆
|
|
31
|
-
const candidates = await engine.getCandidates();
|
|
32
|
-
|
|
33
|
-
// 审核候选
|
|
34
|
-
await engine.approveCandidate('mem:123');
|
|
35
|
-
await engine.rejectCandidate('mem:456');
|
|
31
|
+
// On the next session, the host calls the MCP tool memory_request_distill.
|
|
32
|
+
// The host model summarizes the leased events and saves observations through
|
|
33
|
+
// memory_save_distilled. No local model service is required.
|
|
36
34
|
```
|
|
37
35
|
|
|
38
36
|
## Features
|
|
39
37
|
|
|
40
|
-
- **
|
|
41
|
-
- **
|
|
42
|
-
-
|
|
38
|
+
- **Session event storage** — 持久化 hook tool events,供下一会话按批次租约读取
|
|
39
|
+
- **Host-model distillation** — `memory_request_distill` 返回待处理事件,宿主模型生成 observations
|
|
40
|
+
- **Signal analysis** — 规则评分用于缓冲和分组,不生成 LLM 记忆
|
|
41
|
+
- **候选审核** — 结构化记忆支持 approve/reject 生命周期
|
|
43
42
|
- **分层管理** — 项目层(技术栈、约定)、用户层(偏好)、全局层(最佳实践)
|
|
44
43
|
- **BM25 搜索** — 按内容和类型搜索已审核记忆
|
|
45
44
|
|
|
46
45
|
## API
|
|
47
46
|
|
|
48
|
-
- `new
|
|
47
|
+
- `new MemoryGate(projectRoot)` — 创建项目记忆入口
|
|
48
|
+
- `.forceWarmUp()` — 初始化项目存储
|
|
49
49
|
- `.getAll()` — 获取所有已审核记忆
|
|
50
50
|
- `.getCandidates()` — 获取待审核的候选记忆
|
|
51
51
|
- `.approveCandidate(id)` — 批准候选记忆
|
|
52
52
|
- `.rejectCandidate(id)` — 拒绝候选记忆(标记为 rejected)
|
|
53
|
-
- `.recordEvent(event)` —
|
|
54
|
-
- `.
|
|
53
|
+
- `.recordEvent(event)` — 将 hook event 写入 `session_events`
|
|
54
|
+
- `.getPendingDistillData()` — 为 MCP `memory_request_distill` 租约并返回待处理事件
|
|
55
|
+
- `.saveDistilledObservationsByBatch(batchId, observations)` — 保存宿主模型提炼结果
|
|
55
56
|
- `.search(query, options?)` — 搜索记忆(支持 category/source 过滤)
|
|
56
57
|
- `.listMemories(filter?)` — 按条件列出记忆
|
|
57
58
|
- `.getUserPreferences()` — 获取用户偏好
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consolidator.d.ts","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"consolidator.d.ts","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,wBAAsB,WAAW,CAC/B,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE;IACR,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,OAAO,CAAC,kBAAkB,CAAC,CA6E7B"}
|
package/dist/consolidator.js
CHANGED
|
@@ -1,23 +1,3 @@
|
|
|
1
|
-
import { callLLM } from './llm-provider.js';
|
|
2
|
-
// ── Prompt ──────────────────────────────────────────────
|
|
3
|
-
const CONSOLIDATOR_SYSTEM = `You are a knowledge consolidator for DevFlow.
|
|
4
|
-
Given multiple observations about the same concept from different coding sessions,
|
|
5
|
-
produce ONE consolidated memory entry.
|
|
6
|
-
|
|
7
|
-
Output JSON:
|
|
8
|
-
{
|
|
9
|
-
"type": "semantic" | "pattern" | "constraint" | "preference" | "lesson",
|
|
10
|
-
"title": "≤80 chars",
|
|
11
|
-
"content": "1-2 sentence consolidated knowledge",
|
|
12
|
-
"confidence": 0.0-1.0,
|
|
13
|
-
"supersedes_observation_ids": ["obs:...", ...]
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
Rules:
|
|
17
|
-
- Merge complementary facts; discard contradictory outliers
|
|
18
|
-
- Higher confidence when observations are consistent across sessions
|
|
19
|
-
- If observations disagree, pick the majority view and note uncertainty in content
|
|
20
|
-
- Return ONLY the JSON object`;
|
|
21
1
|
// ── Consolidator ────────────────────────────────────────
|
|
22
2
|
export async function consolidate(store, projectRoot, options) {
|
|
23
3
|
const start = Date.now();
|
|
@@ -46,139 +26,44 @@ export async function consolidate(store, projectRoot, options) {
|
|
|
46
26
|
.map(id => store.getObservation(id))
|
|
47
27
|
.filter(Boolean),
|
|
48
28
|
}));
|
|
49
|
-
// 3.
|
|
50
|
-
let provider = 'rules';
|
|
29
|
+
// 3. Rule-based: pick highest-importance observation per cluster as representative
|
|
51
30
|
let consolidated = 0;
|
|
52
|
-
|
|
53
|
-
const
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
type: 'consolidation',
|
|
82
|
-
weight: entry.confidence ?? 0.7,
|
|
83
|
-
sourceObservationIds: obsIds,
|
|
84
|
-
summary: `Consolidated from ${obsIds.length} observations about "${entry.title ?? entry.type}"`,
|
|
85
|
-
});
|
|
86
|
-
// 7. Audit
|
|
87
|
-
store.recordHistory({
|
|
88
|
-
memoryId: memId,
|
|
89
|
-
event: 'ADD',
|
|
90
|
-
newContent: entry.content,
|
|
91
|
-
reason: 'consolidation',
|
|
92
|
-
});
|
|
93
|
-
consolidated++;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
else {
|
|
97
|
-
// Rule fallback: pick highest-importance observation per cluster as representative
|
|
98
|
-
for (const cluster of clusterData) {
|
|
99
|
-
const best = cluster.observations.reduce((a, b) => (a?.importance ?? 0) >= (b?.importance ?? 0) ? a : b);
|
|
100
|
-
if (!best)
|
|
101
|
-
continue;
|
|
102
|
-
const memId = `mem:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
|
|
103
|
-
store.add({
|
|
104
|
-
id: memId,
|
|
105
|
-
type: 'semantic',
|
|
106
|
-
content: best.narrative || best.title,
|
|
107
|
-
title: best.title,
|
|
108
|
-
source: 'consolidation',
|
|
109
|
-
status: 'active',
|
|
110
|
-
confidence: best.confidence,
|
|
111
|
-
});
|
|
112
|
-
store.linkObservationToMemory(best.id, memId);
|
|
113
|
-
store.addEvidence({
|
|
114
|
-
memoryId: memId,
|
|
115
|
-
type: 'consolidation',
|
|
116
|
-
weight: best.confidence,
|
|
117
|
-
sourceObservationIds: [best.id],
|
|
118
|
-
summary: `Rule-based consolidation from observation about "${cluster.concept}"`,
|
|
119
|
-
});
|
|
120
|
-
store.recordHistory({
|
|
121
|
-
memoryId: memId,
|
|
122
|
-
event: 'ADD',
|
|
123
|
-
newContent: best.narrative ?? best.title,
|
|
124
|
-
reason: 'consolidation_rule_fallback',
|
|
125
|
-
});
|
|
126
|
-
consolidated++;
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
catch (err) {
|
|
131
|
-
console.warn('[Consolidator] Consolidation failed', { error: err.message });
|
|
132
|
-
// Rule fallback on error
|
|
133
|
-
for (const cluster of clusterData) {
|
|
134
|
-
const best = cluster.observations.reduce((a, b) => (a?.importance ?? 0) >= (b?.importance ?? 0) ? a : b);
|
|
135
|
-
if (!best)
|
|
136
|
-
continue;
|
|
137
|
-
const memId = `mem:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
|
|
138
|
-
store.add({
|
|
139
|
-
id: memId,
|
|
140
|
-
type: 'semantic',
|
|
141
|
-
content: best.narrative || best.title,
|
|
142
|
-
title: best.title,
|
|
143
|
-
source: 'consolidation',
|
|
144
|
-
status: 'active',
|
|
145
|
-
confidence: best.confidence,
|
|
146
|
-
});
|
|
147
|
-
store.linkObservationToMemory(best.id, memId);
|
|
148
|
-
store.addEvidence({
|
|
149
|
-
memoryId: memId,
|
|
150
|
-
type: 'consolidation',
|
|
151
|
-
weight: best.confidence,
|
|
152
|
-
sourceObservationIds: [best.id],
|
|
153
|
-
summary: `Error-fallback consolidation from observation about "${cluster.concept}"`,
|
|
154
|
-
});
|
|
155
|
-
store.recordHistory({
|
|
156
|
-
memoryId: memId,
|
|
157
|
-
event: 'ADD',
|
|
158
|
-
newContent: best.narrative ?? best.title,
|
|
159
|
-
reason: 'consolidation_error_fallback',
|
|
160
|
-
});
|
|
161
|
-
consolidated++;
|
|
162
|
-
}
|
|
31
|
+
for (const cluster of clusterData) {
|
|
32
|
+
const best = cluster.observations.reduce((a, b) => (a?.importance ?? 0) >= (b?.importance ?? 0) ? a : b);
|
|
33
|
+
if (!best)
|
|
34
|
+
continue;
|
|
35
|
+
const memId = `mem:${Date.now()}:${Math.random().toString(36).slice(2, 10)}`;
|
|
36
|
+
store.add({
|
|
37
|
+
id: memId,
|
|
38
|
+
type: 'semantic',
|
|
39
|
+
content: best.narrative || best.title,
|
|
40
|
+
title: best.title,
|
|
41
|
+
source: 'consolidation',
|
|
42
|
+
status: 'active',
|
|
43
|
+
confidence: best.confidence,
|
|
44
|
+
});
|
|
45
|
+
store.linkObservationToMemory(best.id, memId);
|
|
46
|
+
store.addEvidence({
|
|
47
|
+
memoryId: memId,
|
|
48
|
+
type: 'consolidation',
|
|
49
|
+
weight: best.confidence,
|
|
50
|
+
sourceObservationIds: [best.id],
|
|
51
|
+
summary: `Consolidation from observation about "${cluster.concept}"`,
|
|
52
|
+
});
|
|
53
|
+
store.recordHistory({
|
|
54
|
+
memoryId: memId,
|
|
55
|
+
event: 'ADD',
|
|
56
|
+
newContent: best.narrative ?? best.title,
|
|
57
|
+
reason: 'consolidation',
|
|
58
|
+
});
|
|
59
|
+
consolidated++;
|
|
163
60
|
}
|
|
164
61
|
const latencyMs = Date.now() - start;
|
|
165
62
|
console.log('[Consolidator] Consolidation complete', {
|
|
166
63
|
consolidated,
|
|
167
64
|
skipped: clusters.length - consolidated,
|
|
168
|
-
provider,
|
|
169
65
|
latencyMs,
|
|
170
66
|
});
|
|
171
|
-
return { consolidated, skipped: clusters.length - consolidated, provider, latencyMs };
|
|
172
|
-
}
|
|
173
|
-
// ── Helpers ─────────────────────────────────────────────
|
|
174
|
-
function buildConsolidationPrompt(clusters) {
|
|
175
|
-
const parts = clusters.map((c, i) => {
|
|
176
|
-
const obsText = c.observations
|
|
177
|
-
.filter(Boolean)
|
|
178
|
-
.map(o => ` - [${o.type}] ${o.title}: ${o.narrative || 'no narrative'} (importance: ${o.importance}, session: ${o.session_id}, id: ${o.id})`)
|
|
179
|
-
.join('\n');
|
|
180
|
-
return `Cluster ${i + 1}: Concept "${c.concept}" (${c.observationCount} observations across ${c.sessionCount} sessions)\n${obsText}`;
|
|
181
|
-
});
|
|
182
|
-
return `Consolidate the following observation clusters into memory entries. Return a JSON array with one entry per cluster.\n\n${parts.join('\n\n')}`;
|
|
67
|
+
return { consolidated, skipped: clusters.length - consolidated, provider: 'rules', latencyMs };
|
|
183
68
|
}
|
|
184
69
|
//# sourceMappingURL=consolidator.js.map
|
package/dist/consolidator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consolidator.js","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"consolidator.js","sourceRoot":"","sources":["../src/consolidator.ts"],"names":[],"mappings":"AASA,2DAA2D;AAE3D,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAkB,EAClB,WAAmB,EACnB,OAIC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,OAAO,EAAE,eAAe,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,EAAE,WAAW,IAAI,CAAC,CAAC;IAE1C,2DAA2D;IAC3D,MAAM,QAAQ,GAAG,KAAK,CAAC,oBAAoB,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;QAC3F,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAC1F,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,gCAAgC,EAAE;QAC5C,KAAK,EAAE,QAAQ,CAAC,MAAM;QACtB,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3B,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;YACpC,YAAY,EAAE,CAAC,CAAC,YAAY;SAC7B,CAAC,CAAC;KACJ,CAAC,CAAC;IAEH,kDAAkD;IAClD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrC,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;QACpC,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,YAAY,EAAE,CAAC,CAAC,cAAc;aAC3B,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;aACnC,MAAM,CAAC,OAAO,CAAC;KACnB,CAAC,CAAC,CAAC;IAEJ,mFAAmF;IACnF,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAChD,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACrD,CAAC;QACF,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAC7E,KAAK,CAAC,GAAG,CAAC;YACR,EAAE,EAAE,KAAK;YACT,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK;YACrC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,eAAe;YACvB,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;QAEH,KAAK,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC9C,KAAK,CAAC,WAAW,CAAC;YAChB,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE,IAAI,CAAC,UAAU;YACvB,oBAAoB,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,yCAAyC,OAAO,CAAC,OAAO,GAAG;SACrE,CAAC,CAAC;QACH,KAAK,CAAC,aAAa,CAAC;YAClB,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK;YACxC,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;QAEH,YAAY,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,uCAAuC,EAAE;QACnD,YAAY;QACZ,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY;QACvC,SAAS;KACV,CAAC,CAAC;IAEH,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AACjG,CAAC"}
|
|
@@ -1,8 +1,28 @@
|
|
|
1
|
+
export type EmbeddingStatus = "idle" | "warming" | "ready" | "failed";
|
|
2
|
+
type EmbeddingPipeline = (texts: string[]) => Promise<Array<{
|
|
3
|
+
data: Float32Array;
|
|
4
|
+
}>>;
|
|
1
5
|
export declare class EmbeddingProvider {
|
|
2
|
-
private
|
|
6
|
+
private readonly pipelineLoader;
|
|
7
|
+
private status;
|
|
8
|
+
private lastError;
|
|
9
|
+
private initializationPromise;
|
|
10
|
+
private pipeline;
|
|
11
|
+
private errorLogPath;
|
|
12
|
+
constructor(pipelineLoader?: () => Promise<EmbeddingPipeline>);
|
|
3
13
|
isReady(): boolean;
|
|
14
|
+
getStatus(): EmbeddingStatus;
|
|
15
|
+
getLastError(): string | null;
|
|
4
16
|
initialize(): Promise<void>;
|
|
17
|
+
warmup(options?: {
|
|
18
|
+
timeoutMs?: number;
|
|
19
|
+
errorLogPath?: string;
|
|
20
|
+
}): Promise<boolean>;
|
|
5
21
|
embed(text: string): Promise<Float32Array>;
|
|
6
22
|
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
23
|
+
private markReady;
|
|
24
|
+
private recordFailure;
|
|
25
|
+
private logFailure;
|
|
7
26
|
}
|
|
27
|
+
export {};
|
|
8
28
|
//# sourceMappingURL=embedding-provider.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../src/embedding-provider.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../src/embedding-provider.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtE,KAAK,iBAAiB,GAAG,CACvB,KAAK,EAAE,MAAM,EAAE,KACZ,OAAO,CAAC,KAAK,CAAC;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,CAAC,CAAC,CAAC;AA0E5C,qBAAa,iBAAiB;IAa1B,OAAO,CAAC,QAAQ,CAAC,cAAc;IAZjC,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,qBAAqB,CAA8B;IAC3D,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,YAAY,CAKlB;gBAGiB,cAAc,GAAE,MAAM,OAAO,CAAC,iBAAiB,CAC5C;IAGtB,OAAO,IAAI,OAAO;IAIlB,SAAS,IAAI,eAAe;IAI5B,YAAY,IAAI,MAAM,GAAG,IAAI;IAI7B,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BrB,MAAM,CAAC,OAAO,GAAE;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,OAAO,CAAC;IA2BnB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAY1C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAY1D,OAAO,CAAC,SAAS;YAKH,aAAa;YAOb,UAAU;CAazB"}
|
|
@@ -1,35 +1,168 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
1
|
+
import { appendFile, chmod, mkdir } from "node:fs/promises";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
function splitBatchTensor(result, expectedCount) {
|
|
5
|
+
const [batchSize, embeddingSize] = result.dims ?? [];
|
|
6
|
+
if (batchSize === expectedCount
|
|
7
|
+
&& Number.isInteger(embeddingSize)
|
|
8
|
+
&& embeddingSize > 0
|
|
9
|
+
&& result.data.length === batchSize * embeddingSize) {
|
|
10
|
+
return Array.from({ length: batchSize }, (_, index) => ({
|
|
11
|
+
data: result.data.slice(index * embeddingSize, (index + 1) * embeddingSize),
|
|
12
|
+
}));
|
|
13
|
+
}
|
|
14
|
+
return [{ data: result.data }];
|
|
15
|
+
}
|
|
16
|
+
let sharedPipeline = null;
|
|
17
|
+
let sharedPipelinePromise = null;
|
|
18
|
+
function loadSharedPipeline() {
|
|
19
|
+
if (sharedPipeline)
|
|
20
|
+
return Promise.resolve(sharedPipeline);
|
|
21
|
+
if (sharedPipelinePromise)
|
|
22
|
+
return sharedPipelinePromise;
|
|
23
|
+
sharedPipelinePromise = (async () => {
|
|
24
|
+
const { pipeline: transformersPipeline, env: transformersEnv, } = await import("@xenova/transformers");
|
|
25
|
+
const remoteHost = process.env.DEVFLOW_HF_ENDPOINT ?? process.env.HF_ENDPOINT;
|
|
26
|
+
if (remoteHost) {
|
|
27
|
+
transformersEnv.remoteHost = remoteHost.endsWith("/")
|
|
28
|
+
? remoteHost
|
|
29
|
+
: `${remoteHost}/`;
|
|
30
|
+
}
|
|
31
|
+
const raw = await transformersPipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
|
|
32
|
+
const loaded = async (texts) => {
|
|
33
|
+
const result = await raw(texts, { pooling: "mean", normalize: true });
|
|
34
|
+
return Array.isArray(result)
|
|
35
|
+
? result
|
|
36
|
+
: splitBatchTensor(result, texts.length);
|
|
37
|
+
};
|
|
38
|
+
sharedPipeline = loaded;
|
|
39
|
+
return loaded;
|
|
40
|
+
})().catch((error) => {
|
|
41
|
+
sharedPipelinePromise = null;
|
|
42
|
+
throw error;
|
|
43
|
+
});
|
|
44
|
+
return sharedPipelinePromise;
|
|
45
|
+
}
|
|
46
|
+
function errorMessage(error) {
|
|
47
|
+
if (!(error instanceof Error))
|
|
48
|
+
return String(error);
|
|
49
|
+
const cause = error.cause instanceof Error ? error.cause.message : null;
|
|
50
|
+
return cause && cause !== error.message
|
|
51
|
+
? `${error.message}: ${cause}`
|
|
52
|
+
: error.message;
|
|
12
53
|
}
|
|
13
54
|
export class EmbeddingProvider {
|
|
14
|
-
|
|
55
|
+
pipelineLoader;
|
|
56
|
+
status = "idle";
|
|
57
|
+
lastError = null;
|
|
58
|
+
initializationPromise = null;
|
|
59
|
+
pipeline = null;
|
|
60
|
+
errorLogPath = join(homedir(), ".devflow", "errors", "embedder.log");
|
|
61
|
+
constructor(pipelineLoader = loadSharedPipeline) {
|
|
62
|
+
this.pipelineLoader = pipelineLoader;
|
|
63
|
+
}
|
|
15
64
|
isReady() {
|
|
16
|
-
return this.ready;
|
|
65
|
+
return this.status === "ready";
|
|
66
|
+
}
|
|
67
|
+
getStatus() {
|
|
68
|
+
return this.status;
|
|
17
69
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
this.
|
|
70
|
+
getLastError() {
|
|
71
|
+
return this.lastError;
|
|
72
|
+
}
|
|
73
|
+
initialize() {
|
|
74
|
+
if (this.pipeline) {
|
|
75
|
+
this.markReady();
|
|
76
|
+
return Promise.resolve();
|
|
77
|
+
}
|
|
78
|
+
if (this.initializationPromise)
|
|
79
|
+
return this.initializationPromise;
|
|
80
|
+
this.status = "warming";
|
|
81
|
+
const initialization = this.pipelineLoader()
|
|
82
|
+
.then((pipeline) => {
|
|
83
|
+
this.pipeline = pipeline;
|
|
84
|
+
this.status = "ready";
|
|
85
|
+
this.lastError = null;
|
|
86
|
+
})
|
|
87
|
+
.catch(async (error) => {
|
|
88
|
+
await this.recordFailure(error);
|
|
89
|
+
if (this.initializationPromise === initialization) {
|
|
90
|
+
this.initializationPromise = null;
|
|
91
|
+
}
|
|
92
|
+
throw error;
|
|
93
|
+
});
|
|
94
|
+
this.initializationPromise = initialization;
|
|
95
|
+
return this.initializationPromise;
|
|
96
|
+
}
|
|
97
|
+
async warmup(options = {}) {
|
|
98
|
+
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
99
|
+
if (options.errorLogPath)
|
|
100
|
+
this.errorLogPath = options.errorLogPath;
|
|
101
|
+
let timeout;
|
|
102
|
+
try {
|
|
103
|
+
await Promise.race([
|
|
104
|
+
this.initialize(),
|
|
105
|
+
new Promise((_, reject) => {
|
|
106
|
+
timeout = setTimeout(() => reject(new Error(`Embedding model warmup timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
107
|
+
}),
|
|
108
|
+
]);
|
|
109
|
+
return this.status === "ready";
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
const message = errorMessage(error);
|
|
113
|
+
if (this.lastError !== message) {
|
|
114
|
+
await this.recordFailure(error);
|
|
115
|
+
}
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
if (timeout)
|
|
120
|
+
clearTimeout(timeout);
|
|
121
|
+
}
|
|
23
122
|
}
|
|
24
123
|
async embed(text) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
124
|
+
await this.initialize();
|
|
125
|
+
try {
|
|
126
|
+
const results = await this.pipeline([text]);
|
|
127
|
+
this.markReady();
|
|
128
|
+
return results[0].data;
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
await this.recordFailure(error);
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
28
134
|
}
|
|
29
135
|
async embedBatch(texts) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
136
|
+
await this.initialize();
|
|
137
|
+
try {
|
|
138
|
+
const results = await this.pipeline(texts);
|
|
139
|
+
this.markReady();
|
|
140
|
+
return results.map((result) => result.data);
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
await this.recordFailure(error);
|
|
144
|
+
throw error;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
markReady() {
|
|
148
|
+
this.status = "ready";
|
|
149
|
+
this.lastError = null;
|
|
150
|
+
}
|
|
151
|
+
async recordFailure(error) {
|
|
152
|
+
const message = errorMessage(error);
|
|
153
|
+
this.status = "failed";
|
|
154
|
+
this.lastError = message;
|
|
155
|
+
await this.logFailure(message);
|
|
156
|
+
}
|
|
157
|
+
async logFailure(message) {
|
|
158
|
+
try {
|
|
159
|
+
await mkdir(dirname(this.errorLogPath), { recursive: true, mode: 0o700 });
|
|
160
|
+
await appendFile(this.errorLogPath, `[${new Date().toISOString()}] ${message}\n`, { encoding: "utf8", mode: 0o600 });
|
|
161
|
+
await chmod(this.errorLogPath, 0o600);
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
// Failure telemetry must not replace the original embedding error.
|
|
165
|
+
}
|
|
33
166
|
}
|
|
34
167
|
}
|
|
35
168
|
//# sourceMappingURL=embedding-provider.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-provider.js","sourceRoot":"","sources":["../src/embedding-provider.ts"],"names":[],"mappings":"AAAA,IAAI,IAAI,
|
|
1
|
+
{"version":3,"file":"embedding-provider.js","sourceRoot":"","sources":["../src/embedding-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAa1C,SAAS,gBAAgB,CACvB,MAAuB,EACvB,aAAqB;IAErB,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;IACrD,IACE,SAAS,KAAK,aAAa;WACxB,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC;WAC/B,aAAc,GAAG,CAAC;WAClB,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,GAAG,aAAc,EACpD,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACtD,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CACrB,KAAK,GAAG,aAAc,EACtB,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,aAAc,CAC7B;SACF,CAAC,CAAC,CAAC;IACN,CAAC;IACD,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;AACjC,CAAC;AAED,IAAI,cAAc,GAA6B,IAAI,CAAC;AACpD,IAAI,qBAAqB,GAAsC,IAAI,CAAC;AAEpE,SAAS,kBAAkB;IACzB,IAAI,cAAc;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC3D,IAAI,qBAAqB;QAAE,OAAO,qBAAqB,CAAC;IAExD,qBAAqB,GAAG,CAAC,KAAK,IAAI,EAAE;QAClC,MAAM,EACJ,QAAQ,EAAE,oBAAoB,EAC9B,GAAG,EAAE,eAAe,GACrB,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;QAC9E,IAAI,UAAU,EAAE,CAAC;YACf,eAAe,CAAC,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC;gBACnD,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC;QACvB,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,oBAAoB,CACpC,oBAAoB,EACpB,yBAAyB,CAC1B,CAAC;QACF,MAAM,MAAM,GAAsB,KAAK,EAAE,KAAK,EAAE,EAAE;YAChD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC1B,CAAC,CAAE,MAAwC;gBAC3C,CAAC,CAAC,gBAAgB,CAAC,MAAyB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAChE,CAAC,CAAC;QACF,cAAc,GAAG,MAAM,CAAC;QACxB,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QAC5B,qBAAqB,GAAG,IAAI,CAAC;QAC7B,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,qBAAqB,CAAC;AAC/B,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACxE,OAAO,KAAK,IAAI,KAAK,KAAK,KAAK,CAAC,OAAO;QACrC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,EAAE;QAC9B,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;AACpB,CAAC;AAED,MAAM,OAAO,iBAAiB;IAaT;IAZX,MAAM,GAAoB,MAAM,CAAC;IACjC,SAAS,GAAkB,IAAI,CAAC;IAChC,qBAAqB,GAAyB,IAAI,CAAC;IACnD,QAAQ,GAA6B,IAAI,CAAC;IAC1C,YAAY,GAAG,IAAI,CACzB,OAAO,EAAE,EACT,UAAU,EACV,QAAQ,EACR,cAAc,CACf,CAAC;IAEF,YACmB,iBACf,kBAAkB;QADH,mBAAc,GAAd,cAAc,CACX;IACnB,CAAC;IAEJ,OAAO;QACL,OAAO,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC;IACjC,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,UAAU;QACR,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QACD,IAAI,IAAI,CAAC,qBAAqB;YAAE,OAAO,IAAI,CAAC,qBAAqB,CAAC;QAElE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,EAAE;aACzC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;YACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;YACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC,CAAC;aACD,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;YAC9B,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,qBAAqB,KAAK,cAAc,EAAE,CAAC;gBAClD,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACpC,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACL,IAAI,CAAC,qBAAqB,GAAG,cAAc,CAAC;QAE5C,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAGT,EAAE;QACJ,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;QAC9C,IAAI,OAAO,CAAC,YAAY;YAAE,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QAEnE,IAAI,OAAkD,CAAC;QACvD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,IAAI,CAAC,UAAU,EAAE;gBACjB,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;oBAC/B,OAAO,GAAG,UAAU,CAClB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,0CAA0C,SAAS,IAAI,CAAC,CAAC,EAChF,SAAS,CACV,CAAC;gBACJ,CAAC,CAAC;aACH,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,MAAM,KAAK,OAAO,CAAC;QACjC,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,IAAI,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;gBAC/B,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;gBAAS,CAAC;YACT,IAAI,OAAO;gBAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7C,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAS,CAAC,KAAK,CAAC,CAAC;YAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,SAAS;QACf,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAAc;QACxC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,OAAe;QACtC,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1E,MAAM,UAAU,CACd,IAAI,CAAC,YAAY,EACjB,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,EAC5C,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAClC,CAAC;YACF,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,mEAAmE;QACrE,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event-grouper.d.ts","sourceRoot":"","sources":["../src/event-grouper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"event-grouper.d.ts","sourceRoot":"","sources":["../src/event-grouper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAIlD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;CACjB;AAeD,wBAAgB,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,UAAU,EAAE,CAiE5D;AAID,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,SAAS,SAAI,GAAG,UAAU,EAAE,CAqC9E"}
|
package/dist/event-grouper.js
CHANGED
|
@@ -44,6 +44,12 @@ export function groupEvents(events) {
|
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
if (fixGroup.length > 1) {
|
|
47
|
+
const prefix = currentGroup.slice(0, -1);
|
|
48
|
+
if (prefix.length > 0) {
|
|
49
|
+
const prefixGroup = buildGroup(prefix, groupIdx++);
|
|
50
|
+
if (prefixGroup)
|
|
51
|
+
groups.push(prefixGroup);
|
|
52
|
+
}
|
|
47
53
|
const group = buildGroup(fixGroup, groupIdx++);
|
|
48
54
|
if (group)
|
|
49
55
|
groups.push(group);
|
|
@@ -127,10 +133,46 @@ function buildGroup(events, idx) {
|
|
|
127
133
|
primaryTool,
|
|
128
134
|
hasError,
|
|
129
135
|
hasFix,
|
|
130
|
-
signalTypes:
|
|
136
|
+
signalTypes: collectSignalTypes(events),
|
|
131
137
|
summary: summary || `${events.length} tool calls with ${primaryTool}`,
|
|
132
138
|
};
|
|
133
139
|
}
|
|
140
|
+
function collectSignalTypes(events) {
|
|
141
|
+
const signalTypes = new Set();
|
|
142
|
+
for (const event of events) {
|
|
143
|
+
const payload = parsePayload(event.payload);
|
|
144
|
+
const explicitTypes = payload.signalTypes ?? payload.signal_types;
|
|
145
|
+
if (Array.isArray(explicitTypes)) {
|
|
146
|
+
for (const signalType of explicitTypes) {
|
|
147
|
+
if (typeof signalType === 'string' && signalType.trim())
|
|
148
|
+
signalTypes.add(signalType);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (typeof payload.signalType === 'string' && payload.signalType.trim()) {
|
|
152
|
+
signalTypes.add(payload.signalType);
|
|
153
|
+
}
|
|
154
|
+
// Event rows retain aggregate analyzer scores even when signal rows are not
|
|
155
|
+
// available to this grouping boundary. Preserve those triage signals.
|
|
156
|
+
if (event.l1_score > 0)
|
|
157
|
+
signalTypes.add('l1');
|
|
158
|
+
if (event.dialog_score > 0)
|
|
159
|
+
signalTypes.add('dialog');
|
|
160
|
+
}
|
|
161
|
+
return [...signalTypes];
|
|
162
|
+
}
|
|
163
|
+
function parsePayload(raw) {
|
|
164
|
+
if (!raw)
|
|
165
|
+
return {};
|
|
166
|
+
try {
|
|
167
|
+
const parsed = JSON.parse(raw);
|
|
168
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
169
|
+
? parsed
|
|
170
|
+
: {};
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
return {};
|
|
174
|
+
}
|
|
175
|
+
}
|
|
134
176
|
function isErrorEvent(e) {
|
|
135
177
|
return e.exit_code !== null && e.exit_code !== undefined && e.exit_code !== 0;
|
|
136
178
|
}
|