@memextend/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedding/index.d.ts +3 -0
- package/dist/embedding/index.d.ts.map +1 -0
- package/dist/embedding/index.js +4 -0
- package/dist/embedding/index.js.map +1 -0
- package/dist/embedding/local.d.ts +54 -0
- package/dist/embedding/local.d.ts.map +1 -0
- package/dist/embedding/local.js +162 -0
- package/dist/embedding/local.js.map +1 -0
- package/dist/embedding/local.test.d.ts +2 -0
- package/dist/embedding/local.test.d.ts.map +1 -0
- package/dist/embedding/local.test.js +73 -0
- package/dist/embedding/local.test.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/memory/capture.d.ts +102 -0
- package/dist/memory/capture.d.ts.map +1 -0
- package/dist/memory/capture.js +249 -0
- package/dist/memory/capture.js.map +1 -0
- package/dist/memory/capture.test.d.ts +2 -0
- package/dist/memory/capture.test.d.ts.map +1 -0
- package/dist/memory/capture.test.js +168 -0
- package/dist/memory/capture.test.js.map +1 -0
- package/dist/memory/deduplicate.d.ts +31 -0
- package/dist/memory/deduplicate.d.ts.map +1 -0
- package/dist/memory/deduplicate.js +64 -0
- package/dist/memory/deduplicate.js.map +1 -0
- package/dist/memory/deduplicate.test.d.ts +2 -0
- package/dist/memory/deduplicate.test.d.ts.map +1 -0
- package/dist/memory/deduplicate.test.js +116 -0
- package/dist/memory/deduplicate.test.js.map +1 -0
- package/dist/memory/index.d.ts +8 -0
- package/dist/memory/index.d.ts.map +1 -0
- package/dist/memory/index.js +7 -0
- package/dist/memory/index.js.map +1 -0
- package/dist/memory/retrieve.d.ts +62 -0
- package/dist/memory/retrieve.d.ts.map +1 -0
- package/dist/memory/retrieve.js +218 -0
- package/dist/memory/retrieve.js.map +1 -0
- package/dist/memory/retrieve.test.d.ts +2 -0
- package/dist/memory/retrieve.test.d.ts.map +1 -0
- package/dist/memory/retrieve.test.js +177 -0
- package/dist/memory/retrieve.test.js.map +1 -0
- package/dist/memory/types.d.ts +36 -0
- package/dist/memory/types.d.ts.map +1 -0
- package/dist/memory/types.js +4 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/memory/types.test.d.ts +2 -0
- package/dist/memory/types.test.d.ts.map +1 -0
- package/dist/memory/types.test.js +31 -0
- package/dist/memory/types.test.js.map +1 -0
- package/dist/storage/index.d.ts +4 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +5 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/lancedb.d.ts +24 -0
- package/dist/storage/lancedb.d.ts.map +1 -0
- package/dist/storage/lancedb.js +106 -0
- package/dist/storage/lancedb.js.map +1 -0
- package/dist/storage/lancedb.test.d.ts +2 -0
- package/dist/storage/lancedb.test.d.ts.map +1 -0
- package/dist/storage/lancedb.test.js +67 -0
- package/dist/storage/lancedb.test.js.map +1 -0
- package/dist/storage/sqlite.d.ts +68 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/storage/sqlite.js +354 -0
- package/dist/storage/sqlite.js.map +1 -0
- package/dist/storage/sqlite.test.d.ts +2 -0
- package/dist/storage/sqlite.test.d.ts.map +1 -0
- package/dist/storage/sqlite.test.js +137 -0
- package/dist/storage/sqlite.test.js.map +1 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +4 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/project.d.ts +13 -0
- package/dist/utils/project.d.ts.map +1 -0
- package/dist/utils/project.js +37 -0
- package/dist/utils/project.js.map +1 -0
- package/package.json +61 -0
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// packages/core/src/memory/capture.ts
|
|
2
|
+
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
|
+
// Minimum length for text to be considered worth capturing
|
|
4
|
+
const MIN_TEXT_LENGTH = 100;
|
|
5
|
+
// Default max lengths
|
|
6
|
+
const DEFAULT_MAX_REASONING_LENGTH = 10000;
|
|
7
|
+
const DEFAULT_MAX_TOOL_OUTPUT_LENGTH = 2000;
|
|
8
|
+
// All available tools that can be configured
|
|
9
|
+
export const CONFIGURABLE_TOOLS = ['Edit', 'Write', 'Bash', 'Task'];
|
|
10
|
+
// Default: all tools disabled
|
|
11
|
+
const DEFAULT_TOOL_CONFIG = {
|
|
12
|
+
Edit: false,
|
|
13
|
+
Write: false,
|
|
14
|
+
Bash: false,
|
|
15
|
+
Task: false
|
|
16
|
+
};
|
|
17
|
+
export class TranscriptParser {
|
|
18
|
+
toolConfig;
|
|
19
|
+
maxReasoningLength;
|
|
20
|
+
maxToolOutputLength;
|
|
21
|
+
captureReasoning;
|
|
22
|
+
constructor(options = {}) {
|
|
23
|
+
// Support legacy captureTools/skipTools or new toolConfig
|
|
24
|
+
if (options.toolConfig) {
|
|
25
|
+
this.toolConfig = { ...DEFAULT_TOOL_CONFIG, ...options.toolConfig };
|
|
26
|
+
}
|
|
27
|
+
else if (options.captureTools) {
|
|
28
|
+
// Legacy support: convert Set to toolConfig
|
|
29
|
+
this.toolConfig = { ...DEFAULT_TOOL_CONFIG };
|
|
30
|
+
for (const tool of CONFIGURABLE_TOOLS) {
|
|
31
|
+
this.toolConfig[tool] = options.captureTools.has(tool);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
this.toolConfig = { ...DEFAULT_TOOL_CONFIG };
|
|
36
|
+
}
|
|
37
|
+
// Support legacy maxContentLength or new separate limits
|
|
38
|
+
if (options.maxContentLength !== undefined) {
|
|
39
|
+
// Legacy: use same limit for both
|
|
40
|
+
this.maxReasoningLength = options.maxContentLength;
|
|
41
|
+
this.maxToolOutputLength = options.maxContentLength;
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
this.maxReasoningLength = options.maxReasoningLength ?? DEFAULT_MAX_REASONING_LENGTH;
|
|
45
|
+
this.maxToolOutputLength = options.maxToolOutputLength ?? DEFAULT_MAX_TOOL_OUTPUT_LENGTH;
|
|
46
|
+
}
|
|
47
|
+
this.captureReasoning = options.captureReasoning ?? true;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Parse a Claude Code JSONL transcript and extract captures
|
|
51
|
+
*
|
|
52
|
+
* Captures two types of content:
|
|
53
|
+
* 1. Claude's text responses (reasoning, explanations, decisions)
|
|
54
|
+
* 2. Tool calls for code changes (Edit, Write)
|
|
55
|
+
*
|
|
56
|
+
* Claude Code transcript format:
|
|
57
|
+
* - type: "assistant" with message.content containing text and tool_use objects
|
|
58
|
+
* - type: "user" with message.content containing tool_result objects
|
|
59
|
+
*/
|
|
60
|
+
parse(transcript) {
|
|
61
|
+
const lines = transcript.trim().split('\n');
|
|
62
|
+
const captures = [];
|
|
63
|
+
// Map of tool_use_id -> { tool, input }
|
|
64
|
+
const pendingToolUses = new Map();
|
|
65
|
+
for (const line of lines) {
|
|
66
|
+
if (!line.trim())
|
|
67
|
+
continue;
|
|
68
|
+
try {
|
|
69
|
+
const entry = JSON.parse(line);
|
|
70
|
+
// Skip non-message entries
|
|
71
|
+
if (!entry.message?.content || typeof entry.message.content === 'string') {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
const contentArray = entry.message.content;
|
|
75
|
+
// Process assistant messages for text and tool_use
|
|
76
|
+
if (entry.type === 'assistant') {
|
|
77
|
+
// Capture Claude's text reasoning
|
|
78
|
+
if (this.captureReasoning) {
|
|
79
|
+
for (const block of contentArray) {
|
|
80
|
+
if (block.type === 'text' && block.text) {
|
|
81
|
+
const text = block.text.trim();
|
|
82
|
+
// Only capture substantial text (not just "Let me..." fragments)
|
|
83
|
+
if (text.length >= MIN_TEXT_LENGTH && this.isSubstantialContent(text)) {
|
|
84
|
+
captures.push({
|
|
85
|
+
type: 'reasoning',
|
|
86
|
+
content: this.truncate(text, this.maxReasoningLength)
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
// Capture tool_use for code changes
|
|
93
|
+
for (const block of contentArray) {
|
|
94
|
+
if (block.type === 'tool_use' && block.name && block.id && block.input) {
|
|
95
|
+
if (this.shouldCapture(block.name)) {
|
|
96
|
+
pendingToolUses.set(block.id, {
|
|
97
|
+
tool: block.name,
|
|
98
|
+
input: block.input
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
// Process user messages for tool_result
|
|
105
|
+
if (entry.type === 'user') {
|
|
106
|
+
for (const block of contentArray) {
|
|
107
|
+
if (block.type === 'tool_result' && block.tool_use_id) {
|
|
108
|
+
const pending = pendingToolUses.get(block.tool_use_id);
|
|
109
|
+
if (pending) {
|
|
110
|
+
// Extract output text from content
|
|
111
|
+
let output = '';
|
|
112
|
+
if (typeof block.content === 'string') {
|
|
113
|
+
output = block.content;
|
|
114
|
+
}
|
|
115
|
+
else if (Array.isArray(block.content)) {
|
|
116
|
+
output = block.content
|
|
117
|
+
.filter(c => c.type === 'text' && c.text)
|
|
118
|
+
.map(c => c.text)
|
|
119
|
+
.join('\n');
|
|
120
|
+
}
|
|
121
|
+
captures.push({
|
|
122
|
+
tool: pending.tool,
|
|
123
|
+
input: pending.input,
|
|
124
|
+
output: this.truncate(output, this.maxToolOutputLength)
|
|
125
|
+
});
|
|
126
|
+
pendingToolUses.delete(block.tool_use_id);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
// Skip malformed lines
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return captures;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Check if text content is substantial (not just filler/transitional)
|
|
140
|
+
*/
|
|
141
|
+
isSubstantialContent(text) {
|
|
142
|
+
// Skip common transitional/filler phrases that aren't valuable to remember
|
|
143
|
+
const skipPatterns = [
|
|
144
|
+
/^(let me|i'll|i will|now i|looking at|checking|searching)/i,
|
|
145
|
+
/^(here's|here is) (the|what|how)/i,
|
|
146
|
+
/^(done|completed|finished|success)/i,
|
|
147
|
+
/^(ok|okay|sure|yes|no|alright)/i,
|
|
148
|
+
];
|
|
149
|
+
for (const pattern of skipPatterns) {
|
|
150
|
+
if (pattern.test(text.slice(0, 50))) {
|
|
151
|
+
// But if the text is long enough, it might still be valuable
|
|
152
|
+
if (text.length < 200) {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return true;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Legacy method for backward compatibility - returns only tool captures
|
|
161
|
+
*/
|
|
162
|
+
parseToolCaptures(transcript) {
|
|
163
|
+
return this.parse(transcript).filter((c) => 'tool' in c);
|
|
164
|
+
}
|
|
165
|
+
shouldCapture(tool) {
|
|
166
|
+
// Check if tool is in our configurable list and enabled
|
|
167
|
+
if (CONFIGURABLE_TOOLS.includes(tool)) {
|
|
168
|
+
return this.toolConfig[tool] === true;
|
|
169
|
+
}
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
truncate(str, maxLen) {
|
|
173
|
+
if (str.length <= maxLen)
|
|
174
|
+
return str;
|
|
175
|
+
return str.slice(0, maxLen - 3) + '...';
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Type guard to check if a capture is a TextCapture
|
|
180
|
+
*/
|
|
181
|
+
export function isTextCapture(capture) {
|
|
182
|
+
return 'type' in capture && capture.type === 'reasoning';
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Type guard to check if a capture is a ToolCapture
|
|
186
|
+
*/
|
|
187
|
+
export function isToolCapture(capture) {
|
|
188
|
+
return 'tool' in capture;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Format a capture into a human-readable memory content string
|
|
192
|
+
*/
|
|
193
|
+
export function formatCaptureContent(capture) {
|
|
194
|
+
if (isTextCapture(capture)) {
|
|
195
|
+
return capture.content;
|
|
196
|
+
}
|
|
197
|
+
return formatToolMemoryContent(capture.tool, capture.input, capture.output);
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Format a tool capture into a human-readable memory content string
|
|
201
|
+
*/
|
|
202
|
+
export function formatToolMemoryContent(tool, input, output) {
|
|
203
|
+
const maxOutputLen = 200;
|
|
204
|
+
switch (tool) {
|
|
205
|
+
case 'Edit': {
|
|
206
|
+
const filePath = input.file_path || 'unknown file';
|
|
207
|
+
const oldStr = input.old_string || '';
|
|
208
|
+
const newStr = input.new_string || '';
|
|
209
|
+
let description = '';
|
|
210
|
+
if (!oldStr && newStr) {
|
|
211
|
+
description = 'Added new content';
|
|
212
|
+
}
|
|
213
|
+
else if (oldStr && !newStr) {
|
|
214
|
+
description = 'Removed content';
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
description = 'Modified content';
|
|
218
|
+
}
|
|
219
|
+
return `[Edit] ${filePath}\n${description}. ${truncate(output, maxOutputLen)}`;
|
|
220
|
+
}
|
|
221
|
+
case 'Write': {
|
|
222
|
+
const filePath = input.file_path || 'unknown file';
|
|
223
|
+
const content = input.content || '';
|
|
224
|
+
const preview = content.slice(0, 100);
|
|
225
|
+
return `[Write] ${filePath}\nCreated new file. Preview: ${truncate(preview, maxOutputLen)}`;
|
|
226
|
+
}
|
|
227
|
+
case 'Bash': {
|
|
228
|
+
const command = input.command || 'unknown command';
|
|
229
|
+
return `[Bash] ${truncate(command, 100)}\nOutput: ${truncate(output, maxOutputLen)}`;
|
|
230
|
+
}
|
|
231
|
+
case 'Task': {
|
|
232
|
+
const description = input.description || 'Agent task';
|
|
233
|
+
const prompt = input.prompt || '';
|
|
234
|
+
return `[Task] ${description}\n${truncate(prompt, 100)}\nResult: ${truncate(output, 300)}`;
|
|
235
|
+
}
|
|
236
|
+
default:
|
|
237
|
+
return `[${tool}] ${truncate(output, maxOutputLen)}`;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* @deprecated Use formatCaptureContent instead
|
|
242
|
+
*/
|
|
243
|
+
export const formatMemoryContent = formatToolMemoryContent;
|
|
244
|
+
function truncate(str, maxLen) {
|
|
245
|
+
if (str.length <= maxLen)
|
|
246
|
+
return str;
|
|
247
|
+
return str.slice(0, maxLen - 3) + '...';
|
|
248
|
+
}
|
|
249
|
+
//# sourceMappingURL=capture.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"capture.js","sourceRoot":"","sources":["../../src/memory/capture.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,8CAA8C;AAkC9C,2DAA2D;AAC3D,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,sBAAsB;AACtB,MAAM,4BAA4B,GAAG,KAAK,CAAC;AAC3C,MAAM,8BAA8B,GAAG,IAAI,CAAC;AAE5C,6CAA6C;AAC7C,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAU,CAAC;AAW7E,8BAA8B;AAC9B,MAAM,mBAAmB,GAAsB;IAC7C,IAAI,EAAE,KAAK;IACX,KAAK,EAAE,KAAK;IACZ,IAAI,EAAE,KAAK;IACX,IAAI,EAAE,KAAK;CACZ,CAAC;AAkBF,MAAM,OAAO,gBAAgB;IACnB,UAAU,CAAoB;IAC9B,kBAAkB,CAAS;IAC3B,mBAAmB,CAAS;IAC5B,gBAAgB,CAAU;IAElC,YAAY,UAAmC,EAAE;QAC/C,0DAA0D;QAC1D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;QACtE,CAAC;aAAM,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YAChC,4CAA4C;YAC5C,IAAI,CAAC,UAAU,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;YAC7C,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;gBACtC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,GAAG,EAAE,GAAG,mBAAmB,EAAE,CAAC;QAC/C,CAAC;QAED,yDAAyD;QACzD,IAAI,OAAO,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;YAC3C,kCAAkC;YAClC,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,gBAAgB,CAAC;YACnD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,4BAA4B,CAAC;YACrF,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,8BAA8B,CAAC;QAC3F,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,IAAI,CAAC;IAC3D,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,UAAkB;QACtB,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAc,EAAE,CAAC;QAE/B,wCAAwC;QACxC,MAAM,eAAe,GAAG,IAAI,GAAG,EAA4D,CAAC;QAE5F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAS;YAE3B,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAoB,CAAC;gBAElD,2BAA2B;gBAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,IAAI,OAAO,KAAK,CAAC,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACzE,SAAS;gBACX,CAAC;gBAED,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;gBAE3C,mDAAmD;gBACnD,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC/B,kCAAkC;oBAClC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;wBAC1B,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;4BACjC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gCACxC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gCAC/B,iEAAiE;gCACjE,IAAI,IAAI,CAAC,MAAM,IAAI,eAAe,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;oCACtE,QAAQ,CAAC,IAAI,CAAC;wCACZ,IAAI,EAAE,WAAW;wCACjB,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC;qCACtD,CAAC,CAAC;gCACL,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,oCAAoC;oBACpC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;wBACjC,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;4BACvE,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gCACnC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;oCAC5B,IAAI,EAAE,KAAK,CAAC,IAAI;oCAChB,KAAK,EAAE,KAAK,CAAC,KAAK;iCACnB,CAAC,CAAC;4BACL,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,wCAAwC;gBACxC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC1B,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;wBACjC,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;4BACtD,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;4BACvD,IAAI,OAAO,EAAE,CAAC;gCACZ,mCAAmC;gCACnC,IAAI,MAAM,GAAG,EAAE,CAAC;gCAChB,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oCACtC,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC;gCACzB,CAAC;qCAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oCACxC,MAAM,GAAG,KAAK,CAAC,OAAO;yCACnB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC;yCACxC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;yCAChB,IAAI,CAAC,IAAI,CAAC,CAAC;gCAChB,CAAC;gCAED,QAAQ,CAAC,IAAI,CAAC;oCACZ,IAAI,EAAE,OAAO,CAAC,IAAkB;oCAChC,KAAK,EAAE,OAAO,CAAC,KAAK;oCACpB,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC;iCACxD,CAAC,CAAC;gCAEH,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;4BAC5C,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uBAAuB;YACzB,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,IAAY;QACvC,2EAA2E;QAC3E,MAAM,YAAY,GAAG;YACnB,4DAA4D;YAC5D,mCAAmC;YACnC,qCAAqC;YACrC,iCAAiC;SAClC,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpC,6DAA6D;gBAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBACtB,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,UAAkB;QAClC,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAClC,CAAC,CAAC,EAAoB,EAAE,CAAC,MAAM,IAAI,CAAC,CACrC,CAAC;IACJ,CAAC;IAEO,aAAa,CAAC,IAAY;QAChC,wDAAwD;QACxD,IAAI,kBAAkB,CAAC,QAAQ,CAAC,IAAwB,CAAC,EAAE,CAAC;YAC1D,OAAO,IAAI,CAAC,UAAU,CAAC,IAAwB,CAAC,KAAK,IAAI,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,QAAQ,CAAC,GAAW,EAAE,MAAc;QAC1C,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;YAAE,OAAO,GAAG,CAAC;QACrC,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC1C,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,OAAO,MAAM,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,OAAO,MAAM,IAAI,OAAO,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAgB;IACnD,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC,OAAO,CAAC;IACzB,CAAC;IAED,OAAO,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC9E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,IAAgB,EAChB,KAA8B,EAC9B,MAAc;IAEd,MAAM,YAAY,GAAG,GAAG,CAAC;IAEzB,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAmB,IAAI,cAAc,CAAC;YAC7D,MAAM,MAAM,GAAG,KAAK,CAAC,UAAoB,IAAI,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,KAAK,CAAC,UAAoB,IAAI,EAAE,CAAC;YAEhD,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,IAAI,MAAM,EAAE,CAAC;gBACtB,WAAW,GAAG,mBAAmB,CAAC;YACpC,CAAC;iBAAM,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC7B,WAAW,GAAG,iBAAiB,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,WAAW,GAAG,kBAAkB,CAAC;YACnC,CAAC;YAED,OAAO,UAAU,QAAQ,KAAK,WAAW,KAAK,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC;QACjF,CAAC;QAED,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAmB,IAAI,cAAc,CAAC;YAC7D,MAAM,OAAO,GAAG,KAAK,CAAC,OAAiB,IAAI,EAAE,CAAC;YAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACtC,OAAO,WAAW,QAAQ,gCAAgC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;QAC9F,CAAC;QAED,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,OAAO,GAAG,KAAK,CAAC,OAAiB,IAAI,iBAAiB,CAAC;YAC7D,OAAO,UAAU,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC;QACvF,CAAC;QAED,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,WAAW,GAAG,KAAK,CAAC,WAAqB,IAAI,YAAY,CAAC;YAChE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAgB,IAAI,EAAE,CAAC;YAC5C,OAAO,UAAU,WAAW,KAAK,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7F,CAAC;QAED;YACE,OAAO,IAAI,IAAI,KAAK,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC;IACzD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,uBAAuB,CAAC;AAE3D,SAAS,QAAQ,CAAC,GAAW,EAAE,MAAc;IAC3C,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,GAAG,CAAC;IACrC,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"capture.test.d.ts","sourceRoot":"","sources":["../../src/memory/capture.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// packages/core/src/memory/capture.test.ts
|
|
2
|
+
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
|
+
import { describe, it, expect } from 'vitest';
|
|
4
|
+
import { TranscriptParser, formatMemoryContent, isToolCapture, isTextCapture } from './capture.js';
|
|
5
|
+
// Claude Code JSONL transcript format
|
|
6
|
+
const sampleTranscript = `{"type":"assistant","message":{"content":[{"type":"text","text":"Redis caching is the best approach for this use case. The implementation requires creating a Redis client instance and connecting to the server. This is a substantial response with enough content to be captured as reasoning since we need at least 100 characters to make the cut."},{"type":"tool_use","id":"edit1","name":"Edit","input":{"file_path":"/src/cache.ts","old_string":"","new_string":"import Redis from 'ioredis';"}}]}}
|
|
7
|
+
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"edit1","content":"File edited successfully"}]}}
|
|
8
|
+
{"type":"assistant","message":{"content":[{"type":"tool_use","id":"read1","name":"Read","input":{"file_path":"/src/config.ts"}}]}}
|
|
9
|
+
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"read1","content":"export const config = {}"}]}}
|
|
10
|
+
{"type":"assistant","message":{"content":[{"type":"tool_use","id":"write1","name":"Write","input":{"file_path":"/src/redis.ts","content":"// Redis client\\nexport const redis = new Redis();"}}]}}
|
|
11
|
+
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"write1","content":"File created successfully"}]}}`;
|
|
12
|
+
describe('TranscriptParser', () => {
|
|
13
|
+
it('should capture only reasoning by default (no tool capture)', () => {
|
|
14
|
+
const parser = new TranscriptParser();
|
|
15
|
+
const captures = parser.parse(sampleTranscript);
|
|
16
|
+
// Default: only reasoning, no tools
|
|
17
|
+
const textCaptures = captures.filter(isTextCapture);
|
|
18
|
+
const toolCaptures = captures.filter(isToolCapture);
|
|
19
|
+
expect(textCaptures.length).toBeGreaterThan(0);
|
|
20
|
+
expect(toolCaptures.length).toBe(0);
|
|
21
|
+
});
|
|
22
|
+
it('should capture tool calls when explicitly enabled', () => {
|
|
23
|
+
const parser = new TranscriptParser({
|
|
24
|
+
toolConfig: { Edit: true, Write: true },
|
|
25
|
+
captureReasoning: false
|
|
26
|
+
});
|
|
27
|
+
const captures = parser.parseToolCaptures(sampleTranscript);
|
|
28
|
+
// Should capture Edit and Write when enabled
|
|
29
|
+
expect(captures.length).toBe(2);
|
|
30
|
+
expect(captures[0].tool).toBe('Edit');
|
|
31
|
+
expect(captures[1].tool).toBe('Write');
|
|
32
|
+
});
|
|
33
|
+
it('should allow enabling individual tools', () => {
|
|
34
|
+
const parser = new TranscriptParser({
|
|
35
|
+
toolConfig: { Edit: true, Write: false },
|
|
36
|
+
captureReasoning: false
|
|
37
|
+
});
|
|
38
|
+
const captures = parser.parseToolCaptures(sampleTranscript);
|
|
39
|
+
// Should only capture Edit, not Write
|
|
40
|
+
expect(captures.length).toBe(1);
|
|
41
|
+
expect(captures[0].tool).toBe('Edit');
|
|
42
|
+
});
|
|
43
|
+
it('should capture reasoning text from assistant messages', () => {
|
|
44
|
+
const parser = new TranscriptParser({ captureReasoning: true });
|
|
45
|
+
const captures = parser.parse(sampleTranscript);
|
|
46
|
+
const textCaptures = captures.filter(isTextCapture);
|
|
47
|
+
expect(textCaptures.length).toBeGreaterThan(0);
|
|
48
|
+
expect(textCaptures[0].content).toContain('Redis caching');
|
|
49
|
+
});
|
|
50
|
+
it('should skip filtered tools', () => {
|
|
51
|
+
const parser = new TranscriptParser({ captureReasoning: false });
|
|
52
|
+
const captures = parser.parseToolCaptures(sampleTranscript);
|
|
53
|
+
const readCapture = captures.find(c => c.tool === 'Read');
|
|
54
|
+
expect(readCapture).toBeUndefined();
|
|
55
|
+
});
|
|
56
|
+
it('should preserve tool input and output', () => {
|
|
57
|
+
const parser = new TranscriptParser({
|
|
58
|
+
toolConfig: { Edit: true },
|
|
59
|
+
captureReasoning: false
|
|
60
|
+
});
|
|
61
|
+
const captures = parser.parseToolCaptures(sampleTranscript);
|
|
62
|
+
const editCapture = captures[0];
|
|
63
|
+
expect(editCapture.input.file_path).toBe('/src/cache.ts');
|
|
64
|
+
expect(editCapture.output).toBe('File edited successfully');
|
|
65
|
+
});
|
|
66
|
+
it('should support legacy captureTools option', () => {
|
|
67
|
+
const parser = new TranscriptParser({
|
|
68
|
+
captureTools: new Set(['Edit', 'Write']),
|
|
69
|
+
captureReasoning: false
|
|
70
|
+
});
|
|
71
|
+
const captures = parser.parseToolCaptures(sampleTranscript);
|
|
72
|
+
expect(captures.length).toBe(2);
|
|
73
|
+
expect(captures[0].tool).toBe('Edit');
|
|
74
|
+
expect(captures[1].tool).toBe('Write');
|
|
75
|
+
});
|
|
76
|
+
it('should handle malformed lines gracefully', () => {
|
|
77
|
+
const transcript = `{"type":"user","message":"test"}
|
|
78
|
+
not valid json
|
|
79
|
+
{"type":"assistant","message":{"content":[{"type":"tool_use","id":"e1","name":"Edit","input":{"file_path":"test.ts"}}]}}
|
|
80
|
+
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"e1","content":"success"}]}}`;
|
|
81
|
+
const parser = new TranscriptParser({
|
|
82
|
+
toolConfig: { Edit: true },
|
|
83
|
+
captureReasoning: false
|
|
84
|
+
});
|
|
85
|
+
const captures = parser.parseToolCaptures(transcript);
|
|
86
|
+
expect(captures.length).toBe(1);
|
|
87
|
+
expect(captures[0].tool).toBe('Edit');
|
|
88
|
+
});
|
|
89
|
+
it('should truncate long tool output', () => {
|
|
90
|
+
const longOutput = 'x'.repeat(3000);
|
|
91
|
+
const transcript = `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"b1","name":"Write","input":{"file_path":"test.txt","content":"test"}}]}}
|
|
92
|
+
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"b1","content":"${longOutput}"}]}}`;
|
|
93
|
+
const parser = new TranscriptParser({
|
|
94
|
+
toolConfig: { Write: true },
|
|
95
|
+
maxToolOutputLength: 100,
|
|
96
|
+
captureReasoning: false
|
|
97
|
+
});
|
|
98
|
+
const captures = parser.parseToolCaptures(transcript);
|
|
99
|
+
expect(captures[0].output.length).toBeLessThanOrEqual(100);
|
|
100
|
+
expect(captures[0].output.endsWith('...')).toBe(true);
|
|
101
|
+
});
|
|
102
|
+
it('should have separate limits for reasoning vs tool output', () => {
|
|
103
|
+
const longText = 'This is a substantial reasoning response. '.repeat(50); // ~2100 chars
|
|
104
|
+
const transcript = `{"type":"assistant","message":{"content":[{"type":"text","text":"${longText}"}]}}`;
|
|
105
|
+
// With default limits (10000 reasoning, 2000 tool)
|
|
106
|
+
const parser = new TranscriptParser({ captureReasoning: true });
|
|
107
|
+
const captures = parser.parse(transcript);
|
|
108
|
+
const textCaptures = captures.filter(isTextCapture);
|
|
109
|
+
expect(textCaptures.length).toBe(1);
|
|
110
|
+
// Should not be truncated at 2000
|
|
111
|
+
expect(textCaptures[0].content.length).toBeGreaterThan(2000);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
describe('formatMemoryContent', () => {
|
|
115
|
+
it('should format Edit capture for new content', () => {
|
|
116
|
+
const content = formatMemoryContent('Edit', {
|
|
117
|
+
file_path: '/src/cache.ts',
|
|
118
|
+
old_string: '',
|
|
119
|
+
new_string: "import Redis from 'ioredis';"
|
|
120
|
+
}, 'File edited successfully');
|
|
121
|
+
expect(content).toContain('[Edit]');
|
|
122
|
+
expect(content).toContain('/src/cache.ts');
|
|
123
|
+
expect(content).toContain('Added new content');
|
|
124
|
+
});
|
|
125
|
+
it('should format Edit capture for removed content', () => {
|
|
126
|
+
const content = formatMemoryContent('Edit', {
|
|
127
|
+
file_path: '/src/cache.ts',
|
|
128
|
+
old_string: 'old code',
|
|
129
|
+
new_string: ''
|
|
130
|
+
}, 'File edited successfully');
|
|
131
|
+
expect(content).toContain('Removed content');
|
|
132
|
+
});
|
|
133
|
+
it('should format Edit capture for modified content', () => {
|
|
134
|
+
const content = formatMemoryContent('Edit', {
|
|
135
|
+
file_path: '/src/cache.ts',
|
|
136
|
+
old_string: 'old code',
|
|
137
|
+
new_string: 'new code'
|
|
138
|
+
}, 'File edited successfully');
|
|
139
|
+
expect(content).toContain('Modified content');
|
|
140
|
+
});
|
|
141
|
+
it('should format Bash capture', () => {
|
|
142
|
+
const content = formatMemoryContent('Bash', {
|
|
143
|
+
command: 'npm install ioredis'
|
|
144
|
+
}, 'added 1 package');
|
|
145
|
+
expect(content).toContain('[Bash]');
|
|
146
|
+
expect(content).toContain('npm install ioredis');
|
|
147
|
+
expect(content).toContain('added 1 package');
|
|
148
|
+
});
|
|
149
|
+
it('should format Write capture', () => {
|
|
150
|
+
const content = formatMemoryContent('Write', {
|
|
151
|
+
file_path: '/src/redis.ts',
|
|
152
|
+
content: '// Redis client\nexport const redis = new Redis();'
|
|
153
|
+
}, 'File created successfully');
|
|
154
|
+
expect(content).toContain('[Write]');
|
|
155
|
+
expect(content).toContain('/src/redis.ts');
|
|
156
|
+
expect(content).toContain('Created new file');
|
|
157
|
+
});
|
|
158
|
+
it('should format Task capture', () => {
|
|
159
|
+
const content = formatMemoryContent('Task', {
|
|
160
|
+
description: 'Deploy to staging',
|
|
161
|
+
prompt: 'Deploy the application to staging environment'
|
|
162
|
+
}, 'Deployment completed successfully');
|
|
163
|
+
expect(content).toContain('[Task]');
|
|
164
|
+
expect(content).toContain('Deploy to staging');
|
|
165
|
+
expect(content).toContain('completed successfully');
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
//# sourceMappingURL=capture.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"capture.test.js","sourceRoot":"","sources":["../../src/memory/capture.test.ts"],"names":[],"mappings":"AAAA,2CAA2C;AAC3C,8CAA8C;AAE9C,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAEnG,sCAAsC;AACtC,MAAM,gBAAgB,GAAG;;;;;4HAKmG,CAAC;AAE7H,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAEhD,oCAAoC;QACpC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAEpD,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAClC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE;YACvC,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAE5D,6CAA6C;QAC7C,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAClC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;YACxC,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAE5D,sCAAsC;QACtC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAEhD,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACpD,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAE5D,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAE,CAAC,CAAC,IAAe,KAAK,MAAM,CAAC,CAAC;QACtE,MAAM,CAAC,WAAW,CAAC,CAAC,aAAa,EAAE,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAClC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;YAC1B,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAE5D,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC1D,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAClC,YAAY,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACxC,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAE5D,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,UAAU,GAAG;;;sGAG+E,CAAC;QAEnG,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAClC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;YAC1B,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEtD,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG;2FACoE,UAAU,OAAO,CAAC;QAEzG,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAClC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;YAC3B,mBAAmB,EAAE,GAAG;YACxB,gBAAgB,EAAE,KAAK;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEtD,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAC3D,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,QAAQ,GAAG,4CAA4C,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc;QACxF,MAAM,UAAU,GAAG,oEAAoE,QAAQ,OAAO,CAAC;QAEvG,mDAAmD;QACnD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE1C,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACpD,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpC,kCAAkC;QAClC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,EAAE;YAC1C,SAAS,EAAE,eAAe;YAC1B,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,8BAA8B;SAC3C,EAAE,0BAA0B,CAAC,CAAC;QAE/B,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,EAAE;YAC1C,SAAS,EAAE,eAAe;YAC1B,UAAU,EAAE,UAAU;YACtB,UAAU,EAAE,EAAE;SACf,EAAE,0BAA0B,CAAC,CAAC;QAE/B,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,EAAE;YAC1C,SAAS,EAAE,eAAe;YAC1B,UAAU,EAAE,UAAU;YACtB,UAAU,EAAE,UAAU;SACvB,EAAE,0BAA0B,CAAC,CAAC;QAE/B,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,EAAE;YAC1C,OAAO,EAAE,qBAAqB;SAC/B,EAAE,iBAAiB,CAAC,CAAC;QAEtB,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QACjD,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,EAAE;YAC3C,SAAS,EAAE,eAAe;YAC1B,OAAO,EAAE,oDAAoD;SAC9D,EAAE,2BAA2B,CAAC,CAAC;QAEhC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,EAAE;YAC1C,WAAW,EAAE,mBAAmB;YAChC,MAAM,EAAE,+CAA+C;SACxD,EAAE,mCAAmC,CAAC,CAAC;QAExC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Memory } from './types.js';
|
|
2
|
+
export interface DeduplicationOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Similarity threshold (0-1). Memories with similarity above this
|
|
5
|
+
* are considered duplicates. Default: 0.85
|
|
6
|
+
*/
|
|
7
|
+
similarityThreshold?: number;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Deduplicate memories using cosine similarity.
|
|
11
|
+
*
|
|
12
|
+
* Algorithm:
|
|
13
|
+
* 1. Sort memories by date (newest first)
|
|
14
|
+
* 2. For each memory, check similarity against already-selected memories
|
|
15
|
+
* 3. If similarity > threshold, skip it (newer version already selected)
|
|
16
|
+
* 4. Result: diverse set with newest version of each "topic"
|
|
17
|
+
*
|
|
18
|
+
* @param memories - Array of memories to deduplicate
|
|
19
|
+
* @param vectors - Map of memory ID to embedding vector
|
|
20
|
+
* @param options - Deduplication options
|
|
21
|
+
* @returns Deduplicated array of memories (newest of each similar group)
|
|
22
|
+
*/
|
|
23
|
+
export declare function deduplicateMemories(memories: Memory[], vectors: Map<string, number[]>, options?: DeduplicationOptions): Memory[];
|
|
24
|
+
/**
|
|
25
|
+
* Get statistics about deduplication
|
|
26
|
+
*/
|
|
27
|
+
export declare function getDeduplicationStats(originalCount: number, deduplicatedCount: number): {
|
|
28
|
+
removed: number;
|
|
29
|
+
percentage: number;
|
|
30
|
+
};
|
|
31
|
+
//# sourceMappingURL=deduplicate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deduplicate.d.ts","sourceRoot":"","sources":["../../src/memory/deduplicate.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGzC,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAID;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAC9B,OAAO,CAAC,EAAE,oBAAoB,GAC7B,MAAM,EAAE,CAyCV;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,aAAa,EAAE,MAAM,EACrB,iBAAiB,EAAE,MAAM,GACxB;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAMzC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// packages/core/src/memory/deduplicate.ts
|
|
2
|
+
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
|
+
import { cosineSimilarity } from '../embedding/index.js';
|
|
4
|
+
const DEFAULT_THRESHOLD = 0.85;
|
|
5
|
+
/**
|
|
6
|
+
* Deduplicate memories using cosine similarity.
|
|
7
|
+
*
|
|
8
|
+
* Algorithm:
|
|
9
|
+
* 1. Sort memories by date (newest first)
|
|
10
|
+
* 2. For each memory, check similarity against already-selected memories
|
|
11
|
+
* 3. If similarity > threshold, skip it (newer version already selected)
|
|
12
|
+
* 4. Result: diverse set with newest version of each "topic"
|
|
13
|
+
*
|
|
14
|
+
* @param memories - Array of memories to deduplicate
|
|
15
|
+
* @param vectors - Map of memory ID to embedding vector
|
|
16
|
+
* @param options - Deduplication options
|
|
17
|
+
* @returns Deduplicated array of memories (newest of each similar group)
|
|
18
|
+
*/
|
|
19
|
+
export function deduplicateMemories(memories, vectors, options) {
|
|
20
|
+
if (memories.length === 0)
|
|
21
|
+
return [];
|
|
22
|
+
const threshold = options?.similarityThreshold ?? DEFAULT_THRESHOLD;
|
|
23
|
+
// Sort by date (newest first)
|
|
24
|
+
const sorted = [...memories].sort((a, b) => {
|
|
25
|
+
return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime();
|
|
26
|
+
});
|
|
27
|
+
const selected = [];
|
|
28
|
+
const selectedVectors = [];
|
|
29
|
+
for (const memory of sorted) {
|
|
30
|
+
const vector = vectors.get(memory.id);
|
|
31
|
+
// If no vector, include the memory (can't compare)
|
|
32
|
+
if (!vector) {
|
|
33
|
+
selected.push(memory);
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
// Check similarity against all already-selected memories
|
|
37
|
+
let isDuplicate = false;
|
|
38
|
+
for (const selectedVector of selectedVectors) {
|
|
39
|
+
const similarity = cosineSimilarity(vector, selectedVector);
|
|
40
|
+
if (similarity > threshold) {
|
|
41
|
+
// This memory is too similar to an already-selected one
|
|
42
|
+
// Since we sorted by newest first, the selected one is newer
|
|
43
|
+
isDuplicate = true;
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (!isDuplicate) {
|
|
48
|
+
selected.push(memory);
|
|
49
|
+
selectedVectors.push(vector);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return selected;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Get statistics about deduplication
|
|
56
|
+
*/
|
|
57
|
+
export function getDeduplicationStats(originalCount, deduplicatedCount) {
|
|
58
|
+
const removed = originalCount - deduplicatedCount;
|
|
59
|
+
const percentage = originalCount > 0
|
|
60
|
+
? Math.round((removed / originalCount) * 100)
|
|
61
|
+
: 0;
|
|
62
|
+
return { removed, percentage };
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=deduplicate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deduplicate.js","sourceRoot":"","sources":["../../src/memory/deduplicate.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAG9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CACjC,QAAkB,EAClB,OAA8B,EAC9B,OAA8B;IAE9B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,SAAS,GAAG,OAAO,EAAE,mBAAmB,IAAI,iBAAiB,CAAC;IAEpE,8BAA8B;IAC9B,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,eAAe,GAAe,EAAE,CAAC;IAEvC,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAEtC,mDAAmD;QACnD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QAED,yDAAyD;QACzD,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,KAAK,MAAM,cAAc,IAAI,eAAe,EAAE,CAAC;YAC7C,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAC5D,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;gBAC3B,wDAAwD;gBACxD,6DAA6D;gBAC7D,WAAW,GAAG,IAAI,CAAC;gBACnB,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtB,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,aAAqB,EACrB,iBAAyB;IAEzB,MAAM,OAAO,GAAG,aAAa,GAAG,iBAAiB,CAAC;IAClD,MAAM,UAAU,GAAG,aAAa,GAAG,CAAC;QAClC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,aAAa,CAAC,GAAG,GAAG,CAAC;QAC7C,CAAC,CAAC,CAAC,CAAC;IACN,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AACjC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deduplicate.test.d.ts","sourceRoot":"","sources":["../../src/memory/deduplicate.test.ts"],"names":[],"mappings":""}
|