@lov3kaizen/agentsea-debugger 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +364 -0
- package/dist/Recorder-Dc9qR2V2.d.ts +138 -0
- package/dist/ReplayEngine-Dyyqy5bw.d.ts +54 -0
- package/dist/analysis/index.d.ts +4 -0
- package/dist/analysis/index.js +1840 -0
- package/dist/analysis/index.js.map +1 -0
- package/dist/diff-CLShBdWe.d.ts +24 -0
- package/dist/index-1W27DYJt.d.ts +366 -0
- package/dist/index-DRrKPSo9.d.ts +233 -0
- package/dist/index.d.ts +29 -0
- package/dist/index.js +6232 -0
- package/dist/index.js.map +1 -0
- package/dist/integrations/agentsea/index.d.ts +8 -0
- package/dist/integrations/agentsea/index.js +5826 -0
- package/dist/integrations/agentsea/index.js.map +1 -0
- package/dist/recording/index.d.ts +91 -0
- package/dist/recording/index.js +1207 -0
- package/dist/recording/index.js.map +1 -0
- package/dist/recording.types-Ck7pbikw.d.ts +281 -0
- package/dist/replay/index.d.ts +80 -0
- package/dist/replay/index.js +1112 -0
- package/dist/replay/index.js.map +1 -0
- package/dist/replay.types-C9hJizI-.d.ts +76 -0
- package/dist/storage/index.d.ts +135 -0
- package/dist/storage/index.js +588 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/visualization/index.d.ts +123 -0
- package/dist/visualization/index.js +789 -0
- package/dist/visualization/index.js.map +1 -0
- package/dist/visualization.types-dWjGE037.d.ts +98 -0
- package/package.json +99 -0
|
@@ -0,0 +1,1840 @@
|
|
|
1
|
+
import { EventEmitter } from 'eventemitter3';
|
|
2
|
+
import { nanoid } from 'nanoid';
|
|
3
|
+
|
|
4
|
+
// src/analysis/WhatIfEngine.ts
|
|
5
|
+
function generateId(prefix = "") {
|
|
6
|
+
const id = nanoid(12);
|
|
7
|
+
return prefix ? `${prefix}_${id}` : id;
|
|
8
|
+
}
|
|
9
|
+
function now() {
|
|
10
|
+
return Date.now();
|
|
11
|
+
}
|
|
12
|
+
function deepClone(obj) {
|
|
13
|
+
if (obj === null || typeof obj !== "object") {
|
|
14
|
+
return obj;
|
|
15
|
+
}
|
|
16
|
+
if (Array.isArray(obj)) {
|
|
17
|
+
return obj.map((item) => deepClone(item));
|
|
18
|
+
}
|
|
19
|
+
if (obj instanceof Date) {
|
|
20
|
+
return new Date(obj.getTime());
|
|
21
|
+
}
|
|
22
|
+
if (obj instanceof Map) {
|
|
23
|
+
const clonedMap = /* @__PURE__ */ new Map();
|
|
24
|
+
obj.forEach((value, key) => {
|
|
25
|
+
clonedMap.set(deepClone(key), deepClone(value));
|
|
26
|
+
});
|
|
27
|
+
return clonedMap;
|
|
28
|
+
}
|
|
29
|
+
if (obj instanceof Set) {
|
|
30
|
+
const clonedSet = /* @__PURE__ */ new Set();
|
|
31
|
+
obj.forEach((value) => {
|
|
32
|
+
clonedSet.add(deepClone(value));
|
|
33
|
+
});
|
|
34
|
+
return clonedSet;
|
|
35
|
+
}
|
|
36
|
+
const clonedObj = {};
|
|
37
|
+
for (const key in obj) {
|
|
38
|
+
if (Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
39
|
+
clonedObj[key] = deepClone(
|
|
40
|
+
obj[key]
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return clonedObj;
|
|
45
|
+
}
|
|
46
|
+
function sleep(ms) {
|
|
47
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// src/utils/diff.ts
|
|
51
|
+
function diff(lhs, rhs, path = []) {
|
|
52
|
+
const differences = [];
|
|
53
|
+
if (lhs === rhs) {
|
|
54
|
+
return differences;
|
|
55
|
+
}
|
|
56
|
+
if (lhs === null || lhs === void 0) {
|
|
57
|
+
if (rhs !== null && rhs !== void 0) {
|
|
58
|
+
differences.push({ path, kind: "N", rhs });
|
|
59
|
+
}
|
|
60
|
+
return differences;
|
|
61
|
+
}
|
|
62
|
+
if (rhs === null || rhs === void 0) {
|
|
63
|
+
differences.push({ path, kind: "D", lhs });
|
|
64
|
+
return differences;
|
|
65
|
+
}
|
|
66
|
+
const lhsType = typeof lhs;
|
|
67
|
+
const rhsType = typeof rhs;
|
|
68
|
+
if (lhsType !== rhsType) {
|
|
69
|
+
differences.push({ path, kind: "E", lhs, rhs });
|
|
70
|
+
return differences;
|
|
71
|
+
}
|
|
72
|
+
if (lhsType !== "object") {
|
|
73
|
+
if (lhs !== rhs) {
|
|
74
|
+
differences.push({ path, kind: "E", lhs, rhs });
|
|
75
|
+
}
|
|
76
|
+
return differences;
|
|
77
|
+
}
|
|
78
|
+
if (Array.isArray(lhs) && Array.isArray(rhs)) {
|
|
79
|
+
const maxLen = Math.max(lhs.length, rhs.length);
|
|
80
|
+
for (let i = 0; i < maxLen; i++) {
|
|
81
|
+
if (i >= lhs.length) {
|
|
82
|
+
differences.push({
|
|
83
|
+
path,
|
|
84
|
+
kind: "A",
|
|
85
|
+
index: i,
|
|
86
|
+
item: { path: [...path, String(i)], kind: "N", rhs: rhs[i] }
|
|
87
|
+
});
|
|
88
|
+
} else if (i >= rhs.length) {
|
|
89
|
+
differences.push({
|
|
90
|
+
path,
|
|
91
|
+
kind: "A",
|
|
92
|
+
index: i,
|
|
93
|
+
item: { path: [...path, String(i)], kind: "D", lhs: lhs[i] }
|
|
94
|
+
});
|
|
95
|
+
} else {
|
|
96
|
+
const itemDiffs = diff(lhs[i], rhs[i], [...path, String(i)]);
|
|
97
|
+
differences.push(...itemDiffs);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return differences;
|
|
101
|
+
}
|
|
102
|
+
if (typeof lhs === "object" && typeof rhs === "object") {
|
|
103
|
+
const lhsObj = lhs;
|
|
104
|
+
const rhsObj = rhs;
|
|
105
|
+
const allKeys = /* @__PURE__ */ new Set([...Object.keys(lhsObj), ...Object.keys(rhsObj)]);
|
|
106
|
+
for (const key of allKeys) {
|
|
107
|
+
const keyPath = [...path, key];
|
|
108
|
+
if (!(key in lhsObj)) {
|
|
109
|
+
differences.push({ path: keyPath, kind: "N", rhs: rhsObj[key] });
|
|
110
|
+
} else if (!(key in rhsObj)) {
|
|
111
|
+
differences.push({ path: keyPath, kind: "D", lhs: lhsObj[key] });
|
|
112
|
+
} else {
|
|
113
|
+
const propDiffs = diff(lhsObj[key], rhsObj[key], keyPath);
|
|
114
|
+
differences.push(...propDiffs);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return differences;
|
|
119
|
+
}
|
|
120
|
+
function toReplayDifferences(diffs, stepIndex) {
|
|
121
|
+
return diffs.map((d) => ({
|
|
122
|
+
stepIndex,
|
|
123
|
+
path: d.path.join("."),
|
|
124
|
+
original: d.lhs,
|
|
125
|
+
replayed: d.rhs,
|
|
126
|
+
type: d.kind === "N" ? "added" : d.kind === "D" ? "removed" : "changed"
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
function applyPatches(target, patches) {
|
|
130
|
+
let result = target;
|
|
131
|
+
for (const patch of patches) {
|
|
132
|
+
result = applyPatch(result, patch);
|
|
133
|
+
}
|
|
134
|
+
return result;
|
|
135
|
+
}
|
|
136
|
+
function applyPatch(target, patch) {
|
|
137
|
+
const result = JSON.parse(JSON.stringify(target));
|
|
138
|
+
if (patch.path.length === 0) {
|
|
139
|
+
return patch.rhs;
|
|
140
|
+
}
|
|
141
|
+
let current = result;
|
|
142
|
+
for (let i = 0; i < patch.path.length - 1; i++) {
|
|
143
|
+
const key = patch.path[i];
|
|
144
|
+
if (!(key in current)) {
|
|
145
|
+
current[key] = {};
|
|
146
|
+
}
|
|
147
|
+
current = current[key];
|
|
148
|
+
}
|
|
149
|
+
const lastKey = patch.path[patch.path.length - 1];
|
|
150
|
+
switch (patch.kind) {
|
|
151
|
+
case "N":
|
|
152
|
+
case "E":
|
|
153
|
+
current[lastKey] = patch.rhs;
|
|
154
|
+
break;
|
|
155
|
+
case "D":
|
|
156
|
+
delete current[lastKey];
|
|
157
|
+
break;
|
|
158
|
+
case "A":
|
|
159
|
+
if (Array.isArray(current[lastKey]) && patch.item) {
|
|
160
|
+
const arr = current[lastKey];
|
|
161
|
+
if (patch.item.kind === "N") {
|
|
162
|
+
arr.splice(patch.index, 0, patch.item.rhs);
|
|
163
|
+
} else if (patch.item.kind === "D") {
|
|
164
|
+
arr.splice(patch.index, 1);
|
|
165
|
+
} else if (patch.item.kind === "E") {
|
|
166
|
+
arr[patch.index] = patch.item.rhs;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
return result;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// src/replay/StateRestorer.ts
|
|
175
|
+
var DEFAULT_OPTIONS = {
|
|
176
|
+
includeMemory: true,
|
|
177
|
+
includeContext: true,
|
|
178
|
+
includeMessages: true,
|
|
179
|
+
includeTools: true
|
|
180
|
+
};
|
|
181
|
+
var StateRestorer = class {
|
|
182
|
+
options;
|
|
183
|
+
stateCache = /* @__PURE__ */ new Map();
|
|
184
|
+
constructor(options) {
|
|
185
|
+
this.options = {
|
|
186
|
+
...DEFAULT_OPTIONS,
|
|
187
|
+
...options
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Restore state at a specific step
|
|
192
|
+
*/
|
|
193
|
+
restore(recording, stepIndex) {
|
|
194
|
+
const cacheKey = recording.id;
|
|
195
|
+
const recordingCache = this.stateCache.get(cacheKey);
|
|
196
|
+
if (recordingCache?.has(stepIndex)) {
|
|
197
|
+
return deepClone(recordingCache.get(stepIndex));
|
|
198
|
+
}
|
|
199
|
+
if (stepIndex < 0) {
|
|
200
|
+
const state2 = deepClone(recording.initialState);
|
|
201
|
+
if (!recordingCache) {
|
|
202
|
+
this.stateCache.set(cacheKey, /* @__PURE__ */ new Map());
|
|
203
|
+
}
|
|
204
|
+
this.stateCache.get(cacheKey).set(stepIndex, deepClone(state2));
|
|
205
|
+
return state2;
|
|
206
|
+
}
|
|
207
|
+
let state;
|
|
208
|
+
let startStep;
|
|
209
|
+
const checkpoint = this.findClosestCheckpoint(recording, stepIndex);
|
|
210
|
+
if (checkpoint) {
|
|
211
|
+
if (checkpoint.stepIndex === stepIndex) {
|
|
212
|
+
state = deepClone(recording.initialState);
|
|
213
|
+
startStep = 0;
|
|
214
|
+
} else {
|
|
215
|
+
state = deepClone(recording.initialState);
|
|
216
|
+
startStep = 0;
|
|
217
|
+
}
|
|
218
|
+
} else {
|
|
219
|
+
state = deepClone(recording.initialState);
|
|
220
|
+
startStep = 0;
|
|
221
|
+
}
|
|
222
|
+
for (let i = startStep; i <= stepIndex && i < recording.steps.length; i++) {
|
|
223
|
+
state = this.applyStep(state, recording.steps[i]);
|
|
224
|
+
}
|
|
225
|
+
if (!recordingCache) {
|
|
226
|
+
this.stateCache.set(cacheKey, /* @__PURE__ */ new Map());
|
|
227
|
+
}
|
|
228
|
+
this.stateCache.get(cacheKey).set(stepIndex, deepClone(state));
|
|
229
|
+
return state;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Restore state from a checkpoint
|
|
233
|
+
*/
|
|
234
|
+
restoreFromCheckpoint(checkpoint) {
|
|
235
|
+
return deepClone(checkpoint.state);
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Apply a step to state
|
|
239
|
+
*/
|
|
240
|
+
applyStep(state, step) {
|
|
241
|
+
const newState = deepClone(state);
|
|
242
|
+
switch (step.type) {
|
|
243
|
+
case "input":
|
|
244
|
+
if (this.options.includeMessages) {
|
|
245
|
+
newState.messages.push({
|
|
246
|
+
role: "user",
|
|
247
|
+
content: String(step.input)
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
break;
|
|
251
|
+
case "prompt":
|
|
252
|
+
break;
|
|
253
|
+
case "response":
|
|
254
|
+
if (this.options.includeMessages) {
|
|
255
|
+
newState.messages.push({
|
|
256
|
+
role: "assistant",
|
|
257
|
+
content: String(step.output)
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
break;
|
|
261
|
+
case "tool-call":
|
|
262
|
+
if (this.options.includeMessages && step.toolCall) {
|
|
263
|
+
newState.messages.push({
|
|
264
|
+
role: "assistant",
|
|
265
|
+
content: `[Calling tool: ${step.toolCall.name}]`,
|
|
266
|
+
toolCalls: [
|
|
267
|
+
{
|
|
268
|
+
id: step.toolCall.id,
|
|
269
|
+
name: step.toolCall.name,
|
|
270
|
+
arguments: step.toolCall.arguments
|
|
271
|
+
}
|
|
272
|
+
]
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
break;
|
|
276
|
+
case "tool-result":
|
|
277
|
+
if (this.options.includeMessages && step.toolCall) {
|
|
278
|
+
newState.messages.push({
|
|
279
|
+
role: "tool",
|
|
280
|
+
content: String(step.toolCall.result ?? ""),
|
|
281
|
+
toolCallId: step.toolCall.id
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
break;
|
|
285
|
+
case "memory-write":
|
|
286
|
+
if (this.options.includeMemory && step.output) {
|
|
287
|
+
this.applyMemoryChange(
|
|
288
|
+
newState,
|
|
289
|
+
step.output
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
break;
|
|
293
|
+
case "memory-read":
|
|
294
|
+
break;
|
|
295
|
+
case "decision":
|
|
296
|
+
if (this.options.includeContext && step.decision) {
|
|
297
|
+
newState.context["lastDecision"] = {
|
|
298
|
+
options: step.decision.options.map((o) => o.description),
|
|
299
|
+
chosen: step.decision.chosen.description,
|
|
300
|
+
confidence: step.decision.confidence
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
break;
|
|
304
|
+
case "handoff":
|
|
305
|
+
case "delegation":
|
|
306
|
+
if (this.options.includeContext && step.output) {
|
|
307
|
+
const output = step.output;
|
|
308
|
+
newState.context["delegatedTo"] = output.agentId ?? output.agentName;
|
|
309
|
+
}
|
|
310
|
+
break;
|
|
311
|
+
case "error":
|
|
312
|
+
newState.context["lastError"] = step.error;
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
return newState;
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Apply memory changes to state
|
|
319
|
+
*/
|
|
320
|
+
applyMemoryChange(state, changes) {
|
|
321
|
+
for (const [key, value] of Object.entries(changes)) {
|
|
322
|
+
if (key === "working") {
|
|
323
|
+
state.memory.working = {
|
|
324
|
+
...state.memory.working,
|
|
325
|
+
...value
|
|
326
|
+
};
|
|
327
|
+
} else if (key === "shortTerm" && Array.isArray(value)) {
|
|
328
|
+
state.memory.shortTerm = [...state.memory.shortTerm ?? [], ...value];
|
|
329
|
+
} else if (key === "longTermSummary" && typeof value === "string") {
|
|
330
|
+
state.memory.longTermSummary = value;
|
|
331
|
+
} else if (key === "size" && typeof value === "number") {
|
|
332
|
+
state.memory.size = value;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Find closest checkpoint before or at step
|
|
338
|
+
*/
|
|
339
|
+
findClosestCheckpoint(recording, stepIndex) {
|
|
340
|
+
let closest;
|
|
341
|
+
for (const checkpoint of recording.checkpoints) {
|
|
342
|
+
if (checkpoint.stepIndex <= stepIndex) {
|
|
343
|
+
if (!closest || checkpoint.stepIndex > closest.stepIndex) {
|
|
344
|
+
closest = checkpoint;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return closest;
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Apply a diff patch to state
|
|
352
|
+
*/
|
|
353
|
+
applyPatch(state, differences) {
|
|
354
|
+
return applyPatches(state, differences);
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Validate restored state
|
|
358
|
+
*/
|
|
359
|
+
validate(state) {
|
|
360
|
+
const errors = [];
|
|
361
|
+
const warnings = [];
|
|
362
|
+
if (!state.agentId) {
|
|
363
|
+
errors.push("Missing agentId");
|
|
364
|
+
}
|
|
365
|
+
if (!state.agentName) {
|
|
366
|
+
warnings.push("Missing agentName");
|
|
367
|
+
}
|
|
368
|
+
if (!state.model) {
|
|
369
|
+
warnings.push("Missing model");
|
|
370
|
+
}
|
|
371
|
+
if (!state.memory) {
|
|
372
|
+
errors.push("Missing memory object");
|
|
373
|
+
} else {
|
|
374
|
+
if (typeof state.memory.size !== "number") {
|
|
375
|
+
warnings.push("Memory size should be a number");
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
if (!Array.isArray(state.messages)) {
|
|
379
|
+
errors.push("Messages should be an array");
|
|
380
|
+
} else {
|
|
381
|
+
for (let i = 0; i < state.messages.length; i++) {
|
|
382
|
+
const msg = state.messages[i];
|
|
383
|
+
if (!msg.role) {
|
|
384
|
+
errors.push(`Message ${i} missing role`);
|
|
385
|
+
}
|
|
386
|
+
if (msg.content === void 0 && !msg.toolCalls) {
|
|
387
|
+
warnings.push(`Message ${i} has no content or tool calls`);
|
|
388
|
+
}
|
|
389
|
+
if (typeof msg.content === "string" && msg.content.trim() === "") {
|
|
390
|
+
warnings.push(`Message ${i} has empty content`);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
if (!state.context || typeof state.context !== "object") {
|
|
395
|
+
warnings.push("Context should be an object");
|
|
396
|
+
}
|
|
397
|
+
if (!Array.isArray(state.tools)) {
|
|
398
|
+
warnings.push("Tools should be an array");
|
|
399
|
+
}
|
|
400
|
+
return {
|
|
401
|
+
valid: errors.length === 0,
|
|
402
|
+
errors,
|
|
403
|
+
warnings
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Merge two states
|
|
408
|
+
*/
|
|
409
|
+
merge(base, overlay) {
|
|
410
|
+
const merged = deepClone(base);
|
|
411
|
+
if (overlay.agentId) {
|
|
412
|
+
merged.agentId = overlay.agentId;
|
|
413
|
+
}
|
|
414
|
+
if (overlay.agentName) {
|
|
415
|
+
merged.agentName = overlay.agentName;
|
|
416
|
+
}
|
|
417
|
+
if (overlay.model) {
|
|
418
|
+
merged.model = overlay.model;
|
|
419
|
+
}
|
|
420
|
+
if (overlay.memory) {
|
|
421
|
+
merged.memory = {
|
|
422
|
+
...merged.memory,
|
|
423
|
+
...overlay.memory
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
if (overlay.context) {
|
|
427
|
+
merged.context = {
|
|
428
|
+
...merged.context,
|
|
429
|
+
...overlay.context
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
if (overlay.tools) {
|
|
433
|
+
merged.tools = [...overlay.tools];
|
|
434
|
+
}
|
|
435
|
+
if (overlay.messages) {
|
|
436
|
+
merged.messages = [...overlay.messages];
|
|
437
|
+
}
|
|
438
|
+
return merged;
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Create a minimal state
|
|
442
|
+
*/
|
|
443
|
+
createMinimalState(agentId, agentName, model) {
|
|
444
|
+
return {
|
|
445
|
+
agentId,
|
|
446
|
+
agentName,
|
|
447
|
+
model,
|
|
448
|
+
memory: {
|
|
449
|
+
size: 0
|
|
450
|
+
},
|
|
451
|
+
context: {},
|
|
452
|
+
tools: [],
|
|
453
|
+
messages: []
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Clear state cache
|
|
458
|
+
*/
|
|
459
|
+
clearCache() {
|
|
460
|
+
this.stateCache.clear();
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Clear cache for specific recording
|
|
464
|
+
*/
|
|
465
|
+
clearRecordingCache(recordingId) {
|
|
466
|
+
this.stateCache.delete(recordingId);
|
|
467
|
+
}
|
|
468
|
+
/**
|
|
469
|
+
* Get cache size
|
|
470
|
+
*/
|
|
471
|
+
getCacheSize() {
|
|
472
|
+
let size = 0;
|
|
473
|
+
for (const cache of this.stateCache.values()) {
|
|
474
|
+
size += cache.size;
|
|
475
|
+
}
|
|
476
|
+
return size;
|
|
477
|
+
}
|
|
478
|
+
};
|
|
479
|
+
var ReplayController = class extends EventEmitter {
|
|
480
|
+
recording;
|
|
481
|
+
session;
|
|
482
|
+
config;
|
|
483
|
+
playbackState;
|
|
484
|
+
stateHistory = /* @__PURE__ */ new Map();
|
|
485
|
+
checkpointMap = /* @__PURE__ */ new Map();
|
|
486
|
+
constructor(recording, session, config) {
|
|
487
|
+
super();
|
|
488
|
+
this.recording = recording;
|
|
489
|
+
this.session = session;
|
|
490
|
+
this.config = config;
|
|
491
|
+
this.playbackState = {
|
|
492
|
+
currentStep: session.currentStep,
|
|
493
|
+
state: deepClone(recording.initialState),
|
|
494
|
+
isPaused: false
|
|
495
|
+
};
|
|
496
|
+
this.stateHistory.set(-1, deepClone(recording.initialState));
|
|
497
|
+
for (const checkpoint of recording.checkpoints) {
|
|
498
|
+
this.checkpointMap.set(checkpoint.id, checkpoint);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* Step forward one step
|
|
503
|
+
*/
|
|
504
|
+
stepForward() {
|
|
505
|
+
const nextStep = this.playbackState.currentStep + 1;
|
|
506
|
+
if (nextStep >= this.recording.steps.length) {
|
|
507
|
+
return void 0;
|
|
508
|
+
}
|
|
509
|
+
const step = this.recording.steps[nextStep];
|
|
510
|
+
const newState = this.applyStep(this.playbackState.state, step);
|
|
511
|
+
this.stateHistory.set(nextStep, deepClone(newState));
|
|
512
|
+
this.playbackState.currentStep = nextStep;
|
|
513
|
+
this.playbackState.state = newState;
|
|
514
|
+
this.session.currentStep = nextStep;
|
|
515
|
+
this.emit("step:replayed", step, newState);
|
|
516
|
+
this.checkPauseConditions(step);
|
|
517
|
+
return step;
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Step backward one step
|
|
521
|
+
*/
|
|
522
|
+
stepBackward() {
|
|
523
|
+
if (this.playbackState.currentStep < 0) {
|
|
524
|
+
return void 0;
|
|
525
|
+
}
|
|
526
|
+
const prevStep = this.playbackState.currentStep - 1;
|
|
527
|
+
let state = this.stateHistory.get(prevStep);
|
|
528
|
+
if (!state) {
|
|
529
|
+
state = this.rebuildStateAt(prevStep);
|
|
530
|
+
this.stateHistory.set(prevStep, deepClone(state));
|
|
531
|
+
}
|
|
532
|
+
this.playbackState.currentStep = prevStep;
|
|
533
|
+
this.playbackState.state = deepClone(state);
|
|
534
|
+
this.session.currentStep = Math.max(0, prevStep);
|
|
535
|
+
const stepIndex = prevStep - 1;
|
|
536
|
+
if (stepIndex >= 0 && stepIndex < this.recording.steps.length) {
|
|
537
|
+
const step = this.recording.steps[stepIndex];
|
|
538
|
+
this.emit("step:replayed", step, this.playbackState.state);
|
|
539
|
+
return step;
|
|
540
|
+
}
|
|
541
|
+
return void 0;
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Jump to a specific step
|
|
545
|
+
*/
|
|
546
|
+
jumpToStep(stepIndex) {
|
|
547
|
+
if (stepIndex < -1 || stepIndex >= this.recording.steps.length) {
|
|
548
|
+
return void 0;
|
|
549
|
+
}
|
|
550
|
+
let state = this.stateHistory.get(stepIndex);
|
|
551
|
+
if (!state) {
|
|
552
|
+
state = this.rebuildStateAt(stepIndex);
|
|
553
|
+
this.stateHistory.set(stepIndex, deepClone(state));
|
|
554
|
+
}
|
|
555
|
+
this.playbackState.currentStep = stepIndex;
|
|
556
|
+
this.playbackState.state = deepClone(state);
|
|
557
|
+
this.session.currentStep = Math.max(0, stepIndex);
|
|
558
|
+
if (stepIndex >= 0) {
|
|
559
|
+
const step = this.recording.steps[stepIndex];
|
|
560
|
+
this.emit("step:replayed", step, this.playbackState.state);
|
|
561
|
+
return step;
|
|
562
|
+
}
|
|
563
|
+
return void 0;
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* Jump to a checkpoint
|
|
567
|
+
*/
|
|
568
|
+
jumpToCheckpoint(checkpointId) {
|
|
569
|
+
const checkpoint = this.checkpointMap.get(checkpointId);
|
|
570
|
+
if (!checkpoint) {
|
|
571
|
+
return false;
|
|
572
|
+
}
|
|
573
|
+
this.playbackState.currentStep = checkpoint.stepIndex;
|
|
574
|
+
this.playbackState.state = deepClone(checkpoint.state);
|
|
575
|
+
this.session.currentStep = checkpoint.stepIndex;
|
|
576
|
+
this.stateHistory.set(checkpoint.stepIndex, deepClone(checkpoint.state));
|
|
577
|
+
this.emit("checkpoint:reached", checkpoint);
|
|
578
|
+
if (checkpoint.stepIndex >= 0 && checkpoint.stepIndex < this.recording.steps.length) {
|
|
579
|
+
const step = this.recording.steps[checkpoint.stepIndex];
|
|
580
|
+
this.emit("step:replayed", step, this.playbackState.state);
|
|
581
|
+
}
|
|
582
|
+
return true;
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* Get next checkpoint
|
|
586
|
+
*/
|
|
587
|
+
getNextCheckpoint() {
|
|
588
|
+
const currentStep = this.playbackState.currentStep;
|
|
589
|
+
for (const checkpoint of this.recording.checkpoints) {
|
|
590
|
+
if (checkpoint.stepIndex > currentStep) {
|
|
591
|
+
return checkpoint;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
return void 0;
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Get previous checkpoint
|
|
598
|
+
*/
|
|
599
|
+
getPreviousCheckpoint() {
|
|
600
|
+
const currentStep = this.playbackState.currentStep;
|
|
601
|
+
let lastCheckpoint;
|
|
602
|
+
for (const checkpoint of this.recording.checkpoints) {
|
|
603
|
+
if (checkpoint.stepIndex < currentStep) {
|
|
604
|
+
lastCheckpoint = checkpoint;
|
|
605
|
+
} else {
|
|
606
|
+
break;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
return lastCheckpoint;
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Pause playback
|
|
613
|
+
*/
|
|
614
|
+
pause(reason) {
|
|
615
|
+
this.playbackState.isPaused = true;
|
|
616
|
+
this.playbackState.pauseReason = reason;
|
|
617
|
+
this.emit("paused", reason ?? "Manual pause");
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* Resume playback
|
|
621
|
+
*/
|
|
622
|
+
resume() {
|
|
623
|
+
this.playbackState.isPaused = false;
|
|
624
|
+
this.playbackState.pauseReason = void 0;
|
|
625
|
+
this.emit("resumed");
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Get current step
|
|
629
|
+
*/
|
|
630
|
+
getCurrentStep() {
|
|
631
|
+
const index = this.playbackState.currentStep;
|
|
632
|
+
if (index >= 0 && index < this.recording.steps.length) {
|
|
633
|
+
return this.recording.steps[index];
|
|
634
|
+
}
|
|
635
|
+
return void 0;
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Get current state
|
|
639
|
+
*/
|
|
640
|
+
getCurrentState() {
|
|
641
|
+
return deepClone(this.playbackState.state);
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Get playback state
|
|
645
|
+
*/
|
|
646
|
+
getPlaybackState() {
|
|
647
|
+
return { ...this.playbackState };
|
|
648
|
+
}
|
|
649
|
+
/**
|
|
650
|
+
* Check if at beginning
|
|
651
|
+
*/
|
|
652
|
+
isAtBeginning() {
|
|
653
|
+
return this.playbackState.currentStep <= 0;
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* Check if at end
|
|
657
|
+
*/
|
|
658
|
+
isAtEnd() {
|
|
659
|
+
return this.playbackState.currentStep >= this.recording.steps.length - 1;
|
|
660
|
+
}
|
|
661
|
+
/**
|
|
662
|
+
* Get progress percentage
|
|
663
|
+
*/
|
|
664
|
+
getProgress() {
|
|
665
|
+
if (this.recording.steps.length === 0) {
|
|
666
|
+
return 100;
|
|
667
|
+
}
|
|
668
|
+
return (this.playbackState.currentStep + 1) / this.recording.steps.length * 100;
|
|
669
|
+
}
|
|
670
|
+
/**
|
|
671
|
+
* Apply a step to state
|
|
672
|
+
*/
|
|
673
|
+
applyStep(state, step) {
|
|
674
|
+
const newState = deepClone(state);
|
|
675
|
+
switch (step.type) {
|
|
676
|
+
case "input":
|
|
677
|
+
newState.messages.push({
|
|
678
|
+
role: "user",
|
|
679
|
+
content: String(step.input)
|
|
680
|
+
});
|
|
681
|
+
break;
|
|
682
|
+
case "response":
|
|
683
|
+
newState.messages.push({
|
|
684
|
+
role: "assistant",
|
|
685
|
+
content: String(step.output)
|
|
686
|
+
});
|
|
687
|
+
break;
|
|
688
|
+
case "tool-call":
|
|
689
|
+
if (step.toolCall) {
|
|
690
|
+
newState.messages.push({
|
|
691
|
+
role: "assistant",
|
|
692
|
+
content: `[Tool Call: ${step.toolCall.name}]`
|
|
693
|
+
});
|
|
694
|
+
}
|
|
695
|
+
break;
|
|
696
|
+
case "tool-result":
|
|
697
|
+
if (step.toolCall) {
|
|
698
|
+
newState.messages.push({
|
|
699
|
+
role: "tool",
|
|
700
|
+
content: String(step.toolCall.result)
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
break;
|
|
704
|
+
case "memory-write":
|
|
705
|
+
if (step.output && typeof step.output === "object") {
|
|
706
|
+
Object.assign(newState.memory, step.output);
|
|
707
|
+
}
|
|
708
|
+
break;
|
|
709
|
+
}
|
|
710
|
+
return newState;
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Rebuild state at a specific step
|
|
714
|
+
*/
|
|
715
|
+
rebuildStateAt(targetStep) {
|
|
716
|
+
let startStep = -1;
|
|
717
|
+
let state = deepClone(this.recording.initialState);
|
|
718
|
+
for (let i = targetStep; i >= -1; i--) {
|
|
719
|
+
const cached = this.stateHistory.get(i);
|
|
720
|
+
if (cached) {
|
|
721
|
+
startStep = i;
|
|
722
|
+
state = deepClone(cached);
|
|
723
|
+
break;
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
for (const checkpoint of this.recording.checkpoints) {
|
|
727
|
+
if (checkpoint.stepIndex > startStep && checkpoint.stepIndex <= targetStep) {
|
|
728
|
+
startStep = checkpoint.stepIndex;
|
|
729
|
+
state = deepClone(checkpoint.state);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
for (let i = startStep + 1; i <= targetStep; i++) {
|
|
733
|
+
if (i >= 0 && i < this.recording.steps.length) {
|
|
734
|
+
state = this.applyStep(state, this.recording.steps[i]);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
return state;
|
|
738
|
+
}
|
|
739
|
+
/**
|
|
740
|
+
* Check pause conditions
|
|
741
|
+
*/
|
|
742
|
+
checkPauseConditions(step) {
|
|
743
|
+
if (this.config.pauseOnDecisions && step.type === "decision") {
|
|
744
|
+
this.pause("Decision point");
|
|
745
|
+
}
|
|
746
|
+
if (this.config.pauseOnErrors && step.error) {
|
|
747
|
+
this.pause("Error occurred");
|
|
748
|
+
}
|
|
749
|
+
if (this.config.pauseOnToolCalls && step.type === "tool-call") {
|
|
750
|
+
this.pause("Tool call");
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
/**
|
|
754
|
+
* Get all checkpoints
|
|
755
|
+
*/
|
|
756
|
+
getCheckpoints() {
|
|
757
|
+
return this.recording.checkpoints;
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* Get steps count
|
|
761
|
+
*/
|
|
762
|
+
get stepsCount() {
|
|
763
|
+
return this.recording.steps.length;
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
|
|
767
|
+
// src/replay/ReplayEngine.ts
|
|
768
|
+
var DEFAULT_CONFIG = {
|
|
769
|
+
speedMultiplier: 1,
|
|
770
|
+
pauseOnDecisions: false,
|
|
771
|
+
pauseOnErrors: true,
|
|
772
|
+
pauseOnToolCalls: false,
|
|
773
|
+
executeTools: false,
|
|
774
|
+
executeLLM: false,
|
|
775
|
+
compareResults: true,
|
|
776
|
+
trackDifferences: true
|
|
777
|
+
};
|
|
778
|
+
var ReplayEngine = class extends EventEmitter {
|
|
779
|
+
config;
|
|
780
|
+
sessions = /* @__PURE__ */ new Map();
|
|
781
|
+
currentSession;
|
|
782
|
+
stateRestorer;
|
|
783
|
+
controller;
|
|
784
|
+
isRunning = false;
|
|
785
|
+
constructor(config) {
|
|
786
|
+
super();
|
|
787
|
+
this.config = {
|
|
788
|
+
...DEFAULT_CONFIG,
|
|
789
|
+
...config
|
|
790
|
+
};
|
|
791
|
+
this.stateRestorer = new StateRestorer();
|
|
792
|
+
}
|
|
793
|
+
/**
|
|
794
|
+
* Start a replay session
|
|
795
|
+
*/
|
|
796
|
+
start(recording, options) {
|
|
797
|
+
const session = {
|
|
798
|
+
id: generateId("replay"),
|
|
799
|
+
recordingId: recording.id,
|
|
800
|
+
state: "idle",
|
|
801
|
+
currentStep: options?.startStep ?? 0,
|
|
802
|
+
totalSteps: recording.steps.length,
|
|
803
|
+
speed: options?.speed ?? "normal",
|
|
804
|
+
startedAt: now(),
|
|
805
|
+
modifications: options?.modifications ?? [],
|
|
806
|
+
differences: []
|
|
807
|
+
};
|
|
808
|
+
this.sessions.set(session.id, session);
|
|
809
|
+
this.currentSession = session;
|
|
810
|
+
this.controller = new ReplayController(recording, session, {
|
|
811
|
+
...this.config,
|
|
812
|
+
speedMultiplier: this.getSpeedMultiplier(session.speed)
|
|
813
|
+
});
|
|
814
|
+
this.controller.on("step:replayed", (step, state) => {
|
|
815
|
+
this.emit("step:replayed", step, state);
|
|
816
|
+
});
|
|
817
|
+
this.controller.on("paused", () => {
|
|
818
|
+
session.state = "paused";
|
|
819
|
+
this.emit("replay:paused", session);
|
|
820
|
+
});
|
|
821
|
+
this.controller.on("error", (error) => {
|
|
822
|
+
this.emit("error", error);
|
|
823
|
+
});
|
|
824
|
+
this.emit("replay:started", session);
|
|
825
|
+
this.runReplay(recording, session, options).catch((error) => {
|
|
826
|
+
this.emit("error", error);
|
|
827
|
+
});
|
|
828
|
+
return session;
|
|
829
|
+
}
|
|
830
|
+
/**
|
|
831
|
+
* Run the replay loop
|
|
832
|
+
*/
|
|
833
|
+
async runReplay(recording, session, options) {
|
|
834
|
+
this.isRunning = true;
|
|
835
|
+
const startStep = options?.startStep ?? 0;
|
|
836
|
+
const endStep = options?.endStep ?? recording.steps.length - 1;
|
|
837
|
+
let currentState = this.stateRestorer.restore(
|
|
838
|
+
recording,
|
|
839
|
+
startStep > 0 ? startStep - 1 : 0
|
|
840
|
+
);
|
|
841
|
+
const replayedSteps = [];
|
|
842
|
+
const differences = [];
|
|
843
|
+
for (let i = startStep; i <= endStep && this.isRunning; i++) {
|
|
844
|
+
while (session.state === "paused" && this.isRunning) {
|
|
845
|
+
await sleep(100);
|
|
846
|
+
}
|
|
847
|
+
if (!this.isRunning) {
|
|
848
|
+
break;
|
|
849
|
+
}
|
|
850
|
+
const originalStep = recording.steps[i];
|
|
851
|
+
let step = deepClone(originalStep);
|
|
852
|
+
const modification = options?.modifications?.find(
|
|
853
|
+
(m) => m.stepIndex === i
|
|
854
|
+
);
|
|
855
|
+
if (modification) {
|
|
856
|
+
step = this.applyModification(step, modification);
|
|
857
|
+
this.emit("step:modified", originalStep, step);
|
|
858
|
+
}
|
|
859
|
+
const result2 = await this.executeStep(step, currentState, options);
|
|
860
|
+
if (this.config.trackDifferences && result2.executed) {
|
|
861
|
+
const stepDiffs = this.compareResults(originalStep, result2.step);
|
|
862
|
+
if (stepDiffs.length > 0) {
|
|
863
|
+
differences.push(...stepDiffs);
|
|
864
|
+
session.differences = differences;
|
|
865
|
+
this.emit("divergence:detected", stepDiffs);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
replayedSteps.push(result2.step);
|
|
869
|
+
currentState = result2.state;
|
|
870
|
+
session.currentStep = i;
|
|
871
|
+
const delay = this.getStepDelay(step, session.speed);
|
|
872
|
+
if (delay > 0) {
|
|
873
|
+
await sleep(delay);
|
|
874
|
+
}
|
|
875
|
+
this.emit("step:replayed", result2.step, currentState);
|
|
876
|
+
if (this.shouldPause(step)) {
|
|
877
|
+
session.state = "paused";
|
|
878
|
+
this.emit("replay:paused", session);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
const result = {
|
|
882
|
+
sessionId: session.id,
|
|
883
|
+
recordingId: recording.id,
|
|
884
|
+
success: this.isRunning,
|
|
885
|
+
stepsReplayed: replayedSteps.length,
|
|
886
|
+
differences,
|
|
887
|
+
finalState: currentState,
|
|
888
|
+
startedAt: session.startedAt,
|
|
889
|
+
completedAt: now(),
|
|
890
|
+
durationMs: now() - session.startedAt
|
|
891
|
+
};
|
|
892
|
+
session.state = "completed";
|
|
893
|
+
session.completedAt = result.completedAt;
|
|
894
|
+
this.isRunning = false;
|
|
895
|
+
this.emit("replay:completed", result);
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* Execute a step during replay
|
|
899
|
+
*/
|
|
900
|
+
async executeStep(step, state, options) {
|
|
901
|
+
const newStep = deepClone(step);
|
|
902
|
+
let newState = deepClone(state);
|
|
903
|
+
let executed = false;
|
|
904
|
+
if (step.type === "tool-call" && options?.executeTools && options?.onToolCall) {
|
|
905
|
+
try {
|
|
906
|
+
const result = await options.onToolCall(step);
|
|
907
|
+
newStep.toolCall = {
|
|
908
|
+
...newStep.toolCall,
|
|
909
|
+
result,
|
|
910
|
+
success: true
|
|
911
|
+
};
|
|
912
|
+
executed = true;
|
|
913
|
+
} catch (error) {
|
|
914
|
+
newStep.toolCall = {
|
|
915
|
+
...newStep.toolCall,
|
|
916
|
+
result: error.message,
|
|
917
|
+
success: false
|
|
918
|
+
};
|
|
919
|
+
executed = true;
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
if (step.type === "response" && options?.executeLLM && options?.onLLMCall) {
|
|
923
|
+
try {
|
|
924
|
+
const response = await options.onLLMCall(step);
|
|
925
|
+
newStep.output = response;
|
|
926
|
+
executed = true;
|
|
927
|
+
} catch (error) {
|
|
928
|
+
newStep.error = {
|
|
929
|
+
name: "LLMError",
|
|
930
|
+
message: error.message
|
|
931
|
+
};
|
|
932
|
+
executed = true;
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
newState = this.updateState(newState, newStep);
|
|
936
|
+
return { step: newStep, state: newState, executed };
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Update state based on step
|
|
940
|
+
*/
|
|
941
|
+
updateState(state, step) {
|
|
942
|
+
const newState = deepClone(state);
|
|
943
|
+
if (step.type === "input") {
|
|
944
|
+
newState.messages.push({
|
|
945
|
+
role: "user",
|
|
946
|
+
content: String(step.input)
|
|
947
|
+
});
|
|
948
|
+
} else if (step.type === "response") {
|
|
949
|
+
newState.messages.push({
|
|
950
|
+
role: "assistant",
|
|
951
|
+
content: String(step.output)
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
return newState;
|
|
955
|
+
}
|
|
956
|
+
/**
|
|
957
|
+
* Apply a modification to a step
|
|
958
|
+
*/
|
|
959
|
+
applyModification(step, modification) {
|
|
960
|
+
const modified = deepClone(step);
|
|
961
|
+
switch (modification.type) {
|
|
962
|
+
case "skip":
|
|
963
|
+
modified.metadata = {
|
|
964
|
+
...modified.metadata,
|
|
965
|
+
skipped: true
|
|
966
|
+
};
|
|
967
|
+
break;
|
|
968
|
+
case "modify":
|
|
969
|
+
if (modification.data) {
|
|
970
|
+
Object.assign(modified, modification.data);
|
|
971
|
+
}
|
|
972
|
+
break;
|
|
973
|
+
case "insert":
|
|
974
|
+
break;
|
|
975
|
+
case "replace":
|
|
976
|
+
if (modification.data) {
|
|
977
|
+
return {
|
|
978
|
+
...modified,
|
|
979
|
+
...modification.data
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
break;
|
|
983
|
+
}
|
|
984
|
+
return modified;
|
|
985
|
+
}
|
|
986
|
+
/**
|
|
987
|
+
* Compare original and replayed results
|
|
988
|
+
*/
|
|
989
|
+
compareResults(original, replayed) {
|
|
990
|
+
const differences = diff(original, replayed);
|
|
991
|
+
return toReplayDifferences(differences, original.index);
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* Check if should pause on this step
|
|
995
|
+
*/
|
|
996
|
+
shouldPause(step) {
|
|
997
|
+
if (this.config.pauseOnDecisions && step.type === "decision") {
|
|
998
|
+
return true;
|
|
999
|
+
}
|
|
1000
|
+
if (this.config.pauseOnErrors && step.error) {
|
|
1001
|
+
return true;
|
|
1002
|
+
}
|
|
1003
|
+
if (this.config.pauseOnToolCalls && step.type === "tool-call") {
|
|
1004
|
+
return true;
|
|
1005
|
+
}
|
|
1006
|
+
return false;
|
|
1007
|
+
}
|
|
1008
|
+
/**
|
|
1009
|
+
* Get speed multiplier
|
|
1010
|
+
*/
|
|
1011
|
+
getSpeedMultiplier(speed) {
|
|
1012
|
+
switch (speed) {
|
|
1013
|
+
case "slow":
|
|
1014
|
+
return 0.5;
|
|
1015
|
+
case "normal":
|
|
1016
|
+
return 1;
|
|
1017
|
+
case "fast":
|
|
1018
|
+
return 2;
|
|
1019
|
+
case "instant":
|
|
1020
|
+
return 0;
|
|
1021
|
+
default:
|
|
1022
|
+
return 1;
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Get delay for step based on speed
|
|
1027
|
+
*/
|
|
1028
|
+
getStepDelay(step, speed) {
|
|
1029
|
+
if (speed === "instant") {
|
|
1030
|
+
return 0;
|
|
1031
|
+
}
|
|
1032
|
+
const baseDelay = step.durationMs ?? 100;
|
|
1033
|
+
const multiplier = this.getSpeedMultiplier(speed);
|
|
1034
|
+
return baseDelay / multiplier;
|
|
1035
|
+
}
|
|
1036
|
+
/**
|
|
1037
|
+
* Pause current replay
|
|
1038
|
+
*/
|
|
1039
|
+
pause() {
|
|
1040
|
+
if (this.currentSession && this.currentSession.state !== "paused" && this.currentSession.state !== "stopped") {
|
|
1041
|
+
this.currentSession.state = "paused";
|
|
1042
|
+
this.emit("replay:paused", this.currentSession);
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
/**
|
|
1046
|
+
* Resume current replay
|
|
1047
|
+
*/
|
|
1048
|
+
resume() {
|
|
1049
|
+
if (this.currentSession && this.currentSession.state === "paused") {
|
|
1050
|
+
this.currentSession.state = "playing";
|
|
1051
|
+
this.emit("replay:resumed", this.currentSession);
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
/**
|
|
1055
|
+
* Stop current replay
|
|
1056
|
+
*/
|
|
1057
|
+
stop() {
|
|
1058
|
+
this.isRunning = false;
|
|
1059
|
+
if (this.currentSession) {
|
|
1060
|
+
this.currentSession.state = "stopped";
|
|
1061
|
+
this.emit("replay:stopped", this.currentSession);
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
/**
|
|
1065
|
+
* Set replay speed
|
|
1066
|
+
*/
|
|
1067
|
+
setSpeed(speed) {
|
|
1068
|
+
if (this.currentSession) {
|
|
1069
|
+
this.currentSession.speed = speed;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
/**
|
|
1073
|
+
* Jump to step
|
|
1074
|
+
*/
|
|
1075
|
+
jumpToStep(stepIndex) {
|
|
1076
|
+
if (!this.currentSession) {
|
|
1077
|
+
return;
|
|
1078
|
+
}
|
|
1079
|
+
this.currentSession.currentStep = stepIndex;
|
|
1080
|
+
}
|
|
1081
|
+
/**
|
|
1082
|
+
* Get current session
|
|
1083
|
+
*/
|
|
1084
|
+
getSession() {
|
|
1085
|
+
return this.currentSession;
|
|
1086
|
+
}
|
|
1087
|
+
/**
|
|
1088
|
+
* Get session by ID
|
|
1089
|
+
*/
|
|
1090
|
+
getSessionById(id) {
|
|
1091
|
+
return this.sessions.get(id);
|
|
1092
|
+
}
|
|
1093
|
+
/**
|
|
1094
|
+
* Get all sessions
|
|
1095
|
+
*/
|
|
1096
|
+
getSessions() {
|
|
1097
|
+
return Array.from(this.sessions.values());
|
|
1098
|
+
}
|
|
1099
|
+
};
|
|
1100
|
+
|
|
1101
|
+
// src/analysis/WhatIfEngine.ts
|
|
1102
|
+
var WhatIfEngine = class extends EventEmitter {
|
|
1103
|
+
scenarios = /* @__PURE__ */ new Map();
|
|
1104
|
+
results = /* @__PURE__ */ new Map();
|
|
1105
|
+
replayEngine;
|
|
1106
|
+
stateRestorer;
|
|
1107
|
+
constructor() {
|
|
1108
|
+
super();
|
|
1109
|
+
this.replayEngine = new ReplayEngine();
|
|
1110
|
+
this.stateRestorer = new StateRestorer();
|
|
1111
|
+
}
|
|
1112
|
+
/**
|
|
1113
|
+
* Create a what-if scenario
|
|
1114
|
+
*/
|
|
1115
|
+
createScenario(options) {
|
|
1116
|
+
const scenario = {
|
|
1117
|
+
id: generateId("whatif"),
|
|
1118
|
+
name: options.name,
|
|
1119
|
+
description: options.description,
|
|
1120
|
+
baseRecordingId: options.recordingId,
|
|
1121
|
+
modifications: options.modifications,
|
|
1122
|
+
createdAt: now(),
|
|
1123
|
+
status: "pending"
|
|
1124
|
+
};
|
|
1125
|
+
this.scenarios.set(scenario.id, scenario);
|
|
1126
|
+
this.emit("scenario:created", scenario);
|
|
1127
|
+
return scenario;
|
|
1128
|
+
}
|
|
1129
|
+
/**
|
|
1130
|
+
* Run a scenario
|
|
1131
|
+
*/
|
|
1132
|
+
async runScenario(scenarioId, recording, options) {
|
|
1133
|
+
const scenario = this.scenarios.get(scenarioId);
|
|
1134
|
+
if (!scenario) {
|
|
1135
|
+
throw new Error(`Scenario not found: ${scenarioId}`);
|
|
1136
|
+
}
|
|
1137
|
+
scenario.status = "running";
|
|
1138
|
+
this.emit("scenario:started", scenarioId);
|
|
1139
|
+
try {
|
|
1140
|
+
this.replayEngine.start(recording, {
|
|
1141
|
+
modifications: scenario.modifications,
|
|
1142
|
+
executeTools: options?.executeTools,
|
|
1143
|
+
executeLLM: options?.executeLLM,
|
|
1144
|
+
onToolCall: options?.onToolCall,
|
|
1145
|
+
onLLMCall: options?.onLLMCall
|
|
1146
|
+
});
|
|
1147
|
+
await this.waitForReplayCompletion();
|
|
1148
|
+
const replaySession = this.replayEngine.getSession();
|
|
1149
|
+
const differences = replaySession?.differences ?? [];
|
|
1150
|
+
const result = {
|
|
1151
|
+
scenarioId,
|
|
1152
|
+
success: true,
|
|
1153
|
+
originalRecordingId: recording.id,
|
|
1154
|
+
modifiedSteps: scenario.modifications.length,
|
|
1155
|
+
differences,
|
|
1156
|
+
divergencePoint: differences.length > 0 ? differences[0].stepIndex : void 0,
|
|
1157
|
+
finalState: replaySession ? deepClone(recording.finalState ?? recording.initialState) : recording.finalState ?? recording.initialState,
|
|
1158
|
+
executedAt: now(),
|
|
1159
|
+
durationMs: now() - scenario.createdAt
|
|
1160
|
+
};
|
|
1161
|
+
scenario.status = "completed";
|
|
1162
|
+
this.results.set(scenarioId, result);
|
|
1163
|
+
this.emit("scenario:completed", result);
|
|
1164
|
+
return result;
|
|
1165
|
+
} catch (error) {
|
|
1166
|
+
scenario.status = "failed";
|
|
1167
|
+
this.emit("scenario:failed", scenarioId, error);
|
|
1168
|
+
throw error;
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
/**
|
|
1172
|
+
* Wait for replay to complete
|
|
1173
|
+
*/
|
|
1174
|
+
waitForReplayCompletion() {
|
|
1175
|
+
return new Promise((resolve) => {
|
|
1176
|
+
const handler = () => {
|
|
1177
|
+
this.replayEngine.off("replay:completed", handler);
|
|
1178
|
+
this.replayEngine.off("replay:stopped", handler);
|
|
1179
|
+
resolve();
|
|
1180
|
+
};
|
|
1181
|
+
this.replayEngine.on("replay:completed", handler);
|
|
1182
|
+
this.replayEngine.on("replay:stopped", handler);
|
|
1183
|
+
});
|
|
1184
|
+
}
|
|
1185
|
+
/**
|
|
1186
|
+
* Run multiple scenarios in batch
|
|
1187
|
+
*/
|
|
1188
|
+
async runBatch(options, recording) {
|
|
1189
|
+
const scenarios = options.variations.map(
|
|
1190
|
+
(v) => this.createScenario({
|
|
1191
|
+
name: v.name,
|
|
1192
|
+
recordingId: options.recordingId,
|
|
1193
|
+
modifications: v.modifications
|
|
1194
|
+
})
|
|
1195
|
+
);
|
|
1196
|
+
if (options.parallel) {
|
|
1197
|
+
return Promise.all(
|
|
1198
|
+
scenarios.map((s) => this.runScenario(s.id, recording))
|
|
1199
|
+
);
|
|
1200
|
+
} else {
|
|
1201
|
+
const results = [];
|
|
1202
|
+
for (const scenario of scenarios) {
|
|
1203
|
+
const result = await this.runScenario(scenario.id, recording);
|
|
1204
|
+
results.push(result);
|
|
1205
|
+
}
|
|
1206
|
+
return results;
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
/**
|
|
1210
|
+
* Compare original recording with scenario result
|
|
1211
|
+
*/
|
|
1212
|
+
compare(original, result) {
|
|
1213
|
+
const scenario = this.scenarios.get(result.scenarioId);
|
|
1214
|
+
const outcomeChanged = original.status !== (result.success ? "completed" : "failed");
|
|
1215
|
+
const totalSteps = original.steps.length;
|
|
1216
|
+
const divergenceStep = result.divergencePoint ?? totalSteps;
|
|
1217
|
+
const divergencePercentage = (totalSteps - divergenceStep) / totalSteps * 100;
|
|
1218
|
+
return {
|
|
1219
|
+
scenarioId: result.scenarioId,
|
|
1220
|
+
scenarioName: scenario?.name ?? "Unknown",
|
|
1221
|
+
originalRecordingId: original.id,
|
|
1222
|
+
outcomeChanged,
|
|
1223
|
+
divergencePoint: result.divergencePoint,
|
|
1224
|
+
divergencePercentage,
|
|
1225
|
+
differences: result.differences,
|
|
1226
|
+
summary: this.generateComparisonSummary(original, result)
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
/**
|
|
1230
|
+
* Generate a summary of the comparison
|
|
1231
|
+
*/
|
|
1232
|
+
generateComparisonSummary(original, result) {
|
|
1233
|
+
const lines = [];
|
|
1234
|
+
if (result.divergencePoint !== void 0) {
|
|
1235
|
+
lines.push(`Diverged at step ${result.divergencePoint}`);
|
|
1236
|
+
} else {
|
|
1237
|
+
lines.push("No divergence detected");
|
|
1238
|
+
}
|
|
1239
|
+
lines.push(`${result.modifiedSteps} step(s) modified`);
|
|
1240
|
+
lines.push(`${result.differences.length} difference(s) detected`);
|
|
1241
|
+
return lines.join(". ");
|
|
1242
|
+
}
|
|
1243
|
+
/**
|
|
1244
|
+
* Create scenario from a decision point
|
|
1245
|
+
*/
|
|
1246
|
+
createFromDecision(recording, stepIndex, alternativeChoice) {
|
|
1247
|
+
const step = recording.steps[stepIndex];
|
|
1248
|
+
if (step.type !== "decision" || !step.decision) {
|
|
1249
|
+
throw new Error("Step is not a decision point");
|
|
1250
|
+
}
|
|
1251
|
+
const alternative = step.decision.options.find(
|
|
1252
|
+
(o) => o.id === alternativeChoice || o.description === alternativeChoice
|
|
1253
|
+
);
|
|
1254
|
+
if (!alternative) {
|
|
1255
|
+
throw new Error("Alternative choice not found in decision options");
|
|
1256
|
+
}
|
|
1257
|
+
return this.createScenario({
|
|
1258
|
+
name: `What if: ${alternative.description}`,
|
|
1259
|
+
description: `Explore alternative decision at step ${stepIndex}`,
|
|
1260
|
+
recordingId: recording.id,
|
|
1261
|
+
modifications: [
|
|
1262
|
+
{
|
|
1263
|
+
stepIndex,
|
|
1264
|
+
type: "modify",
|
|
1265
|
+
data: {
|
|
1266
|
+
decision: {
|
|
1267
|
+
...step.decision,
|
|
1268
|
+
chosen: alternative
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
]
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1275
|
+
/**
|
|
1276
|
+
* Create scenario from a tool result change
|
|
1277
|
+
*/
|
|
1278
|
+
createFromToolResult(recording, stepIndex, alternativeResult) {
|
|
1279
|
+
const step = recording.steps[stepIndex];
|
|
1280
|
+
if (step.type !== "tool-result" || !step.toolCall) {
|
|
1281
|
+
throw new Error("Step is not a tool result");
|
|
1282
|
+
}
|
|
1283
|
+
return this.createScenario({
|
|
1284
|
+
name: `What if: ${step.toolCall.name} returned different result`,
|
|
1285
|
+
description: `Explore alternative tool result at step ${stepIndex}`,
|
|
1286
|
+
recordingId: recording.id,
|
|
1287
|
+
modifications: [
|
|
1288
|
+
{
|
|
1289
|
+
stepIndex,
|
|
1290
|
+
type: "modify",
|
|
1291
|
+
data: {
|
|
1292
|
+
toolCall: {
|
|
1293
|
+
...step.toolCall,
|
|
1294
|
+
result: alternativeResult
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
]
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
/**
|
|
1302
|
+
* Create scenario from skipping steps
|
|
1303
|
+
*/
|
|
1304
|
+
createFromSkip(recording, stepIndices, name) {
|
|
1305
|
+
return this.createScenario({
|
|
1306
|
+
name: name ?? `What if: Skip steps ${stepIndices.join(", ")}`,
|
|
1307
|
+
description: `Skip specific steps in the execution`,
|
|
1308
|
+
recordingId: recording.id,
|
|
1309
|
+
modifications: stepIndices.map((stepIndex) => ({
|
|
1310
|
+
stepIndex,
|
|
1311
|
+
type: "skip"
|
|
1312
|
+
}))
|
|
1313
|
+
});
|
|
1314
|
+
}
|
|
1315
|
+
/**
|
|
1316
|
+
* Get a scenario by ID
|
|
1317
|
+
*/
|
|
1318
|
+
getScenario(id) {
|
|
1319
|
+
return this.scenarios.get(id);
|
|
1320
|
+
}
|
|
1321
|
+
/**
|
|
1322
|
+
* Get all scenarios
|
|
1323
|
+
*/
|
|
1324
|
+
getScenarios() {
|
|
1325
|
+
return Array.from(this.scenarios.values());
|
|
1326
|
+
}
|
|
1327
|
+
/**
|
|
1328
|
+
* Get scenarios for a recording
|
|
1329
|
+
*/
|
|
1330
|
+
getScenariosForRecording(recordingId) {
|
|
1331
|
+
return this.getScenarios().filter((s) => s.baseRecordingId === recordingId);
|
|
1332
|
+
}
|
|
1333
|
+
/**
|
|
1334
|
+
* Get a result by scenario ID
|
|
1335
|
+
*/
|
|
1336
|
+
getResult(scenarioId) {
|
|
1337
|
+
return this.results.get(scenarioId);
|
|
1338
|
+
}
|
|
1339
|
+
/**
|
|
1340
|
+
* Get all results
|
|
1341
|
+
*/
|
|
1342
|
+
getResults() {
|
|
1343
|
+
return Array.from(this.results.values());
|
|
1344
|
+
}
|
|
1345
|
+
/**
|
|
1346
|
+
* Delete a scenario
|
|
1347
|
+
*/
|
|
1348
|
+
deleteScenario(id) {
|
|
1349
|
+
this.results.delete(id);
|
|
1350
|
+
return this.scenarios.delete(id);
|
|
1351
|
+
}
|
|
1352
|
+
/**
|
|
1353
|
+
* Clear all scenarios
|
|
1354
|
+
*/
|
|
1355
|
+
clear() {
|
|
1356
|
+
this.scenarios.clear();
|
|
1357
|
+
this.results.clear();
|
|
1358
|
+
}
|
|
1359
|
+
};
|
|
1360
|
+
function createWhatIfEngine() {
|
|
1361
|
+
return new WhatIfEngine();
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
// src/analysis/FailureAnalyzer.ts
|
|
1365
|
+
var DEFAULT_PATTERNS = [
|
|
1366
|
+
{
|
|
1367
|
+
id: "repeated_tool_failure",
|
|
1368
|
+
name: "Repeated Tool Failures",
|
|
1369
|
+
description: "The same tool failed multiple times in succession",
|
|
1370
|
+
matcher: (recording, steps) => {
|
|
1371
|
+
const toolFailures = steps.filter(
|
|
1372
|
+
(s) => s.type === "tool-result" && s.toolCall && !s.toolCall.success
|
|
1373
|
+
);
|
|
1374
|
+
if (toolFailures.length < 2) return false;
|
|
1375
|
+
for (let i = 0; i < toolFailures.length - 1; i++) {
|
|
1376
|
+
if (toolFailures[i].toolCall?.name === toolFailures[i + 1].toolCall?.name) {
|
|
1377
|
+
return true;
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
return false;
|
|
1381
|
+
},
|
|
1382
|
+
severity: "high",
|
|
1383
|
+
recommendations: [
|
|
1384
|
+
"Add retry logic with exponential backoff",
|
|
1385
|
+
"Implement fallback tools",
|
|
1386
|
+
"Add input validation before tool calls"
|
|
1387
|
+
]
|
|
1388
|
+
},
|
|
1389
|
+
{
|
|
1390
|
+
id: "low_confidence_decision",
|
|
1391
|
+
name: "Low Confidence Decision",
|
|
1392
|
+
description: "A critical decision was made with low confidence",
|
|
1393
|
+
matcher: (recording, steps) => {
|
|
1394
|
+
return steps.some(
|
|
1395
|
+
(s) => s.type === "decision" && s.decision && s.decision.confidence < 0.5
|
|
1396
|
+
);
|
|
1397
|
+
},
|
|
1398
|
+
severity: "medium",
|
|
1399
|
+
recommendations: [
|
|
1400
|
+
"Gather more context before making decisions",
|
|
1401
|
+
"Request clarification from user for ambiguous cases",
|
|
1402
|
+
"Add confidence thresholds that trigger fallback behavior"
|
|
1403
|
+
]
|
|
1404
|
+
},
|
|
1405
|
+
{
|
|
1406
|
+
id: "infinite_loop",
|
|
1407
|
+
name: "Potential Infinite Loop",
|
|
1408
|
+
description: "Similar steps repeated many times without progress",
|
|
1409
|
+
matcher: (recording, steps) => {
|
|
1410
|
+
const stepTypes = steps.map((s) => s.type);
|
|
1411
|
+
const windowSize = 3;
|
|
1412
|
+
const minRepetitions = 4;
|
|
1413
|
+
for (let i = 0; i <= stepTypes.length - windowSize * minRepetitions; i++) {
|
|
1414
|
+
const window1 = stepTypes.slice(i, i + windowSize).join(",");
|
|
1415
|
+
let repetitions = 1;
|
|
1416
|
+
for (let j = i + windowSize; j <= stepTypes.length - windowSize; j += windowSize) {
|
|
1417
|
+
const nextWindow = stepTypes.slice(j, j + windowSize).join(",");
|
|
1418
|
+
if (nextWindow === window1) {
|
|
1419
|
+
repetitions++;
|
|
1420
|
+
} else {
|
|
1421
|
+
break;
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
if (repetitions >= minRepetitions) {
|
|
1425
|
+
return true;
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
return false;
|
|
1429
|
+
},
|
|
1430
|
+
severity: "critical",
|
|
1431
|
+
recommendations: [
|
|
1432
|
+
"Add loop detection and break conditions",
|
|
1433
|
+
"Track state changes to detect lack of progress",
|
|
1434
|
+
"Implement maximum iteration limits"
|
|
1435
|
+
]
|
|
1436
|
+
},
|
|
1437
|
+
{
|
|
1438
|
+
id: "missing_context",
|
|
1439
|
+
name: "Missing Context",
|
|
1440
|
+
description: "Tool or decision made without required context",
|
|
1441
|
+
matcher: (recording, steps) => {
|
|
1442
|
+
for (let i = 0; i < steps.length - 1; i++) {
|
|
1443
|
+
if (steps[i].type === "input" && steps[i + 1].type === "tool-call") {
|
|
1444
|
+
return true;
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
return false;
|
|
1448
|
+
},
|
|
1449
|
+
severity: "low",
|
|
1450
|
+
recommendations: [
|
|
1451
|
+
"Add context gathering step before tool calls",
|
|
1452
|
+
"Use memory retrieval to provide relevant context",
|
|
1453
|
+
"Implement input analysis before action"
|
|
1454
|
+
]
|
|
1455
|
+
},
|
|
1456
|
+
{
|
|
1457
|
+
id: "memory_overflow",
|
|
1458
|
+
name: "Memory Issues",
|
|
1459
|
+
description: "Memory size grew excessively during execution",
|
|
1460
|
+
matcher: (recording) => {
|
|
1461
|
+
const initialSize = recording.initialState.memory.size;
|
|
1462
|
+
const finalSize = recording.finalState?.memory?.size ?? 0;
|
|
1463
|
+
return finalSize > initialSize * 10 && finalSize > 1024 * 1024;
|
|
1464
|
+
},
|
|
1465
|
+
severity: "medium",
|
|
1466
|
+
recommendations: [
|
|
1467
|
+
"Implement memory compaction",
|
|
1468
|
+
"Add memory limits and eviction policies",
|
|
1469
|
+
"Summarize long conversations"
|
|
1470
|
+
]
|
|
1471
|
+
},
|
|
1472
|
+
{
|
|
1473
|
+
id: "timeout_likely",
|
|
1474
|
+
name: "Slow Execution",
|
|
1475
|
+
description: "Execution took excessively long, possibly due to timeouts",
|
|
1476
|
+
matcher: (recording) => {
|
|
1477
|
+
const avgDuration = recording.durationMs / recording.steps.length;
|
|
1478
|
+
return avgDuration > 1e4;
|
|
1479
|
+
},
|
|
1480
|
+
severity: "medium",
|
|
1481
|
+
recommendations: [
|
|
1482
|
+
"Add timeout handling for external calls",
|
|
1483
|
+
"Implement caching for repeated operations",
|
|
1484
|
+
"Optimize tool implementations"
|
|
1485
|
+
]
|
|
1486
|
+
}
|
|
1487
|
+
];
|
|
1488
|
+
var FailureAnalyzer = class {
|
|
1489
|
+
patterns;
|
|
1490
|
+
options;
|
|
1491
|
+
constructor(options) {
|
|
1492
|
+
this.options = {
|
|
1493
|
+
includeDetailedSteps: options?.includeDetailedSteps ?? true,
|
|
1494
|
+
includeMemoryAnalysis: options?.includeMemoryAnalysis ?? true,
|
|
1495
|
+
includeTimingAnalysis: options?.includeTimingAnalysis ?? true,
|
|
1496
|
+
customPatterns: options?.customPatterns ?? []
|
|
1497
|
+
};
|
|
1498
|
+
this.patterns = [...DEFAULT_PATTERNS, ...this.options.customPatterns];
|
|
1499
|
+
}
|
|
1500
|
+
/**
|
|
1501
|
+
* Analyze a failed recording
|
|
1502
|
+
*/
|
|
1503
|
+
analyze(recording) {
|
|
1504
|
+
const contributingFactors = this.findContributingFactors(recording);
|
|
1505
|
+
const rootCause = this.determineRootCause(recording, contributingFactors);
|
|
1506
|
+
const recommendations = this.generateRecommendations(contributingFactors);
|
|
1507
|
+
const errorStepIndex = this.findErrorStep(recording);
|
|
1508
|
+
return {
|
|
1509
|
+
id: generateId("analysis"),
|
|
1510
|
+
recordingId: recording.id,
|
|
1511
|
+
analyzedAt: now(),
|
|
1512
|
+
rootCause,
|
|
1513
|
+
contributingFactors,
|
|
1514
|
+
recommendations,
|
|
1515
|
+
errorStepIndex,
|
|
1516
|
+
errorMessage: this.getErrorMessage(recording, errorStepIndex),
|
|
1517
|
+
stackTrace: this.getStackTrace(recording, errorStepIndex),
|
|
1518
|
+
severity: this.calculateOverallSeverity(contributingFactors),
|
|
1519
|
+
confidence: this.calculateConfidence(contributingFactors)
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
/**
|
|
1523
|
+
* Find contributing factors
|
|
1524
|
+
*/
|
|
1525
|
+
findContributingFactors(recording) {
|
|
1526
|
+
const factors = [];
|
|
1527
|
+
for (const pattern of this.patterns) {
|
|
1528
|
+
if (pattern.matcher(recording, recording.steps)) {
|
|
1529
|
+
factors.push({
|
|
1530
|
+
id: generateId("factor"),
|
|
1531
|
+
type: pattern.id,
|
|
1532
|
+
description: pattern.description,
|
|
1533
|
+
severity: pattern.severity,
|
|
1534
|
+
stepIndices: this.findPatternSteps(recording, pattern),
|
|
1535
|
+
evidence: this.gatherEvidence(recording, pattern)
|
|
1536
|
+
});
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
const errorSteps = recording.steps.filter((s) => s.error);
|
|
1540
|
+
for (const step of errorSteps) {
|
|
1541
|
+
factors.push({
|
|
1542
|
+
id: generateId("factor"),
|
|
1543
|
+
type: "explicit_error",
|
|
1544
|
+
description: step.error?.message ?? "Unknown error",
|
|
1545
|
+
severity: "high",
|
|
1546
|
+
stepIndices: [step.index],
|
|
1547
|
+
evidence: {
|
|
1548
|
+
errorName: step.error?.name,
|
|
1549
|
+
errorMessage: step.error?.message,
|
|
1550
|
+
errorStack: step.error?.stack
|
|
1551
|
+
}
|
|
1552
|
+
});
|
|
1553
|
+
}
|
|
1554
|
+
const severityOrder = { critical: 0, high: 1, medium: 2, low: 3 };
|
|
1555
|
+
factors.sort(
|
|
1556
|
+
(a, b) => severityOrder[a.severity] - severityOrder[b.severity]
|
|
1557
|
+
);
|
|
1558
|
+
return factors;
|
|
1559
|
+
}
|
|
1560
|
+
/**
|
|
1561
|
+
* Find steps related to a pattern
|
|
1562
|
+
*/
|
|
1563
|
+
findPatternSteps(recording, pattern) {
|
|
1564
|
+
const indices = [];
|
|
1565
|
+
switch (pattern.id) {
|
|
1566
|
+
case "repeated_tool_failure":
|
|
1567
|
+
for (const step of recording.steps) {
|
|
1568
|
+
if (step.type === "tool-result" && step.toolCall && !step.toolCall.success) {
|
|
1569
|
+
indices.push(step.index);
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
break;
|
|
1573
|
+
case "low_confidence_decision":
|
|
1574
|
+
for (const step of recording.steps) {
|
|
1575
|
+
if (step.type === "decision" && step.decision && step.decision.confidence < 0.5) {
|
|
1576
|
+
indices.push(step.index);
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
break;
|
|
1580
|
+
default:
|
|
1581
|
+
for (const step of recording.steps) {
|
|
1582
|
+
if (step.error) {
|
|
1583
|
+
indices.push(step.index);
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
return indices;
|
|
1588
|
+
}
|
|
1589
|
+
/**
|
|
1590
|
+
* Gather evidence for a pattern
|
|
1591
|
+
*/
|
|
1592
|
+
gatherEvidence(recording, pattern) {
|
|
1593
|
+
const evidence = {
|
|
1594
|
+
patternId: pattern.id,
|
|
1595
|
+
patternName: pattern.name
|
|
1596
|
+
};
|
|
1597
|
+
switch (pattern.id) {
|
|
1598
|
+
case "repeated_tool_failure": {
|
|
1599
|
+
const failures = recording.steps.filter(
|
|
1600
|
+
(s) => s.type === "tool-result" && s.toolCall && !s.toolCall.success
|
|
1601
|
+
);
|
|
1602
|
+
evidence.failedTools = failures.map((s) => s.toolCall?.name);
|
|
1603
|
+
evidence.failureCount = failures.length;
|
|
1604
|
+
break;
|
|
1605
|
+
}
|
|
1606
|
+
case "low_confidence_decision": {
|
|
1607
|
+
const lowConfidence = recording.steps.filter(
|
|
1608
|
+
(s) => s.type === "decision" && s.decision && s.decision.confidence < 0.5
|
|
1609
|
+
);
|
|
1610
|
+
evidence.decisions = lowConfidence.map((s) => ({
|
|
1611
|
+
step: s.index,
|
|
1612
|
+
confidence: s.decision?.confidence,
|
|
1613
|
+
reason: s.decision?.reason
|
|
1614
|
+
}));
|
|
1615
|
+
break;
|
|
1616
|
+
}
|
|
1617
|
+
case "timeout_likely":
|
|
1618
|
+
evidence.totalDuration = recording.durationMs;
|
|
1619
|
+
evidence.avgStepDuration = recording.durationMs / recording.steps.length;
|
|
1620
|
+
break;
|
|
1621
|
+
case "memory_overflow":
|
|
1622
|
+
evidence.initialMemorySize = recording.initialState.memory.size;
|
|
1623
|
+
evidence.finalMemorySize = recording.finalState?.memory?.size ?? 0;
|
|
1624
|
+
evidence.growth = (recording.finalState?.memory?.size ?? 0) / recording.initialState.memory.size;
|
|
1625
|
+
break;
|
|
1626
|
+
}
|
|
1627
|
+
return evidence;
|
|
1628
|
+
}
|
|
1629
|
+
/**
|
|
1630
|
+
* Determine the root cause
|
|
1631
|
+
*/
|
|
1632
|
+
determineRootCause(recording, factors) {
|
|
1633
|
+
const explicitError = factors.find((f) => f.type === "explicit_error");
|
|
1634
|
+
if (explicitError) {
|
|
1635
|
+
return explicitError.description;
|
|
1636
|
+
}
|
|
1637
|
+
if (factors.length > 0) {
|
|
1638
|
+
return factors[0].description;
|
|
1639
|
+
}
|
|
1640
|
+
if (recording.status === "failed") {
|
|
1641
|
+
return "Execution failed without clear error indication";
|
|
1642
|
+
}
|
|
1643
|
+
return "No failure detected";
|
|
1644
|
+
}
|
|
1645
|
+
/**
|
|
1646
|
+
* Generate recommendations
|
|
1647
|
+
*/
|
|
1648
|
+
generateRecommendations(factors) {
|
|
1649
|
+
const recommendations = [];
|
|
1650
|
+
const seenRecommendations = /* @__PURE__ */ new Set();
|
|
1651
|
+
for (const factor of factors) {
|
|
1652
|
+
const pattern = this.patterns.find((p) => p.id === factor.type);
|
|
1653
|
+
if (pattern) {
|
|
1654
|
+
for (const rec of pattern.recommendations) {
|
|
1655
|
+
if (!seenRecommendations.has(rec)) {
|
|
1656
|
+
seenRecommendations.add(rec);
|
|
1657
|
+
recommendations.push({
|
|
1658
|
+
id: generateId("rec"),
|
|
1659
|
+
priority: this.getPriorityFromSeverity(factor.severity),
|
|
1660
|
+
title: rec,
|
|
1661
|
+
description: `Based on pattern: ${pattern.name}`,
|
|
1662
|
+
relatedFactors: [factor.id]
|
|
1663
|
+
});
|
|
1664
|
+
}
|
|
1665
|
+
}
|
|
1666
|
+
} else if (factor.type === "explicit_error") {
|
|
1667
|
+
const defaultRecs = [
|
|
1668
|
+
"Review the error message and stack trace",
|
|
1669
|
+
"Add error handling and recovery logic",
|
|
1670
|
+
"Validate inputs before operations"
|
|
1671
|
+
];
|
|
1672
|
+
for (const rec of defaultRecs) {
|
|
1673
|
+
if (!seenRecommendations.has(rec)) {
|
|
1674
|
+
seenRecommendations.add(rec);
|
|
1675
|
+
recommendations.push({
|
|
1676
|
+
id: generateId("rec"),
|
|
1677
|
+
priority: this.getPriorityFromSeverity(factor.severity),
|
|
1678
|
+
title: rec,
|
|
1679
|
+
description: `Based on explicit error: ${factor.description}`,
|
|
1680
|
+
relatedFactors: [factor.id]
|
|
1681
|
+
});
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
recommendations.sort((a, b) => a.priority - b.priority);
|
|
1687
|
+
return recommendations;
|
|
1688
|
+
}
|
|
1689
|
+
/**
|
|
1690
|
+
* Get priority from severity
|
|
1691
|
+
*/
|
|
1692
|
+
getPriorityFromSeverity(severity) {
|
|
1693
|
+
switch (severity) {
|
|
1694
|
+
case "critical":
|
|
1695
|
+
return 1;
|
|
1696
|
+
case "high":
|
|
1697
|
+
return 2;
|
|
1698
|
+
case "medium":
|
|
1699
|
+
return 3;
|
|
1700
|
+
case "low":
|
|
1701
|
+
return 4;
|
|
1702
|
+
default:
|
|
1703
|
+
return 5;
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
/**
|
|
1707
|
+
* Find the error step
|
|
1708
|
+
*/
|
|
1709
|
+
findErrorStep(recording) {
|
|
1710
|
+
for (const step of recording.steps) {
|
|
1711
|
+
if (step.error) {
|
|
1712
|
+
return step.index;
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
for (let i = recording.steps.length - 1; i >= 0; i--) {
|
|
1716
|
+
const step = recording.steps[i];
|
|
1717
|
+
if (step.type === "tool-result" && step.toolCall && !step.toolCall.success) {
|
|
1718
|
+
return step.index;
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
return void 0;
|
|
1722
|
+
}
|
|
1723
|
+
/**
|
|
1724
|
+
* Get error message
|
|
1725
|
+
*/
|
|
1726
|
+
getErrorMessage(recording, stepIndex) {
|
|
1727
|
+
if (stepIndex === void 0) {
|
|
1728
|
+
return void 0;
|
|
1729
|
+
}
|
|
1730
|
+
const step = recording.steps[stepIndex];
|
|
1731
|
+
return step?.error?.message ?? step?.toolCall?.result;
|
|
1732
|
+
}
|
|
1733
|
+
/**
|
|
1734
|
+
* Get stack trace
|
|
1735
|
+
*/
|
|
1736
|
+
getStackTrace(recording, stepIndex) {
|
|
1737
|
+
if (stepIndex === void 0) {
|
|
1738
|
+
return void 0;
|
|
1739
|
+
}
|
|
1740
|
+
const step = recording.steps[stepIndex];
|
|
1741
|
+
return step?.error?.stack;
|
|
1742
|
+
}
|
|
1743
|
+
/**
|
|
1744
|
+
* Calculate overall severity
|
|
1745
|
+
*/
|
|
1746
|
+
calculateOverallSeverity(factors) {
|
|
1747
|
+
if (factors.length === 0) {
|
|
1748
|
+
return "low";
|
|
1749
|
+
}
|
|
1750
|
+
const severities = factors.map((f) => f.severity);
|
|
1751
|
+
if (severities.includes("critical")) return "critical";
|
|
1752
|
+
if (severities.includes("high")) return "high";
|
|
1753
|
+
if (severities.includes("medium")) return "medium";
|
|
1754
|
+
return "low";
|
|
1755
|
+
}
|
|
1756
|
+
/**
|
|
1757
|
+
* Calculate analysis confidence
|
|
1758
|
+
*/
|
|
1759
|
+
calculateConfidence(factors) {
|
|
1760
|
+
if (factors.length === 0) {
|
|
1761
|
+
return 0.3;
|
|
1762
|
+
}
|
|
1763
|
+
let confidenceSum = 0;
|
|
1764
|
+
for (const factor of factors) {
|
|
1765
|
+
let factorConfidence = 0.5;
|
|
1766
|
+
if (factor.type === "explicit_error") {
|
|
1767
|
+
factorConfidence = 0.95;
|
|
1768
|
+
} else if (Object.keys(factor.evidence ?? {}).length > 2) {
|
|
1769
|
+
factorConfidence = 0.8;
|
|
1770
|
+
}
|
|
1771
|
+
confidenceSum += factorConfidence;
|
|
1772
|
+
}
|
|
1773
|
+
return Math.min(confidenceSum / factors.length, 0.99);
|
|
1774
|
+
}
|
|
1775
|
+
/**
|
|
1776
|
+
* Analyze individual steps
|
|
1777
|
+
*/
|
|
1778
|
+
analyzeSteps(steps) {
|
|
1779
|
+
const analyses = [];
|
|
1780
|
+
for (const step of steps) {
|
|
1781
|
+
const analysis = {
|
|
1782
|
+
stepIndex: step.index,
|
|
1783
|
+
type: step.type,
|
|
1784
|
+
suspicious: false,
|
|
1785
|
+
reasons: [],
|
|
1786
|
+
relatedSteps: []
|
|
1787
|
+
};
|
|
1788
|
+
if (step.error) {
|
|
1789
|
+
analysis.suspicious = true;
|
|
1790
|
+
analysis.reasons.push(`Error: ${step.error.message}`);
|
|
1791
|
+
}
|
|
1792
|
+
if (step.type === "tool-result" && step.toolCall && !step.toolCall.success) {
|
|
1793
|
+
analysis.suspicious = true;
|
|
1794
|
+
analysis.reasons.push(`Tool ${step.toolCall.name} failed`);
|
|
1795
|
+
}
|
|
1796
|
+
if (step.type === "decision" && step.decision && step.decision.confidence < 0.5) {
|
|
1797
|
+
analysis.suspicious = true;
|
|
1798
|
+
analysis.reasons.push(
|
|
1799
|
+
`Low confidence decision (${step.decision.confidence})`
|
|
1800
|
+
);
|
|
1801
|
+
}
|
|
1802
|
+
if (step.durationMs && step.durationMs > 3e4) {
|
|
1803
|
+
analysis.suspicious = true;
|
|
1804
|
+
analysis.reasons.push(`Long duration (${step.durationMs}ms)`);
|
|
1805
|
+
}
|
|
1806
|
+
analyses.push(analysis);
|
|
1807
|
+
}
|
|
1808
|
+
return analyses;
|
|
1809
|
+
}
|
|
1810
|
+
/**
|
|
1811
|
+
* Add a custom pattern
|
|
1812
|
+
*/
|
|
1813
|
+
addPattern(pattern) {
|
|
1814
|
+
this.patterns.push(pattern);
|
|
1815
|
+
}
|
|
1816
|
+
/**
|
|
1817
|
+
* Remove a pattern
|
|
1818
|
+
*/
|
|
1819
|
+
removePattern(patternId) {
|
|
1820
|
+
const index = this.patterns.findIndex((p) => p.id === patternId);
|
|
1821
|
+
if (index >= 0) {
|
|
1822
|
+
this.patterns.splice(index, 1);
|
|
1823
|
+
return true;
|
|
1824
|
+
}
|
|
1825
|
+
return false;
|
|
1826
|
+
}
|
|
1827
|
+
/**
|
|
1828
|
+
* Get all patterns
|
|
1829
|
+
*/
|
|
1830
|
+
getPatterns() {
|
|
1831
|
+
return [...this.patterns];
|
|
1832
|
+
}
|
|
1833
|
+
};
|
|
1834
|
+
function createFailureAnalyzer(options) {
|
|
1835
|
+
return new FailureAnalyzer(options);
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
export { FailureAnalyzer, WhatIfEngine, createFailureAnalyzer, createWhatIfEngine };
|
|
1839
|
+
//# sourceMappingURL=index.js.map
|
|
1840
|
+
//# sourceMappingURL=index.js.map
|