@elizaos/plugin-trajectory-logger 2.0.0-alpha.12 → 2.0.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/node/index.node.js +140 -1225
- package/dist/node/index.node.js.map +5 -5
- package/package.json +11 -14
package/dist/node/index.node.js
CHANGED
|
@@ -1,1266 +1,191 @@
|
|
|
1
|
-
import { createRequire } from "node:module";
|
|
2
|
-
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
3
|
-
|
|
4
1
|
// index.ts
|
|
5
|
-
import {
|
|
6
|
-
createUniqueUuid
|
|
7
|
-
} from "@elizaos/core";
|
|
8
|
-
import crypto from "node:crypto";
|
|
2
|
+
import { createUniqueUuid } from "@elizaos/core";
|
|
9
3
|
|
|
10
4
|
// TrajectoryLoggerService.ts
|
|
11
|
-
import {
|
|
5
|
+
import { asUUID, logger } from "@elizaos/core";
|
|
12
6
|
import { v4 as uuidv4 } from "uuid";
|
|
13
|
-
function asNumber(value) {
|
|
14
|
-
if (typeof value === "number" && Number.isFinite(value))
|
|
15
|
-
return value;
|
|
16
|
-
if (typeof value === "string") {
|
|
17
|
-
const parsed = Number(value);
|
|
18
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
19
|
-
}
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
function asString(value) {
|
|
23
|
-
if (typeof value === "string")
|
|
24
|
-
return value;
|
|
25
|
-
if (typeof value === "number" || typeof value === "boolean")
|
|
26
|
-
return String(value);
|
|
27
|
-
if (value instanceof Date)
|
|
28
|
-
return value.toISOString();
|
|
29
|
-
return null;
|
|
30
|
-
}
|
|
31
|
-
function asIsoString(value) {
|
|
32
|
-
if (value instanceof Date)
|
|
33
|
-
return value.toISOString();
|
|
34
|
-
const asText = asString(value);
|
|
35
|
-
if (!asText)
|
|
36
|
-
return new Date(0).toISOString();
|
|
37
|
-
const parsed = new Date(asText);
|
|
38
|
-
if (Number.isNaN(parsed.getTime()))
|
|
39
|
-
return new Date(0).toISOString();
|
|
40
|
-
return parsed.toISOString();
|
|
41
|
-
}
|
|
42
|
-
function pickCell(row, ...keys) {
|
|
43
|
-
for (const key of keys) {
|
|
44
|
-
if (Object.hasOwn(row, key)) {
|
|
45
|
-
return row[key];
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
return;
|
|
49
|
-
}
|
|
50
|
-
function sqlLiteral(v) {
|
|
51
|
-
if (v === null || v === undefined)
|
|
52
|
-
return "NULL";
|
|
53
|
-
if (typeof v === "number")
|
|
54
|
-
return String(v);
|
|
55
|
-
if (typeof v === "boolean")
|
|
56
|
-
return v ? "TRUE" : "FALSE";
|
|
57
|
-
if (typeof v === "object")
|
|
58
|
-
return `'${JSON.stringify(v).replace(/'/g, "''")}'`;
|
|
59
|
-
return `'${String(v).replace(/'/g, "''")}'`;
|
|
60
|
-
}
|
|
61
7
|
|
|
62
|
-
class TrajectoryLoggerService
|
|
63
|
-
|
|
64
|
-
get serviceType() {
|
|
65
|
-
return TrajectoryLoggerService.serviceType;
|
|
66
|
-
}
|
|
67
|
-
capabilityDescription = "Captures and persists LLM calls, provider accesses, and full trajectories for debugging, analysis, and RL training";
|
|
68
|
-
enabled = true;
|
|
69
|
-
initialized = false;
|
|
8
|
+
class TrajectoryLoggerService {
|
|
9
|
+
activeTrajectories = new Map;
|
|
70
10
|
activeStepIds = new Map;
|
|
71
|
-
|
|
72
|
-
writeQueues = new Map;
|
|
73
|
-
static async start(runtime) {
|
|
74
|
-
const service = new this(runtime);
|
|
75
|
-
await service.initialize();
|
|
76
|
-
return service;
|
|
77
|
-
}
|
|
78
|
-
async stop() {
|
|
79
|
-
this.enabled = false;
|
|
80
|
-
}
|
|
81
|
-
setEnabled(enabled) {
|
|
82
|
-
this.enabled = enabled;
|
|
83
|
-
}
|
|
84
|
-
isEnabled() {
|
|
85
|
-
return this.enabled;
|
|
86
|
-
}
|
|
87
|
-
async getSqlHelper() {
|
|
88
|
-
const drizzle = await import("drizzle-orm");
|
|
89
|
-
return drizzle.sql;
|
|
90
|
-
}
|
|
91
|
-
async executeRawSql(sqlText) {
|
|
92
|
-
const runtime = this.runtime;
|
|
93
|
-
if (!runtime?.adapter) {
|
|
94
|
-
throw new Error("Database adapter not available");
|
|
95
|
-
}
|
|
96
|
-
const sqlHelper = await this.getSqlHelper();
|
|
97
|
-
const db = runtime.adapter.db;
|
|
98
|
-
const query = sqlHelper.raw(sqlText);
|
|
99
|
-
const result = await db.execute(query);
|
|
100
|
-
const rows = Array.isArray(result.rows) ? result.rows : [];
|
|
101
|
-
const columns = result.fields && Array.isArray(result.fields) ? result.fields.map((field) => field.name) : rows.length > 0 ? Object.keys(rows[0]) : [];
|
|
102
|
-
return { rows, columns };
|
|
103
|
-
}
|
|
104
|
-
async initialize() {
|
|
105
|
-
if (this.initialized)
|
|
106
|
-
return;
|
|
107
|
-
const runtime = this.runtime;
|
|
108
|
-
if (!runtime?.adapter) {
|
|
109
|
-
logger.warn("[trajectory-logger] No runtime adapter available, skipping initialization");
|
|
110
|
-
return;
|
|
111
|
-
}
|
|
112
|
-
await this.ensureTablesExist();
|
|
113
|
-
await this.backfillTrajectoriesMissingLlmCalls();
|
|
114
|
-
this.initialized = true;
|
|
115
|
-
logger.info("[trajectory-logger] Trajectory logger service initialized");
|
|
116
|
-
}
|
|
117
|
-
async getTableColumnNames(tableName) {
|
|
118
|
-
const names = new Set;
|
|
119
|
-
try {
|
|
120
|
-
const result = await this.executeRawSql(`
|
|
121
|
-
SELECT column_name
|
|
122
|
-
FROM information_schema.columns
|
|
123
|
-
WHERE table_name = ${sqlLiteral(tableName)}
|
|
124
|
-
AND table_schema NOT IN ('pg_catalog', 'information_schema')
|
|
125
|
-
`);
|
|
126
|
-
for (const row of result.rows) {
|
|
127
|
-
const name = asString(pickCell(row, "column_name"));
|
|
128
|
-
if (name)
|
|
129
|
-
names.add(name);
|
|
130
|
-
}
|
|
131
|
-
if (names.size > 0)
|
|
132
|
-
return names;
|
|
133
|
-
} catch {}
|
|
134
|
-
const safeTableName = tableName.replace(/[^a-zA-Z0-9_]/g, "");
|
|
135
|
-
if (!safeTableName)
|
|
136
|
-
return names;
|
|
137
|
-
try {
|
|
138
|
-
const pragma = await this.executeRawSql(`PRAGMA table_info(${safeTableName})`);
|
|
139
|
-
for (const row of pragma.rows) {
|
|
140
|
-
const name = asString(pickCell(row, "name"));
|
|
141
|
-
if (name)
|
|
142
|
-
names.add(name);
|
|
143
|
-
}
|
|
144
|
-
} catch {}
|
|
145
|
-
return names;
|
|
146
|
-
}
|
|
147
|
-
async ensureTrajectoryColumnsExist() {
|
|
148
|
-
const columns = await this.getTableColumnNames("trajectories");
|
|
149
|
-
const requiredColumns = [
|
|
150
|
-
["scenario_id", "TEXT"],
|
|
151
|
-
["episode_id", "TEXT"],
|
|
152
|
-
["batch_id", "TEXT"],
|
|
153
|
-
["group_index", "INTEGER"],
|
|
154
|
-
["steps_json", "JSONB NOT NULL DEFAULT '[]'"],
|
|
155
|
-
["reward_components_json", "JSONB NOT NULL DEFAULT '{}'"],
|
|
156
|
-
["metrics_json", "JSONB NOT NULL DEFAULT '{}'"],
|
|
157
|
-
["metadata_json", "JSONB NOT NULL DEFAULT '{}'"],
|
|
158
|
-
["is_training_data", "BOOLEAN NOT NULL DEFAULT FALSE"],
|
|
159
|
-
["is_evaluation", "BOOLEAN NOT NULL DEFAULT FALSE"],
|
|
160
|
-
["used_in_training", "BOOLEAN NOT NULL DEFAULT FALSE"],
|
|
161
|
-
["judged_at", "TIMESTAMPTZ"]
|
|
162
|
-
];
|
|
163
|
-
for (const [columnName, definition] of requiredColumns) {
|
|
164
|
-
if (columns.has(columnName))
|
|
165
|
-
continue;
|
|
166
|
-
try {
|
|
167
|
-
await this.executeRawSql(`
|
|
168
|
-
ALTER TABLE trajectories
|
|
169
|
-
ADD COLUMN IF NOT EXISTS ${columnName} ${definition}
|
|
170
|
-
`);
|
|
171
|
-
} catch (errWithIfExists) {
|
|
172
|
-
try {
|
|
173
|
-
await this.executeRawSql(`
|
|
174
|
-
ALTER TABLE trajectories
|
|
175
|
-
ADD COLUMN ${columnName} ${definition}
|
|
176
|
-
`);
|
|
177
|
-
} catch (errPlain) {
|
|
178
|
-
const combinedMessage = `${errWithIfExists instanceof Error ? errWithIfExists.message : String(errWithIfExists)} | ${errPlain instanceof Error ? errPlain.message : String(errPlain)}`.toLowerCase();
|
|
179
|
-
if (combinedMessage.includes("already exists") || combinedMessage.includes("duplicate column") || combinedMessage.includes("duplicate_column")) {
|
|
180
|
-
continue;
|
|
181
|
-
}
|
|
182
|
-
logger.warn(`[trajectory-logger] Failed to add trajectories.${columnName} (non-fatal): ${errPlain instanceof Error ? errPlain.message : String(errPlain)}`);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
for (const statement of [
|
|
187
|
-
`ALTER TABLE trajectories
|
|
188
|
-
ALTER COLUMN start_time TYPE BIGINT USING start_time::BIGINT`,
|
|
189
|
-
`ALTER TABLE trajectories
|
|
190
|
-
ALTER COLUMN end_time TYPE BIGINT USING end_time::BIGINT`,
|
|
191
|
-
`ALTER TABLE trajectories
|
|
192
|
-
ALTER COLUMN duration_ms TYPE BIGINT USING duration_ms::BIGINT`
|
|
193
|
-
]) {
|
|
194
|
-
try {
|
|
195
|
-
await this.executeRawSql(statement);
|
|
196
|
-
} catch {}
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
async ensureTablesExist() {
|
|
200
|
-
await this.executeRawSql(`
|
|
201
|
-
CREATE TABLE IF NOT EXISTS trajectories (
|
|
202
|
-
id TEXT PRIMARY KEY,
|
|
203
|
-
agent_id TEXT NOT NULL,
|
|
204
|
-
source TEXT NOT NULL DEFAULT 'chat',
|
|
205
|
-
status TEXT NOT NULL DEFAULT 'active',
|
|
206
|
-
start_time BIGINT NOT NULL,
|
|
207
|
-
end_time BIGINT,
|
|
208
|
-
duration_ms BIGINT,
|
|
209
|
-
step_count INTEGER NOT NULL DEFAULT 0,
|
|
210
|
-
llm_call_count INTEGER NOT NULL DEFAULT 0,
|
|
211
|
-
provider_access_count INTEGER NOT NULL DEFAULT 0,
|
|
212
|
-
total_prompt_tokens INTEGER NOT NULL DEFAULT 0,
|
|
213
|
-
total_completion_tokens INTEGER NOT NULL DEFAULT 0,
|
|
214
|
-
total_reward REAL NOT NULL DEFAULT 0,
|
|
215
|
-
scenario_id TEXT,
|
|
216
|
-
episode_id TEXT,
|
|
217
|
-
batch_id TEXT,
|
|
218
|
-
group_index INTEGER,
|
|
219
|
-
steps_json JSONB NOT NULL DEFAULT '[]',
|
|
220
|
-
reward_components_json JSONB NOT NULL DEFAULT '{}',
|
|
221
|
-
metrics_json JSONB NOT NULL DEFAULT '{}',
|
|
222
|
-
metadata_json JSONB NOT NULL DEFAULT '{}',
|
|
223
|
-
is_training_data BOOLEAN NOT NULL DEFAULT FALSE,
|
|
224
|
-
is_evaluation BOOLEAN NOT NULL DEFAULT FALSE,
|
|
225
|
-
used_in_training BOOLEAN NOT NULL DEFAULT FALSE,
|
|
226
|
-
ai_judge_reward REAL,
|
|
227
|
-
ai_judge_reasoning TEXT,
|
|
228
|
-
judged_at TIMESTAMPTZ,
|
|
229
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
230
|
-
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
231
|
-
)
|
|
232
|
-
`);
|
|
233
|
-
await this.ensureTrajectoryColumnsExist();
|
|
234
|
-
try {
|
|
235
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectories_agent_id ON trajectories(agent_id)`);
|
|
236
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectories_source ON trajectories(source)`);
|
|
237
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectories_status ON trajectories(status)`);
|
|
238
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectories_created_at ON trajectories(created_at)`);
|
|
239
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectories_scenario_id ON trajectories(scenario_id)`);
|
|
240
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectories_batch_id ON trajectories(batch_id)`);
|
|
241
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectories_is_training ON trajectories(is_training_data)`);
|
|
242
|
-
} catch (e) {
|
|
243
|
-
logger.warn(`[trajectory-logger] Failed to create indexes (non-fatal): ${e instanceof Error ? e.message : String(e)}`);
|
|
244
|
-
}
|
|
245
|
-
await this.executeRawSql(`
|
|
246
|
-
CREATE TABLE IF NOT EXISTS trajectory_step_index (
|
|
247
|
-
step_id TEXT PRIMARY KEY,
|
|
248
|
-
trajectory_id TEXT NOT NULL REFERENCES trajectories(id) ON DELETE CASCADE,
|
|
249
|
-
step_number INTEGER NOT NULL DEFAULT 0,
|
|
250
|
-
is_active BOOLEAN NOT NULL DEFAULT FALSE,
|
|
251
|
-
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
252
|
-
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
253
|
-
)
|
|
254
|
-
`);
|
|
255
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectory_step_index_trajectory_id ON trajectory_step_index(trajectory_id)`);
|
|
256
|
-
await this.executeRawSql(`CREATE INDEX IF NOT EXISTS idx_trajectory_step_index_is_active ON trajectory_step_index(is_active)`);
|
|
257
|
-
}
|
|
258
|
-
normalizeTrajectoryStatus(value) {
|
|
259
|
-
switch (value) {
|
|
260
|
-
case "active":
|
|
261
|
-
case "completed":
|
|
262
|
-
case "error":
|
|
263
|
-
case "timeout":
|
|
264
|
-
case "terminated":
|
|
265
|
-
return value;
|
|
266
|
-
default:
|
|
267
|
-
return "completed";
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
async backfillTrajectoriesMissingLlmCalls() {
|
|
271
|
-
const maxRows = 2000;
|
|
272
|
-
let fixed = 0;
|
|
273
|
-
try {
|
|
274
|
-
const rows = await this.executeRawSql(`
|
|
275
|
-
SELECT id, status
|
|
276
|
-
FROM trajectories
|
|
277
|
-
WHERE COALESCE(llm_call_count, 0) = 0
|
|
278
|
-
AND status <> 'active'
|
|
279
|
-
ORDER BY created_at ASC
|
|
280
|
-
LIMIT ${maxRows}
|
|
281
|
-
`);
|
|
282
|
-
for (const row of rows.rows) {
|
|
283
|
-
const trajectoryId = asString(pickCell(row, "id"));
|
|
284
|
-
if (!trajectoryId)
|
|
285
|
-
continue;
|
|
286
|
-
const status = this.normalizeTrajectoryStatus(asString(pickCell(row, "status")));
|
|
287
|
-
await this.withTrajectoryWriteLock(trajectoryId, async () => {
|
|
288
|
-
const trajectory = await this.getTrajectoryById(trajectoryId);
|
|
289
|
-
if (!trajectory)
|
|
290
|
-
return;
|
|
291
|
-
const inserted = this.ensureAtLeastOneLlmCall(trajectory, "backfill");
|
|
292
|
-
if (!inserted)
|
|
293
|
-
return;
|
|
294
|
-
await this.persistTrajectory(trajectoryId, trajectory, status);
|
|
295
|
-
fixed += 1;
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
if (fixed > 0) {
|
|
299
|
-
logger.info(`[trajectory-logger] Backfilled ${fixed} completed trajectories with synthetic LLM calls`);
|
|
300
|
-
}
|
|
301
|
-
if (rows.rows.length >= maxRows) {
|
|
302
|
-
logger.warn(`[trajectory-logger] Backfill hit safety cap (${maxRows}); remaining older trajectories will be fixed lazily when touched`);
|
|
303
|
-
}
|
|
304
|
-
} catch (err) {
|
|
305
|
-
logger.warn(`[trajectory-logger] Failed to backfill trajectories missing LLM calls (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
normalizePurpose(value) {
|
|
309
|
-
switch (value) {
|
|
310
|
-
case "action":
|
|
311
|
-
case "reasoning":
|
|
312
|
-
case "evaluation":
|
|
313
|
-
case "response":
|
|
314
|
-
case "other":
|
|
315
|
-
return value;
|
|
316
|
-
default:
|
|
317
|
-
return "other";
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
defaultEnvironmentState(timestamp = Date.now()) {
|
|
321
|
-
return {
|
|
322
|
-
timestamp,
|
|
323
|
-
agentBalance: 0,
|
|
324
|
-
agentPoints: 0,
|
|
325
|
-
agentPnL: 0,
|
|
326
|
-
openPositions: 0
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
createPendingAction(stepTimestamp) {
|
|
330
|
-
return {
|
|
331
|
-
attemptId: "",
|
|
332
|
-
timestamp: stepTimestamp,
|
|
333
|
-
actionType: "pending",
|
|
334
|
-
actionName: "pending",
|
|
335
|
-
parameters: {},
|
|
336
|
-
success: false
|
|
337
|
-
};
|
|
338
|
-
}
|
|
339
|
-
createStep(stepId, stepNumber, envState) {
|
|
340
|
-
const timestamp = envState.timestamp || Date.now();
|
|
341
|
-
return {
|
|
342
|
-
stepId,
|
|
343
|
-
stepNumber,
|
|
344
|
-
timestamp,
|
|
345
|
-
environmentState: envState,
|
|
346
|
-
observation: {},
|
|
347
|
-
llmCalls: [],
|
|
348
|
-
providerAccesses: [],
|
|
349
|
-
action: this.createPendingAction(timestamp),
|
|
350
|
-
reward: 0,
|
|
351
|
-
done: false
|
|
352
|
-
};
|
|
353
|
-
}
|
|
354
|
-
ensureAtLeastOneLlmCall(trajectory, source) {
|
|
355
|
-
if (this.computeTotals(trajectory.steps).llmCallCount > 0) {
|
|
356
|
-
return false;
|
|
357
|
-
}
|
|
358
|
-
const timestamp = Date.now();
|
|
359
|
-
let step = trajectory.steps[trajectory.steps.length - 1];
|
|
360
|
-
if (!step) {
|
|
361
|
-
step = this.createStep(uuidv4(), 0, this.defaultEnvironmentState(timestamp));
|
|
362
|
-
step.done = true;
|
|
363
|
-
trajectory.steps.push(step);
|
|
364
|
-
}
|
|
365
|
-
if (!Array.isArray(step.llmCalls)) {
|
|
366
|
-
step.llmCalls = [];
|
|
367
|
-
}
|
|
368
|
-
const syntheticCall = {
|
|
369
|
-
callId: uuidv4(),
|
|
370
|
-
timestamp,
|
|
371
|
-
model: "milady/synthetic-trajectory-fallback",
|
|
372
|
-
systemPrompt: "[synthetic] inserted by trajectory logger because no LLM calls were captured",
|
|
373
|
-
userPrompt: "[synthetic] this trajectory completed without recorded model activity",
|
|
374
|
-
response: "[synthetic] placeholder call inserted to enforce minimum llm_call_count=1",
|
|
375
|
-
temperature: 0,
|
|
376
|
-
maxTokens: 0,
|
|
377
|
-
promptTokens: 0,
|
|
378
|
-
completionTokens: 0,
|
|
379
|
-
latencyMs: 0,
|
|
380
|
-
purpose: "other",
|
|
381
|
-
actionType: "TRAJECTORY_FALLBACK"
|
|
382
|
-
};
|
|
383
|
-
step.llmCalls.push(syntheticCall);
|
|
384
|
-
if (!step.metadata)
|
|
385
|
-
step.metadata = {};
|
|
386
|
-
step.metadata.syntheticLlmCall = true;
|
|
387
|
-
step.metadata.syntheticLlmCallSource = source;
|
|
388
|
-
trajectory.metadata.syntheticLlmCall = true;
|
|
389
|
-
trajectory.metadata.syntheticLlmCallSource = source;
|
|
390
|
-
const existingFallbackCount = trajectory.metrics.syntheticLlmFallbackCount;
|
|
391
|
-
trajectory.metrics.syntheticLlmFallbackCount = typeof existingFallbackCount === "number" && Number.isFinite(existingFallbackCount) ? existingFallbackCount + 1 : 1;
|
|
392
|
-
return true;
|
|
393
|
-
}
|
|
394
|
-
computeTotals(steps) {
|
|
395
|
-
let llmCallCount = 0;
|
|
396
|
-
let providerAccessCount = 0;
|
|
397
|
-
let totalPromptTokens = 0;
|
|
398
|
-
let totalCompletionTokens = 0;
|
|
399
|
-
for (const step of steps) {
|
|
400
|
-
const llmCalls = Array.isArray(step.llmCalls) ? step.llmCalls : [];
|
|
401
|
-
const providerAccesses = Array.isArray(step.providerAccesses) ? step.providerAccesses : [];
|
|
402
|
-
llmCallCount += llmCalls.length;
|
|
403
|
-
providerAccessCount += providerAccesses.length;
|
|
404
|
-
for (const call of llmCalls) {
|
|
405
|
-
totalPromptTokens += call.promptTokens ?? 0;
|
|
406
|
-
totalCompletionTokens += call.completionTokens ?? 0;
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
return {
|
|
410
|
-
stepCount: steps.length,
|
|
411
|
-
llmCallCount,
|
|
412
|
-
providerAccessCount,
|
|
413
|
-
totalPromptTokens,
|
|
414
|
-
totalCompletionTokens
|
|
415
|
-
};
|
|
416
|
-
}
|
|
417
|
-
async withTrajectoryWriteLock(trajectoryId, task) {
|
|
418
|
-
const previous = this.writeQueues.get(trajectoryId) ?? Promise.resolve();
|
|
419
|
-
const next = previous.catch(() => {}).then(task);
|
|
420
|
-
this.writeQueues.set(trajectoryId, next);
|
|
421
|
-
try {
|
|
422
|
-
await next;
|
|
423
|
-
} finally {
|
|
424
|
-
if (this.writeQueues.get(trajectoryId) === next) {
|
|
425
|
-
this.writeQueues.delete(trajectoryId);
|
|
426
|
-
}
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
async getTrajectoryById(trajectoryId) {
|
|
430
|
-
const result = await this.executeRawSql(`SELECT * FROM trajectories WHERE id = ${sqlLiteral(trajectoryId)} LIMIT 1`);
|
|
431
|
-
if (result.rows.length === 0)
|
|
432
|
-
return null;
|
|
433
|
-
return this.rowToTrajectory(result.rows[0]);
|
|
434
|
-
}
|
|
435
|
-
async getStepIndex(stepId) {
|
|
436
|
-
const result = await this.executeRawSql(`SELECT trajectory_id, step_number, is_active FROM trajectory_step_index WHERE step_id = ${sqlLiteral(stepId)} LIMIT 1`);
|
|
437
|
-
const row = result.rows[0];
|
|
438
|
-
if (!row)
|
|
439
|
-
return null;
|
|
440
|
-
const trajectoryId = asString(pickCell(row, "trajectory_id"));
|
|
441
|
-
if (!trajectoryId)
|
|
442
|
-
return null;
|
|
443
|
-
const stepNumber = asNumber(pickCell(row, "step_number")) ?? 0;
|
|
444
|
-
const isActiveText = asString(pickCell(row, "is_active"));
|
|
445
|
-
const isActive = isActiveText === "true" || isActiveText === "t" || pickCell(row, "is_active") === true;
|
|
446
|
-
return { trajectoryId, stepNumber, isActive };
|
|
447
|
-
}
|
|
448
|
-
async setStepIndex(stepId, trajectoryId, stepNumber, isActive) {
|
|
449
|
-
await this.executeRawSql(`
|
|
450
|
-
INSERT INTO trajectory_step_index (
|
|
451
|
-
step_id, trajectory_id, step_number, is_active, updated_at
|
|
452
|
-
) VALUES (
|
|
453
|
-
${sqlLiteral(stepId)},
|
|
454
|
-
${sqlLiteral(trajectoryId)},
|
|
455
|
-
${stepNumber},
|
|
456
|
-
${isActive ? "TRUE" : "FALSE"},
|
|
457
|
-
CURRENT_TIMESTAMP
|
|
458
|
-
)
|
|
459
|
-
ON CONFLICT (step_id) DO UPDATE SET
|
|
460
|
-
trajectory_id = EXCLUDED.trajectory_id,
|
|
461
|
-
step_number = EXCLUDED.step_number,
|
|
462
|
-
is_active = EXCLUDED.is_active,
|
|
463
|
-
updated_at = CURRENT_TIMESTAMP
|
|
464
|
-
`);
|
|
465
|
-
}
|
|
466
|
-
async markAllStepsInactive(trajectoryId) {
|
|
467
|
-
await this.executeRawSql(`
|
|
468
|
-
UPDATE trajectory_step_index
|
|
469
|
-
SET is_active = FALSE, updated_at = CURRENT_TIMESTAMP
|
|
470
|
-
WHERE trajectory_id = ${sqlLiteral(trajectoryId)}
|
|
471
|
-
`);
|
|
472
|
-
}
|
|
473
|
-
async resolveTrajectoryId(stepIdOrTrajectoryId) {
|
|
474
|
-
const cached = this.stepToTrajectory.get(stepIdOrTrajectoryId);
|
|
475
|
-
if (cached)
|
|
476
|
-
return cached;
|
|
477
|
-
const byStep = await this.getStepIndex(stepIdOrTrajectoryId);
|
|
478
|
-
if (byStep?.trajectoryId) {
|
|
479
|
-
this.stepToTrajectory.set(stepIdOrTrajectoryId, byStep.trajectoryId);
|
|
480
|
-
return byStep.trajectoryId;
|
|
481
|
-
}
|
|
482
|
-
const byId = await this.executeRawSql(`SELECT id FROM trajectories WHERE id = ${sqlLiteral(stepIdOrTrajectoryId)} LIMIT 1`);
|
|
483
|
-
const row = byId.rows[0];
|
|
484
|
-
const id = row ? asString(pickCell(row, "id")) : null;
|
|
485
|
-
return id;
|
|
486
|
-
}
|
|
487
|
-
async getCurrentStepIdFromDb(trajectoryId) {
|
|
488
|
-
const result = await this.executeRawSql(`
|
|
489
|
-
SELECT step_id
|
|
490
|
-
FROM trajectory_step_index
|
|
491
|
-
WHERE trajectory_id = ${sqlLiteral(trajectoryId)} AND is_active = TRUE
|
|
492
|
-
ORDER BY step_number DESC, updated_at DESC
|
|
493
|
-
LIMIT 1
|
|
494
|
-
`);
|
|
495
|
-
const row = result.rows[0];
|
|
496
|
-
return row ? asString(pickCell(row, "step_id")) : null;
|
|
497
|
-
}
|
|
498
|
-
async persistTrajectory(trajectoryId, trajectory, status = "active") {
|
|
499
|
-
if (status !== "active") {
|
|
500
|
-
this.ensureAtLeastOneLlmCall(trajectory, "finalize");
|
|
501
|
-
}
|
|
502
|
-
const totals = this.computeTotals(trajectory.steps);
|
|
503
|
-
const isFinalStatus = status !== "active";
|
|
504
|
-
const persistedEndTime = isFinalStatus ? trajectory.endTime : null;
|
|
505
|
-
const persistedDuration = isFinalStatus ? trajectory.durationMs : null;
|
|
506
|
-
const updatedAtIso = new Date().toISOString();
|
|
507
|
-
try {
|
|
508
|
-
await this.executeRawSql(`
|
|
509
|
-
UPDATE trajectories SET
|
|
510
|
-
status = ${sqlLiteral(status)},
|
|
511
|
-
end_time = ${sqlLiteral(persistedEndTime)},
|
|
512
|
-
duration_ms = ${sqlLiteral(persistedDuration)},
|
|
513
|
-
step_count = ${totals.stepCount},
|
|
514
|
-
llm_call_count = ${totals.llmCallCount},
|
|
515
|
-
provider_access_count = ${totals.providerAccessCount},
|
|
516
|
-
total_prompt_tokens = ${totals.totalPromptTokens},
|
|
517
|
-
total_completion_tokens = ${totals.totalCompletionTokens},
|
|
518
|
-
total_reward = ${trajectory.totalReward},
|
|
519
|
-
steps_json = ${sqlLiteral(trajectory.steps)},
|
|
520
|
-
reward_components_json = ${sqlLiteral(trajectory.rewardComponents)},
|
|
521
|
-
metrics_json = ${sqlLiteral(trajectory.metrics)},
|
|
522
|
-
metadata_json = ${sqlLiteral(trajectory.metadata)},
|
|
523
|
-
updated_at = ${sqlLiteral(updatedAtIso)}
|
|
524
|
-
WHERE id = ${sqlLiteral(trajectoryId)}
|
|
525
|
-
`);
|
|
526
|
-
} catch (modernErr) {
|
|
527
|
-
await this.executeRawSql(`
|
|
528
|
-
UPDATE trajectories SET
|
|
529
|
-
status = ${sqlLiteral(status)},
|
|
530
|
-
end_time = ${sqlLiteral(persistedEndTime)},
|
|
531
|
-
duration_ms = ${sqlLiteral(persistedDuration)},
|
|
532
|
-
step_count = ${totals.stepCount},
|
|
533
|
-
llm_call_count = ${totals.llmCallCount},
|
|
534
|
-
provider_access_count = ${totals.providerAccessCount},
|
|
535
|
-
total_prompt_tokens = ${totals.totalPromptTokens},
|
|
536
|
-
total_completion_tokens = ${totals.totalCompletionTokens},
|
|
537
|
-
total_reward = ${trajectory.totalReward},
|
|
538
|
-
steps_json = ${sqlLiteral(trajectory.steps)},
|
|
539
|
-
metadata = ${sqlLiteral(trajectory.metadata)},
|
|
540
|
-
updated_at = ${sqlLiteral(updatedAtIso)}
|
|
541
|
-
WHERE id = ${sqlLiteral(trajectoryId)}
|
|
542
|
-
`).catch((legacyErr) => {
|
|
543
|
-
logger.warn({ err: legacyErr, trajectoryId }, `[trajectory-logger] Failed to persist trajectory update after compatibility fallback: ${modernErr instanceof Error ? modernErr.message : String(modernErr)}`);
|
|
544
|
-
throw legacyErr;
|
|
545
|
-
});
|
|
546
|
-
}
|
|
547
|
-
}
|
|
548
|
-
async ensureStepExists(trajectory, stepId) {
|
|
549
|
-
let step = trajectory.steps.find((entry) => entry.stepId === stepId);
|
|
550
|
-
if (step) {
|
|
551
|
-
if (!Array.isArray(step.llmCalls))
|
|
552
|
-
step.llmCalls = [];
|
|
553
|
-
if (!Array.isArray(step.providerAccesses))
|
|
554
|
-
step.providerAccesses = [];
|
|
555
|
-
return step;
|
|
556
|
-
}
|
|
557
|
-
const index = await this.getStepIndex(stepId);
|
|
558
|
-
const stepNumber = index?.stepNumber ?? trajectory.steps.length;
|
|
559
|
-
step = this.createStep(stepId, stepNumber, this.defaultEnvironmentState());
|
|
560
|
-
trajectory.steps.push(step);
|
|
561
|
-
trajectory.steps.sort((a, b) => a.stepNumber - b.stepNumber);
|
|
562
|
-
return step;
|
|
563
|
-
}
|
|
564
|
-
logLlmCall(params) {
|
|
565
|
-
if (!this.enabled)
|
|
566
|
-
return;
|
|
567
|
-
(async () => {
|
|
568
|
-
const trajectoryId = await this.resolveTrajectoryId(params.stepId);
|
|
569
|
-
if (!trajectoryId) {
|
|
570
|
-
logger.debug({ stepId: params.stepId }, "[trajectory-logger] No trajectory mapping for LLM call");
|
|
571
|
-
return;
|
|
572
|
-
}
|
|
573
|
-
await this.withTrajectoryWriteLock(trajectoryId, async () => {
|
|
574
|
-
const trajectory = await this.getTrajectoryById(trajectoryId);
|
|
575
|
-
if (!trajectory)
|
|
576
|
-
return;
|
|
577
|
-
const step = await this.ensureStepExists(trajectory, params.stepId);
|
|
578
|
-
const llmCall = {
|
|
579
|
-
callId: uuidv4(),
|
|
580
|
-
timestamp: Date.now(),
|
|
581
|
-
model: params.model,
|
|
582
|
-
modelVersion: params.modelVersion,
|
|
583
|
-
systemPrompt: params.systemPrompt,
|
|
584
|
-
userPrompt: params.userPrompt,
|
|
585
|
-
response: params.response,
|
|
586
|
-
reasoning: params.reasoning,
|
|
587
|
-
temperature: params.temperature,
|
|
588
|
-
maxTokens: params.maxTokens,
|
|
589
|
-
purpose: this.normalizePurpose(params.purpose),
|
|
590
|
-
actionType: params.actionType,
|
|
591
|
-
promptTokens: params.promptTokens,
|
|
592
|
-
completionTokens: params.completionTokens,
|
|
593
|
-
latencyMs: params.latencyMs
|
|
594
|
-
};
|
|
595
|
-
step.llmCalls.push(llmCall);
|
|
596
|
-
await this.persistTrajectory(trajectoryId, trajectory, "active");
|
|
597
|
-
});
|
|
598
|
-
})().catch((err) => {
|
|
599
|
-
logger.warn({ err, stepId: params.stepId }, "[trajectory-logger] Failed to persist LLM call");
|
|
600
|
-
});
|
|
601
|
-
}
|
|
602
|
-
logLLMCall(stepId, details) {
|
|
603
|
-
this.logLlmCall({
|
|
604
|
-
stepId,
|
|
605
|
-
model: details.model,
|
|
606
|
-
modelVersion: details.modelVersion,
|
|
607
|
-
systemPrompt: details.systemPrompt,
|
|
608
|
-
userPrompt: details.userPrompt,
|
|
609
|
-
response: details.response,
|
|
610
|
-
reasoning: details.reasoning,
|
|
611
|
-
temperature: details.temperature,
|
|
612
|
-
maxTokens: details.maxTokens,
|
|
613
|
-
purpose: details.purpose,
|
|
614
|
-
actionType: details.actionType ?? "",
|
|
615
|
-
latencyMs: details.latencyMs ?? 0,
|
|
616
|
-
promptTokens: details.promptTokens,
|
|
617
|
-
completionTokens: details.completionTokens
|
|
618
|
-
});
|
|
619
|
-
}
|
|
620
|
-
logProviderAccess(arg1, arg2) {
|
|
621
|
-
if (!this.enabled)
|
|
622
|
-
return;
|
|
623
|
-
const params = typeof arg1 === "string" ? {
|
|
624
|
-
stepId: arg1,
|
|
625
|
-
providerName: arg2?.providerName ?? "unknown",
|
|
626
|
-
data: arg2?.data ?? {},
|
|
627
|
-
purpose: arg2?.purpose ?? "other",
|
|
628
|
-
query: arg2?.query
|
|
629
|
-
} : arg1;
|
|
630
|
-
(async () => {
|
|
631
|
-
const trajectoryId = await this.resolveTrajectoryId(params.stepId);
|
|
632
|
-
if (!trajectoryId) {
|
|
633
|
-
logger.debug({ stepId: params.stepId }, "[trajectory-logger] No trajectory mapping for provider access");
|
|
634
|
-
return;
|
|
635
|
-
}
|
|
636
|
-
await this.withTrajectoryWriteLock(trajectoryId, async () => {
|
|
637
|
-
const trajectory = await this.getTrajectoryById(trajectoryId);
|
|
638
|
-
if (!trajectory)
|
|
639
|
-
return;
|
|
640
|
-
const step = await this.ensureStepExists(trajectory, params.stepId);
|
|
641
|
-
const access = {
|
|
642
|
-
providerId: uuidv4(),
|
|
643
|
-
providerName: params.providerName,
|
|
644
|
-
timestamp: Date.now(),
|
|
645
|
-
data: params.data,
|
|
646
|
-
query: params.query,
|
|
647
|
-
purpose: params.purpose
|
|
648
|
-
};
|
|
649
|
-
step.providerAccesses.push(access);
|
|
650
|
-
await this.persistTrajectory(trajectoryId, trajectory, "active");
|
|
651
|
-
});
|
|
652
|
-
})().catch((err) => {
|
|
653
|
-
logger.warn({ err, stepId: params.stepId }, "[trajectory-logger] Failed to persist provider access");
|
|
654
|
-
});
|
|
655
|
-
}
|
|
656
|
-
logProviderAccessByTrajectoryId(trajectoryId, access) {
|
|
657
|
-
const stepId = this.getCurrentStepId(trajectoryId);
|
|
658
|
-
if (!stepId) {
|
|
659
|
-
logger.debug({ trajectoryId }, "[trajectory-logger] No active step for provider access by trajectory");
|
|
660
|
-
return;
|
|
661
|
-
}
|
|
662
|
-
this.logProviderAccess(stepId, access);
|
|
663
|
-
}
|
|
664
|
-
async startTrajectory(stepIdOrAgentId, options = {}) {
|
|
665
|
-
if (!this.enabled)
|
|
666
|
-
return uuidv4();
|
|
667
|
-
const legacyStepId = typeof options.agentId === "string" && options.agentId.length > 0 ? stepIdOrAgentId : null;
|
|
668
|
-
const agentId = (typeof options.agentId === "string" && options.agentId.length > 0 ? options.agentId : stepIdOrAgentId) ?? stepIdOrAgentId;
|
|
11
|
+
startTrajectory(agentId, options = {}) {
|
|
669
12
|
const trajectoryId = uuidv4();
|
|
670
13
|
const now = Date.now();
|
|
671
|
-
const timestampIso = new Date(now).toISOString();
|
|
672
|
-
const metadata = {
|
|
673
|
-
...options.metadata ?? {}
|
|
674
|
-
};
|
|
675
|
-
if (options.roomId)
|
|
676
|
-
metadata.roomId = options.roomId;
|
|
677
|
-
if (options.entityId)
|
|
678
|
-
metadata.entityId = options.entityId;
|
|
679
14
|
const trajectory = {
|
|
680
|
-
trajectoryId,
|
|
681
|
-
agentId,
|
|
15
|
+
trajectoryId: asUUID(trajectoryId),
|
|
16
|
+
agentId: asUUID(agentId),
|
|
682
17
|
startTime: now,
|
|
683
18
|
endTime: now,
|
|
684
19
|
durationMs: 0,
|
|
685
|
-
scenarioId: options.scenarioId,
|
|
686
20
|
episodeId: options.episodeId,
|
|
21
|
+
scenarioId: options.scenarioId,
|
|
687
22
|
batchId: options.batchId,
|
|
688
23
|
groupIndex: options.groupIndex,
|
|
689
24
|
steps: [],
|
|
690
25
|
totalReward: 0,
|
|
691
|
-
rewardComponents: {
|
|
26
|
+
rewardComponents: {
|
|
27
|
+
environmentReward: 0
|
|
28
|
+
},
|
|
692
29
|
metrics: {
|
|
693
30
|
episodeLength: 0,
|
|
694
31
|
finalStatus: "completed"
|
|
695
32
|
},
|
|
696
|
-
metadata: {
|
|
697
|
-
source: options.source ?? "chat",
|
|
698
|
-
...metadata
|
|
699
|
-
}
|
|
33
|
+
metadata: options.metadata || {}
|
|
700
34
|
};
|
|
701
|
-
|
|
702
|
-
try {
|
|
703
|
-
await this.executeRawSql(`
|
|
704
|
-
INSERT INTO trajectories (
|
|
705
|
-
id, agent_id, source, status, start_time, scenario_id, episode_id,
|
|
706
|
-
batch_id, group_index, metadata_json, steps_json, reward_components_json, metrics_json,
|
|
707
|
-
created_at, updated_at
|
|
708
|
-
) VALUES (
|
|
709
|
-
${sqlLiteral(trajectoryId)},
|
|
710
|
-
${sqlLiteral(agentId)},
|
|
711
|
-
${sqlLiteral(options.source ?? "chat")},
|
|
712
|
-
'active',
|
|
713
|
-
${now},
|
|
714
|
-
${sqlLiteral(options.scenarioId ?? null)},
|
|
715
|
-
${sqlLiteral(options.episodeId ?? null)},
|
|
716
|
-
${sqlLiteral(options.batchId ?? null)},
|
|
717
|
-
${options.groupIndex ?? "NULL"},
|
|
718
|
-
${sqlLiteral(trajectory.metadata)},
|
|
719
|
-
${sqlLiteral([])},
|
|
720
|
-
${sqlLiteral(trajectory.rewardComponents)},
|
|
721
|
-
${sqlLiteral(trajectory.metrics)},
|
|
722
|
-
${sqlLiteral(timestampIso)},
|
|
723
|
-
${sqlLiteral(timestampIso)}
|
|
724
|
-
)
|
|
725
|
-
`);
|
|
726
|
-
persistedStart = true;
|
|
727
|
-
} catch (err) {
|
|
728
|
-
try {
|
|
729
|
-
await this.executeRawSql(`
|
|
730
|
-
INSERT INTO trajectories (
|
|
731
|
-
id, agent_id, source, status, start_time, steps_json, metadata, created_at, updated_at
|
|
732
|
-
) VALUES (
|
|
733
|
-
${sqlLiteral(trajectoryId)},
|
|
734
|
-
${sqlLiteral(agentId)},
|
|
735
|
-
${sqlLiteral(options.source ?? "chat")},
|
|
736
|
-
'active',
|
|
737
|
-
${now},
|
|
738
|
-
${sqlLiteral([])},
|
|
739
|
-
${sqlLiteral(trajectory.metadata)},
|
|
740
|
-
${sqlLiteral(timestampIso)},
|
|
741
|
-
${sqlLiteral(timestampIso)}
|
|
742
|
-
)
|
|
743
|
-
`);
|
|
744
|
-
persistedStart = true;
|
|
745
|
-
} catch (legacyErr) {
|
|
746
|
-
logger.warn({ err: legacyErr, trajectoryId }, "[trajectory-logger] Failed to persist trajectory start");
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
if (persistedStart && legacyStepId) {
|
|
750
|
-
this.stepToTrajectory.set(legacyStepId, trajectoryId);
|
|
751
|
-
try {
|
|
752
|
-
await this.setStepIndex(legacyStepId, trajectoryId, -1, false);
|
|
753
|
-
} catch (indexErr) {
|
|
754
|
-
logger.warn({ err: indexErr, trajectoryId, stepId: legacyStepId }, "[trajectory-logger] Failed to persist step index for trajectory start");
|
|
755
|
-
}
|
|
756
|
-
}
|
|
35
|
+
this.activeTrajectories.set(trajectoryId, trajectory);
|
|
757
36
|
return trajectoryId;
|
|
758
37
|
}
|
|
759
38
|
startStep(trajectoryId, envState) {
|
|
760
|
-
if (!this.enabled)
|
|
761
|
-
return uuidv4();
|
|
762
39
|
const stepId = uuidv4();
|
|
40
|
+
const trajectory = this.activeTrajectories.get(trajectoryId);
|
|
41
|
+
if (!trajectory) {
|
|
42
|
+
throw new Error(`Trajectory ${trajectoryId} not found`);
|
|
43
|
+
}
|
|
44
|
+
const step = {
|
|
45
|
+
stepId: asUUID(stepId),
|
|
46
|
+
stepNumber: trajectory.steps.length,
|
|
47
|
+
timestamp: envState.timestamp || Date.now(),
|
|
48
|
+
environmentState: envState,
|
|
49
|
+
observation: {},
|
|
50
|
+
llmCalls: [],
|
|
51
|
+
providerAccesses: [],
|
|
52
|
+
action: {
|
|
53
|
+
attemptId: "",
|
|
54
|
+
timestamp: 0,
|
|
55
|
+
actionType: "pending",
|
|
56
|
+
actionName: "pending",
|
|
57
|
+
parameters: {},
|
|
58
|
+
success: false
|
|
59
|
+
},
|
|
60
|
+
reward: 0,
|
|
61
|
+
done: false
|
|
62
|
+
};
|
|
63
|
+
trajectory.steps.push(step);
|
|
763
64
|
this.activeStepIds.set(trajectoryId, stepId);
|
|
764
|
-
this.stepToTrajectory.set(stepId, trajectoryId);
|
|
765
|
-
this.withTrajectoryWriteLock(trajectoryId, async () => {
|
|
766
|
-
const trajectory = await this.getTrajectoryById(trajectoryId);
|
|
767
|
-
if (!trajectory) {
|
|
768
|
-
logger.warn({ trajectoryId }, "[trajectory-logger] Trajectory not found for startStep");
|
|
769
|
-
return;
|
|
770
|
-
}
|
|
771
|
-
const step = this.createStep(stepId, trajectory.steps.length, envState);
|
|
772
|
-
trajectory.steps.push(step);
|
|
773
|
-
await this.markAllStepsInactive(trajectoryId);
|
|
774
|
-
await this.setStepIndex(stepId, trajectoryId, step.stepNumber, true);
|
|
775
|
-
await this.persistTrajectory(trajectoryId, trajectory, "active");
|
|
776
|
-
}).catch((err) => {
|
|
777
|
-
logger.warn({ err, trajectoryId, stepId }, "[trajectory-logger] Failed to persist startStep");
|
|
778
|
-
});
|
|
779
65
|
return stepId;
|
|
780
66
|
}
|
|
781
|
-
|
|
782
|
-
|
|
67
|
+
logLLMCall(stepId, llmCall) {
|
|
68
|
+
const trajectory = this.findTrajectoryByStepId(stepId);
|
|
69
|
+
if (!trajectory) {
|
|
70
|
+
logger.warn({ stepId }, "Trajectory not found for LLM call");
|
|
783
71
|
return;
|
|
784
|
-
|
|
785
|
-
const
|
|
786
|
-
|
|
787
|
-
|
|
72
|
+
}
|
|
73
|
+
const step = trajectory.steps.find((s) => s.stepId === stepId);
|
|
74
|
+
if (!step) {
|
|
75
|
+
logger.warn({ stepId }, "Step not found for LLM call");
|
|
788
76
|
return;
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
const step = await this.ensureStepExists(trajectory, stepId);
|
|
797
|
-
step.action = {
|
|
798
|
-
attemptId: uuidv4(),
|
|
799
|
-
timestamp: Date.now(),
|
|
800
|
-
...action
|
|
801
|
-
};
|
|
802
|
-
step.done = true;
|
|
803
|
-
if (rewardInfo?.reward !== undefined) {
|
|
804
|
-
step.reward = rewardInfo.reward;
|
|
805
|
-
trajectory.totalReward += rewardInfo.reward;
|
|
806
|
-
}
|
|
807
|
-
if (rewardInfo?.components) {
|
|
808
|
-
trajectory.rewardComponents = {
|
|
809
|
-
...trajectory.rewardComponents,
|
|
810
|
-
...rewardInfo.components
|
|
811
|
-
};
|
|
812
|
-
}
|
|
813
|
-
await this.setStepIndex(stepId, trajectoryId, step.stepNumber, false);
|
|
814
|
-
this.activeStepIds.delete(trajectoryId);
|
|
815
|
-
await this.persistTrajectory(trajectoryId, trajectory, "active");
|
|
816
|
-
}).catch((err) => {
|
|
817
|
-
logger.warn({ err, trajectoryId }, "[trajectory-logger] Failed to complete step");
|
|
818
|
-
});
|
|
77
|
+
}
|
|
78
|
+
const fullLLMCall = {
|
|
79
|
+
callId: uuidv4(),
|
|
80
|
+
timestamp: Date.now(),
|
|
81
|
+
...llmCall
|
|
82
|
+
};
|
|
83
|
+
step.llmCalls.push(fullLLMCall);
|
|
819
84
|
}
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
if (!trajectoryId) {
|
|
825
|
-
logger.debug({ stepIdOrTrajectoryId }, "[trajectory-logger] No trajectory to end");
|
|
85
|
+
logProviderAccess(stepId, access) {
|
|
86
|
+
const trajectory = this.findTrajectoryByStepId(stepId);
|
|
87
|
+
if (!trajectory) {
|
|
88
|
+
logger.warn({ stepId }, "Trajectory not found for provider access");
|
|
826
89
|
return;
|
|
827
90
|
}
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
return;
|
|
833
|
-
}
|
|
834
|
-
const now = Date.now();
|
|
835
|
-
trajectory.endTime = now;
|
|
836
|
-
trajectory.durationMs = now - trajectory.startTime;
|
|
837
|
-
trajectory.metrics.finalStatus = status;
|
|
838
|
-
trajectory.metrics.episodeLength = trajectory.steps.length;
|
|
839
|
-
if (finalMetrics) {
|
|
840
|
-
trajectory.metrics = {
|
|
841
|
-
...trajectory.metrics,
|
|
842
|
-
...finalMetrics
|
|
843
|
-
};
|
|
844
|
-
}
|
|
845
|
-
await this.markAllStepsInactive(trajectoryId);
|
|
846
|
-
this.activeStepIds.delete(trajectoryId);
|
|
847
|
-
await this.persistTrajectory(trajectoryId, trajectory, status);
|
|
848
|
-
});
|
|
849
|
-
for (const [stepId, mappedTrajectoryId] of this.stepToTrajectory.entries()) {
|
|
850
|
-
if (mappedTrajectoryId === trajectoryId) {
|
|
851
|
-
this.stepToTrajectory.delete(stepId);
|
|
852
|
-
}
|
|
91
|
+
const step = trajectory.steps.find((s) => s.stepId === stepId);
|
|
92
|
+
if (!step) {
|
|
93
|
+
logger.warn({ stepId }, "Step not found for provider access");
|
|
94
|
+
return;
|
|
853
95
|
}
|
|
96
|
+
const fullAccess = {
|
|
97
|
+
providerId: uuidv4(),
|
|
98
|
+
timestamp: Date.now(),
|
|
99
|
+
...access
|
|
100
|
+
};
|
|
101
|
+
step.providerAccesses.push(fullAccess);
|
|
854
102
|
}
|
|
855
|
-
|
|
856
|
-
const
|
|
857
|
-
if (!
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
const offset = Math.max(0, options.offset ?? 0);
|
|
861
|
-
const limit = Math.min(500, Math.max(1, options.limit ?? 50));
|
|
862
|
-
const whereClauses = [];
|
|
863
|
-
if (options.status) {
|
|
864
|
-
whereClauses.push(`status = ${sqlLiteral(options.status)}`);
|
|
865
|
-
}
|
|
866
|
-
if (options.source) {
|
|
867
|
-
whereClauses.push(`source = ${sqlLiteral(options.source)}`);
|
|
868
|
-
}
|
|
869
|
-
if (options.scenarioId) {
|
|
870
|
-
whereClauses.push(`scenario_id = ${sqlLiteral(options.scenarioId)}`);
|
|
871
|
-
}
|
|
872
|
-
if (options.batchId) {
|
|
873
|
-
whereClauses.push(`batch_id = ${sqlLiteral(options.batchId)}`);
|
|
874
|
-
}
|
|
875
|
-
if (options.isTrainingData !== undefined) {
|
|
876
|
-
whereClauses.push(`is_training_data = ${options.isTrainingData}`);
|
|
877
|
-
}
|
|
878
|
-
if (options.startDate) {
|
|
879
|
-
whereClauses.push(`created_at >= ${sqlLiteral(options.startDate)}::timestamptz`);
|
|
880
|
-
}
|
|
881
|
-
if (options.endDate) {
|
|
882
|
-
whereClauses.push(`created_at <= ${sqlLiteral(options.endDate)}::timestamptz`);
|
|
883
|
-
}
|
|
884
|
-
if (options.search) {
|
|
885
|
-
const escaped = options.search.replace(/'/g, "''").replace(/%/g, "\\%");
|
|
886
|
-
whereClauses.push(`(
|
|
887
|
-
id ILIKE '%${escaped}%' OR
|
|
888
|
-
agent_id ILIKE '%${escaped}%' OR
|
|
889
|
-
source ILIKE '%${escaped}%' OR
|
|
890
|
-
scenario_id ILIKE '%${escaped}%'
|
|
891
|
-
)`);
|
|
103
|
+
logLLMCallByTrajectoryId(trajectoryId, llmCall) {
|
|
104
|
+
const stepId = this.activeStepIds.get(trajectoryId);
|
|
105
|
+
if (!stepId) {
|
|
106
|
+
logger.warn({ trajectoryId }, "No active step for trajectory");
|
|
107
|
+
return;
|
|
892
108
|
}
|
|
893
|
-
|
|
894
|
-
const countResult = await this.executeRawSql(`SELECT count(*)::int AS total FROM trajectories ${whereClause}`);
|
|
895
|
-
const total = asNumber(pickCell(countResult.rows[0] ?? {}, "total")) ?? 0;
|
|
896
|
-
const rowsResult = await this.executeRawSql(`
|
|
897
|
-
SELECT
|
|
898
|
-
id, agent_id, source, status, start_time, end_time, duration_ms,
|
|
899
|
-
step_count, llm_call_count, total_prompt_tokens, total_completion_tokens,
|
|
900
|
-
total_reward, scenario_id, batch_id, created_at
|
|
901
|
-
FROM trajectories
|
|
902
|
-
${whereClause}
|
|
903
|
-
ORDER BY created_at DESC
|
|
904
|
-
LIMIT ${limit} OFFSET ${offset}
|
|
905
|
-
`);
|
|
906
|
-
const trajectories = rowsResult.rows.map((row) => {
|
|
907
|
-
const status = asString(pickCell(row, "status")) ?? "completed";
|
|
908
|
-
const rawLlmCallCount = asNumber(pickCell(row, "llm_call_count")) ?? 0;
|
|
909
|
-
const llmCallCount = status === "active" ? rawLlmCallCount : Math.max(1, rawLlmCallCount);
|
|
910
|
-
return {
|
|
911
|
-
id: asString(pickCell(row, "id")) ?? "",
|
|
912
|
-
agentId: asString(pickCell(row, "agent_id")) ?? "",
|
|
913
|
-
source: asString(pickCell(row, "source")) ?? "chat",
|
|
914
|
-
status,
|
|
915
|
-
startTime: asNumber(pickCell(row, "start_time")) ?? 0,
|
|
916
|
-
endTime: asNumber(pickCell(row, "end_time")),
|
|
917
|
-
durationMs: asNumber(pickCell(row, "duration_ms")),
|
|
918
|
-
stepCount: asNumber(pickCell(row, "step_count")) ?? 0,
|
|
919
|
-
llmCallCount,
|
|
920
|
-
totalPromptTokens: asNumber(pickCell(row, "total_prompt_tokens")) ?? 0,
|
|
921
|
-
totalCompletionTokens: asNumber(pickCell(row, "total_completion_tokens")) ?? 0,
|
|
922
|
-
totalReward: asNumber(pickCell(row, "total_reward")) ?? 0,
|
|
923
|
-
scenarioId: asString(pickCell(row, "scenario_id")),
|
|
924
|
-
batchId: asString(pickCell(row, "batch_id")),
|
|
925
|
-
createdAt: asIsoString(pickCell(row, "created_at"))
|
|
926
|
-
};
|
|
927
|
-
});
|
|
928
|
-
return { trajectories, total, offset, limit };
|
|
109
|
+
this.logLLMCall(stepId, llmCall);
|
|
929
110
|
}
|
|
930
|
-
|
|
931
|
-
const
|
|
932
|
-
if (!
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
const result = await this.executeRawSql(`SELECT * FROM trajectories WHERE id = '${safeId}' LIMIT 1`);
|
|
936
|
-
if (result.rows.length === 0)
|
|
937
|
-
return null;
|
|
938
|
-
const row = result.rows[0];
|
|
939
|
-
const trajectory = this.rowToTrajectory(row);
|
|
940
|
-
const status = this.normalizeTrajectoryStatus(asString(pickCell(row, "status")));
|
|
941
|
-
if (status !== "active") {
|
|
942
|
-
const inserted = this.ensureAtLeastOneLlmCall(trajectory, "backfill");
|
|
943
|
-
if (inserted) {
|
|
944
|
-
await this.withTrajectoryWriteLock(trajectoryId, async () => {
|
|
945
|
-
await this.persistTrajectory(trajectoryId, trajectory, status);
|
|
946
|
-
});
|
|
947
|
-
}
|
|
111
|
+
logProviderAccessByTrajectoryId(trajectoryId, access) {
|
|
112
|
+
const stepId = this.activeStepIds.get(trajectoryId);
|
|
113
|
+
if (!stepId) {
|
|
114
|
+
logger.warn({ trajectoryId }, "No active step for trajectory");
|
|
115
|
+
return;
|
|
948
116
|
}
|
|
949
|
-
|
|
117
|
+
this.logProviderAccess(stepId, access);
|
|
950
118
|
}
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
totalCompletionTokens: 0,
|
|
960
|
-
averageDurationMs: 0,
|
|
961
|
-
averageReward: 0,
|
|
962
|
-
bySource: {},
|
|
963
|
-
byStatus: {},
|
|
964
|
-
byScenario: {}
|
|
965
|
-
};
|
|
966
|
-
}
|
|
967
|
-
const statsResult = await this.executeRawSql(`
|
|
968
|
-
SELECT
|
|
969
|
-
count(*)::int AS total_trajectories,
|
|
970
|
-
COALESCE(sum(step_count), 0)::int AS total_steps,
|
|
971
|
-
COALESCE(sum(llm_call_count), 0)::int AS total_llm_calls,
|
|
972
|
-
COALESCE(sum(total_prompt_tokens), 0)::int AS total_prompt_tokens,
|
|
973
|
-
COALESCE(sum(total_completion_tokens), 0)::int AS total_completion_tokens,
|
|
974
|
-
COALESCE(avg(duration_ms), 0)::int AS avg_duration_ms,
|
|
975
|
-
COALESCE(avg(total_reward), 0)::real AS avg_reward
|
|
976
|
-
FROM trajectories
|
|
977
|
-
`);
|
|
978
|
-
const sourceResult = await this.executeRawSql(`
|
|
979
|
-
SELECT source, count(*)::int AS cnt
|
|
980
|
-
FROM trajectories
|
|
981
|
-
GROUP BY source
|
|
982
|
-
`);
|
|
983
|
-
const statusResult = await this.executeRawSql(`
|
|
984
|
-
SELECT status, count(*)::int AS cnt
|
|
985
|
-
FROM trajectories
|
|
986
|
-
GROUP BY status
|
|
987
|
-
`);
|
|
988
|
-
const scenarioResult = await this.executeRawSql(`
|
|
989
|
-
SELECT scenario_id, count(*)::int AS cnt
|
|
990
|
-
FROM trajectories
|
|
991
|
-
WHERE scenario_id IS NOT NULL
|
|
992
|
-
GROUP BY scenario_id
|
|
993
|
-
`);
|
|
994
|
-
const stats = statsResult.rows[0] ?? {};
|
|
995
|
-
const bySource = {};
|
|
996
|
-
const byStatus = {};
|
|
997
|
-
const byScenario = {};
|
|
998
|
-
for (const row of sourceResult.rows) {
|
|
999
|
-
const source = asString(pickCell(row, "source"));
|
|
1000
|
-
const cnt = asNumber(pickCell(row, "cnt"));
|
|
1001
|
-
if (source && cnt !== null)
|
|
1002
|
-
bySource[source] = cnt;
|
|
1003
|
-
}
|
|
1004
|
-
for (const row of statusResult.rows) {
|
|
1005
|
-
const status = asString(pickCell(row, "status"));
|
|
1006
|
-
const cnt = asNumber(pickCell(row, "cnt"));
|
|
1007
|
-
if (status && cnt !== null)
|
|
1008
|
-
byStatus[status] = cnt;
|
|
119
|
+
getCurrentStepId(trajectoryId) {
|
|
120
|
+
return this.activeStepIds.get(trajectoryId) || null;
|
|
121
|
+
}
|
|
122
|
+
completeStep(trajectoryId, stepId, action, rewardInfo) {
|
|
123
|
+
const trajectory = this.activeTrajectories.get(trajectoryId);
|
|
124
|
+
if (!trajectory) {
|
|
125
|
+
logger.warn({ trajectoryId }, "Trajectory not found for completeStep");
|
|
126
|
+
return;
|
|
1009
127
|
}
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
byScenario[scenario] = cnt;
|
|
128
|
+
const step = trajectory.steps.find((s) => s.stepId === stepId);
|
|
129
|
+
if (!step) {
|
|
130
|
+
logger.warn({ trajectoryId, stepId }, "Step not found for completeStep");
|
|
131
|
+
return;
|
|
1015
132
|
}
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
totalPromptTokens: asNumber(pickCell(stats, "total_prompt_tokens")) ?? 0,
|
|
1021
|
-
totalCompletionTokens: asNumber(pickCell(stats, "total_completion_tokens")) ?? 0,
|
|
1022
|
-
averageDurationMs: asNumber(pickCell(stats, "avg_duration_ms")) ?? 0,
|
|
1023
|
-
averageReward: asNumber(pickCell(stats, "avg_reward")) ?? 0,
|
|
1024
|
-
bySource,
|
|
1025
|
-
byStatus,
|
|
1026
|
-
byScenario
|
|
133
|
+
step.action = {
|
|
134
|
+
attemptId: uuidv4(),
|
|
135
|
+
timestamp: Date.now(),
|
|
136
|
+
...action
|
|
1027
137
|
};
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
return result.rows.length;
|
|
1038
|
-
}
|
|
1039
|
-
async clearAllTrajectories() {
|
|
1040
|
-
const runtime = this.runtime;
|
|
1041
|
-
if (!runtime?.adapter)
|
|
1042
|
-
return 0;
|
|
1043
|
-
const countResult = await this.executeRawSql(`SELECT count(*)::int AS cnt FROM trajectories`);
|
|
1044
|
-
const count = asNumber(pickCell(countResult.rows[0] ?? {}, "cnt")) ?? 0;
|
|
1045
|
-
await this.executeRawSql(`DELETE FROM trajectories`);
|
|
1046
|
-
return count;
|
|
1047
|
-
}
|
|
1048
|
-
sanitizeZipFolderName(value) {
|
|
1049
|
-
const sanitized = value.trim().replace(/[^a-zA-Z0-9._-]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1050
|
-
return sanitized || "trajectory";
|
|
1051
|
-
}
|
|
1052
|
-
redactTrajectoryPrompts(trajectory) {
|
|
1053
|
-
return {
|
|
1054
|
-
...trajectory,
|
|
1055
|
-
steps: trajectory.steps.map((step) => ({
|
|
1056
|
-
...step,
|
|
1057
|
-
llmCalls: step.llmCalls.map((call) => ({
|
|
1058
|
-
...call,
|
|
1059
|
-
systemPrompt: "[redacted]",
|
|
1060
|
-
userPrompt: "[redacted]",
|
|
1061
|
-
response: "[redacted]"
|
|
1062
|
-
}))
|
|
1063
|
-
}))
|
|
1064
|
-
};
|
|
1065
|
-
}
|
|
1066
|
-
buildZipSummary(trajectory) {
|
|
1067
|
-
const finalStatus = trajectory.metrics?.finalStatus ?? "completed";
|
|
1068
|
-
const normalizedEndTime = typeof trajectory.endTime === "number" && trajectory.endTime > 0 ? trajectory.endTime : null;
|
|
1069
|
-
const status = finalStatus === "timeout" || finalStatus === "terminated" || finalStatus === "error" ? "error" : finalStatus === "completed" ? "completed" : normalizedEndTime ? "completed" : "active";
|
|
1070
|
-
let llmCallCount = 0;
|
|
1071
|
-
let providerAccessCount = 0;
|
|
1072
|
-
let totalPromptTokens = 0;
|
|
1073
|
-
let totalCompletionTokens = 0;
|
|
1074
|
-
for (const step of trajectory.steps) {
|
|
1075
|
-
providerAccessCount += step.providerAccesses.length;
|
|
1076
|
-
llmCallCount += step.llmCalls.length;
|
|
1077
|
-
for (const call of step.llmCalls) {
|
|
1078
|
-
totalPromptTokens += call.promptTokens ?? 0;
|
|
1079
|
-
totalCompletionTokens += call.completionTokens ?? 0;
|
|
1080
|
-
}
|
|
138
|
+
if (rewardInfo?.reward !== undefined) {
|
|
139
|
+
step.reward = rewardInfo.reward;
|
|
140
|
+
trajectory.totalReward += rewardInfo.reward;
|
|
141
|
+
}
|
|
142
|
+
if (rewardInfo?.components) {
|
|
143
|
+
trajectory.rewardComponents = {
|
|
144
|
+
...trajectory.rewardComponents,
|
|
145
|
+
...rewardInfo.components
|
|
146
|
+
};
|
|
1081
147
|
}
|
|
1082
|
-
|
|
1083
|
-
const asNullableString = (value) => typeof value === "string" ? value : null;
|
|
1084
|
-
const source = typeof metadata.source === "string" ? metadata.source : "chat";
|
|
1085
|
-
const normalizedDurationMs = status === "active" ? null : typeof trajectory.durationMs === "number" ? trajectory.durationMs : null;
|
|
1086
|
-
const updatedAtMs = normalizedEndTime ?? (trajectory.startTime || Date.now());
|
|
1087
|
-
return {
|
|
1088
|
-
id: trajectory.trajectoryId,
|
|
1089
|
-
agentId: trajectory.agentId,
|
|
1090
|
-
roomId: asNullableString(metadata.roomId),
|
|
1091
|
-
entityId: asNullableString(metadata.entityId),
|
|
1092
|
-
conversationId: asNullableString(metadata.conversationId),
|
|
1093
|
-
source,
|
|
1094
|
-
status,
|
|
1095
|
-
startTime: trajectory.startTime,
|
|
1096
|
-
endTime: normalizedEndTime,
|
|
1097
|
-
durationMs: normalizedDurationMs,
|
|
1098
|
-
llmCallCount,
|
|
1099
|
-
providerAccessCount,
|
|
1100
|
-
totalPromptTokens,
|
|
1101
|
-
totalCompletionTokens,
|
|
1102
|
-
metadata,
|
|
1103
|
-
createdAt: new Date(trajectory.startTime).toISOString(),
|
|
1104
|
-
updatedAt: new Date(updatedAtMs).toISOString()
|
|
1105
|
-
};
|
|
148
|
+
this.activeStepIds.delete(trajectoryId);
|
|
1106
149
|
}
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
if (
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
startDate: options.startDate,
|
|
1113
|
-
endDate: options.endDate,
|
|
1114
|
-
scenarioId: options.scenarioId,
|
|
1115
|
-
batchId: options.batchId
|
|
1116
|
-
});
|
|
1117
|
-
targetIds = list.trajectories.map((trajectory) => trajectory.id);
|
|
1118
|
-
}
|
|
1119
|
-
const entries = [];
|
|
1120
|
-
const manifestRows = [];
|
|
1121
|
-
for (const trajectoryId of targetIds) {
|
|
1122
|
-
const detail = await this.getTrajectoryDetail(trajectoryId);
|
|
1123
|
-
if (!detail)
|
|
1124
|
-
continue;
|
|
1125
|
-
const exportTrajectory = options.includePrompts === false ? this.redactTrajectoryPrompts(detail) : detail;
|
|
1126
|
-
const summary = this.buildZipSummary(exportTrajectory);
|
|
1127
|
-
const folderName = this.sanitizeZipFolderName(trajectoryId);
|
|
1128
|
-
entries.push({
|
|
1129
|
-
name: `${folderName}/trajectory.json`,
|
|
1130
|
-
data: JSON.stringify(exportTrajectory, null, 2)
|
|
1131
|
-
});
|
|
1132
|
-
entries.push({
|
|
1133
|
-
name: `${folderName}/summary.json`,
|
|
1134
|
-
data: JSON.stringify(summary, null, 2)
|
|
1135
|
-
});
|
|
1136
|
-
manifestRows.push({
|
|
1137
|
-
trajectoryId,
|
|
1138
|
-
folder: folderName,
|
|
1139
|
-
createdAt: summary.createdAt
|
|
1140
|
-
});
|
|
150
|
+
completeCurrentStep(trajectoryId, action, rewardInfo) {
|
|
151
|
+
const stepId = this.activeStepIds.get(trajectoryId);
|
|
152
|
+
if (!stepId) {
|
|
153
|
+
logger.warn({ trajectoryId }, "No active step for trajectory");
|
|
154
|
+
return;
|
|
1141
155
|
}
|
|
1142
|
-
|
|
1143
|
-
name: "manifest.json",
|
|
1144
|
-
data: JSON.stringify({
|
|
1145
|
-
exportedAt: new Date().toISOString(),
|
|
1146
|
-
trajectories: manifestRows
|
|
1147
|
-
}, null, 2)
|
|
1148
|
-
});
|
|
1149
|
-
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
1150
|
-
return {
|
|
1151
|
-
filename: `trajectories-${timestamp}.zip`,
|
|
1152
|
-
entries
|
|
1153
|
-
};
|
|
156
|
+
this.completeStep(trajectoryId, stepId, action, rewardInfo);
|
|
1154
157
|
}
|
|
1155
|
-
async
|
|
1156
|
-
const
|
|
1157
|
-
if (!
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
const whereClauses = [];
|
|
1161
|
-
if (options.trajectoryIds && options.trajectoryIds.length > 0) {
|
|
1162
|
-
const ids = options.trajectoryIds.map(sqlLiteral).join(", ");
|
|
1163
|
-
whereClauses.push(`id IN (${ids})`);
|
|
1164
|
-
}
|
|
1165
|
-
if (options.startDate) {
|
|
1166
|
-
whereClauses.push(`created_at >= ${sqlLiteral(options.startDate)}::timestamptz`);
|
|
1167
|
-
}
|
|
1168
|
-
if (options.endDate) {
|
|
1169
|
-
whereClauses.push(`created_at <= ${sqlLiteral(options.endDate)}::timestamptz`);
|
|
1170
|
-
}
|
|
1171
|
-
if (options.scenarioId) {
|
|
1172
|
-
whereClauses.push(`scenario_id = ${sqlLiteral(options.scenarioId)}`);
|
|
1173
|
-
}
|
|
1174
|
-
if (options.batchId) {
|
|
1175
|
-
whereClauses.push(`batch_id = ${sqlLiteral(options.batchId)}`);
|
|
158
|
+
async endTrajectory(trajectoryId, status, finalMetrics) {
|
|
159
|
+
const trajectory = this.activeTrajectories.get(trajectoryId);
|
|
160
|
+
if (!trajectory) {
|
|
161
|
+
logger.warn({ trajectoryId }, "Trajectory not found for endTrajectory");
|
|
162
|
+
return;
|
|
1176
163
|
}
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
if (
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
for (const t of trajectories) {
|
|
1186
|
-
lines.push([
|
|
1187
|
-
t.trajectoryId,
|
|
1188
|
-
t.agentId,
|
|
1189
|
-
t.metadata.source ?? "chat",
|
|
1190
|
-
t.metrics.finalStatus,
|
|
1191
|
-
t.startTime,
|
|
1192
|
-
t.endTime,
|
|
1193
|
-
t.durationMs,
|
|
1194
|
-
t.steps.length,
|
|
1195
|
-
t.steps.reduce((sum, s) => sum + s.llmCalls.length, 0),
|
|
1196
|
-
t.totalReward,
|
|
1197
|
-
t.scenarioId ?? ""
|
|
1198
|
-
].join(","));
|
|
1199
|
-
}
|
|
1200
|
-
return {
|
|
1201
|
-
data: lines.join(`
|
|
1202
|
-
`),
|
|
1203
|
-
filename: `trajectories-${timestamp}.csv`,
|
|
1204
|
-
mimeType: "text/csv"
|
|
164
|
+
trajectory.endTime = Date.now();
|
|
165
|
+
trajectory.durationMs = trajectory.endTime - trajectory.startTime;
|
|
166
|
+
trajectory.metrics.finalStatus = status;
|
|
167
|
+
trajectory.metrics.episodeLength = trajectory.steps.length;
|
|
168
|
+
if (finalMetrics) {
|
|
169
|
+
trajectory.metrics = {
|
|
170
|
+
...trajectory.metrics,
|
|
171
|
+
...finalMetrics
|
|
1205
172
|
};
|
|
1206
173
|
}
|
|
1207
|
-
|
|
1208
|
-
if (!options.includePrompts) {
|
|
1209
|
-
exportData = trajectories.map((trajectory) => this.redactTrajectoryPrompts(trajectory));
|
|
1210
|
-
}
|
|
1211
|
-
return {
|
|
1212
|
-
data: JSON.stringify(exportData, null, 2),
|
|
1213
|
-
filename: `trajectories-${timestamp}.json`,
|
|
1214
|
-
mimeType: "application/json"
|
|
1215
|
-
};
|
|
1216
|
-
}
|
|
1217
|
-
rowToTrajectory(row) {
|
|
1218
|
-
const parseJson = (cell, fallback) => {
|
|
1219
|
-
if (typeof cell === "string") {
|
|
1220
|
-
try {
|
|
1221
|
-
return JSON.parse(cell);
|
|
1222
|
-
} catch {
|
|
1223
|
-
return fallback;
|
|
1224
|
-
}
|
|
1225
|
-
}
|
|
1226
|
-
if (typeof cell === "object" && cell !== null && !Array.isArray(cell)) {
|
|
1227
|
-
return cell;
|
|
1228
|
-
}
|
|
1229
|
-
return fallback;
|
|
1230
|
-
};
|
|
1231
|
-
return {
|
|
1232
|
-
trajectoryId: asString(pickCell(row, "id")) ?? "",
|
|
1233
|
-
agentId: asString(pickCell(row, "agent_id")) ?? "",
|
|
1234
|
-
startTime: asNumber(pickCell(row, "start_time")) ?? 0,
|
|
1235
|
-
endTime: asNumber(pickCell(row, "end_time")) ?? 0,
|
|
1236
|
-
durationMs: asNumber(pickCell(row, "duration_ms")) ?? 0,
|
|
1237
|
-
scenarioId: asString(pickCell(row, "scenario_id")) ?? undefined,
|
|
1238
|
-
episodeId: asString(pickCell(row, "episode_id")) ?? undefined,
|
|
1239
|
-
batchId: asString(pickCell(row, "batch_id")) ?? undefined,
|
|
1240
|
-
groupIndex: asNumber(pickCell(row, "group_index")) ?? undefined,
|
|
1241
|
-
steps: parseJson(pickCell(row, "steps_json", "steps"), []),
|
|
1242
|
-
totalReward: asNumber(pickCell(row, "total_reward")) ?? 0,
|
|
1243
|
-
rewardComponents: parseJson(pickCell(row, "reward_components_json", "reward_components"), { environmentReward: 0 }),
|
|
1244
|
-
metrics: parseJson(pickCell(row, "metrics_json", "metrics"), {
|
|
1245
|
-
episodeLength: 0,
|
|
1246
|
-
finalStatus: "completed"
|
|
1247
|
-
}),
|
|
1248
|
-
metadata: parseJson(pickCell(row, "metadata_json", "metadata"), {})
|
|
1249
|
-
};
|
|
174
|
+
this.activeStepIds.delete(trajectoryId);
|
|
1250
175
|
}
|
|
1251
176
|
getActiveTrajectory(trajectoryId) {
|
|
1252
|
-
return null;
|
|
1253
|
-
}
|
|
1254
|
-
getCurrentStepId(trajectoryId) {
|
|
1255
|
-
return this.activeStepIds.get(trajectoryId) || null;
|
|
1256
|
-
}
|
|
1257
|
-
getProviderAccessLogs() {
|
|
1258
|
-
return [];
|
|
177
|
+
return this.activeTrajectories.get(trajectoryId) || null;
|
|
1259
178
|
}
|
|
1260
|
-
|
|
1261
|
-
|
|
179
|
+
findTrajectoryByStepId(stepId) {
|
|
180
|
+
for (const trajectory of this.activeTrajectories.values()) {
|
|
181
|
+
if (trajectory.steps.some((s) => s.stepId === stepId)) {
|
|
182
|
+
return trajectory;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return null;
|
|
1262
186
|
}
|
|
1263
187
|
}
|
|
188
|
+
|
|
1264
189
|
// action-interceptor.ts
|
|
1265
190
|
import { logger as logger2 } from "@elizaos/core";
|
|
1266
191
|
var trajectoryContexts = new WeakMap;
|
|
@@ -1718,8 +643,8 @@ async function buildGameStateFromDB(_trajectoryId) {
|
|
|
1718
643
|
async function recomputeTrajectoryRewards(_trajectoryIds) {}
|
|
1719
644
|
// integration.ts
|
|
1720
645
|
import { logger as logger3 } from "@elizaos/core";
|
|
1721
|
-
|
|
1722
|
-
const trajectoryId =
|
|
646
|
+
function startAutonomousTick(trajectoryLogger, context) {
|
|
647
|
+
const trajectoryId = trajectoryLogger.startTrajectory(context.agentId, {
|
|
1723
648
|
scenarioId: context.scenarioId,
|
|
1724
649
|
episodeId: context.episodeId,
|
|
1725
650
|
batchId: context.batchId,
|
|
@@ -1884,9 +809,7 @@ var trajectoryLoggerPlugin = {
|
|
|
1884
809
|
if (!message || !runtime)
|
|
1885
810
|
return;
|
|
1886
811
|
if (!message.metadata) {
|
|
1887
|
-
message.metadata = {
|
|
1888
|
-
type: "message"
|
|
1889
|
-
};
|
|
812
|
+
message.metadata = { type: "message" };
|
|
1890
813
|
}
|
|
1891
814
|
const meta = message.metadata;
|
|
1892
815
|
const logger4 = runtime.getService("trajectory_logger");
|
|
@@ -1918,17 +841,13 @@ var trajectoryLoggerPlugin = {
|
|
|
1918
841
|
trajectoryStepId = normalizedStepId;
|
|
1919
842
|
meta.trajectoryStepId = trajectoryStepId;
|
|
1920
843
|
pendingTrajectoryEndTargetByStepId.set(trajectoryStepId, normalizedTrajectoryId);
|
|
1921
|
-
}
|
|
844
|
+
}
|
|
1922
845
|
if (message.id) {
|
|
1923
846
|
const replyId = createUniqueUuid(runtime, message.id);
|
|
1924
847
|
pendingTrajectoryStepByReplyId.set(replyId, trajectoryStepId);
|
|
1925
848
|
}
|
|
1926
849
|
} catch (err) {
|
|
1927
|
-
runtime.logger?.warn({
|
|
1928
|
-
err,
|
|
1929
|
-
src: "plugin-trajectory-logger",
|
|
1930
|
-
roomId: message.roomId
|
|
1931
|
-
}, "Failed to start trajectory logging");
|
|
850
|
+
runtime.logger?.warn({ err, src: "plugin-trajectory-logger", roomId: message.roomId }, "Failed to start trajectory logging");
|
|
1932
851
|
}
|
|
1933
852
|
}
|
|
1934
853
|
],
|
|
@@ -1952,11 +871,7 @@ var trajectoryLoggerPlugin = {
|
|
|
1952
871
|
const endTarget = pendingTrajectoryEndTargetByStepId.get(trajectoryStepId) ?? trajectoryStepId;
|
|
1953
872
|
await logger4.endTrajectory(endTarget, "completed");
|
|
1954
873
|
} catch (err) {
|
|
1955
|
-
runtime.logger?.warn({
|
|
1956
|
-
err,
|
|
1957
|
-
src: "plugin-trajectory-logger",
|
|
1958
|
-
trajectoryStepId
|
|
1959
|
-
}, "Failed to end trajectory logging");
|
|
874
|
+
runtime.logger?.warn({ err, src: "plugin-trajectory-logger", trajectoryStepId }, "Failed to end trajectory logging");
|
|
1960
875
|
}
|
|
1961
876
|
if (inReplyTo) {
|
|
1962
877
|
pendingTrajectoryStepByReplyId.delete(inReplyTo);
|
|
@@ -2007,4 +922,4 @@ export {
|
|
|
2007
922
|
RewardService
|
|
2008
923
|
};
|
|
2009
924
|
|
|
2010
|
-
//# debugId=
|
|
925
|
+
//# debugId=F0E0006C38150F8464756E2164756E21
|