@sixb/pg 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/agent-storage.d.ts +5 -3
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/checkpoints.d.ts +12 -0
- package/dist/agents/checkpoints.d.ts.map +1 -0
- package/dist/agents/index.d.ts +1 -0
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/messages.d.ts +2 -1
- package/dist/agents/messages.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +18 -1
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +847 -30
- package/dist/index.js.map +12 -10
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts +12 -0
- package/dist/pg-ai-cost-storage.d.ts.map +1 -0
- package/package.json +3 -3
- package/src/agents/agent-storage.ts +6 -3
- package/src/agents/checkpoints.ts +175 -0
- package/src/agents/index.ts +1 -0
- package/src/agents/messages.ts +71 -13
- package/src/agents/rows.ts +45 -0
- package/src/index.ts +7 -1
- package/src/migrations/026-ai-cost-accounting.sql +53 -0
- package/src/migrations/027-agent-context-checkpoints.sql +34 -0
- package/src/migrations/028-object-override-edit-times.sql +3 -0
- package/src/migrations.ts +10 -0
- package/src/ontology-storage/materialization-state.ts +2 -0
- package/src/ontology-storage/materialization-writer.ts +5 -3
- package/src/pg-ai-cost-storage.ts +707 -0
package/dist/index.js
CHANGED
|
@@ -16,8 +16,14 @@ import {
|
|
|
16
16
|
throwNestedStorageTransaction
|
|
17
17
|
} from "@sixb/core/storage";
|
|
18
18
|
|
|
19
|
-
// src/agents/
|
|
20
|
-
import {
|
|
19
|
+
// src/agents/checkpoints.ts
|
|
20
|
+
import {
|
|
21
|
+
agentContextCheckpointMatchesCreateInput,
|
|
22
|
+
assertAgentContextCheckpointAnchors,
|
|
23
|
+
assertAgentContextCheckpointAuthority,
|
|
24
|
+
assertAgentContextCheckpointReplayState,
|
|
25
|
+
assertCreateAgentContextCheckpointInput
|
|
26
|
+
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
21
27
|
import {
|
|
22
28
|
AgentStorageError
|
|
23
29
|
} from "@sixb/core/storage";
|
|
@@ -149,6 +155,28 @@ function rowToMessageRecord(row) {
|
|
|
149
155
|
completedAt: row.completed_at ? new Date(row.completed_at) : undefined
|
|
150
156
|
};
|
|
151
157
|
}
|
|
158
|
+
function rowToContextCheckpointRecord(row) {
|
|
159
|
+
const summaryFormatVersion = Number(row.summary_format_version);
|
|
160
|
+
if (summaryFormatVersion !== 1) {
|
|
161
|
+
throw new Error(`[SixbPg] Unsupported agent context checkpoint summary format version '${summaryFormatVersion}'.`);
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
id: row.id,
|
|
165
|
+
projectId: row.project_id,
|
|
166
|
+
threadId: row.thread_id,
|
|
167
|
+
createdByRunId: row.created_by_run_id,
|
|
168
|
+
previousCheckpointId: row.previous_checkpoint_id ?? undefined,
|
|
169
|
+
reason: row.reason,
|
|
170
|
+
summary: row.summary,
|
|
171
|
+
summaryFormatVersion: 1,
|
|
172
|
+
summarizedThroughSeq: Number(row.summarized_through_seq),
|
|
173
|
+
observedHeadSeq: Number(row.observed_head_seq),
|
|
174
|
+
estimatedInputTokensBefore: Number(row.estimated_input_tokens_before),
|
|
175
|
+
estimatedInputTokensAfter: Number(row.estimated_input_tokens_after),
|
|
176
|
+
summaryModelId: row.summary_model_id,
|
|
177
|
+
createdAt: new Date(row.created_at)
|
|
178
|
+
};
|
|
179
|
+
}
|
|
152
180
|
function principalFromColumns(type, id) {
|
|
153
181
|
return type && id ? { type, id } : undefined;
|
|
154
182
|
}
|
|
@@ -184,7 +212,144 @@ async function queryAgentList(input) {
|
|
|
184
212
|
return { rows, total, hasMore: offset + rows.length < total };
|
|
185
213
|
}
|
|
186
214
|
|
|
215
|
+
// src/agents/checkpoints.ts
|
|
216
|
+
class PgAgentContextCheckpointStore {
|
|
217
|
+
sql;
|
|
218
|
+
constructor(sql) {
|
|
219
|
+
this.sql = sql;
|
|
220
|
+
}
|
|
221
|
+
async create(input) {
|
|
222
|
+
assertCreateAgentContextCheckpointInput(input, "SixbPg");
|
|
223
|
+
try {
|
|
224
|
+
return await runPgTransaction(this.sql, async (tx) => {
|
|
225
|
+
const [runRow] = await tx`
|
|
226
|
+
SELECT * FROM agent_runs
|
|
227
|
+
WHERE project_id = ${input.projectId} AND id = ${input.createdByRunId}
|
|
228
|
+
FOR UPDATE
|
|
229
|
+
`;
|
|
230
|
+
const [threadRow] = await tx`
|
|
231
|
+
SELECT * FROM agent_threads
|
|
232
|
+
WHERE project_id = ${input.projectId} AND id = ${input.threadId}
|
|
233
|
+
FOR UPDATE
|
|
234
|
+
`;
|
|
235
|
+
assertAgentContextCheckpointAuthority({
|
|
236
|
+
create: input,
|
|
237
|
+
run: runRow ? rowToRunRecord(runRow) : null,
|
|
238
|
+
thread: threadRow ? rowToThreadRecord(threadRow) : null,
|
|
239
|
+
prefix: "SixbPg"
|
|
240
|
+
});
|
|
241
|
+
const [latestRow] = await tx`
|
|
242
|
+
SELECT * FROM agent_context_checkpoints
|
|
243
|
+
WHERE project_id = ${input.projectId} AND thread_id = ${input.threadId}
|
|
244
|
+
ORDER BY summarized_through_seq DESC, created_at DESC, id DESC
|
|
245
|
+
LIMIT 1
|
|
246
|
+
`;
|
|
247
|
+
const latest = latestRow ? rowToContextCheckpointRecord(latestRow) : null;
|
|
248
|
+
const [headRow] = await tx`
|
|
249
|
+
SELECT COALESCE(MAX(seq), 0) AS seq FROM agent_messages
|
|
250
|
+
WHERE project_id = ${input.projectId} AND thread_id = ${input.threadId}
|
|
251
|
+
`;
|
|
252
|
+
const headSeq = Number(headRow?.seq ?? 0);
|
|
253
|
+
const [existingForRunRow] = await tx`
|
|
254
|
+
SELECT * FROM agent_context_checkpoints
|
|
255
|
+
WHERE project_id = ${input.projectId} AND created_by_run_id = ${input.createdByRunId}
|
|
256
|
+
`;
|
|
257
|
+
if (existingForRunRow) {
|
|
258
|
+
const record = rowToContextCheckpointRecord(existingForRunRow);
|
|
259
|
+
if (agentContextCheckpointMatchesCreateInput(record, input)) {
|
|
260
|
+
assertAgentContextCheckpointReplayState({
|
|
261
|
+
create: input,
|
|
262
|
+
existing: record,
|
|
263
|
+
latest,
|
|
264
|
+
headSeq,
|
|
265
|
+
prefix: "SixbPg"
|
|
266
|
+
});
|
|
267
|
+
return record;
|
|
268
|
+
}
|
|
269
|
+
throw new AgentStorageError("invalid_state", `[SixbPg] Agent run '${input.createdByRunId}' already created a different context checkpoint.`);
|
|
270
|
+
}
|
|
271
|
+
const [duplicateId] = await tx`
|
|
272
|
+
SELECT 1 AS present FROM agent_context_checkpoints
|
|
273
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
274
|
+
`;
|
|
275
|
+
if (duplicateId) {
|
|
276
|
+
throw new AgentStorageError("duplicate_id", `[SixbPg] Agent context checkpoint '${input.id}' already exists for project '${input.projectId}'.`);
|
|
277
|
+
}
|
|
278
|
+
const [firstRetainedRow] = await tx`
|
|
279
|
+
SELECT * FROM agent_messages
|
|
280
|
+
WHERE
|
|
281
|
+
project_id = ${input.projectId}
|
|
282
|
+
AND thread_id = ${input.threadId}
|
|
283
|
+
AND seq > ${input.summarizedThroughSeq}
|
|
284
|
+
ORDER BY seq ASC
|
|
285
|
+
LIMIT 1
|
|
286
|
+
`;
|
|
287
|
+
assertAgentContextCheckpointAnchors({
|
|
288
|
+
create: input,
|
|
289
|
+
latest,
|
|
290
|
+
headSeq,
|
|
291
|
+
firstRetained: firstRetainedRow ? rowToMessageRecord(firstRetainedRow) : null,
|
|
292
|
+
prefix: "SixbPg"
|
|
293
|
+
});
|
|
294
|
+
const [row] = await tx`
|
|
295
|
+
INSERT INTO agent_context_checkpoints (
|
|
296
|
+
project_id,
|
|
297
|
+
id,
|
|
298
|
+
thread_id,
|
|
299
|
+
created_by_run_id,
|
|
300
|
+
previous_checkpoint_id,
|
|
301
|
+
reason,
|
|
302
|
+
summary,
|
|
303
|
+
summary_format_version,
|
|
304
|
+
summarized_through_seq,
|
|
305
|
+
observed_head_seq,
|
|
306
|
+
estimated_input_tokens_before,
|
|
307
|
+
estimated_input_tokens_after,
|
|
308
|
+
summary_model_id,
|
|
309
|
+
created_at
|
|
310
|
+
) VALUES (
|
|
311
|
+
${input.projectId},
|
|
312
|
+
${input.id},
|
|
313
|
+
${input.threadId},
|
|
314
|
+
${input.createdByRunId},
|
|
315
|
+
${input.expectedPreviousCheckpointId},
|
|
316
|
+
${input.reason},
|
|
317
|
+
${input.summary},
|
|
318
|
+
${input.summaryFormatVersion},
|
|
319
|
+
${input.summarizedThroughSeq},
|
|
320
|
+
${input.observedHeadSeq},
|
|
321
|
+
${input.estimatedInputTokensBefore},
|
|
322
|
+
${input.estimatedInputTokensAfter},
|
|
323
|
+
${input.summaryModelId},
|
|
324
|
+
${input.createdAt ?? new Date}
|
|
325
|
+
)
|
|
326
|
+
RETURNING *
|
|
327
|
+
`;
|
|
328
|
+
return rowToContextCheckpointRecord(row);
|
|
329
|
+
});
|
|
330
|
+
} catch (error) {
|
|
331
|
+
if (isUniqueViolation(error)) {
|
|
332
|
+
throw new AgentStorageError("duplicate_id", `[SixbPg] Agent context checkpoint '${input.id}' already exists for project '${input.projectId}'.`);
|
|
333
|
+
}
|
|
334
|
+
throw error;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
async getLatest(input) {
|
|
338
|
+
const [row] = await this.sql`
|
|
339
|
+
SELECT * FROM agent_context_checkpoints
|
|
340
|
+
WHERE project_id = ${input.projectId} AND thread_id = ${input.threadId}
|
|
341
|
+
ORDER BY summarized_through_seq DESC, created_at DESC, id DESC
|
|
342
|
+
LIMIT 1
|
|
343
|
+
`;
|
|
344
|
+
return row ? rowToContextCheckpointRecord(row) : null;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
187
348
|
// src/agents/messages.ts
|
|
349
|
+
import { AGENT_MESSAGE_CONTENT_VERSION } from "@sixb/core/internal/agents";
|
|
350
|
+
import {
|
|
351
|
+
AgentStorageError as AgentStorageError2
|
|
352
|
+
} from "@sixb/core/storage";
|
|
188
353
|
class PgAgentMessageStore {
|
|
189
354
|
sql;
|
|
190
355
|
constructor(sql) {
|
|
@@ -194,21 +359,23 @@ class PgAgentMessageStore {
|
|
|
194
359
|
const createdAt = input.createdAt ?? new Date;
|
|
195
360
|
try {
|
|
196
361
|
return await runPgTransaction(this.sql, async (tx) => {
|
|
362
|
+
if (input.runId !== null) {
|
|
363
|
+
const [run] = await tx`
|
|
364
|
+
SELECT id FROM agent_runs
|
|
365
|
+
WHERE project_id = ${input.projectId} AND id = ${input.runId}
|
|
366
|
+
FOR UPDATE
|
|
367
|
+
`;
|
|
368
|
+
if (!run) {
|
|
369
|
+
throw new AgentStorageError2("run_not_found", `[SixbPg] Agent run '${input.runId}' not found for project '${input.projectId}'.`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
197
372
|
const [thread] = await tx`
|
|
198
373
|
SELECT id FROM agent_threads
|
|
199
374
|
WHERE project_id = ${input.projectId} AND id = ${input.threadId}
|
|
200
375
|
FOR UPDATE
|
|
201
376
|
`;
|
|
202
377
|
if (!thread) {
|
|
203
|
-
throw new
|
|
204
|
-
}
|
|
205
|
-
if (input.runId !== null) {
|
|
206
|
-
const [run] = await tx`
|
|
207
|
-
SELECT id FROM agent_runs WHERE project_id = ${input.projectId} AND id = ${input.runId}
|
|
208
|
-
`;
|
|
209
|
-
if (!run) {
|
|
210
|
-
throw new AgentStorageError("run_not_found", `[SixbPg] Agent run '${input.runId}' not found for project '${input.projectId}'.`);
|
|
211
|
-
}
|
|
378
|
+
throw new AgentStorageError2("thread_not_found", `[SixbPg] Agent thread '${input.threadId}' not found for project '${input.projectId}'.`);
|
|
212
379
|
}
|
|
213
380
|
const [seqRow] = await tx`
|
|
214
381
|
SELECT COALESCE(MAX(seq), 0) + 1 AS next FROM agent_messages
|
|
@@ -255,11 +422,51 @@ class PgAgentMessageStore {
|
|
|
255
422
|
});
|
|
256
423
|
} catch (error) {
|
|
257
424
|
if (isUniqueViolation(error)) {
|
|
258
|
-
throw new
|
|
425
|
+
throw new AgentStorageError2("duplicate_id", `[SixbPg] Agent message '${input.id}' already exists for project '${input.projectId}'.`);
|
|
259
426
|
}
|
|
260
427
|
throw error;
|
|
261
428
|
}
|
|
262
429
|
}
|
|
430
|
+
async deleteByRunId(input) {
|
|
431
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
432
|
+
const [thread] = await tx`
|
|
433
|
+
SELECT id FROM agent_threads
|
|
434
|
+
WHERE project_id = ${input.projectId} AND id = ${input.threadId}
|
|
435
|
+
FOR UPDATE
|
|
436
|
+
`;
|
|
437
|
+
if (!thread) {
|
|
438
|
+
throw new AgentStorageError2("thread_not_found", `[SixbPg] Agent thread '${input.threadId}' not found for project '${input.projectId}'.`);
|
|
439
|
+
}
|
|
440
|
+
const deleted = await tx`
|
|
441
|
+
DELETE FROM agent_messages
|
|
442
|
+
WHERE project_id = ${input.projectId}
|
|
443
|
+
AND thread_id = ${input.threadId}
|
|
444
|
+
AND run_id = ${input.runId}
|
|
445
|
+
RETURNING id
|
|
446
|
+
`;
|
|
447
|
+
if (deleted.length === 0) {
|
|
448
|
+
return 0;
|
|
449
|
+
}
|
|
450
|
+
const updatedAt = new Date;
|
|
451
|
+
await tx`
|
|
452
|
+
UPDATE agent_threads
|
|
453
|
+
SET
|
|
454
|
+
message_count = (
|
|
455
|
+
SELECT COUNT(*) FROM agent_messages
|
|
456
|
+
WHERE project_id = ${input.projectId} AND thread_id = ${input.threadId}
|
|
457
|
+
),
|
|
458
|
+
last_message_at = (
|
|
459
|
+
SELECT created_at FROM agent_messages
|
|
460
|
+
WHERE project_id = ${input.projectId} AND thread_id = ${input.threadId}
|
|
461
|
+
ORDER BY seq DESC
|
|
462
|
+
LIMIT 1
|
|
463
|
+
),
|
|
464
|
+
updated_at = ${updatedAt}
|
|
465
|
+
WHERE project_id = ${input.projectId} AND id = ${input.threadId}
|
|
466
|
+
`;
|
|
467
|
+
return deleted.length;
|
|
468
|
+
});
|
|
469
|
+
}
|
|
263
470
|
async getById(params) {
|
|
264
471
|
const [row] = await this.sql`
|
|
265
472
|
SELECT * FROM agent_messages WHERE project_id = ${params.projectId} AND id = ${params.id}
|
|
@@ -273,6 +480,10 @@ class PgAgentMessageStore {
|
|
|
273
480
|
const whereClauses = ["project_id = $1", "thread_id = $2"];
|
|
274
481
|
const params = [input.projectId, input.threadId];
|
|
275
482
|
let index = 3;
|
|
483
|
+
if (input.afterSeq !== undefined) {
|
|
484
|
+
whereClauses.push(`seq > $${index++}`);
|
|
485
|
+
params.push(input.afterSeq);
|
|
486
|
+
}
|
|
276
487
|
if (input.roles) {
|
|
277
488
|
const placeholders = input.roles.map(() => `$${index++}`);
|
|
278
489
|
whereClauses.push(`role IN (${placeholders.join(", ")})`);
|
|
@@ -299,7 +510,7 @@ import { normalizeRequesterGroupIds } from "@sixb/core/internal/auth";
|
|
|
299
510
|
import { serializeSixbFailure } from "@sixb/core/internal/errors";
|
|
300
511
|
import {
|
|
301
512
|
AGENT_RUN_FAILURE_CODES as AGENT_RUN_FAILURE_CODES2,
|
|
302
|
-
AgentStorageError as
|
|
513
|
+
AgentStorageError as AgentStorageError3
|
|
303
514
|
} from "@sixb/core/storage";
|
|
304
515
|
|
|
305
516
|
// src/run-list-query.ts
|
|
@@ -384,10 +595,10 @@ class PgAgentRunStore {
|
|
|
384
595
|
FOR UPDATE
|
|
385
596
|
`;
|
|
386
597
|
if (!thread) {
|
|
387
|
-
throw new
|
|
598
|
+
throw new AgentStorageError3("thread_not_found", `[SixbPg] Agent thread '${input.threadId}' not found for project '${input.projectId}'.`);
|
|
388
599
|
}
|
|
389
600
|
if (thread.active_run_id !== null) {
|
|
390
|
-
throw new
|
|
601
|
+
throw new AgentStorageError3("active_run_exists", `[SixbPg] Agent thread '${input.threadId}' already has an active run '${thread.active_run_id}'.`);
|
|
391
602
|
}
|
|
392
603
|
const [row] = await tx`
|
|
393
604
|
INSERT INTO agent_runs (
|
|
@@ -423,7 +634,7 @@ class PgAgentRunStore {
|
|
|
423
634
|
});
|
|
424
635
|
} catch (error) {
|
|
425
636
|
if (isUniqueViolation(error)) {
|
|
426
|
-
throw new
|
|
637
|
+
throw new AgentStorageError3("duplicate_id", `[SixbPg] Agent run '${input.id}' already exists for project '${input.projectId}'.`);
|
|
427
638
|
}
|
|
428
639
|
throw error;
|
|
429
640
|
}
|
|
@@ -483,7 +694,7 @@ class PgAgentRunStore {
|
|
|
483
694
|
return runPgTransaction(this.sql, async (tx) => {
|
|
484
695
|
const run = await this.lockRunning(tx, input.projectId, input.id);
|
|
485
696
|
if (run.execution_token !== input.executionToken) {
|
|
486
|
-
throw new
|
|
697
|
+
throw new AgentStorageError3("execution_lost", `[SixbPg] Execution token is no longer current on agent run '${input.id}'.`);
|
|
487
698
|
}
|
|
488
699
|
const [row] = await tx`
|
|
489
700
|
UPDATE agent_runs
|
|
@@ -503,7 +714,7 @@ class PgAgentRunStore {
|
|
|
503
714
|
return runPgTransaction(this.sql, async (tx) => {
|
|
504
715
|
const run = await this.lockRunning(tx, input.projectId, input.id);
|
|
505
716
|
if (run.execution_token !== input.executionToken) {
|
|
506
|
-
throw new
|
|
717
|
+
throw new AgentStorageError3("execution_lost", `[SixbPg] Execution token is no longer current on agent run '${input.id}'.`);
|
|
507
718
|
}
|
|
508
719
|
const [row] = await tx`
|
|
509
720
|
UPDATE agent_runs
|
|
@@ -584,10 +795,10 @@ class PgAgentRunStore {
|
|
|
584
795
|
FOR UPDATE
|
|
585
796
|
`;
|
|
586
797
|
if (!row) {
|
|
587
|
-
throw new
|
|
798
|
+
throw new AgentStorageError3("run_not_found", `[SixbPg] Agent run '${id}' not found for project '${projectId}'.`);
|
|
588
799
|
}
|
|
589
800
|
if (row.status !== status) {
|
|
590
|
-
throw new
|
|
801
|
+
throw new AgentStorageError3("invalid_state", `[SixbPg] Agent run '${id}' is not ${status} (status '${row.status}').`);
|
|
591
802
|
}
|
|
592
803
|
return row;
|
|
593
804
|
}
|
|
@@ -601,7 +812,7 @@ class PgAgentRunStore {
|
|
|
601
812
|
|
|
602
813
|
// src/agents/threads.ts
|
|
603
814
|
import {
|
|
604
|
-
AgentStorageError as
|
|
815
|
+
AgentStorageError as AgentStorageError4
|
|
605
816
|
} from "@sixb/core/storage";
|
|
606
817
|
class PgAgentThreadStore {
|
|
607
818
|
sql;
|
|
@@ -639,7 +850,7 @@ class PgAgentThreadStore {
|
|
|
639
850
|
return rowToThreadRecord(row);
|
|
640
851
|
} catch (error) {
|
|
641
852
|
if (isUniqueViolation(error)) {
|
|
642
|
-
throw new
|
|
853
|
+
throw new AgentStorageError4("duplicate_id", `[SixbPg] Agent thread '${input.id}' already exists for project '${input.projectId}'.`);
|
|
643
854
|
}
|
|
644
855
|
throw error;
|
|
645
856
|
}
|
|
@@ -695,10 +906,12 @@ class PgAgentStorage {
|
|
|
695
906
|
threads;
|
|
696
907
|
runs;
|
|
697
908
|
messages;
|
|
909
|
+
checkpoints;
|
|
698
910
|
constructor(options) {
|
|
699
911
|
this.threads = new PgAgentThreadStore(options.sql);
|
|
700
912
|
this.runs = new PgAgentRunStore(options.sql, options.executions);
|
|
701
913
|
this.messages = new PgAgentMessageStore(options.sql);
|
|
914
|
+
this.checkpoints = new PgAgentContextCheckpointStore(options.sql);
|
|
702
915
|
}
|
|
703
916
|
}
|
|
704
917
|
// src/auth-storage/auth-storage.ts
|
|
@@ -5915,6 +6128,105 @@ CREATE INDEX idx_connector_connections_status
|
|
|
5915
6128
|
ON connector_connections (project_id, connector_id, status, id);
|
|
5916
6129
|
`;
|
|
5917
6130
|
|
|
6131
|
+
// src/migrations/026-ai-cost-accounting.sql
|
|
6132
|
+
var _026_ai_cost_accounting_default = `CREATE TABLE ai_model_call_valuations (
|
|
6133
|
+
project_id TEXT NOT NULL CHECK (length(btrim(project_id)) > 0),
|
|
6134
|
+
usage_record_id TEXT NOT NULL CHECK (length(btrim(usage_record_id)) > 0),
|
|
6135
|
+
status TEXT NOT NULL CHECK (status IN ('rated', 'unpriceable')),
|
|
6136
|
+
provider_id TEXT CHECK (provider_id IS NULL OR length(btrim(provider_id)) > 0),
|
|
6137
|
+
model_id TEXT CHECK (model_id IS NULL OR length(btrim(model_id)) > 0),
|
|
6138
|
+
currency TEXT CHECK (currency IS NULL OR currency ~ '^[A-Z]{3}$'),
|
|
6139
|
+
amount_nanos BIGINT CHECK (amount_nanos IS NULL OR amount_nanos >= 0),
|
|
6140
|
+
reason TEXT CHECK (
|
|
6141
|
+
reason IS NULL OR reason IN (
|
|
6142
|
+
'missingBillingIdentity',
|
|
6143
|
+
'missingCatalogEntry',
|
|
6144
|
+
'missingUsageMeter',
|
|
6145
|
+
'unsupportedPricingDimension',
|
|
6146
|
+
'invalidUsageForFormula'
|
|
6147
|
+
)
|
|
6148
|
+
),
|
|
6149
|
+
details JSONB NOT NULL CHECK (jsonb_typeof(details) = 'object'),
|
|
6150
|
+
rated_at TIMESTAMPTZ NOT NULL,
|
|
6151
|
+
PRIMARY KEY (project_id, usage_record_id),
|
|
6152
|
+
FOREIGN KEY (project_id, usage_record_id)
|
|
6153
|
+
REFERENCES ai_model_call_usage(project_id, id)
|
|
6154
|
+
ON DELETE RESTRICT,
|
|
6155
|
+
CHECK (
|
|
6156
|
+
(
|
|
6157
|
+
status = 'rated'
|
|
6158
|
+
AND provider_id IS NOT NULL
|
|
6159
|
+
AND model_id IS NOT NULL
|
|
6160
|
+
AND currency IS NOT NULL
|
|
6161
|
+
AND amount_nanos IS NOT NULL
|
|
6162
|
+
AND reason IS NULL
|
|
6163
|
+
)
|
|
6164
|
+
OR
|
|
6165
|
+
(
|
|
6166
|
+
status = 'unpriceable'
|
|
6167
|
+
AND currency IS NULL
|
|
6168
|
+
AND amount_nanos IS NULL
|
|
6169
|
+
AND reason IS NOT NULL
|
|
6170
|
+
AND (
|
|
6171
|
+
(reason = 'missingBillingIdentity' AND provider_id IS NULL AND model_id IS NULL)
|
|
6172
|
+
OR
|
|
6173
|
+
(reason <> 'missingBillingIdentity' AND provider_id IS NOT NULL AND model_id IS NOT NULL)
|
|
6174
|
+
)
|
|
6175
|
+
)
|
|
6176
|
+
)
|
|
6177
|
+
);
|
|
6178
|
+
|
|
6179
|
+
CREATE INDEX idx_ai_model_call_usage_provider_model_time
|
|
6180
|
+
ON ai_model_call_usage (project_id, provider_id, requested_model_id, occurred_at, id);
|
|
6181
|
+
CREATE INDEX idx_ai_model_call_usage_model_time
|
|
6182
|
+
ON ai_model_call_usage (project_id, requested_model_id, occurred_at, id);
|
|
6183
|
+
CREATE INDEX idx_ai_model_call_valuations_summary
|
|
6184
|
+
ON ai_model_call_valuations (project_id, status, currency, usage_record_id);
|
|
6185
|
+
`;
|
|
6186
|
+
|
|
6187
|
+
// src/migrations/027-agent-context-checkpoints.sql
|
|
6188
|
+
var _027_agent_context_checkpoints_default = `CREATE TABLE agent_context_checkpoints (
|
|
6189
|
+
project_id TEXT NOT NULL,
|
|
6190
|
+
id TEXT NOT NULL,
|
|
6191
|
+
thread_id TEXT NOT NULL,
|
|
6192
|
+
created_by_run_id TEXT NOT NULL,
|
|
6193
|
+
previous_checkpoint_id TEXT,
|
|
6194
|
+
reason TEXT NOT NULL CHECK (reason IN ('threshold', 'overflow')),
|
|
6195
|
+
summary TEXT NOT NULL CHECK (length(trim(summary)) > 0),
|
|
6196
|
+
summary_format_version INTEGER NOT NULL CHECK (summary_format_version = 1),
|
|
6197
|
+
summarized_through_seq INTEGER NOT NULL CHECK (summarized_through_seq >= 1),
|
|
6198
|
+
observed_head_seq INTEGER NOT NULL,
|
|
6199
|
+
estimated_input_tokens_before BIGINT NOT NULL CHECK (estimated_input_tokens_before >= 0),
|
|
6200
|
+
estimated_input_tokens_after BIGINT NOT NULL CHECK (estimated_input_tokens_after >= 0),
|
|
6201
|
+
summary_model_id TEXT NOT NULL CHECK (length(trim(summary_model_id)) > 0),
|
|
6202
|
+
created_at TIMESTAMPTZ NOT NULL,
|
|
6203
|
+
PRIMARY KEY (project_id, id),
|
|
6204
|
+
UNIQUE (project_id, created_by_run_id),
|
|
6205
|
+
FOREIGN KEY (project_id, thread_id)
|
|
6206
|
+
REFERENCES agent_threads(project_id, id) ON DELETE CASCADE,
|
|
6207
|
+
FOREIGN KEY (project_id, created_by_run_id)
|
|
6208
|
+
REFERENCES agent_runs(project_id, id) ON DELETE RESTRICT,
|
|
6209
|
+
FOREIGN KEY (project_id, previous_checkpoint_id)
|
|
6210
|
+
REFERENCES agent_context_checkpoints(project_id, id) ON DELETE RESTRICT,
|
|
6211
|
+
CHECK (summarized_through_seq < observed_head_seq)
|
|
6212
|
+
);
|
|
6213
|
+
|
|
6214
|
+
CREATE INDEX idx_agent_context_checkpoints_thread_latest
|
|
6215
|
+
ON agent_context_checkpoints (
|
|
6216
|
+
project_id,
|
|
6217
|
+
thread_id,
|
|
6218
|
+
summarized_through_seq DESC,
|
|
6219
|
+
created_at DESC,
|
|
6220
|
+
id DESC
|
|
6221
|
+
);
|
|
6222
|
+
`;
|
|
6223
|
+
|
|
6224
|
+
// src/migrations/028-object-override-edit-times.sql
|
|
6225
|
+
var _028_object_override_edit_times_default = `ALTER TABLE ontology_object_overrides
|
|
6226
|
+
ADD COLUMN edited_at JSONB NOT NULL DEFAULT '{}'::jsonb
|
|
6227
|
+
CHECK (jsonb_typeof(edited_at) = 'object');
|
|
6228
|
+
`;
|
|
6229
|
+
|
|
5918
6230
|
// src/migrations.ts
|
|
5919
6231
|
var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
|
|
5920
6232
|
function pgSql(id, sqlText) {
|
|
@@ -6114,7 +6426,10 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
6114
6426
|
pgSql("022-projection-executions", _022_projection_executions_default),
|
|
6115
6427
|
pgSql("023-webhook-executions", _023_webhook_executions_default),
|
|
6116
6428
|
pgSql("024-ontology-commit-executions", _024_ontology_commit_executions_default),
|
|
6117
|
-
pgSql("025-connector-connections", _025_connector_connections_default)
|
|
6429
|
+
pgSql("025-connector-connections", _025_connector_connections_default),
|
|
6430
|
+
pgSql("026-ai-cost-accounting", _026_ai_cost_accounting_default),
|
|
6431
|
+
pgSql("027-agent-context-checkpoints", _027_agent_context_checkpoints_default),
|
|
6432
|
+
pgSql("028-object-override-edit-times", _028_object_override_edit_times_default)
|
|
6118
6433
|
]
|
|
6119
6434
|
});
|
|
6120
6435
|
|
|
@@ -7159,6 +7474,7 @@ function storedObjectOverride(row) {
|
|
|
7159
7474
|
return {
|
|
7160
7475
|
ref,
|
|
7161
7476
|
value: structuredClone(row.value),
|
|
7477
|
+
editedAt: structuredClone(row.edited_at),
|
|
7162
7478
|
lastCommitId: row.last_commit_id,
|
|
7163
7479
|
updatedAt: toIsoString2(row.updated_at)
|
|
7164
7480
|
};
|
|
@@ -7592,6 +7908,7 @@ class PgMaterializationWriter {
|
|
|
7592
7908
|
objectTypeId: item.ref.objectTypeId,
|
|
7593
7909
|
primaryId: item.ref.primaryId,
|
|
7594
7910
|
value: item.value,
|
|
7911
|
+
editedAt: item.editedAt,
|
|
7595
7912
|
lastCommitId: item.lastCommitId,
|
|
7596
7913
|
updatedAt: item.updatedAt,
|
|
7597
7914
|
expectedLastCommitId: item.expectedLastCommitId
|
|
@@ -7603,10 +7920,10 @@ class PgMaterializationWriter {
|
|
|
7603
7920
|
SELECT value FROM jsonb_array_elements(${jsonParameter2(this.sql, objectInserts)}::jsonb)
|
|
7604
7921
|
)
|
|
7605
7922
|
INSERT INTO ontology_object_overrides (
|
|
7606
|
-
project_id, object_type_id, primary_id, value, last_commit_id, updated_at
|
|
7923
|
+
project_id, object_type_id, primary_id, value, edited_at, last_commit_id, updated_at
|
|
7607
7924
|
)
|
|
7608
7925
|
SELECT ${projectId}, value->>'objectTypeId', value->>'primaryId', value->'value',
|
|
7609
|
-
value->>'lastCommitId',
|
|
7926
|
+
value->'editedAt', value->>'lastCommitId',
|
|
7610
7927
|
(value->>'updatedAt')::timestamptz
|
|
7611
7928
|
FROM staged
|
|
7612
7929
|
ON CONFLICT DO NOTHING
|
|
@@ -7622,7 +7939,8 @@ class PgMaterializationWriter {
|
|
|
7622
7939
|
SELECT value FROM jsonb_array_elements(${jsonParameter2(this.sql, objectUpdates)}::jsonb)
|
|
7623
7940
|
)
|
|
7624
7941
|
UPDATE ontology_object_overrides AS overrides
|
|
7625
|
-
SET value = staged.value->'value',
|
|
7942
|
+
SET value = staged.value->'value', edited_at = staged.value->'editedAt',
|
|
7943
|
+
last_commit_id = staged.value->>'lastCommitId',
|
|
7626
7944
|
updated_at = (staged.value->>'updatedAt')::timestamptz
|
|
7627
7945
|
FROM staged
|
|
7628
7946
|
WHERE overrides.project_id = ${projectId}
|
|
@@ -9811,6 +10129,502 @@ function rowToActionSubject(row) {
|
|
|
9811
10129
|
};
|
|
9812
10130
|
}
|
|
9813
10131
|
|
|
10132
|
+
// src/pg-ai-cost-storage.ts
|
|
10133
|
+
import {
|
|
10134
|
+
aiModelCallCostDetails,
|
|
10135
|
+
aiModelCallCostMatchesUsage,
|
|
10136
|
+
buildAiAccountingOverviewFromFragments,
|
|
10137
|
+
normalizeAiAccountingQuery,
|
|
10138
|
+
normalizeAiModelCallAccountingQuery,
|
|
10139
|
+
normalizeAiModelCallCostRecord,
|
|
10140
|
+
parseAiModelCallCostDetails,
|
|
10141
|
+
toAccountingItem
|
|
10142
|
+
} from "@sixb/core/internal/ai-cost-storage-provider";
|
|
10143
|
+
import {
|
|
10144
|
+
AiCostStorageError
|
|
10145
|
+
} from "@sixb/core/storage";
|
|
10146
|
+
class PgAiCostStorage {
|
|
10147
|
+
sql;
|
|
10148
|
+
constructor(sql) {
|
|
10149
|
+
this.sql = sql;
|
|
10150
|
+
}
|
|
10151
|
+
async recordModelCallCost(input) {
|
|
10152
|
+
const record = normalizeAiModelCallCostRecord(input);
|
|
10153
|
+
await runPgTransaction(this.sql, async (tx) => {
|
|
10154
|
+
const usage = await findUsage(tx, record.projectId, record.usageRecordId);
|
|
10155
|
+
if (!usage)
|
|
10156
|
+
throw missingUsage(record);
|
|
10157
|
+
assertCostMatchesUsage(record, usage);
|
|
10158
|
+
if (await findCost(tx, record.projectId, record.usageRecordId))
|
|
10159
|
+
return;
|
|
10160
|
+
const rated = record.status === "rated" ? record : undefined;
|
|
10161
|
+
const inserted = await tx`
|
|
10162
|
+
INSERT INTO ai_model_call_valuations (
|
|
10163
|
+
project_id, usage_record_id, status, provider_id, model_id,
|
|
10164
|
+
currency, amount_nanos, reason, details, rated_at
|
|
10165
|
+
) VALUES (
|
|
10166
|
+
${record.projectId}, ${record.usageRecordId}, ${record.status},
|
|
10167
|
+
${record.billingIdentity?.providerId ?? null},
|
|
10168
|
+
${record.billingIdentity?.modelId ?? null}, ${rated?.money.currency ?? null},
|
|
10169
|
+
${rated?.money.amountNanos ?? null},
|
|
10170
|
+
${record.status === "unpriceable" ? record.reason : null},
|
|
10171
|
+
${JSON.stringify(aiModelCallCostDetails(record))}::text::jsonb, ${record.ratedAt}
|
|
10172
|
+
)
|
|
10173
|
+
ON CONFLICT DO NOTHING
|
|
10174
|
+
RETURNING usage_record_id
|
|
10175
|
+
`;
|
|
10176
|
+
if (inserted.length === 0 && !await findCost(tx, record.projectId, record.usageRecordId)) {
|
|
10177
|
+
throw new Error("[SixbPg] AI valuation insert conflicted without a visible row.");
|
|
10178
|
+
}
|
|
10179
|
+
});
|
|
10180
|
+
}
|
|
10181
|
+
async summarizeExecutions(input) {
|
|
10182
|
+
assertNonBlank(input.projectId, "cost summary projectId");
|
|
10183
|
+
for (const id of input.executionIds)
|
|
10184
|
+
assertNonBlank(id, "cost summary executionId");
|
|
10185
|
+
if (input.executionIds.length === 0)
|
|
10186
|
+
return [];
|
|
10187
|
+
const ids = [...new Set(input.executionIds)];
|
|
10188
|
+
const rows = await this.sql`
|
|
10189
|
+
WITH requested(execution_id) AS (
|
|
10190
|
+
SELECT DISTINCT * FROM unnest(${this.sql.array(ids)}::text[])
|
|
10191
|
+
)
|
|
10192
|
+
SELECT usage.execution_id,
|
|
10193
|
+
cost.status,
|
|
10194
|
+
cost.currency,
|
|
10195
|
+
cost.amount_nanos
|
|
10196
|
+
FROM requested
|
|
10197
|
+
JOIN ai_model_call_usage AS usage
|
|
10198
|
+
ON usage.project_id = ${input.projectId} AND usage.execution_id = requested.execution_id
|
|
10199
|
+
LEFT JOIN ai_model_call_valuations AS cost
|
|
10200
|
+
ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
|
|
10201
|
+
`;
|
|
10202
|
+
return aggregateSummaryRows(input.executionIds, rows);
|
|
10203
|
+
}
|
|
10204
|
+
async queryProjectOverview(input) {
|
|
10205
|
+
const query = normalizeAiAccountingQuery(input);
|
|
10206
|
+
const providerId = query.providerId ?? null;
|
|
10207
|
+
const modelId = query.modelId ?? null;
|
|
10208
|
+
const rows = await this.sql`
|
|
10209
|
+
WITH filtered AS (
|
|
10210
|
+
SELECT usage.provider_id,
|
|
10211
|
+
usage.requested_model_id,
|
|
10212
|
+
usage.occurred_at,
|
|
10213
|
+
usage.input_tokens,
|
|
10214
|
+
usage.output_tokens,
|
|
10215
|
+
usage.uncached_input_tokens,
|
|
10216
|
+
usage.cache_read_input_tokens,
|
|
10217
|
+
usage.cache_write_input_tokens,
|
|
10218
|
+
usage.text_output_tokens,
|
|
10219
|
+
usage.reasoning_output_tokens,
|
|
10220
|
+
usage.reporting_status,
|
|
10221
|
+
COALESCE(direct_agent.agent_id, workflow_agent.agent_id) AS attribution_agent_id,
|
|
10222
|
+
workflow_node.workflow_id AS attribution_workflow_id,
|
|
10223
|
+
cost.status AS cost_status,
|
|
10224
|
+
cost.currency AS cost_currency,
|
|
10225
|
+
cost.amount_nanos AS cost_amount_nanos
|
|
10226
|
+
FROM ai_model_call_usage AS usage
|
|
10227
|
+
LEFT JOIN agent_runs AS direct_agent
|
|
10228
|
+
ON direct_agent.project_id = usage.project_id
|
|
10229
|
+
AND direct_agent.execution_id = usage.execution_id
|
|
10230
|
+
LEFT JOIN workflow_agent_node_runs AS workflow_agent
|
|
10231
|
+
ON workflow_agent.project_id = usage.project_id
|
|
10232
|
+
AND workflow_agent.execution_id = usage.execution_id
|
|
10233
|
+
LEFT JOIN workflow_node_runs AS workflow_node
|
|
10234
|
+
ON workflow_node.project_id = workflow_agent.project_id
|
|
10235
|
+
AND workflow_node.id = workflow_agent.node_run_id
|
|
10236
|
+
LEFT JOIN ai_model_call_valuations AS cost
|
|
10237
|
+
ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
|
|
10238
|
+
WHERE usage.project_id = ${query.projectId}
|
|
10239
|
+
AND usage.occurred_at >= ${query.from} AND usage.occurred_at < ${query.to}
|
|
10240
|
+
AND (${providerId}::text IS NULL OR usage.provider_id = ${providerId})
|
|
10241
|
+
AND (${modelId}::text IS NULL OR usage.requested_model_id = ${modelId})
|
|
10242
|
+
), classified AS (
|
|
10243
|
+
SELECT filtered.*,
|
|
10244
|
+
CASE
|
|
10245
|
+
WHEN cost_status = 'rated' THEN 'rated'
|
|
10246
|
+
WHEN cost_status = 'unpriceable' THEN 'unpriceable'
|
|
10247
|
+
ELSE 'unvalued'
|
|
10248
|
+
END AS valuation_status,
|
|
10249
|
+
CASE WHEN cost_status = 'rated' THEN cost_currency ELSE NULL END AS amount_currency,
|
|
10250
|
+
CASE WHEN cost_status = 'rated' THEN cost_amount_nanos ELSE NULL END AS amount_nanos
|
|
10251
|
+
FROM filtered
|
|
10252
|
+
), expanded AS (
|
|
10253
|
+
SELECT 'totals'::text AS dimension,
|
|
10254
|
+
NULL::text AS key_1,
|
|
10255
|
+
NULL::text AS key_2,
|
|
10256
|
+
NULL::timestamptz AS bucket_start,
|
|
10257
|
+
classified.*
|
|
10258
|
+
FROM classified
|
|
10259
|
+
UNION ALL
|
|
10260
|
+
SELECT 'series', NULL, NULL,
|
|
10261
|
+
CASE ${input.bucket}::text
|
|
10262
|
+
WHEN 'hour' THEN
|
|
10263
|
+
date_trunc('hour', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
10264
|
+
WHEN 'day' THEN
|
|
10265
|
+
date_trunc('day', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
10266
|
+
ELSE date_trunc('week', occurred_at AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
|
|
10267
|
+
END,
|
|
10268
|
+
classified.*
|
|
10269
|
+
FROM classified
|
|
10270
|
+
UNION ALL
|
|
10271
|
+
SELECT 'model', provider_id, requested_model_id, NULL, classified.* FROM classified
|
|
10272
|
+
UNION ALL
|
|
10273
|
+
SELECT 'agent', attribution_agent_id, NULL, NULL, classified.*
|
|
10274
|
+
FROM classified WHERE attribution_agent_id IS NOT NULL
|
|
10275
|
+
UNION ALL
|
|
10276
|
+
SELECT 'workflow', attribution_workflow_id, NULL, NULL, classified.*
|
|
10277
|
+
FROM classified WHERE attribution_workflow_id IS NOT NULL
|
|
10278
|
+
)
|
|
10279
|
+
SELECT dimension,
|
|
10280
|
+
key_1,
|
|
10281
|
+
key_2,
|
|
10282
|
+
bucket_start,
|
|
10283
|
+
amount_currency,
|
|
10284
|
+
COUNT(*)::text AS model_call_count,
|
|
10285
|
+
COUNT(*) FILTER (WHERE reporting_status <> 'unavailable')::text
|
|
10286
|
+
AS reported_usage_call_count,
|
|
10287
|
+
COUNT(input_tokens)::text AS input_tokens_count,
|
|
10288
|
+
SUM(input_tokens)::text AS input_tokens_sum,
|
|
10289
|
+
COUNT(output_tokens)::text AS output_tokens_count,
|
|
10290
|
+
SUM(output_tokens)::text AS output_tokens_sum,
|
|
10291
|
+
COUNT(uncached_input_tokens)::text AS uncached_input_tokens_count,
|
|
10292
|
+
SUM(uncached_input_tokens)::text AS uncached_input_tokens_sum,
|
|
10293
|
+
COUNT(cache_read_input_tokens)::text AS cache_read_input_tokens_count,
|
|
10294
|
+
SUM(cache_read_input_tokens)::text AS cache_read_input_tokens_sum,
|
|
10295
|
+
COUNT(cache_write_input_tokens)::text AS cache_write_input_tokens_count,
|
|
10296
|
+
SUM(cache_write_input_tokens)::text AS cache_write_input_tokens_sum,
|
|
10297
|
+
COUNT(text_output_tokens)::text AS text_output_tokens_count,
|
|
10298
|
+
SUM(text_output_tokens)::text AS text_output_tokens_sum,
|
|
10299
|
+
COUNT(reasoning_output_tokens)::text AS reasoning_output_tokens_count,
|
|
10300
|
+
SUM(reasoning_output_tokens)::text AS reasoning_output_tokens_sum,
|
|
10301
|
+
COUNT(*) FILTER (WHERE valuation_status = 'rated')::text AS rated_call_count,
|
|
10302
|
+
COUNT(*) FILTER (WHERE valuation_status = 'unpriceable')::text
|
|
10303
|
+
AS unpriceable_call_count,
|
|
10304
|
+
COUNT(*) FILTER (WHERE valuation_status = 'unvalued')::text
|
|
10305
|
+
AS unvalued_call_count,
|
|
10306
|
+
SUM(amount_nanos)::text AS amount_nanos
|
|
10307
|
+
FROM expanded
|
|
10308
|
+
GROUP BY dimension, key_1, key_2, bucket_start, amount_currency
|
|
10309
|
+
`;
|
|
10310
|
+
return buildAiAccountingOverviewFromFragments(input, rows.map(aggregateFragmentFromRow));
|
|
10311
|
+
}
|
|
10312
|
+
async listModelCalls(input) {
|
|
10313
|
+
const query = normalizeAiModelCallAccountingQuery(input);
|
|
10314
|
+
const providerId = query.providerId ?? null;
|
|
10315
|
+
const modelId = query.modelId ?? null;
|
|
10316
|
+
const executionId = query.executionId ?? null;
|
|
10317
|
+
const valuationStatus = query.valuationStatus ?? null;
|
|
10318
|
+
const [count] = await this.sql`
|
|
10319
|
+
SELECT COUNT(*)::text AS total
|
|
10320
|
+
FROM ai_model_call_usage AS usage
|
|
10321
|
+
LEFT JOIN ai_model_call_valuations AS cost
|
|
10322
|
+
ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
|
|
10323
|
+
WHERE usage.project_id = ${query.projectId}
|
|
10324
|
+
AND usage.occurred_at >= ${query.from} AND usage.occurred_at < ${query.to}
|
|
10325
|
+
AND (${providerId}::text IS NULL OR usage.provider_id = ${providerId})
|
|
10326
|
+
AND (${modelId}::text IS NULL OR usage.requested_model_id = ${modelId})
|
|
10327
|
+
AND (${executionId}::text IS NULL OR usage.execution_id = ${executionId})
|
|
10328
|
+
AND (
|
|
10329
|
+
${valuationStatus}::text IS NULL OR ${valuationStatus} = CASE
|
|
10330
|
+
WHEN cost.status = 'rated' THEN 'rated'
|
|
10331
|
+
WHEN cost.status = 'unpriceable' THEN 'unpriceable'
|
|
10332
|
+
ELSE 'unvalued'
|
|
10333
|
+
END
|
|
10334
|
+
)
|
|
10335
|
+
`;
|
|
10336
|
+
const rows = await this.sql`
|
|
10337
|
+
SELECT usage.*,
|
|
10338
|
+
CASE
|
|
10339
|
+
WHEN direct_agent.id IS NOT NULL THEN 'agent'
|
|
10340
|
+
WHEN workflow_agent.node_run_id IS NOT NULL THEN 'workflowAgent'
|
|
10341
|
+
ELSE NULL
|
|
10342
|
+
END AS attribution_kind,
|
|
10343
|
+
COALESCE(direct_agent.agent_id, workflow_agent.agent_id) AS attribution_agent_id,
|
|
10344
|
+
direct_agent.id AS attribution_agent_run_id,
|
|
10345
|
+
direct_agent.thread_id AS attribution_thread_id,
|
|
10346
|
+
workflow_agent.node_run_id AS attribution_node_run_id,
|
|
10347
|
+
workflow_node.workflow_id AS attribution_workflow_id,
|
|
10348
|
+
workflow_node.workflow_run_id AS attribution_workflow_run_id,
|
|
10349
|
+
COALESCE((
|
|
10350
|
+
SELECT jsonb_agg(groups.group_id ORDER BY groups.group_id)
|
|
10351
|
+
FROM ai_model_call_usage_groups AS groups
|
|
10352
|
+
WHERE groups.project_id = usage.project_id AND groups.usage_record_id = usage.id
|
|
10353
|
+
), '[]'::jsonb) AS requester_group_ids,
|
|
10354
|
+
cost.status AS cost_status,
|
|
10355
|
+
cost.provider_id AS cost_provider_id,
|
|
10356
|
+
cost.model_id AS cost_model_id,
|
|
10357
|
+
cost.currency AS cost_currency,
|
|
10358
|
+
cost.amount_nanos AS cost_amount_nanos,
|
|
10359
|
+
cost.reason AS cost_reason,
|
|
10360
|
+
cost.details AS cost_details,
|
|
10361
|
+
cost.rated_at AS cost_rated_at
|
|
10362
|
+
FROM ai_model_call_usage AS usage
|
|
10363
|
+
LEFT JOIN agent_runs AS direct_agent
|
|
10364
|
+
ON direct_agent.project_id = usage.project_id
|
|
10365
|
+
AND direct_agent.execution_id = usage.execution_id
|
|
10366
|
+
LEFT JOIN workflow_agent_node_runs AS workflow_agent
|
|
10367
|
+
ON workflow_agent.project_id = usage.project_id
|
|
10368
|
+
AND workflow_agent.execution_id = usage.execution_id
|
|
10369
|
+
LEFT JOIN workflow_node_runs AS workflow_node
|
|
10370
|
+
ON workflow_node.project_id = workflow_agent.project_id
|
|
10371
|
+
AND workflow_node.id = workflow_agent.node_run_id
|
|
10372
|
+
LEFT JOIN ai_model_call_valuations AS cost
|
|
10373
|
+
ON cost.project_id = usage.project_id AND cost.usage_record_id = usage.id
|
|
10374
|
+
WHERE usage.project_id = ${query.projectId}
|
|
10375
|
+
AND usage.occurred_at >= ${query.from} AND usage.occurred_at < ${query.to}
|
|
10376
|
+
AND (${providerId}::text IS NULL OR usage.provider_id = ${providerId})
|
|
10377
|
+
AND (${modelId}::text IS NULL OR usage.requested_model_id = ${modelId})
|
|
10378
|
+
AND (${executionId}::text IS NULL OR usage.execution_id = ${executionId})
|
|
10379
|
+
AND (
|
|
10380
|
+
${valuationStatus}::text IS NULL OR ${valuationStatus} = CASE
|
|
10381
|
+
WHEN cost.status = 'rated' THEN 'rated'
|
|
10382
|
+
WHEN cost.status = 'unpriceable' THEN 'unpriceable'
|
|
10383
|
+
ELSE 'unvalued'
|
|
10384
|
+
END
|
|
10385
|
+
)
|
|
10386
|
+
ORDER BY usage.occurred_at DESC, usage.id ASC
|
|
10387
|
+
LIMIT ${query.limit} OFFSET ${query.offset}
|
|
10388
|
+
`;
|
|
10389
|
+
const items = rows.map((row) => toAccountingItem(accountingItemFromRow(row)));
|
|
10390
|
+
if (!count)
|
|
10391
|
+
throw new Error("[SixbPg] AI accounting count query returned no row.");
|
|
10392
|
+
const total = safeSqlCount(count.total, "model-call list total");
|
|
10393
|
+
return structuredClone({
|
|
10394
|
+
items,
|
|
10395
|
+
total,
|
|
10396
|
+
hasMore: query.offset + items.length < total
|
|
10397
|
+
});
|
|
10398
|
+
}
|
|
10399
|
+
}
|
|
10400
|
+
async function findCost(sql, projectId, usageId) {
|
|
10401
|
+
const [row] = await sql`
|
|
10402
|
+
SELECT * FROM ai_model_call_valuations
|
|
10403
|
+
WHERE project_id = ${projectId} AND usage_record_id = ${usageId}
|
|
10404
|
+
`;
|
|
10405
|
+
return row ? rowToCost(row) : null;
|
|
10406
|
+
}
|
|
10407
|
+
async function findUsage(sql, projectId, id) {
|
|
10408
|
+
const [row] = await sql`
|
|
10409
|
+
SELECT * FROM ai_model_call_usage WHERE project_id = ${projectId} AND id = ${id}
|
|
10410
|
+
`;
|
|
10411
|
+
return row ? usageFromRow(row, []) : null;
|
|
10412
|
+
}
|
|
10413
|
+
function aggregateFragmentFromRow(row) {
|
|
10414
|
+
const dimensionKeys = row.dimension === "model" ? { providerId: required2(row.key_1), modelId: required2(row.key_2) } : row.dimension === "agent" ? { agentId: required2(row.key_1) } : row.dimension === "workflow" ? { workflowId: required2(row.key_1) } : row.dimension === "series" ? { start: new Date(required2(row.bucket_start)) } : {};
|
|
10415
|
+
return {
|
|
10416
|
+
dimension: row.dimension,
|
|
10417
|
+
...dimensionKeys,
|
|
10418
|
+
modelCallCount: row.model_call_count,
|
|
10419
|
+
usage: {
|
|
10420
|
+
reportedCallCount: row.reported_usage_call_count,
|
|
10421
|
+
inputTokens: meterFragment(row.input_tokens_count, row.input_tokens_sum),
|
|
10422
|
+
outputTokens: meterFragment(row.output_tokens_count, row.output_tokens_sum),
|
|
10423
|
+
uncachedInputTokens: meterFragment(row.uncached_input_tokens_count, row.uncached_input_tokens_sum),
|
|
10424
|
+
cacheReadInputTokens: meterFragment(row.cache_read_input_tokens_count, row.cache_read_input_tokens_sum),
|
|
10425
|
+
cacheWriteInputTokens: meterFragment(row.cache_write_input_tokens_count, row.cache_write_input_tokens_sum),
|
|
10426
|
+
textOutputTokens: meterFragment(row.text_output_tokens_count, row.text_output_tokens_sum),
|
|
10427
|
+
reasoningOutputTokens: meterFragment(row.reasoning_output_tokens_count, row.reasoning_output_tokens_sum)
|
|
10428
|
+
},
|
|
10429
|
+
costs: {
|
|
10430
|
+
...row.amount_currency === null || row.amount_nanos === null ? {} : {
|
|
10431
|
+
amount: {
|
|
10432
|
+
currency: row.amount_currency,
|
|
10433
|
+
amountNanos: row.amount_nanos
|
|
10434
|
+
}
|
|
10435
|
+
},
|
|
10436
|
+
ratedCallCount: row.rated_call_count,
|
|
10437
|
+
unpriceableCallCount: row.unpriceable_call_count,
|
|
10438
|
+
unvaluedCallCount: row.unvalued_call_count
|
|
10439
|
+
}
|
|
10440
|
+
};
|
|
10441
|
+
}
|
|
10442
|
+
function meterFragment(presentCallCount, total) {
|
|
10443
|
+
return { presentCallCount, total };
|
|
10444
|
+
}
|
|
10445
|
+
function accountingItemFromRow(row) {
|
|
10446
|
+
const usage = usageFromRow(row, parseStringArray(row.requester_group_ids));
|
|
10447
|
+
const cost = row.cost_status === null ? undefined : costFromJoinedRow(row);
|
|
10448
|
+
const attribution = row.attribution_kind === null ? undefined : row.attribution_kind === "agent" ? {
|
|
10449
|
+
kind: "agent",
|
|
10450
|
+
agentId: required2(row.attribution_agent_id),
|
|
10451
|
+
agentRunId: required2(row.attribution_agent_run_id),
|
|
10452
|
+
threadId: required2(row.attribution_thread_id)
|
|
10453
|
+
} : {
|
|
10454
|
+
kind: "workflowAgent",
|
|
10455
|
+
agentId: required2(row.attribution_agent_id),
|
|
10456
|
+
nodeRunId: required2(row.attribution_node_run_id),
|
|
10457
|
+
workflowId: required2(row.attribution_workflow_id),
|
|
10458
|
+
workflowRunId: required2(row.attribution_workflow_run_id)
|
|
10459
|
+
};
|
|
10460
|
+
return {
|
|
10461
|
+
usage,
|
|
10462
|
+
...attribution ? { attribution } : {},
|
|
10463
|
+
...cost ? { cost } : {}
|
|
10464
|
+
};
|
|
10465
|
+
}
|
|
10466
|
+
function usageFromRow(row, requesterGroupIds) {
|
|
10467
|
+
const number = (value) => value === null ? undefined : Number(value);
|
|
10468
|
+
return {
|
|
10469
|
+
id: row.id,
|
|
10470
|
+
projectId: row.project_id,
|
|
10471
|
+
executionId: row.execution_id,
|
|
10472
|
+
attempt: Number(row.attempt),
|
|
10473
|
+
callId: row.call_id,
|
|
10474
|
+
requesterGroupIds,
|
|
10475
|
+
providerId: row.provider_id,
|
|
10476
|
+
requestedModelId: row.requested_model_id,
|
|
10477
|
+
...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
|
|
10478
|
+
responseId: row.response_id,
|
|
10479
|
+
usage: {
|
|
10480
|
+
...number(row.input_tokens) === undefined ? {} : { inputTokens: number(row.input_tokens) },
|
|
10481
|
+
...number(row.output_tokens) === undefined ? {} : { outputTokens: number(row.output_tokens) },
|
|
10482
|
+
...number(row.total_tokens) === undefined ? {} : { totalTokens: number(row.total_tokens) },
|
|
10483
|
+
...number(row.uncached_input_tokens) === undefined ? {} : { uncachedInputTokens: number(row.uncached_input_tokens) },
|
|
10484
|
+
...number(row.cache_read_input_tokens) === undefined ? {} : { cacheReadInputTokens: number(row.cache_read_input_tokens) },
|
|
10485
|
+
...number(row.cache_write_input_tokens) === undefined ? {} : { cacheWriteInputTokens: number(row.cache_write_input_tokens) },
|
|
10486
|
+
...number(row.text_output_tokens) === undefined ? {} : { textOutputTokens: number(row.text_output_tokens) },
|
|
10487
|
+
...number(row.reasoning_output_tokens) === undefined ? {} : { reasoningOutputTokens: number(row.reasoning_output_tokens) },
|
|
10488
|
+
reportingStatus: row.reporting_status
|
|
10489
|
+
},
|
|
10490
|
+
...row.raw_usage === null ? {} : { rawUsage: parseJsonObject(row.raw_usage) },
|
|
10491
|
+
occurredAt: new Date(row.occurred_at),
|
|
10492
|
+
recordedAt: new Date(row.recorded_at)
|
|
10493
|
+
};
|
|
10494
|
+
}
|
|
10495
|
+
function rowToCost(row) {
|
|
10496
|
+
return costRecord({
|
|
10497
|
+
projectId: row.project_id,
|
|
10498
|
+
usageId: row.usage_record_id,
|
|
10499
|
+
status: row.status,
|
|
10500
|
+
providerId: row.provider_id,
|
|
10501
|
+
modelId: row.model_id,
|
|
10502
|
+
currency: row.currency,
|
|
10503
|
+
amountNanos: row.amount_nanos,
|
|
10504
|
+
reason: row.reason,
|
|
10505
|
+
details: row.details,
|
|
10506
|
+
ratedAt: row.rated_at
|
|
10507
|
+
});
|
|
10508
|
+
}
|
|
10509
|
+
function costFromJoinedRow(row) {
|
|
10510
|
+
return costRecord({
|
|
10511
|
+
projectId: row.project_id,
|
|
10512
|
+
usageId: row.id,
|
|
10513
|
+
status: required2(row.cost_status),
|
|
10514
|
+
providerId: row.cost_provider_id,
|
|
10515
|
+
modelId: row.cost_model_id,
|
|
10516
|
+
currency: row.cost_currency,
|
|
10517
|
+
amountNanos: row.cost_amount_nanos,
|
|
10518
|
+
reason: row.cost_reason,
|
|
10519
|
+
details: row.cost_details,
|
|
10520
|
+
ratedAt: required2(row.cost_rated_at)
|
|
10521
|
+
});
|
|
10522
|
+
}
|
|
10523
|
+
function costRecord(input) {
|
|
10524
|
+
const details = parseAiModelCallCostDetails(input.details);
|
|
10525
|
+
const base = {
|
|
10526
|
+
projectId: input.projectId,
|
|
10527
|
+
usageRecordId: input.usageId,
|
|
10528
|
+
pricingContext: details.pricingContext,
|
|
10529
|
+
priceSource: {
|
|
10530
|
+
...details.priceSource,
|
|
10531
|
+
observedAt: new Date(details.priceSource.observedAt)
|
|
10532
|
+
},
|
|
10533
|
+
ratedAt: new Date(input.ratedAt)
|
|
10534
|
+
};
|
|
10535
|
+
if (input.status === "rated") {
|
|
10536
|
+
return normalizeAiModelCallCostRecord({
|
|
10537
|
+
...base,
|
|
10538
|
+
status: "rated",
|
|
10539
|
+
billingIdentity: { providerId: required2(input.providerId), modelId: required2(input.modelId) },
|
|
10540
|
+
money: { currency: required2(input.currency), amountNanos: required2(input.amountNanos) },
|
|
10541
|
+
components: required2(details.components)
|
|
10542
|
+
});
|
|
10543
|
+
}
|
|
10544
|
+
return normalizeAiModelCallCostRecord({
|
|
10545
|
+
...base,
|
|
10546
|
+
status: "unpriceable",
|
|
10547
|
+
...input.providerId && input.modelId ? { billingIdentity: { providerId: input.providerId, modelId: input.modelId } } : {},
|
|
10548
|
+
reason: required2(input.reason),
|
|
10549
|
+
...details.missingMeters ? { missingMeters: details.missingMeters } : {}
|
|
10550
|
+
});
|
|
10551
|
+
}
|
|
10552
|
+
function aggregateSummaryRows(ids, rows) {
|
|
10553
|
+
const map = new Map(ids.map((id) => [id, accumulator()]));
|
|
10554
|
+
for (const row of rows) {
|
|
10555
|
+
const value = map.get(row.execution_id);
|
|
10556
|
+
if (!value)
|
|
10557
|
+
continue;
|
|
10558
|
+
if (row.status === null)
|
|
10559
|
+
value.unvaluedCallCount += 1;
|
|
10560
|
+
else if (row.status === "unpriceable")
|
|
10561
|
+
value.unpriceableCallCount += 1;
|
|
10562
|
+
else {
|
|
10563
|
+
value.ratedCallCount += 1;
|
|
10564
|
+
add(value.amounts, required2(row.currency), required2(row.amount_nanos));
|
|
10565
|
+
}
|
|
10566
|
+
}
|
|
10567
|
+
return ids.map((id) => finish(map.get(id)));
|
|
10568
|
+
}
|
|
10569
|
+
function accumulator() {
|
|
10570
|
+
return {
|
|
10571
|
+
amounts: new Map,
|
|
10572
|
+
ratedCallCount: 0,
|
|
10573
|
+
unpriceableCallCount: 0,
|
|
10574
|
+
unvaluedCallCount: 0
|
|
10575
|
+
};
|
|
10576
|
+
}
|
|
10577
|
+
function finish(value) {
|
|
10578
|
+
return {
|
|
10579
|
+
amounts: [...value.amounts].sort(([left], [right]) => left.localeCompare(right)).map(([currency, amount]) => ({ currency, amountNanos: amount.toString() })),
|
|
10580
|
+
ratedCallCount: value.ratedCallCount,
|
|
10581
|
+
unpriceableCallCount: value.unpriceableCallCount,
|
|
10582
|
+
unvaluedCallCount: value.unvaluedCallCount
|
|
10583
|
+
};
|
|
10584
|
+
}
|
|
10585
|
+
function add(amounts, currency, amount) {
|
|
10586
|
+
amounts.set(currency, (amounts.get(currency) ?? 0n) + BigInt(amount));
|
|
10587
|
+
}
|
|
10588
|
+
function parseStringArray(value) {
|
|
10589
|
+
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
|
10590
|
+
if (!Array.isArray(parsed) || parsed.some((item) => typeof item !== "string")) {
|
|
10591
|
+
throw new Error("[SixbPg] AI requester group ids are invalid.");
|
|
10592
|
+
}
|
|
10593
|
+
return parsed;
|
|
10594
|
+
}
|
|
10595
|
+
function parseJsonObject(value) {
|
|
10596
|
+
return typeof value === "string" ? JSON.parse(value) : value;
|
|
10597
|
+
}
|
|
10598
|
+
function assertCostMatchesUsage(record, usage) {
|
|
10599
|
+
if (!aiModelCallCostMatchesUsage(record, usage)) {
|
|
10600
|
+
throw new AiCostStorageError("cost_mismatch", `[SixbPg] AI valuation for usage '${record.usageRecordId}' does not match its usage record.`);
|
|
10601
|
+
}
|
|
10602
|
+
}
|
|
10603
|
+
function missingUsage(record) {
|
|
10604
|
+
return new AiCostStorageError("missing_usage", `[SixbPg] AI usage '${record.usageRecordId}' does not exist.`);
|
|
10605
|
+
}
|
|
10606
|
+
function assertNonBlank(value, field) {
|
|
10607
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
10608
|
+
throw new TypeError(`[Sixb] AI ${field} must be nonblank.`);
|
|
10609
|
+
}
|
|
10610
|
+
}
|
|
10611
|
+
function safeSqlCount(value, field) {
|
|
10612
|
+
if (!/^(0|[1-9]\d*)$/.test(value)) {
|
|
10613
|
+
throw new Error(`[SixbPg] AI accounting ${field} is invalid.`);
|
|
10614
|
+
}
|
|
10615
|
+
const count = Number(value);
|
|
10616
|
+
if (!Number.isSafeInteger(count)) {
|
|
10617
|
+
throw new TypeError(`[Sixb] AI accounting ${field} exceeds the safe integer range.`);
|
|
10618
|
+
}
|
|
10619
|
+
return count;
|
|
10620
|
+
}
|
|
10621
|
+
function required2(value) {
|
|
10622
|
+
if (value === null || value === undefined) {
|
|
10623
|
+
throw new Error("[SixbPg] AI valuation column is unexpectedly null.");
|
|
10624
|
+
}
|
|
10625
|
+
return value;
|
|
10626
|
+
}
|
|
10627
|
+
|
|
9814
10628
|
// src/pg-ai-usage-storage.ts
|
|
9815
10629
|
import {
|
|
9816
10630
|
AiUsageStorageError,
|
|
@@ -9950,7 +10764,7 @@ class PgAiUsageStorage {
|
|
|
9950
10764
|
...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
|
|
9951
10765
|
responseId: row.response_id,
|
|
9952
10766
|
usage: {
|
|
9953
|
-
...
|
|
10767
|
+
...usageFromRow2(row),
|
|
9954
10768
|
...row.total_tokens === null ? {} : { totalTokens: Number(row.total_tokens) },
|
|
9955
10769
|
reportingStatus: row.reporting_status
|
|
9956
10770
|
},
|
|
@@ -9987,7 +10801,7 @@ function aggregateExecutionRows(executionIds, rows) {
|
|
|
9987
10801
|
for (const executionId of executionIds)
|
|
9988
10802
|
usageByExecutionId.set(executionId, []);
|
|
9989
10803
|
for (const row of rows) {
|
|
9990
|
-
usageByExecutionId.get(row.execution_id)?.push(
|
|
10804
|
+
usageByExecutionId.get(row.execution_id)?.push(usageFromRow2(row));
|
|
9991
10805
|
}
|
|
9992
10806
|
return executionIds.map((executionId) => {
|
|
9993
10807
|
const usages = usageByExecutionId.get(executionId) ?? [];
|
|
@@ -9997,7 +10811,7 @@ function aggregateExecutionRows(executionIds, rows) {
|
|
|
9997
10811
|
};
|
|
9998
10812
|
});
|
|
9999
10813
|
}
|
|
10000
|
-
function
|
|
10814
|
+
function usageFromRow2(row) {
|
|
10001
10815
|
return {
|
|
10002
10816
|
...row.input_tokens === null ? {} : { inputTokens: Number(row.input_tokens) },
|
|
10003
10817
|
...row.output_tokens === null ? {} : { outputTokens: Number(row.output_tokens) },
|
|
@@ -14190,6 +15004,7 @@ class PostgresStorage {
|
|
|
14190
15004
|
executions;
|
|
14191
15005
|
agents;
|
|
14192
15006
|
aiUsage;
|
|
15007
|
+
aiCosts;
|
|
14193
15008
|
actionRuns;
|
|
14194
15009
|
pipelineRuns;
|
|
14195
15010
|
workflowRuns;
|
|
@@ -14239,6 +15054,7 @@ class PostgresStorage {
|
|
|
14239
15054
|
this.executions = createOperationScopedFacade(stores.executions, scope);
|
|
14240
15055
|
this.agents = createAgentOperationScope(stores.agents, scope);
|
|
14241
15056
|
this.aiUsage = createOperationScopedFacade(stores.aiUsage, scope);
|
|
15057
|
+
this.aiCosts = createOperationScopedFacade(stores.aiCosts, scope);
|
|
14242
15058
|
this.actionRuns = createOperationScopedFacade(stores.actionRuns, scope);
|
|
14243
15059
|
this.pipelineRuns = createOperationScopedFacade(stores.pipelineRuns, scope);
|
|
14244
15060
|
this.workflowRuns = createWorkflowRunOperationScope(stores.workflowRuns, scope);
|
|
@@ -14327,6 +15143,7 @@ function createPostgresStores(sql, options) {
|
|
|
14327
15143
|
executions,
|
|
14328
15144
|
agents: new PgAgentStorage({ sql, executions }),
|
|
14329
15145
|
aiUsage: new PgAiUsageStorage(sql),
|
|
15146
|
+
aiCosts: new PgAiCostStorage(sql),
|
|
14330
15147
|
actionRuns: new PgActionRunStorage(sql, executions),
|
|
14331
15148
|
pipelineRuns: new PgPipelineRunStorage(sql, executions),
|
|
14332
15149
|
workflowRuns: new PgWorkflowRunStorage(sql, executions),
|
|
@@ -14353,4 +15170,4 @@ export {
|
|
|
14353
15170
|
PostgresStorage
|
|
14354
15171
|
};
|
|
14355
15172
|
|
|
14356
|
-
//# debugId=
|
|
15173
|
+
//# debugId=687ACDC68EF0C51B64756E2164756E21
|