@axiom-lattice/pg-stores 2.0.4 → 2.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/CHANGELOG.md +15 -0
- package/dist/index.d.mts +8 -1
- package/dist/index.d.ts +8 -1
- package/dist/index.js +141 -18
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +138 -18
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -2
- package/src/__tests__/PostgreSQLEvalStore.test.ts +60 -0
- package/src/__tests__/eval-migrations.test.ts +48 -0
- package/src/migrations/eval_migrations.ts +114 -0
- package/src/stores/PostgreSQLEvalStore.ts +41 -27
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@axiom-lattice/pg-stores",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.6",
|
|
4
4
|
"description": "PG stores implementation for Axiom Lattice framework",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"@langchain/core": "1.1.30",
|
|
26
26
|
"pg": "^8.16.3",
|
|
27
27
|
"uuid": "^9.0.1",
|
|
28
|
-
"@axiom-lattice/core": "3.0.
|
|
28
|
+
"@axiom-lattice/core": "3.0.6",
|
|
29
29
|
"@axiom-lattice/protocols": "3.0.2"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
@@ -33,6 +33,9 @@
|
|
|
33
33
|
"@types/node": "^20.11.24",
|
|
34
34
|
"@types/pg": "^8.11.10",
|
|
35
35
|
"@types/uuid": "^9.0.8",
|
|
36
|
+
"@typescript-eslint/eslint-plugin": "^7.2.0",
|
|
37
|
+
"@typescript-eslint/parser": "^7.2.0",
|
|
38
|
+
"eslint": "^8.57.0",
|
|
36
39
|
"jest": "^29.7.0",
|
|
37
40
|
"rimraf": "^5.0.5",
|
|
38
41
|
"ts-jest": "^29.4.0",
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PostgreSQLEvalStore tests — regression lock for HITL interruptPolicy
|
|
3
|
+
* round-trip. A case's interruptPolicy must survive store writes AND reads;
|
|
4
|
+
* the SELECT queries must include the interrupt_policy column.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { PostgreSQLEvalStore } from "../stores/PostgreSQLEvalStore";
|
|
8
|
+
|
|
9
|
+
const mockQuery = jest.fn();
|
|
10
|
+
|
|
11
|
+
const mockPool = {
|
|
12
|
+
query: mockQuery,
|
|
13
|
+
connect: jest.fn().mockResolvedValue({ query: mockQuery, release: jest.fn() }),
|
|
14
|
+
end: jest.fn(),
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
describe("PostgreSQLEvalStore interruptPolicy round-trip", () => {
|
|
18
|
+
let store: PostgreSQLEvalStore;
|
|
19
|
+
|
|
20
|
+
const caseRow = {
|
|
21
|
+
id: "case-1",
|
|
22
|
+
tenant_id: "t1",
|
|
23
|
+
suite_id: "s1",
|
|
24
|
+
input_message: "approve the payment",
|
|
25
|
+
input_files: "{}",
|
|
26
|
+
steps: JSON.stringify([{ agent_id: "executor-1" }]),
|
|
27
|
+
output_type: "message_content",
|
|
28
|
+
content_assertion: "payment executed",
|
|
29
|
+
rubrics: "[]",
|
|
30
|
+
interrupt_policy: JSON.stringify({ mode: "auto-approve" }),
|
|
31
|
+
created_at: new Date(),
|
|
32
|
+
updated_at: new Date(),
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
beforeEach(() => {
|
|
36
|
+
// External pool → constructor sets initialized=true, no migrations run.
|
|
37
|
+
store = new PostgreSQLEvalStore({ pool: mockPool as never });
|
|
38
|
+
jest.clearAllMocks();
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("getCasesBySuite SELECT includes the interrupt_policy column", async () => {
|
|
42
|
+
mockQuery.mockResolvedValueOnce({ rows: [] });
|
|
43
|
+
await store.getCasesBySuite("t1", "s1");
|
|
44
|
+
const sql = mockQuery.mock.calls[0][0] as string;
|
|
45
|
+
expect(sql).toContain("interrupt_policy");
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("parses interruptPolicy from a case row", async () => {
|
|
49
|
+
mockQuery.mockResolvedValueOnce({ rows: [caseRow] });
|
|
50
|
+
const cases = await store.getCasesBySuite("t1", "s1");
|
|
51
|
+
expect(cases[0].interruptPolicy).toEqual({ mode: "auto-approve" });
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("getCaseById SELECT includes the interrupt_policy column", async () => {
|
|
55
|
+
mockQuery.mockResolvedValueOnce({ rows: [] });
|
|
56
|
+
await store.getCaseById("t1", "case-1");
|
|
57
|
+
const sql = mockQuery.mock.calls[0][0] as string;
|
|
58
|
+
expect(sql).toContain("interrupt_policy");
|
|
59
|
+
});
|
|
60
|
+
});
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval migration tests
|
|
3
|
+
*
|
|
4
|
+
* Verifies the unique-project-name migration executes the deduplication
|
|
5
|
+
* and constraint SQL in order.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { describe, expect, it, jest } from "@jest/globals";
|
|
9
|
+
import { enforceUniqueEvalProjectName } from "../migrations/eval_migrations";
|
|
10
|
+
|
|
11
|
+
describe("enforceUniqueEvalProjectName migration", () => {
|
|
12
|
+
it("repoints children, deduplicates, then adds the unique constraint", async () => {
|
|
13
|
+
const queries: string[] = [];
|
|
14
|
+
const client = {
|
|
15
|
+
query: jest.fn(async (sql: string) => {
|
|
16
|
+
queries.push(sql.replace(/\s+/g, " ").trim());
|
|
17
|
+
}),
|
|
18
|
+
} as any;
|
|
19
|
+
|
|
20
|
+
await enforceUniqueEvalProjectName.up(client);
|
|
21
|
+
|
|
22
|
+
expect(queries).toHaveLength(4);
|
|
23
|
+
// 1. suites repointed to the kept project (history preserved)
|
|
24
|
+
expect(queries[0]).toContain("UPDATE lattice_eval_suites");
|
|
25
|
+
expect(queries[0]).toContain("ORDER BY p2.created_at DESC, p2.id DESC");
|
|
26
|
+
// 2. runs repointed
|
|
27
|
+
expect(queries[1]).toContain("UPDATE lattice_eval_runs");
|
|
28
|
+
// 3. duplicate projects deleted
|
|
29
|
+
expect(queries[2]).toContain("DELETE FROM lattice_eval_projects");
|
|
30
|
+
expect(queries[2]).toContain("p.created_at < p2.created_at");
|
|
31
|
+
// 4. unique constraint added
|
|
32
|
+
expect(queries[3]).toContain("ADD CONSTRAINT lattice_eval_projects_tenant_name_unique UNIQUE (tenant_id, name)");
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it("drops the constraint on down", async () => {
|
|
36
|
+
const queries: string[] = [];
|
|
37
|
+
const client = {
|
|
38
|
+
query: jest.fn(async (sql: string) => {
|
|
39
|
+
queries.push(sql.replace(/\s+/g, " ").trim());
|
|
40
|
+
}),
|
|
41
|
+
} as any;
|
|
42
|
+
|
|
43
|
+
await enforceUniqueEvalProjectName.down(client);
|
|
44
|
+
|
|
45
|
+
expect(queries).toHaveLength(1);
|
|
46
|
+
expect(queries[0]).toContain("DROP CONSTRAINT IF EXISTS lattice_eval_projects_tenant_name_unique");
|
|
47
|
+
});
|
|
48
|
+
});
|
|
@@ -248,6 +248,117 @@ export const addEnvToEvalRuns: Migration = {
|
|
|
248
248
|
},
|
|
249
249
|
};
|
|
250
250
|
|
|
251
|
+
/** Add HITL interrupt tracking: interrupted flag on run results + interrupted count on runs */
|
|
252
|
+
export const addInterruptColumnsToEval: Migration = {
|
|
253
|
+
version: 111,
|
|
254
|
+
name: "add_interrupt_columns_to_eval",
|
|
255
|
+
up: async (client: PoolClient) => {
|
|
256
|
+
await client.query(`
|
|
257
|
+
ALTER TABLE lattice_eval_run_results
|
|
258
|
+
ADD COLUMN IF NOT EXISTS interrupted BOOLEAN NOT NULL DEFAULT FALSE
|
|
259
|
+
`);
|
|
260
|
+
await client.query(`
|
|
261
|
+
ALTER TABLE lattice_eval_runs
|
|
262
|
+
ADD COLUMN IF NOT EXISTS interrupted_cases INTEGER NOT NULL DEFAULT 0
|
|
263
|
+
`);
|
|
264
|
+
},
|
|
265
|
+
down: async (client: PoolClient) => {
|
|
266
|
+
await client.query(`
|
|
267
|
+
ALTER TABLE lattice_eval_run_results DROP COLUMN IF EXISTS interrupted
|
|
268
|
+
`);
|
|
269
|
+
await client.query(`
|
|
270
|
+
ALTER TABLE lattice_eval_runs DROP COLUMN IF EXISTS interrupted_cases
|
|
271
|
+
`);
|
|
272
|
+
},
|
|
273
|
+
};
|
|
274
|
+
|
|
275
|
+
/** Add HITL auto-resolve policy to eval cases */
|
|
276
|
+
export const addInterruptPolicyToEvalCases: Migration = {
|
|
277
|
+
version: 112,
|
|
278
|
+
name: "add_interrupt_policy_to_eval_cases",
|
|
279
|
+
up: async (client: PoolClient) => {
|
|
280
|
+
await client.query(`
|
|
281
|
+
ALTER TABLE lattice_eval_cases
|
|
282
|
+
ADD COLUMN IF NOT EXISTS interrupt_policy JSONB
|
|
283
|
+
`);
|
|
284
|
+
},
|
|
285
|
+
down: async (client: PoolClient) => {
|
|
286
|
+
await client.query(`
|
|
287
|
+
ALTER TABLE lattice_eval_cases DROP COLUMN IF EXISTS interrupt_policy
|
|
288
|
+
`);
|
|
289
|
+
},
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
/** Enforce unique eval project name per tenant (eval-{agent-id} convention) */
|
|
293
|
+
export const enforceUniqueEvalProjectName: Migration = {
|
|
294
|
+
version: 163,
|
|
295
|
+
name: "enforce_unique_eval_project_name",
|
|
296
|
+
up: async (client: PoolClient) => {
|
|
297
|
+
// Keep the newest project (max created_at, tie-break max id) per
|
|
298
|
+
// (tenant_id, name). Repoint child rows (suites, runs) of duplicate
|
|
299
|
+
// projects onto the kept project BEFORE deleting, so no eval history
|
|
300
|
+
// is lost. Cases follow their suite; run results follow their run.
|
|
301
|
+
await client.query(`
|
|
302
|
+
UPDATE lattice_eval_suites s
|
|
303
|
+
SET project_id = (
|
|
304
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
305
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
306
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
307
|
+
LIMIT 1
|
|
308
|
+
)
|
|
309
|
+
FROM lattice_eval_projects p
|
|
310
|
+
WHERE s.project_id = p.id
|
|
311
|
+
AND s.project_id <> (
|
|
312
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
313
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
314
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
315
|
+
LIMIT 1
|
|
316
|
+
)
|
|
317
|
+
`);
|
|
318
|
+
|
|
319
|
+
await client.query(`
|
|
320
|
+
UPDATE lattice_eval_runs r
|
|
321
|
+
SET project_id = (
|
|
322
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
323
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
324
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
325
|
+
LIMIT 1
|
|
326
|
+
)
|
|
327
|
+
FROM lattice_eval_projects p
|
|
328
|
+
WHERE r.project_id = p.id
|
|
329
|
+
AND r.project_id <> (
|
|
330
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
331
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
332
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
333
|
+
LIMIT 1
|
|
334
|
+
)
|
|
335
|
+
`);
|
|
336
|
+
|
|
337
|
+
// Children are repointed; safe to delete the duplicate projects now.
|
|
338
|
+
await client.query(`
|
|
339
|
+
DELETE FROM lattice_eval_projects p
|
|
340
|
+
USING lattice_eval_projects p2
|
|
341
|
+
WHERE p.tenant_id = p2.tenant_id
|
|
342
|
+
AND p.name = p2.name
|
|
343
|
+
AND (
|
|
344
|
+
p.created_at < p2.created_at
|
|
345
|
+
OR (p.created_at = p2.created_at AND p.id < p2.id)
|
|
346
|
+
)
|
|
347
|
+
`);
|
|
348
|
+
|
|
349
|
+
await client.query(`
|
|
350
|
+
ALTER TABLE lattice_eval_projects
|
|
351
|
+
ADD CONSTRAINT lattice_eval_projects_tenant_name_unique UNIQUE (tenant_id, name)
|
|
352
|
+
`);
|
|
353
|
+
},
|
|
354
|
+
down: async (client: PoolClient) => {
|
|
355
|
+
await client.query(`
|
|
356
|
+
ALTER TABLE lattice_eval_projects
|
|
357
|
+
DROP CONSTRAINT IF EXISTS lattice_eval_projects_tenant_name_unique
|
|
358
|
+
`);
|
|
359
|
+
},
|
|
360
|
+
};
|
|
361
|
+
|
|
251
362
|
/** All eval migrations in version order */
|
|
252
363
|
export const evalMigrations: Migration[] = [
|
|
253
364
|
createEvalProjectsTable,
|
|
@@ -257,4 +368,7 @@ export const evalMigrations: Migration[] = [
|
|
|
257
368
|
createEvalRunResultsTable,
|
|
258
369
|
addHoldoutToEvalRuns,
|
|
259
370
|
addEnvToEvalRuns,
|
|
371
|
+
addInterruptColumnsToEval,
|
|
372
|
+
addInterruptPolicyToEvalCases,
|
|
373
|
+
enforceUniqueEvalProjectName,
|
|
260
374
|
];
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
CreateEvalRunRequest,
|
|
17
17
|
EvalRunResult,
|
|
18
18
|
EvalProjectReport,
|
|
19
|
+
InterruptPolicy,
|
|
19
20
|
} from "@axiom-lattice/protocols";
|
|
20
21
|
import { MigrationManager } from "../migrations/migration";
|
|
21
22
|
import { evalMigrations } from "../migrations/eval_migrations";
|
|
@@ -160,6 +161,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
160
161
|
outputType: (row.output_type as EvalCase["outputType"]) || "message_content",
|
|
161
162
|
contentAssertion: (row.content_assertion as string) || "",
|
|
162
163
|
rubrics: this.parseOptionalJson<Array<{ name: string; weight: number; description: string }>>(row.rubrics),
|
|
164
|
+
interruptPolicy: this.parseOptionalJson<InterruptPolicy>(row.interrupt_policy),
|
|
163
165
|
createdAt: new Date(row.created_at as string),
|
|
164
166
|
updatedAt: new Date(row.updated_at as string),
|
|
165
167
|
};
|
|
@@ -175,6 +177,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
175
177
|
totalCases: (row.total_cases as number) ?? 0,
|
|
176
178
|
passedCases: (row.passed_cases as number) ?? 0,
|
|
177
179
|
failedCases: (row.failed_cases as number) ?? 0,
|
|
180
|
+
interruptedCases: (row.interrupted_cases as number) ?? 0,
|
|
178
181
|
avgScore: (row.avg_score as number) ?? 0,
|
|
179
182
|
error: row.error as string | undefined,
|
|
180
183
|
holdout: row.holdout as boolean | undefined,
|
|
@@ -200,6 +203,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
200
203
|
messages: this.parseOptionalJson<Array<{ role: string; content: string; id?: string }>>(row.messages),
|
|
201
204
|
logs: this.parseOptionalJson<Array<{ timestamp: string; level: string; message: string; data?: unknown }>>(row.logs),
|
|
202
205
|
error: row.error as string | undefined,
|
|
206
|
+
interrupted: (row.interrupted as boolean) ?? false,
|
|
203
207
|
createdAt: new Date(row.created_at as string),
|
|
204
208
|
};
|
|
205
209
|
}
|
|
@@ -457,11 +461,12 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
457
461
|
id: string; tenant_id: string; suite_id: string;
|
|
458
462
|
input_message: string; input_files: unknown; steps: unknown;
|
|
459
463
|
output_type: string; content_assertion: string; rubrics: unknown;
|
|
464
|
+
interrupt_policy: unknown;
|
|
460
465
|
created_at: string; updated_at: string;
|
|
461
466
|
}>(
|
|
462
467
|
`SELECT id, tenant_id, suite_id,
|
|
463
468
|
input_message, input_files, steps,
|
|
464
|
-
output_type, content_assertion, rubrics,
|
|
469
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
465
470
|
created_at, updated_at
|
|
466
471
|
FROM lattice_eval_cases
|
|
467
472
|
WHERE tenant_id = $1 AND suite_id = $2
|
|
@@ -478,11 +483,12 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
478
483
|
id: string; tenant_id: string; suite_id: string;
|
|
479
484
|
input_message: string; input_files: unknown; steps: unknown;
|
|
480
485
|
output_type: string; content_assertion: string; rubrics: unknown;
|
|
486
|
+
interrupt_policy: unknown;
|
|
481
487
|
created_at: string; updated_at: string;
|
|
482
488
|
}>(
|
|
483
489
|
`SELECT id, tenant_id, suite_id,
|
|
484
490
|
input_message, input_files, steps,
|
|
485
|
-
output_type, content_assertion, rubrics,
|
|
491
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
486
492
|
created_at, updated_at
|
|
487
493
|
FROM lattice_eval_cases
|
|
488
494
|
WHERE id = $1 AND tenant_id = $2`,
|
|
@@ -504,14 +510,15 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
504
510
|
id: string; tenant_id: string; suite_id: string;
|
|
505
511
|
input_message: string; input_files: unknown; steps: unknown;
|
|
506
512
|
output_type: string; content_assertion: string; rubrics: unknown;
|
|
513
|
+
interrupt_policy: unknown;
|
|
507
514
|
created_at: string; updated_at: string;
|
|
508
515
|
}>(
|
|
509
516
|
`INSERT INTO lattice_eval_cases
|
|
510
|
-
(id, tenant_id, suite_id, input_message, input_files, steps, output_type, content_assertion, rubrics)
|
|
511
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
517
|
+
(id, tenant_id, suite_id, input_message, input_files, steps, output_type, content_assertion, rubrics, interrupt_policy)
|
|
518
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
|
512
519
|
RETURNING id, tenant_id, suite_id,
|
|
513
520
|
input_message, input_files, steps,
|
|
514
|
-
output_type, content_assertion, rubrics,
|
|
521
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
515
522
|
created_at, updated_at`,
|
|
516
523
|
[
|
|
517
524
|
actualId,
|
|
@@ -523,6 +530,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
523
530
|
data.outputType,
|
|
524
531
|
data.contentAssertion || null,
|
|
525
532
|
JSON.stringify(data.rubrics || []),
|
|
533
|
+
data.interruptPolicy ? JSON.stringify(data.interruptPolicy) : null,
|
|
526
534
|
]
|
|
527
535
|
);
|
|
528
536
|
return this.mapRowToCase(rows[0] as unknown as Record<string, unknown>);
|
|
@@ -544,6 +552,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
544
552
|
if (updates.outputType !== undefined) { set.push(`output_type = $${i++}`); vals.push(updates.outputType); }
|
|
545
553
|
if (updates.contentAssertion !== undefined) { set.push(`content_assertion = $${i++}`); vals.push(updates.contentAssertion); }
|
|
546
554
|
if (updates.rubrics !== undefined) { set.push(`rubrics = $${i++}`); vals.push(JSON.stringify(updates.rubrics)); }
|
|
555
|
+
if (updates.interruptPolicy !== undefined) { set.push(`interrupt_policy = $${i++}`); vals.push(updates.interruptPolicy ? JSON.stringify(updates.interruptPolicy) : null); }
|
|
547
556
|
if (set.length === 0) return this.getCaseById(tenantId, id);
|
|
548
557
|
set.push(`updated_at = NOW()`);
|
|
549
558
|
vals.push(id, tenantId);
|
|
@@ -551,13 +560,14 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
551
560
|
id: string; tenant_id: string; suite_id: string;
|
|
552
561
|
input_message: string; input_files: unknown; steps: unknown;
|
|
553
562
|
output_type: string; content_assertion: string; rubrics: unknown;
|
|
563
|
+
interrupt_policy: unknown;
|
|
554
564
|
created_at: string; updated_at: string;
|
|
555
565
|
}>(
|
|
556
566
|
`UPDATE lattice_eval_cases SET ${set.join(", ")}
|
|
557
567
|
WHERE id = $${i} AND tenant_id = $${i + 1}
|
|
558
568
|
RETURNING id, tenant_id, suite_id,
|
|
559
569
|
input_message, input_files, steps,
|
|
560
|
-
output_type, content_assertion, rubrics,
|
|
570
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
561
571
|
created_at, updated_at`,
|
|
562
572
|
vals
|
|
563
573
|
);
|
|
@@ -592,13 +602,13 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
592
602
|
const { rows } = await this.pool.query<{
|
|
593
603
|
id: string; project_id: string; tenant_id: string;
|
|
594
604
|
status: string; concurrency: number; total_cases: number;
|
|
595
|
-
passed_cases: number; failed_cases: number; avg_score: number;
|
|
596
|
-
error: string | null; created_at: string; started_at: string | null;
|
|
605
|
+
passed_cases: number; failed_cases: number; interrupted_cases: number; avg_score: number;
|
|
606
|
+
error: string | null; interrupted: boolean; created_at: string; started_at: string | null;
|
|
597
607
|
completed_at: string | null;
|
|
598
608
|
}>(
|
|
599
609
|
`SELECT id, project_id, tenant_id,
|
|
600
610
|
status, concurrency, total_cases,
|
|
601
|
-
passed_cases, failed_cases, avg_score,
|
|
611
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
602
612
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at
|
|
603
613
|
FROM lattice_eval_runs
|
|
604
614
|
WHERE ${conditions.join(" AND ")}
|
|
@@ -614,13 +624,13 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
614
624
|
const { rows } = await this.pool.query<{
|
|
615
625
|
id: string; project_id: string; tenant_id: string;
|
|
616
626
|
status: string; concurrency: number; total_cases: number;
|
|
617
|
-
passed_cases: number; failed_cases: number; avg_score: number;
|
|
618
|
-
error: string | null; created_at: string; started_at: string | null;
|
|
627
|
+
passed_cases: number; failed_cases: number; interrupted_cases: number; avg_score: number;
|
|
628
|
+
error: string | null; interrupted: boolean; created_at: string; started_at: string | null;
|
|
619
629
|
completed_at: string | null;
|
|
620
630
|
}>(
|
|
621
631
|
`SELECT id, project_id, tenant_id,
|
|
622
632
|
status, concurrency, total_cases,
|
|
623
|
-
passed_cases, failed_cases, avg_score,
|
|
633
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
624
634
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at
|
|
625
635
|
FROM lattice_eval_runs
|
|
626
636
|
WHERE id = $1 AND tenant_id = $2`,
|
|
@@ -641,8 +651,8 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
641
651
|
const { rows } = await this.pool.query<{
|
|
642
652
|
id: string; project_id: string; tenant_id: string;
|
|
643
653
|
status: string; concurrency: number; total_cases: number;
|
|
644
|
-
passed_cases: number; failed_cases: number; avg_score: number;
|
|
645
|
-
error: string | null; created_at: string; started_at: string | null;
|
|
654
|
+
passed_cases: number; failed_cases: number; interrupted_cases: number; avg_score: number;
|
|
655
|
+
error: string | null; interrupted: boolean; created_at: string; started_at: string | null;
|
|
646
656
|
completed_at: string | null;
|
|
647
657
|
}>(
|
|
648
658
|
`INSERT INTO lattice_eval_runs
|
|
@@ -650,7 +660,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
650
660
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
651
661
|
RETURNING id, project_id, tenant_id,
|
|
652
662
|
status, concurrency, total_cases,
|
|
653
|
-
passed_cases, failed_cases, avg_score,
|
|
663
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
654
664
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at`,
|
|
655
665
|
[
|
|
656
666
|
actualId,
|
|
@@ -679,6 +689,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
679
689
|
status?: EvalRun["status"];
|
|
680
690
|
passedCases?: number;
|
|
681
691
|
failedCases?: number;
|
|
692
|
+
interruptedCases?: number;
|
|
682
693
|
avgScore?: number;
|
|
683
694
|
error?: string;
|
|
684
695
|
completedAt?: Date;
|
|
@@ -691,6 +702,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
691
702
|
if (updates.status !== undefined) { set.push(`status = $${i++}`); vals.push(updates.status); }
|
|
692
703
|
if (updates.passedCases !== undefined) { set.push(`passed_cases = $${i++}`); vals.push(updates.passedCases); }
|
|
693
704
|
if (updates.failedCases !== undefined) { set.push(`failed_cases = $${i++}`); vals.push(updates.failedCases); }
|
|
705
|
+
if (updates.interruptedCases !== undefined) { set.push(`interrupted_cases = $${i++}`); vals.push(updates.interruptedCases); }
|
|
694
706
|
if (updates.avgScore !== undefined) { set.push(`avg_score = $${i++}`); vals.push(updates.avgScore); }
|
|
695
707
|
if (updates.completedAt !== undefined) { set.push(`completed_at = $${i++}`); vals.push(updates.completedAt); }
|
|
696
708
|
if (updates.error !== undefined) { set.push(`error = $${i++}`); vals.push(updates.error); }
|
|
@@ -699,15 +711,15 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
699
711
|
const { rows } = await this.pool.query<{
|
|
700
712
|
id: string; project_id: string; tenant_id: string;
|
|
701
713
|
status: string; concurrency: number; total_cases: number;
|
|
702
|
-
passed_cases: number; failed_cases: number; avg_score: number;
|
|
703
|
-
error: string | null; created_at: string; started_at: string | null;
|
|
714
|
+
passed_cases: number; failed_cases: number; interrupted_cases: number; avg_score: number;
|
|
715
|
+
error: string | null; interrupted: boolean; created_at: string; started_at: string | null;
|
|
704
716
|
completed_at: string | null;
|
|
705
717
|
}>(
|
|
706
718
|
`UPDATE lattice_eval_runs SET ${set.join(", ")}
|
|
707
719
|
WHERE id = $${i} AND tenant_id = $${i + 1}
|
|
708
720
|
RETURNING id, project_id, tenant_id,
|
|
709
721
|
status, concurrency, total_cases,
|
|
710
|
-
passed_cases, failed_cases, avg_score,
|
|
722
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
711
723
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at`,
|
|
712
724
|
vals
|
|
713
725
|
);
|
|
@@ -737,11 +749,11 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
737
749
|
case_id: string | null; pass: boolean; score: number;
|
|
738
750
|
summary: string | null; dimension_results: unknown;
|
|
739
751
|
duration_ms: number | null; messages: unknown; logs: unknown;
|
|
740
|
-
error: string | null; created_at: string;
|
|
752
|
+
error: string | null; interrupted: boolean; created_at: string;
|
|
741
753
|
}>(
|
|
742
754
|
`SELECT rr.id, rr.run_id, rr.suite_name, rr.case_id,
|
|
743
755
|
rr.pass, rr.score, rr.summary, rr.dimension_results,
|
|
744
|
-
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.created_at
|
|
756
|
+
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.interrupted, rr.created_at
|
|
745
757
|
FROM lattice_eval_run_results rr
|
|
746
758
|
INNER JOIN lattice_eval_runs r ON r.id = rr.run_id
|
|
747
759
|
WHERE r.tenant_id = $1 AND rr.run_id = $2
|
|
@@ -765,14 +777,14 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
765
777
|
case_id: string | null; pass: boolean; score: number;
|
|
766
778
|
summary: string | null; dimension_results: unknown;
|
|
767
779
|
duration_ms: number | null; messages: unknown; logs: unknown;
|
|
768
|
-
error: string | null; created_at: string;
|
|
780
|
+
error: string | null; interrupted: boolean; created_at: string;
|
|
769
781
|
}>(
|
|
770
782
|
`INSERT INTO lattice_eval_run_results
|
|
771
|
-
(id, run_id, suite_name, case_id, pass, score, summary, dimension_results, duration_ms, messages, logs, error)
|
|
772
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
783
|
+
(id, run_id, suite_name, case_id, pass, score, summary, dimension_results, duration_ms, messages, logs, error, interrupted)
|
|
784
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
773
785
|
RETURNING id, run_id, suite_name, case_id,
|
|
774
786
|
pass, score, summary, dimension_results,
|
|
775
|
-
duration_ms, messages, logs, error, created_at`,
|
|
787
|
+
duration_ms, messages, logs, error, interrupted, created_at`,
|
|
776
788
|
[
|
|
777
789
|
actualId,
|
|
778
790
|
runId,
|
|
@@ -786,6 +798,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
786
798
|
JSON.stringify(data.messages || []),
|
|
787
799
|
JSON.stringify(data.logs || []),
|
|
788
800
|
data.error || null,
|
|
801
|
+
data.interrupted ?? false,
|
|
789
802
|
]
|
|
790
803
|
);
|
|
791
804
|
return this.mapRowToRunResult(rows[0] as unknown as Record<string, unknown>);
|
|
@@ -814,6 +827,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
814
827
|
if (updates.messages !== undefined) { set.push(`messages = $${i++}`); vals.push(JSON.stringify(updates.messages)); }
|
|
815
828
|
if (updates.logs !== undefined) { set.push(`logs = $${i++}`); vals.push(JSON.stringify(updates.logs)); }
|
|
816
829
|
if (updates.error !== undefined) { set.push(`error = $${i++}`); vals.push(updates.error); }
|
|
830
|
+
if (updates.interrupted !== undefined) { set.push(`interrupted = $${i++}`); vals.push(updates.interrupted); }
|
|
817
831
|
if (set.length === 0) return this.getRunResultById(tenantId, id);
|
|
818
832
|
vals.push(id, tenantId);
|
|
819
833
|
const { rows } = await this.pool.query<{
|
|
@@ -821,7 +835,7 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
821
835
|
case_id: string | null; pass: boolean; score: number;
|
|
822
836
|
summary: string | null; dimension_results: unknown;
|
|
823
837
|
duration_ms: number | null; messages: unknown; logs: unknown;
|
|
824
|
-
error: string | null; created_at: string;
|
|
838
|
+
error: string | null; interrupted: boolean; created_at: string;
|
|
825
839
|
}>(
|
|
826
840
|
`UPDATE lattice_eval_run_results SET ${set.join(", ")}
|
|
827
841
|
WHERE id = $${i}
|
|
@@ -854,11 +868,11 @@ export class PostgreSQLEvalStore implements EvalStore {
|
|
|
854
868
|
case_id: string | null; pass: boolean; score: number;
|
|
855
869
|
summary: string | null; dimension_results: unknown;
|
|
856
870
|
duration_ms: number | null; messages: unknown; logs: unknown;
|
|
857
|
-
error: string | null; created_at: string;
|
|
871
|
+
error: string | null; interrupted: boolean; created_at: string;
|
|
858
872
|
}>(
|
|
859
873
|
`SELECT rr.id, rr.run_id, rr.suite_name, rr.case_id,
|
|
860
874
|
rr.pass, rr.score, rr.summary, rr.dimension_results,
|
|
861
|
-
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.created_at
|
|
875
|
+
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.interrupted, rr.created_at
|
|
862
876
|
FROM lattice_eval_run_results rr
|
|
863
877
|
INNER JOIN lattice_eval_runs r ON r.id = rr.run_id
|
|
864
878
|
WHERE rr.id = $1 AND r.tenant_id = $2`,
|