@axiom-lattice/pg-stores 2.0.4 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/CHANGELOG.md +8 -0
- package/dist/index.d.mts +8 -1
- package/dist/index.d.ts +8 -1
- package/dist/index.js +141 -18
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +138 -18
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -2
- package/src/__tests__/PostgreSQLEvalStore.test.ts +60 -0
- package/src/__tests__/eval-migrations.test.ts +48 -0
- package/src/migrations/eval_migrations.ts +114 -0
- package/src/stores/PostgreSQLEvalStore.ts +41 -27
package/dist/index.mjs
CHANGED
|
@@ -3918,6 +3918,103 @@ var addEnvToEvalRuns = {
|
|
|
3918
3918
|
`);
|
|
3919
3919
|
}
|
|
3920
3920
|
};
|
|
3921
|
+
var addInterruptColumnsToEval = {
|
|
3922
|
+
version: 111,
|
|
3923
|
+
name: "add_interrupt_columns_to_eval",
|
|
3924
|
+
up: async (client) => {
|
|
3925
|
+
await client.query(`
|
|
3926
|
+
ALTER TABLE lattice_eval_run_results
|
|
3927
|
+
ADD COLUMN IF NOT EXISTS interrupted BOOLEAN NOT NULL DEFAULT FALSE
|
|
3928
|
+
`);
|
|
3929
|
+
await client.query(`
|
|
3930
|
+
ALTER TABLE lattice_eval_runs
|
|
3931
|
+
ADD COLUMN IF NOT EXISTS interrupted_cases INTEGER NOT NULL DEFAULT 0
|
|
3932
|
+
`);
|
|
3933
|
+
},
|
|
3934
|
+
down: async (client) => {
|
|
3935
|
+
await client.query(`
|
|
3936
|
+
ALTER TABLE lattice_eval_run_results DROP COLUMN IF EXISTS interrupted
|
|
3937
|
+
`);
|
|
3938
|
+
await client.query(`
|
|
3939
|
+
ALTER TABLE lattice_eval_runs DROP COLUMN IF EXISTS interrupted_cases
|
|
3940
|
+
`);
|
|
3941
|
+
}
|
|
3942
|
+
};
|
|
3943
|
+
var addInterruptPolicyToEvalCases = {
|
|
3944
|
+
version: 112,
|
|
3945
|
+
name: "add_interrupt_policy_to_eval_cases",
|
|
3946
|
+
up: async (client) => {
|
|
3947
|
+
await client.query(`
|
|
3948
|
+
ALTER TABLE lattice_eval_cases
|
|
3949
|
+
ADD COLUMN IF NOT EXISTS interrupt_policy JSONB
|
|
3950
|
+
`);
|
|
3951
|
+
},
|
|
3952
|
+
down: async (client) => {
|
|
3953
|
+
await client.query(`
|
|
3954
|
+
ALTER TABLE lattice_eval_cases DROP COLUMN IF EXISTS interrupt_policy
|
|
3955
|
+
`);
|
|
3956
|
+
}
|
|
3957
|
+
};
|
|
3958
|
+
var enforceUniqueEvalProjectName = {
|
|
3959
|
+
version: 163,
|
|
3960
|
+
name: "enforce_unique_eval_project_name",
|
|
3961
|
+
up: async (client) => {
|
|
3962
|
+
await client.query(`
|
|
3963
|
+
UPDATE lattice_eval_suites s
|
|
3964
|
+
SET project_id = (
|
|
3965
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
3966
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
3967
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
3968
|
+
LIMIT 1
|
|
3969
|
+
)
|
|
3970
|
+
FROM lattice_eval_projects p
|
|
3971
|
+
WHERE s.project_id = p.id
|
|
3972
|
+
AND s.project_id <> (
|
|
3973
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
3974
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
3975
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
3976
|
+
LIMIT 1
|
|
3977
|
+
)
|
|
3978
|
+
`);
|
|
3979
|
+
await client.query(`
|
|
3980
|
+
UPDATE lattice_eval_runs r
|
|
3981
|
+
SET project_id = (
|
|
3982
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
3983
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
3984
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
3985
|
+
LIMIT 1
|
|
3986
|
+
)
|
|
3987
|
+
FROM lattice_eval_projects p
|
|
3988
|
+
WHERE r.project_id = p.id
|
|
3989
|
+
AND r.project_id <> (
|
|
3990
|
+
SELECT p2.id FROM lattice_eval_projects p2
|
|
3991
|
+
WHERE p2.tenant_id = p.tenant_id AND p2.name = p.name
|
|
3992
|
+
ORDER BY p2.created_at DESC, p2.id DESC
|
|
3993
|
+
LIMIT 1
|
|
3994
|
+
)
|
|
3995
|
+
`);
|
|
3996
|
+
await client.query(`
|
|
3997
|
+
DELETE FROM lattice_eval_projects p
|
|
3998
|
+
USING lattice_eval_projects p2
|
|
3999
|
+
WHERE p.tenant_id = p2.tenant_id
|
|
4000
|
+
AND p.name = p2.name
|
|
4001
|
+
AND (
|
|
4002
|
+
p.created_at < p2.created_at
|
|
4003
|
+
OR (p.created_at = p2.created_at AND p.id < p2.id)
|
|
4004
|
+
)
|
|
4005
|
+
`);
|
|
4006
|
+
await client.query(`
|
|
4007
|
+
ALTER TABLE lattice_eval_projects
|
|
4008
|
+
ADD CONSTRAINT lattice_eval_projects_tenant_name_unique UNIQUE (tenant_id, name)
|
|
4009
|
+
`);
|
|
4010
|
+
},
|
|
4011
|
+
down: async (client) => {
|
|
4012
|
+
await client.query(`
|
|
4013
|
+
ALTER TABLE lattice_eval_projects
|
|
4014
|
+
DROP CONSTRAINT IF EXISTS lattice_eval_projects_tenant_name_unique
|
|
4015
|
+
`);
|
|
4016
|
+
}
|
|
4017
|
+
};
|
|
3921
4018
|
var evalMigrations = [
|
|
3922
4019
|
createEvalProjectsTable,
|
|
3923
4020
|
createEvalSuitesTable,
|
|
@@ -3925,7 +4022,10 @@ var evalMigrations = [
|
|
|
3925
4022
|
createEvalRunsTable,
|
|
3926
4023
|
createEvalRunResultsTable,
|
|
3927
4024
|
addHoldoutToEvalRuns,
|
|
3928
|
-
addEnvToEvalRuns
|
|
4025
|
+
addEnvToEvalRuns,
|
|
4026
|
+
addInterruptColumnsToEval,
|
|
4027
|
+
addInterruptPolicyToEvalCases,
|
|
4028
|
+
enforceUniqueEvalProjectName
|
|
3929
4029
|
];
|
|
3930
4030
|
|
|
3931
4031
|
// src/stores/PostgreSQLEvalStore.ts
|
|
@@ -4032,6 +4132,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4032
4132
|
outputType: row.output_type || "message_content",
|
|
4033
4133
|
contentAssertion: row.content_assertion || "",
|
|
4034
4134
|
rubrics: this.parseOptionalJson(row.rubrics),
|
|
4135
|
+
interruptPolicy: this.parseOptionalJson(row.interrupt_policy),
|
|
4035
4136
|
createdAt: new Date(row.created_at),
|
|
4036
4137
|
updatedAt: new Date(row.updated_at)
|
|
4037
4138
|
};
|
|
@@ -4046,6 +4147,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4046
4147
|
totalCases: row.total_cases ?? 0,
|
|
4047
4148
|
passedCases: row.passed_cases ?? 0,
|
|
4048
4149
|
failedCases: row.failed_cases ?? 0,
|
|
4150
|
+
interruptedCases: row.interrupted_cases ?? 0,
|
|
4049
4151
|
avgScore: row.avg_score ?? 0,
|
|
4050
4152
|
error: row.error,
|
|
4051
4153
|
holdout: row.holdout,
|
|
@@ -4070,6 +4172,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4070
4172
|
messages: this.parseOptionalJson(row.messages),
|
|
4071
4173
|
logs: this.parseOptionalJson(row.logs),
|
|
4072
4174
|
error: row.error,
|
|
4175
|
+
interrupted: row.interrupted ?? false,
|
|
4073
4176
|
createdAt: new Date(row.created_at)
|
|
4074
4177
|
};
|
|
4075
4178
|
}
|
|
@@ -4289,7 +4392,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4289
4392
|
const { rows } = await this.pool.query(
|
|
4290
4393
|
`SELECT id, tenant_id, suite_id,
|
|
4291
4394
|
input_message, input_files, steps,
|
|
4292
|
-
output_type, content_assertion, rubrics,
|
|
4395
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
4293
4396
|
created_at, updated_at
|
|
4294
4397
|
FROM lattice_eval_cases
|
|
4295
4398
|
WHERE tenant_id = $1 AND suite_id = $2
|
|
@@ -4304,7 +4407,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4304
4407
|
const { rows } = await this.pool.query(
|
|
4305
4408
|
`SELECT id, tenant_id, suite_id,
|
|
4306
4409
|
input_message, input_files, steps,
|
|
4307
|
-
output_type, content_assertion, rubrics,
|
|
4410
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
4308
4411
|
created_at, updated_at
|
|
4309
4412
|
FROM lattice_eval_cases
|
|
4310
4413
|
WHERE id = $1 AND tenant_id = $2`,
|
|
@@ -4318,11 +4421,11 @@ var PostgreSQLEvalStore = class {
|
|
|
4318
4421
|
const actualId = id || uuidv4();
|
|
4319
4422
|
const { rows } = await this.pool.query(
|
|
4320
4423
|
`INSERT INTO lattice_eval_cases
|
|
4321
|
-
(id, tenant_id, suite_id, input_message, input_files, steps, output_type, content_assertion, rubrics)
|
|
4322
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
4424
|
+
(id, tenant_id, suite_id, input_message, input_files, steps, output_type, content_assertion, rubrics, interrupt_policy)
|
|
4425
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
|
4323
4426
|
RETURNING id, tenant_id, suite_id,
|
|
4324
4427
|
input_message, input_files, steps,
|
|
4325
|
-
output_type, content_assertion, rubrics,
|
|
4428
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
4326
4429
|
created_at, updated_at`,
|
|
4327
4430
|
[
|
|
4328
4431
|
actualId,
|
|
@@ -4333,7 +4436,8 @@ var PostgreSQLEvalStore = class {
|
|
|
4333
4436
|
JSON.stringify(data.steps),
|
|
4334
4437
|
data.outputType,
|
|
4335
4438
|
data.contentAssertion || null,
|
|
4336
|
-
JSON.stringify(data.rubrics || [])
|
|
4439
|
+
JSON.stringify(data.rubrics || []),
|
|
4440
|
+
data.interruptPolicy ? JSON.stringify(data.interruptPolicy) : null
|
|
4337
4441
|
]
|
|
4338
4442
|
);
|
|
4339
4443
|
return this.mapRowToCase(rows[0]);
|
|
@@ -4368,6 +4472,10 @@ var PostgreSQLEvalStore = class {
|
|
|
4368
4472
|
set.push(`rubrics = $${i++}`);
|
|
4369
4473
|
vals.push(JSON.stringify(updates.rubrics));
|
|
4370
4474
|
}
|
|
4475
|
+
if (updates.interruptPolicy !== void 0) {
|
|
4476
|
+
set.push(`interrupt_policy = $${i++}`);
|
|
4477
|
+
vals.push(updates.interruptPolicy ? JSON.stringify(updates.interruptPolicy) : null);
|
|
4478
|
+
}
|
|
4371
4479
|
if (set.length === 0) return this.getCaseById(tenantId, id);
|
|
4372
4480
|
set.push(`updated_at = NOW()`);
|
|
4373
4481
|
vals.push(id, tenantId);
|
|
@@ -4376,7 +4484,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4376
4484
|
WHERE id = $${i} AND tenant_id = $${i + 1}
|
|
4377
4485
|
RETURNING id, tenant_id, suite_id,
|
|
4378
4486
|
input_message, input_files, steps,
|
|
4379
|
-
output_type, content_assertion, rubrics,
|
|
4487
|
+
output_type, content_assertion, rubrics, interrupt_policy,
|
|
4380
4488
|
created_at, updated_at`,
|
|
4381
4489
|
vals
|
|
4382
4490
|
);
|
|
@@ -4411,7 +4519,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4411
4519
|
const { rows } = await this.pool.query(
|
|
4412
4520
|
`SELECT id, project_id, tenant_id,
|
|
4413
4521
|
status, concurrency, total_cases,
|
|
4414
|
-
passed_cases, failed_cases, avg_score,
|
|
4522
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
4415
4523
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at
|
|
4416
4524
|
FROM lattice_eval_runs
|
|
4417
4525
|
WHERE ${conditions.join(" AND ")}
|
|
@@ -4426,7 +4534,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4426
4534
|
const { rows } = await this.pool.query(
|
|
4427
4535
|
`SELECT id, project_id, tenant_id,
|
|
4428
4536
|
status, concurrency, total_cases,
|
|
4429
|
-
passed_cases, failed_cases, avg_score,
|
|
4537
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
4430
4538
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at
|
|
4431
4539
|
FROM lattice_eval_runs
|
|
4432
4540
|
WHERE id = $1 AND tenant_id = $2`,
|
|
@@ -4444,7 +4552,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4444
4552
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
4445
4553
|
RETURNING id, project_id, tenant_id,
|
|
4446
4554
|
status, concurrency, total_cases,
|
|
4447
|
-
passed_cases, failed_cases, avg_score,
|
|
4555
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
4448
4556
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at`,
|
|
4449
4557
|
[
|
|
4450
4558
|
actualId,
|
|
@@ -4482,6 +4590,10 @@ var PostgreSQLEvalStore = class {
|
|
|
4482
4590
|
set.push(`failed_cases = $${i++}`);
|
|
4483
4591
|
vals.push(updates.failedCases);
|
|
4484
4592
|
}
|
|
4593
|
+
if (updates.interruptedCases !== void 0) {
|
|
4594
|
+
set.push(`interrupted_cases = $${i++}`);
|
|
4595
|
+
vals.push(updates.interruptedCases);
|
|
4596
|
+
}
|
|
4485
4597
|
if (updates.avgScore !== void 0) {
|
|
4486
4598
|
set.push(`avg_score = $${i++}`);
|
|
4487
4599
|
vals.push(updates.avgScore);
|
|
@@ -4501,7 +4613,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4501
4613
|
WHERE id = $${i} AND tenant_id = $${i + 1}
|
|
4502
4614
|
RETURNING id, project_id, tenant_id,
|
|
4503
4615
|
status, concurrency, total_cases,
|
|
4504
|
-
passed_cases, failed_cases, avg_score,
|
|
4616
|
+
passed_cases, failed_cases, interrupted_cases, avg_score,
|
|
4505
4617
|
error, holdout, env_project_id, env_workspace_id, created_at, started_at, completed_at`,
|
|
4506
4618
|
vals
|
|
4507
4619
|
);
|
|
@@ -4526,7 +4638,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4526
4638
|
const { rows } = await this.pool.query(
|
|
4527
4639
|
`SELECT rr.id, rr.run_id, rr.suite_name, rr.case_id,
|
|
4528
4640
|
rr.pass, rr.score, rr.summary, rr.dimension_results,
|
|
4529
|
-
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.created_at
|
|
4641
|
+
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.interrupted, rr.created_at
|
|
4530
4642
|
FROM lattice_eval_run_results rr
|
|
4531
4643
|
INNER JOIN lattice_eval_runs r ON r.id = rr.run_id
|
|
4532
4644
|
WHERE r.tenant_id = $1 AND rr.run_id = $2
|
|
@@ -4541,11 +4653,11 @@ var PostgreSQLEvalStore = class {
|
|
|
4541
4653
|
const actualId = id || uuidv4();
|
|
4542
4654
|
const { rows } = await this.pool.query(
|
|
4543
4655
|
`INSERT INTO lattice_eval_run_results
|
|
4544
|
-
(id, run_id, suite_name, case_id, pass, score, summary, dimension_results, duration_ms, messages, logs, error)
|
|
4545
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
4656
|
+
(id, run_id, suite_name, case_id, pass, score, summary, dimension_results, duration_ms, messages, logs, error, interrupted)
|
|
4657
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
4546
4658
|
RETURNING id, run_id, suite_name, case_id,
|
|
4547
4659
|
pass, score, summary, dimension_results,
|
|
4548
|
-
duration_ms, messages, logs, error, created_at`,
|
|
4660
|
+
duration_ms, messages, logs, error, interrupted, created_at`,
|
|
4549
4661
|
[
|
|
4550
4662
|
actualId,
|
|
4551
4663
|
runId,
|
|
@@ -4558,7 +4670,8 @@ var PostgreSQLEvalStore = class {
|
|
|
4558
4670
|
data.durationMs ?? null,
|
|
4559
4671
|
JSON.stringify(data.messages || []),
|
|
4560
4672
|
JSON.stringify(data.logs || []),
|
|
4561
|
-
data.error || null
|
|
4673
|
+
data.error || null,
|
|
4674
|
+
data.interrupted ?? false
|
|
4562
4675
|
]
|
|
4563
4676
|
);
|
|
4564
4677
|
return this.mapRowToRunResult(rows[0]);
|
|
@@ -4612,6 +4725,10 @@ var PostgreSQLEvalStore = class {
|
|
|
4612
4725
|
set.push(`error = $${i++}`);
|
|
4613
4726
|
vals.push(updates.error);
|
|
4614
4727
|
}
|
|
4728
|
+
if (updates.interrupted !== void 0) {
|
|
4729
|
+
set.push(`interrupted = $${i++}`);
|
|
4730
|
+
vals.push(updates.interrupted);
|
|
4731
|
+
}
|
|
4615
4732
|
if (set.length === 0) return this.getRunResultById(tenantId, id);
|
|
4616
4733
|
vals.push(id, tenantId);
|
|
4617
4734
|
const { rows } = await this.pool.query(
|
|
@@ -4642,7 +4759,7 @@ var PostgreSQLEvalStore = class {
|
|
|
4642
4759
|
const { rows } = await this.pool.query(
|
|
4643
4760
|
`SELECT rr.id, rr.run_id, rr.suite_name, rr.case_id,
|
|
4644
4761
|
rr.pass, rr.score, rr.summary, rr.dimension_results,
|
|
4645
|
-
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.created_at
|
|
4762
|
+
rr.duration_ms, rr.messages, rr.logs, rr.error, rr.interrupted, rr.created_at
|
|
4646
4763
|
FROM lattice_eval_run_results rr
|
|
4647
4764
|
INNER JOIN lattice_eval_runs r ON r.id = rr.run_id
|
|
4648
4765
|
WHERE rr.id = $1 AND r.tenant_id = $2`,
|
|
@@ -8462,6 +8579,8 @@ export {
|
|
|
8462
8579
|
addEnvToEvalRuns,
|
|
8463
8580
|
addFileContentType,
|
|
8464
8581
|
addHoldoutToEvalRuns,
|
|
8582
|
+
addInterruptColumnsToEval,
|
|
8583
|
+
addInterruptPolicyToEvalCases,
|
|
8465
8584
|
addProjectConfigColumn,
|
|
8466
8585
|
addScheduleTenantId,
|
|
8467
8586
|
addSkillTenantId,
|
|
@@ -8499,6 +8618,7 @@ export {
|
|
|
8499
8618
|
createUsersTable,
|
|
8500
8619
|
createWorkflowTrackingTables,
|
|
8501
8620
|
createWorkspacesTable,
|
|
8621
|
+
enforceUniqueEvalProjectName,
|
|
8502
8622
|
evalMigrations,
|
|
8503
8623
|
mapRowToRunStep,
|
|
8504
8624
|
mapRowToWorkflowRun,
|