@axiom-lattice/local-stores 2.0.4 → 2.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/CHANGELOG.md +14 -0
- package/dist/index.d.mts +3 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +94 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +94 -4
- package/dist/index.mjs.map +1 -1
- package/jest.config.js +19 -0
- package/package.json +9 -2
- package/src/__tests__/LocalEvalStore.test.ts +96 -0
- package/src/stores/LocalEvalStore.ts +97 -7
package/dist/index.mjs
CHANGED
|
@@ -1920,6 +1920,7 @@ CREATE TABLE IF NOT EXISTS lt_eval_cases (
|
|
|
1920
1920
|
output_type TEXT NOT NULL DEFAULT 'message_content',
|
|
1921
1921
|
content_assertion TEXT NOT NULL DEFAULT '',
|
|
1922
1922
|
rubrics TEXT,
|
|
1923
|
+
interrupt_policy TEXT,
|
|
1923
1924
|
created_at TEXT NOT NULL,
|
|
1924
1925
|
updated_at TEXT NOT NULL,
|
|
1925
1926
|
PRIMARY KEY (tenant_id, id)
|
|
@@ -1935,6 +1936,7 @@ CREATE TABLE IF NOT EXISTS lt_eval_runs (
|
|
|
1935
1936
|
total_cases INTEGER NOT NULL DEFAULT 0,
|
|
1936
1937
|
passed_cases INTEGER NOT NULL DEFAULT 0,
|
|
1937
1938
|
failed_cases INTEGER NOT NULL DEFAULT 0,
|
|
1939
|
+
interrupted_cases INTEGER NOT NULL DEFAULT 0,
|
|
1938
1940
|
avg_score REAL NOT NULL DEFAULT 0,
|
|
1939
1941
|
error TEXT,
|
|
1940
1942
|
holdout INTEGER NOT NULL DEFAULT 0,
|
|
@@ -1960,6 +1962,7 @@ CREATE TABLE IF NOT EXISTS lt_eval_run_results (
|
|
|
1960
1962
|
messages TEXT DEFAULT '[]',
|
|
1961
1963
|
logs TEXT DEFAULT '[]',
|
|
1962
1964
|
error TEXT,
|
|
1965
|
+
interrupted INTEGER NOT NULL DEFAULT 0,
|
|
1963
1966
|
created_at TEXT NOT NULL,
|
|
1964
1967
|
PRIMARY KEY (run_id, id)
|
|
1965
1968
|
);
|
|
@@ -1990,6 +1993,76 @@ var LocalEvalStore = class {
|
|
|
1990
1993
|
constructor(db) {
|
|
1991
1994
|
this.db = db;
|
|
1992
1995
|
ensureTable(db, DDL13);
|
|
1996
|
+
this.ensureColumn("lt_eval_runs", "interrupted_cases", "interrupted_cases INTEGER NOT NULL DEFAULT 0");
|
|
1997
|
+
this.ensureColumn("lt_eval_run_results", "interrupted", "interrupted INTEGER NOT NULL DEFAULT 0");
|
|
1998
|
+
this.ensureColumn("lt_eval_cases", "interrupt_policy", "interrupt_policy TEXT");
|
|
1999
|
+
this.db.exec(`
|
|
2000
|
+
UPDATE lt_eval_suites
|
|
2001
|
+
SET project_id = (
|
|
2002
|
+
SELECT p2.id FROM lt_eval_projects p2
|
|
2003
|
+
WHERE p2.tenant_id = (
|
|
2004
|
+
SELECT p.tenant_id FROM lt_eval_projects p WHERE p.id = lt_eval_suites.project_id
|
|
2005
|
+
)
|
|
2006
|
+
AND p2.name = (
|
|
2007
|
+
SELECT p.name FROM lt_eval_projects p WHERE p.id = lt_eval_suites.project_id
|
|
2008
|
+
)
|
|
2009
|
+
ORDER BY p2.created_at DESC, p2.rowid DESC
|
|
2010
|
+
LIMIT 1
|
|
2011
|
+
)
|
|
2012
|
+
WHERE project_id IN (
|
|
2013
|
+
SELECT p3.id FROM lt_eval_projects p3
|
|
2014
|
+
WHERE p3.id <> (
|
|
2015
|
+
SELECT p2.id FROM lt_eval_projects p2
|
|
2016
|
+
WHERE p2.tenant_id = p3.tenant_id AND p2.name = p3.name
|
|
2017
|
+
ORDER BY p2.created_at DESC, p2.rowid DESC
|
|
2018
|
+
LIMIT 1
|
|
2019
|
+
)
|
|
2020
|
+
)
|
|
2021
|
+
`);
|
|
2022
|
+
this.db.exec(`
|
|
2023
|
+
UPDATE lt_eval_runs
|
|
2024
|
+
SET project_id = (
|
|
2025
|
+
SELECT p2.id FROM lt_eval_projects p2
|
|
2026
|
+
WHERE p2.tenant_id = (
|
|
2027
|
+
SELECT p.tenant_id FROM lt_eval_projects p WHERE p.id = lt_eval_runs.project_id
|
|
2028
|
+
)
|
|
2029
|
+
AND p2.name = (
|
|
2030
|
+
SELECT p.name FROM lt_eval_projects p WHERE p.id = lt_eval_runs.project_id
|
|
2031
|
+
)
|
|
2032
|
+
ORDER BY p2.created_at DESC, p2.rowid DESC
|
|
2033
|
+
LIMIT 1
|
|
2034
|
+
)
|
|
2035
|
+
WHERE project_id IN (
|
|
2036
|
+
SELECT p3.id FROM lt_eval_projects p3
|
|
2037
|
+
WHERE p3.id <> (
|
|
2038
|
+
SELECT p2.id FROM lt_eval_projects p2
|
|
2039
|
+
WHERE p2.tenant_id = p3.tenant_id AND p2.name = p3.name
|
|
2040
|
+
ORDER BY p2.created_at DESC, p2.rowid DESC
|
|
2041
|
+
LIMIT 1
|
|
2042
|
+
)
|
|
2043
|
+
)
|
|
2044
|
+
`);
|
|
2045
|
+
this.db.exec(`
|
|
2046
|
+
DELETE FROM lt_eval_projects
|
|
2047
|
+
WHERE id <> (
|
|
2048
|
+
SELECT p2.id FROM lt_eval_projects p2
|
|
2049
|
+
WHERE p2.tenant_id = lt_eval_projects.tenant_id
|
|
2050
|
+
AND p2.name = lt_eval_projects.name
|
|
2051
|
+
ORDER BY p2.created_at DESC, p2.rowid DESC
|
|
2052
|
+
LIMIT 1
|
|
2053
|
+
)
|
|
2054
|
+
`);
|
|
2055
|
+
this.db.exec(`
|
|
2056
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_lt_eval_projects_tenant_name
|
|
2057
|
+
ON lt_eval_projects(tenant_id, name)
|
|
2058
|
+
`);
|
|
2059
|
+
}
|
|
2060
|
+
/** Add a column if it does not exist (SQLite version compatible). */
|
|
2061
|
+
ensureColumn(table, column, ddl) {
|
|
2062
|
+
const cols = this.db.prepare(`PRAGMA table_info(${table})`).all();
|
|
2063
|
+
if (!cols.some((c) => c.name === column)) {
|
|
2064
|
+
this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${ddl}`);
|
|
2065
|
+
}
|
|
1993
2066
|
}
|
|
1994
2067
|
// -------------------------------------------------------------------------
|
|
1995
2068
|
// Projects
|
|
@@ -2134,8 +2207,8 @@ var LocalEvalStore = class {
|
|
|
2134
2207
|
async createCase(tenantId, suiteId, id, data) {
|
|
2135
2208
|
const actualId = id || randomUUID();
|
|
2136
2209
|
this.db.prepare(
|
|
2137
|
-
`INSERT INTO lt_eval_cases (id, tenant_id, suite_id, input_message, input_files, steps, output_type, content_assertion, rubrics, created_at, updated_at)
|
|
2138
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2210
|
+
`INSERT INTO lt_eval_cases (id, tenant_id, suite_id, input_message, input_files, steps, output_type, content_assertion, rubrics, interrupt_policy, created_at, updated_at)
|
|
2211
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2139
2212
|
).run(
|
|
2140
2213
|
actualId,
|
|
2141
2214
|
tenantId,
|
|
@@ -2146,6 +2219,7 @@ var LocalEvalStore = class {
|
|
|
2146
2219
|
data.outputType || "message_content",
|
|
2147
2220
|
data.contentAssertion || "",
|
|
2148
2221
|
data.rubrics ? JSON.stringify(data.rubrics) : null,
|
|
2222
|
+
data.interruptPolicy ? JSON.stringify(data.interruptPolicy) : null,
|
|
2149
2223
|
nowISO(),
|
|
2150
2224
|
nowISO()
|
|
2151
2225
|
);
|
|
@@ -2180,6 +2254,10 @@ var LocalEvalStore = class {
|
|
|
2180
2254
|
set.push("rubrics = ?");
|
|
2181
2255
|
vals.push(JSON.stringify(updates.rubrics));
|
|
2182
2256
|
}
|
|
2257
|
+
if (updates.interruptPolicy !== void 0) {
|
|
2258
|
+
set.push("interrupt_policy = ?");
|
|
2259
|
+
vals.push(updates.interruptPolicy ? JSON.stringify(updates.interruptPolicy) : null);
|
|
2260
|
+
}
|
|
2183
2261
|
set.push("updated_at = ?");
|
|
2184
2262
|
vals.push(nowISO());
|
|
2185
2263
|
vals.push(tenantId, id);
|
|
@@ -2241,6 +2319,10 @@ var LocalEvalStore = class {
|
|
|
2241
2319
|
set.push("failed_cases = ?");
|
|
2242
2320
|
vals.push(updates.failedCases);
|
|
2243
2321
|
}
|
|
2322
|
+
if (updates.interruptedCases !== void 0) {
|
|
2323
|
+
set.push("interrupted_cases = ?");
|
|
2324
|
+
vals.push(updates.interruptedCases);
|
|
2325
|
+
}
|
|
2244
2326
|
if (updates.avgScore !== void 0) {
|
|
2245
2327
|
set.push("avg_score = ?");
|
|
2246
2328
|
vals.push(updates.avgScore);
|
|
@@ -2285,8 +2367,8 @@ var LocalEvalStore = class {
|
|
|
2285
2367
|
async createRunResult(tenantId, runId, id, data) {
|
|
2286
2368
|
const actualId = id || randomUUID();
|
|
2287
2369
|
this.db.prepare(
|
|
2288
|
-
`INSERT INTO lt_eval_run_results (id, run_id, suite_name, case_id, pass, score, summary, dimension_results, duration_ms, messages, logs, error, created_at)
|
|
2289
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2370
|
+
`INSERT INTO lt_eval_run_results (id, run_id, suite_name, case_id, pass, score, summary, dimension_results, duration_ms, messages, logs, error, interrupted, created_at)
|
|
2371
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2290
2372
|
).run(
|
|
2291
2373
|
actualId,
|
|
2292
2374
|
runId,
|
|
@@ -2300,6 +2382,7 @@ var LocalEvalStore = class {
|
|
|
2300
2382
|
JSON.stringify(data.messages || []),
|
|
2301
2383
|
JSON.stringify(data.logs || []),
|
|
2302
2384
|
data.error || null,
|
|
2385
|
+
data.interrupted ? 1 : 0,
|
|
2303
2386
|
nowISO()
|
|
2304
2387
|
);
|
|
2305
2388
|
return await this.getRunResultById(tenantId, actualId);
|
|
@@ -2348,6 +2431,10 @@ var LocalEvalStore = class {
|
|
|
2348
2431
|
set.push("error = ?");
|
|
2349
2432
|
vals.push(updates.error || null);
|
|
2350
2433
|
}
|
|
2434
|
+
if (updates.interrupted !== void 0) {
|
|
2435
|
+
set.push("interrupted = ?");
|
|
2436
|
+
vals.push(updates.interrupted ? 1 : 0);
|
|
2437
|
+
}
|
|
2351
2438
|
if (set.length === 0) return this.getRunResultById(tenantId, id);
|
|
2352
2439
|
vals.push(id, tenantId);
|
|
2353
2440
|
this.db.prepare(
|
|
@@ -2415,6 +2502,7 @@ var LocalEvalStore = class {
|
|
|
2415
2502
|
outputType: row.output_type || "message_content",
|
|
2416
2503
|
contentAssertion: row.content_assertion || "",
|
|
2417
2504
|
rubrics: parseOptionalJson(row.rubrics),
|
|
2505
|
+
interruptPolicy: parseOptionalJson(row.interrupt_policy),
|
|
2418
2506
|
createdAt: parseISO(row.created_at),
|
|
2419
2507
|
updatedAt: parseISO(row.updated_at)
|
|
2420
2508
|
};
|
|
@@ -2429,6 +2517,7 @@ var LocalEvalStore = class {
|
|
|
2429
2517
|
totalCases: row.total_cases ?? 0,
|
|
2430
2518
|
passedCases: row.passed_cases ?? 0,
|
|
2431
2519
|
failedCases: row.failed_cases ?? 0,
|
|
2520
|
+
interruptedCases: row.interrupted_cases ?? 0,
|
|
2432
2521
|
avgScore: row.avg_score ?? 0,
|
|
2433
2522
|
error: row.error,
|
|
2434
2523
|
holdout: row.holdout ? Boolean(row.holdout) : void 0,
|
|
@@ -2453,6 +2542,7 @@ var LocalEvalStore = class {
|
|
|
2453
2542
|
messages: parseOptionalJson(row.messages),
|
|
2454
2543
|
logs: parseOptionalJson(row.logs),
|
|
2455
2544
|
error: row.error,
|
|
2545
|
+
interrupted: row.interrupted === 1,
|
|
2456
2546
|
createdAt: parseISO(row.created_at)
|
|
2457
2547
|
};
|
|
2458
2548
|
}
|