@axiom-lattice/local-stores 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +20 -0
- package/CHANGELOG.md +10 -0
- package/LICENSE +201 -0
- package/dist/index.d.mts +547 -0
- package/dist/index.d.ts +547 -0
- package/dist/index.js +3372 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +3308 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +42 -0
- package/src/createLocalStoreConfig.ts +81 -0
- package/src/database.ts +259 -0
- package/src/index.ts +44 -0
- package/src/stores/LocalA2AApiKeyStore.ts +177 -0
- package/src/stores/LocalAssistantStore.ts +149 -0
- package/src/stores/LocalChannelBindingStore.ts +204 -0
- package/src/stores/LocalChannelInstallationStore.ts +170 -0
- package/src/stores/LocalDatabaseConfigStore.ts +183 -0
- package/src/stores/LocalEvalStore.ts +510 -0
- package/src/stores/LocalMcpServerConfigStore.ts +223 -0
- package/src/stores/LocalMetricsServerConfigStore.ts +162 -0
- package/src/stores/LocalProjectStore.ts +150 -0
- package/src/stores/LocalScheduleStorage.ts +287 -0
- package/src/stores/LocalSkillStore.ts +188 -0
- package/src/stores/LocalTenantStore.ts +128 -0
- package/src/stores/LocalThreadMessageQueueStore.ts +172 -0
- package/src/stores/LocalThreadStore.ts +149 -0
- package/src/stores/LocalUserStore.ts +136 -0
- package/src/stores/LocalUserTenantLinkStore.ts +135 -0
- package/src/stores/LocalWorkflowTrackingStore.ts +299 -0
- package/src/stores/LocalWorkspaceStore.ts +143 -0
- package/tsconfig.json +23 -0
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local SQLite implementation of EvalStore.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { DatabaseWrapper } from "../database";
|
|
6
|
+
import {
|
|
7
|
+
EvalStore,
|
|
8
|
+
EvalProject,
|
|
9
|
+
CreateEvalProjectRequest,
|
|
10
|
+
EvalSuite,
|
|
11
|
+
CreateEvalSuiteRequest,
|
|
12
|
+
EvalCase,
|
|
13
|
+
CreateEvalCaseRequest,
|
|
14
|
+
EvalRun,
|
|
15
|
+
CreateEvalRunRequest,
|
|
16
|
+
EvalRunResult,
|
|
17
|
+
EvalProjectReport,
|
|
18
|
+
} from "@axiom-lattice/protocols";
|
|
19
|
+
import { ensureTable, nowISO, parseISO } from "../database";
|
|
20
|
+
import { randomUUID } from "crypto";
|
|
21
|
+
|
|
22
|
+
const DDL = `
|
|
23
|
+
CREATE TABLE IF NOT EXISTS lt_eval_projects (
|
|
24
|
+
id TEXT NOT NULL,
|
|
25
|
+
tenant_id TEXT NOT NULL,
|
|
26
|
+
name TEXT NOT NULL,
|
|
27
|
+
description TEXT,
|
|
28
|
+
version TEXT,
|
|
29
|
+
judge_model_config TEXT NOT NULL DEFAULT '{}',
|
|
30
|
+
target_server_config TEXT NOT NULL DEFAULT '{}',
|
|
31
|
+
concurrency INTEGER NOT NULL DEFAULT 3,
|
|
32
|
+
report_config TEXT,
|
|
33
|
+
created_at TEXT NOT NULL,
|
|
34
|
+
updated_at TEXT NOT NULL,
|
|
35
|
+
PRIMARY KEY (tenant_id, id)
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
CREATE TABLE IF NOT EXISTS lt_eval_suites (
|
|
39
|
+
id TEXT NOT NULL,
|
|
40
|
+
tenant_id TEXT NOT NULL,
|
|
41
|
+
project_id TEXT NOT NULL,
|
|
42
|
+
name TEXT NOT NULL,
|
|
43
|
+
created_at TEXT NOT NULL,
|
|
44
|
+
updated_at TEXT NOT NULL,
|
|
45
|
+
PRIMARY KEY (tenant_id, id)
|
|
46
|
+
);
|
|
47
|
+
CREATE INDEX IF NOT EXISTS idx_lt_es_project ON lt_eval_suites(tenant_id, project_id);
|
|
48
|
+
|
|
49
|
+
CREATE TABLE IF NOT EXISTS lt_eval_cases (
|
|
50
|
+
id TEXT NOT NULL,
|
|
51
|
+
tenant_id TEXT NOT NULL,
|
|
52
|
+
suite_id TEXT NOT NULL,
|
|
53
|
+
input_message TEXT NOT NULL,
|
|
54
|
+
input_files TEXT,
|
|
55
|
+
steps TEXT NOT NULL DEFAULT '[]',
|
|
56
|
+
output_type TEXT NOT NULL DEFAULT 'message_content',
|
|
57
|
+
content_assertion TEXT NOT NULL DEFAULT '',
|
|
58
|
+
rubrics TEXT,
|
|
59
|
+
created_at TEXT NOT NULL,
|
|
60
|
+
updated_at TEXT NOT NULL,
|
|
61
|
+
PRIMARY KEY (tenant_id, id)
|
|
62
|
+
);
|
|
63
|
+
CREATE INDEX IF NOT EXISTS idx_lt_ec_suite ON lt_eval_cases(tenant_id, suite_id);
|
|
64
|
+
|
|
65
|
+
CREATE TABLE IF NOT EXISTS lt_eval_runs (
|
|
66
|
+
id TEXT NOT NULL,
|
|
67
|
+
project_id TEXT NOT NULL,
|
|
68
|
+
tenant_id TEXT NOT NULL,
|
|
69
|
+
status TEXT NOT NULL DEFAULT 'running',
|
|
70
|
+
concurrency INTEGER NOT NULL DEFAULT 3,
|
|
71
|
+
total_cases INTEGER NOT NULL DEFAULT 0,
|
|
72
|
+
passed_cases INTEGER NOT NULL DEFAULT 0,
|
|
73
|
+
failed_cases INTEGER NOT NULL DEFAULT 0,
|
|
74
|
+
avg_score REAL NOT NULL DEFAULT 0,
|
|
75
|
+
error TEXT,
|
|
76
|
+
created_at TEXT NOT NULL,
|
|
77
|
+
started_at TEXT,
|
|
78
|
+
completed_at TEXT,
|
|
79
|
+
PRIMARY KEY (tenant_id, id)
|
|
80
|
+
);
|
|
81
|
+
CREATE INDEX IF NOT EXISTS idx_lt_er_project ON lt_eval_runs(tenant_id, project_id);
|
|
82
|
+
|
|
83
|
+
CREATE TABLE IF NOT EXISTS lt_eval_run_results (
|
|
84
|
+
id TEXT NOT NULL,
|
|
85
|
+
run_id TEXT NOT NULL,
|
|
86
|
+
suite_name TEXT NOT NULL,
|
|
87
|
+
case_id TEXT,
|
|
88
|
+
pass INTEGER NOT NULL DEFAULT 0,
|
|
89
|
+
score REAL NOT NULL DEFAULT 0,
|
|
90
|
+
summary TEXT,
|
|
91
|
+
dimension_results TEXT DEFAULT '[]',
|
|
92
|
+
duration_ms INTEGER,
|
|
93
|
+
messages TEXT DEFAULT '[]',
|
|
94
|
+
logs TEXT DEFAULT '[]',
|
|
95
|
+
error TEXT,
|
|
96
|
+
created_at TEXT NOT NULL,
|
|
97
|
+
PRIMARY KEY (run_id, id)
|
|
98
|
+
);
|
|
99
|
+
`;
|
|
100
|
+
|
|
101
|
+
function parseJson<T>(val: unknown, fallback: T): T {
|
|
102
|
+
if (val == null) return fallback;
|
|
103
|
+
if (typeof val === "string") {
|
|
104
|
+
try { return JSON.parse(val) as T; } catch { return fallback; }
|
|
105
|
+
}
|
|
106
|
+
return val as T;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function parseOptionalJson<T>(val: unknown): T | undefined {
|
|
110
|
+
if (val == null) return undefined;
|
|
111
|
+
if (typeof val === "string") {
|
|
112
|
+
try { return JSON.parse(val) as T; } catch { return undefined; }
|
|
113
|
+
}
|
|
114
|
+
return val as T;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export class LocalEvalStore implements EvalStore {
|
|
118
|
+
private db: DatabaseWrapper;
|
|
119
|
+
|
|
120
|
+
constructor(db: DatabaseWrapper) {
|
|
121
|
+
this.db = db;
|
|
122
|
+
ensureTable(db, DDL);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// -------------------------------------------------------------------------
|
|
126
|
+
// Projects
|
|
127
|
+
// -------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
async getProjectsByTenant(tenantId: string): Promise<EvalProject[]> {
|
|
130
|
+
const rows = this.db.prepare(
|
|
131
|
+
`SELECT * FROM lt_eval_projects WHERE tenant_id = ? ORDER BY created_at DESC`,
|
|
132
|
+
).all(tenantId) as unknown as Record<string, unknown>[];
|
|
133
|
+
return rows.map(this.mapRowToProject);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async getProjectById(tenantId: string, id: string): Promise<EvalProject | null> {
|
|
137
|
+
const row = this.db.prepare(
|
|
138
|
+
`SELECT * FROM lt_eval_projects WHERE tenant_id = ? AND id = ?`,
|
|
139
|
+
).get(tenantId, id) as unknown as Record<string, unknown> | undefined;
|
|
140
|
+
return row ? this.mapRowToProject(row) : null;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async createProject(tenantId: string, id: string, data: CreateEvalProjectRequest): Promise<EvalProject> {
|
|
144
|
+
const actualId = id || randomUUID();
|
|
145
|
+
this.db.prepare(
|
|
146
|
+
`INSERT INTO lt_eval_projects (id, tenant_id, name, description, version, judge_model_config, target_server_config, concurrency, report_config, created_at, updated_at)
|
|
147
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
148
|
+
).run(actualId, tenantId, data.name, data.description || null, data.version || null,
|
|
149
|
+
JSON.stringify(data.judgeModelConfig), JSON.stringify(data.targetServerConfig), data.concurrency ?? 3,
|
|
150
|
+
data.reportConfig === undefined ? null : JSON.stringify(data.reportConfig),
|
|
151
|
+
nowISO(), nowISO());
|
|
152
|
+
return (await this.getProjectById(tenantId, actualId))!;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async updateProject(tenantId: string, id: string, updates: Partial<CreateEvalProjectRequest>): Promise<EvalProject | null> {
|
|
156
|
+
const existing = await this.getProjectById(tenantId, id);
|
|
157
|
+
if (!existing) return null;
|
|
158
|
+
|
|
159
|
+
const set: string[] = [];
|
|
160
|
+
const vals: unknown[] = [];
|
|
161
|
+
if (updates.name !== undefined) { set.push("name = ?"); vals.push(updates.name); }
|
|
162
|
+
if (updates.description !== undefined) { set.push("description = ?"); vals.push(updates.description || null); }
|
|
163
|
+
if (updates.version !== undefined) { set.push("version = ?"); vals.push(updates.version || null); }
|
|
164
|
+
if (updates.judgeModelConfig !== undefined) { set.push("judge_model_config = ?"); vals.push(JSON.stringify(updates.judgeModelConfig)); }
|
|
165
|
+
if (updates.targetServerConfig !== undefined) { set.push("target_server_config = ?"); vals.push(JSON.stringify(updates.targetServerConfig)); }
|
|
166
|
+
if (updates.concurrency !== undefined) { set.push("concurrency = ?"); vals.push(updates.concurrency); }
|
|
167
|
+
if (updates.reportConfig !== undefined) { set.push("report_config = ?"); vals.push(JSON.stringify(updates.reportConfig)); }
|
|
168
|
+
set.push("updated_at = ?"); vals.push(nowISO());
|
|
169
|
+
vals.push(tenantId, id);
|
|
170
|
+
|
|
171
|
+
this.db.prepare(
|
|
172
|
+
`UPDATE lt_eval_projects SET ${set.join(", ")} WHERE tenant_id = ? AND id = ?`,
|
|
173
|
+
).run(...vals);
|
|
174
|
+
|
|
175
|
+
return this.getProjectById(tenantId, id);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
async deleteProject(tenantId: string, id: string): Promise<boolean> {
|
|
179
|
+
const result = this.db.prepare(`DELETE FROM lt_eval_projects WHERE tenant_id = ? AND id = ?`).run(tenantId, id);
|
|
180
|
+
return result.changes > 0;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// -------------------------------------------------------------------------
|
|
184
|
+
// Suites
|
|
185
|
+
// -------------------------------------------------------------------------
|
|
186
|
+
|
|
187
|
+
async getSuitesByProject(tenantId: string, projectId: string): Promise<EvalSuite[]> {
|
|
188
|
+
const rows = this.db.prepare(
|
|
189
|
+
`SELECT s.*, (SELECT COUNT(*) FROM lt_eval_cases c WHERE c.suite_id = s.id AND c.tenant_id = s.tenant_id) as case_count
|
|
190
|
+
FROM lt_eval_suites s WHERE s.tenant_id = ? AND s.project_id = ? ORDER BY s.created_at DESC`,
|
|
191
|
+
).all(tenantId, projectId) as unknown as Record<string, unknown>[];
|
|
192
|
+
return rows.map(this.mapRowToSuite);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async getSuiteById(tenantId: string, id: string): Promise<EvalSuite | null> {
|
|
196
|
+
const row = this.db.prepare(
|
|
197
|
+
`SELECT s.*, (SELECT COUNT(*) FROM lt_eval_cases c WHERE c.suite_id = s.id AND c.tenant_id = s.tenant_id) as case_count
|
|
198
|
+
FROM lt_eval_suites s WHERE s.tenant_id = ? AND s.id = ?`,
|
|
199
|
+
).get(tenantId, id) as unknown as Record<string, unknown> | undefined;
|
|
200
|
+
return row ? this.mapRowToSuite(row) : null;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
async createSuite(tenantId: string, projectId: string, id: string, data: CreateEvalSuiteRequest): Promise<EvalSuite> {
|
|
204
|
+
const actualId = id || randomUUID();
|
|
205
|
+
this.db.prepare(
|
|
206
|
+
`INSERT INTO lt_eval_suites (id, tenant_id, project_id, name, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
207
|
+
).run(actualId, tenantId, projectId, data.name, nowISO(), nowISO());
|
|
208
|
+
return (await this.getSuiteById(tenantId, actualId))!;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
async updateSuite(tenantId: string, id: string, updates: Partial<CreateEvalSuiteRequest>): Promise<EvalSuite | null> {
|
|
212
|
+
const existing = await this.getSuiteById(tenantId, id);
|
|
213
|
+
if (!existing) return null;
|
|
214
|
+
const set: string[] = [];
|
|
215
|
+
const vals: unknown[] = [];
|
|
216
|
+
if (updates.name !== undefined) { set.push("name = ?"); vals.push(updates.name); }
|
|
217
|
+
set.push("updated_at = ?"); vals.push(nowISO());
|
|
218
|
+
vals.push(tenantId, id);
|
|
219
|
+
|
|
220
|
+
this.db.prepare(
|
|
221
|
+
`UPDATE lt_eval_suites SET ${set.join(", ")} WHERE tenant_id = ? AND id = ?`,
|
|
222
|
+
).run(...vals);
|
|
223
|
+
return this.getSuiteById(tenantId, id);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async deleteSuite(tenantId: string, id: string): Promise<boolean> {
|
|
227
|
+
const result = this.db.prepare(`DELETE FROM lt_eval_suites WHERE tenant_id = ? AND id = ?`).run(tenantId, id);
|
|
228
|
+
return result.changes > 0;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// -------------------------------------------------------------------------
|
|
232
|
+
// Cases
|
|
233
|
+
// -------------------------------------------------------------------------
|
|
234
|
+
|
|
235
|
+
async getCasesBySuite(tenantId: string, suiteId: string): Promise<EvalCase[]> {
|
|
236
|
+
const rows = this.db.prepare(
|
|
237
|
+
`SELECT * FROM lt_eval_cases WHERE tenant_id = ? AND suite_id = ? ORDER BY created_at DESC`,
|
|
238
|
+
).all(tenantId, suiteId) as unknown as Record<string, unknown>[];
|
|
239
|
+
return rows.map(this.mapRowToCase);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async getCaseById(tenantId: string, id: string): Promise<EvalCase | null> {
|
|
243
|
+
const row = this.db.prepare(
|
|
244
|
+
`SELECT * FROM lt_eval_cases WHERE tenant_id = ? AND id = ?`,
|
|
245
|
+
).get(tenantId, id) as unknown as Record<string, unknown> | undefined;
|
|
246
|
+
return row ? this.mapRowToCase(row) : null;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
async createCase(tenantId: string, suiteId: string, id: string, data: CreateEvalCaseRequest): Promise<EvalCase> {
|
|
250
|
+
const actualId = id || randomUUID();
|
|
251
|
+
this.db.prepare(
|
|
252
|
+
`INSERT INTO lt_eval_cases (id, tenant_id, suite_id, input_message, input_files, steps, output_type, content_assertion, rubrics, created_at, updated_at)
|
|
253
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
254
|
+
).run(actualId, tenantId, suiteId, data.inputMessage,
|
|
255
|
+
data.inputFiles ? JSON.stringify(data.inputFiles) : null,
|
|
256
|
+
JSON.stringify(data.steps), data.outputType || "message_content",
|
|
257
|
+
data.contentAssertion || "",
|
|
258
|
+
data.rubrics ? JSON.stringify(data.rubrics) : null,
|
|
259
|
+
nowISO(), nowISO());
|
|
260
|
+
return (await this.getCaseById(tenantId, actualId))!;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
async updateCase(tenantId: string, id: string, updates: Partial<CreateEvalCaseRequest>): Promise<EvalCase | null> {
|
|
264
|
+
const existing = await this.getCaseById(tenantId, id);
|
|
265
|
+
if (!existing) return null;
|
|
266
|
+
|
|
267
|
+
const set: string[] = [];
|
|
268
|
+
const vals: unknown[] = [];
|
|
269
|
+
if (updates.inputMessage !== undefined) { set.push("input_message = ?"); vals.push(updates.inputMessage); }
|
|
270
|
+
if (updates.inputFiles !== undefined) { set.push("input_files = ?"); vals.push(JSON.stringify(updates.inputFiles)); }
|
|
271
|
+
if (updates.steps !== undefined) { set.push("steps = ?"); vals.push(JSON.stringify(updates.steps)); }
|
|
272
|
+
if (updates.outputType !== undefined) { set.push("output_type = ?"); vals.push(updates.outputType); }
|
|
273
|
+
if (updates.contentAssertion !== undefined) { set.push("content_assertion = ?"); vals.push(updates.contentAssertion); }
|
|
274
|
+
if (updates.rubrics !== undefined) { set.push("rubrics = ?"); vals.push(JSON.stringify(updates.rubrics)); }
|
|
275
|
+
set.push("updated_at = ?"); vals.push(nowISO());
|
|
276
|
+
vals.push(tenantId, id);
|
|
277
|
+
|
|
278
|
+
this.db.prepare(
|
|
279
|
+
`UPDATE lt_eval_cases SET ${set.join(", ")} WHERE tenant_id = ? AND id = ?`,
|
|
280
|
+
).run(...vals);
|
|
281
|
+
return this.getCaseById(tenantId, id);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async deleteCase(tenantId: string, id: string): Promise<boolean> {
|
|
285
|
+
const result = this.db.prepare(`DELETE FROM lt_eval_cases WHERE tenant_id = ? AND id = ?`).run(tenantId, id);
|
|
286
|
+
return result.changes > 0;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// -------------------------------------------------------------------------
|
|
290
|
+
// Runs
|
|
291
|
+
// -------------------------------------------------------------------------
|
|
292
|
+
|
|
293
|
+
async getRunsByTenant(tenantId: string, opts?: { projectId?: string; status?: string }): Promise<EvalRun[]> {
|
|
294
|
+
let query = `SELECT * FROM lt_eval_runs WHERE tenant_id = ?`;
|
|
295
|
+
const vals: unknown[] = [tenantId];
|
|
296
|
+
if (opts?.projectId) { query += ` AND project_id = ?`; vals.push(opts.projectId); }
|
|
297
|
+
if (opts?.status) { query += ` AND status = ?`; vals.push(opts.status); }
|
|
298
|
+
query += ` ORDER BY created_at DESC`;
|
|
299
|
+
const rows = this.db.prepare(query).all(...vals) as unknown as Record<string, unknown>[];
|
|
300
|
+
return rows.map(this.mapRowToRun);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async getRunById(tenantId: string, id: string): Promise<EvalRun | null> {
|
|
304
|
+
const row = this.db.prepare(
|
|
305
|
+
`SELECT * FROM lt_eval_runs WHERE tenant_id = ? AND id = ?`,
|
|
306
|
+
).get(tenantId, id) as unknown as Record<string, unknown> | undefined;
|
|
307
|
+
return row ? this.mapRowToRun(row) : null;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
async createRun(tenantId: string, projectId: string, id: string, data: CreateEvalRunRequest): Promise<EvalRun> {
|
|
311
|
+
const actualId = id || randomUUID();
|
|
312
|
+
this.db.prepare(
|
|
313
|
+
`INSERT INTO lt_eval_runs (id, project_id, tenant_id, status, concurrency, total_cases, passed_cases, failed_cases, avg_score, created_at)
|
|
314
|
+
VALUES (?, ?, ?, 'running', ?, ?, 0, 0, 0, ?)`,
|
|
315
|
+
).run(actualId, projectId, tenantId, data.concurrency, data.totalCases, nowISO());
|
|
316
|
+
return (await this.getRunById(tenantId, actualId))!;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
async updateRunStatus(tenantId: string, id: string, updates: {
|
|
320
|
+
status?: EvalRun["status"]; passedCases?: number; failedCases?: number; avgScore?: number; error?: string; completedAt?: Date;
|
|
321
|
+
}): Promise<EvalRun | null> {
|
|
322
|
+
const existing = await this.getRunById(tenantId, id);
|
|
323
|
+
if (!existing) return null;
|
|
324
|
+
|
|
325
|
+
const set: string[] = [];
|
|
326
|
+
const vals: unknown[] = [];
|
|
327
|
+
if (updates.status !== undefined) { set.push("status = ?"); vals.push(updates.status); }
|
|
328
|
+
if (updates.passedCases !== undefined) { set.push("passed_cases = ?"); vals.push(updates.passedCases); }
|
|
329
|
+
if (updates.failedCases !== undefined) { set.push("failed_cases = ?"); vals.push(updates.failedCases); }
|
|
330
|
+
if (updates.avgScore !== undefined) { set.push("avg_score = ?"); vals.push(updates.avgScore); }
|
|
331
|
+
if (updates.error !== undefined) { set.push("error = ?"); vals.push(updates.error || null); }
|
|
332
|
+
if (updates.completedAt !== undefined) { set.push("completed_at = ?"); vals.push(updates.completedAt ? updates.completedAt.toISOString() : null); }
|
|
333
|
+
vals.push(tenantId, id);
|
|
334
|
+
|
|
335
|
+
this.db.prepare(
|
|
336
|
+
`UPDATE lt_eval_runs SET ${set.join(", ")} WHERE tenant_id = ? AND id = ?`,
|
|
337
|
+
).run(...vals);
|
|
338
|
+
return this.getRunById(tenantId, id);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
async deleteRun(tenantId: string, id: string): Promise<boolean> {
|
|
342
|
+
const result = this.db.prepare(`DELETE FROM lt_eval_runs WHERE tenant_id = ? AND id = ?`).run(tenantId, id);
|
|
343
|
+
return result.changes > 0;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// -------------------------------------------------------------------------
|
|
347
|
+
// Run Results
|
|
348
|
+
// -------------------------------------------------------------------------
|
|
349
|
+
|
|
350
|
+
async getResultsByRun(tenantId: string, runId: string): Promise<EvalRunResult[]> {
|
|
351
|
+
const rows = this.db.prepare(
|
|
352
|
+
`SELECT rr.* FROM lt_eval_run_results rr
|
|
353
|
+
INNER JOIN lt_eval_runs r ON r.id = rr.run_id
|
|
354
|
+
WHERE r.tenant_id = ? AND rr.run_id = ? ORDER BY rr.created_at ASC`,
|
|
355
|
+
).all(tenantId, runId) as unknown as Record<string, unknown>[];
|
|
356
|
+
return rows.map(this.mapRowToRunResult);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
private async getRunResultById(tenantId: string, id: string): Promise<EvalRunResult | null> {
|
|
360
|
+
const row = this.db.prepare(
|
|
361
|
+
`SELECT rr.* FROM lt_eval_run_results rr
|
|
362
|
+
INNER JOIN lt_eval_runs r ON r.id = rr.run_id
|
|
363
|
+
WHERE r.tenant_id = ? AND rr.id = ?`,
|
|
364
|
+
).get(tenantId, id) as unknown as Record<string, unknown> | undefined;
|
|
365
|
+
return row ? this.mapRowToRunResult(row) : null;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
async createRunResult(tenantId: string, runId: string, id: string, data: Omit<EvalRunResult, "id" | "runId" | "createdAt">): Promise<EvalRunResult> {
|
|
369
|
+
const actualId = id || randomUUID();
|
|
370
|
+
this.db.prepare(
|
|
371
|
+
`INSERT INTO lt_eval_run_results (id, run_id, suite_name, case_id, pass, score, summary, dimension_results, duration_ms, messages, logs, error, created_at)
|
|
372
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
373
|
+
).run(actualId, runId, data.suiteName, data.caseId || null,
|
|
374
|
+
data.pass ? 1 : 0, data.score, data.summary || null,
|
|
375
|
+
JSON.stringify(data.dimensionResults || []), data.durationMs ?? null,
|
|
376
|
+
JSON.stringify(data.messages || []), JSON.stringify(data.logs || []),
|
|
377
|
+
data.error || null, nowISO());
|
|
378
|
+
return (await this.getRunResultById(tenantId, actualId))!;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
async updateRunResult(tenantId: string, id: string, updates: Partial<EvalRunResult>): Promise<EvalRunResult | null> {
|
|
382
|
+
if (updates.runId !== undefined) throw new Error("runId cannot be updated on an existing result");
|
|
383
|
+
|
|
384
|
+
const set: string[] = [];
|
|
385
|
+
const vals: unknown[] = [];
|
|
386
|
+
if (updates.suiteName !== undefined) { set.push("suite_name = ?"); vals.push(updates.suiteName); }
|
|
387
|
+
if (updates.caseId !== undefined) { set.push("case_id = ?"); vals.push(updates.caseId); }
|
|
388
|
+
if (updates.pass !== undefined) { set.push("pass = ?"); vals.push(updates.pass ? 1 : 0); }
|
|
389
|
+
if (updates.score !== undefined) { set.push("score = ?"); vals.push(updates.score); }
|
|
390
|
+
if (updates.summary !== undefined) { set.push("summary = ?"); vals.push(updates.summary || null); }
|
|
391
|
+
if (updates.dimensionResults !== undefined) { set.push("dimension_results = ?"); vals.push(JSON.stringify(updates.dimensionResults)); }
|
|
392
|
+
if (updates.durationMs !== undefined) { set.push("duration_ms = ?"); vals.push(updates.durationMs ?? null); }
|
|
393
|
+
if (updates.messages !== undefined) { set.push("messages = ?"); vals.push(JSON.stringify(updates.messages)); }
|
|
394
|
+
if (updates.logs !== undefined) { set.push("logs = ?"); vals.push(JSON.stringify(updates.logs)); }
|
|
395
|
+
if (updates.error !== undefined) { set.push("error = ?"); vals.push(updates.error || null); }
|
|
396
|
+
if (set.length === 0) return this.getRunResultById(tenantId, id);
|
|
397
|
+
|
|
398
|
+
vals.push(id, tenantId);
|
|
399
|
+
this.db.prepare(
|
|
400
|
+
`UPDATE lt_eval_run_results SET ${set.join(", ")}
|
|
401
|
+
WHERE id = ? AND run_id IN (SELECT id FROM lt_eval_runs WHERE tenant_id = ?)`,
|
|
402
|
+
).run(...vals);
|
|
403
|
+
|
|
404
|
+
return this.getRunResultById(tenantId, id);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// -------------------------------------------------------------------------
|
|
408
|
+
// Reports
|
|
409
|
+
// -------------------------------------------------------------------------
|
|
410
|
+
|
|
411
|
+
async getProjectReport(tenantId: string, projectId: string): Promise<EvalProjectReport | null> {
|
|
412
|
+
const project = await this.getProjectById(tenantId, projectId);
|
|
413
|
+
if (!project) return null;
|
|
414
|
+
|
|
415
|
+
const runs = await this.getRunsByTenant(tenantId, { projectId });
|
|
416
|
+
const totalRuns = runs.length;
|
|
417
|
+
const latestPassRate = runs.length > 0
|
|
418
|
+
? (runs[0].totalCases > 0 ? runs[0].passedCases / runs[0].totalCases : 0)
|
|
419
|
+
: 0;
|
|
420
|
+
const avgScore = runs.length > 0
|
|
421
|
+
? runs.reduce((sum, r) => sum + r.avgScore, 0) / runs.length
|
|
422
|
+
: 0;
|
|
423
|
+
|
|
424
|
+
return { projectId: project.id, projectName: project.name, totalRuns, latestPassRate, avgScore, runs };
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// -------------------------------------------------------------------------
|
|
428
|
+
// Row mappers
|
|
429
|
+
// -------------------------------------------------------------------------
|
|
430
|
+
|
|
431
|
+
private mapRowToProject(row: Record<string, unknown>): EvalProject {
|
|
432
|
+
return {
|
|
433
|
+
id: row.id as string,
|
|
434
|
+
tenantId: row.tenant_id as string,
|
|
435
|
+
name: row.name as string,
|
|
436
|
+
description: row.description as string | undefined,
|
|
437
|
+
version: row.version as string | undefined,
|
|
438
|
+
judgeModelConfig: parseJson<Record<string, unknown>>(row.judge_model_config, {}),
|
|
439
|
+
targetServerConfig: parseJson<Record<string, unknown>>(row.target_server_config, {}),
|
|
440
|
+
concurrency: (row.concurrency as number) ?? 3,
|
|
441
|
+
reportConfig: parseOptionalJson<Record<string, unknown>>(row.report_config),
|
|
442
|
+
createdAt: parseISO(row.created_at as string),
|
|
443
|
+
updatedAt: parseISO(row.updated_at as string),
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
private mapRowToSuite(row: Record<string, unknown>): EvalSuite {
|
|
448
|
+
return {
|
|
449
|
+
id: row.id as string,
|
|
450
|
+
tenantId: row.tenant_id as string,
|
|
451
|
+
projectId: row.project_id as string,
|
|
452
|
+
name: row.name as string,
|
|
453
|
+
createdAt: parseISO(row.created_at as string),
|
|
454
|
+
updatedAt: parseISO(row.updated_at as string),
|
|
455
|
+
caseCount: row.case_count !== undefined ? (row.case_count as number) : undefined,
|
|
456
|
+
};
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
private mapRowToCase(row: Record<string, unknown>): EvalCase {
|
|
460
|
+
return {
|
|
461
|
+
id: row.id as string,
|
|
462
|
+
tenantId: row.tenant_id as string,
|
|
463
|
+
suiteId: row.suite_id as string,
|
|
464
|
+
inputMessage: row.input_message as string,
|
|
465
|
+
inputFiles: parseOptionalJson<Record<string, string>>(row.input_files),
|
|
466
|
+
steps: parseJson<Array<{ agent_id: string; override_message?: string }>>(row.steps, []),
|
|
467
|
+
outputType: (row.output_type as EvalCase["outputType"]) || "message_content",
|
|
468
|
+
contentAssertion: (row.content_assertion as string) || "",
|
|
469
|
+
rubrics: parseOptionalJson<Array<{ name: string; weight: number; description: string }>>(row.rubrics),
|
|
470
|
+
createdAt: parseISO(row.created_at as string),
|
|
471
|
+
updatedAt: parseISO(row.updated_at as string),
|
|
472
|
+
};
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
private mapRowToRun(row: Record<string, unknown>): EvalRun {
|
|
476
|
+
return {
|
|
477
|
+
id: row.id as string,
|
|
478
|
+
projectId: row.project_id as string,
|
|
479
|
+
tenantId: row.tenant_id as string,
|
|
480
|
+
status: row.status as EvalRun["status"],
|
|
481
|
+
concurrency: (row.concurrency as number) ?? 3,
|
|
482
|
+
totalCases: (row.total_cases as number) ?? 0,
|
|
483
|
+
passedCases: (row.passed_cases as number) ?? 0,
|
|
484
|
+
failedCases: (row.failed_cases as number) ?? 0,
|
|
485
|
+
avgScore: (row.avg_score as number) ?? 0,
|
|
486
|
+
error: row.error as string | undefined,
|
|
487
|
+
createdAt: parseISO(row.created_at as string),
|
|
488
|
+
startedAt: row.started_at ? parseISO(row.started_at as string) : undefined,
|
|
489
|
+
completedAt: row.completed_at ? parseISO(row.completed_at as string) : undefined,
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
private mapRowToRunResult(row: Record<string, unknown>): EvalRunResult {
|
|
494
|
+
return {
|
|
495
|
+
id: row.id as string,
|
|
496
|
+
runId: row.run_id as string,
|
|
497
|
+
suiteName: row.suite_name as string,
|
|
498
|
+
caseId: row.case_id as string | undefined,
|
|
499
|
+
pass: (row.pass as number) === 1,
|
|
500
|
+
score: (row.score as number) ?? 0,
|
|
501
|
+
summary: row.summary as string | undefined,
|
|
502
|
+
dimensionResults: parseOptionalJson<Array<{ name: string; score: number; reason: string }>>(row.dimension_results),
|
|
503
|
+
durationMs: row.duration_ms as number | undefined,
|
|
504
|
+
messages: parseOptionalJson<Array<{ role: string; content: string; id?: string }>>(row.messages),
|
|
505
|
+
logs: parseOptionalJson<Array<{ timestamp: string; level: string; message: string; data?: unknown }>>(row.logs),
|
|
506
|
+
error: row.error as string | undefined,
|
|
507
|
+
createdAt: parseISO(row.created_at as string),
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
}
|