claude-test-bench 1.0.0

package/dist/server/services/storage-test-helpers.d.ts

@@ -0,0 +1,21 @@
+import type { FsAdapter } from './fs-adapter.js';
+import type { Provider, Scenario, Run, Evaluation } from '../types/index.js';
+export declare const BASE_PROVIDER: {
+    kind: "api";
+    baseUrl: string;
+    apiKey: string;
+    model: string;
+};
+export declare function makeProvider(overrides?: Partial<Provider>): Provider;
+export declare function makeScenario(overrides?: Partial<Scenario>): Scenario;
+export declare function makeRun(overrides?: Partial<Run>): Run;
+export declare function makeEvaluation(overrides?: Partial<Evaluation>): Evaluation;
+export interface MockFsState {
+    files: Map<string, string>;
+    dirs: Set<string>;
+}
+export declare function createMockFs(): {
+    fs: FsAdapter;
+    state: MockFsState;
+};
+//# sourceMappingURL=storage-test-helpers.d.ts.map

package/dist/server/services/storage-test-helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-test-helpers.d.ts","sourceRoot":"","sources":["../../../src/server/services/storage-test-helpers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,GAAG,EACH,UAAU,EACX,MAAM,mBAAmB,CAAC;AAI3B,eAAO,MAAM,aAAa;;;;;CAKzB,CAAC;AAEF,wBAAgB,YAAY,CAAC,SAAS,GAAE,OAAO,CAAC,QAAQ,CAAM,GAAG,QAAQ,CAWxE;AAED,wBAAgB,YAAY,CAAC,SAAS,GAAE,OAAO,CAAC,QAAQ,CAAM,GAAG,QAAQ,CAqBxE;AAED,wBAAgB,OAAO,CAAC,SAAS,GAAE,OAAO,CAAC,GAAG,CAAM,GAAG,GAAG,CAiBzD;AAED,wBAAgB,cAAc,CAAC,SAAS,GAAE,OAAO,CAAC,UAAU,CAAM,GAAG,UAAU,CA+B9E;AAID,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACnB;AAED,wBAAgB,YAAY,IAAI;IAAE,EAAE,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAyEpE"}

package/dist/server/services/storage-test-helpers.js

@@ -0,0 +1,158 @@
+import { vi } from 'vitest';
+// ─── Factory helpers ─────────────────────────────────────────────────
+export const BASE_PROVIDER = {
+    kind: 'api',
+    baseUrl: 'https://api.anthropic.com',
+    apiKey: 'sk-test',
+    model: 'claude-sonnet-4-6',
+};
+export function makeProvider(overrides = {}) {
+    return {
+        id: 'provider-1',
+        name: 'Test Provider',
+        description: 'A test provider',
+        provider: BASE_PROVIDER,
+        timeoutSeconds: 300,
+        createdAt: '2026-01-01T00:00:00.000Z',
+        updatedAt: '2026-01-01T00:00:00.000Z',
+        ...overrides,
+    };
+}
+export function makeScenario(overrides = {}) {
+    return {
+        id: 'scenario-1',
+        name: 'Test Scenario',
+        category: 'planning',
+        claudeMdFiles: [],
+        rules: [],
+        skills: [],
+        subagents: [],
+        mcpServers: [],
+        permissionMode: 'default',
+        prompt: 'Do something',
+        workspaceFiles: [],
+        expectedAnswer: 'done',
+        criticalRequirements: [],
+        gradingGuidelines: '',
+        scoringDimensions: [],
+        createdAt: '2026-01-01T00:00:00.000Z',
+        updatedAt: '2026-01-01T00:00:00.000Z',
+        ...overrides,
+    };
+}
+export function makeRun(overrides = {}) {
+    return {
+        id: 'run-1',
+        providerId: 'provider-1',
+        scenarioId: 'scenario-1',
+        status: 'completed',
+        providerSnapshot: makeProvider(),
+        scenarioSnapshot: makeScenario(),
+        messages: [],
+        resultText: 'done',
+        totalCostUsd: 0.01,
+        durationMs: 1000,
+        numTurns: 3,
+        createdAt: '2026-01-01T00:00:00.000Z',
+        updatedAt: '2026-01-01T00:00:01.000Z',
+        ...overrides,
+    };
+}
+export function makeEvaluation(overrides = {}) {
+    return {
+        id: 'eval-1',
+        runId: 'run-1',
+        status: 'completed',
+        evaluators: [],
+        rounds: [],
+        answerComparison: { matches: true, explanation: 'Good', similarity: 0.9 },
+        criticalResults: [],
+        setupCompliance: {
+            instructionCompliance: {
+                followed: [],
+                violated: [],
+                notApplicable: [],
+                overallCompliance: 1,
+            },
+            skillUsage: [],
+            subagentUsage: [],
+        },
+        synthesis: {
+            dimensionScores: {},
+            weightedTotal: 8,
+            confidence: 0.9,
+            dissenting: [],
+        },
+        ledger: [],
+        totalCostUsd: 0.05,
+        createdAt: '2026-01-01T00:00:00.000Z',
+        updatedAt: '2026-01-01T00:00:02.000Z',
+        ...overrides,
+    };
+}
+export function createMockFs() {
+    const state = {
+        files: new Map(),
+        dirs: new Set(),
+    };
+    const mockFs = {
+        mkdir: vi.fn((_dirPath) => {
+            state.dirs.add(_dirPath);
+            return Promise.resolve();
+        }),
+        writeFile: vi.fn((_filePath, data) => {
+            state.files.set(_filePath, data);
+            return Promise.resolve();
+        }),
+        readFile: vi.fn((filePath) => {
+            const data = state.files.get(filePath);
+            if (data === undefined) {
+                return Promise.reject(new Error(`ENOENT: no such file: ${filePath}`));
+            }
+            return Promise.resolve(data);
+        }),
+        readdir: vi.fn((dirPath) => {
+            const prefix = dirPath.endsWith('/') ? dirPath : dirPath + '/';
+            const entries = [];
+            for (const key of state.files.keys()) {
+                if (key.startsWith(prefix)) {
+                    const rest = key.slice(prefix.length);
+                    if (!rest.includes('/')) {
+                        entries.push(rest);
+                    }
+                }
+            }
+            return Promise.resolve(entries);
+        }),
+        unlink: vi.fn((filePath) => {
+            if (!state.files.has(filePath)) {
+                return Promise.reject(new Error(`ENOENT: no such file: ${filePath}`));
+            }
+            state.files.delete(filePath);
+            return Promise.resolve();
+        }),
+        rename: vi.fn((oldPath, newPath) => {
+            const data = state.files.get(oldPath);
+            if (data === undefined) {
+                return Promise.reject(new Error(`ENOENT: no such file: ${oldPath}`));
+            }
+            state.files.set(newPath, data);
+            state.files.delete(oldPath);
+            return Promise.resolve();
+        }),
+        stat: vi.fn((filePath) => {
+            if (!state.files.has(filePath)) {
+                return Promise.reject(new Error(`ENOENT: no such file: ${filePath}`));
+            }
+            return Promise.resolve({ isFile: () => true });
+        }),
+        access: vi.fn((filePath) => {
+            if (!state.files.has(filePath)) {
+                return Promise.reject(new Error(`ENOENT: no such file: ${filePath}`));
+            }
+            return Promise.resolve();
+        }),
+    };
+    return { fs: mockFs, state };
+}
+//# sourceMappingURL=storage-test-helpers.js.map

package/dist/server/services/storage-test-helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-test-helpers.js","sourceRoot":"","sources":["../../../src/server/services/storage-test-helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAS5B,wEAAwE;AAExE,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,KAAc;IACpB,OAAO,EAAE,2BAA2B;IACpC,MAAM,EAAE,SAAS;IACjB,KAAK,EAAE,mBAAmB;CAC3B,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,YAA+B,EAAE;IAC5D,OAAO;QACL,EAAE,EAAE,YAAY;QAChB,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,iBAAiB;QAC9B,QAAQ,EAAE,aAAa;QACvB,cAAc,EAAE,GAAG;QACnB,SAAS,EAAE,0BAA0B;QACrC,SAAS,EAAE,0BAA0B;QACrC,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,YAA+B,EAAE;IAC5D,OAAO;QACL,EAAE,EAAE,YAAY;QAChB,IAAI,EAAE,eAAe;QACrB,QAAQ,EAAE,UAAU;QACpB,aAAa,EAAE,EAAE;QACjB,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;QACd,cAAc,EAAE,SAAS;QACzB,MAAM,EAAE,cAAc;QACtB,cAAc,EAAE,EAAE;QAClB,cAAc,EAAE,MAAM;QACtB,oBAAoB,EAAE,EAAE;QACxB,iBAAiB,EAAE,EAAE;QACrB,iBAAiB,EAAE,EAAE;QACrB,SAAS,EAAE,0BAA0B;QACrC,SAAS,EAAE,0BAA0B;QACrC,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,YAA0B,EAAE;IAClD,OAAO;QACL,EAAE,EAAE,OAAO;QACX,UAAU,EAAE,YAAY;QACxB,UAAU,EAAE,YAAY;QACxB,MAAM,EAAE,WAAW;QACnB,gBAAgB,EAAE,YAAY,EAAE;QAChC,gBAAgB,EAAE,YAAY,EAAE;QAChC,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,MAAM;QAClB,YAAY,EAAE,IAAI;QAClB,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,0BAA0B;QACrC,SAAS,EAAE,0BAA0B;QACrC,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,YAAiC,EAAE;IAChE,OAAO;QACL,EAAE,EAAE,QAAQ;QACZ,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,WAAW;QACnB,UAAU,EAAE,EAAE;QACd,MAAM,EAAE,EAAE;QACV,gBAAgB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE;QACzE,eAAe,EAAE,EAAE;QACnB,eAAe,EAAE;YACf,qBAAqB,EAAE;gBACrB,QAAQ,EAAE,EAAE;gBACZ,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,EAAE;gBACjB,iBAAiB,EAAE,CAAC;aACrB;YACD,UAAU,EAAE,EAAE;YACd,aAAa,EAAE,EAAE;SAClB;QACD,SAAS,EAAE;YACT,eAAe,EAAE,EAAE;YACnB,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,GAAG;YACf,UAAU,EAAE,EAAE;SACf;QACD,MAAM,EAAE,EAAE;QACV,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,0BAA0B;QACrC,SAAS,EAAE,0BAA0B;QACrC,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AASD,MAAM,UAAU,YAAY;IAC1B,MAAM,KAAK,GAAgB;QACzB,KAAK,EAAE,IAAI,GAAG,EAAE;QAChB,IAAI,EAAE,IAAI,GAAG,EAAE;KAChB,CAAC;IAEF,MAAM,MAAM,GAAc;QACxB,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE;YAChC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC;QAEF,SAAS,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,SAAiB,EAAE,IAAY,EAAE,EAAE;YACnD,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACjC,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC;QAEF,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC,CAAC;YACxE,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC,CAAC;QAEF,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,OAAe,EAAE,EAAE;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC;YAC/D,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBACrC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACtC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBACxB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACrB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClC,CAAC,CAAC;QAEF,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE;YACjC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC,CAAC;YACxE,CAAC;YACD,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC7B,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC;QAEF,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,OAAe,EAAE,OAAe,EAAE,EAAE;YACjD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,OAAO,EAAE,CAAC,CAAC,CAAC;YACvE,CAAC;YACD,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC/B,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5B,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC;QAEF,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE;YAC/B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC,CAAC;YACxE,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QACjD,CAAC,CAAC;QAEF,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAgB,EAAE,EAAE;YACjC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC,CAAC;YACxE,CAAC;YACD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC;KACH,CAAC;IAEF,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC/B,CAAC"}

package/dist/server/services/storage.d.ts

@@ -0,0 +1,35 @@
+import type { IStorage, ProviderFilter, ScenarioFilter, RunFilter, EvaluationFilter } from '../interfaces/storage.js';
+import type { Provider, Scenario, Run, Evaluation } from '../types/index.js';
+import type { FsAdapter } from './fs-adapter.js';
+export declare class JsonFileStorage implements IStorage {
+    private readonly basePath;
+    private readonly fs;
+    private initialized;
+    private readonly entities;
+    constructor(basePath?: string, fsAdapter?: FsAdapter);
+    private ensureInit;
+    private entityDir;
+    private entityPath;
+    private getEntity;
+    private listEntities;
+    private saveEntity;
+    private deleteEntity;
+    getProvider(id: string): Promise<Provider | undefined>;
+    listProviders(filter?: ProviderFilter): Promise<readonly Provider[]>;
+    saveProvider(provider: Provider): Promise<void>;
+    deleteProvider(id: string): Promise<boolean>;
+    getScenario(id: string): Promise<Scenario | undefined>;
+    listScenarios(filter?: ScenarioFilter): Promise<readonly Scenario[]>;
+    saveScenario(scenario: Scenario): Promise<void>;
+    deleteScenario(id: string): Promise<boolean>;
+    private normalizeRun;
+    getRun(id: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<readonly Run[]>;
+    saveRun(run: Run): Promise<void>;
+    deleteRun(id: string): Promise<boolean>;
+    getEvaluation(id: string): Promise<Evaluation | undefined>;
+    listEvaluations(filter?: EvaluationFilter): Promise<readonly Evaluation[]>;
+    saveEvaluation(evaluation: Evaluation): Promise<void>;
+    deleteEvaluation(id: string): Promise<boolean>;
+}
+//# sourceMappingURL=storage.d.ts.map

package/dist/server/services/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../../src/server/services/storage.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACtH,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,GAAG,EACH,UAAU,EACX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAQjD,qBAAa,eAAgB,YAAW,QAAQ;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAC/B,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAKvB;gBAEU,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,SAAS;YAgDtC,UAAU;IAwBxB,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,UAAU;YAMJ,SAAS;YAWT,YAAY;YA6BZ,UAAU;YAeV,YAAY;IAc1B,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAItD,aAAa,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,QAAQ,EAAE,CAAC;IAKpE,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAM5C,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAItD,aAAa,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,QAAQ,EAAE,CAAC;IAKpE,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAK5C,OAAO,CAAC,YAAY;IAed,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC;IAI5C,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC;IAI3D,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAChC,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAGvC,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAC1D,eAAe,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,UAAU,EAAE,CAAC;IAC1E,cAAc,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IACrD,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAC/C"}

package/dist/server/services/storage.js

@@ -0,0 +1,219 @@
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { defaultFs } from './fs-adapter.js';
+export class JsonFileStorage {
+    basePath;
+    fs;
+    initialized = false;
+    entities;
+    constructor(basePath, fsAdapter) {
+        this.basePath = basePath ?? path.join(process.cwd(), '.claude-test-bench');
+        this.fs = fsAdapter ?? defaultFs;
+        this.entities = {
+            providers: {
+                subdir: 'providers',
+                sensitive: true,
+                matchesFilter: (s, f) => {
+                    if (!f)
+                        return true;
+                    if (f.provider && s.provider.kind !== f.provider)
+                        return false;
+                    if (f.model && s.provider.model !== f.model)
+                        return false;
+                    return true;
+                },
+            },
+            scenarios: {
+                subdir: 'scenarios/custom',
+                sensitive: false,
+                matchesFilter: (s, f) => {
+                    if (!f)
+                        return true;
+                    if (f.category && s.category !== f.category)
+                        return false;
+                    return true;
+                },
+            },
+            runs: {
+                subdir: 'runs',
+                sensitive: false,
+                matchesFilter: (r, f) => {
+                    if (!f)
+                        return true;
+                    if (f.providerId && r.providerId !== f.providerId)
+                        return false;
+                    if (f.scenarioId && r.scenarioId !== f.scenarioId)
+                        return false;
+                    if (f.status && r.status !== f.status)
+                        return false;
+                    return true;
+                },
+            },
+            evaluations: {
+                subdir: 'evaluations',
+                sensitive: false,
+                matchesFilter: (e, f) => {
+                    if (!f)
+                        return true;
+                    if (f.runId && e.runId !== f.runId)
+                        return false;
+                    if (f.status && e.status !== f.status)
+                        return false;
+                    return true;
+                },
+            },
+        };
+    }
+    async ensureInit() {
+        if (this.initialized)
+            return;
+        // Migrate legacy 'setups' directory to 'providers'
+        const legacyDir = path.join(this.basePath, 'setups');
+        const newDir = path.join(this.basePath, 'providers');
+        try {
+            await this.fs.access(legacyDir);
+            try {
+                await this.fs.access(newDir);
+            }
+            catch {
+                // New dir doesn't exist — rename legacy dir
+                await this.fs.rename(legacyDir, newDir);
+            }
+        }
+        catch {
+            // Legacy dir doesn't exist — nothing to migrate
+        }
+        for (const cfg of Object.values(this.entities)) {
+            await this.fs.mkdir(path.join(this.basePath, cfg.subdir), { recursive: true });
+        }
+        this.initialized = true;
+    }
+    entityDir(subdir) {
+        return path.join(this.basePath, subdir);
+    }
+    entityPath(subdir, id) {
+        return path.join(this.basePath, subdir, `${id}.json`);
+    }
+    // ─── Generic CRUD ──────────────────────────────────────────────────
+    async getEntity(subdir, id) {
+        await this.ensureInit();
+        const filePath = this.entityPath(subdir, id);
+        try {
+            const raw = await this.fs.readFile(filePath, 'utf-8');
+            return JSON.parse(raw);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async listEntities(subdir, matchesFilter, filter) {
+        await this.ensureInit();
+        const dir = this.entityDir(subdir);
+        let files;
+        try {
+            files = await this.fs.readdir(dir);
+        }
+        catch {
+            return [];
+        }
+        const results = [];
+        for (const file of files) {
+            if (!file.endsWith('.json'))
+                continue;
+            try {
+                const raw = await this.fs.readFile(path.join(dir, file), 'utf-8');
+                const entity = JSON.parse(raw);
+                if (matchesFilter(entity, filter)) {
+                    results.push(entity);
+                }
+            }
+            catch {
+                // Skip corrupt files
+            }
+        }
+        return results;
+    }
+    async saveEntity(subdir, entity, sensitive) {
+        await this.ensureInit();
+        const dir = this.entityDir(subdir);
+        const tmpPath = path.join(dir, `.tmp-${randomUUID()}.json`);
+        const finalPath = this.entityPath(subdir, entity.id);
+        const data = JSON.stringify(entity, null, 2);
+        const opts = sensitive ? { mode: 0o600 } : undefined;
+        await this.fs.writeFile(tmpPath, data, opts);
+        await this.fs.rename(tmpPath, finalPath);
+    }
+    async deleteEntity(subdir, id) {
+        await this.ensureInit();
+        const filePath = this.entityPath(subdir, id);
+        try {
+            await this.fs.access(filePath);
+            await this.fs.unlink(filePath);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    // ─── Providers ──────────────────────────────────────────────────────
+    getProvider(id) {
+        return this.getEntity(this.entities.providers.subdir, id);
+    }
+    listProviders(filter) {
+        const cfg = this.entities.providers;
+        return this.listEntities(cfg.subdir, cfg.matchesFilter, filter);
+    }
+    saveProvider(provider) {
+        const cfg = this.entities.providers;
+        return this.saveEntity(cfg.subdir, provider, cfg.sensitive);
+    }
+    deleteProvider(id) {
+        return this.deleteEntity(this.entities.providers.subdir, id);
+    }
+    // ─── Scenarios ─────────────────────────────────────────────────────
+    getScenario(id) {
+        return this.getEntity(this.entities.scenarios.subdir, id);
+    }
+    listScenarios(filter) {
+        const cfg = this.entities.scenarios;
+        return this.listEntities(cfg.subdir, cfg.matchesFilter, filter);
+    }
+    saveScenario(scenario) {
+        const cfg = this.entities.scenarios;
+        return this.saveEntity(cfg.subdir, scenario, cfg.sensitive);
+    }
+    deleteScenario(id) {
+        return this.deleteEntity(this.entities.scenarios.subdir, id);
+    }
+    // ─── Runs ──────────────────────────────────────────────────────────
+    normalizeRun(raw) {
+        // Accept both old and new field names for backward compat
+        const r = raw;
+        if ('setupId' in r && !('providerId' in r)) {
+            return {
+                ...raw,
+                providerId: r['setupId'],
+                providerSnapshot: (r['setupSnapshot'] ?? raw.providerSnapshot),
+                reviewerProviderIds: (r['reviewerSetupIds'] ?? raw.reviewerProviderIds),
+                reviewerProviderSnapshots: (r['reviewerSetupSnapshots'] ?? raw.reviewerProviderSnapshots),
+            };
+        }
+        return raw;
+    }
+    async getRun(id) {
+        const raw = await this.getEntity(this.entities.runs.subdir, id);
+        return raw ? this.normalizeRun(raw) : undefined;
+    }
+    async listRuns(filter) {
+        const results = await this.listEntities(this.entities.runs.subdir, this.entities.runs.matchesFilter, filter);
+        return results.map((r) => this.normalizeRun(r));
+    }
+    saveRun(run) { return this.saveEntity(this.entities.runs.subdir, run, this.entities.runs.sensitive); }
+    deleteRun(id) { return this.deleteEntity(this.entities.runs.subdir, id); }
+    // ─── Evaluations ───────────────────────────────────────────────────
+    getEvaluation(id) { return this.getEntity(this.entities.evaluations.subdir, id); }
+    listEvaluations(filter) { return this.listEntities(this.entities.evaluations.subdir, this.entities.evaluations.matchesFilter, filter); }
+    saveEvaluation(evaluation) { return this.saveEntity(this.entities.evaluations.subdir, evaluation, this.entities.evaluations.sensitive); }
+    deleteEvaluation(id) { return this.deleteEntity(this.entities.evaluations.subdir, id); }
+}
+//# sourceMappingURL=storage.js.map

package/dist/server/services/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../../src/server/services/storage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAS5C,MAAM,OAAO,eAAe;IACT,QAAQ,CAAS;IACjB,EAAE,CAAY;IACvB,WAAW,GAAG,KAAK,CAAC;IAEX,QAAQ,CAKvB;IAEF,YAAY,QAAiB,EAAE,SAAqB;QAClD,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAC3E,IAAI,CAAC,EAAE,GAAG,SAAS,IAAI,SAAS,CAAC;QAEjC,IAAI,CAAC,QAAQ,GAAG;YACd,SAAS,EAAE;gBACT,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,CAAC,CAAW,EAAE,CAAkB,EAAE,EAAE;oBACjD,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC,QAAQ;wBAAE,OAAO,KAAK,CAAC;oBAC/D,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;wBAAE,OAAO,KAAK,CAAC;oBAC1D,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;YACD,SAAS,EAAE;gBACT,MAAM,EAAE,kBAAkB;gBAC1B,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,CAAC,CAAW,EAAE,CAAkB,EAAE,EAAE;oBACjD,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ;wBAAE,OAAO,KAAK,CAAC;oBAC1D,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;YACD,IAAI,EAAE;gBACJ,MAAM,EAAE,MAAM;gBACd,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,CAAC,CAAM,EAAE,CAAa,EAAE,EAAE;oBACvC,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;wBAAE,OAAO,KAAK,CAAC;oBAChE,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;wBAAE,OAAO,KAAK,CAAC;oBAChE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;wBAAE,OAAO,KAAK,CAAC;oBACpD,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;YACD,WAAW,EAAE;gBACX,MAAM,EAAE,aAAa;gBACrB,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,CAAC,CAAa,EAAE,CAAoB,EAAE,EAAE;oBACrD,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;wBAAE,OAAO,KAAK,CAAC;oBACjD,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;wBAAE,OAAO,KAAK,CAAC;oBACpD,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,mDAAmD;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,4CAA4C;gBAC5C,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAEO,SAAS,CAAC,MAAc;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,UAAU,CAAC,MAAc,EAAE,EAAU;QAC3C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,sEAAsE;IAE9D,KAAK,CAAC,SAAS,CAAI,MAAc,EAAE,EAAU;QACnD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,MAAc,EACd,aAAiD,EACjD,MAAU;QAEV,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,OAAO,GAAQ,EAAE,CAAC;QACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;gBAClE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;gBACpC,IAAI,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;oBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,qBAAqB;YACvB,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,MAAc,EACd,MAAS,EACT,SAAkB;QAElB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,UAAU,EAAE,OAAO,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACrD,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,EAAU;QACnD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,uEAAuE;IAEvE,WAAW,CAAC,EAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAW,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,aAAa,CAAC,MAAuB;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,YAAY,CAA2B,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAC5F,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,sEAAsE;IAEtE,WAAW,CAAC,EAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAW,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,aAAa,CAAC,MAAuB;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,YAAY,CAA2B,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAC5F,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,sEAAsE;IAC9D,YAAY,CAAC,GAAQ;QAC3B,0DAA0D;QAC1D,MAAM,CAAC,GAAG,GAAyC,CAAC;QACpD,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC3C,OAAO;gBACL,GAAI,GAAW;gBACf,UAAU,EAAE,CAAC,CAAC,SAAS,CAAW;gBAClC,gBAAgB,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,GAAG,CAAC,gBAAgB,CAA4B;gBACzF,mBAAmB,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,GAAG,CAAC,mBAAmB,CAA+B;gBACrG,yBAAyB,EAAE,CAAC,CAAC,CAAC,wBAAwB,CAAC,IAAI,GAAG,CAAC,yBAAyB,CAAqC;aAC9H,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrE,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,MAAkB;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAiB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC7H,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,CAAC,GAAQ,IAAmB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC1H,SAAS,CAAC,EAAU,IAAsB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAEpG,sEAAsE;IACtE,aAAa,CAAC,EAAU,IAAqC,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3H,eAAe,CAAC,MAAyB,IAAoC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC3L,cAAc,CAAC,UAAsB,IAAmB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACpK,gBAAgB,CAAC,EAAU,IAAsB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;CACnH"}

package/dist/server/services/transcript-formatter.d.ts

@@ -0,0 +1,18 @@
+import type { SDKMessageRecord } from '../types/index.js';
+/** Structured summary extracted from a transcript. */
+export interface TranscriptSummary {
+    readonly toolCallSequence: readonly string[];
+    readonly filesRead: readonly string[];
+    readonly filesModified: readonly string[];
+    readonly commandFailures: readonly string[];
+    readonly retryPatterns: readonly string[];
+    readonly askedClarifyingQuestions: boolean;
+}
+/** Full transcript output. */
+export interface TranscriptResult {
+    readonly text: string;
+    readonly summary: TranscriptSummary;
+}
+/** Format SDK messages into a readable transcript and structured summary. */
+export declare function formatTranscript(messages: readonly SDKMessageRecord[]): TranscriptResult;
+//# sourceMappingURL=transcript-formatter.d.ts.map

package/dist/server/services/transcript-formatter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcript-formatter.d.ts","sourceRoot":"","sources":["../../../src/server/services/transcript-formatter.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAO1D,sDAAsD;AACtD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,wBAAwB,EAAE,OAAO,CAAC;CAC5C;AAED,8BAA8B;AAC9B,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;CACrC;AAED,6EAA6E;AAC7E,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,GAAG,gBAAgB,CAyExF"}

package/dist/server/services/transcript-formatter.js

@@ -0,0 +1,156 @@
+// ---------------------------------------------------------------------------
+// Transcript Formatter — converts SDK messages into evaluator-readable text
+// ---------------------------------------------------------------------------
+const MAX_TRANSCRIPT_CHARS = 100_000;
+const TRUNCATED_MARKER = '\n[transcript truncated]';
+const MAX_TOOL_INPUT_CHARS = 500;
+const MAX_TOOL_OUTPUT_CHARS = 500;
+/** Format SDK messages into a readable transcript and structured summary. */
+export function formatTranscript(messages) {
+    const lines = [];
+    const toolCalls = [];
+    const filesRead = new Set();
+    const filesModified = new Set();
+    const commandFailures = [];
+    const retryPatterns = [];
+    let askedClarifyingQuestions = false;
+    let totalChars = 0;
+    let truncated = false;
+    for (const record of messages) {
+        if (truncated)
+            break;
+        const msg = record.message;
+        const msgType = msg['type'];
+        if (msgType === 'assistant') {
+            const formatted = formatAssistantMessage(msg);
+            if (formatted) {
+                const result = appendLine(lines, formatted, totalChars);
+                totalChars = result.totalChars;
+                truncated = result.truncated;
+                if (looksLikeQuestion(formatted)) {
+                    askedClarifyingQuestions = true;
+                }
+            }
+        }
+        else if (msgType === 'tool_use') {
+            const name = msg['name'] ?? 'unknown_tool';
+            const input = truncateStr(JSON.stringify(msg['input'] ?? ''), MAX_TOOL_INPUT_CHARS);
+            toolCalls.push(name);
+            trackFileAccess(name, msg, filesRead, filesModified);
+            const line = `[Tool Call] ${name}: ${input}`;
+            const result = appendLine(lines, line, totalChars);
+            totalChars = result.totalChars;
+            truncated = result.truncated;
+        }
+        else if (msgType === 'tool_result') {
+            const output = truncateStr(extractToolResultText(msg), MAX_TOOL_OUTPUT_CHARS);
+            const isError = msg['is_error'] === true;
+            if (isError) {
+                commandFailures.push(output);
+                detectRetryPattern(toolCalls, retryPatterns);
+            }
+            const prefix = isError ? '[Tool Error]' : '[Tool Result]';
+            const line = `${prefix} ${output}`;
+            const result = appendLine(lines, line, totalChars);
+            totalChars = result.totalChars;
+            truncated = result.truncated;
+        }
+        else if (msgType === 'result') {
+            const resultText = msg['result'] ?? '';
+            if (resultText) {
+                const line = `[Final Result] ${resultText}`;
+                const result = appendLine(lines, line, totalChars);
+                totalChars = result.totalChars;
+                truncated = result.truncated;
+            }
+        }
+    }
+    if (truncated) {
+        lines.push(TRUNCATED_MARKER);
+    }
+    return {
+        text: lines.join('\n'),
+        summary: {
+            toolCallSequence: toolCalls,
+            filesRead: [...filesRead],
+            filesModified: [...filesModified],
+            commandFailures,
+            retryPatterns,
+            askedClarifyingQuestions,
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function appendLine(lines, line, totalChars) {
+    const newTotal = totalChars + line.length + 1; // +1 for newline
+    if (newTotal > MAX_TRANSCRIPT_CHARS) {
+        return { totalChars, truncated: true };
+    }
+    lines.push(line);
+    return { totalChars: newTotal, truncated: false };
+}
+function formatAssistantMessage(msg) {
+    const content = msg['message'];
+    if (!content)
+        return undefined;
+    const contentBlocks = content['content'];
+    if (!Array.isArray(contentBlocks))
+        return undefined;
+    const textParts = [];
+    for (const block of contentBlocks) {
+        if (block['type'] === 'text' && typeof block['text'] === 'string') {
+            textParts.push(block['text']);
+        }
+    }
+    return textParts.length > 0 ? `[Assistant] ${textParts.join(' ')}` : undefined;
+}
+function extractToolResultText(msg) {
+    const content = msg['content'];
+    if (typeof content === 'string')
+        return content;
+    if (Array.isArray(content)) {
+        const parts = [];
+        for (const block of content) {
+            if (block['type'] === 'text' && typeof block['text'] === 'string') {
+                parts.push(block['text']);
+            }
+        }
+        return parts.join(' ');
+    }
+    return '';
+}
+function truncateStr(s, max) {
+    return s.length > max ? s.slice(0, max) + '...' : s;
+}
+function looksLikeQuestion(text) {
+    return /\?\s*$/.test(text.trim());
+}
+function trackFileAccess(toolName, msg, filesRead, filesModified) {
+    const input = msg['input'];
+    if (!input)
+        return;
+    const filePath = (input['file_path'] ?? input['path'] ?? input['filename']);
+    if (!filePath)
+        return;
+    if (toolName === 'Read' || toolName === 'Glob' || toolName === 'Grep') {
+        filesRead.add(filePath);
+    }
+    else if (toolName === 'Edit' || toolName === 'Write' || toolName === 'NotebookEdit') {
+        filesModified.add(filePath);
+    }
+}
+function detectRetryPattern(toolCalls, retryPatterns) {
+    if (toolCalls.length < 2)
+        return;
+    const last = toolCalls[toolCalls.length - 1];
+    const prev = toolCalls[toolCalls.length - 2];
+    if (last === prev) {
+        const pattern = `Repeated ${last} after error`;
+        if (!retryPatterns.includes(pattern)) {
+            retryPatterns.push(pattern);
+        }
+    }
+}
+//# sourceMappingURL=transcript-formatter.js.map

package/dist/server/services/transcript-formatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcript-formatter.js","sourceRoot":"","sources":["../../../src/server/services/transcript-formatter.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,8EAA8E;AAI9E,MAAM,oBAAoB,GAAG,OAAO,CAAC;AACrC,MAAM,gBAAgB,GAAG,0BAA0B,CAAC;AACpD,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAkBlC,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB,CAAC,QAAqC;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAgB,IAAI,GAAG,EAAE,CAAC;IACzC,MAAM,aAAa,GAAgB,IAAI,GAAG,EAAE,CAAC;IAC7C,MAAM,eAAe,GAAa,EAAE,CAAC;IACrC,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,IAAI,wBAAwB,GAAG,KAAK,CAAC;IACrC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,SAAS;YAAE,MAAM;QACrB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;QAC3B,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAuB,CAAC;QAElD,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;YAC9C,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;gBACxD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;gBAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;gBAC7B,IAAI,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;oBACjC,wBAAwB,GAAG,IAAI,CAAC;gBAClC,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAI,GAAG,CAAC,MAAM,CAAY,IAAI,cAAc,CAAC;YACvD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,oBAAoB,CAAC,CAAC;YACpF,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,eAAe,CAAC,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,eAAe,IAAI,KAAK,KAAK,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YACnD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAC/B,CAAC;aAAM,IAAI,OAAO,KAAK,aAAa,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,WAAW,CAAC,qBAAqB,CAAC,GAAG,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAC9E,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC;YACzC,IAAI,OAAO,EAAE,CAAC;gBACZ,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC7B,kBAAkB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;YAC/C,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;YAC1D,MAAM,IAAI,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YACnD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAC/B,CAAC;aAAM,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,UAAU,GAAI,GAAG,CAAC,QAAQ,CAAY,IAAI,EAAE,CAAC;YACnD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,kBAAkB,UAAU,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;gBACnD,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;gBAC/B,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACtB,OAAO,EAAE;YACP,gBAAgB,EAAE,SAAS;YAC3B,SAAS,EAAE,CAAC,GAAG,SAAS,CAAC;YACzB,aAAa,EAAE,CAAC,GAAG,aAAa,CAAC;YACjC,eAAe;YACf,aAAa;YACb,wBAAwB;SACzB;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,UAAU,CACjB,KAAe,EACf,IAAY,EACZ,UAAkB;IAElB,MAAM,QAAQ,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;IAChE,IAAI,QAAQ,GAAG,oBAAoB,EAAE,CAAC;QACpC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACzC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,sBAAsB,CAAC,GAAsC;IACpE,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAwC,CAAC;IACtE,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,CAAuD,CAAC;IAC/F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;QAAE,OAAO,SAAS,CAAC;IAEpD,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;YAClE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACjF,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAsC;IACnE,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAgE,CAAC;IAC9F,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAChD,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAClE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,CAAS,EAAE,GAAW;IACzC,OAAO,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CACtB,QAAgB,EAChB,GAAsC,EACtC,SAAsB,EACtB,aAA0B;IAE1B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAwC,CAAC;IAClE,IAAI,CAAC,KAAK;QAAE,OAAO;IACnB,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAuB,CAAC;IAClG,IAAI,CAAC,QAAQ;QAAE,OAAO;IAEtB,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACtE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1B,CAAC;SAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,cAAc,EAAE,CAAC;QACtF,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,SAA4B,EAAE,aAAuB;IAC/E,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO;IACjC,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,YAAY,IAAI,cAAc,CAAC;QAC/C,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;AACH,CAAC"}