model-test-bench 1.0.2

package/dist/src/server/services/runner.js

@@ -0,0 +1,99 @@
+// ---------------------------------------------------------------------------
+// AiSdkRunner — executes a single run via Vercel AI SDK (streaming)
+// ---------------------------------------------------------------------------
+import { streamText, stepCountIs } from 'ai';
+import { createModel } from './model-factory.js';
+import { getEnabledTools } from './tools.js';
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+export class AiSdkRunner {
+    logger;
+    constructor(logger) {
+        this.logger = logger;
+    }
+    async executeRun(provider, scenario, run, callbacks, externalAbortController) {
+        const startTime = Date.now();
+        const messages = [];
+        const abortController = externalAbortController ?? new AbortController();
+        let timeoutId;
+        try {
+            // Set up timeout
+            const timeoutMs = provider.timeoutSeconds * 1000;
+            timeoutId = setTimeout(() => abortController.abort(), timeoutMs);
+            callbacks.onStatusChange('running');
+            // Build model and tools
+            const model = createModel(provider);
+            const tools = getEnabledTools(scenario.enabledTools);
+            const hasTools = Object.keys(tools).length > 0;
+            // Stream the query
+            const result = streamText({
+                model,
+                system: scenario.systemPrompt || undefined,
+                prompt: scenario.prompt,
+                temperature: provider.temperature,
+                maxOutputTokens: provider.maxTokens,
+                topP: provider.topP,
+                tools: hasTools ? tools : undefined,
+                stopWhen: stepCountIs(hasTools ? 10 : 1),
+                abortSignal: abortController.signal,
+            });
+            // Consume the stream, forwarding each part to the frontend via callbacks
+            for await (const part of result.fullStream) {
+                const record = {
+                    timestamp: new Date().toISOString(),
+                    message: part,
+                };
+                messages.push(record);
+                callbacks.onMessage(record);
+            }
+            // Await final aggregated values
+            const finalText = await result.text;
+            const usage = await result.usage;
+            const steps = await result.steps;
+            const durationMs = Date.now() - startTime;
+            const completedRun = {
+                ...run,
+                status: 'completed',
+                messages,
+                resultText: finalText,
+                totalCostUsd: 0,
+                durationMs,
+                numTurns: steps.length,
+                updatedAt: new Date().toISOString(),
+            };
+            callbacks.onStatusChange('completed');
+            this.logger.info('Run completed', {
+                runId: run.id,
+                durationMs,
+                inputTokens: usage?.inputTokens ?? 0,
+                outputTokens: usage?.outputTokens ?? 0,
+                steps: steps.length,
+            });
+            return completedRun;
+        }
+        catch (err) {
+            const durationMs = Date.now() - startTime;
+            const errorMessage = err instanceof Error ? err.message : String(err);
+            const isAbort = abortController.signal.aborted;
+            const errorRun = {
+                ...run,
+                status: isAbort ? 'cancelled' : 'failed',
+                messages,
+                resultText: '',
+                totalCostUsd: 0,
+                durationMs,
+                numTurns: 0,
+                error: isAbort ? `Timeout after ${provider.timeoutSeconds}s` : errorMessage,
+                updatedAt: new Date().toISOString(),
+            };
+            callbacks.onStatusChange(errorRun.status);
+            return errorRun;
+        }
+        finally {
+            if (timeoutId !== undefined)
+                clearTimeout(timeoutId);
+        }
+    }
+}
+//# sourceMappingURL=runner.js.map

package/dist/src/server/services/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../../src/server/services/runner.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oEAAoE;AACpE,8EAA8E;AAE9E,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAI7C,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAE7C,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,OAAO,WAAW;IACO;IAA7B,YAA6B,MAAe;QAAf,WAAM,GAAN,MAAM,CAAS;IAAG,CAAC;IAEhD,KAAK,CAAC,UAAU,CACd,QAAkB,EAClB,QAAkB,EAClB,GAAQ,EACR,SAAuB,EACvB,uBAAyC;QAEzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAuB,EAAE,CAAC;QACxC,MAAM,eAAe,GAAG,uBAAuB,IAAI,IAAI,eAAe,EAAE,CAAC;QAEzE,IAAI,SAAoD,CAAC;QAEzD,IAAI,CAAC;YACH,iBAAiB;YACjB,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;YACjD,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;YACjE,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAEpC,wBAAwB;YACxB,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;YACpC,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;YAE/C,mBAAmB;YACnB,MAAM,MAAM,GAAG,UAAU,CAAC;gBACxB,KAAK;gBACL,MAAM,EAAE,QAAQ,CAAC,YAAY,IAAI,SAAS;gBAC1C,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,eAAe,EAAE,QAAQ,CAAC,SAAS;gBACnC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;gBACnC,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,WAAW,EAAE,eAAe,CAAC,MAAM;aACpC,CAAC,CAAC;YAEH,yEAAyE;YACzE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC3C,MAAM,MAAM,GAAqB;oBAC/B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,OAAO,EAAE,IAA0C;iBACpD,CAAC;gBACF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtB,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YAED,gCAAgC;YAChC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAE1C,MAAM,YAAY,GAAQ;gBACxB,GAAG,GAAG;gBACN,MAAM,EAAE,WAAW;gBACnB,QAAQ;gBACR,UAAU,EAAE,SAAS;gBACrB,YAAY,EAAE,CAAC;gBACf,UAAU;gBACV,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;YACF,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAEtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE;gBAChC,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,UAAU;gBACV,WAAW,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;gBACpC,YAAY,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;gBACtC,KAAK,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAC;YAEH,OAAO,YAAY,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,MAAM,YAAY,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACtE,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;YAE/C,MAAM,QAAQ,GAAQ;gBACpB,GAAG,GAAG;gBACN,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;gBACxC,QAAQ;gBACR,UAAU,EAAE,EAAE;gBACd,YAAY,EAAE,CAAC;gBACf,UAAU;gBACV,QAAQ,EAAE,CAAC;gBACX,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,iBAAiB,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,YAAY;gBAC3E,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;YACF,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC1C,OAAO,QAAQ,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,IAAI,SAAS,KAAK,SAAS;gBAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;CACF"}

package/dist/src/server/services/seeder.d.ts

@@ -0,0 +1,5 @@
+import type { IStorage } from '../interfaces/storage.js';
+import type { ILogger } from '../interfaces/logger.js';
+/** Seed storage with example scenarios and providers when empty. */
+export declare function seedIfEmpty(storage: IStorage, logger: ILogger): Promise<void>;
+//# sourceMappingURL=seeder.d.ts.map

package/dist/src/server/services/seeder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"seeder.d.ts","sourceRoot":"","sources":["../../../../src/server/services/seeder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AA8CvD,oEAAoE;AACpE,wBAAsB,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAoCnF"}

package/dist/src/server/services/seeder.js

@@ -0,0 +1,79 @@
+import fs from 'node:fs';
+import path from 'node:path';
+/**
+ * Seed files that ship with the project under docs/schemas/.
+ * Each entry maps a source file to the storage method used to persist it.
+ * Seeding only runs when the respective list is empty — existing data is never overwritten.
+ */
+const SEED_SCENARIOS = [
+    'scenario-baseline.example.json',
+    'scenario-with-system-prompt.example.json',
+    'scenario-carwash-baseline.example.json',
+    'scenario-carwash-with-system-prompt.example.json',
+    'scenario-negative-analysis-baseline.example.json',
+    'scenario-negative-analysis-with-system-prompt.example.json',
+    'scenario-golden-rules-baseline.example.json',
+    'scenario-golden-rules-with-system-prompt.example.json',
+];
+const SEED_PROVIDERS = [
+    'provider-api.example.json',
+    'provider-openai.example.json',
+];
+function resolveSchemasDir() {
+    // Try common locations: relative to cwd, relative to this file's compiled location
+    const candidates = [
+        path.join(process.cwd(), 'docs', 'schemas'),
+        path.join(process.cwd(), '..', 'docs', 'schemas'),
+    ];
+    for (const dir of candidates) {
+        if (fs.existsSync(dir))
+            return dir;
+    }
+    return candidates[0]; // fallback
+}
+function loadJsonFile(filePath) {
+    try {
+        const raw = fs.readFileSync(filePath, 'utf-8');
+        return JSON.parse(raw);
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Seed storage with example scenarios and providers when empty. */
+export async function seedIfEmpty(storage, logger) {
+    const schemasDir = resolveSchemasDir();
+    // Seed scenarios
+    const existingScenarios = await storage.listScenarios();
+    if (existingScenarios.length === 0) {
+        let seeded = 0;
+        for (const file of SEED_SCENARIOS) {
+            const scenario = loadJsonFile(path.join(schemasDir, file));
+            if (scenario) {
+                await storage.saveScenario(scenario);
+                seeded++;
+                logger.info('Seeded scenario', { name: scenario.name, id: scenario.id });
+            }
+        }
+        if (seeded === 0) {
+            logger.info('No seed scenarios found in ' + schemasDir);
+        }
+    }
+    // Seed providers
+    const existingProviders = await storage.listProviders();
+    if (existingProviders.length === 0) {
+        let seeded = 0;
+        for (const file of SEED_PROVIDERS) {
+            const provider = loadJsonFile(path.join(schemasDir, file));
+            if (provider) {
+                await storage.saveProvider(provider);
+                seeded++;
+                logger.info('Seeded provider', { name: provider.name, id: provider.id });
+            }
+        }
+        if (seeded === 0) {
+            logger.info('No seed providers found in ' + schemasDir);
+        }
+    }
+}
+//# sourceMappingURL=seeder.js.map

package/dist/src/server/services/seeder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"seeder.js","sourceRoot":"","sources":["../../../../src/server/services/seeder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAM7B;;;;GAIG;AACH,MAAM,cAAc,GAAG;IACrB,gCAAgC;IAChC,0CAA0C;IAC1C,wCAAwC;IACxC,kDAAkD;IAClD,kDAAkD;IAClD,4DAA4D;IAC5D,6CAA6C;IAC7C,uDAAuD;CACxD,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,2BAA2B;IAC3B,8BAA8B;CAC/B,CAAC;AAEF,SAAS,iBAAiB;IACxB,mFAAmF;IACnF,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC;QAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC;KAClD,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC;IACrC,CAAC;IACD,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW;AACnC,CAAC;AAED,SAAS,YAAY,CAAI,QAAgB;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAiB,EAAE,MAAe;IAClE,MAAM,UAAU,GAAG,iBAAiB,EAAE,CAAC;IAEvC,iBAAiB;IACjB,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC;IACxD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,YAAY,CAAW,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;YACrE,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACrC,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QACD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,6BAA6B,GAAG,UAAU,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,iBAAiB;IACjB,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC;IACxD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,YAAY,CAAW,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;YACrE,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;gBACrC,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QACD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,6BAA6B,GAAG,UAAU,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/src/server/services/storage.d.ts

@@ -0,0 +1,35 @@
+import type { IStorage, ProviderFilter, ScenarioFilter, RunFilter, EvaluationFilter } from '../interfaces/storage.js';
+import type { Provider, Scenario, Run, Evaluation } from '../types/index.js';
+import type { FsAdapter } from './fs-adapter.js';
+export declare class JsonFileStorage implements IStorage {
+    private readonly basePath;
+    private readonly fs;
+    private initialized;
+    private readonly entities;
+    constructor(basePath?: string, fsAdapter?: FsAdapter);
+    private ensureInit;
+    private entityDir;
+    private entityPath;
+    private getEntity;
+    private listEntities;
+    private saveEntity;
+    private deleteEntity;
+    getProvider(id: string): Promise<Provider | undefined>;
+    listProviders(filter?: ProviderFilter): Promise<readonly Provider[]>;
+    saveProvider(provider: Provider): Promise<void>;
+    deleteProvider(id: string): Promise<boolean>;
+    getScenario(id: string): Promise<Scenario | undefined>;
+    listScenarios(filter?: ScenarioFilter): Promise<readonly Scenario[]>;
+    saveScenario(scenario: Scenario): Promise<void>;
+    deleteScenario(id: string): Promise<boolean>;
+    private normalizeRun;
+    getRun(id: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<readonly Run[]>;
+    saveRun(run: Run): Promise<void>;
+    deleteRun(id: string): Promise<boolean>;
+    getEvaluation(id: string): Promise<Evaluation | undefined>;
+    listEvaluations(filter?: EvaluationFilter): Promise<readonly Evaluation[]>;
+    saveEvaluation(evaluation: Evaluation): Promise<void>;
+    deleteEvaluation(id: string): Promise<boolean>;
+}
+//# sourceMappingURL=storage.d.ts.map

package/dist/src/server/services/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../../../src/server/services/storage.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACtH,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,GAAG,EACH,UAAU,EACX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAQjD,qBAAa,eAAgB,YAAW,QAAQ;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAY;IAC/B,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAKvB;gBAEU,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,SAAS;YAgDtC,UAAU;IAwBxB,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,UAAU;YAMJ,SAAS;YAWT,YAAY;YA6BZ,UAAU;YAeV,YAAY;IAc1B,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAItD,aAAa,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,QAAQ,EAAE,CAAC;IAKpE,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAM5C,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAItD,aAAa,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,QAAQ,EAAE,CAAC;IAKpE,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAK5C,OAAO,CAAC,YAAY;IAed,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC;IAI5C,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC;IAI3D,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAChC,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAGvC,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAC1D,eAAe,CAAC,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,UAAU,EAAE,CAAC;IAC1E,cAAc,CAAC,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IACrD,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAC/C"}

package/dist/src/server/services/storage.js

@@ -0,0 +1,219 @@
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { defaultFs } from './fs-adapter.js';
+export class JsonFileStorage {
+    basePath;
+    fs;
+    initialized = false;
+    entities;
+    constructor(basePath, fsAdapter) {
+        this.basePath = basePath ?? path.join(process.cwd(), '.model-test-bench');
+        this.fs = fsAdapter ?? defaultFs;
+        this.entities = {
+            providers: {
+                subdir: 'providers',
+                sensitive: true,
+                matchesFilter: (s, f) => {
+                    if (!f)
+                        return true;
+                    if (f.provider && s.providerName !== f.provider)
+                        return false;
+                    if (f.model && s.model !== f.model)
+                        return false;
+                    return true;
+                },
+            },
+            scenarios: {
+                subdir: 'scenarios/custom',
+                sensitive: false,
+                matchesFilter: (s, f) => {
+                    if (!f)
+                        return true;
+                    if (f.category && s.category !== f.category)
+                        return false;
+                    return true;
+                },
+            },
+            runs: {
+                subdir: 'runs',
+                sensitive: false,
+                matchesFilter: (r, f) => {
+                    if (!f)
+                        return true;
+                    if (f.providerId && r.providerId !== f.providerId)
+                        return false;
+                    if (f.scenarioId && r.scenarioId !== f.scenarioId)
+                        return false;
+                    if (f.status && r.status !== f.status)
+                        return false;
+                    return true;
+                },
+            },
+            evaluations: {
+                subdir: 'evaluations',
+                sensitive: false,
+                matchesFilter: (e, f) => {
+                    if (!f)
+                        return true;
+                    if (f.runId && e.runId !== f.runId)
+                        return false;
+                    if (f.status && e.status !== f.status)
+                        return false;
+                    return true;
+                },
+            },
+        };
+    }
+    async ensureInit() {
+        if (this.initialized)
+            return;
+        // Migrate legacy 'setups' directory to 'providers'
+        const legacyDir = path.join(this.basePath, 'setups');
+        const newDir = path.join(this.basePath, 'providers');
+        try {
+            await this.fs.access(legacyDir);
+            try {
+                await this.fs.access(newDir);
+            }
+            catch {
+                // New dir doesn't exist — rename legacy dir
+                await this.fs.rename(legacyDir, newDir);
+            }
+        }
+        catch {
+            // Legacy dir doesn't exist — nothing to migrate
+        }
+        for (const cfg of Object.values(this.entities)) {
+            await this.fs.mkdir(path.join(this.basePath, cfg.subdir), { recursive: true });
+        }
+        this.initialized = true;
+    }
+    entityDir(subdir) {
+        return path.join(this.basePath, subdir);
+    }
+    entityPath(subdir, id) {
+        return path.join(this.basePath, subdir, `${id}.json`);
+    }
+    // ─── Generic CRUD ──────────────────────────────────────────────────
+    async getEntity(subdir, id) {
+        await this.ensureInit();
+        const filePath = this.entityPath(subdir, id);
+        try {
+            const raw = await this.fs.readFile(filePath, 'utf-8');
+            return JSON.parse(raw);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async listEntities(subdir, matchesFilter, filter) {
+        await this.ensureInit();
+        const dir = this.entityDir(subdir);
+        let files;
+        try {
+            files = await this.fs.readdir(dir);
+        }
+        catch {
+            return [];
+        }
+        const results = [];
+        for (const file of files) {
+            if (!file.endsWith('.json'))
+                continue;
+            try {
+                const raw = await this.fs.readFile(path.join(dir, file), 'utf-8');
+                const entity = JSON.parse(raw);
+                if (matchesFilter(entity, filter)) {
+                    results.push(entity);
+                }
+            }
+            catch {
+                // Skip corrupt files
+            }
+        }
+        return results;
+    }
+    async saveEntity(subdir, entity, sensitive) {
+        await this.ensureInit();
+        const dir = this.entityDir(subdir);
+        const tmpPath = path.join(dir, `.tmp-${randomUUID()}.json`);
+        const finalPath = this.entityPath(subdir, entity.id);
+        const data = JSON.stringify(entity, null, 2);
+        const opts = sensitive ? { mode: 0o600 } : undefined;
+        await this.fs.writeFile(tmpPath, data, opts);
+        await this.fs.rename(tmpPath, finalPath);
+    }
+    async deleteEntity(subdir, id) {
+        await this.ensureInit();
+        const filePath = this.entityPath(subdir, id);
+        try {
+            await this.fs.access(filePath);
+            await this.fs.unlink(filePath);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    // ─── Providers ──────────────────────────────────────────────────────
+    getProvider(id) {
+        return this.getEntity(this.entities.providers.subdir, id);
+    }
+    listProviders(filter) {
+        const cfg = this.entities.providers;
+        return this.listEntities(cfg.subdir, cfg.matchesFilter, filter);
+    }
+    saveProvider(provider) {
+        const cfg = this.entities.providers;
+        return this.saveEntity(cfg.subdir, provider, cfg.sensitive);
+    }
+    deleteProvider(id) {
+        return this.deleteEntity(this.entities.providers.subdir, id);
+    }
+    // ─── Scenarios ─────────────────────────────────────────────────────
+    getScenario(id) {
+        return this.getEntity(this.entities.scenarios.subdir, id);
+    }
+    listScenarios(filter) {
+        const cfg = this.entities.scenarios;
+        return this.listEntities(cfg.subdir, cfg.matchesFilter, filter);
+    }
+    saveScenario(scenario) {
+        const cfg = this.entities.scenarios;
+        return this.saveEntity(cfg.subdir, scenario, cfg.sensitive);
+    }
+    deleteScenario(id) {
+        return this.deleteEntity(this.entities.scenarios.subdir, id);
+    }
+    // ─── Runs ──────────────────────────────────────────────────────────
+    normalizeRun(raw) {
+        // Accept both old and new field names for backward compat
+        const r = raw;
+        if ('setupId' in r && !('providerId' in r)) {
+            return {
+                ...raw,
+                providerId: r['setupId'],
+                providerSnapshot: (r['setupSnapshot'] ?? raw.providerSnapshot),
+                reviewerProviderIds: (r['reviewerSetupIds'] ?? raw.reviewerProviderIds),
+                reviewerProviderSnapshots: (r['reviewerSetupSnapshots'] ?? raw.reviewerProviderSnapshots),
+            };
+        }
+        return raw;
+    }
+    async getRun(id) {
+        const raw = await this.getEntity(this.entities.runs.subdir, id);
+        return raw ? this.normalizeRun(raw) : undefined;
+    }
+    async listRuns(filter) {
+        const results = await this.listEntities(this.entities.runs.subdir, this.entities.runs.matchesFilter, filter);
+        return results.map((r) => this.normalizeRun(r));
+    }
+    saveRun(run) { return this.saveEntity(this.entities.runs.subdir, run, this.entities.runs.sensitive); }
+    deleteRun(id) { return this.deleteEntity(this.entities.runs.subdir, id); }
+    // ─── Evaluations ───────────────────────────────────────────────────
+    getEvaluation(id) { return this.getEntity(this.entities.evaluations.subdir, id); }
+    listEvaluations(filter) { return this.listEntities(this.entities.evaluations.subdir, this.entities.evaluations.matchesFilter, filter); }
+    saveEvaluation(evaluation) { return this.saveEntity(this.entities.evaluations.subdir, evaluation, this.entities.evaluations.sensitive); }
+    deleteEvaluation(id) { return this.deleteEntity(this.entities.evaluations.subdir, id); }
+}
+//# sourceMappingURL=storage.js.map

package/dist/src/server/services/storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.js","sourceRoot":"","sources":["../../../../src/server/services/storage.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAS5C,MAAM,OAAO,eAAe;IACT,QAAQ,CAAS;IACjB,EAAE,CAAY;IACvB,WAAW,GAAG,KAAK,CAAC;IAEX,QAAQ,CAKvB;IAEF,YAAY,QAAiB,EAAE,SAAqB;QAClD,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,mBAAmB,CAAC,CAAC;QAC1E,IAAI,CAAC,EAAE,GAAG,SAAS,IAAI,SAAS,CAAC;QAEjC,IAAI,CAAC,QAAQ,GAAG;YACd,SAAS,EAAE;gBACT,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,CAAC,CAAW,EAAE,CAAkB,EAAE,EAAE;oBACjD,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,QAAQ;wBAAE,OAAO,KAAK,CAAC;oBAC9D,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;wBAAE,OAAO,KAAK,CAAC;oBACjD,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;YACD,SAAS,EAAE;gBACT,MAAM,EAAE,kBAAkB;gBAC1B,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,CAAC,CAAW,EAAE,CAAkB,EAAE,EAAE;oBACjD,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ;wBAAE,OAAO,KAAK,CAAC;oBAC1D,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;YACD,IAAI,EAAE;gBACJ,MAAM,EAAE,MAAM;gBACd,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,CAAC,CAAM,EAAE,CAAa,EAAE,EAAE;oBACvC,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;wBAAE,OAAO,KAAK,CAAC;oBAChE,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;wBAAE,OAAO,KAAK,CAAC;oBAChE,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;wBAAE,OAAO,KAAK,CAAC;oBACpD,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;YACD,WAAW,EAAE;gBACX,MAAM,EAAE,aAAa;gBACrB,SAAS,EAAE,KAAK;gBAChB,aAAa,EAAE,CAAC,CAAa,EAAE,CAAoB,EAAE,EAAE;oBACrD,IAAI,CAAC,CAAC;wBAAE,OAAO,IAAI,CAAC;oBACpB,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK;wBAAE,OAAO,KAAK,CAAC;oBACjD,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;wBAAE,OAAO,KAAK,CAAC;oBACpD,OAAO,IAAI,CAAC;gBACd,CAAC;aACF;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,mDAAmD;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,4CAA4C;gBAC5C,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAEO,SAAS,CAAC,MAAc;QAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,UAAU,CAAC,MAAc,EAAE,EAAU;QAC3C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,sEAAsE;IAE9D,KAAK,CAAC,SAAS,CAAI,MAAc,EAAE,EAAU;QACnD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,MAAc,EACd,aAAiD,EACjD,MAAU;QAEV,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,OAAO,GAAQ,EAAE,CAAC;QACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;gBAClE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAM,CAAC;gBACpC,IAAI,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;oBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,qBAAqB;YACvB,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,MAAc,EACd,MAAS,EACT,SAAkB;QAElB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,UAAU,EAAE,OAAO,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACrD,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC7C,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,EAAU;QACnD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,uEAAuE;IAEvE,WAAW,CAAC,EAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAW,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,aAAa,CAAC,MAAuB;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,YAAY,CAA2B,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAC5F,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,sEAAsE;IAEtE,WAAW,CAAC,EAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAW,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,aAAa,CAAC,MAAuB;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,YAAY,CAA2B,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAC5F,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;QACpC,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,sEAAsE;IAC9D,YAAY,CAAC,GAAQ;QAC3B,0DAA0D;QAC1D,MAAM,CAAC,GAAG,GAAyC,CAAC;QACpD,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC3C,OAAO;gBACL,GAAI,GAAW;gBACf,UAAU,EAAE,CAAC,CAAC,SAAS,CAAW;gBAClC,gBAAgB,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,GAAG,CAAC,gBAAgB,CAA4B;gBACzF,mBAAmB,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,GAAG,CAAC,mBAAmB,CAA+B;gBACrG,yBAAyB,EAAE,CAAC,CAAC,CAAC,wBAAwB,CAAC,IAAI,GAAG,CAAC,yBAAyB,CAAqC;aAC9H,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrE,OAAO,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClD,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,MAAkB;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAiB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC7H,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,CAAC,GAAQ,IAAmB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC1H,SAAS,CAAC,EAAU,IAAsB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAEpG,sEAAsE;IACtE,aAAa,CAAC,EAAU,IAAqC,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3H,eAAe,CAAC,MAAyB,IAAoC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAC3L,cAAc,CAAC,UAAsB,IAAmB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACpK,gBAAgB,CAAC,EAAU,IAAsB,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;CACnH"}

package/dist/src/server/services/tools.d.ts

@@ -0,0 +1,6 @@
+import { tool } from 'ai';
+/** Get the list of all available tool names. */
+export declare function getAvailableToolNames(): string[];
+/** Get enabled tools filtered by name. Returns empty object if none enabled. */
+export declare function getEnabledTools(enabledToolNames: readonly string[]): Record<string, ReturnType<typeof tool<any, any>>>;
+//# sourceMappingURL=tools.d.ts.map

package/dist/src/server/services/tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../../../src/server/services/tools.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AA8E1B,gDAAgD;AAChD,wBAAgB,qBAAqB,IAAI,MAAM,EAAE,CAEhD;AAED,gFAAgF;AAEhF,wBAAgB,eAAe,CAC7B,gBAAgB,EAAE,SAAS,MAAM,EAAE,GAClC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CASnD"}

package/dist/src/server/services/tools.js

@@ -0,0 +1,94 @@
+// ---------------------------------------------------------------------------
+// Built-in Tools — tool definitions for AI SDK generateText()
+// ---------------------------------------------------------------------------
+import { tool } from 'ai';
+import { z } from 'zod';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+/** Read a file by path and return its content. */
+const readFileTool = tool({
+    description: 'Read a file at the given path and return its content.',
+    inputSchema: z.object({
+        path: z.string().describe('Absolute or relative file path to read'),
+    }),
+    execute: async ({ path: filePath }) => {
+        const resolved = path.resolve(filePath);
+        const content = await fs.readFile(resolved, 'utf-8');
+        return content;
+    },
+});
+/** Search file contents by pattern (grep-like). */
+const searchFileTool = tool({
+    description: 'Search file contents by regex pattern. Returns matching lines with file:line references.',
+    inputSchema: z.object({
+        pattern: z.string().describe('Regex pattern to search for'),
+        path: z.string().optional().describe('Directory or file path to search in (defaults to cwd)'),
+    }),
+    execute: async ({ pattern, path: searchPath }) => {
+        const dir = searchPath ? path.resolve(searchPath) : process.cwd();
+        const regex = new RegExp(pattern, 'g');
+        const results = [];
+        async function searchDir(dirPath) {
+            const entries = await fs.readdir(dirPath, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = path.join(dirPath, entry.name);
+                if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules') {
+                    await searchDir(fullPath);
+                }
+                else if (entry.isFile()) {
+                    try {
+                        const content = await fs.readFile(fullPath, 'utf-8');
+                        const lines = content.split('\n');
+                        for (let i = 0; i < lines.length; i++) {
+                            if (regex.test(lines[i])) {
+                                results.push(`${fullPath}:${i + 1}: ${lines[i].trim()}`);
+                                regex.lastIndex = 0;
+                            }
+                        }
+                    }
+                    catch { /* skip binary/unreadable files */ }
+                }
+                if (results.length >= 50)
+                    return;
+            }
+        }
+        await searchDir(dir);
+        return results.length > 0
+            ? results.join('\n')
+            : `No matches found for pattern: ${pattern}`;
+    },
+});
+/** Web search (placeholder — not available in benchmark mode). */
+const webSearchTool = tool({
+    description: 'Search the web for information. Returns search results.',
+    inputSchema: z.object({
+        query: z.string().describe('Search query'),
+    }),
+    execute: async ({ query }) => {
+        return `Web search is not available in benchmark mode. Query was: "${query}"`;
+    },
+});
+/** Registry of all built-in tools. */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const TOOL_REGISTRY = {
+    read_file: readFileTool,
+    search_file: searchFileTool,
+    web_search: webSearchTool,
+};
+/** Get the list of all available tool names. */
+export function getAvailableToolNames() {
+    return Object.keys(TOOL_REGISTRY);
+}
+/** Get enabled tools filtered by name. Returns empty object if none enabled. */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function getEnabledTools(enabledToolNames) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const result = {};
+    for (const name of enabledToolNames) {
+        if (TOOL_REGISTRY[name]) {
+            result[name] = TOOL_REGISTRY[name];
+        }
+    }
+    return result;
+}
+//# sourceMappingURL=tools.js.map

package/dist/src/server/services/tools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../../../src/server/services/tools.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,8DAA8D;AAC9D,8EAA8E;AAE9E,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,kDAAkD;AAClD,MAAM,YAAY,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,uDAAuD;IACpE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;KACpE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF,CAAC,CAAC;AAEH,mDAAmD;AACnD,MAAM,cAAc,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,0FAA0F;IACvG,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;QAC3D,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uDAAuD,CAAC;KAC9F,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE;QAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACvC,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,UAAU,SAAS,CAAC,OAAe;YACtC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;oBACxF,MAAM,SAAS,CAAC,QAAQ,CAAC,CAAC;gBAC5B,CAAC;qBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;oBAC1B,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;wBACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;4BACtC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gCACzB,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gCACzD,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;4BACtB,CAAC;wBACH,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC,CAAC,kCAAkC,CAAC,CAAC;gBAChD,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE;oBAAE,OAAO;YACnC,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC;YACvB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YACpB,CAAC,CAAC,iCAAiC,OAAO,EAAE,CAAC;IACjD,CAAC;CACF,CAAC,CAAC;AAEH,kEAAkE;AAClE,MAAM,aAAa,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,yDAAyD;IACtE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;KAC3C,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,OAAO,8DAA8D,KAAK,GAAG,CAAC;IAChF,CAAC;CACF,CAAC,CAAC;AAEH,sCAAsC;AACtC,8DAA8D;AAC9D,MAAM,aAAa,GAAsD;IACvE,SAAS,EAAE,YAAY;IACvB,WAAW,EAAE,cAAc;IAC3B,UAAU,EAAE,aAAa;CAC1B,CAAC;AAEF,gDAAgD;AAChD,MAAM,UAAU,qBAAqB;IACnC,OAAO,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AACpC,CAAC;AAED,gFAAgF;AAChF,8DAA8D;AAC9D,MAAM,UAAU,eAAe,CAC7B,gBAAmC;IAEnC,8DAA8D;IAC9D,MAAM,MAAM,GAAsD,EAAE,CAAC;IACrE,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACpC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/server/services/transcript-formatter.d.ts

@@ -0,0 +1,18 @@
+import type { SDKMessageRecord } from '../types/index.js';
+/** Structured summary extracted from a transcript. */
+export interface TranscriptSummary {
+    readonly toolCallSequence: readonly string[];
+    readonly filesRead: readonly string[];
+    readonly filesModified: readonly string[];
+    readonly commandFailures: readonly string[];
+    readonly retryPatterns: readonly string[];
+    readonly askedClarifyingQuestions: boolean;
+}
+/** Full transcript output. */
+export interface TranscriptResult {
+    readonly text: string;
+    readonly summary: TranscriptSummary;
+}
+/** Format SDK messages into a readable transcript and structured summary. */
+export declare function formatTranscript(messages: readonly SDKMessageRecord[]): TranscriptResult;
+//# sourceMappingURL=transcript-formatter.d.ts.map

package/dist/src/server/services/transcript-formatter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcript-formatter.d.ts","sourceRoot":"","sources":["../../../../src/server/services/transcript-formatter.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAO1D,sDAAsD;AACtD,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,wBAAwB,EAAE,OAAO,CAAC;CAC5C;AAED,8BAA8B;AAC9B,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;CACrC;AAED,6EAA6E;AAC7E,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,GAAG,gBAAgB,CAsIxF"}