@monoes/monomindcli 1.11.12 → 1.11.14

package/dist/src/eval/dataset-manager.js

@@ -0,0 +1,107 @@
+/**
+ * DatasetManager - JSONL-based eval dataset management (Task 33)
+ */
+import { randomUUID } from 'crypto';
+import { appendFileSync, readFileSync, writeFileSync, renameSync, existsSync, statSync } from 'fs';
+import { resolve, sep } from 'path';
+import { parseJsonl } from '../utils/parse-jsonl.js';
+export class DatasetManager {
+    datasetsPath;
+    entriesPath;
+    constructor(datasetsPath, entriesPath) {
+        this.datasetsPath = datasetsPath;
+        this.entriesPath = entriesPath;
+    }
+    /**
+     * Create a dataset from a set of filtered traces.
+     */
+    createFromTraces(opts) {
+        const now = new Date().toISOString();
+        const agentSlugs = opts.agentSlugs ?? [...new Set(opts.traces.map((t) => t.agentSlug))];
+        const dataset = {
+            datasetId: randomUUID(),
+            name: opts.name,
+            description: opts.description,
+            agentSlugs,
+            createdAt: now,
+            updatedAt: now,
+            entryCount: opts.traces.length,
+        };
+        appendFileSync(this.datasetsPath, JSON.stringify(dataset) + '\n', 'utf-8');
+        for (const trace of opts.traces) {
+            const entry = {
+                entryId: randomUUID(),
+                datasetId: dataset.datasetId,
+                traceId: trace.traceId,
+                addedAt: now,
+            };
+            appendFileSync(this.entriesPath, JSON.stringify(entry) + '\n', 'utf-8');
+        }
+        return dataset;
+    }
+    /**
+     * List all datasets.
+     */
+    listDatasets() {
+        if (!existsSync(this.datasetsPath))
+            return [];
+        if (statSync(this.datasetsPath).size > 50 * 1024 * 1024) {
+            throw new Error('Dataset file exceeds 50MB — run cleanup');
+        }
+        const content = readFileSync(this.datasetsPath, 'utf-8');
+        return parseJsonl(content);
+    }
+    /**
+     * Get entries for a specific dataset.
+     */
+    getEntries(datasetId) {
+        if (!existsSync(this.entriesPath))
+            return [];
+        if (statSync(this.entriesPath).size > 50 * 1024 * 1024) {
+            throw new Error('Entries file exceeds 50MB — run cleanup');
+        }
+        const content = readFileSync(this.entriesPath, 'utf-8');
+        return parseJsonl(content).filter((e) => e.datasetId === datasetId);
+    }
+    /**
+     * Add a single trace to an existing dataset.
+     */
+    addTraceToDataset(datasetId, traceId) {
+        const entry = {
+            entryId: randomUUID(),
+            datasetId,
+            traceId,
+            addedAt: new Date().toISOString(),
+        };
+        appendFileSync(this.entriesPath, JSON.stringify(entry) + '\n', 'utf-8');
+        // Update dataset entryCount via atomic write
+        const datasets = this.listDatasets();
+        const updated = datasets.map((d) => {
+            if (d.datasetId === datasetId) {
+                return { ...d, entryCount: d.entryCount + 1, updatedAt: new Date().toISOString() };
+            }
+            return d;
+        });
+        const tmp = `${this.datasetsPath}.${randomUUID()}.tmp`;
+        writeFileSync(tmp, updated.map((d) => JSON.stringify(d)).join('\n') + '\n', 'utf-8');
+        renameSync(tmp, this.datasetsPath);
+        return entry;
+    }
+    /**
+     * Export a dataset to a JSON file. Output path must be within `allowedRoot`.
+     */
+    exportToFile(datasetId, outputPath, allowedRoot) {
+        if (allowedRoot) {
+            const resolvedOut = resolve(outputPath);
+            const resolvedRoot = resolve(allowedRoot);
+            if (!resolvedOut.startsWith(resolvedRoot + sep) && resolvedOut !== resolvedRoot) {
+                throw new Error(`Export path escapes allowed root: ${resolvedOut}`);
+            }
+        }
+        const datasets = this.listDatasets();
+        const dataset = datasets.find((d) => d.datasetId === datasetId);
+        const entries = this.getEntries(datasetId);
+        writeFileSync(outputPath, JSON.stringify({ dataset, entries }, null, 2), 'utf-8');
+    }
+}
+//# sourceMappingURL=dataset-manager.js.map

package/dist/src/eval/dataset-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-manager.js","sourceRoot":"","sources":["../../../src/eval/dataset-manager.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACnG,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AASrD,MAAM,OAAO,cAAc;IACjB,YAAY,CAAS;IACrB,WAAW,CAAS;IAE5B,YAAY,YAAoB,EAAE,WAAmB;QACnD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,IAA0B;QACzC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACxF,MAAM,OAAO,GAAgB;YAC3B,SAAS,EAAE,UAAU,EAAE;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU;YACV,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,GAAG;YACd,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;SAC/B,CAAC;QAEF,cAAc,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAE3E,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,KAAK,GAAqB;gBAC9B,OAAO,EAAE,UAAU,EAAE;gBACrB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,OAAO,EAAE,GAAG;aACb,CAAC;YACF,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,IAAI,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACxD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzD,OAAO,UAAU,CAAc,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,SAAiB;QAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC;YAAE,OAAO,EAAE,CAAC;QAC7C,IAAI,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACxD,OAAO,UAAU,CAAmB,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACxF,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,SAAiB,EAAE,OAAe;QAClD,MAAM,KAAK,GAAqB;YAC9B,OAAO,EAAE,UAAU,EAAE;YACrB,SAAS;YACT,OAAO;YACP,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SAClC,CAAC;QACF,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAExE,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACjC,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC9B,OAAO,EAAE,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;YACrF,CAAC;YACD,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,YAAY,IAAI,UAAU,EAAE,MAAM,CAAC;QACvD,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QACrF,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAEnC,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,SAAiB,EAAE,UAAkB,EAAE,WAAoB;QACtE,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,YAAY,GAAG,GAAG,CAAC,IAAI,WAAW,KAAK,YAAY,EAAE,CAAC;gBAChF,MAAM,IAAI,KAAK,CAAC,qCAAqC,WAAW,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC3C,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;CACF"}

package/dist/src/eval/dataset-runner.d.ts

@@ -0,0 +1,23 @@
+import type { EvalRunResult, EvalTrace } from '../../../shared/src/types/eval.js';
+export interface AgentRunnerResult {
+    agentOutput: string;
+    outcome: 'success' | 'failure' | 'timeout';
+    qualityScore: number;
+    latencyMs: number;
+}
+export interface DatasetRunOpts {
+    datasetId: string;
+    agentVersion: string;
+    traces: EvalTrace[];
+    agentRunner: (trace: EvalTrace) => Promise<AgentRunnerResult>;
+    baselineResult?: EvalRunResult;
+    regressionThreshold?: number;
+}
+export declare class DatasetRunner {
+    /**
+     * Run all traces through the agent runner and compute stats.
+     * Optionally compare against a baseline to detect regressions.
+     */
+    run(opts: DatasetRunOpts): Promise<EvalRunResult>;
+}
+//# sourceMappingURL=dataset-runner.d.ts.map

package/dist/src/eval/dataset-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-runner.d.ts","sourceRoot":"","sources":["../../../src/eval/dataset-runner.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAoB,MAAM,mCAAmC,CAAC;AAEpG,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,WAAW,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC9D,cAAc,CAAC,EAAE,aAAa,CAAC;IAC/B,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,aAAa;IACxB;;;OAGG;IACG,GAAG,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;CA6DxD"}

package/dist/src/eval/dataset-runner.js

@@ -0,0 +1,59 @@
+/**
+ * DatasetRunner - Run eval datasets against agents and detect regressions (Task 33)
+ */
+import { randomUUID } from 'crypto';
+export class DatasetRunner {
+    /**
+     * Run all traces through the agent runner and compute stats.
+     * Optionally compare against a baseline to detect regressions.
+     */
+    async run(opts) {
+        const { datasetId, agentVersion, traces, agentRunner, baselineResult, regressionThreshold = 0.1, } = opts;
+        const results = [];
+        for (const trace of traces) {
+            const result = await agentRunner(trace);
+            results.push({ trace, result });
+        }
+        const passCount = results.filter((r) => r.result.outcome === 'success').length;
+        const failCount = results.length - passCount;
+        const totalQuality = results.reduce((sum, r) => sum + r.result.qualityScore, 0);
+        const totalLatency = results.reduce((sum, r) => sum + r.result.latencyMs, 0);
+        const avgQualityScore = results.length > 0 ? totalQuality / results.length : 0;
+        const avgLatencyMs = results.length > 0 ? totalLatency / results.length : 0;
+        // Regression detection
+        const regressionDetails = [];
+        let regressionDetected = false;
+        if (baselineResult) {
+            const delta = baselineResult.avgQualityScore - avgQualityScore;
+            if (delta > regressionThreshold) {
+                regressionDetected = true;
+                // Report per-trace regressions for traces with quality below baseline average
+                for (const { trace, result } of results) {
+                    if (result.qualityScore < baselineResult.avgQualityScore) {
+                        regressionDetails.push({
+                            traceId: trace.traceId,
+                            agentSlug: trace.agentSlug,
+                            baselineScore: baselineResult.avgQualityScore,
+                            currentScore: result.qualityScore,
+                            delta: baselineResult.avgQualityScore - result.qualityScore,
+                        });
+                    }
+                }
+            }
+        }
+        return {
+            runId: randomUUID(),
+            datasetId,
+            runAt: new Date().toISOString(),
+            agentVersion,
+            entriesTested: results.length,
+            passCount,
+            failCount,
+            avgQualityScore,
+            avgLatencyMs,
+            regressionDetected,
+            regressionDetails,
+        };
+    }
+}
+//# sourceMappingURL=dataset-runner.js.map

package/dist/src/eval/dataset-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-runner.js","sourceRoot":"","sources":["../../../src/eval/dataset-runner.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAmBpC,MAAM,OAAO,aAAa;IACxB;;;OAGG;IACH,KAAK,CAAC,GAAG,CAAC,IAAoB;QAC5B,MAAM,EACJ,SAAS,EACT,YAAY,EACZ,MAAM,EACN,WAAW,EACX,cAAc,EACd,mBAAmB,GAAG,GAAG,GAC1B,GAAG,IAAI,CAAC;QAET,MAAM,OAAO,GAA2D,EAAE,CAAC;QAE3E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;QAC/E,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;QAC7C,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC7E,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/E,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5E,uBAAuB;QACvB,MAAM,iBAAiB,GAAuB,EAAE,CAAC;QACjD,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAE/B,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,cAAc,CAAC,eAAe,GAAG,eAAe,CAAC;YAC/D,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;gBAChC,kBAAkB,GAAG,IAAI,CAAC;gBAC1B,8EAA8E;gBAC9E,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;oBACxC,IAAI,MAAM,CAAC,YAAY,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;wBACzD,iBAAiB,CAAC,IAAI,CAAC;4BACrB,OAAO,EAAE,KAAK,CAAC,OAAO;4BACtB,SAAS,EAAE,KAAK,CAAC,SAAS;4BAC1B,aAAa,EAAE,cAAc,CAAC,eAAe;4BAC7C,YAAY,EAAE,MAAM,CAAC,YAAY;4BACjC,KAAK,EAAE,cAAc,CAAC,eAAe,GAAG,MAAM,CAAC,YAAY;yBAC5D,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,UAAU,EAAE;YACnB,SAAS;YACT,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC/B,YAAY;YACZ,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,SAAS;YACT,SAAS;YACT,eAAe;YACf,YAAY;YACZ,kBAAkB;YAClB,iBAAiB;SAClB,CAAC;IACJ,CAAC;CACF"}

package/dist/src/eval/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Eval Module - Automated eval dataset from production traces (Task 33)
+ */
+export { TraceCollector } from './trace-collector.js';
+export type { RecordTraceInput } from './trace-collector.js';
+export { DatasetManager } from './dataset-manager.js';
+export type { CreateFromTracesOpts } from './dataset-manager.js';
+export { DatasetRunner } from './dataset-runner.js';
+export type { AgentRunnerResult, DatasetRunOpts } from './dataset-runner.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,YAAY,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/src/eval/index.js

@@ -0,0 +1,7 @@
+/**
+ * Eval Module - Automated eval dataset from production traces (Task 33)
+ */
+export { TraceCollector } from './trace-collector.js';
+export { DatasetManager } from './dataset-manager.js';
+export { DatasetRunner } from './dataset-runner.js';
+//# sourceMappingURL=index.js.map

package/dist/src/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGtD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/src/eval/trace-collector.d.ts

@@ -0,0 +1,40 @@
+import type { EvalTrace } from '../../../shared/src/types/eval.js';
+export interface RecordTraceInput {
+    agentSlug: string;
+    agentVersion: string;
+    taskDescription: string;
+    taskInput: string;
+    agentOutput: string;
+    retryCount: number;
+    qualityScore?: number;
+    outcome: 'success' | 'failure' | 'timeout';
+    latencyMs: number;
+    tokenCount?: number;
+    costUsd?: number;
+    correctedOutput?: string;
+}
+export declare class TraceCollector {
+    private filePath;
+    constructor(filePath: string);
+    /**
+     * Determine auto review status based on trace quality signals.
+     */
+    autoReviewStatus(input: RecordTraceInput): 'pending' | 'approved';
+    /**
+     * Auto-generate tags based on trace characteristics.
+     */
+    autoTag(input: RecordTraceInput): string[];
+    /**
+     * Record a trace, auto-generating traceId, capturedAt, reviewStatus, and tags.
+     */
+    record(input: RecordTraceInput): EvalTrace;
+    /**
+     * Read all traces from the JSONL file.
+     */
+    readAll(): EvalTrace[];
+    /**
+     * Get traces pending review, with optional limit.
+     */
+    getTracesPendingReview(limit?: number): EvalTrace[];
+}
+//# sourceMappingURL=trace-collector.d.ts.map

package/dist/src/eval/trace-collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace-collector.d.ts","sourceRoot":"","sources":["../../../src/eval/trace-collector.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mCAAmC,CAAC;AAGnE,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAEb,QAAQ,EAAE,MAAM;IAI5B;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,GAAG,SAAS,GAAG,UAAU;IAOjE;;OAEG;IACH,OAAO,CAAC,KAAK,EAAE,gBAAgB,GAAG,MAAM,EAAE;IAQ1C;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,gBAAgB,GAAG,SAAS;IAsC1C;;OAEG;IACH,OAAO,IAAI,SAAS,EAAE;IAWtB;;OAEG;IACH,sBAAsB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE;CAKpD"}

package/dist/src/eval/trace-collector.js

@@ -0,0 +1,102 @@
+/**
+ * TraceCollector - JSONL-based production trace collection (Task 33)
+ */
+import { randomUUID } from 'crypto';
+import { appendFileSync, readFileSync, existsSync, statSync } from 'fs';
+import { parseJsonl } from '../utils/parse-jsonl.js';
+export class TraceCollector {
+    filePath;
+    constructor(filePath) {
+        this.filePath = filePath;
+    }
+    /**
+     * Determine auto review status based on trace quality signals.
+     */
+    autoReviewStatus(input) {
+        if (input.retryCount > 1)
+            return 'pending';
+        if (input.qualityScore !== undefined && input.qualityScore < 0.6)
+            return 'pending';
+        if (input.outcome === 'failure')
+            return 'pending';
+        return 'approved';
+    }
+    /**
+     * Auto-generate tags based on trace characteristics.
+     */
+    autoTag(input) {
+        const tags = [];
+        if (input.retryCount > 1)
+            tags.push('high-retry');
+        if (input.outcome === 'failure')
+            tags.push('failure');
+        if (input.outcome === 'timeout')
+            tags.push('timeout');
+        return tags;
+    }
+    /**
+     * Record a trace, auto-generating traceId, capturedAt, reviewStatus, and tags.
+     */
+    record(input) {
+        const trace = {
+            traceId: randomUUID(),
+            agentSlug: input.agentSlug,
+            agentVersion: input.agentVersion,
+            taskDescription: input.taskDescription,
+            taskInput: input.taskInput,
+            agentOutput: input.agentOutput,
+            retryCount: input.retryCount,
+            qualityScore: input.qualityScore,
+            outcome: input.outcome,
+            latencyMs: input.latencyMs,
+            tokenCount: input.tokenCount,
+            costUsd: input.costUsd,
+            capturedAt: new Date().toISOString(),
+            reviewStatus: this.autoReviewStatus(input),
+            correctedOutput: input.correctedOutput,
+            tags: this.autoTag(input),
+        };
+        // Defensive serialization — agent outputs may contain circular references
+        // or BigInt; without this guard the writer crashes mid-trace.
+        let serialized;
+        try {
+            serialized = JSON.stringify(trace);
+        }
+        catch {
+            serialized = JSON.stringify({
+                traceId: trace.traceId,
+                agentSlug: trace.agentSlug,
+                capturedAt: trace.capturedAt,
+                reviewStatus: trace.reviewStatus,
+                outcome: 'serialize_failed',
+            });
+        }
+        appendFileSync(this.filePath, serialized + '\n', 'utf-8');
+        return trace;
+    }
+    /**
+     * Read all traces from the JSONL file.
+     */
+    readAll() {
+        if (!existsSync(this.filePath))
+            return [];
+        const stat = statSync(this.filePath);
+        if (stat.size > 256 * 1024 * 1024) {
+            throw new Error(`Trace file exceeds 256MB (${stat.size} bytes). Run rotation/cleanup.`);
+        }
+        const content = readFileSync(this.filePath, 'utf-8').trim();
+        if (!content)
+            return [];
+        return parseJsonl(content);
+    }
+    /**
+     * Get traces pending review, with optional limit.
+     */
+    getTracesPendingReview(limit) {
+        const all = this.readAll().filter((t) => t.reviewStatus === 'pending');
+        if (limit !== undefined)
+            return all.slice(0, limit);
+        return all;
+    }
+}
+//# sourceMappingURL=trace-collector.js.map

package/dist/src/eval/trace-collector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace-collector.js","sourceRoot":"","sources":["../../../src/eval/trace-collector.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAExE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAiBrD,MAAM,OAAO,cAAc;IACjB,QAAQ,CAAS;IAEzB,YAAY,QAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAuB;QACtC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC;YAAE,OAAO,SAAS,CAAC;QAC3C,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,IAAI,KAAK,CAAC,YAAY,GAAG,GAAG;YAAE,OAAO,SAAS,CAAC;QACnF,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAClD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,KAAuB;QAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAuB;QAC5B,MAAM,KAAK,GAAc;YACvB,OAAO,EAAE,UAAU,EAAE;YACrB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACpC,YAAY,EAAE,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;YAC1C,eAAe,EAAE,KAAK,CAAC,eAAe;YACtC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;SAC1B,CAAC;QAEF,0EAA0E;QAC1E,8DAA8D;QAC9D,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACH,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,OAAO,EAAE,kBAAkB;aAC5B,CAAC,CAAC;QACL,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,IAAI,gCAAgC,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACxB,OAAO,UAAU,CAAY,OAAO,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,KAAc;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC;QACvE,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACpD,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}