@agentmark-ai/cli 0.9.0 → 0.10.0

package/dist/server/routes/experiments/index.js

@@ -0,0 +1,192 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getExperimentById = exports.getExperiments = void 0;
+const database_1 = __importDefault(require("../../database"));
+const getExperiments = () => __awaiter(void 0, void 0, void 0, function* () {
+    // Root spans have dataset metadata; child spans with Model set have cost/promptName.
+    // Use Model != '' instead of Type = 'GENERATION' since type classification may not
+    // match for all SDK versions (e.g. OpenTelemetry GenAI vs Vercel AI SDK conventions).
+    // Scores are stored with resource_id = TraceId (not SpanId).
+    const sql = `
+    SELECT
+      root.DatasetRunId AS id,
+      COALESCE(NULLIF(root.DatasetRunName, ''), root.DatasetRunId) AS name,
+      COALESCE(MAX(NULLIF(gen.PromptName, '')), MAX(NULLIF(root.PromptName, ''))) AS promptName,
+      MAX(NULLIF(root.DatasetPath, '')) AS datasetPath,
+      COUNT(DISTINCT root.TraceId) AS itemCount,
+      AVG(root.Duration) AS avgLatencyMs,
+      COALESCE(SUM(gen.item_cost), 0.0) AS totalCost,
+      MIN(root.CreatedAt) AS createdAt
+    FROM traces root
+    LEFT JOIN (
+      SELECT TraceId,
+        SUM(COALESCE(Cost, 0.0)) AS item_cost,
+        MAX(NULLIF(PromptName, '')) AS PromptName
+      FROM traces
+      WHERE Model IS NOT NULL AND Model != ''
+      GROUP BY TraceId
+    ) gen ON gen.TraceId = root.TraceId
+    WHERE root.DatasetRunId IS NOT NULL
+      AND root.DatasetRunId != ''
+      AND root.ParentSpanId IS NULL
+    GROUP BY root.DatasetRunId
+    ORDER BY MIN(root.CreatedAt) DESC
+  `;
+    const rows = database_1.default.prepare(sql).all();
+    // Fetch average scores — scores.resource_id stores the OTEL TraceId
+    return rows.map((row) => {
+        var _a;
+        const scoresSql = `
+      SELECT AVG(s.score) AS avg_score
+      FROM scores s
+      WHERE s.resource_id IN (
+        SELECT DISTINCT TraceId FROM traces WHERE DatasetRunId = ?
+      )
+    `;
+        const scoreRow = database_1.default.prepare(scoresSql).get(row.id);
+        return {
+            id: row.id,
+            name: row.name,
+            promptName: row.promptName || "",
+            datasetPath: row.datasetPath || "",
+            itemCount: row.itemCount,
+            avgLatencyMs: Math.round(row.avgLatencyMs || 0),
+            totalCost: row.totalCost || 0,
+            avgScore: (_a = scoreRow === null || scoreRow === void 0 ? void 0 : scoreRow.avg_score) !== null && _a !== void 0 ? _a : null,
+            createdAt: row.createdAt,
+        };
+    });
+});
+exports.getExperiments = getExperiments;
+const getExperimentById = (experimentId) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a;
+    // Summary: root spans for dataset metadata, JOIN child model spans for cost/promptName
+    const summarySql = `
+    SELECT
+      root.DatasetRunId AS id,
+      COALESCE(NULLIF(root.DatasetRunName, ''), root.DatasetRunId) AS name,
+      COALESCE(MAX(NULLIF(gen.PromptName, '')), MAX(NULLIF(root.PromptName, ''))) AS promptName,
+      MAX(NULLIF(root.DatasetPath, '')) AS datasetPath,
+      COUNT(DISTINCT root.TraceId) AS itemCount,
+      AVG(root.Duration) AS avgLatencyMs,
+      COALESCE(SUM(gen.item_cost), 0.0) AS totalCost,
+      MIN(root.CreatedAt) AS createdAt
+    FROM traces root
+    LEFT JOIN (
+      SELECT TraceId,
+        SUM(COALESCE(Cost, 0.0)) AS item_cost,
+        MAX(NULLIF(PromptName, '')) AS PromptName
+      FROM traces
+      WHERE Model IS NOT NULL AND Model != ''
+      GROUP BY TraceId
+    ) gen ON gen.TraceId = root.TraceId
+    WHERE root.DatasetRunId = ?
+      AND root.ParentSpanId IS NULL
+    GROUP BY root.DatasetRunId
+  `;
+    const summaryRow = database_1.default.prepare(summarySql).get(experimentId);
+    if (!summaryRow)
+        return null;
+    // Scores use resource_id = TraceId (OTEL trace ID)
+    const scoresSql = `
+    SELECT AVG(s.score) AS avg_score
+    FROM scores s
+    WHERE s.resource_id IN (
+      SELECT DISTINCT TraceId FROM traces WHERE DatasetRunId = ?
+    )
+  `;
+    const scoreRow = database_1.default.prepare(scoresSql).get(experimentId);
+    const summary = {
+        id: summaryRow.id,
+        name: summaryRow.name,
+        promptName: summaryRow.promptName || "",
+        datasetPath: summaryRow.datasetPath || "",
+        itemCount: summaryRow.itemCount,
+        avgLatencyMs: Math.round(summaryRow.avgLatencyMs || 0),
+        totalCost: summaryRow.totalCost || 0,
+        avgScore: (_a = scoreRow === null || scoreRow === void 0 ? void 0 : scoreRow.avg_score) !== null && _a !== void 0 ? _a : null,
+        createdAt: summaryRow.createdAt,
+    };
+    // Items: root span for dataset metadata, JOIN child model spans for input/output/cost.
+    // Input/Output may be in normalized columns (Vercel AI SDK) or in SpanAttributes JSON
+    // as gen_ai.prompt / gen_ai.completion (OpenTelemetry GenAI conventions).
+    const itemsSql = `
+    SELECT
+      root.TraceId AS traceId,
+      COALESCE(NULLIF(root.DatasetItemName, ''), root.SpanName, 'Item') AS itemName,
+      COALESCE(
+        NULLIF(gen.input, ''),
+        NULLIF(gen.genai_input, ''),
+        NULLIF(root.Input, ''),
+        ''
+      ) AS input,
+      COALESCE(root.DatasetExpectedOutput, '') AS expectedOutput,
+      COALESCE(
+        NULLIF(gen.output, ''),
+        NULLIF(gen.outputObject, ''),
+        NULLIF(gen.genai_output, ''),
+        NULLIF(root.Output, ''),
+        NULLIF(root.OutputObject, ''),
+        ''
+      ) AS actualOutput,
+      COALESCE(root.Duration, 0) AS latencyMs,
+      COALESCE(gen.item_cost, root.Cost, 0.0) AS cost
+    FROM traces root
+    LEFT JOIN (
+      SELECT TraceId,
+        MAX(Input) AS input,
+        MAX(Output) AS output,
+        MAX(OutputObject) AS outputObject,
+        SUM(COALESCE(Cost, 0.0)) AS item_cost,
+        MAX(json_extract(SpanAttributes, '$."gen_ai.prompt"')) AS genai_input,
+        MAX(json_extract(SpanAttributes, '$."gen_ai.completion"')) AS genai_output
+      FROM traces
+      WHERE Model IS NOT NULL AND Model != ''
+      GROUP BY TraceId
+    ) gen ON gen.TraceId = root.TraceId
+    WHERE root.DatasetRunId = ?
+      AND root.ParentSpanId IS NULL
+    ORDER BY CAST(root.Timestamp AS REAL) ASC
+  `;
+    const itemRows = database_1.default.prepare(itemsSql).all(experimentId);
+    const items = itemRows.map((item) => {
+        // Scores use resource_id = TraceId (OTEL trace ID, not SpanId)
+        const itemScoresSql = `
+      SELECT name, score, label, reason
+      FROM scores
+      WHERE resource_id = ?
+      ORDER BY created_at ASC
+    `;
+        const itemScores = database_1.default.prepare(itemScoresSql).all(item.traceId);
+        return {
+            traceId: item.traceId,
+            itemName: item.itemName,
+            input: item.input,
+            expectedOutput: item.expectedOutput,
+            actualOutput: item.actualOutput,
+            latencyMs: item.latencyMs,
+            cost: item.cost,
+            scores: itemScores.map((s) => ({
+                name: s.name,
+                score: s.score,
+                label: s.label,
+                reason: s.reason,
+            })),
+        };
+    });
+    return { summary, items };
+});
+exports.getExperimentById = getExperimentById;
+//# sourceMappingURL=index.js.map

package/dist/server/routes/experiments/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../cli-src/server/routes/experiments/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,8DAAgC;AAyBzB,MAAM,cAAc,GAAG,GAAuC,EAAE;IACrE,qFAAqF;IACrF,mFAAmF;IACnF,sFAAsF;IACtF,6DAA6D;IAC7D,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;;;;;GAwBX,CAAC;IAEF,MAAM,IAAI,GAAG,kBAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAW,CAAC;IAE5C,oEAAoE;IACpE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;;QACtB,MAAM,SAAS,GAAG;;;;;;KAMjB,CAAC;QACF,MAAM,QAAQ,GAAG,kBAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAQ,CAAC;QAE1D,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,EAAE;YAChC,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;YAClC,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC;YAC/C,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC;YAC7B,QAAQ,EAAE,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,SAAS,mCAAI,IAAI;YACrC,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAA,CAAC;AAxDW,QAAA,cAAc,kBAwDzB;AAEK,MAAM,iBAAiB,GAAG,CAC/B,YAAoB,EACqD,EAAE;;IAC3E,uFAAuF;IACvF,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;GAsBlB,CAAC;IAEF,MAAM,UAAU,GAAG,kBAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,YAAY,CAAQ,CAAC;IACnE,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAE7B,mDAAmD;IACnD,MAAM,SAAS,GAAG;;;;;;GAMjB,CAAC;IACF,MAAM,QAAQ,GAAG,kBAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,YAAY,CAAQ,CAAC;IAEhE,MAAM,OAAO,GAAsB;QACjC,EAAE,EAAE,UAAU,CAAC,EAAE;QACjB,IAAI,EAAE,UAAU,CAAC,IAAI;QACrB,UAAU,EAAE,UAAU,CAAC,UAAU,IAAI,EAAE;QACvC,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,EAAE;QACzC,SAAS,EAAE,UAAU,CAAC,SAAS;QAC/B,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,YAAY,IAAI,CAAC,CAAC;QACtD,SAAS,EAAE,UAAU,CAAC,SAAS,IAAI,CAAC;QACpC,QAAQ,EAAE,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,SAAS,mCAAI,IAAI;QACrC,SAAS,EAAE,UAAU,CAAC,SAAS;KAChC,CAAC;IAEF,uFAAuF;IACvF,sFAAsF;IACtF,0EAA0E;IAC1E,MAAM,QAAQ,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqChB,CAAC;IAEF,MAAM,QAAQ,GAAG,kBAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,YAAY,CAAU,CAAC;IAEjE,MAAM,KAAK,GAAqB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACpD,+DAA+D;QAC/D,MAAM,aAAa,GAAG;;;;;KAKrB,CAAC;QACF,MAAM,UAAU,GAAG,kBAAE,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAU,CAAC;QAExE,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBAClC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC,CAAA,CAAC;AA7HW,QAAA,iBAAiB,qBA6H5B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentmark-ai/cli",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "Agentmark's CLI",
   "main": "dist/index.js",
   "exports": {
@@ -57,7 +57,7 @@
     "yaml": "^2.7.0"
   },
   "devDependencies": {
-    "@agentmark-ai/ui-components": "0.3.6",
+    "@agentmark-ai/ui-components": "0.4.0",
     "@eslint/eslintrc": "^3",
     "@types/better-sqlite3": "^7.6.13",
     "@types/cors": "^2",