@agent-native/core 0.7.11 → 0.7.13

package/dist/observability/evals.js

@@ -0,0 +1,371 @@
+import { getTraceSummary, insertEvalResult, getEvalDataset } from "./store.js";
+import { getRunById, getRunEventsSince } from "../agent/run-store.js";
+import { resolveEngine, getStoredModelForEngine, } from "../agent/engine/index.js";
+const LATENCY_BASELINE_PER_TOOL_MS = 10_000;
+const COST_BASELINE_PER_TOOL_CX100 = 50;
+const LLM_JUDGE_TIMEOUT_MS = 30_000;
+function makeEvalResult(opts) {
+    return {
+        id: crypto.randomUUID(),
+        runId: opts.runId,
+        threadId: opts.threadId,
+        userId: opts.userId,
+        evalType: opts.evalType,
+        criteria: opts.criteria,
+        score: Math.max(0, Math.min(1, opts.score)),
+        reasoning: opts.reasoning ?? null,
+        metadata: opts.metadata ?? null,
+        createdAt: Date.now(),
+    };
+}
+/** Lift the (runId, threadId, userId) triple off a TraceSummary —
+ *  every automated scorer pulls these together. */
+function fromSummary(summary) {
+    return {
+        runId: summary.runId,
+        threadId: summary.threadId,
+        userId: summary.userId,
+    };
+}
+// ─── Layer 1: Automated deterministic scorers ────────────────────────
+function scoreToolSuccessRate(summary) {
+    const total = summary.toolCalls;
+    const score = total > 0 ? summary.successfulTools / total : 1.0;
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "tool_success_rate",
+        score,
+        metadata: {
+            totalTools: total,
+            successfulTools: summary.successfulTools,
+            failedTools: summary.failedTools,
+        },
+    });
+}
+function scoreStepEfficiency(summary) {
+    // No tool calls = simple Q&A, maximally efficient.
+    // With tools: penalize excessive LLM iterations relative to tool calls.
+    const score = summary.toolCalls === 0
+        ? 1.0
+        : summary.llmCalls > 0
+            ? Math.min(1, summary.toolCalls / summary.llmCalls)
+            : 1.0;
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "step_efficiency",
+        score,
+        metadata: { llmCalls: summary.llmCalls, toolCalls: summary.toolCalls },
+    });
+}
+function scoreLatency(summary) {
+    const expectedMs = Math.max(LATENCY_BASELINE_PER_TOOL_MS, summary.toolCalls * LATENCY_BASELINE_PER_TOOL_MS);
+    const score = Math.max(0, 1 - summary.totalDurationMs / expectedMs);
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "latency_score",
+        score,
+        metadata: { actualMs: summary.totalDurationMs, expectedMs },
+    });
+}
+function scoreCostEfficiency(summary) {
+    const expectedCx100 = Math.max(COST_BASELINE_PER_TOOL_CX100, summary.toolCalls * COST_BASELINE_PER_TOOL_CX100);
+    const score = Math.max(0, 1 - summary.totalCostCentsX100 / expectedCx100);
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "cost_efficiency",
+        score,
+        metadata: { actualCx100: summary.totalCostCentsX100, expectedCx100 },
+    });
+}
+function scoreErrorRecovery(summary, runStatus) {
+    const hadErrors = summary.failedTools > 0;
+    let score;
+    if (!hadErrors) {
+        score = 1.0;
+    }
+    else if (runStatus === "completed") {
+        score = 1.0;
+    }
+    else {
+        score = 0;
+    }
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "error_recovery",
+        score,
+        metadata: { hadErrors, runStatus },
+    });
+}
+export async function runAutomatedEvals(runId) {
+    const [summary, run] = await Promise.all([
+        getTraceSummary(runId),
+        getRunById(runId),
+    ]);
+    if (!summary)
+        return [];
+    const runStatus = run?.status ?? "unknown";
+    const results = [
+        scoreToolSuccessRate(summary),
+        scoreStepEfficiency(summary),
+        scoreLatency(summary),
+        scoreCostEfficiency(summary),
+        scoreErrorRecovery(summary, runStatus),
+    ];
+    for (const result of results) {
+        insertEvalResult(result).catch(() => { });
+    }
+    return results;
+}
+// ─── Layer 2: LLM-as-Judge ───────────────────────────────────────────
+function buildConversationTranscript(events) {
+    const lines = [];
+    for (const { eventData } of events) {
+        try {
+            const event = JSON.parse(eventData);
+            if (event.type === "user-message") {
+                lines.push(`[User]: ${event.text ?? JSON.stringify(event.content)}`);
+            }
+            else if (event.type === "text-delta" || event.type === "text") {
+                lines.push(`[Agent]: ${event.text}`);
+            }
+            else if (event.type === "tool-call") {
+                lines.push(`[Tool Call: ${event.name}] ${JSON.stringify(event.input)}`);
+            }
+            else if (event.type === "tool-result") {
+                const snippet = typeof event.content === "string"
+                    ? event.content.slice(0, 500)
+                    : JSON.stringify(event.content).slice(0, 500);
+                lines.push(`[Tool Result${event.isError ? " (ERROR)" : ""}]: ${snippet}`);
+            }
+        }
+        catch {
+            // Skip unparseable events
+        }
+    }
+    return lines.join("\n");
+}
+function buildJudgePrompt(transcript, criteria) {
+    let prompt = `You are an expert evaluator. Assess the following agent conversation against the given criteria.
+
+## Criteria
+Name: ${criteria.name}
+Description: ${criteria.description}`;
+    if (criteria.rubric) {
+        prompt += `\nRubric: ${criteria.rubric}`;
+    }
+    const min = criteria.scoreRange?.min ?? 0;
+    const max = criteria.scoreRange?.max ?? 1;
+    prompt += `
+
+## Conversation Transcript
+${transcript}
+
+## Instructions
+Evaluate the conversation and respond with ONLY a JSON object (no markdown, no explanation outside the JSON):
+{"score": <number between ${min} and ${max}>, "reasoning": "<brief explanation>"}`;
+    return prompt;
+}
+export async function runLlmJudgeEval(runId, criteria, opts) {
+    try {
+        const [events, run] = await Promise.all([
+            getRunEventsSince(runId, 0),
+            getRunById(runId),
+        ]);
+        if (events.length === 0)
+            return null;
+        const transcript = buildConversationTranscript(events);
+        if (!transcript.trim())
+            return null;
+        const engine = opts?.engine ?? (await resolveEngine({ engineOption: undefined }));
+        const model = opts?.model ??
+            (await getStoredModelForEngine(engine)) ??
+            engine.defaultModel;
+        const judgePrompt = buildJudgePrompt(transcript, criteria);
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), LLM_JUDGE_TIMEOUT_MS);
+        let responseText = "";
+        try {
+            const stream = engine.stream({
+                model,
+                systemPrompt: "You are an evaluation judge. Respond only with valid JSON.",
+                messages: [
+                    { role: "user", content: [{ type: "text", text: judgePrompt }] },
+                ],
+                tools: [],
+                abortSignal: controller.signal,
+                maxOutputTokens: 512,
+                temperature: 0,
+            });
+            for await (const event of stream) {
+                if (event.type === "text-delta") {
+                    responseText += event.text;
+                }
+            }
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+        const jsonMatch = responseText.match(/\{[\s\S]*\}/);
+        if (!jsonMatch)
+            return null;
+        const parsed = JSON.parse(jsonMatch[0]);
+        const min = criteria.scoreRange?.min ?? 0;
+        const max = criteria.scoreRange?.max ?? 1;
+        const normalizedScore = max > min ? (parsed.score - min) / (max - min) : parsed.score;
+        const result = makeEvalResult({
+            runId,
+            threadId: run?.threadId ?? null,
+            userId: opts?.userId ?? null,
+            evalType: "llm_judge",
+            criteria: criteria.name,
+            score: normalizedScore,
+            reasoning: parsed.reasoning,
+            metadata: { model, rawScore: parsed.score, scoreRange: { min, max } },
+        });
+        insertEvalResult(result).catch(() => { });
+        return result;
+    }
+    catch {
+        return null;
+    }
+}
+// ─── Layer 3: Dataset evaluation ─────────────────────────────────────
+export async function runDatasetEval(datasetId, opts) {
+    const dataset = await getEvalDataset(datasetId);
+    if (!dataset) {
+        return { datasetId, totalCases: 0, avgScore: 0, results: [] };
+    }
+    const engine = opts?.engine ?? (await resolveEngine({ engineOption: undefined }));
+    const model = opts?.model ??
+        (await getStoredModelForEngine(engine)) ??
+        engine.defaultModel;
+    const criteria = opts?.criteria ?? [
+        {
+            name: "response_quality",
+            description: "How well the agent's response addresses the user's input, considering accuracy, completeness, and helpfulness.",
+        },
+    ];
+    const allResults = [];
+    for (const testCase of dataset.entries) {
+        const transcript = buildTestCaseTranscript(testCase, engine, model);
+        for (const c of criteria) {
+            const result = await evaluateTestCase(datasetId, testCase, transcript, c, engine, model);
+            if (result)
+                allResults.push(result);
+        }
+    }
+    const avgScore = allResults.length > 0
+        ? allResults.reduce((sum, r) => sum + r.score, 0) / allResults.length
+        : 0;
+    return {
+        datasetId,
+        totalCases: dataset.entries.length,
+        avgScore,
+        results: allResults,
+    };
+}
+function buildTestCaseTranscript(testCase, _engine, _model) {
+    let transcript = `[User]: ${testCase.input}`;
+    if (testCase.expectedOutput) {
+        transcript += `\n[Expected Output]: ${testCase.expectedOutput}`;
+    }
+    if (testCase.context) {
+        transcript += `\n[Context]: ${JSON.stringify(testCase.context)}`;
+    }
+    return transcript;
+}
+async function evaluateTestCase(datasetId, testCase, transcript, criteria, engine, model) {
+    try {
+        const judgePrompt = buildJudgePrompt(transcript, criteria);
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), LLM_JUDGE_TIMEOUT_MS);
+        let responseText = "";
+        try {
+            const stream = engine.stream({
+                model,
+                systemPrompt: "You are an evaluation judge. Respond only with valid JSON.",
+                messages: [
+                    { role: "user", content: [{ type: "text", text: judgePrompt }] },
+                ],
+                tools: [],
+                abortSignal: controller.signal,
+                maxOutputTokens: 512,
+                temperature: 0,
+            });
+            for await (const event of stream) {
+                if (event.type === "text-delta") {
+                    responseText += event.text;
+                }
+            }
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+        const jsonMatch = responseText.match(/\{[\s\S]*\}/);
+        if (!jsonMatch)
+            return null;
+        const parsed = JSON.parse(jsonMatch[0]);
+        const min = criteria.scoreRange?.min ?? 0;
+        const max = criteria.scoreRange?.max ?? 1;
+        const normalizedScore = max > min ? (parsed.score - min) / (max - min) : parsed.score;
+        // Dataset evals use a synthetic runId since there's no real run
+        const syntheticRunId = `dataset:${datasetId}:${crypto.randomUUID()}`;
+        // Dataset evals are administrative — there's no per-user runId, so
+        // we leave userId null. Per-user reads filter null rows out, which
+        // is the right default; admins can fetch dataset evals via the
+        // unfiltered call path.
+        const result = makeEvalResult({
+            runId: syntheticRunId,
+            threadId: null,
+            userId: null,
+            evalType: "llm_judge",
+            criteria: criteria.name,
+            score: normalizedScore,
+            reasoning: parsed.reasoning,
+            metadata: {
+                datasetId,
+                model,
+                testCaseInput: testCase.input,
+                expectedOutput: testCase.expectedOutput ?? null,
+                tags: testCase.tags ?? [],
+                rawScore: parsed.score,
+                scoreRange: { min, max },
+            },
+        });
+        insertEvalResult(result).catch(() => { });
+        return result;
+    }
+    catch {
+        return null;
+    }
+}
+// ─── Orchestrator ────────────────────────────────────────────────────
+export async function evaluateRun(runId, opts) {
+    const results = await runAutomatedEvals(runId);
+    const userId = results[0]?.userId ?? null;
+    const sampleRate = opts?.sampleRate ?? 0;
+    if (sampleRate > 0 && Math.random() < sampleRate) {
+        const defaultCriteria = [
+            {
+                name: "overall_quality",
+                description: "Overall quality of the agent's response, considering helpfulness, accuracy, and appropriate tool usage.",
+            },
+            {
+                name: "task_completion",
+                description: "Whether the agent successfully completed the user's requested task.",
+            },
+        ];
+        const judgeResults = await Promise.all(defaultCriteria.map((c) => runLlmJudgeEval(runId, c, { userId })));
+        for (const r of judgeResults) {
+            if (r)
+                results.push(r);
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=evals.js.map

package/dist/observability/evals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evals.js","sourceRoot":"","sources":["../../src/observability/evals.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAEtE,OAAO,EACL,aAAa,EACb,uBAAuB,GACxB,MAAM,0BAA0B,CAAC;AAElC,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAC5C,MAAM,4BAA4B,GAAG,EAAE,CAAC;AACxC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAapC,SAAS,cAAc,CAAC,IAAwB;IAC9C,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;QACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,IAAI;QAC/B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;AACJ,CAAC;AAED;mDACmD;AACnD,SAAS,WAAW,CAAC,OAAqB;IAKxC,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;AACJ,CAAC;AAED,wEAAwE;AAExE,SAAS,oBAAoB,CAAC,OAAqB;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC;IAChC,MAAM,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;IAChE,OAAO,cAAc,CAAC;QACpB,GAAG,WAAW,CAAC,OAAO,CAAC;QACvB,QAAQ,EAAE,WAAW;QACrB,QAAQ,EAAE,mBAAmB;QAC7B,KAAK;QACL,QAAQ,EAAE;YACR,UAAU,EAAE,KAAK;YACjB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC;KACF,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAqB;IAChD,mDAAmD;IACnD,wEAAwE;IACxE,MAAM,KAAK,GACT,OAAO,CAAC,SAAS,KAAK,CAAC;QACrB,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC;YACpB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC;YACnD,CAAC,CAAC,GAAG,CAAC;IACZ,OAAO,cAAc,CAAC;QACpB,GAAG,WAAW,CAAC,OAAO,CAAC;QACvB,QAAQ,EAAE,WAAW;QACrB,QAAQ,EAAE,iBAAiB;QAC3B,KAAK;QACL,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;KACvE,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,OAAqB;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,4BAA4B,EAC5B,OAAO,CAAC,SAAS,GAAG,4BAA4B,CACjD,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,eAAe,GAAG,UAAU,CAAC,CAAC;IACpE,OAAO,cAAc,CAAC;QACpB,GAAG,WAAW,CAAC,OAAO,CAAC;QACvB,QAAQ,EAAE,WAAW;QACrB,QAAQ,EAAE,eAAe;QACzB,KAAK;QACL,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,eAAe,EAAE,UAAU,EAAE;KAC5D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAqB;IAChD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,4BAA4B,EAC5B,OAAO,CAAC,SAAS,GAAG,4BAA4B,CACjD,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,kBAAkB,GAAG,aAAa,CAAC,CAAC;IAC1E,OAAO,cAAc,CAAC;QACpB,GAAG,WAAW,CAAC,OAAO,CAAC;QACvB,QAAQ,EAAE,WAAW;QACrB,QAAQ,EAAE,iBAAiB;QAC3B,KAAK;QACL,QAAQ,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,kBAAkB,EAAE,aAAa,EAAE;KACrE,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqB,EACrB,SAAiB;IAEjB,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC;IAC1C,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;SAAM,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;QACrC,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;SAAM,CAAC;QACN,KAAK,GAAG,CAAC,CAAC;IACZ,CAAC;IACD,OAAO,cAAc,CAAC;QACpB,GAAG,WAAW,CAAC,OAAO,CAAC;QACvB,QAAQ,EAAE,WAAW;QACrB,QAAQ,EAAE,gBAAgB;QAC1B,KAAK;QACL,QAAQ,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE;KACnC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAAa;IACnD,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACvC,eAAe,CAAC,KAAK,CAAC;QACtB,UAAU,CAAC,KAAK,CAAC;KAClB,CAAC,CAAC;IAEH,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAExB,MAAM,SAAS,GAAG,GAAG,EAAE,MAAM,IAAI,SAAS,CAAC;IAC3C,MAAM,OAAO,GAAG;QACd,oBAAoB,CAAC,OAAO,CAAC;QAC7B,mBAAmB,CAAC,OAAO,CAAC;QAC5B,YAAY,CAAC,OAAO,CAAC;QACrB,mBAAmB,CAAC,OAAO,CAAC;QAC5B,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC;KACvC,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,wEAAwE;AAExE,SAAS,2BAA2B,CAClC,MAAiD;IAEjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,EAAE,SAAS,EAAE,IAAI,MAAM,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACpC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAClC,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACvE,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YACvC,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1E,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBACxC,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;oBAC/B,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBAC7B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAClD,KAAK,CAAC,IAAI,CACR,eAAe,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,OAAO,EAAE,CAC9D,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,UAAkB,EAAE,QAAsB;IAClE,IAAI,MAAM,GAAG;;;QAGP,QAAQ,CAAC,IAAI;eACN,QAAQ,CAAC,WAAW,EAAE,CAAC;IAEpC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,aAAa,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IAE1C,MAAM,IAAI;;;EAGV,UAAU;;;;4BAIgB,GAAG,QAAQ,GAAG,wCAAwC,CAAC;IAEjF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,QAAsB,EACtB,IAAuE;IAEvE,IAAI,CAAC;QACH,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACtC,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC;YAC3B,UAAU,CAAC,KAAK,CAAC;SAClB,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAErC,MAAM,UAAU,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;QACvD,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QAEpC,MAAM,MAAM,GACV,IAAI,EAAE,MAAM,IAAI,CAAC,MAAM,aAAa,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;QACrE,MAAM,KAAK,GACT,IAAI,EAAE,KAAK;YACX,CAAC,MAAM,uBAAuB,CAAC,MAAM,CAAC,CAAC;YACvC,MAAM,CAAC,YAAY,CAAC;QAEtB,MAAM,WAAW,GAAG,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAE3D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAE3E,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC3B,KAAK;gBACL,YAAY,EACV,4DAA4D;gBAC9D,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE;iBACjE;gBACD,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,UAAU,CAAC,MAAM;gBAC9B,eAAe,EAAE,GAAG;gBACpB,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;YAEH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACpD,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAGrC,CAAC;QAEF,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,eAAe,GACnB,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAEhE,MAAM,MAAM,GAAG,cAAc,CAAC;YAC5B,KAAK;YACL,QAAQ,EAAE,GAAG,EAAE,QAAQ,IAAI,IAAI;YAC/B,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,IAAI;YAC5B,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,KAAK,EAAE,eAAe;YACtB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;SACtE,CAAC,CAAC;QAEH,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,wEAAwE;AAExE,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,IAA0E;IAO1E,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAChE,CAAC;IAED,MAAM,MAAM,GACV,IAAI,EAAE,MAAM,IAAI,CAAC,MAAM,aAAa,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IACrE,MAAM,KAAK,GACT,IAAI,EAAE,KAAK;QACX,CAAC,MAAM,uBAAuB,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,YAAY,CAAC;IAEtB,MAAM,QAAQ,GAAG,IAAI,EAAE,QAAQ,IAAI;QACjC;YACE,IAAI,EAAE,kBAAkB;YACxB,WAAW,EACT,gHAAgH;SACnH;KACF,CAAC;IAEF,MAAM,UAAU,GAAiB,EAAE,CAAC;IAEpC,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,UAAU,GAAG,uBAAuB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAEpE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM,gBAAgB,CACnC,SAAS,EACT,QAAQ,EACR,UAAU,EACV,CAAC,EACD,MAAM,EACN,KAAK,CACN,CAAC;YACF,IAAI,MAAM;gBAAE,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GACZ,UAAU,CAAC,MAAM,GAAG,CAAC;QACnB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;QACrE,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,SAAS;QACT,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;QAClC,QAAQ;QACR,OAAO,EAAE,UAAU;KACpB,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,QAAsB,EACtB,OAAoB,EACpB,MAAc;IAEd,IAAI,UAAU,GAAG,WAAW,QAAQ,CAAC,KAAK,EAAE,CAAC;IAC7C,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC5B,UAAU,IAAI,wBAAwB,QAAQ,CAAC,cAAc,EAAE,CAAC;IAClE,CAAC;IACD,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrB,UAAU,IAAI,gBAAgB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;IACnE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAiB,EACjB,QAAsB,EACtB,UAAkB,EAClB,QAAsB,EACtB,MAAmB,EACnB,KAAa;IAEb,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAE3D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,oBAAoB,CAAC,CAAC;QAE3E,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBAC3B,KAAK;gBACL,YAAY,EACV,4DAA4D;gBAC9D,QAAQ,EAAE;oBACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE;iBACjE;gBACD,KAAK,EAAE,EAAE;gBACT,WAAW,EAAE,UAAU,CAAC,MAAM;gBAC9B,eAAe,EAAE,GAAG;gBACpB,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;YAEH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,YAAY,IAAI,KAAK,CAAC,IAAI,CAAC;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACpD,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAGrC,CAAC;QAEF,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,eAAe,GACnB,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAEhE,gEAAgE;QAChE,MAAM,cAAc,GAAG,WAAW,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;QAErE,mEAAmE;QACnE,mEAAmE;QACnE,+DAA+D;QAC/D,wBAAwB;QACxB,MAAM,MAAM,GAAG,cAAc,CAAC;YAC5B,KAAK,EAAE,cAAc;YACrB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,KAAK,EAAE,eAAe;YACtB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,QAAQ,EAAE;gBACR,SAAS;gBACT,KAAK;gBACL,aAAa,EAAE,QAAQ,CAAC,KAAK;gBAC7B,cAAc,EAAE,QAAQ,CAAC,cAAc,IAAI,IAAI;gBAC/C,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;gBACzB,QAAQ,EAAE,MAAM,CAAC,KAAK;gBACtB,UAAU,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE;aACzB;SACF,CAAC,CAAC;QAEH,gBAAgB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,wEAAwE;AAExE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,IAA8B;IAE9B,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,IAAI,CAAC;IAE1C,MAAM,UAAU,GAAG,IAAI,EAAE,UAAU,IAAI,CAAC,CAAC;IACzC,IAAI,UAAU,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACjD,MAAM,eAAe,GAAmB;YACtC;gBACE,IAAI,EAAE,iBAAiB;gBACvB,WAAW,EACT,yGAAyG;aAC5G;YACD;gBACE,IAAI,EAAE,iBAAiB;gBACvB,WAAW,EACT,qEAAqE;aACxE;SACF,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,CAClE,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;YAC7B,IAAI,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/observability/experiments.d.ts

@@ -0,0 +1,24 @@
+import type { Experiment, ExperimentVariant, ExperimentMetricResult } from "./types.js";
+export declare function createExperiment(opts: {
+    name: string;
+    variants: Array<{
+        id: string;
+        weight: number;
+        config: Record<string, unknown>;
+    }>;
+    metrics: string[];
+    assignmentLevel?: "user" | "session";
+}): Promise<Experiment>;
+export declare function startExperiment(id: string): Promise<void>;
+export declare function pauseExperiment(id: string): Promise<void>;
+export declare function completeExperiment(id: string): Promise<void>;
+export declare function resolveVariant(experimentId: string, userId: string): Promise<ExperimentVariant>;
+export declare function resolveActiveExperimentConfig(userId: string): Promise<{
+    configs: Record<string, unknown>;
+    assignments: Array<{
+        experimentId: string;
+        variantId: string;
+    }>;
+} | null>;
+export declare function computeExperimentResults(experimentId: string): Promise<ExperimentMetricResult[]>;
+//# sourceMappingURL=experiments.d.ts.map

package/dist/observability/experiments.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../src/observability/experiments.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAkDpB,wBAAsB,gBAAgB,CAAC,IAAI,EAAE;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACjC,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,OAAO,CAAC,UAAU,CAAC,CActB;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG/D;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG/D;AAED,wBAAsB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;AAID,wBAAsB,cAAc,CAClC,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,iBAAiB,CAAC,CA8C5B;AAED,wBAAsB,6BAA6B,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,WAAW,EAAE,KAAK,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,IAAI,CAAC,CAmBR;AAID,wBAAsB,wBAAwB,CAC5C,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,sBAAsB,EAAE,CAAC,CA2JnC"}