@zhivex-ai/core 0.7.0 → 0.8.0

package/dist/workflow-evaluation-diff.js

@@ -0,0 +1,141 @@
+const summarizeReport = (report) => ({
+    ok: report.ok,
+    total: report.total,
+    passed: report.passed,
+    failed: report.failed,
+    passRate: report.passRate
+});
+const failuresFor = (testCase) => new Set(testCase?.failures ?? []);
+const difference = (left, right) => [...left].filter((value) => !right.has(value)).sort();
+const reportCaseMap = (report) => new Map(report.cases.map((testCase) => [testCase.name, testCase]));
+const caseChanged = (base, target, newFailures, resolvedFailures) => base.ok !== target.ok ||
+    base.status !== target.status ||
+    newFailures.length > 0 ||
+    resolvedFailures.length > 0;
+export const compareWorkflowEvaluationReports = (base, target, options = {}) => {
+    const baseByName = reportCaseMap(base);
+    const targetByName = reportCaseMap(target);
+    const names = [...new Set([...baseByName.keys(), ...targetByName.keys()])].sort();
+    const cases = [];
+    const regressions = [];
+    const improvements = [];
+    if (target.passRate < base.passRate) {
+        regressions.push(`Pass rate decreased from ${base.passRate} to ${target.passRate}.`);
+    }
+    else if (target.passRate > base.passRate) {
+        improvements.push(`Pass rate increased from ${base.passRate} to ${target.passRate}.`);
+    }
+    if (target.ok !== base.ok) {
+        if (base.ok) {
+            regressions.push("Evaluation changed from passing to failing.");
+        }
+        else {
+            improvements.push("Evaluation changed from failing to passing.");
+        }
+    }
+    for (const name of names) {
+        const baseCase = baseByName.get(name);
+        const targetCase = targetByName.get(name);
+        const newFailures = difference(failuresFor(targetCase), failuresFor(baseCase));
+        const resolvedFailures = difference(failuresFor(baseCase), failuresFor(targetCase));
+        if (!baseCase && targetCase) {
+            cases.push({
+                name,
+                status: "added",
+                targetOk: targetCase.ok,
+                targetStatus: targetCase.status,
+                newFailures,
+                resolvedFailures
+            });
+            if (!targetCase.ok || newFailures.length > 0) {
+                regressions.push(`Failing case "${name}" was added.`);
+            }
+            continue;
+        }
+        if (baseCase && !targetCase) {
+            cases.push({
+                name,
+                status: "removed",
+                baseOk: baseCase.ok,
+                baseStatus: baseCase.status,
+                newFailures,
+                resolvedFailures
+            });
+            if (!baseCase.ok || resolvedFailures.length > 0) {
+                improvements.push(`Failing case "${name}" was removed.`);
+            }
+            continue;
+        }
+        if (!baseCase || !targetCase) {
+            continue;
+        }
+        const status = caseChanged(baseCase, targetCase, newFailures, resolvedFailures)
+            ? "changed"
+            : "unchanged";
+        cases.push({
+            name,
+            status,
+            baseOk: baseCase.ok,
+            targetOk: targetCase.ok,
+            baseStatus: baseCase.status,
+            targetStatus: targetCase.status,
+            newFailures,
+            resolvedFailures
+        });
+        if (baseCase.ok && !targetCase.ok) {
+            regressions.push(`Case "${name}" regressed.`);
+        }
+        else if (!baseCase.ok && targetCase.ok) {
+            improvements.push(`Case "${name}" recovered.`);
+        }
+        if (newFailures.length > 0) {
+            regressions.push(`Case "${name}" has ${newFailures.length} new failure(s).`);
+        }
+        if (resolvedFailures.length > 0) {
+            improvements.push(`Case "${name}" resolved ${resolvedFailures.length} failure(s).`);
+        }
+        if (baseCase.status !== targetCase.status) {
+            const message = `Case "${name}" status changed from "${baseCase.status}" to "${targetCase.status}".`;
+            if (targetCase.ok || !baseCase.ok) {
+                improvements.push(message);
+            }
+            else {
+                regressions.push(message);
+            }
+        }
+    }
+    return {
+        ok: regressions.length === 0,
+        base: summarizeReport(base),
+        target: summarizeReport(target),
+        delta: {
+            total: target.total - base.total,
+            passed: target.passed - base.passed,
+            failed: target.failed - base.failed,
+            passRate: target.passRate - base.passRate
+        },
+        regressions,
+        improvements,
+        cases,
+        metadata: options.metadata
+    };
+};
+export const createWorkflowEvaluationDiffReport = (diff) => ({
+    ...diff,
+    cases: diff.cases.map((testCase) => ({
+        ...testCase,
+        newFailures: [...testCase.newFailures],
+        resolvedFailures: [...testCase.resolvedFailures]
+    })),
+    regressions: [...diff.regressions],
+    improvements: [...diff.improvements],
+    summary: {
+        added: diff.cases.filter((testCase) => testCase.status === "added").length,
+        removed: diff.cases.filter((testCase) => testCase.status === "removed").length,
+        changed: diff.cases.filter((testCase) => testCase.status === "changed").length,
+        unchanged: diff.cases.filter((testCase) => testCase.status === "unchanged").length,
+        newFailures: diff.cases.reduce((total, testCase) => total + testCase.newFailures.length, 0),
+        resolvedFailures: diff.cases.reduce((total, testCase) => total + testCase.resolvedFailures.length, 0)
+    }
+});
+//# sourceMappingURL=workflow-evaluation-diff.js.map

package/dist/workflow-evaluation-diff.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow-evaluation-diff.js","sourceRoot":"","sources":["../src/workflow-evaluation-diff.ts"],"names":[],"mappings":"AAoDA,MAAM,eAAe,GAAG,CAAC,MAAgC,EAAiC,EAAE,CAAC,CAAC;IAC5F,EAAE,EAAE,MAAM,CAAC,EAAE;IACb,KAAK,EAAE,MAAM,CAAC,KAAK;IACnB,MAAM,EAAE,MAAM,CAAC,MAAM;IACrB,MAAM,EAAE,MAAM,CAAC,MAAM;IACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;CAC1B,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,QAAkD,EAAe,EAAE,CACtF,IAAI,GAAG,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC;AAEpC,MAAM,UAAU,GAAG,CAAC,IAAiB,EAAE,KAAkB,EAAY,EAAE,CACrE,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAExD,MAAM,aAAa,GAAG,CAAC,MAAgC,EAA6C,EAAE,CACpG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;AAErE,MAAM,WAAW,GAAG,CAClB,IAAkC,EAClC,MAAoC,EACpC,WAAqB,EACrB,gBAA0B,EACjB,EAAE,CACX,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE;IACrB,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;IAC7B,WAAW,CAAC,MAAM,GAAG,CAAC;IACtB,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;AAE9B,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAC9C,IAA8B,EAC9B,MAAgC,EAChC,UAAoD,EAAE,EAC9B,EAAE;IAC1B,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClF,MAAM,KAAK,GAAiC,EAAE,CAAC;IAC/C,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,YAAY,GAAa,EAAE,CAAC;IAElC,IAAI,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,WAAW,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,QAAQ,OAAO,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvF,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,YAAY,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,QAAQ,OAAO,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;IACxF,CAAC;IAED,IAAI,MAAM,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,WAAW,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC/E,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC;QAEpF,IAAI,CAAC,QAAQ,IAAI,UAAU,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI;gBACJ,MAAM,EAAE,OAAO;gBACf,QAAQ,EAAE,UAAU,CAAC,EAAE;gBACvB,YAAY,EAAE,UAAU,CAAC,MAAM;gBAC/B,WAAW;gBACX,gBAAgB;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7C,WAAW,CAAC,IAAI,CAAC,iBAAiB,IAAI,cAAc,CAAC,CAAC;YACxD,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI;gBACJ,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACnB,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,WAAW;gBACX,gBAAgB;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChD,YAAY,CAAC,IAAI,CAAC,iBAAiB,IAAI,gBAAgB,CAAC,CAAC;YAC3D,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,EAAE,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAqC,WAAW,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,gBAAgB,CAAC;YAC/G,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,EAAE;YACnB,QAAQ,EAAE,UAAU,CAAC,EAAE;YACvB,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,YAAY,EAAE,UAAU,CAAC,MAAM;YAC/B,WAAW;YACX,gBAAgB;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YAClC,WAAW,CAAC,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC,CAAC;QAChD,CAAC;aAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,UAAU,CAAC,EAAE,EAAE,CAAC;YACzC,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,WAAW,CAAC,IAAI,CAAC,SAAS,IAAI,SAAS,WAAW,CAAC,MAAM,kBAAkB,CAAC,CAAC;QAC/E,CAAC;QACD,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,cAAc,gBAAgB,CAAC,MAAM,cAAc,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,SAAS,IAAI,0BAA0B,QAAQ,CAAC,MAAM,SAAS,UAAU,CAAC,MAAM,IAAI,CAAC;YACrG,IAAI,UAAU,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAClC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,EAAE,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC;QAC5B,IAAI,EAAE,eAAe,CAAC,IAAI,CAAC;QAC3B,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE;YACL,KAAK,EAAE,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;SAC1C;QACD,WAAW;QACX,YAAY;QACZ,KAAK;QACL,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAChD,IAA4B,EACE,EAAE,CAAC,CAAC;IAClC,GAAG,IAAI;IACP,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnC,GAAG,QAAQ;QACX,WAAW,EAAE,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC;QACtC,gBAAgB,EAAE,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC;KACjD,CAAC,CAAC;IACH,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;IAClC,YAAY,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;IACpC,OAAO,EAAE;QACP,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;QAC1E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC9E,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC9E,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM;QAClF,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,GAAG,QAAQ,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3F,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAC;KACtG;CACF,CAAC,CAAC"}

package/dist/workflow-evaluation.d.ts

@@ -0,0 +1,94 @@
+import type { JsonValue, LanguageModel } from "./types.js";
+import { type WorkflowDefinition, type WorkflowReplayTimelineEvent, type WorkflowRunInput, type WorkflowRunOutput, type WorkflowStatus, type WorkflowStepStatus } from "./workflow.js";
+export interface WorkflowEvaluationExpectations {
+    status?: WorkflowStatus;
+    outputs?: Record<string, JsonValue>;
+    outputContains?: Record<string, string> | string[];
+    stepStatuses?: Record<string, WorkflowStepStatus>;
+    stepOutputContains?: Record<string, string>;
+    errorContains?: string;
+    timelineContains?: WorkflowReplayTimelineEvent["type"][];
+}
+export interface WorkflowEvaluationCase {
+    name: string;
+    input: WorkflowRunInput;
+    expectations?: WorkflowEvaluationExpectations;
+    metadata?: Record<string, JsonValue>;
+}
+export interface WorkflowEvaluationCaseResult {
+    name: string;
+    ok: boolean;
+    output: WorkflowRunOutput;
+    failures: string[];
+    metadata?: Record<string, JsonValue>;
+}
+export interface WorkflowEvaluationResult {
+    ok: boolean;
+    cases: WorkflowEvaluationCaseResult[];
+}
+export interface WorkflowEvaluationJudgeResult {
+    score: number;
+    feedback?: string;
+    metadata?: Record<string, JsonValue>;
+}
+export type WorkflowEvaluationJudge = ((result: WorkflowEvaluationResult) => WorkflowEvaluationJudgeResult | Promise<WorkflowEvaluationJudgeResult>) | {
+    model: LanguageModel;
+    prompt?: string;
+};
+export interface RunWorkflowEvaluationOptions {
+    workflow: WorkflowDefinition | ((testCase: WorkflowEvaluationCase) => WorkflowDefinition | Promise<WorkflowDefinition>);
+}
+export interface WorkflowEvaluationFixture {
+    name?: string;
+    dataset: WorkflowEvaluationCase[];
+    expectedOk?: boolean;
+    metadata?: Record<string, JsonValue>;
+    createdAt?: number;
+}
+export interface WorkflowEvaluationReportCase {
+    name: string;
+    ok: boolean;
+    status: WorkflowStatus;
+    failures: string[];
+    outputPreview: string;
+    outputKeys: string[];
+    stepCount: number;
+    stepStatusCounts: Record<string, number>;
+    timelineEventCounts: Record<string, number>;
+    durationMs?: number;
+    metadata?: Record<string, JsonValue>;
+}
+export interface WorkflowEvaluationReport {
+    ok: boolean;
+    total: number;
+    passed: number;
+    failed: number;
+    passRate: number;
+    statusCounts: Record<string, number>;
+    stepCount: number;
+    stepStatusCounts: Record<string, number>;
+    timelineEventCounts: Record<string, number>;
+    failures: Array<{
+        name: string;
+        failures: string[];
+    }>;
+    cases: WorkflowEvaluationReportCase[];
+    judge?: WorkflowEvaluationJudgeResult;
+    metadata?: Record<string, JsonValue>;
+}
+export declare const runWorkflowEvaluation: (dataset: WorkflowEvaluationCase[], options: RunWorkflowEvaluationOptions) => Promise<WorkflowEvaluationResult>;
+export declare const createWorkflowEvaluationFixture: (options: {
+    name?: string;
+    dataset: WorkflowEvaluationCase[];
+    expectedOk?: boolean;
+    metadata?: Record<string, JsonValue>;
+    createdAt?: number;
+}) => WorkflowEvaluationFixture;
+export declare const runWorkflowEvaluationFixture: (fixture: WorkflowEvaluationFixture, options: RunWorkflowEvaluationOptions) => Promise<WorkflowEvaluationResult>;
+export declare const createWorkflowEvaluationReport: (result: WorkflowEvaluationResult, options?: {
+    judge?: WorkflowEvaluationJudgeResult;
+    metadata?: Record<string, JsonValue>;
+    outputPreviewLength?: number;
+}) => WorkflowEvaluationReport;
+export declare const judgeWorkflowEvaluation: (result: WorkflowEvaluationResult, judge: WorkflowEvaluationJudge) => Promise<WorkflowEvaluationJudgeResult>;
+//# sourceMappingURL=workflow-evaluation.d.ts.map

package/dist/workflow-evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow-evaluation.d.ts","sourceRoot":"","sources":["../src/workflow-evaluation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3D,OAAO,EAGL,KAAK,kBAAkB,EACvB,KAAK,2BAA2B,EAChC,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EAEnB,KAAK,kBAAkB,EACxB,MAAM,eAAe,CAAC;AAEvB,MAAM,WAAW,8BAA8B;IAC7C,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;IACnD,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAClD,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,2BAA2B,CAAC,MAAM,CAAC,EAAE,CAAC;CAC1D;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,gBAAgB,CAAC;IACxB,YAAY,CAAC,EAAE,8BAA8B,CAAC;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,4BAA4B,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,6BAA6B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,MAAM,uBAAuB,GAC/B,CAAC,CAAC,MAAM,EAAE,wBAAwB,KAAK,6BAA6B,GAAG,OAAO,CAAC,6BAA6B,CAAC,CAAC,GAC9G;IACE,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEN,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EACJ,kBAAkB,GAClB,CAAC,CAAC,QAAQ,EAAE,sBAAsB,KAAK,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;CAC9F;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,sBAAsB,EAAE,CAAC;IAClC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,cAAc,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IACtD,KAAK,EAAE,4BAA4B,EAAE,CAAC;IACtC,KAAK,CAAC,EAAE,6BAA6B,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AA4GD,eAAO,MAAM,qBAAqB,GAChC,SAAS,sBAAsB,EAAE,EACjC,SAAS,4BAA4B,KACpC,OAAO,CAAC,wBAAwB,CAoBlC,CAAC;AAEF,eAAO,MAAM,+BAA+B,GAAI,SAAS;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,sBAAsB,EAAE,CAAC;IAClC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,yBAMF,CAAC;AAEH,eAAO,MAAM,4BAA4B,GACvC,SAAS,yBAAyB,EAClC,SAAS,4BAA4B,KACpC,OAAO,CAAC,wBAAwB,CAAoD,CAAC;AAWxF,eAAO,MAAM,8BAA8B,GACzC,QAAQ,wBAAwB,EAChC,UAAS;IACP,KAAK,CAAC,EAAE,6BAA6B,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACrC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CACzB,KACL,wBA6DF,CAAC;AAiBF,eAAO,MAAM,uBAAuB,GAClC,QAAQ,wBAAwB,EAChC,OAAO,uBAAuB,KAC7B,OAAO,CAAC,6BAA6B,CAgBvC,CAAC"}

package/dist/workflow-evaluation.js

@@ -0,0 +1,210 @@
+import { generateText } from "./generate-text.js";
+import { createTextMessage, serializeJsonValue } from "./messages.js";
+import { replayWorkflowRun, runWorkflow } from "./workflow.js";
+const preview = (text, length) => text.length > length ? `${text.slice(0, Math.max(0, length))}...` : text;
+const flattenStepResults = (steps) => steps.flatMap((step) => [step, ...flattenStepResults(step.children ?? [])]);
+const createStepMap = (steps) => {
+    const map = new Map();
+    for (const step of flattenStepResults(steps)) {
+        map.set(step.id, [...(map.get(step.id) ?? []), step]);
+    }
+    return map;
+};
+const getLatestStep = (stepMap, stepId) => stepMap.get(stepId)?.at(-1);
+const valueEquals = (left, right) => JSON.stringify(left) === JSON.stringify(right);
+const evaluateExpectations = (output, expectations) => {
+    const failures = [];
+    if (!expectations) {
+        return failures;
+    }
+    if (expectations.status && output.status !== expectations.status) {
+        failures.push(`Expected status "${expectations.status}", received "${output.status}".`);
+    }
+    if (expectations.outputs) {
+        for (const [key, expected] of Object.entries(expectations.outputs)) {
+            if (!(key in output.outputs)) {
+                failures.push(`Expected output key "${key}".`);
+                continue;
+            }
+            if (!valueEquals(output.outputs[key], expected)) {
+                failures.push(`Expected output "${key}" to equal ${JSON.stringify(expected)}.`);
+            }
+        }
+    }
+    if (Array.isArray(expectations.outputContains)) {
+        const serializedOutputs = JSON.stringify(output.outputs);
+        for (const expectedText of expectations.outputContains) {
+            if (!serializedOutputs.includes(expectedText)) {
+                failures.push(`Expected serialized outputs to contain "${expectedText}".`);
+            }
+        }
+    }
+    else if (expectations.outputContains) {
+        for (const [key, expectedText] of Object.entries(expectations.outputContains)) {
+            const value = output.outputs[key];
+            if (value === undefined || !String(value).includes(expectedText)) {
+                failures.push(`Expected output "${key}" to contain "${expectedText}".`);
+            }
+        }
+    }
+    const stepMap = createStepMap(output.steps);
+    if (expectations.stepStatuses) {
+        for (const [stepId, expectedStatus] of Object.entries(expectations.stepStatuses)) {
+            const step = getLatestStep(stepMap, stepId);
+            if (!step) {
+                failures.push(`Expected step "${stepId}".`);
+                continue;
+            }
+            if (step.status !== expectedStatus) {
+                failures.push(`Expected step "${stepId}" status "${expectedStatus}", received "${step.status}".`);
+            }
+        }
+    }
+    if (expectations.stepOutputContains) {
+        for (const [stepId, expectedText] of Object.entries(expectations.stepOutputContains)) {
+            const step = getLatestStep(stepMap, stepId);
+            if (!step?.outputText?.includes(expectedText)) {
+                failures.push(`Expected step "${stepId}" output to contain "${expectedText}".`);
+            }
+        }
+    }
+    if (expectations.errorContains) {
+        const errors = flattenStepResults(output.steps).flatMap((step) => step.error?.message ? [step.error.message] : []);
+        if (!errors.some((message) => message.includes(expectations.errorContains))) {
+            failures.push(`Expected workflow error to contain "${expectations.errorContains}".`);
+        }
+    }
+    if (expectations.timelineContains?.length) {
+        const replay = replayWorkflowRun(output.state);
+        const eventTypes = new Set(replay.timeline.map((event) => event.type));
+        for (const eventType of expectations.timelineContains) {
+            if (!eventTypes.has(eventType)) {
+                failures.push(`Expected replay timeline event "${eventType}".`);
+            }
+        }
+    }
+    return failures;
+};
+export const runWorkflowEvaluation = async (dataset, options) => {
+    const cases = [];
+    for (const testCase of dataset) {
+        const workflow = typeof options.workflow === "function" ? await options.workflow(testCase) : options.workflow;
+        const output = await runWorkflow(workflow, testCase.input);
+        const failures = evaluateExpectations(output, testCase.expectations);
+        cases.push({
+            name: testCase.name,
+            ok: failures.length === 0,
+            output,
+            failures,
+            metadata: testCase.metadata
+        });
+    }
+    return {
+        ok: cases.every((testCase) => testCase.ok),
+        cases
+    };
+};
+export const createWorkflowEvaluationFixture = (options) => ({
+    name: options.name,
+    dataset: serializeJsonValue(options.dataset),
+    expectedOk: options.expectedOk,
+    metadata: options.metadata,
+    createdAt: options.createdAt ?? Date.now()
+});
+export const runWorkflowEvaluationFixture = async (fixture, options) => runWorkflowEvaluation(fixture.dataset, options);
+const workflowDuration = (output) => output.state.createdAt !== undefined && output.state.updatedAt !== undefined
+    ? Math.max(0, output.state.updatedAt - output.state.createdAt)
+    : undefined;
+const incrementCounts = (counts, key) => {
+    counts[key] = (counts[key] ?? 0) + 1;
+};
+export const createWorkflowEvaluationReport = (result, options = {}) => {
+    const outputPreviewLength = options.outputPreviewLength ?? 500;
+    const cases = result.cases.map((testCase) => {
+        const steps = flattenStepResults(testCase.output.steps);
+        const stepStatusCounts = {};
+        const timelineEventCounts = {};
+        const replay = replayWorkflowRun(testCase.output.state);
+        for (const step of steps) {
+            incrementCounts(stepStatusCounts, step.status);
+        }
+        for (const event of replay.timeline) {
+            incrementCounts(timelineEventCounts, event.type);
+        }
+        return {
+            name: testCase.name,
+            ok: testCase.ok,
+            status: testCase.output.status,
+            failures: [...testCase.failures],
+            outputPreview: preview(JSON.stringify(testCase.output.outputs), outputPreviewLength),
+            outputKeys: Object.keys(testCase.output.outputs),
+            stepCount: steps.length,
+            stepStatusCounts,
+            timelineEventCounts,
+            durationMs: workflowDuration(testCase.output),
+            metadata: testCase.metadata
+        };
+    });
+    const statusCounts = {};
+    const stepStatusCounts = {};
+    const timelineEventCounts = {};
+    for (const testCase of cases) {
+        incrementCounts(statusCounts, testCase.status);
+        for (const [status, count] of Object.entries(testCase.stepStatusCounts)) {
+            stepStatusCounts[status] = (stepStatusCounts[status] ?? 0) + count;
+        }
+        for (const [eventType, count] of Object.entries(testCase.timelineEventCounts)) {
+            timelineEventCounts[eventType] = (timelineEventCounts[eventType] ?? 0) + count;
+        }
+    }
+    const passed = cases.filter((testCase) => testCase.ok).length;
+    const failed = cases.length - passed;
+    return {
+        ok: result.ok,
+        total: cases.length,
+        passed,
+        failed,
+        passRate: cases.length ? passed / cases.length : 1,
+        statusCounts,
+        stepCount: cases.reduce((total, testCase) => total + testCase.stepCount, 0),
+        stepStatusCounts,
+        timelineEventCounts,
+        failures: cases.filter((testCase) => testCase.failures.length).map((testCase) => ({
+            name: testCase.name,
+            failures: testCase.failures
+        })),
+        cases,
+        judge: options.judge,
+        metadata: options.metadata
+    };
+};
+const parseJudgeResponse = (text) => {
+    try {
+        const parsed = JSON.parse(text);
+        return {
+            score: typeof parsed.score === "number" ? Math.max(0, Math.min(1, parsed.score)) : 0,
+            feedback: parsed.feedback
+        };
+    }
+    catch {
+        return {
+            score: 0,
+            feedback: text
+        };
+    }
+};
+export const judgeWorkflowEvaluation = async (result, judge) => {
+    if (typeof judge === "function") {
+        return judge(result);
+    }
+    const prompt = judge.prompt ??
+        "Score this workflow evaluation from 0 to 1. Return JSON with fields score and feedback.";
+    const response = await generateText({
+        model: judge.model,
+        messages: [
+            createTextMessage("user", `${prompt}\n\n${JSON.stringify(serializeJsonValue(result), null, 2)}`)
+        ]
+    });
+    return parseJudgeResponse(response.text);
+};
+//# sourceMappingURL=workflow-evaluation.js.map

package/dist/workflow-evaluation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow-evaluation.js","sourceRoot":"","sources":["../src/workflow-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEtE,OAAO,EACL,iBAAiB,EACjB,WAAW,EAQZ,MAAM,eAAe,CAAC;AAyFvB,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,MAAc,EAAU,EAAE,CACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAE3E,MAAM,kBAAkB,GAAG,CAAC,KAA2B,EAAwB,EAAE,CAC/E,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,kBAAkB,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAE9E,MAAM,aAAa,GAAG,CAAC,KAA2B,EAAqC,EAAE;IACvF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAgC,CAAC;IACpD,KAAK,MAAM,IAAI,IAAI,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CACpB,OAA0C,EAC1C,MAAc,EACkB,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAEjE,MAAM,WAAW,GAAG,CAAC,IAAe,EAAE,KAAgB,EAAW,EAAE,CACjE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAEjD,MAAM,oBAAoB,GAAG,CAC3B,MAAyB,EACzB,YAAwD,EAC9C,EAAE;IACZ,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,oBAAoB,YAAY,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;IAC1F,CAAC;IAED,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;YACnE,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC7B,QAAQ,CAAC,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACjD,QAAQ,CAAC,IAAI,CAAC,oBAAoB,GAAG,cAAc,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC;QAC/C,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzD,KAAK,MAAM,YAAY,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;YACvD,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC9C,QAAQ,CAAC,IAAI,CAAC,2CAA2C,YAAY,IAAI,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC;QACvC,KAAK,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,CAAC;YAC9E,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACjE,QAAQ,CAAC,IAAI,CAAC,oBAAoB,GAAG,iBAAiB,YAAY,IAAI,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,YAAY,CAAC,YAAY,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,QAAQ,CAAC,IAAI,CAAC,kBAAkB,MAAM,IAAI,CAAC,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,kBAAkB,MAAM,aAAa,cAAc,gBAAgB,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YACpG,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,kBAAkB,EAAE,CAAC;QACpC,KAAK,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACrF,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC9C,QAAQ,CAAC,IAAI,CAAC,kBAAkB,MAAM,wBAAwB,YAAY,IAAI,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,aAAa,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACnH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,aAAc,CAAC,CAAC,EAAE,CAAC;YAC7E,QAAQ,CAAC,IAAI,CAAC,uCAAuC,YAAY,CAAC,aAAa,IAAI,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACvE,KAAK,MAAM,SAAS,IAAI,YAAY,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC/B,QAAQ,CAAC,IAAI,CAAC,mCAAmC,SAAS,IAAI,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EACxC,OAAiC,EACjC,OAAqC,EACF,EAAE;IACrC,MAAM,KAAK,GAAmC,EAAE,CAAC;IAEjD,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,OAAO,OAAO,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC9G,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;QACrE,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,EAAE,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YACzB,MAAM;YACN,QAAQ;YACR,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,KAAK;KACN,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,OAM/C,EAA6B,EAAE,CAAC,CAAC;IAChC,IAAI,EAAE,OAAO,CAAC,IAAI;IAClB,OAAO,EAAE,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAwC;IACnF,UAAU,EAAE,OAAO,CAAC,UAAU;IAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;IAC1B,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;CAC3C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,4BAA4B,GAAG,KAAK,EAC/C,OAAkC,EAClC,OAAqC,EACF,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAExF,MAAM,gBAAgB,GAAG,CAAC,MAAyB,EAAsB,EAAE,CACzE,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS;IAC1E,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9D,CAAC,CAAC,SAAS,CAAC;AAEhB,MAAM,eAAe,GAAG,CAAC,MAA8B,EAAE,GAAW,EAAE,EAAE;IACtE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;AACvC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,MAAgC,EAChC,UAII,EAAE,EACoB,EAAE;IAC5B,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAgC,EAAE;QACxE,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,MAAM,gBAAgB,GAA2B,EAAE,CAAC;QACpD,MAAM,mBAAmB,GAA2B,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,eAAe,CAAC,gBAAgB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACjD,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpC,eAAe,CAAC,mBAAmB,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM;YAC9B,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAChC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,mBAAmB,CAAC;YACpF,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;YAChD,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,gBAAgB;YAChB,mBAAmB;YACnB,UAAU,EAAE,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC7C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,MAAM,gBAAgB,GAA2B,EAAE,CAAC;IACpD,MAAM,mBAAmB,GAA2B,EAAE,CAAC;IACvD,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,eAAe,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/C,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACxE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;QACrE,CAAC;QACD,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;YAC9E,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;QACjF,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IAErC,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,MAAM;QACN,MAAM;QACN,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAClD,YAAY;QACZ,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QAC3E,gBAAgB;QAChB,mBAAmB;QACnB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAChF,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC,CAAC;QACH,KAAK;QACL,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAiC,EAAE;IACzE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA2C,CAAC;QAC1E,OAAO;YACL,KAAK,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpF,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAC1C,MAAgC,EAChC,KAA8B,EACU,EAAE;IAC1C,IAAI,OAAO,KAAK,KAAK,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GACV,KAAK,CAAC,MAAM;QACZ,yFAAyF,CAAC;IAC5F,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC;QAClC,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE;YACR,iBAAiB,CAAC,MAAM,EAAE,GAAG,MAAM,OAAO,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SACjG;KACF,CAAC,CAAC;IAEH,OAAO,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC,CAAC"}

package/dist/workflow-state-service.d.ts

@@ -0,0 +1,67 @@
+import type { PostgresClientLike, SqliteDatabaseLike } from "./types.js";
+import { type PersistedWorkflowRunState, type WorkflowStatus } from "./workflow.js";
+export declare const WORKFLOW_STATE_RECORD_SCHEMA_VERSION: 1;
+export interface WorkflowStateLookup {
+    appName: string;
+    userId: string;
+    sessionId: string;
+    workflowKey: string;
+}
+export interface WorkflowStateListInput {
+    appName: string;
+    userId: string;
+    sessionId?: string;
+    workflowKey?: string;
+    status?: WorkflowStatus;
+}
+export interface WorkflowStateSaveInput extends WorkflowStateLookup {
+    state: PersistedWorkflowRunState;
+    expectedRevision?: number;
+}
+export interface WorkflowStateRecord extends WorkflowStateLookup {
+    schemaVersion: typeof WORKFLOW_STATE_RECORD_SCHEMA_VERSION;
+    revision: number;
+    state: PersistedWorkflowRunState;
+    status: WorkflowStatus;
+    runId: string;
+    createdAt: number;
+    updatedAt: number;
+}
+export interface WorkflowStateService {
+    saveWorkflowState(input: WorkflowStateSaveInput): Promise<WorkflowStateRecord> | WorkflowStateRecord;
+    loadWorkflowState(input: WorkflowStateLookup): Promise<WorkflowStateRecord | undefined> | WorkflowStateRecord | undefined;
+    listWorkflowStates(input: WorkflowStateListInput): Promise<WorkflowStateRecord[]> | WorkflowStateRecord[];
+    deleteWorkflowState(input: WorkflowStateLookup): Promise<void> | void;
+}
+export interface FileWorkflowStateServiceOptions {
+    directory: string;
+}
+export interface FileWorkflowStateStorePruneOptions extends FileWorkflowStateServiceOptions {
+    olderThanMs?: number;
+    keepLast?: number;
+    now?: number;
+    dryRun?: boolean;
+}
+export interface FileWorkflowStateStorePruneResult {
+    directory: string;
+    dryRun: boolean;
+    deletedWorkflowStateKeys: string[];
+    keptWorkflowStateKeys: string[];
+}
+export interface SqliteWorkflowStateServiceOptions {
+    db: SqliteDatabaseLike;
+    tableName?: string;
+}
+export interface PostgresWorkflowStateServiceOptions {
+    client: PostgresClientLike;
+    tableName?: string;
+}
+export type WorkflowStateRecordMigrationTarget = typeof WORKFLOW_STATE_RECORD_SCHEMA_VERSION;
+export declare const normalizeWorkflowStateRecord: (value: unknown) => WorkflowStateRecord;
+export declare const migrateWorkflowStateRecord: (value: unknown, targetVersion?: WorkflowStateRecordMigrationTarget) => WorkflowStateRecord;
+export declare const createInMemoryWorkflowStateService: () => WorkflowStateService;
+export declare const createFileWorkflowStateService: (options: FileWorkflowStateServiceOptions) => WorkflowStateService;
+export declare const pruneFileWorkflowStateStore: (options: FileWorkflowStateStorePruneOptions) => Promise<FileWorkflowStateStorePruneResult>;
+export declare const createSqliteWorkflowStateService: (options: SqliteWorkflowStateServiceOptions) => WorkflowStateService;
+export declare const createPostgresWorkflowStateService: (options: PostgresWorkflowStateServiceOptions) => WorkflowStateService;
+//# sourceMappingURL=workflow-state-service.d.ts.map

package/dist/workflow-state-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workflow-state-service.d.ts","sourceRoot":"","sources":["../src/workflow-state-service.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAa,kBAAkB,EAAE,kBAAkB,EAAuB,MAAM,YAAY,CAAC;AACzG,OAAO,EAAgE,KAAK,yBAAyB,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAKlJ,eAAO,MAAM,oCAAoC,EAAG,CAAU,CAAC;AAE/D,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,MAAM,WAAW,sBAAuB,SAAQ,mBAAmB;IACjE,KAAK,EAAE,yBAAyB,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAoB,SAAQ,mBAAmB;IAC9D,aAAa,EAAE,OAAO,oCAAoC,CAAC;IAC3D,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,yBAAyB,CAAC;IACjC,MAAM,EAAE,cAAc,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,mBAAmB,CAAC,GAAG,mBAAmB,CAAC;IACrG,iBAAiB,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,GAAG,mBAAmB,GAAG,SAAS,CAAC;IAC1H,kBAAkB,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,GAAG,mBAAmB,EAAE,CAAC;IAC1G,mBAAmB,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CACvE;AAED,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,kCAAmC,SAAQ,+BAA+B;IACzF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,iCAAiC;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,wBAAwB,EAAE,MAAM,EAAE,CAAC;IACnC,qBAAqB,EAAE,MAAM,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,iCAAiC;IAChD,EAAE,EAAE,kBAAkB,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,mCAAmC;IAClD,MAAM,EAAE,kBAAkB,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,kCAAkC,GAAG,OAAO,oCAAoC,CAAC;AAQ7F,eAAO,MAAM,4BAA4B,GAAI,OAAO,OAAO,KAAG,mBAkC7D,CAAC;AAEF,eAAO,MAAM,0BAA0B,GACrC,OAAO,OAAO,EACd,gBAAe,kCAAyE,KACvF,mBAKF,CAAC;AA2GF,eAAO,MAAM,kCAAkC,QAAO,oBAwBrD,CAAC;AAEF,eAAO,MAAM,8BAA8B,GAAI,SAAS,+BAA+B,KAAG,oBAwDzF,CAAC;AAEF,eAAO,MAAM,2BAA2B,GACtC,SAAS,kCAAkC,KAC1C,OAAO,CAAC,iCAAiC,CAgD3C,CAAC;AAEF,eAAO,MAAM,gCAAgC,GAAI,SAAS,iCAAiC,KAAG,oBAkI7F,CAAC;AAEF,eAAO,MAAM,kCAAkC,GAAI,SAAS,mCAAmC,KAAG,oBA4HjG,CAAC"}