@prismatic-io/lux 0.0.1

package/lib/orchestrator/experiment-report.d.ts

@@ -0,0 +1,450 @@
+import { z } from "zod";
+import { type ExperimentRun } from "./experiment-runs.js";
+export declare const ExperimentDecisionReportSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<1>;
+    experimentDir: z.ZodString;
+    experimentId: z.ZodString;
+    campaignId: z.ZodString;
+    mode: z.ZodEnum<{
+        experiment: "experiment";
+        optimize: "optimize";
+    }>;
+    evaluationKind: z.ZodEnum<{
+        agent: "agent";
+        grader: "grader";
+    }>;
+    status: z.ZodEnum<{
+        aborted: "aborted";
+        failed: "failed";
+        passed: "passed";
+        running: "running";
+    }>;
+    outcome: z.ZodEnum<{
+        error: "error";
+        incomplete: "incomplete";
+        promote: "promote";
+        reject: "reject";
+        retain: "retain";
+    }>;
+    error: z.ZodNullable<z.ZodString>;
+    candidateAndAccountingVerified: z.ZodBoolean;
+    championCandidateId: z.ZodNullable<z.ZodString>;
+    ranking: z.ZodArray<z.ZodString>;
+    eligibleCandidateIds: z.ZodArray<z.ZodString>;
+    source: z.ZodObject<{
+        baselineBytes: z.ZodNullable<z.ZodNumber>;
+        championBytes: z.ZodNullable<z.ZodNumber>;
+        deltaBytes: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    gates: z.ZodArray<z.ZodObject<{
+        metric: z.ZodString;
+        profile: z.ZodNullable<z.ZodString>;
+        split: z.ZodEnum<{
+            test: "test";
+            train: "train";
+            validation: "validation";
+        }>;
+        passed: z.ZodBoolean;
+        values: z.ZodArray<z.ZodNumber>;
+        interval: z.ZodNullable<z.ZodString>;
+        note: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    evidence: z.ZodArray<z.ZodObject<{
+        candidateId: z.ZodString;
+        profileId: z.ZodString;
+        split: z.ZodEnum<{
+            test: "test";
+            train: "train";
+            validation: "validation";
+        }>;
+        repetitions: z.ZodNumber;
+        caseIds: z.ZodArray<z.ZodString>;
+        passRate: z.ZodNumber;
+        passRateSamples: z.ZodNullable<z.ZodNumber>;
+        subjectInputTokens: z.ZodNullable<z.ZodNumber>;
+        subjectOutputTokens: z.ZodNullable<z.ZodNumber>;
+        subjectTotalTokens: z.ZodNullable<z.ZodNumber>;
+        subjectTokenSamples: z.ZodNullable<z.ZodNumber>;
+        evaluationTokens: z.ZodNullable<z.ZodNumber>;
+        meanQuestions: z.ZodNullable<z.ZodNumber>;
+        questionSamples: z.ZodNullable<z.ZodNumber>;
+        meanToolCalls: z.ZodNullable<z.ZodNumber>;
+        toolSamples: z.ZodNullable<z.ZodNumber>;
+        reportedCostUsd: z.ZodNullable<z.ZodNumber>;
+        unpricedModelCalls: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    deltas: z.ZodArray<z.ZodObject<{
+        profileId: z.ZodString;
+        split: z.ZodEnum<{
+            test: "test";
+            train: "train";
+            validation: "validation";
+        }>;
+        passRate: z.ZodNullable<z.ZodNumber>;
+        subjectTotalTokens: z.ZodNullable<z.ZodNumber>;
+        meanQuestions: z.ZodNullable<z.ZodNumber>;
+        meanToolCalls: z.ZodNullable<z.ZodNumber>;
+        reportedCostUsd: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    spend: z.ZodObject<{
+        candidates: z.ZodNumber;
+        metricCalls: z.ZodNumber;
+        judgeCalls: z.ZodNumber;
+        proposalCalls: z.ZodNumber;
+        totalTokens: z.ZodNumber;
+        costUsd: z.ZodNumber;
+        unpricedModelCalls: z.ZodNullable<z.ZodNumber>;
+        activeTimeMs: z.ZodNumber;
+    }, z.core.$strict>;
+    budget: z.ZodObject<{
+        maxCandidates: z.ZodNumber;
+        maxMetricCalls: z.ZodNumber;
+        maxJudgeCalls: z.ZodNumber;
+        maxProposalCalls: z.ZodNumber;
+        maxTokens: z.ZodNumber;
+        maxCostUsd: z.ZodNumber;
+        maxUnpricedModelCalls: z.ZodNumber;
+        maxWallTimeMs: z.ZodNumber;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export type ExperimentDecisionReport = z.infer<typeof ExperimentDecisionReportSchema>;
+export declare const VerifiedExperimentDecisionSnapshotSchema: z.ZodObject<{
+    experiment: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<1>;
+        experimentId: z.ZodString;
+        campaignId: z.ZodString;
+        campaignPath: z.ZodString;
+        campaignHash: z.ZodString;
+        campaignSnapshot: z.ZodObject<{
+            id: z.ZodString;
+            subject: z.ZodObject<{
+                root: z.ZodString;
+                mutable: z.ZodArray<z.ZodString>;
+                exclude: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                validate: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    command: z.ZodString;
+                    args: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    timeoutMs: z.ZodDefault<z.ZodNumber>;
+                    unsafeHostExecution: z.ZodLiteral<true>;
+                }, z.core.$strict>>>;
+            }, z.core.$strict>;
+            evaluation: z.ZodDefault<z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"agent">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"grader">;
+                annotations: z.ZodString;
+                allowedAssertionAdditions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>>;
+                scoreTolerance: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>], "kind">>;
+            profiles: z.ZodArray<z.ZodString>;
+            splits: z.ZodObject<{
+                train: z.ZodOptional<z.ZodObject<{
+                    patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                }, z.core.$strict>>;
+                validation: z.ZodObject<{
+                    patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                }, z.core.$strict>;
+                test: z.ZodOptional<z.ZodObject<{
+                    patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                    tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                }, z.core.$strict>>;
+            }, z.core.$strict>;
+            repetitions: z.ZodDefault<z.ZodObject<{
+                search: z.ZodDefault<z.ZodNumber>;
+                promotion: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            variants: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                name: z.ZodOptional<z.ZodString>;
+                parentIds: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                components: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
+            }, z.core.$strict>>>;
+            objectives: z.ZodDefault<z.ZodObject<{
+                constraints: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    metric: z.ZodString;
+                    min: z.ZodOptional<z.ZodNumber>;
+                    max: z.ZodOptional<z.ZodNumber>;
+                    profile: z.ZodOptional<z.ZodString>;
+                    split: z.ZodDefault<z.ZodEnum<{
+                        test: "test";
+                        train: "train";
+                        validation: "validation";
+                    }>>;
+                    confidence: z.ZodOptional<z.ZodObject<{
+                        method: z.ZodEnum<{
+                            hoeffding: "hoeffding";
+                            "percentile-bootstrap": "percentile-bootstrap";
+                            wilson: "wilson";
+                        }>;
+                        level: z.ZodDefault<z.ZodNumber>;
+                        resamples: z.ZodDefault<z.ZodNumber>;
+                        seed: z.ZodDefault<z.ZodNumber>;
+                        minSamples: z.ZodDefault<z.ZodNumber>;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>>;
+                minimize: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                    metric: z.ZodString;
+                    profile: z.ZodOptional<z.ZodString>;
+                    split: z.ZodDefault<z.ZodEnum<{
+                        test: "test";
+                        train: "train";
+                        validation: "validation";
+                    }>>;
+                }, z.core.$strict>>>;
+            }, z.core.$strict>>;
+            optimizer: z.ZodOptional<z.ZodObject<{
+                algorithm: z.ZodLiteral<"reflective">;
+                config: z.ZodOptional<z.ZodUnknown>;
+                candidateSelection: z.ZodDefault<z.ZodEnum<{
+                    "current-best": "current-best";
+                    pareto: "pareto";
+                }>>;
+                mergeFrontier: z.ZodDefault<z.ZodBoolean>;
+                promotionTopK: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            budget: z.ZodDefault<z.ZodObject<{
+                maxCandidates: z.ZodDefault<z.ZodNumber>;
+                patience: z.ZodDefault<z.ZodNumber>;
+                maxMetricCalls: z.ZodDefault<z.ZodNumber>;
+                maxJudgeCalls: z.ZodDefault<z.ZodNumber>;
+                maxProposalCalls: z.ZodDefault<z.ZodNumber>;
+                maxTokens: z.ZodDefault<z.ZodNumber>;
+                maxCostUsd: z.ZodDefault<z.ZodNumber>;
+                maxUnpricedModelCalls: z.ZodDefault<z.ZodNumber>;
+                maxWallTimeMs: z.ZodDefault<z.ZodNumber>;
+            }, z.core.$strict>>;
+            meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>;
+        resolvedProfiles: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        evaluatorVersion: z.ZodLiteral<1>;
+        luxVersion: z.ZodString;
+        evaluationKind: z.ZodDefault<z.ZodEnum<{
+            agent: "agent";
+            grader: "grader";
+        }>>;
+        evaluationHash: z.ZodExactOptional<z.ZodString>;
+        mode: z.ZodEnum<{
+            experiment: "experiment";
+            optimize: "optimize";
+        }>;
+        status: z.ZodEnum<{
+            aborted: "aborted";
+            failed: "failed";
+            passed: "passed";
+            running: "running";
+        }>;
+        startedAt: z.ZodString;
+        endedAt: z.ZodNullable<z.ZodString>;
+        activeTimeMs: z.ZodDefault<z.ZodNumber>;
+        activeSegmentStartedAt: z.ZodExactOptional<z.ZodString>;
+        subjectRoot: z.ZodString;
+        baseSourceHash: z.ZodString;
+        candidateIntegrityVersion: z.ZodExactOptional<z.ZodLiteral<1>>;
+        candidates: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            path: z.ZodString;
+            candidateHash: z.ZodString;
+            sourceBytes: z.ZodNumber;
+            candidateEnvelopeHash: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>>;
+        evaluations: z.ZodArray<z.ZodObject<{
+            profileId: z.ZodString;
+            split: z.ZodEnum<{
+                test: "test";
+                train: "train";
+                validation: "validation";
+            }>;
+            suiteDir: z.ZodExactOptional<z.ZodString>;
+            passRate: z.ZodNumber;
+            casePassRates: z.ZodRecord<z.ZodString, z.ZodNumber>;
+            metrics: z.ZodRecord<z.ZodString, z.ZodNumber>;
+            metricCalls: z.ZodExactOptional<z.ZodNumber>;
+            judgeCalls: z.ZodExactOptional<z.ZodNumber>;
+            unpricedModelCalls: z.ZodExactOptional<z.ZodNumber>;
+            usage: z.ZodExactOptional<z.ZodObject<{
+                input: z.ZodNumber;
+                output: z.ZodNumber;
+                cacheRead: z.ZodNumber;
+                cacheCreation: z.ZodNumber;
+            }, z.core.$strip>>;
+            costUsd: z.ZodExactOptional<z.ZodNumber>;
+            samples: z.ZodExactOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodNumber>>>;
+            failedCases: z.ZodArray<z.ZodObject<{
+                caseId: z.ZodString;
+                score: z.ZodNullable<z.ZodNumber>;
+                feedback: z.ZodArray<z.ZodString>;
+            }, z.core.$strip>>;
+            candidateId: z.ZodString;
+            repetitions: z.ZodNumber;
+            summaryPath: z.ZodExactOptional<z.ZodString>;
+            evidencePath: z.ZodExactOptional<z.ZodString>;
+        }, z.core.$strip>>;
+        accountedEvaluationUnits: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        accountingJournalEntries: z.ZodDefault<z.ZodNumber>;
+        budgetUsage: z.ZodObject<{
+            metricCalls: z.ZodNumber;
+            judgeCalls: z.ZodNumber;
+            unpricedModelCalls: z.ZodOptional<z.ZodNumber>;
+            proposalCalls: z.ZodNumber;
+            usage: z.ZodObject<{
+                input: z.ZodNumber;
+                output: z.ZodNumber;
+                cacheRead: z.ZodNumber;
+                cacheCreation: z.ZodNumber;
+            }, z.core.$strip>;
+            costUsd: z.ZodNumber;
+        }, z.core.$strip>;
+        championCandidateId: z.ZodExactOptional<z.ZodString>;
+        selection: z.ZodExactOptional<z.ZodObject<{
+            decidedAt: z.ZodString;
+            decisionCandidateIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            promotionFinalistCandidateIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            rankedCandidateIds: z.ZodArray<z.ZodString>;
+            eligibleCandidateIds: z.ZodArray<z.ZodString>;
+            championCandidateId: z.ZodExactOptional<z.ZodString>;
+            constraints: z.ZodArray<z.ZodObject<{
+                candidateId: z.ZodString;
+                metric: z.ZodString;
+                profile: z.ZodExactOptional<z.ZodString>;
+                split: z.ZodEnum<{
+                    test: "test";
+                    train: "train";
+                    validation: "validation";
+                }>;
+                values: z.ZodArray<z.ZodNumber>;
+                passed: z.ZodBoolean;
+                interval: z.ZodExactOptional<z.ZodObject<{
+                    method: z.ZodEnum<{
+                        hoeffding: "hoeffding";
+                        "percentile-bootstrap": "percentile-bootstrap";
+                        wilson: "wilson";
+                    }>;
+                    level: z.ZodNumber;
+                    estimate: z.ZodNumber;
+                    lower: z.ZodNumber;
+                    upper: z.ZodNumber;
+                    n: z.ZodNumber;
+                    resamples: z.ZodNumber;
+                }, z.core.$strict>>;
+                note: z.ZodExactOptional<z.ZodString>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+        errorMessage: z.ZodExactOptional<z.ZodString>;
+    }, z.core.$strip>;
+    report: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<1>;
+        experimentDir: z.ZodString;
+        experimentId: z.ZodString;
+        campaignId: z.ZodString;
+        mode: z.ZodEnum<{
+            experiment: "experiment";
+            optimize: "optimize";
+        }>;
+        evaluationKind: z.ZodEnum<{
+            agent: "agent";
+            grader: "grader";
+        }>;
+        status: z.ZodEnum<{
+            aborted: "aborted";
+            failed: "failed";
+            passed: "passed";
+            running: "running";
+        }>;
+        outcome: z.ZodEnum<{
+            error: "error";
+            incomplete: "incomplete";
+            promote: "promote";
+            reject: "reject";
+            retain: "retain";
+        }>;
+        error: z.ZodNullable<z.ZodString>;
+        candidateAndAccountingVerified: z.ZodBoolean;
+        championCandidateId: z.ZodNullable<z.ZodString>;
+        ranking: z.ZodArray<z.ZodString>;
+        eligibleCandidateIds: z.ZodArray<z.ZodString>;
+        source: z.ZodObject<{
+            baselineBytes: z.ZodNullable<z.ZodNumber>;
+            championBytes: z.ZodNullable<z.ZodNumber>;
+            deltaBytes: z.ZodNullable<z.ZodNumber>;
+        }, z.core.$strict>;
+        gates: z.ZodArray<z.ZodObject<{
+            metric: z.ZodString;
+            profile: z.ZodNullable<z.ZodString>;
+            split: z.ZodEnum<{
+                test: "test";
+                train: "train";
+                validation: "validation";
+            }>;
+            passed: z.ZodBoolean;
+            values: z.ZodArray<z.ZodNumber>;
+            interval: z.ZodNullable<z.ZodString>;
+            note: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>>;
+        evidence: z.ZodArray<z.ZodObject<{
+            candidateId: z.ZodString;
+            profileId: z.ZodString;
+            split: z.ZodEnum<{
+                test: "test";
+                train: "train";
+                validation: "validation";
+            }>;
+            repetitions: z.ZodNumber;
+            caseIds: z.ZodArray<z.ZodString>;
+            passRate: z.ZodNumber;
+            passRateSamples: z.ZodNullable<z.ZodNumber>;
+            subjectInputTokens: z.ZodNullable<z.ZodNumber>;
+            subjectOutputTokens: z.ZodNullable<z.ZodNumber>;
+            subjectTotalTokens: z.ZodNullable<z.ZodNumber>;
+            subjectTokenSamples: z.ZodNullable<z.ZodNumber>;
+            evaluationTokens: z.ZodNullable<z.ZodNumber>;
+            meanQuestions: z.ZodNullable<z.ZodNumber>;
+            questionSamples: z.ZodNullable<z.ZodNumber>;
+            meanToolCalls: z.ZodNullable<z.ZodNumber>;
+            toolSamples: z.ZodNullable<z.ZodNumber>;
+            reportedCostUsd: z.ZodNullable<z.ZodNumber>;
+            unpricedModelCalls: z.ZodNullable<z.ZodNumber>;
+        }, z.core.$strict>>;
+        deltas: z.ZodArray<z.ZodObject<{
+            profileId: z.ZodString;
+            split: z.ZodEnum<{
+                test: "test";
+                train: "train";
+                validation: "validation";
+            }>;
+            passRate: z.ZodNullable<z.ZodNumber>;
+            subjectTotalTokens: z.ZodNullable<z.ZodNumber>;
+            meanQuestions: z.ZodNullable<z.ZodNumber>;
+            meanToolCalls: z.ZodNullable<z.ZodNumber>;
+            reportedCostUsd: z.ZodNullable<z.ZodNumber>;
+        }, z.core.$strict>>;
+        spend: z.ZodObject<{
+            candidates: z.ZodNumber;
+            metricCalls: z.ZodNumber;
+            judgeCalls: z.ZodNumber;
+            proposalCalls: z.ZodNumber;
+            totalTokens: z.ZodNumber;
+            costUsd: z.ZodNumber;
+            unpricedModelCalls: z.ZodNullable<z.ZodNumber>;
+            activeTimeMs: z.ZodNumber;
+        }, z.core.$strict>;
+        budget: z.ZodObject<{
+            maxCandidates: z.ZodNumber;
+            maxMetricCalls: z.ZodNumber;
+            maxJudgeCalls: z.ZodNumber;
+            maxProposalCalls: z.ZodNumber;
+            maxTokens: z.ZodNumber;
+            maxCostUsd: z.ZodNumber;
+            maxUnpricedModelCalls: z.ZodNumber;
+            maxWallTimeMs: z.ZodNumber;
+        }, z.core.$strict>;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export type VerifiedExperimentDecisionSnapshot = z.infer<typeof VerifiedExperimentDecisionSnapshotSchema>;
+export declare const verifyExperimentDecisionIntegrity: (experiment: ExperimentRun) => void;
+export declare const createExperimentDecisionReport: (experimentDir: string, experiment: ExperimentRun) => ExperimentDecisionReport;
+export declare const loadVerifiedExperimentDecisionSnapshot: (experimentDir: string) => Promise<VerifiedExperimentDecisionSnapshot>;
+export declare const loadExperimentDecisionReport: (experimentDir: string) => Promise<ExperimentDecisionReport>;
+export declare const formatExperimentDecisionMarkdown: (report: ExperimentDecisionReport) => string;
+//# sourceMappingURL=experiment-report.d.ts.map

package/lib/orchestrator/experiment-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-report.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-report.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EACL,KAAK,aAAa,EAKnB,MAAM,sBAAsB,CAAC;AAK9B,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiGhC,CAAC;AACZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AAEtF,eAAO,MAAM,wCAAwC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAK1C,CAAC;AACZ,MAAM,MAAM,kCAAkC,GAAG,CAAC,CAAC,KAAK,CACtD,OAAO,wCAAwC,CAChD,CAAC;AA2SF,eAAO,MAAM,iCAAiC,eAAgB,aAAa,KAAG,IAe7E,CAAC;AA+MF,eAAO,MAAM,8BAA8B,kBAC1B,MAAM,cACT,aAAa,KACxB,wBAA6F,CAAC;AAEjG,eAAO,MAAM,sCAAsC,kBAClC,MAAM,KACpB,OAAO,CAAC,kCAAkC,CAU5C,CAAC;AAEF,eAAO,MAAM,4BAA4B,kBACxB,MAAM,KACpB,OAAO,CAAC,wBAAwB,CACmC,CAAC;AA0CvE,eAAO,MAAM,gCAAgC,WAAY,wBAAwB,KAAG,MA+EnF,CAAC"}