@tuned-tensor/local 0.1.0

package/dist/contracts.js

@@ -0,0 +1,253 @@
+import { z } from "zod";
+import { canonicalizeTrainingModel } from "./model-registry.js";
+export const documentInputAssetSchema = z.object({
+    type: z.literal("image").default("image"),
+    mime_type: z.string().min(1).optional(),
+    image: z.string().min(1).optional(),
+    data_uri: z.string().min(1).optional(),
+    uri: z.string().min(1).optional(),
+    path: z.string().min(1).optional(),
+    page: z.number().int().min(1).optional(),
+}).refine((asset) => Boolean(asset.image || asset.data_uri || asset.uri || asset.path), { message: "image asset must include image, data_uri, uri, or path" });
+export const behaviorSpecExampleSchema = z.object({
+    input: z.string().min(1),
+    output: z.string().min(1),
+    input_assets: z.array(documentInputAssetSchema).optional(),
+    modality: z.literal("document_ocr").optional(),
+});
+export const specSnapshotSchema = z.object({
+    name: z.string().min(1),
+    description: z.string().default(""),
+    system_prompt: z.string().default(""),
+    guidelines: z.array(z.string()).default([]),
+    examples: z.array(behaviorSpecExampleSchema).default([]),
+    constraints: z.array(z.string()).default([]),
+    base_model: z.string().transform((value) => canonicalizeTrainingModel(value)),
+});
+export const datasetPrebuiltSchema = z.object({
+    training: z.string().min(1),
+    validation: z.string().min(1).optional(),
+    test: z.string().min(1).optional(),
+    format: z.enum(["chat_jsonl", "document_ocr_chat_jsonl"]).optional(),
+});
+export const fineTuneHyperparametersSchema = z.object({
+    n_epochs: z.number().int().min(1).max(20).default(3),
+    learning_rate: z.number().positive().optional(),
+    batch_size: z.number().int().min(1).optional(),
+    lora_rank: z.number().int().min(1).optional(),
+    lora_alpha: z.number().int().min(1).optional(),
+    lora_dropout: z.number().min(0).max(1).optional(),
+    max_seq_length: z.number().int().min(128).max(32768).optional(),
+    gradient_accumulation_steps: z.number().int().min(1).optional(),
+    save_adapter_only: z.boolean().default(false),
+    augment: z.boolean().default(false),
+    use_llm_judge: z.boolean().default(false),
+    max_eval_examples: z.number().int().min(1).optional(),
+});
+export const localArtifactsSchema = z.object({
+    prefix: z.string().min(1).optional(),
+});
+export const fineTuneRunRequestSchema = z.object({
+    run_id: z.string().uuid(),
+    user_id: z.string().min(1),
+    behavior_spec_id: z.string().uuid(),
+    run_number: z.number().int().min(1),
+    spec_snapshot: specSnapshotSchema,
+    hyperparameters: fineTuneHyperparametersSchema.default({
+        n_epochs: 3,
+        save_adapter_only: false,
+        augment: false,
+        use_llm_judge: false,
+    }),
+    artifacts: localArtifactsSchema.optional(),
+    dataset_prebuilt: datasetPrebuiltSchema.optional(),
+}).superRefine((request, ctx) => {
+    if (request.spec_snapshot.examples.length === 0 && !request.dataset_prebuilt) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ["spec_snapshot", "examples"],
+            message: "spec_snapshot.examples must have at least 1 example unless dataset_prebuilt is provided",
+        });
+    }
+});
+export const localBehaviorSpecFileSchema = specSnapshotSchema.extend({
+    id: z.string().uuid().optional(),
+    user_id: z.string().min(1).default("local-user"),
+    run_number: z.number().int().min(1).default(1),
+    hyperparameters: fineTuneHyperparametersSchema.optional(),
+    artifacts: localArtifactsSchema.optional(),
+    dataset_prebuilt: datasetPrebuiltSchema.optional(),
+});
+export const evalExampleResultSchema = z.object({
+    prompt: z.string(),
+    expected: z.string(),
+    actual: z.string(),
+    passed: z.boolean(),
+    score: z.number().min(0).max(1),
+    reasoning: z.string().nullable(),
+    latency_ms: z.number().int().nonnegative(),
+});
+export const evalReportSchema = z.object({
+    kind: z.enum(["baseline", "candidate"]),
+    model_id: z.string(),
+    total: z.number().int().nonnegative(),
+    eval_examples_total: z.number().int().nonnegative(),
+    eval_examples_used: z.number().int().nonnegative(),
+    eval_truncated: z.boolean(),
+    avg_score: z.number().min(0).max(1),
+    pass_rate: z.number().min(0).max(1),
+    exact_match_rate: z.number().min(0).max(1),
+    avg_latency_ms: z.number().int().nonnegative(),
+    results: z.array(evalExampleResultSchema),
+    artifact_uri: z.string(),
+    scoring_method: z.enum(["heuristic", "command", "llm_judge", "exact_match"]),
+    judge_model_id: z.string().nullable().optional(),
+    inference_provider: z.enum(["none", "command", "transformers"]).optional(),
+    scoring_mode: z.enum(["exact_match", "llm_judge"]).optional(),
+    generation_config: z.record(z.string(), z.unknown()).optional(),
+});
+export const comparisonReportSchema = z.object({
+    avg_score_delta: z.number(),
+    pass_rate_delta: z.number(),
+    exact_match_rate_delta: z.number(),
+    regressions: z.number().int().nonnegative(),
+    improvements: z.number().int().nonnegative(),
+    regressed_examples: z.array(z.object({
+        prompt: z.string(),
+        old_score: z.number(),
+        new_score: z.number(),
+    })),
+});
+export const trainingReportSchema = z.object({
+    provider: z.literal("local-uv"),
+    training_job_name: z.string(),
+    model_artifact_uri: z.string().optional(),
+    base_model_artifact_uri: z.string().optional(),
+    metrics: z.record(z.string(), z.unknown()).nullable(),
+    exit_code: z.number().int().nullable(),
+    log_uri: z.string(),
+    command: z.array(z.string()).optional(),
+});
+export const runReportSchema = z.object({
+    run_id: z.string().uuid(),
+    behavior_spec_id: z.string().uuid(),
+    user_id: z.string(),
+    run_number: z.number().int().min(1),
+    base_model: z.string(),
+    fine_tuned_model_id: z.string(),
+    status: z.enum(["completed", "failed"]),
+    baseline: evalReportSchema,
+    candidate: evalReportSchema,
+    comparison: comparisonReportSchema,
+    training: trainingReportSchema,
+    artifact_uris: z.object({
+        dataset: z.string(),
+        baseline_eval: z.string(),
+        candidate_eval: z.string(),
+        report: z.string(),
+    }),
+    run_metadata: z.object({
+        base_model: z.string(),
+        fine_tuned_model_id: z.string(),
+        dataset_prebuilt: z.boolean(),
+        dataset_uri: z.string(),
+        spec_example_count: z.number().int().nonnegative(),
+        training_example_count: z.number().int().nonnegative().nullable(),
+        eval_examples_total: z.number().int().nonnegative(),
+        eval_examples_used: z.number().int().nonnegative(),
+        started_at: z.string(),
+        completed_at: z.string(),
+        elapsed_ms: z.number().int().nonnegative(),
+        elapsed_seconds: z.number().nonnegative(),
+    }),
+    created_at: z.string(),
+});
+export const commandSchema = z.array(z.string().min(1)).min(1);
+export const localRunnerConfigSchema = z.object({
+    storeRoot: z.string().optional(),
+    artifactRoot: z.string().default(".tt-local/artifacts"),
+    dryRun: z.boolean().default(false),
+    training: z.object({
+        backend: z.literal("uv").default("uv"),
+        project: z.string().optional(),
+        cwd: z.string().optional(),
+        module: z.string().optional(),
+        script: z.string().optional(),
+        args: z.array(z.string()).default([]),
+        with: z.array(z.string()).default([]),
+        env: z.record(z.string(), z.string()).default({}),
+    }).default({
+        backend: "uv",
+        args: [],
+        with: [],
+        env: {},
+    }),
+    paths: z.object({
+        baseModel: z.string().optional(),
+        modelCache: z.string().optional(),
+    }).default({}),
+    evaluation: z.object({
+        mode: z.enum(["heuristic", "command", "llm_judge"]).default("heuristic"),
+        baselineCommand: commandSchema.optional(),
+        candidateCommand: commandSchema.optional(),
+        inference: z.object({
+            provider: z.enum(["transformers", "none"]).default("transformers"),
+            project: z.string().optional(),
+            cwd: z.string().optional(),
+            module: z.string().optional(),
+            script: z.string().default("training/sft-local/src/evaluate.py"),
+            with: z.array(z.string()).default([]),
+            env: z.record(z.string(), z.string()).default({}),
+            maxNewTokens: z.number().int().min(1).default(256),
+            temperature: z.number().min(0).default(0),
+            topP: z.number().min(0).max(1).default(1),
+            trustRemoteCode: z.boolean().default(true),
+            device: z.enum(["auto", "cpu", "cuda", "mps"]).default("auto"),
+        }).default({
+            provider: "transformers",
+            script: "training/sft-local/src/evaluate.py",
+            with: [],
+            env: {},
+            maxNewTokens: 256,
+            temperature: 0,
+            topP: 1,
+            trustRemoteCode: true,
+            device: "auto",
+        }),
+        scoring: z.object({
+            mode: z.enum(["exact_match", "llm_judge"]).default("llm_judge"),
+            fallback: z.enum(["exact_match", "fail"]).default("exact_match"),
+        }).default({
+            mode: "llm_judge",
+            fallback: "exact_match",
+        }),
+        timeoutMs: z.number().int().min(100).default(120_000),
+        maxExamples: z.number().int().min(1).optional(),
+    }).default({
+        mode: "heuristic",
+        inference: {
+            provider: "transformers",
+            script: "training/sft-local/src/evaluate.py",
+            with: [],
+            env: {},
+            maxNewTokens: 256,
+            temperature: 0,
+            topP: 1,
+            trustRemoteCode: true,
+            device: "auto",
+        },
+        scoring: {
+            mode: "llm_judge",
+            fallback: "exact_match",
+        },
+        timeoutMs: 120_000,
+    }),
+    llm: z.object({
+        provider: z.literal("openrouter").default("openrouter"),
+        model: z.string().min(1).default("openai/gpt-5.5"),
+        apiKeyEnv: z.string().min(1).default("OPENROUTER_API_KEY"),
+        appName: z.string().optional(),
+        siteUrl: z.string().url().optional(),
+    }).optional(),
+});
+//# sourceMappingURL=contracts.js.map

package/dist/contracts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contracts.js","sourceRoot":"","sources":["../src/contracts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAEhE,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IACzC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACvC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACjC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACzC,CAAC,CAAC,MAAM,CACP,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,EAC5E,EAAE,OAAO,EAAE,wDAAwD,EAAE,CACtE,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,QAAQ,EAAE;IAC1D,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;CAC/C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACnC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC3C,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACxD,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC5C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC;CAC9E,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACxC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,yBAAyB,CAAC,CAAC,CAAC,QAAQ,EAAE;CACrE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IACpD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACpD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC/C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACjD,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;IAC/D,2BAA2B,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC/D,iBAAiB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IAC7C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACzC,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACtD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACrC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;IACzB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;IACnC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,aAAa,EAAE,kBAAkB;IACjC,eAAe,EAAE,6BAA6B,CAAC,OAAO,CAAC;QACrD,QAAQ,EAAE,CAAC;QACX,iBAAiB,EAAE,KAAK;QACxB,OAAO,EAAE,KAAK;QACd,aAAa,EAAE,KAAK;KACrB,CAAC;IACF,SAAS,EAAE,oBAAoB,CAAC,QAAQ,EAAE;IAC1C,gBAAgB,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;IAC9B,IAAI,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7E,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,IAAI,EAAE,CAAC,eAAe,EAAE,UAAU,CAAC;YACnC,OAAO,EAAE,yFAAyF;SACnG,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,2BAA2B,GAAG,kBAAkB,CAAC,MAAM,CAAC;IACnE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;IAChC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;IAChD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,eAAe,EAAE,6BAA6B,CAAC,QAAQ,EAAE;IACzD,SAAS,EAAE,oBAAoB,CAAC,QAAQ,EAAE;IAC1C,gBAAgB,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CAC3C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IACvC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACrC,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACnD,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAClD,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAC9C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;IACzC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IAC5E,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,kBAAkB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC1E,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7D,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CAChE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;IAC3B,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE;IAClC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAC3C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAC5C,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;QACnC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC,CAAC;CACJ,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAC/B,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC7B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzC,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9C,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;IACzB,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;IACnC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE;IAC/B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IACvC,QAAQ,EAAE,gBAAgB;IAC1B,SAAS,EAAE,gBAAgB;IAC3B,UAAU,EAAE,sBAAsB;IAClC,QAAQ,EAAE,oBAAoB;IAC9B,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC;QACtB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;QAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;KACnB,CAAC;IACF,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC;QACrB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE;QAC/B,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE;QAC7B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAClD,sBAAsB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;QACjE,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QACnD,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAClD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;QACtB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;QACxB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;QAC1C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE;KAC1C,CAAC;IACF,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;CACvB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC;IACvD,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IAClC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;QACjB,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QACtC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC9B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC7B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC,OAAO,CAAC;QACT,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,EAAE;QACR,IAAI,EAAE,EAAE;QACR,GAAG,EAAE,EAAE;KACR,CAAC;IACF,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;QACd,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAChC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAClC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACd,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;QACxE,eAAe,EAAE,aAAa,CAAC,QAAQ,EAAE;QACzC,gBAAgB,EAAE,aAAa,CAAC,QAAQ,EAAE;QAC1C,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC;YAClB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;YAClE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC9B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC7B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC;YAChE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACjD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;YAClD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACzC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACzC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;YAC1C,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;SAC/D,CAAC,CAAC,OAAO,CAAC;YACT,QAAQ,EAAE,cAAc;YACxB,MAAM,EAAE,oCAAoC;YAC5C,IAAI,EAAE,EAAE;YACR,GAAG,EAAE,EAAE;YACP,YAAY,EAAE,GAAG;YACjB,WAAW,EAAE,CAAC;YACd,IAAI,EAAE,CAAC;YACP,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,MAAM;SACf,CAAC;QACF,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;YAChB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;YAC/D,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC;SACjE,CAAC,CAAC,OAAO,CAAC;YACT,IAAI,EAAE,WAAW;YACjB,QAAQ,EAAE,aAAa;SACxB,CAAC;QACF,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACrD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC,CAAC,OAAO,CAAC;QACT,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE;YACT,QAAQ,EAAE,cAAc;YACxB,MAAM,EAAE,oCAAoC;YAC5C,IAAI,EAAE,EAAE;YACR,GAAG,EAAE,EAAE;YACP,YAAY,EAAE,GAAG;YACjB,WAAW,EAAE,CAAC;YACd,IAAI,EAAE,CAAC;YACP,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,MAAM;SACf;QACD,OAAO,EAAE;YACP,IAAI,EAAE,WAAW;YACjB,QAAQ,EAAE,aAAa;SACxB;QACD,SAAS,EAAE,OAAO;KACnB,CAAC;IACF,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC;QACZ,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;QACvD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;QAClD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC;QAC1D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC9B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;KACrC,CAAC,CAAC,QAAQ,EAAE;CACd,CAAC,CAAC"}

package/dist/dataset.d.ts

@@ -0,0 +1,12 @@
+import type { BehaviorSpecExample, SpecSnapshot } from "./contracts.js";
+export interface ChatJsonlRow {
+    messages: Array<{
+        role: "system" | "user" | "assistant";
+        content: string | Array<Record<string, unknown>>;
+    }>;
+}
+export declare function buildSystemMessage(spec: SpecSnapshot): string;
+export declare function exampleToChatRow(spec: SpecSnapshot, example: BehaviorSpecExample): ChatJsonlRow;
+export declare function compileSpecToJsonl(spec: SpecSnapshot): string;
+export declare function examplesFromSpec(spec: SpecSnapshot): BehaviorSpecExample[];
+export declare function examplesFromChatJsonl(path: string): Promise<BehaviorSpecExample[]>;

package/dist/dataset.js

@@ -0,0 +1,60 @@
+import { readFile } from "node:fs/promises";
+export function buildSystemMessage(spec) {
+    const parts = [];
+    if (spec.system_prompt.trim())
+        parts.push(spec.system_prompt.trim());
+    if (spec.guidelines.length > 0) {
+        parts.push(`Guidelines:\n${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
+    }
+    if (spec.constraints.length > 0) {
+        parts.push(`Constraints:\n${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
+    }
+    return parts.join("\n\n");
+}
+export function exampleToChatRow(spec, example) {
+    const system = buildSystemMessage(spec);
+    const userContent = example.input_assets?.length
+        ? [
+            ...example.input_assets.map((asset) => ({
+                type: "image",
+                image: asset.image ?? asset.data_uri ?? asset.uri ?? asset.path,
+                ...(asset.mime_type ? { mime_type: asset.mime_type } : {}),
+                ...(asset.page ? { page: asset.page } : {}),
+            })),
+            { type: "text", text: example.input },
+        ]
+        : example.input;
+    return {
+        messages: [
+            { role: "system", content: system },
+            { role: "user", content: userContent },
+            { role: "assistant", content: example.output },
+        ],
+    };
+}
+export function compileSpecToJsonl(spec) {
+    return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
+}
+export function examplesFromSpec(spec) {
+    return spec.examples;
+}
+export async function examplesFromChatJsonl(path) {
+    const text = await readFile(path, "utf8");
+    const examples = [];
+    for (const [index, line] of text.split(/\r?\n/).entries()) {
+        if (!line.trim())
+            continue;
+        const row = JSON.parse(line);
+        const user = row.messages.find((message) => message.role === "user");
+        const assistant = [...row.messages].reverse().find((message) => message.role === "assistant");
+        if (!user || !assistant || typeof assistant.content !== "string") {
+            throw new Error(`Invalid chat JSONL row ${index + 1}: expected user and assistant messages`);
+        }
+        examples.push({
+            input: typeof user.content === "string" ? user.content : JSON.stringify(user.content),
+            output: assistant.content,
+        });
+    }
+    return examples;
+}
+//# sourceMappingURL=dataset.js.map

package/dist/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../src/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAU5C,MAAM,UAAU,kBAAkB,CAAC,IAAkB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;IACrE,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAkB,EAAE,OAA4B;IAC/E,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,MAAM;QAC9C,CAAC,CAAC;YACE,GAAG,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtC,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI;gBAC/D,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C,CAAC,CAAC;YACH,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE;SACtC;QACH,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAElB,OAAO;QACL,QAAQ,EAAE;YACR,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;YACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;YACtC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE;SAC/C;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAkB;IACnD,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACpG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAkB;IACjD,OAAO,IAAI,CAAC,QAAQ,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY;IACtD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAA0B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiB,CAAC;QAC7C,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QACrE,MAAM,SAAS,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;QAC9F,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,GAAG,CAAC,wCAAwC,CAAC,CAAC;QAC/F,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;YACrF,MAAM,EAAE,SAAS,CAAC,OAAO;SAC1B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/docker-training.d.ts

@@ -0,0 +1,8 @@
+import type { FineTuneRunRequest, LocalRunnerConfig, TrainingReport } from "./contracts.js";
+import type { RunArtifacts } from "./artifacts.js";
+export declare function buildTrainingHyperparameters(request: FineTuneRunRequest): Record<string, string>;
+export declare function launchDockerTraining(args: {
+    request: FineTuneRunRequest;
+    artifacts: RunArtifacts;
+    config: LocalRunnerConfig;
+}): Promise<TrainingReport>;

package/dist/docker-training.js

@@ -0,0 +1,123 @@
+import { spawn } from "node:child_process";
+import { createWriteStream } from "node:fs";
+import { access, copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { copyDatasetToTrainingChannel, fileUri, writeJson } from "./artifacts.js";
+import { resolveTrainingModel } from "./model-registry.js";
+export function buildTrainingHyperparameters(request) {
+    const model = resolveTrainingModel(request.spec_snapshot.base_model);
+    const hyper = request.hyperparameters;
+    return {
+        run_id: request.run_id,
+        base_model: model.id,
+        model_family: model.family,
+        model_loader: model.loader,
+        n_epochs: String(hyper.n_epochs),
+        learning_rate: String(hyper.learning_rate ?? model.defaultLearningRate),
+        per_device_train_batch_size: String(hyper.batch_size ?? model.defaultPerDeviceBatchSize),
+        gradient_accumulation_steps: String(hyper.gradient_accumulation_steps ?? model.defaultGradientAccumulationSteps),
+        lora_rank: String(hyper.lora_rank ?? model.defaultLoraRank),
+        lora_alpha: String(hyper.lora_alpha ?? model.defaultLoraAlpha),
+        lora_dropout: String(hyper.lora_dropout ?? model.defaultLoraDropout),
+        max_seq_length: String(hyper.max_seq_length ?? model.defaultMaxSeqLength),
+        save_adapter_only: String(hyper.save_adapter_only),
+        requires_hf_token: String(model.requiresHfToken),
+        trust_remote_code: String(model.trustRemoteCode),
+    };
+}
+async function pathExists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function launchDockerTraining(args) {
+    const { request, artifacts, config } = args;
+    const jobName = `tt-local-${request.run_id}`;
+    await mkdir(artifacts.trainingDir, { recursive: true });
+    await copyDatasetToTrainingChannel(artifacts);
+    const hyperparameters = buildTrainingHyperparameters(request);
+    await writeJson(join(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
+    if (config.dryRun) {
+        await writeFile(artifacts.trainingLog, "Dry run enabled. Docker training was not launched.\n", "utf8");
+        return {
+            provider: "local-docker",
+            training_job_name: jobName,
+            model_artifact_uri: fileUri(artifacts.trainingModelDir),
+            base_model_artifact_uri: config.paths.baseModel ? fileUri(config.paths.baseModel) : undefined,
+            metrics: { dry_run: true },
+            exit_code: 0,
+            log_uri: fileUri(artifacts.trainingLog),
+        };
+    }
+    if (!config.docker.image) {
+        throw new Error("Local Docker training requires docker.image in the runner config, or dryRun=true.");
+    }
+    const dockerArgs = [
+        "run",
+        "--rm",
+        "--name",
+        jobName,
+        "--gpus",
+        config.docker.gpus,
+        "-v",
+        `${resolve(artifacts.trainingInputDir)}:/opt/ml/input/data/training:ro`,
+        "-v",
+        `${resolve(artifacts.trainingConfigDir)}:/opt/ml/input/config:ro`,
+        "-v",
+        `${resolve(artifacts.trainingOutputDir)}:/opt/ml/output`,
+        "-v",
+        `${resolve(artifacts.trainingModelDir)}:/opt/ml/model`,
+        "-e",
+        "BACKEND=local",
+        ...Object.entries(config.docker.env).flatMap(([key, value]) => ["-e", `${key}=${value}`]),
+    ];
+    if (config.paths.baseModel) {
+        dockerArgs.push("-v", `${resolve(config.paths.baseModel)}:/opt/ml/input/data/base_model:ro`);
+    }
+    if (config.paths.modelCache) {
+        await mkdir(config.paths.modelCache, { recursive: true });
+        dockerArgs.push("-v", `${resolve(config.paths.modelCache)}:/root/.cache/huggingface`);
+    }
+    if (config.docker.network) {
+        dockerArgs.push("--network", config.docker.network);
+    }
+    dockerArgs.push(...config.docker.extraArgs, config.docker.image);
+    const logStream = createWriteStream(artifacts.trainingLog, { flags: "w" });
+    const exitCode = await new Promise((resolvePromise, reject) => {
+        const child = spawn("docker", dockerArgs, { stdio: ["ignore", "pipe", "pipe"] });
+        child.stdout.pipe(logStream, { end: false });
+        child.stderr.pipe(logStream, { end: false });
+        child.on("error", reject);
+        child.on("close", (code) => resolvePromise(code ?? 1));
+    }).finally(() => {
+        logStream.end();
+    });
+    const metricsPath = join(artifacts.trainingModelDir, "training-metrics.json");
+    const metrics = await pathExists(metricsPath)
+        ? JSON.parse(await readFile(metricsPath, "utf8"))
+        : null;
+    if (exitCode !== 0) {
+        throw new Error(`Docker training exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
+    }
+    const modelTarPath = join(artifacts.trainingOutputDir, "model.tar.gz");
+    const modelUri = await pathExists(modelTarPath)
+        ? fileUri(modelTarPath)
+        : fileUri(artifacts.trainingModelDir);
+    if (await pathExists(modelTarPath)) {
+        await copyFile(modelTarPath, join(artifacts.runDir, "model.tar.gz")).catch(() => undefined);
+    }
+    return {
+        provider: "local-docker",
+        training_job_name: jobName,
+        model_artifact_uri: modelUri,
+        base_model_artifact_uri: config.paths.baseModel ? fileUri(config.paths.baseModel) : undefined,
+        metrics,
+        exit_code: exitCode,
+        log_uri: fileUri(artifacts.trainingLog),
+    };
+}
+//# sourceMappingURL=docker-training.js.map

package/dist/docker-training.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docker-training.js","sourceRoot":"","sources":["../src/docker-training.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,4BAA4B,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D,MAAM,UAAU,4BAA4B,CAAC,OAA2B;IACtE,MAAM,KAAK,GAAG,oBAAoB,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,OAAO,CAAC,eAAe,CAAC;IACtC,OAAO;QACL,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,KAAK,CAAC,EAAE;QACpB,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;QAChC,aAAa,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,mBAAmB,CAAC;QACvE,2BAA2B,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,yBAAyB,CAAC;QACxF,2BAA2B,EAAE,MAAM,CACjC,KAAK,CAAC,2BAA2B,IAAI,KAAK,CAAC,gCAAgC,CAC5E;QACD,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,eAAe,CAAC;QAC3D,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,gBAAgB,CAAC;QAC9D,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,kBAAkB,CAAC;QACpE,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,cAAc,IAAI,KAAK,CAAC,mBAAmB,CAAC;QACzE,iBAAiB,EAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAClD,iBAAiB,EAAE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC;QAChD,iBAAiB,EAAE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAI1C;IACC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC5C,MAAM,OAAO,GAAG,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC;IAC7C,MAAM,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,4BAA4B,CAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,eAAe,GAAG,4BAA4B,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,sBAAsB,CAAC,EAAE,eAAe,CAAC,CAAC;IAE5F,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,SAAS,CACb,SAAS,CAAC,WAAW,EACrB,sDAAsD,EACtD,MAAM,CACP,CAAC;QACF,OAAO;YACL,QAAQ,EAAE,cAAc;YACxB,iBAAiB,EAAE,OAAO;YAC1B,kBAAkB,EAAE,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC;YACvD,uBAAuB,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;YAC7F,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;YAC1B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAC;IACvG,CAAC;IAED,MAAM,UAAU,GAAG;QACjB,KAAK;QACL,MAAM;QACN,QAAQ;QACR,OAAO;QACP,QAAQ;QACR,MAAM,CAAC,MAAM,CAAC,IAAI;QAClB,IAAI;QACJ,GAAG,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC,iCAAiC;QACvE,IAAI;QACJ,GAAG,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,0BAA0B;QACjE,IAAI;QACJ,GAAG,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,iBAAiB;QACxD,IAAI;QACJ,GAAG,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC,gBAAgB;QACtD,IAAI;QACJ,eAAe;QACf,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;KAC1F,CAAC;IAEF,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAC3B,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;QAC5B,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC1B,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEjE,MAAM,SAAS,GAAG,iBAAiB,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,CAAS,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE;QACpE,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACjF,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;QACd,SAAS,CAAC,GAAG,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,uBAAuB,CAAC,CAAC;IAC9E,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC;QAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAA4B;QAC5E,CAAC,CAAC,IAAI,CAAC;IAET,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,SAAS,SAAS,CAAC,WAAW,GAAG,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC;QAC7C,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;QACvB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;IACxC,IAAI,MAAM,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACnC,MAAM,QAAQ,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC9F,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,cAAc;QACxB,iBAAiB,EAAE,OAAO;QAC1B,kBAAkB,EAAE,QAAQ;QAC5B,uBAAuB,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QAC7F,OAAO;QACP,SAAS,EAAE,QAAQ;QACnB,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC;KACxC,CAAC;AACJ,CAAC"}

package/dist/doctor.d.ts

@@ -0,0 +1,7 @@
+import type { LocalRunnerConfig } from "./contracts.js";
+export interface DoctorCheck {
+    name: string;
+    ok: boolean;
+    message: string;
+}
+export declare function runDoctor(config: LocalRunnerConfig): Promise<DoctorCheck[]>;

package/dist/doctor.js

@@ -0,0 +1,74 @@
+import { spawn } from "node:child_process";
+async function runCommand(command, args, timeoutMs = 10_000) {
+    return await new Promise((resolve) => {
+        const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
+        let stdout = "";
+        let stderr = "";
+        const timer = setTimeout(() => {
+            child.kill("SIGTERM");
+            resolve({ code: null, stdout, stderr, error: `timed out after ${timeoutMs}ms` });
+        }, timeoutMs);
+        child.stdout.setEncoding("utf8");
+        child.stderr.setEncoding("utf8");
+        child.stdout.on("data", (chunk) => { stdout += chunk; });
+        child.stderr.on("data", (chunk) => { stderr += chunk; });
+        child.on("error", (error) => {
+            clearTimeout(timer);
+            resolve({ code: null, stdout, stderr, error: error.message });
+        });
+        child.on("close", (code) => {
+            clearTimeout(timer);
+            resolve({ code, stdout, stderr });
+        });
+    });
+}
+function firstLine(value) {
+    return value.trim().split(/\r?\n/)[0] ?? "";
+}
+export async function runDoctor(config) {
+    const checks = [];
+    const nodeVersion = process.versions.node;
+    const nodeMajor = Number(nodeVersion.split(".")[0]);
+    checks.push({
+        name: "node",
+        ok: nodeMajor >= 22,
+        message: `Node ${nodeVersion}`,
+    });
+    const uvVersion = await runCommand("uv", ["--version"]);
+    checks.push({
+        name: "uv",
+        ok: uvVersion.code === 0,
+        message: uvVersion.code === 0
+            ? firstLine(uvVersion.stdout)
+            : uvVersion.error ?? (firstLine(uvVersion.stderr) || "uv is not available"),
+    });
+    const pythonVersion = await runCommand("uv", ["run", "python", "--version"]);
+    checks.push({
+        name: "uv-python",
+        ok: pythonVersion.code === 0,
+        message: pythonVersion.code === 0
+            ? firstLine(pythonVersion.stdout || pythonVersion.stderr)
+            : pythonVersion.error ?? (firstLine(pythonVersion.stderr) || "uv could not run Python"),
+    });
+    const nvidiaSmi = await runCommand("nvidia-smi", []);
+    checks.push({
+        name: "nvidia-smi",
+        ok: nvidiaSmi.code === 0,
+        message: nvidiaSmi.code === 0
+            ? firstLine(nvidiaSmi.stdout)
+            : nvidiaSmi.error ?? (firstLine(nvidiaSmi.stderr) || "nvidia-smi not available"),
+    });
+    checks.push({
+        name: "training-entrypoint",
+        ok: Boolean(config.training.module || config.training.script || config.dryRun),
+        message: config.training.module
+            ? `uv will run module ${config.training.module}`
+            : config.training.script
+                ? `uv will run script ${config.training.script}`
+                : config.dryRun
+                    ? "No training entrypoint configured; dryRun is enabled"
+                    : "No training module or script configured",
+    });
+    return checks;
+}
+//# sourceMappingURL=doctor.js.map

package/dist/doctor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../src/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAS3C,KAAK,UAAU,UAAU,CAAC,OAAe,EAAE,IAAc,EAAE,SAAS,GAAG,MAAM;IAM3E,OAAO,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC1E,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtB,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,SAAS,IAAI,EAAE,CAAC,CAAC;QACnF,CAAC,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC1B,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAyB;IACvD,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,CAAC,IAAI,CAAC;QACV,IAAI,EAAE,MAAM;QACZ,EAAE,EAAE,SAAS,IAAI,EAAE;QACnB,OAAO,EAAE,QAAQ,WAAW,EAAE;KAC/B,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;IACxD,MAAM,CAAC,IAAI,CAAC;QACV,IAAI,EAAE,IAAI;QACV,EAAE,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC;QACxB,OAAO,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC;YAC3B,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,qBAAqB,CAAC;KAC9E,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;IAC7E,MAAM,CAAC,IAAI,CAAC;QACV,IAAI,EAAE,WAAW;QACjB,EAAE,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC;QAC5B,OAAO,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC;YAC/B,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,IAAI,aAAa,CAAC,MAAM,CAAC;YACzD,CAAC,CAAC,aAAa,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,yBAAyB,CAAC;KAC1F,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACrD,MAAM,CAAC,IAAI,CAAC;QACV,IAAI,EAAE,YAAY;QAClB,EAAE,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC;QACxB,OAAO,EAAE,SAAS,CAAC,IAAI,KAAK,CAAC;YAC3B,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC;YAC7B,CAAC,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,0BAA0B,CAAC;KACnF,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC;QACV,IAAI,EAAE,qBAAqB;QAC3B,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;QAC9E,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;YAC7B,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE;YAChD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM;gBACtB,CAAC,CAAC,sBAAsB,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE;gBAChD,CAAC,CAAC,MAAM,CAAC,MAAM;oBACb,CAAC,CAAC,sDAAsD;oBACxD,CAAC,CAAC,yCAAyC;KAClD,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/evaluation.d.ts

@@ -0,0 +1,23 @@
+import type { BehaviorSpecExample, EvalReport, LocalRunnerConfig } from "./contracts.js";
+export declare function evaluateExamples(args: {
+    kind: "baseline" | "candidate";
+    modelId: string;
+    baseModelId?: string;
+    adapterPath?: string;
+    examples: BehaviorSpecExample[];
+    system: string;
+    config: LocalRunnerConfig;
+    outputPath: string;
+}): Promise<EvalReport>;
+export declare function compareEvalReports(baseline: EvalReport, candidate: EvalReport): {
+    avg_score_delta: number;
+    pass_rate_delta: number;
+    exact_match_rate_delta: number;
+    regressions: number;
+    improvements: number;
+    regressed_examples: {
+        prompt: string;
+        old_score: number;
+        new_score: number;
+    }[];
+};