@interf/compiler 0.6.1 → 0.6.4

package/dist/packages/testing/lib/schema.d.ts

@@ -0,0 +1,261 @@
+import { z } from "zod";
+import { type TestCaseExpect, type TestTargetType } from "../../compiler/lib/schema.js";
+export declare const TestCaseSchema: z.ZodObject<{
+    id: z.ZodString;
+    question: z.ZodString;
+    file: z.ZodOptional<z.ZodString>;
+    answer: z.ZodOptional<z.ZodString>;
+    expect: z.ZodOptional<z.ZodObject<{
+        must_include: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        must_include_one_of: z.ZodOptional<z.ZodArray<z.ZodArray<z.ZodString>>>;
+        must_not_include: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        min_words: z.ZodOptional<z.ZodNumber>;
+        max_words: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    strictness: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare const TestSpecSchema: z.ZodObject<{
+    type: z.ZodEnum<{
+        compiled: "compiled";
+        raw: "raw";
+    }>;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    cases: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        question: z.ZodString;
+        file: z.ZodOptional<z.ZodString>;
+        answer: z.ZodOptional<z.ZodString>;
+        expect: z.ZodOptional<z.ZodObject<{
+            must_include: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            must_include_one_of: z.ZodOptional<z.ZodArray<z.ZodArray<z.ZodString>>>;
+            must_not_include: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            min_words: z.ZodOptional<z.ZodNumber>;
+            max_words: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>>;
+        strictness: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const TestCheckResultSchema: z.ZodObject<{
+    label: z.ZodString;
+    ok: z.ZodBoolean;
+    detail: z.ZodString;
+}, z.core.$strip>;
+export declare const TestCaseResultSchema: z.ZodObject<{
+    caseId: z.ZodString;
+    question: z.ZodString;
+    file: z.ZodOptional<z.ZodString>;
+    ok: z.ZodBoolean;
+    wordCount: z.ZodNumber;
+    passedChecks: z.ZodNumber;
+    totalChecks: z.ZodNumber;
+    checks: z.ZodArray<z.ZodObject<{
+        label: z.ZodString;
+        ok: z.ZodBoolean;
+        detail: z.ZodString;
+    }, z.core.$strip>>;
+    answer: z.ZodOptional<z.ZodString>;
+    trace: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+}, z.core.$strip>;
+export declare const TestTargetResultSchema: z.ZodObject<{
+    target: z.ZodObject<{
+        type: z.ZodEnum<{
+            compiled: "compiled";
+            raw: "raw";
+        }>;
+        name: z.ZodString;
+        path: z.ZodString;
+        workflow: z.ZodString;
+    }, z.core.$strip>;
+    sandbox_path: z.ZodOptional<z.ZodString>;
+    ok: z.ZodBoolean;
+    passedCases: z.ZodNumber;
+    totalCases: z.ZodNumber;
+    passedChecks: z.ZodNumber;
+    totalChecks: z.ZodNumber;
+    caseResults: z.ZodArray<z.ZodObject<{
+        caseId: z.ZodString;
+        question: z.ZodString;
+        file: z.ZodOptional<z.ZodString>;
+        ok: z.ZodBoolean;
+        wordCount: z.ZodNumber;
+        passedChecks: z.ZodNumber;
+        totalChecks: z.ZodNumber;
+        checks: z.ZodArray<z.ZodObject<{
+            label: z.ZodString;
+            ok: z.ZodBoolean;
+            detail: z.ZodString;
+        }, z.core.$strip>>;
+        answer: z.ZodOptional<z.ZodString>;
+        trace: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const TestTargetRunSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"interf-test-target-run">;
+    version: z.ZodLiteral<1>;
+    generated_at: z.ZodString;
+    spec: z.ZodObject<{
+        id: z.ZodString;
+        name: z.ZodString;
+        type: z.ZodEnum<{
+            compiled: "compiled";
+            raw: "raw";
+        }>;
+        file: z.ZodString;
+        description: z.ZodOptional<z.ZodString>;
+        case_count: z.ZodNumber;
+        cases: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            question: z.ZodString;
+            file: z.ZodOptional<z.ZodString>;
+            answer: z.ZodOptional<z.ZodString>;
+            strictness: z.ZodOptional<z.ZodString>;
+            expect: z.ZodOptional<z.ZodObject<{
+                must_include: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                must_include_one_of: z.ZodOptional<z.ZodArray<z.ZodArray<z.ZodString>>>;
+                must_not_include: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                min_words: z.ZodOptional<z.ZodNumber>;
+                max_words: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    source_path: z.ZodString;
+    executor: z.ZodOptional<z.ZodObject<{
+        kind: z.ZodEnum<{
+            "local-agent": "local-agent";
+            "connected-provider": "connected-provider";
+            managed: "managed";
+        }>;
+        name: z.ZodString;
+        display_name: z.ZodString;
+        command: z.ZodNullable<z.ZodString>;
+        model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        effort: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        profile: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        timeout_ms: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strip>>;
+    target_count: z.ZodNumber;
+    results: z.ZodArray<z.ZodObject<{
+        target: z.ZodObject<{
+            type: z.ZodEnum<{
+                compiled: "compiled";
+                raw: "raw";
+            }>;
+            name: z.ZodString;
+            path: z.ZodString;
+            workflow: z.ZodString;
+        }, z.core.$strip>;
+        sandbox_path: z.ZodOptional<z.ZodString>;
+        ok: z.ZodBoolean;
+        passedCases: z.ZodNumber;
+        totalCases: z.ZodNumber;
+        passedChecks: z.ZodNumber;
+        totalChecks: z.ZodNumber;
+        caseResults: z.ZodArray<z.ZodObject<{
+            caseId: z.ZodString;
+            question: z.ZodString;
+            file: z.ZodOptional<z.ZodString>;
+            ok: z.ZodBoolean;
+            wordCount: z.ZodNumber;
+            passedChecks: z.ZodNumber;
+            totalChecks: z.ZodNumber;
+            checks: z.ZodArray<z.ZodObject<{
+                label: z.ZodString;
+                ok: z.ZodBoolean;
+                detail: z.ZodString;
+            }, z.core.$strip>>;
+            answer: z.ZodOptional<z.ZodString>;
+            trace: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+        }, z.core.$strip>>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const TestRunModeSchema: z.ZodEnum<{
+    compiled: "compiled";
+    raw: "raw";
+    both: "both";
+}>;
+export declare const TestRunTargetSummarySchema: z.ZodObject<{
+    label: z.ZodString;
+    run_path: z.ZodString;
+    ok: z.ZodBoolean;
+    passed_cases: z.ZodNumber;
+    total_cases: z.ZodNumber;
+    passed_checks: z.ZodNumber;
+    total_checks: z.ZodNumber;
+    target: z.ZodObject<{
+        type: z.ZodEnum<{
+            compiled: "compiled";
+            raw: "raw";
+        }>;
+        name: z.ZodString;
+        path: z.ZodString;
+        workflow: z.ZodString;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export declare const TestRunComparisonSchema: z.ZodObject<{
+    kind: z.ZodLiteral<"interf-test-run">;
+    version: z.ZodLiteral<1>;
+    generated_at: z.ZodString;
+    mode: z.ZodEnum<{
+        compiled: "compiled";
+        raw: "raw";
+        both: "both";
+    }>;
+    source_path: z.ZodString;
+    checks_fingerprint: z.ZodOptional<z.ZodString>;
+    dataset: z.ZodObject<{
+        name: z.ZodString;
+        compiled_path: z.ZodNullable<z.ZodString>;
+    }, z.core.$strip>;
+    raw: z.ZodNullable<z.ZodObject<{
+        label: z.ZodString;
+        run_path: z.ZodString;
+        ok: z.ZodBoolean;
+        passed_cases: z.ZodNumber;
+        total_cases: z.ZodNumber;
+        passed_checks: z.ZodNumber;
+        total_checks: z.ZodNumber;
+        target: z.ZodObject<{
+            type: z.ZodEnum<{
+                compiled: "compiled";
+                raw: "raw";
+            }>;
+            name: z.ZodString;
+            path: z.ZodString;
+            workflow: z.ZodString;
+        }, z.core.$strip>;
+    }, z.core.$strip>>;
+    compiled: z.ZodNullable<z.ZodObject<{
+        label: z.ZodString;
+        run_path: z.ZodString;
+        ok: z.ZodBoolean;
+        passed_cases: z.ZodNumber;
+        total_cases: z.ZodNumber;
+        passed_checks: z.ZodNumber;
+        total_checks: z.ZodNumber;
+        target: z.ZodObject<{
+            type: z.ZodEnum<{
+                compiled: "compiled";
+                raw: "raw";
+            }>;
+            name: z.ZodString;
+            path: z.ZodString;
+            workflow: z.ZodString;
+        }, z.core.$strip>;
+    }, z.core.$strip>>;
+    summary: z.ZodObject<{
+        raw_pass_rate: z.ZodNullable<z.ZodNumber>;
+        compiled_pass_rate: z.ZodNullable<z.ZodNumber>;
+        pass_rate_delta: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export type { TestCaseExpect, TestTargetType };
+export type TestCase = z.infer<typeof TestCaseSchema>;
+export type TestSpec = z.infer<typeof TestSpecSchema>;
+export type TestCheckResult = z.infer<typeof TestCheckResultSchema>;
+export type TestCaseResult = z.infer<typeof TestCaseResultSchema>;
+export type TestTargetResult = z.infer<typeof TestTargetResultSchema>;
+export type TestTargetRun = z.infer<typeof TestTargetRunSchema>;
+export type TestRunMode = z.infer<typeof TestRunModeSchema>;
+export type TestRunTargetSummary = z.infer<typeof TestRunTargetSummarySchema>;
+export type TestRunComparison = z.infer<typeof TestRunComparisonSchema>;

package/dist/packages/testing/lib/schema.js

@@ -0,0 +1,119 @@
+import { z } from "zod";
+import { RuntimeExecutorInfoSchema, TestCaseExpectSchema, TestTargetTypeSchema, } from "../../compiler/lib/schema.js";
+import { DatasetNameSchema } from "../../project-model/lib/schema.js";
+const TestCaseCoreSchema = z.object({
+    id: z.string().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
+    question: z.string().min(1),
+    file: z.string().min(1).optional(),
+    answer: z.string().min(1).optional(),
+    expect: TestCaseExpectSchema.optional(),
+    strictness: z.string().min(1).optional(),
+}).superRefine((value, ctx) => {
+    if (!value.file && !value.answer && !value.expect) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "Test cases need at least one of file, answer, or expect.",
+        });
+    }
+});
+export const TestCaseSchema = TestCaseCoreSchema;
+const TestSpecCoreSchema = z.object({
+    type: TestTargetTypeSchema,
+    name: z.string().min(1),
+    description: z.string().min(1).optional(),
+    cases: z.array(TestCaseSchema).min(1),
+});
+export const TestSpecSchema = TestSpecCoreSchema;
+export const TestCheckResultSchema = z.object({
+    label: z.string(),
+    ok: z.boolean(),
+    detail: z.string(),
+});
+const JsonRecordSchema = z.record(z.string(), z.unknown());
+export const TestCaseResultSchema = z.object({
+    caseId: z.string(),
+    question: z.string(),
+    file: z.string().optional(),
+    ok: z.boolean(),
+    wordCount: z.number(),
+    passedChecks: z.number(),
+    totalChecks: z.number(),
+    checks: z.array(TestCheckResultSchema),
+    answer: z.string().optional(),
+    trace: JsonRecordSchema.nullable().optional(),
+});
+export const TestTargetResultSchema = z.object({
+    target: z.object({
+        type: TestTargetTypeSchema,
+        name: z.string(),
+        path: z.string(),
+        workflow: z.string(),
+    }),
+    sandbox_path: z.string().optional(),
+    ok: z.boolean(),
+    passedCases: z.number(),
+    totalCases: z.number(),
+    passedChecks: z.number(),
+    totalChecks: z.number(),
+    caseResults: z.array(TestCaseResultSchema),
+});
+export const TestTargetRunSchema = z.object({
+    kind: z.literal("interf-test-target-run"),
+    version: z.literal(1),
+    generated_at: z.string(),
+    spec: z.object({
+        id: z.string(),
+        name: z.string(),
+        type: TestTargetTypeSchema,
+        file: z.string(),
+        description: z.string().optional(),
+        case_count: z.number(),
+        cases: z.array(z.object({
+            id: z.string(),
+            question: z.string(),
+            file: z.string().optional(),
+            answer: z.string().optional(),
+            strictness: z.string().optional(),
+            expect: TestCaseExpectSchema.optional(),
+        })),
+    }),
+    source_path: z.string(),
+    executor: RuntimeExecutorInfoSchema.optional(),
+    target_count: z.number(),
+    results: z.array(TestTargetResultSchema),
+});
+export const TestRunModeSchema = z.enum(["raw", "compiled", "both"]);
+export const TestRunTargetSummarySchema = z.object({
+    label: z.string(),
+    run_path: z.string(),
+    ok: z.boolean(),
+    passed_cases: z.number(),
+    total_cases: z.number(),
+    passed_checks: z.number(),
+    total_checks: z.number(),
+    target: z.object({
+        type: TestTargetTypeSchema,
+        name: z.string(),
+        path: z.string(),
+        workflow: z.string(),
+    }),
+});
+export const TestRunComparisonSchema = z.object({
+    kind: z.literal("interf-test-run"),
+    version: z.literal(1),
+    generated_at: z.string(),
+    mode: TestRunModeSchema,
+    source_path: z.string(),
+    checks_fingerprint: z.string().min(1).optional(),
+    dataset: z.object({
+        name: DatasetNameSchema,
+        compiled_path: z.string().nullable(),
+    }),
+    raw: TestRunTargetSummarySchema.nullable(),
+    compiled: TestRunTargetSummarySchema.nullable(),
+    summary: z.object({
+        raw_pass_rate: z.number().nullable(),
+        compiled_pass_rate: z.number().nullable(),
+        pass_rate_delta: z.number().nullable(),
+    }),
+});

package/dist/packages/testing/test-execution.d.ts

@@ -0,0 +1,14 @@
+import { type WorkflowExecutor } from "../agents/lib/executors.js";
+import type { TestTargetRun, TestTargetCandidate, LoadedTestSpec } from "./test-types.js";
+import { type TestSandboxRetentionMode } from "./test-sandbox.js";
+export declare function runTargetTests(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[]): TestTargetRun;
+export declare function runTargetTestsWithJudge(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[], executor: WorkflowExecutor, options?: {
+    preserveSandboxes?: TestSandboxRetentionMode;
+    artifactRootPath?: string;
+}): Promise<TestTargetRun>;
+export declare function runTargetTestsAuto(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[], options?: {
+    executor?: WorkflowExecutor | null;
+    preserveSandboxes?: TestSandboxRetentionMode;
+    artifactRootPath?: string;
+}): Promise<TestTargetRun>;
+export declare function saveTargetTestRun(artifactRootPath: string, result: TestTargetRun): string;