@mzhub/promptc 0.1.0 → 0.1.2

package/dist/src/compiler/CandidatePool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CandidatePool.d.ts","sourceRoot":"","sources":["../../../src/compiler/CandidatePool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,qBAAa,aAAa,CAAC,CAAC,EAAE,CAAC;IAGjB,OAAO,CAAC,QAAQ;IAF5B,OAAO,CAAC,GAAG,CAAe;gBAEN,QAAQ,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM;IAI5D,yBAAyB,CACvB,KAAK,EAAE,MAAM,EACb,oBAAoB,EAAE,MAAM,GAC3B,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAazB,gBAAgB,CAAC,UAAU,GAAE,MAAY,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;IAM3D,OAAO,CAAC,YAAY;CAOrB"}

package/dist/src/compiler/CandidatePool.js

@@ -0,0 +1,29 @@
+export class CandidatePool {
+    trainset;
+    rng;
+    constructor(trainset, seed) {
+        this.trainset = trainset;
+        this.rng = seed !== undefined ? this.seededRandom(seed) : Math.random;
+    }
+    generateFewShotCandidates(count, examplesPerCandidate) {
+        const candidates = [];
+        for (let i = 0; i < count; i++) {
+            const shuffled = [...this.trainset].sort(() => this.rng() - 0.5);
+            candidates.push(shuffled.slice(0, Math.min(examplesPerCandidate, shuffled.length)));
+        }
+        return candidates;
+    }
+    getValidationSet(splitRatio = 0.3) {
+        const count = Math.max(1, Math.floor(this.trainset.length * splitRatio));
+        const shuffled = [...this.trainset].sort(() => this.rng() - 0.5);
+        return shuffled.slice(0, count);
+    }
+    seededRandom(seed) {
+        let s = seed;
+        return () => {
+            s = Math.sin(s) * 10000;
+            return s - Math.floor(s);
+        };
+    }
+}
+//# sourceMappingURL=CandidatePool.js.map

package/dist/src/compiler/CandidatePool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CandidatePool.js","sourceRoot":"","sources":["../../../src/compiler/CandidatePool.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,aAAa;IAGJ;IAFZ,GAAG,CAAe;IAE1B,YAAoB,QAAyB,EAAE,IAAa;QAAxC,aAAQ,GAAR,QAAQ,CAAiB;QAC3C,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IACxE,CAAC;IAED,yBAAyB,CACvB,KAAa,EACb,oBAA4B;QAE5B,MAAM,UAAU,GAA2B,EAAE,CAAC;QAE9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;YACjE,UAAU,CAAC,IAAI,CACb,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CACnE,CAAC;QACJ,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,gBAAgB,CAAC,aAAqB,GAAG;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;QACjE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAEO,YAAY,CAAC,IAAY;QAC/B,IAAI,CAAC,GAAG,IAAI,CAAC;QACb,OAAO,GAAG,EAAE;YACV,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;YACxB,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC,CAAC;IACJ,CAAC;CACF"}

package/dist/src/compiler/CompiledProgram.d.ts

@@ -0,0 +1,43 @@
+import { ZodRawShape } from "zod";
+import type { ProgramConfig, ProgramOutput } from "../program/Program.js";
+import type { Example } from "./types.js";
+export interface CompiledProgramMeta {
+    score: number;
+    compiledAt: string;
+    strategy: string;
+    tokenUsage: {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+        calls: number;
+    };
+}
+export interface CompiledProgram<I, O> {
+    run(input: I, overrides?: Partial<ProgramConfig>): Promise<ProgramOutput<O>>;
+    meta: CompiledProgramMeta;
+    config: {
+        instructions: string;
+        fewShotExamples: Array<Example<unknown, unknown>>;
+    };
+    toJSON(): string;
+}
+export declare function createCompiledProgram<I extends ZodRawShape, O extends ZodRawShape, InputType = unknown, OutputType = unknown>(originalProgram: {
+    run(input: InputType, config?: ProgramConfig): Promise<ProgramOutput<OutputType>>;
+}, compilationResult: {
+    meta: CompiledProgramMeta;
+    config: {
+        instructions: string;
+        fewShotExamples: Array<Example<unknown, unknown>>;
+    };
+}): CompiledProgram<InputType, OutputType>;
+export interface SerializedCompiledProgram {
+    meta: CompiledProgramMeta;
+    config: {
+        instructions: string;
+        fewShotExamples: Array<Example<unknown, unknown>>;
+    };
+}
+export declare function loadCompiledProgram<InputType = unknown, OutputType = unknown>(jsonOrPath: string | SerializedCompiledProgram, program: {
+    run(input: InputType, config?: ProgramConfig): Promise<ProgramOutput<OutputType>>;
+}): CompiledProgram<InputType, OutputType>;
+//# sourceMappingURL=CompiledProgram.d.ts.map

package/dist/src/compiler/CompiledProgram.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"CompiledProgram.d.ts","sourceRoot":"","sources":["../../../src/compiler/CompiledProgram.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAa,MAAM,KAAK,CAAC;AAC7C,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE;QACV,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,eAAe,CAAC,CAAC,EAAE,CAAC;IACnC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;KACnD,CAAC;IACF,MAAM,IAAI,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,CACnC,CAAC,SAAS,WAAW,EACrB,CAAC,SAAS,WAAW,EACrB,SAAS,GAAG,OAAO,EACnB,UAAU,GAAG,OAAO,EAEpB,eAAe,EAAE;IACf,GAAG,CACD,KAAK,EAAE,SAAS,EAChB,MAAM,CAAC,EAAE,aAAa,GACrB,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;CACvC,EACD,iBAAiB,EAAE;IACjB,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;KACnD,CAAC;CACH,GACA,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CA8BxC;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,mBAAmB,CAAC;IAC1B,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;KACnD,CAAC;CACH;AAED,wBAAgB,mBAAmB,CAAC,SAAS,GAAG,OAAO,EAAE,UAAU,GAAG,OAAO,EAC3E,UAAU,EAAE,MAAM,GAAG,yBAAyB,EAC9C,OAAO,EAAE;IACP,GAAG,CACD,KAAK,EAAE,SAAS,EAChB,MAAM,CAAC,EAAE,aAAa,GACrB,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC;CACvC,GACA,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC,CAkBxC"}

package/dist/src/compiler/CompiledProgram.js

@@ -0,0 +1,41 @@
+export function createCompiledProgram(originalProgram, compilationResult) {
+    return {
+        async run(input, overrides) {
+            return originalProgram.run(input, {
+                instructions: overrides?.instructions ?? compilationResult.config.instructions,
+                fewShotExamples: overrides?.fewShotExamples ??
+                    compilationResult.config.fewShotExamples.map((ex) => ({
+                        input: ex.input,
+                        output: ex.output,
+                    })),
+            });
+        },
+        meta: compilationResult.meta,
+        config: compilationResult.config,
+        toJSON() {
+            return JSON.stringify({
+                meta: compilationResult.meta,
+                config: compilationResult.config,
+            }, null, 2);
+        },
+    };
+}
+export function loadCompiledProgram(jsonOrPath, program) {
+    let data;
+    if (typeof jsonOrPath === "string") {
+        try {
+            data = JSON.parse(jsonOrPath);
+        }
+        catch {
+            throw new Error("Invalid JSON string provided to loadCompiledProgram");
+        }
+    }
+    else {
+        data = jsonOrPath;
+    }
+    if (!data.meta || !data.config) {
+        throw new Error("Invalid compiled program format: missing meta or config");
+    }
+    return createCompiledProgram(program, data);
+}
+//# sourceMappingURL=CompiledProgram.js.map

package/dist/src/compiler/CompiledProgram.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CompiledProgram.js","sourceRoot":"","sources":["../../../src/compiler/CompiledProgram.ts"],"names":[],"mappings":"AA0BA,MAAM,UAAU,qBAAqB,CAMnC,eAKC,EACD,iBAMC;IAED,OAAO;QACL,KAAK,CAAC,GAAG,CACP,KAAgB,EAChB,SAAkC;YAElC,OAAO,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE;gBAChC,YAAY,EACV,SAAS,EAAE,YAAY,IAAI,iBAAiB,CAAC,MAAM,CAAC,YAAY;gBAClE,eAAe,EACb,SAAS,EAAE,eAAe;oBAC1B,iBAAiB,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;wBACpD,KAAK,EAAE,EAAE,CAAC,KAAgC;wBAC1C,MAAM,EAAE,EAAE,CAAC,MAAiC;qBAC7C,CAAC,CAAC;aACN,CAAC,CAAC;QACL,CAAC;QACD,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,MAAM,EAAE,iBAAiB,CAAC,MAAM;QAChC,MAAM;YACJ,OAAO,IAAI,CAAC,SAAS,CACnB;gBACE,IAAI,EAAE,iBAAiB,CAAC,IAAI;gBAC5B,MAAM,EAAE,iBAAiB,CAAC,MAAM;aACjC,EACD,IAAI,EACJ,CAAC,CACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAUD,MAAM,UAAU,mBAAmB,CACjC,UAA8C,EAC9C,OAKC;IAED,IAAI,IAA+B,CAAC;IAEpC,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IAED,OAAO,qBAAqB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAC9C,CAAC"}

package/dist/src/compiler/InstructionRewrite.d.ts

@@ -0,0 +1,19 @@
+import { ZodRawShape } from "zod";
+import { Program } from "../program/Program.js";
+import type { LLMProvider } from "../providers/types.js";
+import type { Evaluator, Example, CompilationResult, CompileOptions } from "./types.js";
+export interface InstructionRewriteOptions extends CompileOptions {
+    instructionVariations?: number;
+    provider: LLMProvider;
+}
+export declare class InstructionRewrite<O> {
+    private evaluator;
+    constructor(evaluator: Evaluator<O>);
+    compile<I extends ZodRawShape, OShape extends ZodRawShape>(program: Program<I, OShape>, trainset: Example<unknown, O>[], options: InstructionRewriteOptions): Promise<CompilationResult<unknown, O>>;
+    private generateInstructionVariants;
+    estimateCost(trainsetSize: number, options?: Partial<InstructionRewriteOptions>): {
+        estimatedCalls: number;
+        estimatedTokens: number;
+    };
+}
+//# sourceMappingURL=InstructionRewrite.d.ts.map

package/dist/src/compiler/InstructionRewrite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"InstructionRewrite.d.ts","sourceRoot":"","sources":["../../../src/compiler/InstructionRewrite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,KAAK,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAIhD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EACV,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,yBAA0B,SAAQ,cAAc;IAC/D,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,EAAE,WAAW,CAAC;CACvB;AAED,qBAAa,kBAAkB,CAAC,CAAC;IACnB,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;IAErC,OAAO,CAAC,CAAC,SAAS,WAAW,EAAE,MAAM,SAAS,WAAW,EAC7D,OAAO,EAAE,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,EAC3B,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAC/B,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,iBAAiB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAiH3B,2BAA2B;IAoCzC,YAAY,CACV,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,OAAO,CAAC,yBAAyB,CAAM,GAC/C;QACD,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB;CAmBF"}

package/dist/src/compiler/InstructionRewrite.js

@@ -0,0 +1,117 @@
+import { createConcurrencyManager } from "../runtime/concurrency.js";
+import { CostTracker } from "../runtime/costTracker.js";
+import { CandidatePool } from "./CandidatePool.js";
+export class InstructionRewrite {
+    evaluator;
+    constructor(evaluator) {
+        this.evaluator = evaluator;
+    }
+    async compile(program, trainset, options) {
+        const { candidates: candidateCount = 5, instructionVariations = 5, concurrency = 5, examplesPerCandidate = 3, validationSplit = 0.3, seed, budget, provider, } = options;
+        const baseDescription = program["schema"].description;
+        const pool = new CandidatePool(trainset, seed);
+        const validationSet = pool.getValidationSet(validationSplit);
+        const limiter = createConcurrencyManager(concurrency);
+        const costTracker = new CostTracker();
+        const instructionVariants = await this.generateInstructionVariants(provider, baseDescription, instructionVariations);
+        const fewShotCandidates = pool.generateFewShotCandidates(candidateCount, examplesPerCandidate);
+        const allCandidates = [];
+        for (const instruction of instructionVariants) {
+            for (const examples of fewShotCandidates) {
+                allCandidates.push({ instructions: instruction, examples });
+            }
+        }
+        const results = [];
+        const evaluationPromises = allCandidates.map((candidate) => limiter.run(async () => {
+            if (budget?.maxTokens && costTracker.exceedsBudget(budget.maxTokens)) {
+                budget.onBudgetWarning?.(costTracker.total.totalTokens, budget.maxTokens);
+                return { score: -1, ...candidate };
+            }
+            let totalScore = 0;
+            let validCount = 0;
+            for (const testCase of validationSet) {
+                try {
+                    const prediction = await program.run(testCase.input, {
+                        instructions: candidate.instructions,
+                        fewShotExamples: candidate.examples.map((ex) => ({
+                            input: ex.input,
+                            output: ex.output,
+                        })),
+                    });
+                    costTracker.record(prediction.trace.usage);
+                    const score = await this.evaluator(prediction.result, testCase.output);
+                    totalScore += score;
+                    validCount++;
+                }
+                catch {
+                    continue;
+                }
+            }
+            return {
+                score: validCount > 0 ? totalScore / validCount : 0,
+                ...candidate,
+            };
+        }));
+        const candidateResults = await Promise.all(evaluationPromises);
+        results.push(...candidateResults.filter((r) => r.score >= 0));
+        const winner = results.reduce((best, current) => (current.score > best.score ? current : best), {
+            score: -1,
+            instructions: baseDescription,
+            examples: [],
+        });
+        return {
+            meta: {
+                score: winner.score,
+                compiledAt: new Date().toISOString(),
+                strategy: "InstructionRewrite",
+                tokenUsage: costTracker.total,
+            },
+            config: {
+                instructions: winner.instructions,
+                fewShotExamples: winner.examples,
+            },
+        };
+    }
+    async generateInstructionVariants(provider, baseInstruction, count) {
+        const prompt = `You are an expert prompt engineer. Given a base instruction for an LLM task, generate ${count} different variations that might perform better.
+
+Base instruction: "${baseInstruction}"
+
+Requirements:
+- Each variation should convey the same task but with different wording, tone, or structure
+- Try variations like: more specific, more concise, role-based ("You are an expert..."), step-by-step, formal, casual
+- Output as a JSON array of strings
+
+Respond with ONLY a JSON array, no additional text:
+["variation 1", "variation 2", ...]`;
+        const response = await provider.complete({
+            prompt,
+            temperature: 0.8,
+            maxTokens: 1024,
+            responseFormat: "json",
+        });
+        try {
+            const jsonMatch = response.content.match(/\[[\s\S]*\]/);
+            if (!jsonMatch) {
+                return [baseInstruction];
+            }
+            const variations = JSON.parse(jsonMatch[0]);
+            return [baseInstruction, ...variations.slice(0, count)];
+        }
+        catch {
+            return [baseInstruction];
+        }
+    }
+    estimateCost(trainsetSize, options = {}) {
+        const { candidates = 5, instructionVariations = 5, validationSplit = 0.3, } = options;
+        const validationSize = Math.max(1, Math.floor(trainsetSize * validationSplit));
+        const totalCandidates = candidates * instructionVariations;
+        const estimatedCalls = totalCandidates * validationSize + 1;
+        const avgTokensPerCall = 500;
+        return {
+            estimatedCalls,
+            estimatedTokens: estimatedCalls * avgTokensPerCall,
+        };
+    }
+}
+//# sourceMappingURL=InstructionRewrite.js.map

package/dist/src/compiler/InstructionRewrite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InstructionRewrite.js","sourceRoot":"","sources":["../../../src/compiler/InstructionRewrite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAcnD,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,SAAuB;QAAvB,cAAS,GAAT,SAAS,CAAc;IAAG,CAAC;IAE/C,KAAK,CAAC,OAAO,CACX,OAA2B,EAC3B,QAA+B,EAC/B,OAAkC;QAElC,MAAM,EACJ,UAAU,EAAE,cAAc,GAAG,CAAC,EAC9B,qBAAqB,GAAG,CAAC,EACzB,WAAW,GAAG,CAAC,EACf,oBAAoB,GAAG,CAAC,EACxB,eAAe,GAAG,GAAG,EACrB,IAAI,EACJ,MAAM,EACN,QAAQ,GACT,GAAG,OAAO,CAAC;QAEZ,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC;QACtD,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QAEtC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAChE,QAAQ,EACR,eAAe,EACf,qBAAqB,CACtB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,yBAAyB,CACtD,cAAc,EACd,oBAAoB,CACrB,CAAC;QAEF,MAAM,aAAa,GAGd,EAAE,CAAC;QACR,KAAK,MAAM,WAAW,IAAI,mBAAmB,EAAE,CAAC;YAC9C,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;gBACzC,aAAa,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAIR,EAAE,CAAC;QAER,MAAM,kBAAkB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACzD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACrB,IAAI,MAAM,EAAE,SAAS,IAAI,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrE,MAAM,CAAC,eAAe,EAAE,CACtB,WAAW,CAAC,KAAK,CAAC,WAAW,EAC7B,MAAM,CAAC,SAAS,CACjB,CAAC;gBACF,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,CAAC;YACrC,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAY,EAAE;wBAC1D,YAAY,EAAE,SAAS,CAAC,YAAY;wBACpC,eAAe,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;4BAC/C,KAAK,EAAE,EAAE,CAAC,KAAgC;4BAC1C,MAAM,EAAE,EAAE,CAAC,MAAiC;yBAC7C,CAAC,CAAC;qBACJ,CAAC,CAAC;oBAEH,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAChC,UAAU,CAAC,MAAW,EACtB,QAAQ,CAAC,MAAM,CAChB,CAAC;oBACF,UAAU,IAAI,KAAK,CAAC;oBACpB,UAAU,EAAE,CAAC;gBACf,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;YACH,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBACnD,GAAG,SAAS;aACb,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;QAE9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAC3B,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAChE;YACE,KAAK,EAAE,CAAC,CAAC;YACT,YAAY,EAAE,eAAe;YAC7B,QAAQ,EAAE,EAA2B;SACtC,CACF,CAAC;QAEF,OAAO;YACL,IAAI,EAAE;gBACJ,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACpC,QAAQ,EAAE,oBAAoB;gBAC9B,UAAU,EAAE,WAAW,CAAC,KAAK;aAC9B;YACD,MAAM,EAAE;gBACN,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,eAAe,EAAE,MAAM,CAAC,QAAQ;aACjC;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,QAAqB,EACrB,eAAuB,EACvB,KAAa;QAEb,MAAM,MAAM,GAAG,yFAAyF,KAAK;;qBAE5F,eAAe;;;;;;;;oCAQA,CAAC;QAEjC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC;YACvC,MAAM;YACN,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;YACf,cAAc,EAAE,MAAM;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACxD,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,eAAe,CAAC,CAAC;YAC3B,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAa,CAAC;YACxD,OAAO,CAAC,eAAe,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,eAAe,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,YAAY,CACV,YAAoB,EACpB,UAA8C,EAAE;QAKhD,MAAM,EACJ,UAAU,GAAG,CAAC,EACd,qBAAqB,GAAG,CAAC,EACzB,eAAe,GAAG,GAAG,GACtB,GAAG,OAAO,CAAC;QACZ,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC7B,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,eAAe,CAAC,CAC3C,CAAC;QACF,MAAM,eAAe,GAAG,UAAU,GAAG,qBAAqB,CAAC;QAC3D,MAAM,cAAc,GAAG,eAAe,GAAG,cAAc,GAAG,CAAC,CAAC;QAC5D,MAAM,gBAAgB,GAAG,GAAG,CAAC;QAE7B,OAAO;YACL,cAAc;YACd,eAAe,EAAE,cAAc,GAAG,gBAAgB;SACnD,CAAC;IACJ,CAAC;CACF"}

package/dist/src/compiler/index.d.ts

@@ -0,0 +1,8 @@
+export { BootstrapFewShot } from "./BootstrapFewShot.js";
+export { InstructionRewrite } from "./InstructionRewrite.js";
+export type { InstructionRewriteOptions } from "./InstructionRewrite.js";
+export { CandidatePool } from "./CandidatePool.js";
+export { createCompiledProgram, loadCompiledProgram, } from "./CompiledProgram.js";
+export type { CompiledProgram, CompiledProgramMeta, SerializedCompiledProgram, } from "./CompiledProgram.js";
+export type { Evaluator, Example, CompilationResult, CompileOptions, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/compiler/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/compiler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,YAAY,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,eAAe,EACf,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,GACf,MAAM,YAAY,CAAC"}

package/dist/src/compiler/index.js

@@ -0,0 +1,5 @@
+export { BootstrapFewShot } from "./BootstrapFewShot.js";
+export { InstructionRewrite } from "./InstructionRewrite.js";
+export { CandidatePool } from "./CandidatePool.js";
+export { createCompiledProgram, loadCompiledProgram, } from "./CompiledProgram.js";
+//# sourceMappingURL=index.js.map

package/dist/src/compiler/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/compiler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC"}

package/dist/src/compiler/types.d.ts

@@ -0,0 +1,41 @@
+export type Evaluator<O> = (prediction: O, groundTruth: O) => number | Promise<number>;
+export interface Example<I, O> {
+    input: I;
+    output: O;
+}
+export interface CompilationResult<I, O> {
+    meta: {
+        score: number;
+        compiledAt: string;
+        strategy: string;
+        tokenUsage: {
+            inputTokens: number;
+            outputTokens: number;
+            totalTokens: number;
+            calls: number;
+        };
+    };
+    config: {
+        instructions: string;
+        fewShotExamples: Array<Example<I, O>>;
+    };
+}
+export interface CompileOptions {
+    candidates?: number;
+    concurrency?: number;
+    examplesPerCandidate?: number;
+    validationSplit?: number;
+    seed?: number;
+    earlyStopThreshold?: number;
+    budget?: {
+        maxTokens?: number;
+        onBudgetWarning?: (used: number, max: number) => void;
+    };
+    onProgress?: (progress: {
+        candidatesEvaluated: number;
+        totalCandidates: number;
+        currentBestScore: number;
+        tokensUsed: number;
+    }) => void;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/src/compiler/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/compiler/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CACzB,UAAU,EAAE,CAAC,EACb,WAAW,EAAE,CAAC,KACX,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAE9B,MAAM,WAAW,OAAO,CAAC,CAAC,EAAE,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAC;IACT,MAAM,EAAE,CAAC,CAAC;CACX;AAED,MAAM,WAAW,iBAAiB,CAAC,CAAC,EAAE,CAAC;IACrC,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE;YACV,WAAW,EAAE,MAAM,CAAC;YACpB,YAAY,EAAE,MAAM,CAAC;YACrB,WAAW,EAAE,MAAM,CAAC;YACpB,KAAK,EAAE,MAAM,CAAC;SACf,CAAC;KACH,CAAC;IACF,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;KACvC,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,MAAM,CAAC,EAAE;QACP,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;KACvD,CAAC;IACF,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QACtB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;KACpB,KAAK,IAAI,CAAC;CACZ"}

package/dist/src/compiler/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/src/compiler/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/compiler/types.ts"],"names":[],"mappings":""}

package/dist/src/eval/exactMatch.d.ts

@@ -0,0 +1,5 @@
+import type { Evaluator } from "./types.js";
+export declare function exactMatch<O>(): Evaluator<O>;
+export declare function partialMatch<O extends Record<string, unknown>>(): Evaluator<O>;
+export declare function arrayOverlap<T>(): Evaluator<T[]>;
+//# sourceMappingURL=exactMatch.d.ts.map

package/dist/src/eval/exactMatch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exactMatch.d.ts","sourceRoot":"","sources":["../../../src/eval/exactMatch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAyB5C,wBAAgB,UAAU,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAI5C;AAED,wBAAgB,YAAY,CAC1B,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,SAAS,CAAC,CAAC,CAAC,CAchB;AAED,wBAAgB,YAAY,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,EAAE,CAAC,CAehD"}

package/dist/src/eval/exactMatch.js

@@ -0,0 +1,58 @@
+function deepEqual(a, b) {
+    if (a === b)
+        return true;
+    if (typeof a !== typeof b)
+        return false;
+    if (a === null || b === null)
+        return a === b;
+    if (Array.isArray(a) && Array.isArray(b)) {
+        if (a.length !== b.length)
+            return false;
+        return a.every((item, i) => deepEqual(item, b[i]));
+    }
+    if (typeof a === "object" && typeof b === "object") {
+        const aObj = a;
+        const bObj = b;
+        const aKeys = Object.keys(aObj);
+        const bKeys = Object.keys(bObj);
+        if (aKeys.length !== bKeys.length)
+            return false;
+        return aKeys.every((key) => deepEqual(aObj[key], bObj[key]));
+    }
+    return false;
+}
+export function exactMatch() {
+    return (prediction, groundTruth) => {
+        return deepEqual(prediction, groundTruth) ? 1.0 : 0.0;
+    };
+}
+export function partialMatch() {
+    return (prediction, groundTruth) => {
+        const keys = Object.keys(groundTruth);
+        if (keys.length === 0)
+            return 1.0;
+        let matchCount = 0;
+        for (const key of keys) {
+            if (deepEqual(prediction[key], groundTruth[key])) {
+                matchCount++;
+            }
+        }
+        return matchCount / keys.length;
+    };
+}
+export function arrayOverlap() {
+    return (prediction, groundTruth) => {
+        if (groundTruth.length === 0)
+            return prediction.length === 0 ? 1.0 : 0.0;
+        const predSet = new Set(prediction.map((x) => JSON.stringify(x)));
+        const truthSet = new Set(groundTruth.map((x) => JSON.stringify(x)));
+        let intersection = 0;
+        for (const item of predSet) {
+            if (truthSet.has(item))
+                intersection++;
+        }
+        const union = new Set([...predSet, ...truthSet]).size;
+        return union > 0 ? intersection / union : 0;
+    };
+}
+//# sourceMappingURL=exactMatch.js.map

package/dist/src/eval/exactMatch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exactMatch.js","sourceRoot":"","sources":["../../../src/eval/exactMatch.ts"],"names":[],"mappings":"AAEA,SAAS,SAAS,CAAC,CAAU,EAAE,CAAU;IACvC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAE7C,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACxC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,CAA4B,CAAC;QAC1C,MAAM,IAAI,GAAG,CAA4B,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAChD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,UAAU;IACxB,OAAO,CAAC,UAAa,EAAE,WAAc,EAAU,EAAE;QAC/C,OAAO,SAAS,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACxD,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY;IAG1B,OAAO,CAAC,UAAa,EAAE,WAAc,EAAU,EAAE;QAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAElC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACjD,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,OAAO,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;IAClC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,OAAO,CAAC,UAAe,EAAE,WAAgB,EAAU,EAAE;QACnD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAEzE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,YAAY,EAAE,CAAC;QACzC,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QACtD,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/eval/index.d.ts

@@ -0,0 +1,5 @@
+export type { Evaluator } from "./types.js";
+export { exactMatch, partialMatch, arrayOverlap } from "./exactMatch.js";
+export { llmJudge } from "./llmJudge.js";
+export type { LLMJudgeConfig } from "./llmJudge.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/eval/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,YAAY,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC"}

package/dist/src/eval/index.js

@@ -0,0 +1,3 @@
+export { exactMatch, partialMatch, arrayOverlap } from "./exactMatch.js";
+export { llmJudge } from "./llmJudge.js";
+//# sourceMappingURL=index.js.map

package/dist/src/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/eval/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC"}

package/dist/src/eval/llmJudge.d.ts

@@ -0,0 +1,9 @@
+import type { LLMProvider } from "../providers/types.js";
+import type { Evaluator } from "./types.js";
+export interface LLMJudgeConfig {
+    provider: LLMProvider;
+    criteria?: string;
+    model?: string;
+}
+export declare function llmJudge<O>(config: LLMJudgeConfig): Evaluator<O>;
+//# sourceMappingURL=llmJudge.d.ts.map

package/dist/src/eval/llmJudge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmJudge.d.ts","sourceRoot":"","sources":["../../../src/eval/llmJudge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,SAAS,CAAC,CAAC,CAAC,CAmChE"}

package/dist/src/eval/llmJudge.js

@@ -0,0 +1,33 @@
+export function llmJudge(config) {
+    const { provider, criteria = "accuracy and completeness", model } = config;
+    return async (prediction, groundTruth) => {
+        const prompt = `You are evaluating the quality of an AI response.
+
+EXPECTED OUTPUT:
+${JSON.stringify(groundTruth, null, 2)}
+
+ACTUAL OUTPUT:
+${JSON.stringify(prediction, null, 2)}
+
+EVALUATION CRITERIA: ${criteria}
+
+Rate the actual output on a scale from 0.0 to 1.0, where:
+- 1.0 = Perfect match or equivalent
+- 0.5 = Partially correct
+- 0.0 = Completely wrong
+
+Respond with ONLY a number between 0.0 and 1.0, nothing else.`;
+        const response = await provider.complete({
+            prompt,
+            model,
+            temperature: 0,
+            maxTokens: 10,
+        });
+        const score = parseFloat(response.content.trim());
+        if (isNaN(score) || score < 0 || score > 1) {
+            return 0;
+        }
+        return score;
+    };
+}
+//# sourceMappingURL=llmJudge.js.map

package/dist/src/eval/llmJudge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmJudge.js","sourceRoot":"","sources":["../../../src/eval/llmJudge.ts"],"names":[],"mappings":"AASA,MAAM,UAAU,QAAQ,CAAI,MAAsB;IAChD,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,2BAA2B,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAE3E,OAAO,KAAK,EAAE,UAAa,EAAE,WAAc,EAAmB,EAAE;QAC9D,MAAM,MAAM,GAAG;;;EAGjB,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;;;EAGpC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;;uBAEd,QAAQ;;;;;;;8DAO+B,CAAC;QAE3D,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC;YACvC,MAAM;YACN,KAAK;YACL,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,EAAE;SACd,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC"}

package/dist/src/eval/types.d.ts

@@ -0,0 +1,2 @@
+export type Evaluator<O> = (prediction: O, groundTruth: O) => number | Promise<number>;
+//# sourceMappingURL=types.d.ts.map

package/dist/src/eval/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/eval/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CACzB,UAAU,EAAE,CAAC,EACb,WAAW,EAAE,CAAC,KACX,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC"}

package/dist/src/eval/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/src/eval/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/eval/types.ts"],"names":[],"mappings":""}

package/dist/src/index.d.ts

@@ -0,0 +1,14 @@
+export { z } from "zod";
+export { Schema, defineSchema } from "./schema/index.js";
+export type { SchemaDefinition } from "./schema/index.js";
+export { createProvider, OpenAIProvider, AnthropicProvider, GoogleProvider, OllamaProvider, GroqProvider, CerebrasProvider, } from "./providers/index.js";
+export type { LLMProvider, CompletionParams, CompletionResult, ProviderConfig, ProviderName, } from "./providers/index.js";
+export { Program, Predict, ChainOfThought } from "./program/index.js";
+export type { ProgramConfig, ProgramOutput, ProgramTrace, } from "./program/index.js";
+export { BootstrapFewShot, InstructionRewrite, CandidatePool, createCompiledProgram, loadCompiledProgram, } from "./compiler/index.js";
+export type { Evaluator, Example, CompilationResult, CompileOptions, CompiledProgram, CompiledProgramMeta, SerializedCompiledProgram, InstructionRewriteOptions, } from "./compiler/index.js";
+export { exactMatch, partialMatch, arrayOverlap, llmJudge, } from "./eval/index.js";
+export type { LLMJudgeConfig } from "./eval/index.js";
+export { createConcurrencyManager, CostTracker, estimateCost, withRetry, createRetryWrapper, PromptCache, createCache, } from "./runtime/index.js";
+export type { ConcurrencyManager, TokenUsage, CostEstimate, RetryOptions, CacheOptions, } from "./runtime/index.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAE1D,OAAO,EACL,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,YAAY,EACZ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,GACb,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,YAAY,EACV,aAAa,EACb,aAAa,EACb,YAAY,GACb,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACb,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACV,SAAS,EACT,OAAO,EACP,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,mBAAmB,EACnB,yBAAyB,EACzB,yBAAyB,GAC1B,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,QAAQ,GACT,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EACL,wBAAwB,EACxB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EACV,kBAAkB,EAClB,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,YAAY,GACb,MAAM,oBAAoB,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,8 @@
+export { z } from "zod";
+export { Schema, defineSchema } from "./schema/index.js";
+export { createProvider, OpenAIProvider, AnthropicProvider, GoogleProvider, OllamaProvider, GroqProvider, CerebrasProvider, } from "./providers/index.js";
+export { Program, Predict, ChainOfThought } from "./program/index.js";
+export { BootstrapFewShot, InstructionRewrite, CandidatePool, createCompiledProgram, loadCompiledProgram, } from "./compiler/index.js";
+export { exactMatch, partialMatch, arrayOverlap, llmJudge, } from "./eval/index.js";
+export { createConcurrencyManager, CostTracker, estimateCost, withRetry, createRetryWrapper, PromptCache, createCache, } from "./runtime/index.js";
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAGzD,OAAO,EACL,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,YAAY,EACZ,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAS9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAOtE,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,EACb,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,qBAAqB,CAAC;AAY7B,OAAO,EACL,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,QAAQ,GACT,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,wBAAwB,EACxB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,oBAAoB,CAAC"}

package/dist/src/program/ChainOfThought.d.ts

@@ -0,0 +1,6 @@
+import { z, ZodRawShape, ZodObject } from "zod";
+import { Program, ProgramConfig, ProgramOutput } from "./Program.js";
+export declare class ChainOfThought<I extends ZodRawShape, O extends ZodRawShape> extends Program<I, O> {
+    run(input: z.infer<ZodObject<I>>, config?: ProgramConfig): Promise<ProgramOutput<z.infer<ZodObject<O>>>>;
+}
+//# sourceMappingURL=ChainOfThought.d.ts.map

package/dist/src/program/ChainOfThought.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChainOfThought.d.ts","sourceRoot":"","sources":["../../../src/program/ChainOfThought.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAErE,qBAAa,cAAc,CACzB,CAAC,SAAS,WAAW,EACrB,CAAC,SAAS,WAAW,CACrB,SAAQ,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACf,GAAG,CACP,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,GAAE,aAAkB,GACzB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAgDjD"}

package/dist/src/program/ChainOfThought.js

@@ -0,0 +1,44 @@
+import { Program } from "./Program.js";
+export class ChainOfThought extends Program {
+    async run(input, config = {}) {
+        const startTime = Date.now();
+        const validatedInput = this.schema.validateInput(input);
+        const instructions = config.instructions || this.schema.description;
+        const fewShotSection = this.buildFewShotString(config.fewShotExamples);
+        const outputKeys = this.schema.getOutputKeys();
+        const outputFormat = outputKeys
+            .map((k) => `"${k}": <value>`)
+            .join(",\n    ");
+        const prompt = `${instructions}
+
+${fewShotSection ? `Here are some examples:\n\n${fewShotSection}\n\n` : ""}Think through this step-by-step before providing your answer.
+
+Input: ${JSON.stringify(validatedInput)}
+
+Respond with valid JSON in this exact format:
+{
+  "reasoning": "Your step-by-step thinking process...",
+  ${outputFormat}
+}
+
+Respond ONLY with valid JSON, no additional text.`;
+        const response = await this.provider.complete({
+            prompt,
+            responseFormat: "json",
+        });
+        const parsed = this.parseJsonResponse(response.content);
+        const reasoning = parsed.reasoning;
+        delete parsed.reasoning;
+        const result = this.schema.validateOutput(parsed);
+        return {
+            result,
+            trace: {
+                promptUsed: prompt,
+                reasoning,
+                usage: response.usage,
+                latencyMs: Date.now() - startTime,
+            },
+        };
+    }
+}
+//# sourceMappingURL=ChainOfThought.js.map