@mzhub/promptc 0.1.0

package/src/cli.ts

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+
+import { parseArgs } from "node:util";
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { resolve } from "node:path";
+
+const { values, positionals } = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    config: { type: "string", short: "c" },
+    output: { type: "string", short: "o", default: "compiled-prompt.json" },
+    candidates: { type: "string", default: "10" },
+    concurrency: { type: "string", default: "5" },
+    provider: { type: "string", short: "p", default: "openai" },
+    model: { type: "string", short: "m" },
+    help: { type: "boolean", short: "h" },
+  },
+  allowPositionals: true,
+});
+
+function printHelp() {
+  console.log(`
+promptc - Type-safe LLM program compiler
+
+Usage:
+  promptc compile <trainset.json> [options]
+  promptc validate <compiled.json>
+
+Commands:
+  compile     Compile a program using a training set
+  validate    Validate a compiled prompt JSON file
+
+Options:
+  -c, --config <file>      Path to config file
+  -o, --output <file>      Output file (default: compiled-prompt.json)
+  -p, --provider <name>    LLM provider: openai, anthropic, google, groq, cerebras, ollama
+  -m, --model <name>       Model to use (overrides provider default)
+  --candidates <n>         Number of candidates to try (default: 10)
+  --concurrency <n>        Parallel evaluations (default: 5)
+  -h, --help               Show this help
+
+Examples:
+  promptc compile trainset.json -p openai -o prompt.json
+  promptc validate prompt.json
+`);
+}
+
+async function validateCommand(filePath: string) {
+  if (!existsSync(filePath)) {
+    console.error(`Error: File not found: ${filePath}`);
+    process.exit(1);
+  }
+
+  try {
+    const content = readFileSync(filePath, "utf-8");
+    const data = JSON.parse(content);
+
+    if (!data.meta || !data.config) {
+      console.error("Invalid format: missing 'meta' or 'config' fields");
+      process.exit(1);
+    }
+
+    console.log("✓ Valid compiled prompt file");
+    console.log(`  Score: ${data.meta.score}`);
+    console.log(`  Strategy: ${data.meta.strategy}`);
+    console.log(`  Compiled: ${data.meta.compiledAt}`);
+    console.log(`  Examples: ${data.config.fewShotExamples?.length || 0}`);
+    console.log(`  Tokens used: ${data.meta.tokenUsage?.totalTokens || "N/A"}`);
+  } catch (error) {
+    console.error("Invalid JSON file");
+    process.exit(1);
+  }
+}
+
+async function main() {
+  if (values.help || positionals.length === 0) {
+    printHelp();
+    process.exit(0);
+  }
+
+  const command = positionals[0];
+
+  switch (command) {
+    case "validate":
+      if (!positionals[1]) {
+        console.error("Error: Please provide a file to validate");
+        process.exit(1);
+      }
+      await validateCommand(resolve(positionals[1]));
+      break;
+
+    case "compile":
+      console.log(`
+Note: The 'compile' command requires a custom config file that defines:
+- Your schema (using defineSchema)
+- Your program (Predict or ChainOfThought)
+- Your evaluator
+
+Example config file (promptc.config.ts):
+
+  import { defineSchema, ChainOfThought, createProvider, exactMatch, z } from 'promptc';
+  
+  export const schema = defineSchema({...});
+  export const provider = createProvider('openai', { apiKey: process.env.OPENAI_API_KEY });
+  export const program = new ChainOfThought(schema, provider);
+  export const evaluator = exactMatch();
+
+Run with: promptc compile trainset.json -c promptc.config.ts
+`);
+      break;
+
+    default:
+      console.error(`Unknown command: ${command}`);
+      printHelp();
+      process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error("Error:", error.message);
+  process.exit(1);
+});

package/src/compiler/BootstrapFewShot.ts

@@ -0,0 +1,149 @@
+import { ZodRawShape } from "zod";
+import { Program } from "../program/Program.js";
+import { createConcurrencyManager } from "../runtime/concurrency.js";
+import { CostTracker } from "../runtime/costTracker.js";
+import { CandidatePool } from "./CandidatePool.js";
+import type {
+  Evaluator,
+  Example,
+  CompilationResult,
+  CompileOptions,
+} from "./types.js";
+
+export class BootstrapFewShot<O> {
+  constructor(private evaluator: Evaluator<O>) {}
+
+  async compile<I extends ZodRawShape, OShape extends ZodRawShape>(
+    program: Program<I, OShape>,
+    trainset: Example<unknown, O>[],
+    options: CompileOptions = {}
+  ): Promise<CompilationResult<unknown, O>> {
+    const {
+      candidates: candidateCount = 10,
+      concurrency = 5,
+      examplesPerCandidate = 3,
+      validationSplit = 0.3,
+      seed,
+      earlyStopThreshold = 0,
+      budget,
+      onProgress,
+    } = options;
+
+    const pool = new CandidatePool(trainset, seed);
+    const candidates = pool.generateFewShotCandidates(
+      candidateCount,
+      examplesPerCandidate
+    );
+    const validationSet = pool.getValidationSet(validationSplit);
+    const limiter = createConcurrencyManager(concurrency);
+    const costTracker = new CostTracker();
+
+    const results: Array<{ score: number; examples: Example<unknown, O>[] }> =
+      [];
+
+    const evaluationPromises = candidates.map((fewShotSet) =>
+      limiter.run(async () => {
+        if (budget?.maxTokens && costTracker.exceedsBudget(budget.maxTokens)) {
+          budget.onBudgetWarning?.(
+            costTracker.total.totalTokens,
+            budget.maxTokens
+          );
+          return { score: -1, examples: fewShotSet };
+        }
+
+        let totalScore = 0;
+        let validCount = 0;
+
+        for (const testCase of validationSet) {
+          try {
+            const prediction = await program.run(testCase.input as any, {
+              fewShotExamples: fewShotSet.map((ex) => ({
+                input: ex.input as Record<string, unknown>,
+                output: ex.output as Record<string, unknown>,
+              })),
+            });
+
+            costTracker.record(prediction.trace.usage);
+            const score = await this.evaluator(
+              prediction.result as O,
+              testCase.output
+            );
+            totalScore += score;
+            validCount++;
+
+            if (earlyStopThreshold > 0 && validCount >= 2) {
+              const avgSoFar = totalScore / validCount;
+              if (avgSoFar < earlyStopThreshold) {
+                break;
+              }
+            }
+          } catch {
+            continue;
+          }
+        }
+
+        return {
+          score: validCount > 0 ? totalScore / validCount : 0,
+          examples: fewShotSet,
+        };
+      })
+    );
+
+    const candidateResults = await Promise.all(evaluationPromises);
+
+    let currentBestScore = -1;
+    for (const result of candidateResults) {
+      if (result.score >= 0) {
+        results.push(result);
+        if (result.score > currentBestScore) {
+          currentBestScore = result.score;
+        }
+        onProgress?.({
+          candidatesEvaluated: results.length,
+          totalCandidates: candidateCount,
+          currentBestScore,
+          tokensUsed: costTracker.total.totalTokens,
+        });
+      }
+    }
+
+    const winner = results.reduce(
+      (best, current) => (current.score > best.score ? current : best),
+      { score: -1, examples: [] as Example<unknown, O>[] }
+    );
+
+    return {
+      meta: {
+        score: winner.score,
+        compiledAt: new Date().toISOString(),
+        strategy: "BootstrapFewShot",
+        tokenUsage: costTracker.total,
+      },
+      config: {
+        instructions: program["schema"].description,
+        fewShotExamples: winner.examples,
+      },
+    };
+  }
+
+  estimateCost(
+    trainsetSize: number,
+    options: CompileOptions = {}
+  ): {
+    estimatedCalls: number;
+    estimatedTokens: number;
+  } {
+    const { candidates = 10, validationSplit = 0.3 } = options;
+    const validationSize = Math.max(
+      1,
+      Math.floor(trainsetSize * validationSplit)
+    );
+    const estimatedCalls = candidates * validationSize;
+    const avgTokensPerCall = 500;
+
+    return {
+      estimatedCalls,
+      estimatedTokens: estimatedCalls * avgTokensPerCall,
+    };
+  }
+}

package/src/compiler/CandidatePool.ts

@@ -0,0 +1,39 @@
+import type { Example } from "./types.js";
+
+export class CandidatePool<I, O> {
+  private rng: () => number;
+
+  constructor(private trainset: Example<I, O>[], seed?: number) {
+    this.rng = seed !== undefined ? this.seededRandom(seed) : Math.random;
+  }
+
+  generateFewShotCandidates(
+    count: number,
+    examplesPerCandidate: number
+  ): Array<Example<I, O>[]> {
+    const candidates: Array<Example<I, O>[]> = [];
+
+    for (let i = 0; i < count; i++) {
+      const shuffled = [...this.trainset].sort(() => this.rng() - 0.5);
+      candidates.push(
+        shuffled.slice(0, Math.min(examplesPerCandidate, shuffled.length))
+      );
+    }
+
+    return candidates;
+  }
+
+  getValidationSet(splitRatio: number = 0.3): Example<I, O>[] {
+    const count = Math.max(1, Math.floor(this.trainset.length * splitRatio));
+    const shuffled = [...this.trainset].sort(() => this.rng() - 0.5);
+    return shuffled.slice(0, count);
+  }
+
+  private seededRandom(seed: number): () => number {
+    let s = seed;
+    return () => {
+      s = Math.sin(s) * 10000;
+      return s - Math.floor(s);
+    };
+  }
+}

package/src/compiler/CompiledProgram.ts

@@ -0,0 +1,112 @@
+import { ZodRawShape, ZodObject } from "zod";
+import type { ProgramConfig, ProgramOutput } from "../program/Program.js";
+import type { Example } from "./types.js";
+
+export interface CompiledProgramMeta {
+  score: number;
+  compiledAt: string;
+  strategy: string;
+  tokenUsage: {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    calls: number;
+  };
+}
+
+export interface CompiledProgram<I, O> {
+  run(input: I, overrides?: Partial<ProgramConfig>): Promise<ProgramOutput<O>>;
+  meta: CompiledProgramMeta;
+  config: {
+    instructions: string;
+    fewShotExamples: Array<Example<unknown, unknown>>;
+  };
+  toJSON(): string;
+}
+
+export function createCompiledProgram<
+  I extends ZodRawShape,
+  O extends ZodRawShape,
+  InputType = unknown,
+  OutputType = unknown
+>(
+  originalProgram: {
+    run(
+      input: InputType,
+      config?: ProgramConfig
+    ): Promise<ProgramOutput<OutputType>>;
+  },
+  compilationResult: {
+    meta: CompiledProgramMeta;
+    config: {
+      instructions: string;
+      fewShotExamples: Array<Example<unknown, unknown>>;
+    };
+  }
+): CompiledProgram<InputType, OutputType> {
+  return {
+    async run(
+      input: InputType,
+      overrides?: Partial<ProgramConfig>
+    ): Promise<ProgramOutput<OutputType>> {
+      return originalProgram.run(input, {
+        instructions:
+          overrides?.instructions ?? compilationResult.config.instructions,
+        fewShotExamples:
+          overrides?.fewShotExamples ??
+          compilationResult.config.fewShotExamples.map((ex) => ({
+            input: ex.input as Record<string, unknown>,
+            output: ex.output as Record<string, unknown>,
+          })),
+      });
+    },
+    meta: compilationResult.meta,
+    config: compilationResult.config,
+    toJSON(): string {
+      return JSON.stringify(
+        {
+          meta: compilationResult.meta,
+          config: compilationResult.config,
+        },
+        null,
+        2
+      );
+    },
+  };
+}
+
+export interface SerializedCompiledProgram {
+  meta: CompiledProgramMeta;
+  config: {
+    instructions: string;
+    fewShotExamples: Array<Example<unknown, unknown>>;
+  };
+}
+
+export function loadCompiledProgram<InputType = unknown, OutputType = unknown>(
+  jsonOrPath: string | SerializedCompiledProgram,
+  program: {
+    run(
+      input: InputType,
+      config?: ProgramConfig
+    ): Promise<ProgramOutput<OutputType>>;
+  }
+): CompiledProgram<InputType, OutputType> {
+  let data: SerializedCompiledProgram;
+
+  if (typeof jsonOrPath === "string") {
+    try {
+      data = JSON.parse(jsonOrPath);
+    } catch {
+      throw new Error("Invalid JSON string provided to loadCompiledProgram");
+    }
+  } else {
+    data = jsonOrPath;
+  }
+
+  if (!data.meta || !data.config) {
+    throw new Error("Invalid compiled program format: missing meta or config");
+  }
+
+  return createCompiledProgram(program, data);
+}

package/src/compiler/InstructionRewrite.ts

@@ -0,0 +1,200 @@
+import { ZodRawShape } from "zod";
+import { Program } from "../program/Program.js";
+import { createConcurrencyManager } from "../runtime/concurrency.js";
+import { CostTracker } from "../runtime/costTracker.js";
+import { CandidatePool } from "./CandidatePool.js";
+import type { LLMProvider } from "../providers/types.js";
+import type {
+  Evaluator,
+  Example,
+  CompilationResult,
+  CompileOptions,
+} from "./types.js";
+
+export interface InstructionRewriteOptions extends CompileOptions {
+  instructionVariations?: number;
+  provider: LLMProvider;
+}
+
+export class InstructionRewrite<O> {
+  constructor(private evaluator: Evaluator<O>) {}
+
+  async compile<I extends ZodRawShape, OShape extends ZodRawShape>(
+    program: Program<I, OShape>,
+    trainset: Example<unknown, O>[],
+    options: InstructionRewriteOptions
+  ): Promise<CompilationResult<unknown, O>> {
+    const {
+      candidates: candidateCount = 5,
+      instructionVariations = 5,
+      concurrency = 5,
+      examplesPerCandidate = 3,
+      validationSplit = 0.3,
+      seed,
+      budget,
+      provider,
+    } = options;
+
+    const baseDescription = program["schema"].description;
+    const pool = new CandidatePool(trainset, seed);
+    const validationSet = pool.getValidationSet(validationSplit);
+    const limiter = createConcurrencyManager(concurrency);
+    const costTracker = new CostTracker();
+
+    const instructionVariants = await this.generateInstructionVariants(
+      provider,
+      baseDescription,
+      instructionVariations
+    );
+
+    const fewShotCandidates = pool.generateFewShotCandidates(
+      candidateCount,
+      examplesPerCandidate
+    );
+
+    const allCandidates: Array<{
+      instructions: string;
+      examples: Example<unknown, O>[];
+    }> = [];
+    for (const instruction of instructionVariants) {
+      for (const examples of fewShotCandidates) {
+        allCandidates.push({ instructions: instruction, examples });
+      }
+    }
+
+    const results: Array<{
+      score: number;
+      instructions: string;
+      examples: Example<unknown, O>[];
+    }> = [];
+
+    const evaluationPromises = allCandidates.map((candidate) =>
+      limiter.run(async () => {
+        if (budget?.maxTokens && costTracker.exceedsBudget(budget.maxTokens)) {
+          budget.onBudgetWarning?.(
+            costTracker.total.totalTokens,
+            budget.maxTokens
+          );
+          return { score: -1, ...candidate };
+        }
+
+        let totalScore = 0;
+        let validCount = 0;
+
+        for (const testCase of validationSet) {
+          try {
+            const prediction = await program.run(testCase.input as any, {
+              instructions: candidate.instructions,
+              fewShotExamples: candidate.examples.map((ex) => ({
+                input: ex.input as Record<string, unknown>,
+                output: ex.output as Record<string, unknown>,
+              })),
+            });
+
+            costTracker.record(prediction.trace.usage);
+            const score = await this.evaluator(
+              prediction.result as O,
+              testCase.output
+            );
+            totalScore += score;
+            validCount++;
+          } catch {
+            continue;
+          }
+        }
+
+        return {
+          score: validCount > 0 ? totalScore / validCount : 0,
+          ...candidate,
+        };
+      })
+    );
+
+    const candidateResults = await Promise.all(evaluationPromises);
+    results.push(...candidateResults.filter((r) => r.score >= 0));
+
+    const winner = results.reduce(
+      (best, current) => (current.score > best.score ? current : best),
+      {
+        score: -1,
+        instructions: baseDescription,
+        examples: [] as Example<unknown, O>[],
+      }
+    );
+
+    return {
+      meta: {
+        score: winner.score,
+        compiledAt: new Date().toISOString(),
+        strategy: "InstructionRewrite",
+        tokenUsage: costTracker.total,
+      },
+      config: {
+        instructions: winner.instructions,
+        fewShotExamples: winner.examples,
+      },
+    };
+  }
+
+  private async generateInstructionVariants(
+    provider: LLMProvider,
+    baseInstruction: string,
+    count: number
+  ): Promise<string[]> {
+    const prompt = `You are an expert prompt engineer. Given a base instruction for an LLM task, generate ${count} different variations that might perform better.
+
+Base instruction: "${baseInstruction}"
+
+Requirements:
+- Each variation should convey the same task but with different wording, tone, or structure
+- Try variations like: more specific, more concise, role-based ("You are an expert..."), step-by-step, formal, casual
+- Output as a JSON array of strings
+
+Respond with ONLY a JSON array, no additional text:
+["variation 1", "variation 2", ...]`;
+
+    const response = await provider.complete({
+      prompt,
+      temperature: 0.8,
+      maxTokens: 1024,
+      responseFormat: "json",
+    });
+
+    try {
+      const jsonMatch = response.content.match(/\[[\s\S]*\]/);
+      if (!jsonMatch) {
+        return [baseInstruction];
+      }
+      const variations = JSON.parse(jsonMatch[0]) as string[];
+      return [baseInstruction, ...variations.slice(0, count)];
+    } catch {
+      return [baseInstruction];
+    }
+  }
+
+  estimateCost(
+    trainsetSize: number,
+    options: Partial<InstructionRewriteOptions> = {}
+  ): {
+    estimatedCalls: number;
+    estimatedTokens: number;
+  } {
+    const {
+      candidates = 5,
+      instructionVariations = 5,
+      validationSplit = 0.3,
+    } = options;
+    const validationSize = Math.max(
+      1,
+      Math.floor(trainsetSize * validationSplit)
+    );
+    const totalCandidates = candidates * instructionVariations;
+    const estimatedCalls = totalCandidates * validationSize + 1;
+    const avgTokensPerCall = 500;
+
+    return {
+      estimatedCalls,
+      estimatedTokens: estimatedCalls * avgTokensPerCall,
+    };
+  }
+}

package/src/compiler/index.ts

@@ -0,0 +1,19 @@
+export { BootstrapFewShot } from "./BootstrapFewShot.js";
+export { InstructionRewrite } from "./InstructionRewrite.js";
+export type { InstructionRewriteOptions } from "./InstructionRewrite.js";
+export { CandidatePool } from "./CandidatePool.js";
+export {
+  createCompiledProgram,
+  loadCompiledProgram,
+} from "./CompiledProgram.js";
+export type {
+  CompiledProgram,
+  CompiledProgramMeta,
+  SerializedCompiledProgram,
+} from "./CompiledProgram.js";
+export type {
+  Evaluator,
+  Example,
+  CompilationResult,
+  CompileOptions,
+} from "./types.js";

package/src/compiler/types.ts

@@ -0,0 +1,46 @@
+export type Evaluator<O> = (
+  prediction: O,
+  groundTruth: O
+) => number | Promise<number>;
+
+export interface Example<I, O> {
+  input: I;
+  output: O;
+}
+
+export interface CompilationResult<I, O> {
+  meta: {
+    score: number;
+    compiledAt: string;
+    strategy: string;
+    tokenUsage: {
+      inputTokens: number;
+      outputTokens: number;
+      totalTokens: number;
+      calls: number;
+    };
+  };
+  config: {
+    instructions: string;
+    fewShotExamples: Array<Example<I, O>>;
+  };
+}
+
+export interface CompileOptions {
+  candidates?: number;
+  concurrency?: number;
+  examplesPerCandidate?: number;
+  validationSplit?: number;
+  seed?: number;
+  earlyStopThreshold?: number;
+  budget?: {
+    maxTokens?: number;
+    onBudgetWarning?: (used: number, max: number) => void;
+  };
+  onProgress?: (progress: {
+    candidatesEvaluated: number;
+    totalCandidates: number;
+    currentBestScore: number;
+    tokensUsed: number;
+  }) => void;
+}