@mzhub/promptc 0.1.0

package/src/providers/ollama.ts

@@ -0,0 +1,54 @@
+import type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+} from "./types.js";
+
+export class OllamaProvider implements LLMProvider {
+  name = "ollama";
+  defaultModel: string;
+  private baseUrl: string;
+
+  constructor(config: ProviderConfig = {}) {
+    this.baseUrl = config.baseUrl || "http://localhost:11434";
+    this.defaultModel = config.defaultModel || "llama3.2";
+  }
+
+  async complete(params: CompletionParams): Promise<CompletionResult> {
+    const response = await fetch(`${this.baseUrl}/api/generate`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model: params.model || this.defaultModel,
+        prompt: params.prompt,
+        stream: false,
+        options: {
+          temperature: params.temperature ?? 0.7,
+          num_predict: params.maxTokens ?? 1024,
+        },
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Ollama API error: ${response.status} - ${error}`);
+    }
+
+    const data = (await response.json()) as {
+      response: string;
+      prompt_eval_count?: number;
+      eval_count?: number;
+    };
+
+    return {
+      content: data.response,
+      usage: {
+        inputTokens: data.prompt_eval_count || 0,
+        outputTokens: data.eval_count || 0,
+      },
+    };
+  }
+}

package/src/providers/openai.ts

@@ -0,0 +1,57 @@
+import type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+} from "./types.js";
+
+export class OpenAIProvider implements LLMProvider {
+  name = "openai";
+  defaultModel: string;
+  private apiKey: string;
+  private baseUrl: string;
+
+  constructor(config: ProviderConfig) {
+    this.apiKey = config.apiKey || process.env.OPENAI_API_KEY || "";
+    this.baseUrl = config.baseUrl || "https://api.openai.com/v1";
+    this.defaultModel = config.defaultModel || "gpt-4o-mini";
+  }
+
+  async complete(params: CompletionParams): Promise<CompletionResult> {
+    const response = await fetch(`${this.baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: params.model || this.defaultModel,
+        messages: [{ role: "user", content: params.prompt }],
+        temperature: params.temperature ?? 0.7,
+        max_tokens: params.maxTokens ?? 1024,
+        response_format:
+          params.responseFormat === "json"
+            ? { type: "json_object" }
+            : undefined,
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+    }
+
+    const data = (await response.json()) as {
+      choices: Array<{ message: { content: string } }>;
+      usage: { prompt_tokens: number; completion_tokens: number };
+    };
+
+    return {
+      content: data.choices[0].message.content,
+      usage: {
+        inputTokens: data.usage.prompt_tokens,
+        outputTokens: data.usage.completion_tokens,
+      },
+    };
+  }
+}

package/src/providers/types.ts

@@ -0,0 +1,27 @@
+export interface CompletionParams {
+  prompt: string;
+  model?: string;
+  temperature?: number;
+  maxTokens?: number;
+  responseFormat?: "text" | "json";
+}
+
+export interface CompletionResult {
+  content: string;
+  usage: {
+    inputTokens: number;
+    outputTokens: number;
+  };
+}
+
+export interface LLMProvider {
+  name: string;
+  defaultModel: string;
+  complete(params: CompletionParams): Promise<CompletionResult>;
+}
+
+export interface ProviderConfig {
+  apiKey?: string;
+  baseUrl?: string;
+  defaultModel?: string;
+}

package/src/runtime/cache.ts

@@ -0,0 +1,65 @@
+export interface CacheOptions {
+  maxSize?: number;
+  ttlMs?: number;
+}
+
+interface CacheEntry<T> {
+  value: T;
+  timestamp: number;
+}
+
+export class PromptCache<T> {
+  private cache = new Map<string, CacheEntry<T>>();
+  private maxSize: number;
+  private ttlMs: number;
+
+  constructor(options: CacheOptions = {}) {
+    this.maxSize = options.maxSize ?? 1000;
+    this.ttlMs = options.ttlMs ?? 0;
+  }
+
+  private makeKey(prompt: string, input: unknown): string {
+    return `${prompt}::${JSON.stringify(input)}`;
+  }
+
+  get(prompt: string, input: unknown): T | undefined {
+    const key = this.makeKey(prompt, input);
+    const entry = this.cache.get(key);
+
+    if (!entry) return undefined;
+
+    if (this.ttlMs > 0 && Date.now() - entry.timestamp > this.ttlMs) {
+      this.cache.delete(key);
+      return undefined;
+    }
+
+    return entry.value;
+  }
+
+  set(prompt: string, input: unknown, value: T): void {
+    const key = this.makeKey(prompt, input);
+
+    if (this.cache.size >= this.maxSize) {
+      const firstKey = this.cache.keys().next().value;
+      if (firstKey) this.cache.delete(firstKey);
+    }
+
+    this.cache.set(key, { value, timestamp: Date.now() });
+  }
+
+  has(prompt: string, input: unknown): boolean {
+    return this.get(prompt, input) !== undefined;
+  }
+
+  clear(): void {
+    this.cache.clear();
+  }
+
+  get size(): number {
+    return this.cache.size;
+  }
+}
+
+export function createCache<T>(options: CacheOptions = {}): PromptCache<T> {
+  return new PromptCache<T>(options);
+}

package/src/runtime/concurrency.ts

@@ -0,0 +1,21 @@
+import pLimit from "p-limit";
+
+export interface ConcurrencyManager {
+  run<T>(fn: () => Promise<T>): Promise<T>;
+  get pending(): number;
+  get active(): number;
+}
+
+export function createConcurrencyManager(limit: number): ConcurrencyManager {
+  const limiter = pLimit(limit);
+
+  return {
+    run: <T>(fn: () => Promise<T>) => limiter(fn),
+    get pending() {
+      return limiter.pendingCount;
+    },
+    get active() {
+      return limiter.activeCount;
+    },
+  };
+}

package/src/runtime/costTracker.ts

@@ -0,0 +1,58 @@
+export interface TokenUsage {
+  inputTokens: number;
+  outputTokens: number;
+}
+
+export interface CostEstimate {
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+  estimatedCalls: number;
+}
+
+export class CostTracker {
+  private totalInputTokens = 0;
+  private totalOutputTokens = 0;
+  private callCount = 0;
+
+  record(usage: TokenUsage): void {
+    this.totalInputTokens += usage.inputTokens;
+    this.totalOutputTokens += usage.outputTokens;
+    this.callCount++;
+  }
+
+  get total(): TokenUsage & { totalTokens: number; calls: number } {
+    return {
+      inputTokens: this.totalInputTokens,
+      outputTokens: this.totalOutputTokens,
+      totalTokens: this.totalInputTokens + this.totalOutputTokens,
+      calls: this.callCount,
+    };
+  }
+
+  exceedsBudget(maxTokens: number): boolean {
+    return this.totalInputTokens + this.totalOutputTokens > maxTokens;
+  }
+
+  reset(): void {
+    this.totalInputTokens = 0;
+    this.totalOutputTokens = 0;
+    this.callCount = 0;
+  }
+}
+
+export function estimateCost(
+  candidateCount: number,
+  testCaseCount: number,
+  avgTokensPerCall: number = 500
+): CostEstimate {
+  const estimatedCalls = candidateCount * testCaseCount;
+  const totalTokens = estimatedCalls * avgTokensPerCall;
+
+  return {
+    inputTokens: Math.round(totalTokens * 0.7),
+    outputTokens: Math.round(totalTokens * 0.3),
+    totalTokens,
+    estimatedCalls,
+  };
+}

package/src/runtime/index.ts

@@ -0,0 +1,8 @@
+export { createConcurrencyManager } from "./concurrency.js";
+export type { ConcurrencyManager } from "./concurrency.js";
+export { CostTracker, estimateCost } from "./costTracker.js";
+export type { TokenUsage, CostEstimate } from "./costTracker.js";
+export { withRetry, createRetryWrapper } from "./retry.js";
+export type { RetryOptions } from "./retry.js";
+export { PromptCache, createCache } from "./cache.js";
+export type { CacheOptions } from "./cache.js";

package/src/runtime/retry.ts

@@ -0,0 +1,59 @@
+export interface RetryOptions {
+  maxRetries?: number;
+  initialDelayMs?: number;
+  maxDelayMs?: number;
+  backoffMultiplier?: number;
+  retryOn?: (error: Error) => boolean;
+}
+
+const defaultRetryOptions: Required<RetryOptions> = {
+  maxRetries: 3,
+  initialDelayMs: 1000,
+  maxDelayMs: 30000,
+  backoffMultiplier: 2,
+  retryOn: (error: Error) => {
+    const message = error.message.toLowerCase();
+    return (
+      message.includes("rate limit") ||
+      message.includes("429") ||
+      message.includes("503") ||
+      message.includes("timeout")
+    );
+  },
+};
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  options: RetryOptions = {}
+): Promise<T> {
+  const opts = { ...defaultRetryOptions, ...options };
+  let lastError: Error | null = null;
+  let delay = opts.initialDelayMs;
+
+  for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+
+      if (attempt === opts.maxRetries || !opts.retryOn(lastError)) {
+        throw lastError;
+      }
+
+      await sleep(delay);
+      delay = Math.min(delay * opts.backoffMultiplier, opts.maxDelayMs);
+    }
+  }
+
+  throw lastError;
+}
+
+export function createRetryWrapper(
+  options: RetryOptions = {}
+): <T>(fn: () => Promise<T>) => Promise<T> {
+  return <T>(fn: () => Promise<T>) => withRetry(fn, options);
+}

package/src/schema/defineSchema.ts

@@ -0,0 +1,44 @@
+import { z, ZodObject, ZodRawShape } from "zod";
+
+export interface SchemaDefinition<
+  I extends ZodRawShape,
+  O extends ZodRawShape
+> {
+  description: string;
+  inputs: I;
+  outputs: O;
+}
+
+export class Schema<I extends ZodRawShape, O extends ZodRawShape> {
+  public readonly inputSchema: ZodObject<I>;
+  public readonly outputSchema: ZodObject<O>;
+  public readonly description: string;
+
+  constructor(definition: SchemaDefinition<I, O>) {
+    this.description = definition.description;
+    this.inputSchema = z.object(definition.inputs);
+    this.outputSchema = z.object(definition.outputs);
+  }
+
+  validateInput(input: unknown): z.infer<ZodObject<I>> {
+    return this.inputSchema.parse(input);
+  }
+
+  validateOutput(output: unknown): z.infer<ZodObject<O>> {
+    return this.outputSchema.parse(output);
+  }
+
+  getInputKeys(): string[] {
+    return Object.keys(this.inputSchema.shape);
+  }
+
+  getOutputKeys(): string[] {
+    return Object.keys(this.outputSchema.shape);
+  }
+}
+
+export function defineSchema<I extends ZodRawShape, O extends ZodRawShape>(
+  definition: SchemaDefinition<I, O>
+): Schema<I, O> {
+  return new Schema(definition);
+}

package/src/schema/index.ts

@@ -0,0 +1,2 @@
+export { Schema, defineSchema } from "./defineSchema.js";
+export type { SchemaDefinition } from "./defineSchema.js";

package/tests/candidatePool.test.ts

@@ -0,0 +1,46 @@
+import { describe, it, expect } from "vitest";
+import { CandidatePool } from "../src/compiler/CandidatePool.js";
+
+describe("CandidatePool", () => {
+  const trainset = [
+    { input: { text: "A" }, output: { name: "Alice" } },
+    { input: { text: "B" }, output: { name: "Bob" } },
+    { input: { text: "C" }, output: { name: "Charlie" } },
+    { input: { text: "D" }, output: { name: "Diana" } },
+    { input: { text: "E" }, output: { name: "Eve" } },
+  ];
+
+  it("should generate candidates with correct count", () => {
+    const pool = new CandidatePool(trainset);
+    const candidates = pool.generateFewShotCandidates(3, 2);
+
+    expect(candidates.length).toBe(3);
+    candidates.forEach((c) => expect(c.length).toBe(2));
+  });
+
+  it("should be deterministic with seed", () => {
+    const pool1 = new CandidatePool(trainset, 42);
+    const pool2 = new CandidatePool(trainset, 42);
+
+    const candidates1 = pool1.generateFewShotCandidates(5, 3);
+    const candidates2 = pool2.generateFewShotCandidates(5, 3);
+
+    expect(candidates1).toEqual(candidates2);
+  });
+
+  it("should generate different candidates without seed", () => {
+    const pool = new CandidatePool(trainset);
+    const c1 = pool.generateFewShotCandidates(10, 2);
+    const c2 = pool.generateFewShotCandidates(10, 2);
+
+    const same = c1.every((_, i) => c1[i].every((ex, j) => ex === c2[i][j]));
+    expect(same).toBe(false);
+  });
+
+  it("should get validation set with correct ratio", () => {
+    const pool = new CandidatePool(trainset);
+    const validationSet = pool.getValidationSet(0.4);
+
+    expect(validationSet.length).toBe(2);
+  });
+});

package/tests/evaluators.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from "vitest";
+import {
+  exactMatch,
+  partialMatch,
+  arrayOverlap,
+} from "../src/eval/exactMatch.js";
+
+describe("exactMatch", () => {
+  const evaluator = exactMatch<{ name: string }>();
+
+  it("should return 1.0 for identical objects", () => {
+    const score = evaluator({ name: "Alice" }, { name: "Alice" });
+    expect(score).toBe(1.0);
+  });
+
+  it("should return 0.0 for different objects", () => {
+    const score = evaluator({ name: "Alice" }, { name: "Bob" });
+    expect(score).toBe(0.0);
+  });
+
+  it("should handle arrays", () => {
+    const arrEval = exactMatch<string[]>();
+    expect(arrEval(["a", "b"], ["a", "b"])).toBe(1.0);
+    expect(arrEval(["a", "b"], ["a", "c"])).toBe(0.0);
+  });
+});
+
+describe("partialMatch", () => {
+  const evaluator = partialMatch<{ a: number; b: number; c: number }>();
+
+  it("should return 1.0 when all fields match", () => {
+    const score = evaluator({ a: 1, b: 2, c: 3 }, { a: 1, b: 2, c: 3 });
+    expect(score).toBe(1.0);
+  });
+
+  it("should return fraction for partial matches", () => {
+    const score = evaluator({ a: 1, b: 2, c: 3 }, { a: 1, b: 2, c: 4 });
+    expect(score).toBeCloseTo(2 / 3);
+  });
+
+  it("should return 0.0 for no matches", () => {
+    const score = evaluator({ a: 9, b: 9, c: 9 }, { a: 1, b: 2, c: 3 });
+    expect(score).toBe(0.0);
+  });
+});
+
+describe("arrayOverlap", () => {
+  const evaluator = arrayOverlap<string>();
+
+  it("should return 1.0 for identical arrays", () => {
+    const score = evaluator(["a", "b", "c"], ["a", "b", "c"]);
+    expect(score).toBe(1.0);
+  });
+
+  it("should return Jaccard similarity", () => {
+    const score = evaluator(["a", "b"], ["b", "c"]);
+    expect(score).toBeCloseTo(1 / 3);
+  });
+
+  it("should return 0.0 for no overlap", () => {
+    const score = evaluator(["a", "b"], ["c", "d"]);
+    expect(score).toBe(0.0);
+  });
+
+  it("should handle empty arrays", () => {
+    expect(evaluator([], [])).toBe(1.0);
+    expect(evaluator(["a"], [])).toBe(0.0);
+  });
+});

package/tests/runtime.test.ts

@@ -0,0 +1,106 @@
+import { describe, it, expect } from "vitest";
+import { withRetry } from "../src/runtime/retry.js";
+import { PromptCache } from "../src/runtime/cache.js";
+import { CostTracker } from "../src/runtime/costTracker.js";
+
+describe("withRetry", () => {
+  it("should succeed on first try", async () => {
+    let attempts = 0;
+    const result = await withRetry(async () => {
+      attempts++;
+      return "success";
+    });
+
+    expect(result).toBe("success");
+    expect(attempts).toBe(1);
+  });
+
+  it("should retry on failure and eventually succeed", async () => {
+    let attempts = 0;
+    const result = await withRetry(
+      async () => {
+        attempts++;
+        if (attempts < 3) {
+          throw new Error("rate limit");
+        }
+        return "success";
+      },
+      { maxRetries: 3, initialDelayMs: 10 }
+    );
+
+    expect(result).toBe("success");
+    expect(attempts).toBe(3);
+  });
+
+  it("should throw after max retries", async () => {
+    let attempts = 0;
+    await expect(
+      withRetry(
+        async () => {
+          attempts++;
+          throw new Error("rate limit");
+        },
+        { maxRetries: 2, initialDelayMs: 10 }
+      )
+    ).rejects.toThrow("rate limit");
+
+    expect(attempts).toBe(3);
+  });
+});
+
+describe("PromptCache", () => {
+  it("should cache and retrieve values", () => {
+    const cache = new PromptCache<string>();
+    cache.set("prompt1", { key: "value" }, "result1");
+
+    expect(cache.get("prompt1", { key: "value" })).toBe("result1");
+    expect(cache.get("prompt1", { key: "other" })).toBeUndefined();
+  });
+
+  it("should respect maxSize", () => {
+    const cache = new PromptCache<string>({ maxSize: 2 });
+    cache.set("p1", {}, "r1");
+    cache.set("p2", {}, "r2");
+    cache.set("p3", {}, "r3");
+
+    expect(cache.size).toBe(2);
+  });
+
+  it("should check has correctly", () => {
+    const cache = new PromptCache<number>();
+    cache.set("p", {}, 42);
+
+    expect(cache.has("p", {})).toBe(true);
+    expect(cache.has("other", {})).toBe(false);
+  });
+});
+
+describe("CostTracker", () => {
+  it("should track token usage", () => {
+    const tracker = new CostTracker();
+    tracker.record({ inputTokens: 100, outputTokens: 50 });
+    tracker.record({ inputTokens: 200, outputTokens: 100 });
+
+    const total = tracker.total;
+    expect(total.inputTokens).toBe(300);
+    expect(total.outputTokens).toBe(150);
+    expect(total.totalTokens).toBe(450);
+    expect(total.calls).toBe(2);
+  });
+
+  it("should check budget", () => {
+    const tracker = new CostTracker();
+    tracker.record({ inputTokens: 400, outputTokens: 100 });
+
+    expect(tracker.exceedsBudget(1000)).toBe(false);
+    expect(tracker.exceedsBudget(400)).toBe(true);
+  });
+
+  it("should reset", () => {
+    const tracker = new CostTracker();
+    tracker.record({ inputTokens: 100, outputTokens: 50 });
+    tracker.reset();
+
+    expect(tracker.total.totalTokens).toBe(0);
+  });
+});

package/tests/schema.test.ts

@@ -0,0 +1,59 @@
+import { describe, it, expect } from "vitest";
+import { defineSchema } from "../src/schema/defineSchema.js";
+import { z } from "zod";
+
+describe("defineSchema", () => {
+  it("should create a schema with description, inputs, and outputs", () => {
+    const schema = defineSchema({
+      description: "Extract names from text",
+      inputs: { text: z.string() },
+      outputs: { names: z.array(z.string()) },
+    });
+
+    expect(schema.description).toBe("Extract names from text");
+    expect(schema.getInputKeys()).toEqual(["text"]);
+    expect(schema.getOutputKeys()).toEqual(["names"]);
+  });
+
+  it("should validate correct input", () => {
+    const schema = defineSchema({
+      description: "Test",
+      inputs: { text: z.string() },
+      outputs: { result: z.number() },
+    });
+
+    const result = schema.validateInput({ text: "hello" });
+    expect(result.text).toBe("hello");
+  });
+
+  it("should throw on invalid input", () => {
+    const schema = defineSchema({
+      description: "Test",
+      inputs: { text: z.string() },
+      outputs: { result: z.number() },
+    });
+
+    expect(() => schema.validateInput({ text: 123 })).toThrow();
+  });
+
+  it("should validate correct output", () => {
+    const schema = defineSchema({
+      description: "Test",
+      inputs: { text: z.string() },
+      outputs: { names: z.array(z.string()) },
+    });
+
+    const result = schema.validateOutput({ names: ["Alice", "Bob"] });
+    expect(result.names).toEqual(["Alice", "Bob"]);
+  });
+
+  it("should throw on invalid output", () => {
+    const schema = defineSchema({
+      description: "Test",
+      inputs: { text: z.string() },
+      outputs: { names: z.array(z.string()) },
+    });
+
+    expect(() => schema.validateOutput({ names: "not an array" })).toThrow();
+  });
+});