@mzhub/promptc 0.1.0

package/src/eval/exactMatch.ts

@@ -0,0 +1,65 @@
+import type { Evaluator } from "./types.js";
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (a === null || b === null) return a === b;
+
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((item, i) => deepEqual(item, b[i]));
+  }
+
+  if (typeof a === "object" && typeof b === "object") {
+    const aObj = a as Record<string, unknown>;
+    const bObj = b as Record<string, unknown>;
+    const aKeys = Object.keys(aObj);
+    const bKeys = Object.keys(bObj);
+
+    if (aKeys.length !== bKeys.length) return false;
+    return aKeys.every((key) => deepEqual(aObj[key], bObj[key]));
+  }
+
+  return false;
+}
+
+export function exactMatch<O>(): Evaluator<O> {
+  return (prediction: O, groundTruth: O): number => {
+    return deepEqual(prediction, groundTruth) ? 1.0 : 0.0;
+  };
+}
+
+export function partialMatch<
+  O extends Record<string, unknown>
+>(): Evaluator<O> {
+  return (prediction: O, groundTruth: O): number => {
+    const keys = Object.keys(groundTruth);
+    if (keys.length === 0) return 1.0;
+
+    let matchCount = 0;
+    for (const key of keys) {
+      if (deepEqual(prediction[key], groundTruth[key])) {
+        matchCount++;
+      }
+    }
+
+    return matchCount / keys.length;
+  };
+}
+
+export function arrayOverlap<T>(): Evaluator<T[]> {
+  return (prediction: T[], groundTruth: T[]): number => {
+    if (groundTruth.length === 0) return prediction.length === 0 ? 1.0 : 0.0;
+
+    const predSet = new Set(prediction.map((x) => JSON.stringify(x)));
+    const truthSet = new Set(groundTruth.map((x) => JSON.stringify(x)));
+
+    let intersection = 0;
+    for (const item of predSet) {
+      if (truthSet.has(item)) intersection++;
+    }
+
+    const union = new Set([...predSet, ...truthSet]).size;
+    return union > 0 ? intersection / union : 0;
+  };
+}

package/src/eval/index.ts

@@ -0,0 +1,4 @@
+export type { Evaluator } from "./types.js";
+export { exactMatch, partialMatch, arrayOverlap } from "./exactMatch.js";
+export { llmJudge } from "./llmJudge.js";
+export type { LLMJudgeConfig } from "./llmJudge.js";

package/src/eval/llmJudge.ts

@@ -0,0 +1,45 @@
+import type { LLMProvider } from "../providers/types.js";
+import type { Evaluator } from "./types.js";
+
+export interface LLMJudgeConfig {
+  provider: LLMProvider;
+  criteria?: string;
+  model?: string;
+}
+
+export function llmJudge<O>(config: LLMJudgeConfig): Evaluator<O> {
+  const { provider, criteria = "accuracy and completeness", model } = config;
+
+  return async (prediction: O, groundTruth: O): Promise<number> => {
+    const prompt = `You are evaluating the quality of an AI response.
+
+EXPECTED OUTPUT:
+${JSON.stringify(groundTruth, null, 2)}
+
+ACTUAL OUTPUT:
+${JSON.stringify(prediction, null, 2)}
+
+EVALUATION CRITERIA: ${criteria}
+
+Rate the actual output on a scale from 0.0 to 1.0, where:
+- 1.0 = Perfect match or equivalent
+- 0.5 = Partially correct
+- 0.0 = Completely wrong
+
+Respond with ONLY a number between 0.0 and 1.0, nothing else.`;
+
+    const response = await provider.complete({
+      prompt,
+      model,
+      temperature: 0,
+      maxTokens: 10,
+    });
+
+    const score = parseFloat(response.content.trim());
+    if (isNaN(score) || score < 0 || score > 1) {
+      return 0;
+    }
+
+    return score;
+  };
+}

package/src/eval/types.ts

@@ -0,0 +1,4 @@
+export type Evaluator<O> = (
+  prediction: O,
+  groundTruth: O
+) => number | Promise<number>;

package/src/index.ts

@@ -0,0 +1,71 @@
+export { z } from "zod";
+
+export { Schema, defineSchema } from "./schema/index.js";
+export type { SchemaDefinition } from "./schema/index.js";
+
+export {
+  createProvider,
+  OpenAIProvider,
+  AnthropicProvider,
+  GoogleProvider,
+  OllamaProvider,
+  GroqProvider,
+  CerebrasProvider,
+} from "./providers/index.js";
+export type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+  ProviderName,
+} from "./providers/index.js";
+
+export { Program, Predict, ChainOfThought } from "./program/index.js";
+export type {
+  ProgramConfig,
+  ProgramOutput,
+  ProgramTrace,
+} from "./program/index.js";
+
+export {
+  BootstrapFewShot,
+  InstructionRewrite,
+  CandidatePool,
+  createCompiledProgram,
+  loadCompiledProgram,
+} from "./compiler/index.js";
+export type {
+  Evaluator,
+  Example,
+  CompilationResult,
+  CompileOptions,
+  CompiledProgram,
+  CompiledProgramMeta,
+  SerializedCompiledProgram,
+  InstructionRewriteOptions,
+} from "./compiler/index.js";
+
+export {
+  exactMatch,
+  partialMatch,
+  arrayOverlap,
+  llmJudge,
+} from "./eval/index.js";
+export type { LLMJudgeConfig } from "./eval/index.js";
+
+export {
+  createConcurrencyManager,
+  CostTracker,
+  estimateCost,
+  withRetry,
+  createRetryWrapper,
+  PromptCache,
+  createCache,
+} from "./runtime/index.js";
+export type {
+  ConcurrencyManager,
+  TokenUsage,
+  CostEstimate,
+  RetryOptions,
+  CacheOptions,
+} from "./runtime/index.js";

package/src/program/ChainOfThought.ts

@@ -0,0 +1,59 @@
+import { z, ZodRawShape, ZodObject } from "zod";
+import { Program, ProgramConfig, ProgramOutput } from "./Program.js";
+
+export class ChainOfThought<
+  I extends ZodRawShape,
+  O extends ZodRawShape
+> extends Program<I, O> {
+  async run(
+    input: z.infer<ZodObject<I>>,
+    config: ProgramConfig = {}
+  ): Promise<ProgramOutput<z.infer<ZodObject<O>>>> {
+    const startTime = Date.now();
+    const validatedInput = this.schema.validateInput(input);
+
+    const instructions = config.instructions || this.schema.description;
+    const fewShotSection = this.buildFewShotString(config.fewShotExamples);
+    const outputKeys = this.schema.getOutputKeys();
+    const outputFormat = outputKeys
+      .map((k) => `"${k}": <value>`)
+      .join(",\n    ");
+
+    const prompt = `${instructions}
+
+${
+  fewShotSection ? `Here are some examples:\n\n${fewShotSection}\n\n` : ""
+}Think through this step-by-step before providing your answer.
+
+Input: ${JSON.stringify(validatedInput)}
+
+Respond with valid JSON in this exact format:
+{
+  "reasoning": "Your step-by-step thinking process...",
+  ${outputFormat}
+}
+
+Respond ONLY with valid JSON, no additional text.`;
+
+    const response = await this.provider.complete({
+      prompt,
+      responseFormat: "json",
+    });
+
+    const parsed = this.parseJsonResponse(response.content);
+    const reasoning = parsed.reasoning as string | undefined;
+    delete parsed.reasoning;
+
+    const result = this.schema.validateOutput(parsed);
+
+    return {
+      result,
+      trace: {
+        promptUsed: prompt,
+        reasoning,
+        usage: response.usage,
+        latencyMs: Date.now() - startTime,
+      },
+    };
+  }
+}

package/src/program/Predict.ts

@@ -0,0 +1,47 @@
+import { z, ZodRawShape, ZodObject } from "zod";
+import { Program, ProgramConfig, ProgramOutput } from "./Program.js";
+
+export class Predict<
+  I extends ZodRawShape,
+  O extends ZodRawShape
+> extends Program<I, O> {
+  async run(
+    input: z.infer<ZodObject<I>>,
+    config: ProgramConfig = {}
+  ): Promise<ProgramOutput<z.infer<ZodObject<O>>>> {
+    const startTime = Date.now();
+    const validatedInput = this.schema.validateInput(input);
+
+    const instructions = config.instructions || this.schema.description;
+    const fewShotSection = this.buildFewShotString(config.fewShotExamples);
+    const outputFormat = this.buildOutputFormat();
+
+    const prompt = `${instructions}
+
+${
+  fewShotSection ? `Here are some examples:\n\n${fewShotSection}\n\n` : ""
+}Now process this input and respond with valid JSON matching this format:
+${outputFormat}
+
+Input: ${JSON.stringify(validatedInput)}
+
+Respond ONLY with valid JSON, no additional text.`;
+
+    const response = await this.provider.complete({
+      prompt,
+      responseFormat: "json",
+    });
+
+    const parsed = this.parseJsonResponse(response.content);
+    const result = this.schema.validateOutput(parsed);
+
+    return {
+      result,
+      trace: {
+        promptUsed: prompt,
+        usage: response.usage,
+        latencyMs: Date.now() - startTime,
+      },
+    };
+  }
+}

package/src/program/Program.ts

@@ -0,0 +1,64 @@
+import { z, ZodRawShape, ZodObject } from "zod";
+import { Schema } from "../schema/defineSchema.js";
+import type { LLMProvider } from "../providers/types.js";
+
+export interface ProgramConfig {
+  instructions?: string;
+  fewShotExamples?: Array<{
+    input: Record<string, unknown>;
+    output: Record<string, unknown>;
+  }>;
+}
+
+export interface ProgramTrace {
+  promptUsed: string;
+  reasoning?: string;
+  usage: { inputTokens: number; outputTokens: number };
+  latencyMs: number;
+}
+
+export interface ProgramOutput<O> {
+  result: O;
+  trace: ProgramTrace;
+}
+
+export abstract class Program<I extends ZodRawShape, O extends ZodRawShape> {
+  constructor(
+    protected schema: Schema<I, O>,
+    protected provider: LLMProvider
+  ) {}
+
+  abstract run(
+    input: z.infer<ZodObject<I>>,
+    config?: ProgramConfig
+  ): Promise<ProgramOutput<z.infer<ZodObject<O>>>>;
+
+  protected buildFewShotString(
+    examples: ProgramConfig["fewShotExamples"]
+  ): string {
+    if (!examples || examples.length === 0) return "";
+
+    return examples
+      .map(
+        (ex, i) =>
+          `Example ${i + 1}:\nInput: ${JSON.stringify(
+            ex.input
+          )}\nOutput: ${JSON.stringify(ex.output)}`
+      )
+      .join("\n\n");
+  }
+
+  protected buildOutputFormat(): string {
+    const keys = this.schema.getOutputKeys();
+    const format = keys.map((k) => `"${k}": <value>`).join(",\n  ");
+    return `{\n  ${format}\n}`;
+  }
+
+  protected parseJsonResponse(content: string): Record<string, unknown> {
+    const jsonMatch = content.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) {
+      throw new Error("No JSON found in response");
+    }
+    return JSON.parse(jsonMatch[0]);
+  }
+}

package/src/program/index.ts

@@ -0,0 +1,4 @@
+export { Program } from "./Program.js";
+export type { ProgramConfig, ProgramOutput, ProgramTrace } from "./Program.js";
+export { Predict } from "./Predict.js";
+export { ChainOfThought } from "./ChainOfThought.js";

package/src/providers/anthropic.ts

@@ -0,0 +1,55 @@
+import type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+} from "./types.js";
+
+export class AnthropicProvider implements LLMProvider {
+  name = "anthropic";
+  defaultModel: string;
+  private apiKey: string;
+  private baseUrl: string;
+
+  constructor(config: ProviderConfig) {
+    this.apiKey = config.apiKey || process.env.ANTHROPIC_API_KEY || "";
+    this.baseUrl = config.baseUrl || "https://api.anthropic.com/v1";
+    this.defaultModel = config.defaultModel || "claude-3-5-sonnet-20241022";
+  }
+
+  async complete(params: CompletionParams): Promise<CompletionResult> {
+    const response = await fetch(`${this.baseUrl}/messages`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": this.apiKey,
+        "anthropic-version": "2023-06-01",
+      },
+      body: JSON.stringify({
+        model: params.model || this.defaultModel,
+        max_tokens: params.maxTokens ?? 1024,
+        messages: [{ role: "user", content: params.prompt }],
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Anthropic API error: ${response.status} - ${error}`);
+    }
+
+    const data = (await response.json()) as {
+      content: Array<{ type: string; text: string }>;
+      usage: { input_tokens: number; output_tokens: number };
+    };
+
+    const textContent = data.content.find((c) => c.type === "text");
+
+    return {
+      content: textContent?.text || "",
+      usage: {
+        inputTokens: data.usage.input_tokens,
+        outputTokens: data.usage.output_tokens,
+      },
+    };
+  }
+}

package/src/providers/cerebras.ts

@@ -0,0 +1,53 @@
+import type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+} from "./types.js";
+
+export class CerebrasProvider implements LLMProvider {
+  name = "cerebras";
+  defaultModel: string;
+  private apiKey: string;
+  private baseUrl: string;
+
+  constructor(config: ProviderConfig) {
+    this.apiKey = config.apiKey || process.env.CEREBRAS_API_KEY || "";
+    this.baseUrl = config.baseUrl || "https://api.cerebras.ai/v1";
+    this.defaultModel = config.defaultModel || "llama3.1-8b";
+  }
+
+  async complete(params: CompletionParams): Promise<CompletionResult> {
+    const response = await fetch(`${this.baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: params.model || this.defaultModel,
+        messages: [{ role: "user", content: params.prompt }],
+        temperature: params.temperature ?? 0.7,
+        max_tokens: params.maxTokens ?? 1024,
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Cerebras API error: ${response.status} - ${error}`);
+    }
+
+    const data = (await response.json()) as {
+      choices: Array<{ message: { content: string } }>;
+      usage: { prompt_tokens: number; completion_tokens: number };
+    };
+
+    return {
+      content: data.choices[0].message.content,
+      usage: {
+        inputTokens: data.usage.prompt_tokens,
+        outputTokens: data.usage.completion_tokens,
+      },
+    };
+  }
+}

package/src/providers/google.ts

@@ -0,0 +1,57 @@
+import type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+} from "./types.js";
+
+export class GoogleProvider implements LLMProvider {
+  name = "google";
+  defaultModel: string;
+  private apiKey: string;
+  private baseUrl: string;
+
+  constructor(config: ProviderConfig) {
+    this.apiKey = config.apiKey || process.env.GOOGLE_API_KEY || "";
+    this.baseUrl =
+      config.baseUrl || "https://generativelanguage.googleapis.com/v1beta";
+    this.defaultModel = config.defaultModel || "gemini-2.0-flash";
+  }
+
+  async complete(params: CompletionParams): Promise<CompletionResult> {
+    const model = params.model || this.defaultModel;
+    const url = `${this.baseUrl}/models/${model}:generateContent?key=${this.apiKey}`;
+
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        contents: [{ parts: [{ text: params.prompt }] }],
+        generationConfig: {
+          temperature: params.temperature ?? 0.7,
+          maxOutputTokens: params.maxTokens ?? 1024,
+        },
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Google API error: ${response.status} - ${error}`);
+    }
+
+    const data = (await response.json()) as {
+      candidates: Array<{ content: { parts: Array<{ text: string }> } }>;
+      usageMetadata: { promptTokenCount: number; candidatesTokenCount: number };
+    };
+
+    return {
+      content: data.candidates[0]?.content?.parts[0]?.text || "",
+      usage: {
+        inputTokens: data.usageMetadata?.promptTokenCount || 0,
+        outputTokens: data.usageMetadata?.candidatesTokenCount || 0,
+      },
+    };
+  }
+}

package/src/providers/groq.ts

@@ -0,0 +1,57 @@
+import type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+} from "./types.js";
+
+export class GroqProvider implements LLMProvider {
+  name = "groq";
+  defaultModel: string;
+  private apiKey: string;
+  private baseUrl: string;
+
+  constructor(config: ProviderConfig) {
+    this.apiKey = config.apiKey || process.env.GROQ_API_KEY || "";
+    this.baseUrl = config.baseUrl || "https://api.groq.com/openai/v1";
+    this.defaultModel = config.defaultModel || "llama-3.3-70b-versatile";
+  }
+
+  async complete(params: CompletionParams): Promise<CompletionResult> {
+    const response = await fetch(`${this.baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: params.model || this.defaultModel,
+        messages: [{ role: "user", content: params.prompt }],
+        temperature: params.temperature ?? 0.7,
+        max_tokens: params.maxTokens ?? 1024,
+        response_format:
+          params.responseFormat === "json"
+            ? { type: "json_object" }
+            : undefined,
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Groq API error: ${response.status} - ${error}`);
+    }
+
+    const data = (await response.json()) as {
+      choices: Array<{ message: { content: string } }>;
+      usage: { prompt_tokens: number; completion_tokens: number };
+    };
+
+    return {
+      content: data.choices[0].message.content,
+      usage: {
+        inputTokens: data.usage.prompt_tokens,
+        outputTokens: data.usage.completion_tokens,
+      },
+    };
+  }
+}

package/src/providers/index.ts

@@ -0,0 +1,50 @@
+import { OpenAIProvider } from "./openai.js";
+import { AnthropicProvider } from "./anthropic.js";
+import { GoogleProvider } from "./google.js";
+import { OllamaProvider } from "./ollama.js";
+import { GroqProvider } from "./groq.js";
+import { CerebrasProvider } from "./cerebras.js";
+import type { LLMProvider, ProviderConfig } from "./types.js";
+
+export type ProviderName =
+  | "openai"
+  | "anthropic"
+  | "google"
+  | "ollama"
+  | "groq"
+  | "cerebras";
+
+export function createProvider(
+  name: ProviderName,
+  config: ProviderConfig = {}
+): LLMProvider {
+  switch (name) {
+    case "openai":
+      return new OpenAIProvider(config);
+    case "anthropic":
+      return new AnthropicProvider(config);
+    case "google":
+      return new GoogleProvider(config);
+    case "ollama":
+      return new OllamaProvider(config);
+    case "groq":
+      return new GroqProvider(config);
+    case "cerebras":
+      return new CerebrasProvider(config);
+    default:
+      throw new Error(`Unknown provider: ${name}`);
+  }
+}
+
+export { OpenAIProvider } from "./openai.js";
+export { AnthropicProvider } from "./anthropic.js";
+export { GoogleProvider } from "./google.js";
+export { OllamaProvider } from "./ollama.js";
+export { GroqProvider } from "./groq.js";
+export { CerebrasProvider } from "./cerebras.js";
+export type {
+  LLMProvider,
+  CompletionParams,
+  CompletionResult,
+  ProviderConfig,
+} from "./types.js";