@elsium-ai/testing 0.1.6

package/dist/index.js

@@ -0,0 +1,1263 @@
+// @bun
+// ../core/src/errors.ts
+class ElsiumError extends Error {
+  code;
+  provider;
+  model;
+  statusCode;
+  retryable;
+  retryAfterMs;
+  cause;
+  metadata;
+  constructor(details) {
+    super(details.message);
+    this.name = "ElsiumError";
+    this.code = details.code;
+    this.provider = details.provider;
+    this.model = details.model;
+    this.statusCode = details.statusCode;
+    this.retryable = details.retryable;
+    this.retryAfterMs = details.retryAfterMs;
+    this.cause = details.cause;
+    this.metadata = details.metadata;
+  }
+  toJSON() {
+    return {
+      name: this.name,
+      code: this.code,
+      message: this.message,
+      provider: this.provider,
+      model: this.model,
+      statusCode: this.statusCode,
+      retryable: this.retryable,
+      retryAfterMs: this.retryAfterMs,
+      metadata: this.metadata
+    };
+  }
+  static providerError(message, opts) {
+    return new ElsiumError({
+      code: "PROVIDER_ERROR",
+      message,
+      provider: opts.provider,
+      statusCode: opts.statusCode,
+      retryable: opts.retryable ?? false,
+      cause: opts.cause
+    });
+  }
+  static rateLimit(provider, retryAfterMs) {
+    return new ElsiumError({
+      code: "RATE_LIMIT",
+      message: `Rate limited by ${provider}`,
+      provider,
+      statusCode: 429,
+      retryable: true,
+      retryAfterMs
+    });
+  }
+  static authError(provider) {
+    return new ElsiumError({
+      code: "AUTH_ERROR",
+      message: `Authentication failed for ${provider}. Check your API key.`,
+      provider,
+      statusCode: 401,
+      retryable: false
+    });
+  }
+  static timeout(provider, timeoutMs) {
+    return new ElsiumError({
+      code: "TIMEOUT",
+      message: `Request to ${provider} timed out after ${timeoutMs}ms`,
+      provider,
+      retryable: true
+    });
+  }
+  static validation(message, metadata) {
+    return new ElsiumError({
+      code: "VALIDATION_ERROR",
+      message,
+      retryable: false,
+      metadata
+    });
+  }
+  static budgetExceeded(spent, budget) {
+    return new ElsiumError({
+      code: "BUDGET_EXCEEDED",
+      message: `Token budget exceeded: spent ${spent}, budget ${budget}`,
+      retryable: false,
+      metadata: { spent, budget }
+    });
+  }
+}
+// ../core/src/utils.ts
+import { randomBytes } from "crypto";
+function cryptoHex(bytes) {
+  return randomBytes(bytes).toString("hex");
+}
+function generateId(prefix = "els") {
+  const timestamp = Date.now().toString(36);
+  const random = cryptoHex(4);
+  return `${prefix}_${timestamp}_${random}`;
+}
+function generateTraceId() {
+  const timestamp = Date.now().toString(36);
+  const random = cryptoHex(6);
+  return `trc_${timestamp}_${random}`;
+}
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// ../core/src/stream.ts
+function shouldEmitCheckpoint(lastCheckpointTime, intervalMs, textLength) {
+  const elapsed = Date.now() - lastCheckpointTime;
+  return elapsed >= intervalMs && textLength > 0;
+}
+function createCheckpoint(textAccumulator, eventIndex, now) {
+  return {
+    id: generateId("ckpt"),
+    timestamp: now,
+    text: textAccumulator,
+    tokensSoFar: Math.ceil(textAccumulator.length / 1.5),
+    eventIndex
+  };
+}
+function toError(err) {
+  return err instanceof Error ? err : new Error(String(err));
+}
+function* emitErrorEvent(err, textAccumulator, onPartialRecovery) {
+  const error = toError(err);
+  if (textAccumulator.length > 0) {
+    onPartialRecovery?.(textAccumulator, error);
+    yield { type: "recovery", partialText: textAccumulator, error };
+  } else {
+    yield { type: "error", error };
+  }
+}
+
+class ElsiumStream {
+  source;
+  iterating = false;
+  constructor(source) {
+    this.source = source;
+  }
+  async* [Symbol.asyncIterator]() {
+    if (this.iterating) {
+      throw new Error("ElsiumStream supports only a single consumer");
+    }
+    this.iterating = true;
+    yield* this.source;
+  }
+  text() {
+    const source = this.source;
+    return {
+      async* [Symbol.asyncIterator]() {
+        for await (const event of source) {
+          if (event.type === "text_delta") {
+            yield event.text;
+          }
+        }
+      }
+    };
+  }
+  async toText() {
+    const parts = [];
+    for await (const text of this.text()) {
+      parts.push(text);
+    }
+    return parts.join("");
+  }
+  async toTextWithTimeout(timeoutMs) {
+    const parts = [];
+    const deadline = Date.now() + timeoutMs;
+    const iterator = this.source[Symbol.asyncIterator]();
+    try {
+      while (true) {
+        const remaining = deadline - Date.now();
+        if (remaining <= 0)
+          break;
+        let timer;
+        const timeoutPromise = new Promise((resolve) => {
+          timer = setTimeout(() => resolve({ value: undefined, done: true }), remaining);
+        });
+        const result = await Promise.race([iterator.next(), timeoutPromise]);
+        if (timer !== undefined)
+          clearTimeout(timer);
+        if (result.done)
+          break;
+        const event = result.value;
+        if (event.type === "text_delta") {
+          parts.push(event.text);
+        }
+      }
+    } catch (err) {
+      if (parts.length === 0)
+        throw err;
+    } finally {
+      await iterator.return?.();
+    }
+    return parts.join("");
+  }
+  async toResponse() {
+    const parts = [];
+    let usage = null;
+    let stopReason = null;
+    for await (const event of this.source) {
+      switch (event.type) {
+        case "text_delta":
+          parts.push(event.text);
+          break;
+        case "message_end":
+          usage = event.usage;
+          stopReason = event.stopReason;
+          break;
+      }
+    }
+    return { text: parts.join(""), usage, stopReason };
+  }
+  pipe(transform) {
+    return new ElsiumStream(transform(this.source));
+  }
+  resilient(options = {}) {
+    const { checkpointIntervalMs = 1000, onCheckpoint, onPartialRecovery } = options;
+    const source = this.source;
+    const resilientSource = {
+      async* [Symbol.asyncIterator]() {
+        let lastCheckpointTime = Date.now();
+        let textAccumulator = "";
+        let eventIndex = 0;
+        try {
+          for await (const event of source) {
+            eventIndex++;
+            if (event.type === "text_delta") {
+              textAccumulator += event.text;
+            }
+            yield event;
+            if (shouldEmitCheckpoint(lastCheckpointTime, checkpointIntervalMs, textAccumulator.length)) {
+              const now = Date.now();
+              const checkpoint = createCheckpoint(textAccumulator, eventIndex, now);
+              onCheckpoint?.(checkpoint);
+              yield { type: "checkpoint", checkpoint };
+              lastCheckpointTime = now;
+            }
+          }
+        } catch (err) {
+          yield* emitErrorEvent(err, textAccumulator, onPartialRecovery);
+        }
+      }
+    };
+    return new ElsiumStream(resilientSource);
+  }
+}
+var MAX_BUFFER_SIZE = 1e4;
+function createStream(executor) {
+  let resolve = null;
+  const buffer = [];
+  let done = false;
+  let error = null;
+  let dropped = 0;
+  const source = {
+    [Symbol.asyncIterator]() {
+      return {
+        next() {
+          if (buffer.length > 0) {
+            const value = buffer.shift();
+            return Promise.resolve({ value, done: false });
+          }
+          if (done) {
+            return Promise.resolve({ value: undefined, done: true });
+          }
+          if (error) {
+            return Promise.reject(error);
+          }
+          return new Promise((r) => {
+            resolve = r;
+          });
+        }
+      };
+    }
+  };
+  const emit = (event) => {
+    if (resolve) {
+      const r = resolve;
+      resolve = null;
+      r({ value: event, done: false });
+    } else {
+      if (buffer.length < MAX_BUFFER_SIZE) {
+        buffer.push(event);
+      } else {
+        dropped++;
+      }
+    }
+  };
+  executor(emit).then(() => {
+    if (dropped > 0) {
+      emit({
+        type: "error",
+        error: new Error(`Stream buffer overflow: ${dropped} events dropped`)
+      });
+    }
+    done = true;
+    if (resolve) {
+      const r = resolve;
+      resolve = null;
+      r({ value: undefined, done: true });
+    }
+  }).catch((e) => {
+    error = e instanceof Error ? e : new Error(String(e));
+    if (resolve) {
+      resolve({ value: { type: "error", error }, done: false });
+      resolve = null;
+    }
+  });
+  return new ElsiumStream(source);
+}
+// src/mock-provider.ts
+function mockProvider(options = {}) {
+  const { responses = [], defaultResponse, onRequest } = options;
+  const calls = [];
+  let callIndex = 0;
+  function getNextResponse() {
+    if (callIndex < responses.length) {
+      return responses[callIndex++];
+    }
+    if (defaultResponse) {
+      callIndex++;
+      return defaultResponse;
+    }
+    callIndex++;
+    return { content: "" };
+  }
+  async function emitStreamEvents(emit, config) {
+    if (config.delay) {
+      await new Promise((r) => setTimeout(r, config.delay));
+    }
+    emit({
+      type: "message_start",
+      id: generateId("msg"),
+      model: config.model ?? "mock-model"
+    });
+    const content = config.content ?? "";
+    if (content) {
+      const words = content.split(" ");
+      for (const word of words) {
+        emit({ type: "text_delta", text: `${word} ` });
+      }
+    }
+    emit({
+      type: "message_end",
+      usage: {
+        inputTokens: config.usage?.inputTokens ?? 10,
+        outputTokens: config.usage?.outputTokens ?? 5,
+        totalTokens: config.usage?.totalTokens ?? 15
+      },
+      stopReason: config.stopReason ?? "end_turn"
+    });
+  }
+  function buildResponse(config, request) {
+    const model = config.model ?? request.model ?? "mock-model";
+    const content = config.content ?? "";
+    const toolCalls = config.toolCalls?.map((tc) => ({
+      id: tc.id ?? generateId("tc"),
+      name: tc.name,
+      arguments: tc.arguments
+    }));
+    const usage = {
+      inputTokens: config.usage?.inputTokens ?? Math.ceil(content.length / 4),
+      outputTokens: config.usage?.outputTokens ?? Math.ceil(content.length / 4),
+      totalTokens: 0,
+      ...config.usage
+    };
+    usage.totalTokens = usage.inputTokens + usage.outputTokens;
+    const message = {
+      role: "assistant",
+      content,
+      ...toolCalls?.length ? { toolCalls } : {}
+    };
+    return {
+      id: generateId("msg"),
+      message,
+      usage,
+      cost: { inputCost: 0, outputCost: 0, totalCost: 0, currency: "USD" },
+      model,
+      provider: "mock",
+      stopReason: config.stopReason ?? (toolCalls?.length ? "tool_use" : "end_turn"),
+      latencyMs: config.delay ?? 0,
+      traceId: generateTraceId()
+    };
+  }
+  return {
+    name: "mock",
+    defaultModel: "mock-model",
+    get calls() {
+      return calls;
+    },
+    get callCount() {
+      return calls.length;
+    },
+    async complete(request) {
+      calls.push(request);
+      onRequest?.(request);
+      const config = getNextResponse();
+      if (config.delay) {
+        await new Promise((r) => setTimeout(r, config.delay));
+      }
+      return buildResponse(config, request);
+    },
+    stream(request) {
+      calls.push(request);
+      onRequest?.(request);
+      const config = getNextResponse();
+      return createStream((emit) => emitStreamEvents(emit, config));
+    },
+    async listModels() {
+      return ["mock-model"];
+    },
+    reset() {
+      calls.length = 0;
+      callIndex = 0;
+    }
+  };
+}
+// src/fixtures.ts
+import { createHash } from "crypto";
+function hashMessages(messages) {
+  const content = messages.map((m) => `${m.role}:${m.content}`).join("|");
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function createFixture(name, entries) {
+  return {
+    name,
+    entries,
+    toProvider(options) {
+      if (options?.matching === "request-hash") {
+        const hashMap = new Map;
+        for (const entry of entries) {
+          const hash = hashMessages(entry.request.messages);
+          hashMap.set(hash, entry.response);
+        }
+        const provider = mockProvider({
+          responses: entries.map((e) => e.response)
+        });
+        const originalComplete = provider.complete.bind(provider);
+        const wrapped = Object.create(provider);
+        wrapped.complete = async (request) => {
+          const reqMessages = request.messages.map((m) => ({
+            role: m.role,
+            content: typeof m.content === "string" ? m.content : "[complex]"
+          }));
+          const hash = hashMessages(reqMessages);
+          const matched = hashMap.get(hash);
+          if (matched) {
+            return mockProvider({ responses: [matched] }).complete(request);
+          }
+          return originalComplete(request);
+        };
+        return wrapped;
+      }
+      return mockProvider({
+        responses: entries.map((e) => e.response)
+      });
+    },
+    toJSON() {
+      return JSON.stringify({
+        name,
+        entries: entries.map((e) => ({
+          ...e,
+          timestamp: e.timestamp ?? new Date().toISOString()
+        }))
+      }, null, 2);
+    }
+  };
+}
+function loadFixture(json) {
+  const data = JSON.parse(json);
+  return createFixture(data.name, data.entries);
+}
+function createRecorder() {
+  const entries = [];
+  return {
+    wrap(provider) {
+      const originalComplete = provider.complete.bind(provider);
+      const wrapped = Object.create(provider);
+      wrapped.complete = async (request) => {
+        const response = await originalComplete(request);
+        entries.push({
+          request: {
+            messages: request.messages.map((m) => ({
+              role: m.role,
+              content: typeof m.content === "string" ? m.content : "[complex]"
+            })),
+            model: request.model,
+            system: request.system
+          },
+          response: {
+            content: typeof response.message.content === "string" ? response.message.content : "",
+            toolCalls: response.message.toolCalls,
+            stopReason: response.stopReason,
+            usage: response.usage,
+            model: response.model
+          },
+          timestamp: new Date().toISOString()
+        });
+        return response;
+      };
+      return wrapped;
+    },
+    getEntries() {
+      return [...entries];
+    },
+    toFixture(name) {
+      return createFixture(name, [...entries]);
+    },
+    clear() {
+      entries.length = 0;
+    }
+  };
+}
+// src/eval.ts
+function evaluateContains(output, criterion) {
+  const target = criterion.caseSensitive ? criterion.value : criterion.value.toLowerCase();
+  const haystack = criterion.caseSensitive ? output : output.toLowerCase();
+  const passed = haystack.includes(target);
+  return {
+    type: "contains",
+    passed,
+    message: passed ? `Contains "${criterion.value}"` : `Does not contain "${criterion.value}"`
+  };
+}
+function evaluateNotContains(output, criterion) {
+  const target = criterion.caseSensitive ? criterion.value : criterion.value.toLowerCase();
+  const haystack = criterion.caseSensitive ? output : output.toLowerCase();
+  const passed = !haystack.includes(target);
+  return {
+    type: "not_contains",
+    passed,
+    message: passed ? `Does not contain "${criterion.value}"` : `Contains "${criterion.value}" (should not)`
+  };
+}
+function evaluateMatches(output, criterion) {
+  const regex = new RegExp(criterion.pattern, criterion.flags);
+  const passed = regex.test(output);
+  return {
+    type: "matches",
+    passed,
+    message: passed ? `Matches /${criterion.pattern}/` : `Does not match /${criterion.pattern}/`
+  };
+}
+function evaluateLengthMin(output, criterion) {
+  const passed = output.length >= criterion.value;
+  return {
+    type: "length_min",
+    passed,
+    message: passed ? `Length ${output.length} >= ${criterion.value}` : `Length ${output.length} < ${criterion.value}`
+  };
+}
+function evaluateLengthMax(output, criterion) {
+  const passed = output.length <= criterion.value;
+  return {
+    type: "length_max",
+    passed,
+    message: passed ? `Length ${output.length} <= ${criterion.value}` : `Length ${output.length} > ${criterion.value}`
+  };
+}
+function evaluateJsonValid(output) {
+  try {
+    JSON.parse(output);
+    return { type: "json_valid", passed: true, message: "Valid JSON" };
+  } catch {
+    return { type: "json_valid", passed: false, message: "Invalid JSON" };
+  }
+}
+function evaluateJsonMatches(output, criterion) {
+  try {
+    const parsed = JSON.parse(output);
+    const passed = matchesSchema(parsed, criterion.schema);
+    return {
+      type: "json_matches",
+      passed,
+      message: passed ? "JSON matches schema" : "JSON does not match schema"
+    };
+  } catch {
+    return { type: "json_matches", passed: false, message: "Invalid JSON" };
+  }
+}
+function evaluateCustom(output, criterion) {
+  const passed = criterion.fn(output);
+  return {
+    type: `custom:${criterion.name}`,
+    passed,
+    message: passed ? `Custom check "${criterion.name}" passed` : `Custom check "${criterion.name}" failed`
+  };
+}
+function evaluateSemanticSimilarity(output, criterion) {
+  const refWords = new Set(criterion.reference.toLowerCase().split(/\s+/).filter((w) => w.length > 3));
+  const outWords = output.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
+  const overlap = outWords.filter((w) => refWords.has(w)).length;
+  const score = refWords.size > 0 ? overlap / refWords.size : 0;
+  const threshold = criterion.threshold ?? 0.7;
+  const passed = score >= threshold;
+  return {
+    type: "semantic_similarity",
+    passed,
+    message: passed ? `Semantic similarity ${(score * 100).toFixed(0)}% >= ${(threshold * 100).toFixed(0)}%` : `Semantic similarity ${(score * 100).toFixed(0)}% < ${(threshold * 100).toFixed(0)}%`
+  };
+}
+function evaluateFactualAccuracy(output, criterion) {
+  const facts = criterion.facts;
+  let matchedFacts = 0;
+  const outputLower = output.toLowerCase();
+  for (const fact of facts) {
+    const factWords = fact.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
+    const matches = factWords.filter((w) => outputLower.includes(w)).length;
+    if (matches / Math.max(factWords.length, 1) > 0.5) {
+      matchedFacts++;
+    }
+  }
+  const score = facts.length > 0 ? matchedFacts / facts.length : 1;
+  const threshold = criterion.threshold ?? 0.7;
+  const passed = score >= threshold;
+  return {
+    type: "factual_accuracy",
+    passed,
+    message: passed ? `Factual accuracy: ${matchedFacts}/${facts.length} facts verified` : `Factual accuracy: only ${matchedFacts}/${facts.length} facts found`
+  };
+}
+function evaluateCriterion(output, criterion) {
+  switch (criterion.type) {
+    case "contains":
+      return evaluateContains(output, criterion);
+    case "not_contains":
+      return evaluateNotContains(output, criterion);
+    case "matches":
+      return evaluateMatches(output, criterion);
+    case "length_min":
+      return evaluateLengthMin(output, criterion);
+    case "length_max":
+      return evaluateLengthMax(output, criterion);
+    case "json_valid":
+      return evaluateJsonValid(output);
+    case "json_matches":
+      return evaluateJsonMatches(output, criterion);
+    case "custom":
+      return evaluateCustom(output, criterion);
+    case "llm_judge":
+      return { type: "llm_judge", passed: false, message: "LLM judge requires async evaluation" };
+    case "semantic_similarity":
+      return evaluateSemanticSimilarity(output, criterion);
+    case "factual_accuracy":
+      return evaluateFactualAccuracy(output, criterion);
+  }
+}
+function matchesSchema(value, schema) {
+  if (typeof value !== "object" || value === null)
+    return false;
+  const obj = value;
+  for (const key of Object.keys(schema)) {
+    if (!(key in obj))
+      return false;
+    const expectedType = schema[key];
+    if (typeof expectedType === "string") {
+      const actualType = typeof obj[key];
+      if (actualType !== expectedType)
+        return false;
+    }
+  }
+  return true;
+}
+function makeRunnerErrorResult(evalCase, error, startTime) {
+  return {
+    name: evalCase.name,
+    passed: false,
+    score: 0,
+    criteria: [
+      {
+        type: "error",
+        passed: false,
+        message: `Runner error: ${error instanceof Error ? error.message : String(error)}`
+      }
+    ],
+    input: evalCase.input,
+    output: "",
+    durationMs: Math.round(performance.now() - startTime),
+    tags: evalCase.tags ?? []
+  };
+}
+function checkExpected(output, expected) {
+  const passed = output.includes(expected);
+  return {
+    type: "expected",
+    passed,
+    message: passed ? "Output contains expected text" : `Output does not contain expected "${expected}"`
+  };
+}
+async function evaluateLlmJudge(output, criterion) {
+  try {
+    const fullPrompt = `${criterion.prompt}
+
+Output to evaluate:
+${output}`;
+    const result = await criterion.judge(fullPrompt);
+    const threshold = criterion.threshold ?? 0.7;
+    const passed = result.score >= threshold;
+    return {
+      type: "llm_judge",
+      passed,
+      message: passed ? `LLM judge score: ${result.score.toFixed(2)} (${result.reasoning})` : `LLM judge score: ${result.score.toFixed(2)} < ${threshold} (${result.reasoning})`
+    };
+  } catch (error) {
+    return {
+      type: "llm_judge",
+      passed: false,
+      message: `LLM judge error: ${error instanceof Error ? error.message : String(error)}`
+    };
+  }
+}
+async function evaluateAllCriteria(output, evalCase) {
+  const criteriaResults = [];
+  if (evalCase.expected !== undefined) {
+    criteriaResults.push(checkExpected(output, evalCase.expected));
+  }
+  for (const criterion of evalCase.criteria ?? []) {
+    if (criterion.type === "llm_judge") {
+      criteriaResults.push(await evaluateLlmJudge(output, criterion));
+    } else {
+      criteriaResults.push(evaluateCriterion(output, criterion));
+    }
+  }
+  return criteriaResults;
+}
+async function runCase(evalCase, runner) {
+  const startTime = performance.now();
+  let output;
+  try {
+    output = await runner(evalCase.input);
+  } catch (error) {
+    return makeRunnerErrorResult(evalCase, error, startTime);
+  }
+  const criteriaResults = await evaluateAllCriteria(output, evalCase);
+  const passedCount = criteriaResults.filter((c) => c.passed).length;
+  const totalCount = criteriaResults.length;
+  const allPassed = totalCount === 0 || passedCount === totalCount;
+  const score = totalCount === 0 ? 1 : passedCount / totalCount;
+  return {
+    name: evalCase.name,
+    passed: allPassed,
+    score,
+    criteria: criteriaResults,
+    input: evalCase.input,
+    output,
+    durationMs: Math.round(performance.now() - startTime),
+    tags: evalCase.tags ?? []
+  };
+}
+async function runEvalSuite(config) {
+  const startTime = performance.now();
+  const concurrency = config.concurrency ?? 1;
+  const results = [];
+  if (concurrency <= 1) {
+    for (const evalCase of config.cases) {
+      results.push(await runCase(evalCase, config.runner));
+    }
+  } else {
+    for (let i = 0;i < config.cases.length; i += concurrency) {
+      const batch = config.cases.slice(i, i + concurrency);
+      const batchResults = await Promise.all(batch.map((c) => runCase(c, config.runner)));
+      results.push(...batchResults);
+    }
+  }
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.length - passed;
+  return {
+    name: config.name,
+    total: results.length,
+    passed,
+    failed,
+    score: results.length > 0 ? passed / results.length : 0,
+    results,
+    durationMs: Math.round(performance.now() - startTime)
+  };
+}
+function formatEvalReport(result) {
+  const lines = [];
+  lines.push(`
+  Eval Suite: ${result.name}`);
+  lines.push(`  ${"\u2500".repeat(50)}`);
+  for (const r of result.results) {
+    const icon = r.passed ? "PASS" : "FAIL";
+    lines.push(`  [${icon}] ${r.name} (${r.durationMs}ms)`);
+    if (!r.passed) {
+      for (const c of r.criteria) {
+        if (!c.passed) {
+          lines.push(`         ${c.message}`);
+        }
+      }
+    }
+  }
+  lines.push(`  ${"\u2500".repeat(50)}`);
+  lines.push(`  Score: ${(result.score * 100).toFixed(1)}% | ${result.passed}/${result.total} passed | ${result.durationMs}ms`);
+  lines.push("");
+  return lines.join(`
+`);
+}
+// src/snapshot.ts
+import { createHash as createHash2 } from "crypto";
+function createSnapshotStore(existing) {
+  const snapshots = new Map;
+  if (existing) {
+    for (const s of existing) {
+      snapshots.set(s.name, s);
+    }
+  }
+  return {
+    get(name) {
+      return snapshots.get(name);
+    },
+    set(name, snapshot) {
+      snapshots.set(name, snapshot);
+    },
+    getAll() {
+      return Array.from(snapshots.values());
+    },
+    toJSON() {
+      return JSON.stringify(Array.from(snapshots.values()), null, 2);
+    }
+  };
+}
+function hashOutput(output) {
+  return createHash2("sha256").update(output).digest("hex");
+}
+async function testSnapshot(name, store, runner, request) {
+  const output = await runner();
+  const currentHash = hashOutput(output);
+  const existing = store.get(name);
+  const snapshot = {
+    name,
+    request: {
+      system: request?.system,
+      messages: request?.messages?.map((m) => ({
+        role: m.role,
+        content: typeof m.content === "string" ? m.content : "[complex]"
+      })) ?? [],
+      model: request?.model
+    },
+    outputHash: currentHash,
+    timestamp: new Date().toISOString()
+  };
+  if (!existing) {
+    store.set(name, snapshot);
+    return { name, status: "new", currentHash, output };
+  }
+  if (existing.outputHash === currentHash) {
+    return {
+      name,
+      status: "match",
+      previousHash: existing.outputHash,
+      currentHash,
+      output
+    };
+  }
+  store.set(name, snapshot);
+  return {
+    name,
+    status: "changed",
+    previousHash: existing.outputHash,
+    currentHash,
+    output
+  };
+}
+// src/prompts.ts
+function definePrompt(config) {
+  return { ...config };
+}
+function compareLine(fromLine, toLine, lineNumber) {
+  if (fromLine === undefined) {
+    return [{ type: "added", lineNumber, content: toLine }];
+  }
+  if (toLine === undefined) {
+    return [{ type: "removed", lineNumber, content: fromLine }];
+  }
+  if (fromLine !== toLine) {
+    return [
+      { type: "removed", lineNumber, content: fromLine },
+      { type: "added", lineNumber, content: toLine }
+    ];
+  }
+  return [{ type: "unchanged", lineNumber, content: fromLine }];
+}
+function computeLineChanges(fromLines, toLines) {
+  const changes = [];
+  const maxLen = Math.max(fromLines.length, toLines.length);
+  for (let i = 0;i < maxLen; i++) {
+    changes.push(...compareLine(fromLines[i], toLines[i], i + 1));
+  }
+  return changes;
+}
+function createPromptRegistry() {
+  const store = new Map;
+  function compareVersions(a, b) {
+    const aParts = a.split(".").map(Number);
+    const bParts = b.split(".").map(Number);
+    for (let i = 0;i < Math.max(aParts.length, bParts.length); i++) {
+      const aVal = aParts[i] ?? 0;
+      const bVal = bParts[i] ?? 0;
+      if (aVal !== bVal)
+        return aVal - bVal;
+    }
+    return 0;
+  }
+  return {
+    register(name, prompt) {
+      if (!store.has(name)) {
+        store.set(name, new Map);
+      }
+      const versions = store.get(name);
+      if (versions) {
+        versions.set(prompt.version, prompt);
+      }
+    },
+    get(name, version) {
+      const versions = store.get(name);
+      if (!versions)
+        return;
+      if (version)
+        return versions.get(version);
+      return this.getLatest(name);
+    },
+    getLatest(name) {
+      const versions = store.get(name);
+      if (!versions || versions.size === 0)
+        return;
+      const sorted = [...versions.keys()].sort(compareVersions);
+      return versions.get(sorted[sorted.length - 1]);
+    },
+    list() {
+      const result = [];
+      for (const [name, versions] of store) {
+        result.push({
+          name,
+          versions: [...versions.keys()].sort(compareVersions)
+        });
+      }
+      return result;
+    },
+    getVersions(name) {
+      const versions = store.get(name);
+      if (!versions)
+        return [];
+      return [...versions.keys()].sort(compareVersions);
+    },
+    diff(name, fromVersion, toVersion) {
+      const versions = store.get(name);
+      if (!versions)
+        return null;
+      const from = versions.get(fromVersion);
+      const to = versions.get(toVersion);
+      if (!from || !to)
+        return null;
+      const fromLines = from.content.split(`
+`);
+      const toLines = to.content.split(`
+`);
+      const changes = computeLineChanges(fromLines, toLines);
+      return { name, fromVersion, toVersion, changes };
+    },
+    render(name, variables, version) {
+      const prompt = this.get(name, version);
+      if (!prompt) {
+        throw new Error(`Prompt "${name}" not found${version ? ` (version ${version})` : ""}`);
+      }
+      let rendered = prompt.content;
+      for (const [key, value] of Object.entries(variables)) {
+        rendered = rendered.replaceAll(`{{${key}}}`, value);
+      }
+      return rendered;
+    }
+  };
+}
+// src/regression.ts
+import { mkdirSync, readFileSync, writeFileSync } from "fs";
+import { dirname } from "path";
+function makeEmptyResult(name) {
+  return {
+    name,
+    totalCases: 0,
+    regressions: [],
+    improvements: [],
+    unchanged: 0,
+    overallScore: 0,
+    baselineScore: 0
+  };
+}
+async function scoreCase(input, currentOutput, baselineOutput, scorer) {
+  if (scorer)
+    return scorer(input, currentOutput);
+  return currentOutput === baselineOutput ? 1 : 0.5;
+}
+function classifyDetail(detail, regressions, improvements) {
+  if (detail.delta < -0.1) {
+    regressions.push(detail);
+    return false;
+  }
+  if (detail.delta > 0.1) {
+    improvements.push(detail);
+    return false;
+  }
+  return true;
+}
+async function compareWithBaseline(name, baseline, runner, scorer) {
+  const regressions = [];
+  const improvements = [];
+  let unchanged = 0;
+  let totalCurrentScore = 0;
+  const baselineScore = baseline.cases.reduce((sum, c) => sum + c.score, 0) / baseline.cases.length;
+  for (const baselineCase of baseline.cases) {
+    const currentOutput = await runner(baselineCase.input);
+    const currentScore = await scoreCase(baselineCase.input, currentOutput, baselineCase.output, scorer);
+    totalCurrentScore += currentScore;
+    const detail = {
+      input: baselineCase.input,
+      baselineOutput: baselineCase.output,
+      currentOutput,
+      baselineScore: baselineCase.score,
+      currentScore,
+      delta: currentScore - baselineCase.score
+    };
+    if (classifyDetail(detail, regressions, improvements)) {
+      unchanged++;
+    }
+  }
+  return {
+    name,
+    totalCases: baseline.cases.length,
+    regressions,
+    improvements,
+    unchanged,
+    overallScore: totalCurrentScore / baseline.cases.length,
+    baselineScore
+  };
+}
+function createRegressionSuite(name) {
+  let baseline = null;
+  return {
+    get baseline() {
+      return baseline;
+    },
+    async load(path) {
+      try {
+        const data = readFileSync(path, "utf-8");
+        baseline = JSON.parse(data);
+      } catch {
+        baseline = null;
+      }
+    },
+    async save(path) {
+      if (!baseline) {
+        baseline = {
+          name,
+          cases: [],
+          createdAt: Date.now(),
+          updatedAt: Date.now()
+        };
+      }
+      mkdirSync(dirname(path), { recursive: true });
+      writeFileSync(path, JSON.stringify(baseline, null, 2));
+    },
+    addCase(input, output, score) {
+      if (!baseline) {
+        baseline = {
+          name,
+          cases: [],
+          createdAt: Date.now(),
+          updatedAt: Date.now()
+        };
+      }
+      const existing = baseline.cases.findIndex((c) => c.input === input);
+      if (existing >= 0) {
+        baseline.cases[existing] = { input, output, score, timestamp: Date.now() };
+      } else {
+        baseline.cases.push({ input, output, score, timestamp: Date.now() });
+      }
+      baseline.updatedAt = Date.now();
+    },
+    async run(runner, scorer) {
+      if (!baseline || baseline.cases.length === 0) {
+        return makeEmptyResult(name);
+      }
+      return compareWithBaseline(name, baseline, runner, scorer);
+    }
+  };
+}
+// src/replay.ts
+function createReplayRecorder() {
+  const entries = [];
+  return {
+    wrap(completeFn) {
+      return async (request) => {
+        const response = await completeFn(request);
+        entries.push({
+          request,
+          response,
+          timestamp: Date.now()
+        });
+        return response;
+      };
+    },
+    getEntries() {
+      return [...entries];
+    },
+    toJSON() {
+      return JSON.stringify(entries, null, 2);
+    },
+    clear() {
+      entries.length = 0;
+    }
+  };
+}
+function createReplayPlayer(entriesOrJson) {
+  const entries = typeof entriesOrJson === "string" ? JSON.parse(entriesOrJson) : [...entriesOrJson];
+  let index = 0;
+  return {
+    get remaining() {
+      return entries.length - index;
+    },
+    async complete(_request) {
+      if (index >= entries.length) {
+        throw new Error("Replay exhausted: no more recorded responses");
+      }
+      const entry = entries[index];
+      index++;
+      return entry.response;
+    }
+  };
+}
+// src/pinning.ts
+import { createHash as createHash3 } from "crypto";
+function sha256(input) {
+  return createHash3("sha256").update(input).digest("hex");
+}
+function createPinStore(existing) {
+  const pins = new Map;
+  if (existing) {
+    for (const pin of existing) {
+      const key = `${pin.promptHash}:${pin.configHash}`;
+      pins.set(key, pin);
+    }
+  }
+  return {
+    get(key) {
+      return pins.get(key);
+    },
+    set(key, pin) {
+      pins.set(key, pin);
+    },
+    delete(key) {
+      return pins.delete(key);
+    },
+    getAll() {
+      return Array.from(pins.values());
+    },
+    toJSON() {
+      return JSON.stringify(Array.from(pins.values()), null, 2);
+    }
+  };
+}
+async function pinOutput(name, store, runner, config, options) {
+  const promptHash = sha256(config.prompt);
+  const configHash = sha256(JSON.stringify({
+    model: config.model,
+    temperature: config.temperature,
+    seed: config.seed
+  }));
+  const key = `${promptHash}:${configHash}`;
+  const output = await runner();
+  const outputHash = sha256(output);
+  const pin = {
+    promptHash,
+    configHash,
+    outputHash,
+    outputText: output,
+    model: config.model,
+    createdAt: Date.now()
+  };
+  const existing = store.get(key);
+  if (!existing) {
+    store.set(key, pin);
+    return { status: "new", pin };
+  }
+  if (existing.outputHash === outputHash) {
+    return { status: "match", pin, previousPin: existing };
+  }
+  if (options?.assert) {
+    throw ElsiumError.validation(`Pin mismatch for "${name}": expected hash ${existing.outputHash}, got ${outputHash}`);
+  }
+  store.set(key, pin);
+  return { status: "mismatch", pin, previousPin: existing };
+}
+// src/determinism.ts
+async function assertDeterministic(fn, options) {
+  const runs = options?.runs ?? 5;
+  const seed = options?.seed;
+  const tolerance = options?.tolerance ?? 0;
+  const outputs = [];
+  for (let i = 0;i < runs; i++) {
+    const output = await fn(seed);
+    outputs.push(output);
+  }
+  const unique = new Set(outputs);
+  const uniqueOutputs = unique.size;
+  const variance = runs > 1 ? (uniqueOutputs - 1) / (runs - 1) : 0;
+  const deterministic = variance <= tolerance;
+  if (!deterministic && tolerance === 0) {
+    throw ElsiumError.validation(`Non-deterministic output: ${uniqueOutputs} unique outputs across ${runs} runs (variance: ${variance.toFixed(3)})`);
+  }
+  return {
+    deterministic,
+    runs,
+    uniqueOutputs,
+    outputs,
+    variance
+  };
+}
+async function assertStable(fn, options) {
+  const intervalMs = options?.intervalMs ?? 100;
+  const runs = options?.runs ?? 3;
+  const seed = options?.seed;
+  const outputs = [];
+  for (let i = 0;i < runs; i++) {
+    if (i > 0) {
+      await sleep(intervalMs);
+    }
+    const output = await fn(seed);
+    outputs.push({ output, timestamp: Date.now() });
+  }
+  const unique = new Set(outputs.map((o) => o.output));
+  const uniqueOutputs = unique.size;
+  const variance = runs > 1 ? (uniqueOutputs - 1) / (runs - 1) : 0;
+  return {
+    stable: uniqueOutputs === 1,
+    runs,
+    uniqueOutputs,
+    outputs,
+    variance
+  };
+}
+export {
+  testSnapshot,
+  runEvalSuite,
+  pinOutput,
+  mockProvider,
+  loadFixture,
+  hashOutput,
+  formatEvalReport,
+  definePrompt,
+  createSnapshotStore,
+  createReplayRecorder,
+  createReplayPlayer,
+  createRegressionSuite,
+  createRecorder,
+  createPromptRegistry,
+  createPinStore,
+  createFixture,
+  assertStable,
+  assertDeterministic
+};