@elsium-ai/cli 0.1.6

package/dist/cli.js

@@ -0,0 +1,2382 @@
+#!/usr/bin/env bun
+// @bun
+var __defProp = Object.defineProperty;
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+
+// ../core/src/errors.ts
+var ElsiumError;
+var init_errors = __esm(() => {
+  ElsiumError = class ElsiumError extends Error {
+    code;
+    provider;
+    model;
+    statusCode;
+    retryable;
+    retryAfterMs;
+    cause;
+    metadata;
+    constructor(details) {
+      super(details.message);
+      this.name = "ElsiumError";
+      this.code = details.code;
+      this.provider = details.provider;
+      this.model = details.model;
+      this.statusCode = details.statusCode;
+      this.retryable = details.retryable;
+      this.retryAfterMs = details.retryAfterMs;
+      this.cause = details.cause;
+      this.metadata = details.metadata;
+    }
+    toJSON() {
+      return {
+        name: this.name,
+        code: this.code,
+        message: this.message,
+        provider: this.provider,
+        model: this.model,
+        statusCode: this.statusCode,
+        retryable: this.retryable,
+        retryAfterMs: this.retryAfterMs,
+        metadata: this.metadata
+      };
+    }
+    static providerError(message, opts) {
+      return new ElsiumError({
+        code: "PROVIDER_ERROR",
+        message,
+        provider: opts.provider,
+        statusCode: opts.statusCode,
+        retryable: opts.retryable ?? false,
+        cause: opts.cause
+      });
+    }
+    static rateLimit(provider, retryAfterMs) {
+      return new ElsiumError({
+        code: "RATE_LIMIT",
+        message: `Rate limited by ${provider}`,
+        provider,
+        statusCode: 429,
+        retryable: true,
+        retryAfterMs
+      });
+    }
+    static authError(provider) {
+      return new ElsiumError({
+        code: "AUTH_ERROR",
+        message: `Authentication failed for ${provider}. Check your API key.`,
+        provider,
+        statusCode: 401,
+        retryable: false
+      });
+    }
+    static timeout(provider, timeoutMs) {
+      return new ElsiumError({
+        code: "TIMEOUT",
+        message: `Request to ${provider} timed out after ${timeoutMs}ms`,
+        provider,
+        retryable: true
+      });
+    }
+    static validation(message, metadata) {
+      return new ElsiumError({
+        code: "VALIDATION_ERROR",
+        message,
+        retryable: false,
+        metadata
+      });
+    }
+    static budgetExceeded(spent, budget) {
+      return new ElsiumError({
+        code: "BUDGET_EXCEEDED",
+        message: `Token budget exceeded: spent ${spent}, budget ${budget}`,
+        retryable: false,
+        metadata: { spent, budget }
+      });
+    }
+  };
+});
+// ../core/src/utils.ts
+import { randomBytes } from "crypto";
+function cryptoHex(bytes) {
+  return randomBytes(bytes).toString("hex");
+}
+function generateId(prefix = "els") {
+  const timestamp = Date.now().toString(36);
+  const random = cryptoHex(4);
+  return `${prefix}_${timestamp}_${random}`;
+}
+function generateTraceId() {
+  const timestamp = Date.now().toString(36);
+  const random = cryptoHex(6);
+  return `trc_${timestamp}_${random}`;
+}
+async function sleep(ms) {
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
+}
+var init_utils = () => {};
+
+// ../core/src/stream.ts
+function shouldEmitCheckpoint(lastCheckpointTime, intervalMs, textLength) {
+  const elapsed = Date.now() - lastCheckpointTime;
+  return elapsed >= intervalMs && textLength > 0;
+}
+function createCheckpoint(textAccumulator, eventIndex, now) {
+  return {
+    id: generateId("ckpt"),
+    timestamp: now,
+    text: textAccumulator,
+    tokensSoFar: Math.ceil(textAccumulator.length / 1.5),
+    eventIndex
+  };
+}
+function toError(err) {
+  return err instanceof Error ? err : new Error(String(err));
+}
+function* emitErrorEvent(err, textAccumulator, onPartialRecovery) {
+  const error = toError(err);
+  if (textAccumulator.length > 0) {
+    onPartialRecovery?.(textAccumulator, error);
+    yield { type: "recovery", partialText: textAccumulator, error };
+  } else {
+    yield { type: "error", error };
+  }
+}
+function createStream(executor) {
+  let resolve2 = null;
+  const buffer = [];
+  let done = false;
+  let error = null;
+  let dropped = 0;
+  const source = {
+    [Symbol.asyncIterator]() {
+      return {
+        next() {
+          if (buffer.length > 0) {
+            const value = buffer.shift();
+            return Promise.resolve({ value, done: false });
+          }
+          if (done) {
+            return Promise.resolve({ value: undefined, done: true });
+          }
+          if (error) {
+            return Promise.reject(error);
+          }
+          return new Promise((r) => {
+            resolve2 = r;
+          });
+        }
+      };
+    }
+  };
+  const emit = (event) => {
+    if (resolve2) {
+      const r = resolve2;
+      resolve2 = null;
+      r({ value: event, done: false });
+    } else {
+      if (buffer.length < MAX_BUFFER_SIZE) {
+        buffer.push(event);
+      } else {
+        dropped++;
+      }
+    }
+  };
+  executor(emit).then(() => {
+    if (dropped > 0) {
+      emit({
+        type: "error",
+        error: new Error(`Stream buffer overflow: ${dropped} events dropped`)
+      });
+    }
+    done = true;
+    if (resolve2) {
+      const r = resolve2;
+      resolve2 = null;
+      r({ value: undefined, done: true });
+    }
+  }).catch((e) => {
+    error = e instanceof Error ? e : new Error(String(e));
+    if (resolve2) {
+      resolve2({ value: { type: "error", error }, done: false });
+      resolve2 = null;
+    }
+  });
+  return new ElsiumStream(source);
+}
+var ElsiumStream, MAX_BUFFER_SIZE = 1e4;
+var init_stream = __esm(() => {
+  init_utils();
+  ElsiumStream = class ElsiumStream {
+    source;
+    iterating = false;
+    constructor(source) {
+      this.source = source;
+    }
+    async* [Symbol.asyncIterator]() {
+      if (this.iterating) {
+        throw new Error("ElsiumStream supports only a single consumer");
+      }
+      this.iterating = true;
+      yield* this.source;
+    }
+    text() {
+      const source = this.source;
+      return {
+        async* [Symbol.asyncIterator]() {
+          for await (const event of source) {
+            if (event.type === "text_delta") {
+              yield event.text;
+            }
+          }
+        }
+      };
+    }
+    async toText() {
+      const parts = [];
+      for await (const text of this.text()) {
+        parts.push(text);
+      }
+      return parts.join("");
+    }
+    async toTextWithTimeout(timeoutMs) {
+      const parts = [];
+      const deadline = Date.now() + timeoutMs;
+      const iterator = this.source[Symbol.asyncIterator]();
+      try {
+        while (true) {
+          const remaining = deadline - Date.now();
+          if (remaining <= 0)
+            break;
+          let timer;
+          const timeoutPromise = new Promise((resolve2) => {
+            timer = setTimeout(() => resolve2({ value: undefined, done: true }), remaining);
+          });
+          const result = await Promise.race([iterator.next(), timeoutPromise]);
+          if (timer !== undefined)
+            clearTimeout(timer);
+          if (result.done)
+            break;
+          const event = result.value;
+          if (event.type === "text_delta") {
+            parts.push(event.text);
+          }
+        }
+      } catch (err) {
+        if (parts.length === 0)
+          throw err;
+      } finally {
+        await iterator.return?.();
+      }
+      return parts.join("");
+    }
+    async toResponse() {
+      const parts = [];
+      let usage = null;
+      let stopReason = null;
+      for await (const event of this.source) {
+        switch (event.type) {
+          case "text_delta":
+            parts.push(event.text);
+            break;
+          case "message_end":
+            usage = event.usage;
+            stopReason = event.stopReason;
+            break;
+        }
+      }
+      return { text: parts.join(""), usage, stopReason };
+    }
+    pipe(transform) {
+      return new ElsiumStream(transform(this.source));
+    }
+    resilient(options = {}) {
+      const { checkpointIntervalMs = 1000, onCheckpoint, onPartialRecovery } = options;
+      const source = this.source;
+      const resilientSource = {
+        async* [Symbol.asyncIterator]() {
+          let lastCheckpointTime = Date.now();
+          let textAccumulator = "";
+          let eventIndex = 0;
+          try {
+            for await (const event of source) {
+              eventIndex++;
+              if (event.type === "text_delta") {
+                textAccumulator += event.text;
+              }
+              yield event;
+              if (shouldEmitCheckpoint(lastCheckpointTime, checkpointIntervalMs, textAccumulator.length)) {
+                const now = Date.now();
+                const checkpoint = createCheckpoint(textAccumulator, eventIndex, now);
+                onCheckpoint?.(checkpoint);
+                yield { type: "checkpoint", checkpoint };
+                lastCheckpointTime = now;
+              }
+            }
+          } catch (err) {
+            yield* emitErrorEvent(err, textAccumulator, onPartialRecovery);
+          }
+        }
+      };
+      return new ElsiumStream(resilientSource);
+    }
+  };
+});
+
+// ../core/src/logger.ts
+var init_logger = () => {};
+
+// ../core/src/config.ts
+var init_config = __esm(() => {
+  init_errors();
+});
+
+// ../core/src/circuit-breaker.ts
+var init_circuit_breaker = __esm(() => {
+  init_errors();
+});
+
+// ../core/src/dedup.ts
+var init_dedup = __esm(() => {
+  init_errors();
+});
+
+// ../core/src/policy.ts
+var init_policy = __esm(() => {
+  init_errors();
+  init_utils();
+});
+
+// ../core/src/shutdown.ts
+var init_shutdown = __esm(() => {
+  init_errors();
+});
+
+// ../core/src/index.ts
+var init_src = __esm(() => {
+  init_errors();
+  init_stream();
+  init_logger();
+  init_config();
+  init_utils();
+  init_circuit_breaker();
+  init_dedup();
+  init_policy();
+  init_shutdown();
+});
+
+// ../testing/src/mock-provider.ts
+function mockProvider(options = {}) {
+  const { responses = [], defaultResponse, onRequest } = options;
+  const calls = [];
+  let callIndex = 0;
+  function getNextResponse() {
+    if (callIndex < responses.length) {
+      return responses[callIndex++];
+    }
+    if (defaultResponse) {
+      callIndex++;
+      return defaultResponse;
+    }
+    callIndex++;
+    return { content: "" };
+  }
+  async function emitStreamEvents(emit, config) {
+    if (config.delay) {
+      await new Promise((r) => setTimeout(r, config.delay));
+    }
+    emit({
+      type: "message_start",
+      id: generateId("msg"),
+      model: config.model ?? "mock-model"
+    });
+    const content = config.content ?? "";
+    if (content) {
+      const words = content.split(" ");
+      for (const word of words) {
+        emit({ type: "text_delta", text: `${word} ` });
+      }
+    }
+    emit({
+      type: "message_end",
+      usage: {
+        inputTokens: config.usage?.inputTokens ?? 10,
+        outputTokens: config.usage?.outputTokens ?? 5,
+        totalTokens: config.usage?.totalTokens ?? 15
+      },
+      stopReason: config.stopReason ?? "end_turn"
+    });
+  }
+  function buildResponse(config, request) {
+    const model = config.model ?? request.model ?? "mock-model";
+    const content = config.content ?? "";
+    const toolCalls = config.toolCalls?.map((tc) => ({
+      id: tc.id ?? generateId("tc"),
+      name: tc.name,
+      arguments: tc.arguments
+    }));
+    const usage = {
+      inputTokens: config.usage?.inputTokens ?? Math.ceil(content.length / 4),
+      outputTokens: config.usage?.outputTokens ?? Math.ceil(content.length / 4),
+      totalTokens: 0,
+      ...config.usage
+    };
+    usage.totalTokens = usage.inputTokens + usage.outputTokens;
+    const message = {
+      role: "assistant",
+      content,
+      ...toolCalls?.length ? { toolCalls } : {}
+    };
+    return {
+      id: generateId("msg"),
+      message,
+      usage,
+      cost: { inputCost: 0, outputCost: 0, totalCost: 0, currency: "USD" },
+      model,
+      provider: "mock",
+      stopReason: config.stopReason ?? (toolCalls?.length ? "tool_use" : "end_turn"),
+      latencyMs: config.delay ?? 0,
+      traceId: generateTraceId()
+    };
+  }
+  return {
+    name: "mock",
+    defaultModel: "mock-model",
+    get calls() {
+      return calls;
+    },
+    get callCount() {
+      return calls.length;
+    },
+    async complete(request) {
+      calls.push(request);
+      onRequest?.(request);
+      const config = getNextResponse();
+      if (config.delay) {
+        await new Promise((r) => setTimeout(r, config.delay));
+      }
+      return buildResponse(config, request);
+    },
+    stream(request) {
+      calls.push(request);
+      onRequest?.(request);
+      const config = getNextResponse();
+      return createStream((emit) => emitStreamEvents(emit, config));
+    },
+    async listModels() {
+      return ["mock-model"];
+    },
+    reset() {
+      calls.length = 0;
+      callIndex = 0;
+    }
+  };
+}
+var init_mock_provider = __esm(() => {
+  init_src();
+});
+
+// ../testing/src/fixtures.ts
+import { createHash } from "crypto";
+function hashMessages(messages) {
+  const content = messages.map((m) => `${m.role}:${m.content}`).join("|");
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function createFixture(name, entries) {
+  return {
+    name,
+    entries,
+    toProvider(options) {
+      if (options?.matching === "request-hash") {
+        const hashMap = new Map;
+        for (const entry of entries) {
+          const hash = hashMessages(entry.request.messages);
+          hashMap.set(hash, entry.response);
+        }
+        const provider = mockProvider({
+          responses: entries.map((e) => e.response)
+        });
+        const originalComplete = provider.complete.bind(provider);
+        const wrapped = Object.create(provider);
+        wrapped.complete = async (request) => {
+          const reqMessages = request.messages.map((m) => ({
+            role: m.role,
+            content: typeof m.content === "string" ? m.content : "[complex]"
+          }));
+          const hash = hashMessages(reqMessages);
+          const matched = hashMap.get(hash);
+          if (matched) {
+            return mockProvider({ responses: [matched] }).complete(request);
+          }
+          return originalComplete(request);
+        };
+        return wrapped;
+      }
+      return mockProvider({
+        responses: entries.map((e) => e.response)
+      });
+    },
+    toJSON() {
+      return JSON.stringify({
+        name,
+        entries: entries.map((e) => ({
+          ...e,
+          timestamp: e.timestamp ?? new Date().toISOString()
+        }))
+      }, null, 2);
+    }
+  };
+}
+function loadFixture(json) {
+  const data = JSON.parse(json);
+  return createFixture(data.name, data.entries);
+}
+function createRecorder() {
+  const entries = [];
+  return {
+    wrap(provider) {
+      const originalComplete = provider.complete.bind(provider);
+      const wrapped = Object.create(provider);
+      wrapped.complete = async (request) => {
+        const response = await originalComplete(request);
+        entries.push({
+          request: {
+            messages: request.messages.map((m) => ({
+              role: m.role,
+              content: typeof m.content === "string" ? m.content : "[complex]"
+            })),
+            model: request.model,
+            system: request.system
+          },
+          response: {
+            content: typeof response.message.content === "string" ? response.message.content : "",
+            toolCalls: response.message.toolCalls,
+            stopReason: response.stopReason,
+            usage: response.usage,
+            model: response.model
+          },
+          timestamp: new Date().toISOString()
+        });
+        return response;
+      };
+      return wrapped;
+    },
+    getEntries() {
+      return [...entries];
+    },
+    toFixture(name) {
+      return createFixture(name, [...entries]);
+    },
+    clear() {
+      entries.length = 0;
+    }
+  };
+}
+var init_fixtures = __esm(() => {
+  init_mock_provider();
+});
+
+// ../testing/src/eval.ts
+function evaluateContains(output, criterion) {
+  const target = criterion.caseSensitive ? criterion.value : criterion.value.toLowerCase();
+  const haystack = criterion.caseSensitive ? output : output.toLowerCase();
+  const passed = haystack.includes(target);
+  return {
+    type: "contains",
+    passed,
+    message: passed ? `Contains "${criterion.value}"` : `Does not contain "${criterion.value}"`
+  };
+}
+function evaluateNotContains(output, criterion) {
+  const target = criterion.caseSensitive ? criterion.value : criterion.value.toLowerCase();
+  const haystack = criterion.caseSensitive ? output : output.toLowerCase();
+  const passed = !haystack.includes(target);
+  return {
+    type: "not_contains",
+    passed,
+    message: passed ? `Does not contain "${criterion.value}"` : `Contains "${criterion.value}" (should not)`
+  };
+}
+function evaluateMatches(output, criterion) {
+  const regex = new RegExp(criterion.pattern, criterion.flags);
+  const passed = regex.test(output);
+  return {
+    type: "matches",
+    passed,
+    message: passed ? `Matches /${criterion.pattern}/` : `Does not match /${criterion.pattern}/`
+  };
+}
+function evaluateLengthMin(output, criterion) {
+  const passed = output.length >= criterion.value;
+  return {
+    type: "length_min",
+    passed,
+    message: passed ? `Length ${output.length} >= ${criterion.value}` : `Length ${output.length} < ${criterion.value}`
+  };
+}
+function evaluateLengthMax(output, criterion) {
+  const passed = output.length <= criterion.value;
+  return {
+    type: "length_max",
+    passed,
+    message: passed ? `Length ${output.length} <= ${criterion.value}` : `Length ${output.length} > ${criterion.value}`
+  };
+}
+function evaluateJsonValid(output) {
+  try {
+    JSON.parse(output);
+    return { type: "json_valid", passed: true, message: "Valid JSON" };
+  } catch {
+    return { type: "json_valid", passed: false, message: "Invalid JSON" };
+  }
+}
+function evaluateJsonMatches(output, criterion) {
+  try {
+    const parsed = JSON.parse(output);
+    const passed = matchesSchema(parsed, criterion.schema);
+    return {
+      type: "json_matches",
+      passed,
+      message: passed ? "JSON matches schema" : "JSON does not match schema"
+    };
+  } catch {
+    return { type: "json_matches", passed: false, message: "Invalid JSON" };
+  }
+}
+function evaluateCustom(output, criterion) {
+  const passed = criterion.fn(output);
+  return {
+    type: `custom:${criterion.name}`,
+    passed,
+    message: passed ? `Custom check "${criterion.name}" passed` : `Custom check "${criterion.name}" failed`
+  };
+}
+function evaluateSemanticSimilarity(output, criterion) {
+  const refWords = new Set(criterion.reference.toLowerCase().split(/\s+/).filter((w) => w.length > 3));
+  const outWords = output.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
+  const overlap = outWords.filter((w) => refWords.has(w)).length;
+  const score = refWords.size > 0 ? overlap / refWords.size : 0;
+  const threshold = criterion.threshold ?? 0.7;
+  const passed = score >= threshold;
+  return {
+    type: "semantic_similarity",
+    passed,
+    message: passed ? `Semantic similarity ${(score * 100).toFixed(0)}% >= ${(threshold * 100).toFixed(0)}%` : `Semantic similarity ${(score * 100).toFixed(0)}% < ${(threshold * 100).toFixed(0)}%`
+  };
+}
+function evaluateFactualAccuracy(output, criterion) {
+  const facts = criterion.facts;
+  let matchedFacts = 0;
+  const outputLower = output.toLowerCase();
+  for (const fact of facts) {
+    const factWords = fact.toLowerCase().split(/\s+/).filter((w) => w.length > 3);
+    const matches = factWords.filter((w) => outputLower.includes(w)).length;
+    if (matches / Math.max(factWords.length, 1) > 0.5) {
+      matchedFacts++;
+    }
+  }
+  const score = facts.length > 0 ? matchedFacts / facts.length : 1;
+  const threshold = criterion.threshold ?? 0.7;
+  const passed = score >= threshold;
+  return {
+    type: "factual_accuracy",
+    passed,
+    message: passed ? `Factual accuracy: ${matchedFacts}/${facts.length} facts verified` : `Factual accuracy: only ${matchedFacts}/${facts.length} facts found`
+  };
+}
+function evaluateCriterion(output, criterion) {
+  switch (criterion.type) {
+    case "contains":
+      return evaluateContains(output, criterion);
+    case "not_contains":
+      return evaluateNotContains(output, criterion);
+    case "matches":
+      return evaluateMatches(output, criterion);
+    case "length_min":
+      return evaluateLengthMin(output, criterion);
+    case "length_max":
+      return evaluateLengthMax(output, criterion);
+    case "json_valid":
+      return evaluateJsonValid(output);
+    case "json_matches":
+      return evaluateJsonMatches(output, criterion);
+    case "custom":
+      return evaluateCustom(output, criterion);
+    case "llm_judge":
+      return { type: "llm_judge", passed: false, message: "LLM judge requires async evaluation" };
+    case "semantic_similarity":
+      return evaluateSemanticSimilarity(output, criterion);
+    case "factual_accuracy":
+      return evaluateFactualAccuracy(output, criterion);
+  }
+}
+function matchesSchema(value, schema) {
+  if (typeof value !== "object" || value === null)
+    return false;
+  const obj = value;
+  for (const key of Object.keys(schema)) {
+    if (!(key in obj))
+      return false;
+    const expectedType = schema[key];
+    if (typeof expectedType === "string") {
+      const actualType = typeof obj[key];
+      if (actualType !== expectedType)
+        return false;
+    }
+  }
+  return true;
+}
+function makeRunnerErrorResult(evalCase, error, startTime) {
+  return {
+    name: evalCase.name,
+    passed: false,
+    score: 0,
+    criteria: [
+      {
+        type: "error",
+        passed: false,
+        message: `Runner error: ${error instanceof Error ? error.message : String(error)}`
+      }
+    ],
+    input: evalCase.input,
+    output: "",
+    durationMs: Math.round(performance.now() - startTime),
+    tags: evalCase.tags ?? []
+  };
+}
+function checkExpected(output, expected) {
+  const passed = output.includes(expected);
+  return {
+    type: "expected",
+    passed,
+    message: passed ? "Output contains expected text" : `Output does not contain expected "${expected}"`
+  };
+}
+async function evaluateLlmJudge(output, criterion) {
+  try {
+    const fullPrompt = `${criterion.prompt}
+
+Output to evaluate:
+${output}`;
+    const result = await criterion.judge(fullPrompt);
+    const threshold = criterion.threshold ?? 0.7;
+    const passed = result.score >= threshold;
+    return {
+      type: "llm_judge",
+      passed,
+      message: passed ? `LLM judge score: ${result.score.toFixed(2)} (${result.reasoning})` : `LLM judge score: ${result.score.toFixed(2)} < ${threshold} (${result.reasoning})`
+    };
+  } catch (error) {
+    return {
+      type: "llm_judge",
+      passed: false,
+      message: `LLM judge error: ${error instanceof Error ? error.message : String(error)}`
+    };
+  }
+}
+async function evaluateAllCriteria(output, evalCase) {
+  const criteriaResults = [];
+  if (evalCase.expected !== undefined) {
+    criteriaResults.push(checkExpected(output, evalCase.expected));
+  }
+  for (const criterion of evalCase.criteria ?? []) {
+    if (criterion.type === "llm_judge") {
+      criteriaResults.push(await evaluateLlmJudge(output, criterion));
+    } else {
+      criteriaResults.push(evaluateCriterion(output, criterion));
+    }
+  }
+  return criteriaResults;
+}
+async function runCase(evalCase, runner) {
+  const startTime = performance.now();
+  let output;
+  try {
+    output = await runner(evalCase.input);
+  } catch (error) {
+    return makeRunnerErrorResult(evalCase, error, startTime);
+  }
+  const criteriaResults = await evaluateAllCriteria(output, evalCase);
+  const passedCount = criteriaResults.filter((c) => c.passed).length;
+  const totalCount = criteriaResults.length;
+  const allPassed = totalCount === 0 || passedCount === totalCount;
+  const score = totalCount === 0 ? 1 : passedCount / totalCount;
+  return {
+    name: evalCase.name,
+    passed: allPassed,
+    score,
+    criteria: criteriaResults,
+    input: evalCase.input,
+    output,
+    durationMs: Math.round(performance.now() - startTime),
+    tags: evalCase.tags ?? []
+  };
+}
+async function runEvalSuite(config) {
+  const startTime = performance.now();
+  const concurrency = config.concurrency ?? 1;
+  const results = [];
+  if (concurrency <= 1) {
+    for (const evalCase of config.cases) {
+      results.push(await runCase(evalCase, config.runner));
+    }
+  } else {
+    for (let i = 0;i < config.cases.length; i += concurrency) {
+      const batch = config.cases.slice(i, i + concurrency);
+      const batchResults = await Promise.all(batch.map((c) => runCase(c, config.runner)));
+      results.push(...batchResults);
+    }
+  }
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.length - passed;
+  return {
+    name: config.name,
+    total: results.length,
+    passed,
+    failed,
+    score: results.length > 0 ? passed / results.length : 0,
+    results,
+    durationMs: Math.round(performance.now() - startTime)
+  };
+}
+function formatEvalReport(result) {
+  const lines = [];
+  lines.push(`
+  Eval Suite: ${result.name}`);
+  lines.push(`  ${"\u2500".repeat(50)}`);
+  for (const r of result.results) {
+    const icon = r.passed ? "PASS" : "FAIL";
+    lines.push(`  [${icon}] ${r.name} (${r.durationMs}ms)`);
+    if (!r.passed) {
+      for (const c of r.criteria) {
+        if (!c.passed) {
+          lines.push(`         ${c.message}`);
+        }
+      }
+    }
+  }
+  lines.push(`  ${"\u2500".repeat(50)}`);
+  lines.push(`  Score: ${(result.score * 100).toFixed(1)}% | ${result.passed}/${result.total} passed | ${result.durationMs}ms`);
+  lines.push("");
+  return lines.join(`
+`);
+}
+
+// ../testing/src/snapshot.ts
+import { createHash as createHash2 } from "crypto";
+function createSnapshotStore(existing) {
+  const snapshots = new Map;
+  if (existing) {
+    for (const s of existing) {
+      snapshots.set(s.name, s);
+    }
+  }
+  return {
+    get(name) {
+      return snapshots.get(name);
+    },
+    set(name, snapshot) {
+      snapshots.set(name, snapshot);
+    },
+    getAll() {
+      return Array.from(snapshots.values());
+    },
+    toJSON() {
+      return JSON.stringify(Array.from(snapshots.values()), null, 2);
+    }
+  };
+}
+function hashOutput(output) {
+  return createHash2("sha256").update(output).digest("hex");
+}
+async function testSnapshot(name, store, runner, request) {
+  const output = await runner();
+  const currentHash = hashOutput(output);
+  const existing = store.get(name);
+  const snapshot = {
+    name,
+    request: {
+      system: request?.system,
+      messages: request?.messages?.map((m) => ({
+        role: m.role,
+        content: typeof m.content === "string" ? m.content : "[complex]"
+      })) ?? [],
+      model: request?.model
+    },
+    outputHash: currentHash,
+    timestamp: new Date().toISOString()
+  };
+  if (!existing) {
+    store.set(name, snapshot);
+    return { name, status: "new", currentHash, output };
+  }
+  if (existing.outputHash === currentHash) {
+    return {
+      name,
+      status: "match",
+      previousHash: existing.outputHash,
+      currentHash,
+      output
+    };
+  }
+  store.set(name, snapshot);
+  return {
+    name,
+    status: "changed",
+    previousHash: existing.outputHash,
+    currentHash,
+    output
+  };
+}
+var init_snapshot = () => {};
+
+// ../testing/src/prompts.ts
+function definePrompt(config) {
+  return { ...config };
+}
+function compareLine(fromLine, toLine, lineNumber) {
+  if (fromLine === undefined) {
+    return [{ type: "added", lineNumber, content: toLine }];
+  }
+  if (toLine === undefined) {
+    return [{ type: "removed", lineNumber, content: fromLine }];
+  }
+  if (fromLine !== toLine) {
+    return [
+      { type: "removed", lineNumber, content: fromLine },
+      { type: "added", lineNumber, content: toLine }
+    ];
+  }
+  return [{ type: "unchanged", lineNumber, content: fromLine }];
+}
+function computeLineChanges(fromLines, toLines) {
+  const changes = [];
+  const maxLen = Math.max(fromLines.length, toLines.length);
+  for (let i = 0;i < maxLen; i++) {
+    changes.push(...compareLine(fromLines[i], toLines[i], i + 1));
+  }
+  return changes;
+}
+function createPromptRegistry() {
+  const store = new Map;
+  function compareVersions(a, b) {
+    const aParts = a.split(".").map(Number);
+    const bParts = b.split(".").map(Number);
+    for (let i = 0;i < Math.max(aParts.length, bParts.length); i++) {
+      const aVal = aParts[i] ?? 0;
+      const bVal = bParts[i] ?? 0;
+      if (aVal !== bVal)
+        return aVal - bVal;
+    }
+    return 0;
+  }
+  return {
+    register(name, prompt) {
+      if (!store.has(name)) {
+        store.set(name, new Map);
+      }
+      const versions = store.get(name);
+      if (versions) {
+        versions.set(prompt.version, prompt);
+      }
+    },
+    get(name, version) {
+      const versions = store.get(name);
+      if (!versions)
+        return;
+      if (version)
+        return versions.get(version);
+      return this.getLatest(name);
+    },
+    getLatest(name) {
+      const versions = store.get(name);
+      if (!versions || versions.size === 0)
+        return;
+      const sorted = [...versions.keys()].sort(compareVersions);
+      return versions.get(sorted[sorted.length - 1]);
+    },
+    list() {
+      const result = [];
+      for (const [name, versions] of store) {
+        result.push({
+          name,
+          versions: [...versions.keys()].sort(compareVersions)
+        });
+      }
+      return result;
+    },
+    getVersions(name) {
+      const versions = store.get(name);
+      if (!versions)
+        return [];
+      return [...versions.keys()].sort(compareVersions);
+    },
+    diff(name, fromVersion, toVersion) {
+      const versions = store.get(name);
+      if (!versions)
+        return null;
+      const from = versions.get(fromVersion);
+      const to = versions.get(toVersion);
+      if (!from || !to)
+        return null;
+      const fromLines = from.content.split(`
+`);
+      const toLines = to.content.split(`
+`);
+      const changes = computeLineChanges(fromLines, toLines);
+      return { name, fromVersion, toVersion, changes };
+    },
+    render(name, variables, version) {
+      const prompt = this.get(name, version);
+      if (!prompt) {
+        throw new Error(`Prompt "${name}" not found${version ? ` (version ${version})` : ""}`);
+      }
+      let rendered = prompt.content;
+      for (const [key, value] of Object.entries(variables)) {
+        rendered = rendered.replaceAll(`{{${key}}}`, value);
+      }
+      return rendered;
+    }
+  };
+}
+
+// ../testing/src/regression.ts
+import { mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
+import { dirname } from "path";
+function makeEmptyResult(name) {
+  return {
+    name,
+    totalCases: 0,
+    regressions: [],
+    improvements: [],
+    unchanged: 0,
+    overallScore: 0,
+    baselineScore: 0
+  };
+}
+async function scoreCase(input, currentOutput, baselineOutput, scorer) {
+  if (scorer)
+    return scorer(input, currentOutput);
+  return currentOutput === baselineOutput ? 1 : 0.5;
+}
+function classifyDetail(detail, regressions, improvements) {
+  if (detail.delta < -0.1) {
+    regressions.push(detail);
+    return false;
+  }
+  if (detail.delta > 0.1) {
+    improvements.push(detail);
+    return false;
+  }
+  return true;
+}
+async function compareWithBaseline(name, baseline, runner, scorer) {
+  const regressions = [];
+  const improvements = [];
+  let unchanged = 0;
+  let totalCurrentScore = 0;
+  const baselineScore = baseline.cases.reduce((sum, c) => sum + c.score, 0) / baseline.cases.length;
+  for (const baselineCase of baseline.cases) {
+    const currentOutput = await runner(baselineCase.input);
+    const currentScore = await scoreCase(baselineCase.input, currentOutput, baselineCase.output, scorer);
+    totalCurrentScore += currentScore;
+    const detail = {
+      input: baselineCase.input,
+      baselineOutput: baselineCase.output,
+      currentOutput,
+      baselineScore: baselineCase.score,
+      currentScore,
+      delta: currentScore - baselineCase.score
+    };
+    if (classifyDetail(detail, regressions, improvements)) {
+      unchanged++;
+    }
+  }
+  return {
+    name,
+    totalCases: baseline.cases.length,
+    regressions,
+    improvements,
+    unchanged,
+    overallScore: totalCurrentScore / baseline.cases.length,
+    baselineScore
+  };
+}
+function createRegressionSuite(name) {
+  let baseline = null;
+  return {
+    get baseline() {
+      return baseline;
+    },
+    async load(path) {
+      try {
+        const data = readFileSync2(path, "utf-8");
+        baseline = JSON.parse(data);
+      } catch {
+        baseline = null;
+      }
+    },
+    async save(path) {
+      if (!baseline) {
+        baseline = {
+          name,
+          cases: [],
+          createdAt: Date.now(),
+          updatedAt: Date.now()
+        };
+      }
+      mkdirSync(dirname(path), { recursive: true });
+      writeFileSync(path, JSON.stringify(baseline, null, 2));
+    },
+    addCase(input, output, score) {
+      if (!baseline) {
+        baseline = {
+          name,
+          cases: [],
+          createdAt: Date.now(),
+          updatedAt: Date.now()
+        };
+      }
+      const existing = baseline.cases.findIndex((c) => c.input === input);
+      if (existing >= 0) {
+        baseline.cases[existing] = { input, output, score, timestamp: Date.now() };
+      } else {
+        baseline.cases.push({ input, output, score, timestamp: Date.now() });
+      }
+      baseline.updatedAt = Date.now();
+    },
+    async run(runner, scorer) {
+      if (!baseline || baseline.cases.length === 0) {
+        return makeEmptyResult(name);
+      }
+      return compareWithBaseline(name, baseline, runner, scorer);
+    }
+  };
+}
+var init_regression = () => {};
+
+// ../testing/src/replay.ts
+function createReplayRecorder() {
+  const entries = [];
+  return {
+    wrap(completeFn) {
+      return async (request) => {
+        const response = await completeFn(request);
+        entries.push({
+          request,
+          response,
+          timestamp: Date.now()
+        });
+        return response;
+      };
+    },
+    getEntries() {
+      return [...entries];
+    },
+    toJSON() {
+      return JSON.stringify(entries, null, 2);
+    },
+    clear() {
+      entries.length = 0;
+    }
+  };
+}
+function createReplayPlayer(entriesOrJson) {
+  const entries = typeof entriesOrJson === "string" ? JSON.parse(entriesOrJson) : [...entriesOrJson];
+  let index = 0;
+  return {
+    get remaining() {
+      return entries.length - index;
+    },
+    async complete(_request) {
+      if (index >= entries.length) {
+        throw new Error("Replay exhausted: no more recorded responses");
+      }
+      const entry = entries[index];
+      index++;
+      return entry.response;
+    }
+  };
+}
+
+// ../testing/src/pinning.ts
+import { createHash as createHash3 } from "crypto";
+function sha256(input) {
+  return createHash3("sha256").update(input).digest("hex");
+}
+function createPinStore(existing) {
+  const pins = new Map;
+  if (existing) {
+    for (const pin of existing) {
+      const key = `${pin.promptHash}:${pin.configHash}`;
+      pins.set(key, pin);
+    }
+  }
+  return {
+    get(key) {
+      return pins.get(key);
+    },
+    set(key, pin) {
+      pins.set(key, pin);
+    },
+    delete(key) {
+      return pins.delete(key);
+    },
+    getAll() {
+      return Array.from(pins.values());
+    },
+    toJSON() {
+      return JSON.stringify(Array.from(pins.values()), null, 2);
+    }
+  };
+}
+async function pinOutput(name, store, runner, config, options) {
+  const promptHash = sha256(config.prompt);
+  const configHash = sha256(JSON.stringify({
+    model: config.model,
+    temperature: config.temperature,
+    seed: config.seed
+  }));
+  const key = `${promptHash}:${configHash}`;
+  const output = await runner();
+  const outputHash = sha256(output);
+  const pin = {
+    promptHash,
+    configHash,
+    outputHash,
+    outputText: output,
+    model: config.model,
+    createdAt: Date.now()
+  };
+  const existing = store.get(key);
+  if (!existing) {
+    store.set(key, pin);
+    return { status: "new", pin };
+  }
+  if (existing.outputHash === outputHash) {
+    return { status: "match", pin, previousPin: existing };
+  }
+  if (options?.assert) {
+    throw ElsiumError.validation(`Pin mismatch for "${name}": expected hash ${existing.outputHash}, got ${outputHash}`);
+  }
+  store.set(key, pin);
+  return { status: "mismatch", pin, previousPin: existing };
+}
+var init_pinning = __esm(() => {
+  init_src();
+});
+
+// ../testing/src/determinism.ts
+async function assertDeterministic(fn, options) {
+  const runs = options?.runs ?? 5;
+  const seed = options?.seed;
+  const tolerance = options?.tolerance ?? 0;
+  const outputs = [];
+  for (let i = 0;i < runs; i++) {
+    const output = await fn(seed);
+    outputs.push(output);
+  }
+  const unique = new Set(outputs);
+  const uniqueOutputs = unique.size;
+  const variance = runs > 1 ? (uniqueOutputs - 1) / (runs - 1) : 0;
+  const deterministic = variance <= tolerance;
+  if (!deterministic && tolerance === 0) {
+    throw ElsiumError.validation(`Non-deterministic output: ${uniqueOutputs} unique outputs across ${runs} runs (variance: ${variance.toFixed(3)})`);
+  }
+  return {
+    deterministic,
+    runs,
+    uniqueOutputs,
+    outputs,
+    variance
+  };
+}
+async function assertStable(fn, options) {
+  const intervalMs = options?.intervalMs ?? 100;
+  const runs = options?.runs ?? 3;
+  const seed = options?.seed;
+  const outputs = [];
+  for (let i = 0;i < runs; i++) {
+    if (i > 0) {
+      await sleep(intervalMs);
+    }
+    const output = await fn(seed);
+    outputs.push({ output, timestamp: Date.now() });
+  }
+  const unique = new Set(outputs.map((o) => o.output));
+  const uniqueOutputs = unique.size;
+  const variance = runs > 1 ? (uniqueOutputs - 1) / (runs - 1) : 0;
+  return {
+    stable: uniqueOutputs === 1,
+    runs,
+    uniqueOutputs,
+    outputs,
+    variance
+  };
+}
+var init_determinism = __esm(() => {
+  init_src();
+});
+
+// ../testing/src/index.ts
+var exports_src = {};
+__export(exports_src, {
+  testSnapshot: () => testSnapshot,
+  runEvalSuite: () => runEvalSuite,
+  pinOutput: () => pinOutput,
+  mockProvider: () => mockProvider,
+  loadFixture: () => loadFixture,
+  hashOutput: () => hashOutput,
+  formatEvalReport: () => formatEvalReport,
+  definePrompt: () => definePrompt,
+  createSnapshotStore: () => createSnapshotStore,
+  createReplayRecorder: () => createReplayRecorder,
+  createReplayPlayer: () => createReplayPlayer,
+  createRegressionSuite: () => createRegressionSuite,
+  createRecorder: () => createRecorder,
+  createPromptRegistry: () => createPromptRegistry,
+  createPinStore: () => createPinStore,
+  createFixture: () => createFixture,
+  assertStable: () => assertStable,
+  assertDeterministic: () => assertDeterministic
+});
+var init_src2 = __esm(() => {
+  init_mock_provider();
+  init_fixtures();
+  init_snapshot();
+  init_regression();
+  init_pinning();
+  init_determinism();
+});
+
+// src/commands/cost.ts
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+var COST_FILE = ".elsium/cost-report.json";
+async function costCommand(args) {
+  const costPath = join(process.cwd(), COST_FILE);
+  if (!existsSync(costPath)) {
+    console.log(`
+  No cost report found.
+
+  Cost reports are generated automatically when you run your app
+  with observability enabled:
+
+    const app = createApp({
+      observe: {
+        costTracking: true,
+      },
+    })
+
+  The report will be saved to .elsium/cost-report.json
+`);
+    return;
+  }
+  try {
+    const data = JSON.parse(readFileSync(costPath, "utf-8"));
+    console.log(`
+  ElsiumAI Cost Report`);
+    console.log(`  ${"\u2500".repeat(50)}`);
+    console.log(`  Generated: ${data.timestamp}`);
+    console.log();
+    console.log(`  Total Cost:          $${data.totalCost.toFixed(6)}`);
+    console.log(`  Total Tokens:        ${data.totalTokens.toLocaleString()}`);
+    console.log(`    Input Tokens:      ${data.totalInputTokens.toLocaleString()}`);
+    console.log(`    Output Tokens:     ${data.totalOutputTokens.toLocaleString()}`);
+    console.log(`  Total API Calls:     ${data.callCount}`);
+    console.log();
+    if (Object.keys(data.byModel).length > 0) {
+      console.log("  By Model:");
+      console.log(`  ${"\u2500".repeat(50)}`);
+      for (const [model, stats] of Object.entries(data.byModel)) {
+        console.log(`    ${model}`);
+        console.log(`      Cost:    $${stats.cost.toFixed(6)}`);
+        console.log(`      Tokens:  ${stats.tokens.toLocaleString()}`);
+        console.log(`      Calls:   ${stats.calls}`);
+      }
+    }
+    console.log();
+  } catch (err) {
+    console.error("Failed to read cost report:", err instanceof Error ? err.message : err);
+    process.exit(1);
+  }
+}
+
+// src/commands/dev.ts
+import { spawn } from "child_process";
+import { existsSync as existsSync2 } from "fs";
+import { resolve } from "path";
+async function devCommand(args) {
+  const entryFile = args[0] ?? "src/index.ts";
+  const cwd = process.cwd();
+  const fullPath = resolve(cwd, entryFile);
+  if (!fullPath.startsWith(`${cwd}/`) && fullPath !== cwd) {
+    console.error("Error: entry file must be within the project directory");
+    process.exit(1);
+  }
+  if (!existsSync2(fullPath)) {
+    console.error(`Entry file not found: ${entryFile}`);
+    console.error("Run this command from your ElsiumAI project root.");
+    process.exit(1);
+  }
+  console.log(`
+  ElsiumAI Dev Server`);
+  console.log(`  Watching: ${entryFile}`);
+  console.log(`  Press Ctrl+C to stop
+`);
+  const child = spawn("bun", ["--watch", entryFile], {
+    stdio: "inherit",
+    cwd: process.cwd()
+  });
+  child.on("error", (err) => {
+    console.error("Failed to start dev server:", err.message);
+    process.exit(1);
+  });
+  child.on("exit", (code) => {
+    process.exit(code ?? 0);
+  });
+  process.on("SIGINT", () => {
+    child.kill("SIGINT");
+  });
+}
+
+// src/commands/eval.ts
+import { existsSync as existsSync3 } from "fs";
+import { join as join2 } from "path";
+async function evalCommand(args) {
+  const evalFile = args[0];
+  if (!evalFile) {
+    console.log(`
+  Usage: elsium eval <file>
+
+  Run an evaluation suite against your prompts.
+
+  Examples:
+    elsium eval ./evals/suite.ts
+    elsium eval ./evals/quality.ts
+
+  Eval file should export a default EvalSuiteConfig:
+
+    import { type EvalSuiteConfig } from '@elsium-ai/testing'
+
+    export default {
+      name: 'my-eval',
+      cases: [
+        {
+          name: 'test-1',
+          input: 'What is TypeScript?',
+          criteria: [
+            { type: 'contains', value: 'typed' },
+            { type: 'length_min', value: 20 },
+          ],
+        },
+      ],
+      runner: async (input) => {
+        // Call your agent/LLM here
+        return response
+      },
+    } satisfies EvalSuiteConfig
+`);
+    return;
+  }
+  const fullPath = join2(process.cwd(), evalFile);
+  if (!existsSync3(fullPath)) {
+    console.error(`Eval file not found: ${evalFile}`);
+    process.exit(1);
+  }
+  try {
+    const mod = await import(fullPath);
+    const config = mod.default ?? mod;
+    if (!config.name || !config.cases || !config.runner) {
+      console.error("Eval file must export a valid EvalSuiteConfig with name, cases, and runner.");
+      process.exit(1);
+    }
+    const { runEvalSuite: runEvalSuite2, formatEvalReport: formatEvalReport2 } = await Promise.resolve().then(() => (init_src2(), exports_src));
+    console.log(`
+  Running eval suite: ${config.name}`);
+    console.log(`  Cases: ${config.cases.length}
+`);
+    const result = await runEvalSuite2(config);
+    console.log(formatEvalReport2(result));
+    if (result.failed > 0) {
+      process.exit(1);
+    }
+  } catch (err2) {
+    console.error("Failed to run eval:", err2 instanceof Error ? err2.message : err2);
+    process.exit(1);
+  }
+}
+
+// src/commands/init.ts
+import { existsSync as existsSync4, mkdirSync as mkdirSync2, writeFileSync as writeFileSync2 } from "fs";
+import { join as join3 } from "path";
+async function initCommand(args) {
+  const projectName = args[0] ?? "my-elsium-app";
+  const projectDir = join3(process.cwd(), projectName);
+  if (existsSync4(projectDir)) {
+    console.error(`Directory "${projectName}" already exists.`);
+    process.exit(1);
+  }
+  console.log(`
+  Creating ElsiumAI project: ${projectName}
+`);
+  const dirs = [
+    "src/agents",
+    "src/tools",
+    "src/policies",
+    "src/gateway",
+    "src/workflows",
+    "evals",
+    "test/agents",
+    ".elsium/baselines",
+    ".elsium/recordings"
+  ];
+  for (const dir of dirs) {
+    mkdirSync2(join3(projectDir, dir), { recursive: true });
+  }
+  const files = [
+    ["package.json", packageJsonContent(projectName)],
+    ["tsconfig.json", tsconfigContent()],
+    ["biome.json", biomeJsonContent()],
+    [".env.example", envExampleContent()],
+    [".gitignore", gitignoreContent()],
+    ["elsium.config.ts", configContent()],
+    ["src/index.ts", indexContent()],
+    ["src/gateway/mesh.ts", meshContent()],
+    ["src/policies/default.ts", policiesContent()],
+    ["src/tools/example.ts", toolContent()],
+    ["src/agents/assistant.ts", agentContent()],
+    ["src/workflows/example.ts", workflowContent()],
+    ["evals/quality.eval.ts", qualityEvalContent()],
+    ["evals/determinism.eval.ts", determinismEvalContent()],
+    ["test/agents/assistant.test.ts", testContent()],
+    [".elsium/baselines/.gitkeep", ""],
+    [".elsium/recordings/.gitkeep", ""],
+    ["README.md", readmeContent(projectName)]
+  ];
+  for (const [filePath, content] of files) {
+    writeFileSync2(join3(projectDir, filePath), content);
+  }
+  console.log("  Created files:");
+  for (const [filePath] of files) {
+    console.log(`    ${projectName}/${filePath}`);
+  }
+  console.log();
+  console.log("  Next steps:");
+  console.log(`    cd ${projectName}`);
+  console.log("    cp .env.example .env   # add your API keys");
+  console.log("    bun install");
+  console.log("    bun run dev");
+  console.log();
+}
+function packageJsonContent(projectName) {
+  return `${JSON.stringify({
+    name: projectName,
+    version: "0.1.0",
+    type: "module",
+    scripts: {
+      dev: "elsium dev",
+      start: "bun src/index.ts",
+      test: "vitest run",
+      eval: "elsium eval evals/quality.eval.ts",
+      "eval:determinism": "elsium eval evals/determinism.eval.ts",
+      lint: "biome check .",
+      format: "biome check --write ."
+    },
+    dependencies: {
+      "@elsium-ai/core": "^0.1.0",
+      "@elsium-ai/gateway": "^0.1.0",
+      "@elsium-ai/agents": "^0.1.0",
+      "@elsium-ai/tools": "^0.1.0",
+      "@elsium-ai/workflows": "^0.1.0",
+      "@elsium-ai/observe": "^0.1.0",
+      "@elsium-ai/app": "^0.1.0",
+      zod: "^3.23.0"
+    },
+    devDependencies: {
+      "@elsium-ai/testing": "^0.1.0",
+      "@biomejs/biome": "^1.9.0",
+      "bun-types": "^1.3.0",
+      typescript: "^5.7.0",
+      vitest: "^3.0.0"
+    }
+  }, null, 2)}
+`;
+}
+function tsconfigContent() {
+  return `${JSON.stringify({
+    compilerOptions: {
+      target: "ESNext",
+      module: "ESNext",
+      moduleResolution: "bundler",
+      strict: true,
+      esModuleInterop: true,
+      skipLibCheck: true,
+      types: ["bun-types"]
+    },
+    include: ["src", "evals", "test", "elsium.config.ts"]
+  }, null, 2)}
+`;
+}
+function biomeJsonContent() {
+  return `${JSON.stringify({
+    $schema: "https://biomejs.dev/schemas/1.9.0/schema.json",
+    organizeImports: { enabled: true },
+    linter: {
+      enabled: true,
+      rules: { recommended: true }
+    },
+    formatter: {
+      enabled: true,
+      indentStyle: "tab",
+      lineWidth: 100
+    }
+  }, null, 2)}
+`;
+}
+function envExampleContent() {
+  return `# Provider API keys \u2014 add at least one
+ANTHROPIC_API_KEY=your-anthropic-key-here
+OPENAI_API_KEY=your-openai-key-here
+`;
+}
+function gitignoreContent() {
+  return `node_modules/
+dist/
+.env
+.env.*
+!.env.example
+.elsium/recordings/*.json
+`;
+}
+function configContent() {
+  return `import type { AppConfig } from '@elsium-ai/app'
+import { env } from '@elsium-ai/core'
+
+const config = {
+	gateway: {
+		providers: {
+			anthropic: { apiKey: env('ANTHROPIC_API_KEY') },
+		},
+		defaultModel: 'claude-sonnet-4-6',
+	},
+	observe: {
+		tracing: true,
+		costTracking: true,
+	},
+	server: {
+		port: 3000,
+	},
+} satisfies Omit<AppConfig, 'agents'>
+
+export default config
+`;
+}
+function indexContent() {
+  return `import { createApp } from '@elsium-ai/app'
+import config from '../elsium.config'
+import { assistant } from './agents/assistant'
+
+const app = createApp({
+	...config,
+	agents: [assistant],
+})
+
+app.listen()
+`;
+}
+function meshContent() {
+  return `import { createProviderMesh } from '@elsium-ai/gateway'
+import { env } from '@elsium-ai/core'
+
+export const mesh = createProviderMesh({
+	providers: [
+		{ name: 'anthropic', config: { apiKey: env('ANTHROPIC_API_KEY') }, priority: 1 },
+		{ name: 'openai', config: { apiKey: env('OPENAI_API_KEY') }, priority: 2 },
+	],
+	strategy: 'fallback',
+	circuitBreaker: {
+		failureThreshold: 3,
+		resetTimeoutMs: 30_000,
+	},
+})
+`;
+}
+function policiesContent() {
+  return `import { createPolicySet, modelAccessPolicy, costLimitPolicy } from '@elsium-ai/core'
+
+export const policies = createPolicySet([
+	modelAccessPolicy(['claude-sonnet-4-6', 'claude-haiku-4-5', 'gpt-4o']),
+	costLimitPolicy(5.0),
+])
+`;
+}
+function toolContent() {
+  return `import { defineTool } from '@elsium-ai/tools'
+import { z } from 'zod'
+
+export const calculatorTool = defineTool({
+	name: 'calculator',
+	description: 'Add two numbers together',
+	input: z.object({
+		a: z.number().describe('First number'),
+		b: z.number().describe('Second number'),
+	}),
+	handler: async ({ a, b }) => {
+		return { result: a + b }
+	},
+})
+`;
+}
+function agentContent() {
+  return `import { defineAgent } from '@elsium-ai/agents'
+import { mesh } from '../gateway/mesh'
+import { calculatorTool } from '../tools/example'
+
+export const assistant = defineAgent(
+	{
+		name: 'assistant',
+		system: 'You are a helpful AI assistant. Use the calculator tool for math questions.',
+		model: 'claude-sonnet-4-6',
+		tools: [calculatorTool],
+		guardrails: {
+			maxIterations: 10,
+			semantic: {
+				relevance: { enabled: true, threshold: 0.5 },
+			},
+		},
+		confidence: {
+			hallucinationRisk: true,
+			relevanceScore: true,
+		},
+	},
+	{ complete: (req) => mesh.complete(req) },
+)
+`;
+}
+function workflowContent() {
+  return `import { defineWorkflow, step } from '@elsium-ai/workflows'
+import { assistant } from '../agents/assistant'
+import { extractText } from '@elsium-ai/core'
+
+export const researchWorkflow = defineWorkflow({
+	name: 'research-pipeline',
+	steps: [
+		step('research', {
+			handler: async (input: string) => {
+				const result = await assistant.run(\`Research this topic: \${input}\`)
+				return extractText(result.message.content)
+			},
+		}),
+		step('summarise', {
+			handler: async (input: string) => {
+				const result = await assistant.run(\`Summarise in two sentences: \${input}\`)
+				return extractText(result.message.content)
+			},
+		}),
+	],
+})
+`;
+}
+function qualityEvalContent() {
+  return `import type { EvalSuiteConfig } from '@elsium-ai/testing'
+import { assistant } from '../src/agents/assistant'
+import { extractText } from '@elsium-ai/core'
+
+export default {
+	name: 'quality',
+	cases: [
+		{
+			name: 'factual-answer',
+			input: 'What is 2 + 2?',
+			criteria: [{ type: 'contains', value: '4' }],
+		},
+		{
+			name: 'polite-greeting',
+			input: 'Hello!',
+			criteria: [{ type: 'contains', value: 'Hello' }],
+		},
+	],
+	runner: async (input) => {
+		const result = await assistant.run(input)
+		return extractText(result.message.content)
+	},
+} satisfies EvalSuiteConfig
+`;
+}
+function determinismEvalContent() {
+  return `import type { EvalSuiteConfig } from '@elsium-ai/testing'
+import { assertDeterministic } from '@elsium-ai/testing'
+import { assistant } from '../src/agents/assistant'
+import { extractText } from '@elsium-ai/core'
+
+export default {
+	name: 'determinism',
+	cases: [
+		{
+			name: 'stable-math',
+			input: 'What is 2 + 2? Reply with just the number.',
+		},
+	],
+	runner: async (input) => {
+		const result = await assertDeterministic(
+			async () => {
+				const res = await assistant.run(input)
+				return extractText(res.message.content)
+			},
+			{ runs: 3, tolerance: 0 },
+		)
+		return result.outputs[0]
+	},
+} satisfies EvalSuiteConfig
+`;
+}
+function testContent() {
+  return `import { describe, it, expect } from 'vitest'
+import { mockProvider, createReplayRecorder, createReplayPlayer } from '@elsium-ai/testing'
+import { defineAgent } from '@elsium-ai/agents'
+import { calculatorTool } from '../../src/tools/example'
+
+describe('assistant agent', () => {
+	it('should respond to a greeting', async () => {
+		const mock = mockProvider({
+			responses: [{ content: 'Hello! How can I help you today?' }],
+		})
+
+		const agent = defineAgent(
+			{
+				name: 'test-assistant',
+				system: 'You are a helpful AI assistant.',
+				model: 'mock',
+				tools: [calculatorTool],
+			},
+			{ complete: (req) => mock.complete(req) },
+		)
+
+		const result = await agent.run('Hello!')
+		expect(result.message.content).toContain('Hello')
+		expect(mock.callCount).toBe(1)
+	})
+
+	it('should replay recorded interactions', async () => {
+		const recorder = createReplayRecorder()
+		const mock = mockProvider({
+			responses: [{ content: 'The answer is 4.' }],
+		})
+
+		const wrappedComplete = recorder.wrap((req) => mock.complete(req))
+		await wrappedComplete({
+			model: 'mock',
+			messages: [{ role: 'user', content: [{ type: 'text', text: 'What is 2+2?' }] }],
+		})
+
+		const player = createReplayPlayer(recorder.toJSON())
+		const replayed = await player.complete({
+			model: 'mock',
+			messages: [{ role: 'user', content: [{ type: 'text', text: 'What is 2+2?' }] }],
+		})
+
+		expect(replayed).toBeDefined()
+	})
+})
+`;
+}
+function readmeContent(projectName) {
+  return `# ${projectName}
+
+Built with [ElsiumAI](https://github.com/elsium-ai/elsium-ai).
+
+## Quick start
+
+\`\`\`bash
+cp .env.example .env   # add your API keys
+bun install
+bun run dev
+\`\`\`
+
+## Scripts
+
+| Command | Description |
+| --- | --- |
+| \`bun run dev\` | Start the dev server |
+| \`bun run test\` | Run unit tests |
+| \`bun run eval\` | Run quality eval suite |
+| \`bun run eval:determinism\` | Run determinism eval |
+| \`bun run lint\` | Lint with Biome |
+| \`bun run format\` | Auto-format with Biome |
+
+## Project structure
+
+- **src/agents/** \u2014 Agent definitions with guardrails
+- **src/tools/** \u2014 Tool schemas validated by Zod
+- **src/policies/** \u2014 Policy sets (model allowlist, cost caps)
+- **src/gateway/** \u2014 Provider mesh with circuit breaker
+- **src/workflows/** \u2014 Multi-step workflows
+- **evals/** \u2014 Eval suites (quality + determinism)
+- **test/** \u2014 Unit tests with mock providers and replay
+`;
+}
+
+// src/commands/prompt.ts
+import { existsSync as existsSync5, readFileSync as readFileSync3, readdirSync } from "fs";
+import { join as join4 } from "path";
+var PROMPTS_DIR = ".elsium/prompts";
+function showHelp() {
+  console.log(`
+  ElsiumAI Prompt Manager
+
+  Usage:
+    elsium prompt list                   List all registered prompts
+    elsium prompt diff <name> <v1> <v2>  Show diff between versions
+    elsium prompt history <name>         Show version history
+    elsium prompt show <name> [version]  Show prompt content
+
+  Prompts are stored in .elsium/prompts/ as JSON files.
+`);
+}
+function loadPromptFiles(promptsPath) {
+  if (!existsSync5(promptsPath)) {
+    return [];
+  }
+  return readdirSync(promptsPath).filter((f) => f.endsWith(".json")).map((f) => {
+    try {
+      return JSON.parse(readFileSync3(join4(promptsPath, f), "utf-8"));
+    } catch {
+      return null;
+    }
+  }).filter((p) => p !== null);
+}
+function handleList(promptsPath) {
+  const allPrompts = loadPromptFiles(promptsPath);
+  if (allPrompts.length === 0) {
+    console.log(`
+  No prompts found. Store prompts in .elsium/prompts/
+`);
+    return;
+  }
+  const prompts = new Map;
+  for (const data of allPrompts) {
+    if (!prompts.has(data.name))
+      prompts.set(data.name, []);
+    const versions = prompts.get(data.name);
+    if (versions)
+      versions.push(data.version);
+  }
+  console.log(`
+  Registered Prompts (${prompts.size})`);
+  console.log(`  ${"\u2500".repeat(50)}`);
+  for (const [name, versions] of prompts) {
+    console.log(`  ${name} \u2014 ${versions.length} version(s): ${versions.join(", ")}`);
+  }
+  console.log();
+}
+function handleHistory(promptsPath, name) {
+  const files = loadPromptFiles(promptsPath).filter((p) => p.name === name).sort((a, b) => a.version.localeCompare(b.version));
+  if (files.length === 0) {
+    console.log(`
+  No versions found for prompt "${name}".
+`);
+    return;
+  }
+  console.log(`
+  Prompt History: ${name} (${files.length} versions)`);
+  console.log(`  ${"\u2500".repeat(50)}`);
+  for (const p of files) {
+    console.log(`  v${p.version} \u2014 ${p.variables.length} variables: ${p.variables.join(", ") || "none"}`);
+  }
+  console.log();
+}
+function handleShow(promptsPath, name, version) {
+  const files = loadPromptFiles(promptsPath).filter((p) => p.name === name);
+  const prompt = version ? files.find((p) => p.version === version) : files.sort((a, b) => b.version.localeCompare(a.version))[0];
+  if (!prompt) {
+    console.log(`
+  Prompt "${name}"${version ? ` v${version}` : ""} not found.
+`);
+    return;
+  }
+  console.log(`
+  Prompt: ${prompt.name} v${prompt.version}`);
+  console.log(`  Variables: ${prompt.variables.join(", ") || "none"}`);
+  console.log(`  ${"\u2500".repeat(50)}`);
+  console.log(prompt.content);
+  console.log(`  ${"\u2500".repeat(50)}
+`);
+}
+function printDiffLines(fromLines, toLines) {
+  const maxLen = Math.max(fromLines.length, toLines.length);
+  for (let i = 0;i < maxLen; i++) {
+    const fl = fromLines[i];
+    const tl = toLines[i];
+    if (fl === undefined) {
+      console.log(`  + ${tl}`);
+    } else if (tl === undefined) {
+      console.log(`  - ${fl}`);
+    } else if (fl !== tl) {
+      console.log(`  - ${fl}`);
+      console.log(`  + ${tl}`);
+    } else {
+      console.log(`    ${fl}`);
+    }
+  }
+}
+function handleDiff(promptsPath, name, v1, v2) {
+  const files = loadPromptFiles(promptsPath).filter((p) => p.name === name);
+  const from = files.find((p) => p.version === v1);
+  const to = files.find((p) => p.version === v2);
+  if (!from || !to) {
+    console.error(`  Could not find both versions: ${v1} and ${v2}`);
+    process.exit(1);
+  }
+  console.log(`
+  Diff: ${name} v${v1} \u2192 v${v2}`);
+  console.log(`  ${"\u2500".repeat(50)}`);
+  printDiffLines(from.content.split(`
+`), to.content.split(`
+`));
+  console.log(`  ${"\u2500".repeat(50)}
+`);
+}
+async function promptCommand(args) {
+  const subcommand = args[0];
+  if (!subcommand || subcommand === "--help" || subcommand === "-h") {
+    showHelp();
+    return;
+  }
+  const promptsPath = join4(process.cwd(), PROMPTS_DIR);
+  switch (subcommand) {
+    case "list":
+      handleList(promptsPath);
+      break;
+    case "history": {
+      const name = args[1];
+      if (!name) {
+        console.error("  Please provide a prompt name: elsium prompt history <name>");
+        process.exit(1);
+      }
+      handleHistory(promptsPath, name);
+      break;
+    }
+    case "show": {
+      const name = args[1];
+      if (!name) {
+        console.error("  Please provide a prompt name: elsium prompt show <name> [version]");
+        process.exit(1);
+      }
+      handleShow(promptsPath, name, args[2]);
+      break;
+    }
+    case "diff": {
+      const name = args[1];
+      const v1 = args[2];
+      const v2 = args[3];
+      if (!name || !v1 || !v2) {
+        console.error("  Usage: elsium prompt diff <name> <v1> <v2>");
+        process.exit(1);
+      }
+      handleDiff(promptsPath, name, v1, v2);
+      break;
+    }
+    default:
+      console.error(`  Unknown subcommand: ${subcommand}`);
+      console.log('  Run "elsium prompt --help" for usage information.');
+      process.exit(1);
+  }
+}
+
+// src/commands/trace.ts
+import { existsSync as existsSync6, readFileSync as readFileSync4, readdirSync as readdirSync2 } from "fs";
+import { join as join5 } from "path";
+var TRACES_DIR = ".elsium/traces";
+function formatStatus(status) {
+  if (status === "ok")
+    return "OK";
+  if (status === "error")
+    return "ERR";
+  return "...";
+}
+function formatDuration(durationMs) {
+  return durationMs ? `${durationMs}ms` : "?";
+}
+function showNoTracesHelp() {
+  console.log(`
+  No traces found.
+
+  Traces are recorded when you run your app with tracing enabled:
+
+    const app = createApp({
+      observe: {
+        tracing: true,
+      },
+    })
+
+  Usage:
+    elsium trace           List recent traces
+    elsium trace <id>      Inspect a specific trace
+`);
+}
+function listTraces(tracesPath) {
+  if (!existsSync6(tracesPath)) {
+    showNoTracesHelp();
+    return;
+  }
+  try {
+    const files = readdirSync2(tracesPath).filter((f) => f.endsWith(".json")).sort().reverse().slice(0, 20);
+    if (files.length === 0) {
+      console.log(`
+  No traces recorded yet.
+`);
+      return;
+    }
+    console.log(`
+  Recent Traces (${files.length})`);
+    console.log(`  ${"\u2500".repeat(60)}`);
+    for (const file of files) {
+      const data = JSON.parse(readFileSync4(join5(tracesPath, file), "utf-8"));
+      const root = data.find((s) => !s.parentId) ?? data[0];
+      if (root) {
+        const status = formatStatus(root.status);
+        const duration = formatDuration(root.durationMs);
+        console.log(`  [${status}] ${root.traceId}  ${root.name}  ${duration}`);
+      }
+    }
+    console.log();
+  } catch (err2) {
+    console.error("Failed to read traces:", err2 instanceof Error ? err2.message : err2);
+  }
+}
+function showTrace(tracesPath, traceId) {
+  const traceFile = join5(tracesPath, `${traceId}.json`);
+  if (!existsSync6(traceFile)) {
+    console.error(`Trace not found: ${traceId}`);
+    process.exit(1);
+  }
+  try {
+    const spans = JSON.parse(readFileSync4(traceFile, "utf-8"));
+    console.log(`
+  Trace: ${traceId}`);
+    console.log(`  Spans: ${spans.length}`);
+    console.log(`  ${"\u2500".repeat(60)}`);
+    const roots = spans.filter((s) => !s.parentId);
+    for (const root of roots) {
+      printSpanTree(root, spans, 0);
+    }
+    console.log();
+  } catch (err2) {
+    console.error("Failed to read trace:", err2 instanceof Error ? err2.message : err2);
+    process.exit(1);
+  }
+}
+async function traceCommand(args) {
+  const traceId = args[0];
+  const tracesPath = join5(process.cwd(), TRACES_DIR);
+  if (!traceId) {
+    listTraces(tracesPath);
+    return;
+  }
+  if (!/^[a-zA-Z0-9_-]+$/.test(traceId)) {
+    console.error("Invalid trace ID format");
+    process.exit(1);
+  }
+  showTrace(tracesPath, traceId);
+}
+function printSpanTree(span, allSpans, depth) {
+  const indent = `  ${"  ".repeat(depth)}`;
+  const status = formatStatus(span.status);
+  const duration = span.durationMs ? `${span.durationMs}ms` : "";
+  const kind = span.kind ? `[${span.kind}]` : "";
+  console.log(`${indent}${kind} ${span.name} (${status}) ${duration}`);
+  for (const [key, value] of Object.entries(span.metadata)) {
+    console.log(`${indent}  ${key}: ${JSON.stringify(value)}`);
+  }
+  for (const event of span.events) {
+    console.log(`${indent}  > ${event.name}${event.data ? `: ${JSON.stringify(event.data)}` : ""}`);
+  }
+  const children = allSpans.filter((s) => s.parentId === span.id);
+  for (const child of children) {
+    printSpanTree(child, allSpans, depth + 1);
+  }
+}
+
+// src/commands/xray.ts
+import { existsSync as existsSync7, readFileSync as readFileSync5 } from "fs";
+import { join as join6 } from "path";
+var XRAY_FILE = ".elsium/xray-history.json";
+async function xrayCommand(args) {
+  const flag = args[0];
+  if (flag === "--help" || flag === "-h") {
+    console.log(`
+  ElsiumAI X-Ray \u2014 Inspect LLM calls
+
+  Usage:
+    elsium xray                Show last call
+    elsium xray --last N       Show last N calls
+    elsium xray --trace <id>   Show call by trace ID
+    elsium xray --raw          Show raw request/response
+
+  X-Ray data is captured when xray mode is enabled:
+
+    const gw = gateway({ ..., xray: true })
+`);
+    return;
+  }
+  const xrayPath = join6(process.cwd(), XRAY_FILE);
+  if (!existsSync7(xrayPath)) {
+    console.log(`
+  No X-Ray data found.
+
+  Enable X-Ray mode on your gateway to capture LLM call details:
+
+    const gw = gateway({ provider: 'anthropic', apiKey: '...', xray: true })
+    await gw.complete({ messages: [...] })
+    console.log(gw.lastCall())
+
+  X-Ray data will be saved to .elsium/xray-history.json
+`);
+    return;
+  }
+  try {
+    const entries = JSON.parse(readFileSync5(xrayPath, "utf-8"));
+    if (flag === "--trace") {
+      const traceId = args[1];
+      if (!traceId) {
+        console.error("  Please provide a trace ID: elsium xray --trace <id>");
+        process.exit(1);
+      }
+      const entry = entries.find((e) => e.traceId === traceId);
+      if (!entry) {
+        console.error(`  Trace not found: ${traceId}`);
+        process.exit(1);
+      }
+      printEntry(entry, args.includes("--raw"));
+      return;
+    }
+    const count = flag === "--last" ? Number.parseInt(args[1] ?? "5", 10) : 1;
+    const showRaw = args.includes("--raw");
+    const toShow = entries.slice(0, count);
+    if (toShow.length === 0) {
+      console.log(`
+  No X-Ray data recorded yet.
+`);
+      return;
+    }
+    console.log(`
+  ElsiumAI X-Ray \u2014 ${toShow.length} call(s)`);
+    console.log(`  ${"\u2500".repeat(60)}`);
+    for (const entry of toShow) {
+      printEntry(entry, showRaw);
+    }
+  } catch (err2) {
+    console.error("Failed to read X-Ray data:", err2 instanceof Error ? err2.message : err2);
+    process.exit(1);
+  }
+}
+function printEntry(entry, raw = false) {
+  console.log(`
+  Trace:    ${entry.traceId}
+  Time:     ${new Date(entry.timestamp).toISOString()}
+  Provider: ${entry.provider}
+  Model:    ${entry.model}
+  Latency:  ${entry.latencyMs}ms
+  Tokens:   ${entry.usage.inputTokens} in / ${entry.usage.outputTokens} out (${entry.usage.totalTokens} total)
+  Cost:     $${entry.cost.totalCost.toFixed(6)}`);
+  if (raw) {
+    console.log(`
+  \u2500\u2500 Request \u2500\u2500
+  ${entry.request.method} ${entry.request.url}
+  Headers: ${JSON.stringify(entry.request.headers, null, 4)}
+  Body: ${JSON.stringify(entry.request.body, null, 4)}
+
+  \u2500\u2500 Response \u2500\u2500
+  Status: ${entry.response.status}
+  Headers: ${JSON.stringify(entry.response.headers, null, 4)}
+  Body: ${JSON.stringify(entry.response.body, null, 4)}`);
+  }
+  console.log(`  ${"\u2500".repeat(60)}`);
+}
+
+// src/cli.ts
+var VERSION = "0.1.0";
+var HELP = `
+  ElsiumAI CLI v${VERSION}
+
+  Usage: elsium <command> [options]
+
+  Commands:
+    init [name]       Scaffold a new ElsiumAI project
+    dev               Start development server with hot reload
+    eval [file]       Run evaluation suite
+    cost              Show cost report from last run
+    trace [id]        Inspect trace from last run
+    xray              Inspect LLM calls (X-Ray mode)
+    prompt            Manage prompt versions
+
+  Options:
+    --help, -h        Show this help message
+    --version, -v     Show version
+
+  Examples:
+    elsium init my-ai-app
+    elsium dev
+    elsium eval ./evals/suite.ts
+    elsium cost
+    elsium trace trc_abc123
+    elsium xray --last 5
+    elsium prompt list
+`;
+async function main() {
+  const args = process.argv.slice(2);
+  const command = args[0];
+  if (!command || command === "--help" || command === "-h") {
+    console.log(HELP);
+    process.exit(0);
+  }
+  if (command === "--version" || command === "-v") {
+    console.log(VERSION);
+    process.exit(0);
+  }
+  switch (command) {
+    case "init":
+      await initCommand(args.slice(1));
+      break;
+    case "dev":
+      await devCommand(args.slice(1));
+      break;
+    case "eval":
+      await evalCommand(args.slice(1));
+      break;
+    case "cost":
+      await costCommand(args.slice(1));
+      break;
+    case "trace":
+      await traceCommand(args.slice(1));
+      break;
+    case "xray":
+      await xrayCommand(args.slice(1));
+      break;
+    case "prompt":
+      await promptCommand(args.slice(1));
+      break;
+    default:
+      console.error(`Unknown command: ${command}`);
+      console.log(HELP);
+      process.exit(1);
+  }
+}
+main().catch((err2) => {
+  const message = err2 instanceof Error ? err2.message : String(err2);
+  console.error("Error:", message);
+  process.exit(1);
+});