@axiom-lattice/gateway 3.0.2 → 3.0.3

package/src/services/eval-runner.ts

@@ -1,283 +0,0 @@
-import { EventEmitter } from "events";
-import {
-  getStoreLattice,
-  modelLatticeManager,
-  LatticeEvalProject,
-} from "@axiom-lattice/core";
-import type { EvalStore } from "@axiom-lattice/protocols";
-import type {
-  CaseRunResult,
-  LatticeEvalBatchReport,
-  LatticeEvalProjectType,
-  LatticeEvalSuiteType,
-  LatticeEvalLogEvent,
-} from "@axiom-lattice/core";
-import { v4 as uuidv4 } from "uuid";
-
-export interface EvalStreamEvent {
-  type: "progress" | "completed" | "error";
-  runId: string;
-  data: Record<string, unknown>;
-}
-
-interface RunContext {
-  runId: string;
-  projectId: string;
-  tenantId: string;
-  abortController: AbortController;
-  promise: Promise<{ report: LatticeEvalBatchReport }>;
-}
-
-function mapLogs(logs: LatticeEvalLogEvent[]): Array<{ timestamp: string; level: string; message: string; data?: unknown }> {
-  return logs.map((l) => ({
-    timestamp: l.ts,
-    level: l.level,
-    message: l.message,
-    data: l.data,
-  }));
-}
-
-/**
- * EvalRunner — in-process evaluation run scheduler.
- *
- * @remarks
- * Single-instance assumption: run state lives in an in-memory Map plus a
- * per-instance EventEmitter. Deploying multiple gateway instances requires
- * a distributed lease (runner instance id + heartbeat) before the orphan
- * cleanup below is safe — without it, instance B could mark instance A's
- * live run failed. The createdAt guard limits cleanup to runs created
- * BEFORE this process started, which are true orphans of a restart.
- */
-class EvalRunner {
-  private runs = new Map<string, RunContext>();
-  private eventEmitter = new EventEmitter();
-  private readonly processStartedAt = new Date();
-
-  getEventEmitter(): EventEmitter {
-    return this.eventEmitter;
-  }
-
-  async startRun(tenantId: string, projectId: string, suiteIds?: string[]): Promise<string> {
-    const store = this.getEvalStore();
-    const project = await store.getProjectById(tenantId, projectId);
-    if (!project) throw new Error("Project not found");
-
-    for (const ctx of this.runs.values()) {
-      if (ctx.projectId === projectId) {
-        throw new Error("A run is already in progress for this project");
-      }
-    }
-
-    const existingRuns = await store.getRunsByTenant(tenantId, { projectId, status: "running" });
-    for (const run of existingRuns) {
-      // Only clean runs created before this process started: they are true
-      // orphans of a gateway restart. Runs created after startup but missing
-      // from memory belong to another live instance — leave them alone.
-      const orphaned = !this.runs.has(run.id) && run.createdAt < this.processStartedAt;
-      if (orphaned) {
-        await store.updateRunStatus(tenantId, run.id, {
-          status: "failed",
-          error: "Gateway restarted — run orphaned",
-          completedAt: new Date(),
-        });
-      }
-    }
-    const stillRunning = await store.getRunsByTenant(tenantId, { projectId, status: "running" });
-    if (stillRunning.length > 0) {
-      throw new Error("A run is already in progress for this project");
-    }
-
-    let suites = await store.getSuitesByProject(tenantId, projectId);
-    if (suiteIds && suiteIds.length > 0) {
-      suites = suites.filter((s) => suiteIds.includes(s.id));
-      if (suites.length === 0) {
-        throw new Error("No suites match the requested suiteIds filter");
-      }
-    }
-    const evalSuites: LatticeEvalSuiteType[] = [];
-    let totalCases = 0;
-
-    for (const suite of suites) {
-      const cases = await store.getCasesBySuite(tenantId, suite.id);
-      totalCases += cases.length;
-      evalSuites.push({
-        suiteName: suite.name,
-        cases: cases.map((c) => ({
-          caseId: c.id,
-          input: { message: c.inputMessage, files: c.inputFiles },
-          steps: c.steps,
-          output: { type: "message_content" },
-          eval: {
-            content_assertion: c.contentAssertion,
-            eval_rubrics: c.rubrics?.map((r) => ({
-              dimension: r.name,
-              weight: r.weight,
-              description: r.description,
-            })),
-          },
-        })),
-      });
-    }
-
-    const judgeCfg = (project.judgeModelConfig ?? {}) as Record<string, unknown>;
-    const judgeModelKey = (judgeCfg.modelKey as string) || "";
-    let resolvedJudgeModelKey: string;
-
-    if (judgeModelKey) {
-      if (!modelLatticeManager.hasLattice(judgeModelKey)) {
-        throw new Error(`Judge model "${judgeModelKey}" is not registered`);
-      }
-      resolvedJudgeModelKey = judgeModelKey;
-    } else {
-      const firstModel = modelLatticeManager.getAllLattices()[0];
-      if (!firstModel) {
-        throw new Error("No model registered — cannot run eval without a judge model");
-      }
-      resolvedJudgeModelKey = firstModel.key;
-      console.warn(`[eval-runner] Project "${project.name}" has no judge modelKey — falling back to first registered model "${firstModel.key}"`);
-    }
-
-    const runId = uuidv4();
-    const concurrency = Math.max(1, Math.floor((project as unknown as Record<string, unknown>).concurrency as number) || 3);
-    await store.createRun(tenantId, projectId, runId, { totalCases, concurrency });
-
-    const tsc = (project.targetServerConfig ?? {}) as Record<string, unknown>;
-    const workspaceId = (tsc.workspace_id as string) || "";
-    const projectIdFromConfig = (tsc.project_id as string) || "";
-
-    const projectConfig: LatticeEvalProjectType = {
-      projectName: project.name,
-      version: project.version,
-      description: project.description,
-      suites: evalSuites,
-      judge_agent_config: { modelKey: resolvedJudgeModelKey },
-      lattice_server_config: {
-        tenant_id: tenantId,
-        workspace_id: workspaceId,
-        project_id: projectIdFromConfig,
-      },
-      concurrency,
-    };
-
-    const abortController = new AbortController();
-    const stats = { passed: 0, failed: 0, totalScore: 0 };
-
-    const onCaseComplete = async (result: CaseRunResult, suiteName: string): Promise<void> => {
-      const passed = result.result?.pass ?? false;
-      const score = result.result?.final_score ?? 0;
-
-      if (passed) stats.passed++; else stats.failed++;
-      stats.totalScore += score;
-
-      const completedCount = stats.passed + stats.failed;
-
-      await store.createRunResult(tenantId, runId, uuidv4(), {
-        suiteName,
-        caseId: result.caseId,
-        pass: passed,
-        score,
-        summary: result.result?.summary,
-        dimensionResults: result.result?.dimension_results?.map((d) => ({
-          name: d.name,
-          score: d.score,
-          reason: d.reason,
-        })),
-        durationMs: result.duration_ms,
-        messages: result.messages,
-        logs: mapLogs(result.logs),
-        error: result.error,
-      });
-
-      const avgScore = completedCount > 0 ? stats.totalScore / completedCount : 0;
-      await store.updateRunStatus(tenantId, runId, {
-        passedCases: stats.passed,
-        failedCases: stats.failed,
-        avgScore,
-      });
-
-      this.eventEmitter.emit(`run:${runId}`, {
-        type: "progress",
-        runId,
-        data: { suiteName, caseId: result.caseId, pass: passed, score, completed: completedCount, total: totalCases },
-      } satisfies EvalStreamEvent);
-    };
-
-    const runPromise = (async () => {
-      try {
-        const evalProject = new LatticeEvalProject(projectConfig, onCaseComplete);
-        const calibration = await evalProject.calibrateJudge();
-        if (!calibration.ok) {
-          await store.updateRunStatus(tenantId, runId, {
-            status: "failed",
-            error: `Judge calibration failed: ${calibration.reason}`,
-            completedAt: new Date(),
-          });
-          throw new Error(`Judge calibration failed: ${calibration.reason}`);
-        }
-        const { report } = await evalProject.runAllSuitesBatch(concurrency, abortController.signal);
-
-        if (!abortController.signal.aborted) {
-          const completedCount = stats.passed + stats.failed;
-          await store.updateRunStatus(tenantId, runId, {
-            status: "completed",
-            completedAt: new Date(),
-            passedCases: stats.passed,
-            failedCases: stats.failed,
-            avgScore: completedCount > 0 ? stats.totalScore / completedCount : 0,
-          });
-
-          this.eventEmitter.emit(`run:${runId}`, {
-            type: "completed",
-            runId,
-            data: { passed: stats.passed, failed: stats.failed, avgScore: completedCount > 0 ? stats.totalScore / completedCount : 0 },
-          } satisfies EvalStreamEvent);
-        }
-
-        return { report };
-      } catch (err) {
-        const errorMsg = (err as Error).message;
-        if (!abortController.signal.aborted) {
-          await store.updateRunStatus(tenantId, runId, {
-            status: "failed",
-            error: errorMsg,
-            completedAt: new Date(),
-          });
-
-          this.eventEmitter.emit(`run:${runId}`, {
-            type: "error",
-            runId,
-            data: { message: errorMsg },
-          } satisfies EvalStreamEvent);
-        }
-
-        throw err;
-      } finally {
-        this.runs.delete(runId);
-      }
-    })();
-
-    this.runs.set(runId, { runId, projectId, tenantId, abortController, promise: runPromise });
-    runPromise.catch(() => {});
-
-    return runId;
-  }
-
-  async abortRun(runId: string): Promise<boolean> {
-    const ctx = this.runs.get(runId);
-    if (!ctx) return false;
-    ctx.abortController.abort();
-    const store = this.getEvalStore();
-    await store.updateRunStatus(ctx.tenantId, runId, { status: "aborted", completedAt: new Date() });
-    return true;
-  }
-
-  isRunning(runId: string): boolean {
-    return this.runs.has(runId);
-  }
-
-  private getEvalStore(): EvalStore {
-    return getStoreLattice("default", "eval").store as EvalStore;
-  }
-}
-
-export const evalRunner = new EvalRunner();

package/src/services/local-a2a.ts

@@ -1,354 +0,0 @@
-import { spawn, type ChildProcess } from "node:child_process";
-import { existsSync, statSync } from "node:fs";
-import { dirname, join, parse, resolve } from "node:path";
-import { setTimeout as delay } from "node:timers/promises";
-import {
-  type LocalA2AProviderState,
-  type LocalRuntimeConfig,
-} from "@axiom-lattice/protocols";
-
-interface LocalA2AProcessState extends LocalA2AProviderState {
-  enabled: boolean;
-  localRuntime?: LocalRuntimeConfig;
-  process?: ChildProcess;
-}
-
-const STDERR_TAIL_LIMIT = 4096;
-
-function readBooleanEnv(
-  name: string,
-  defaultValue: boolean,
-  env: NodeJS.ProcessEnv = process.env,
-): boolean {
-  const value = env[name];
-  if (value === undefined) return defaultValue;
-  return value === "1" || value.toLowerCase() === "true";
-}
-
-export function findWorkspaceRoot(startDir: string): string | null {
-  let current = resolve(startDir);
-
-  const { root } = parse(current);
-
-  while (current !== root) {
-    if (existsSync(join(current, "pnpm-workspace.yaml"))) {
-      return current;
-    }
-
-    const parent = dirname(current);
-    if (parent === current) return null;
-    current = parent;
-  }
-
-  return null;
-}
-
-export function resolveLocalA2AWorkingDirectory(options: {
-  envWorkdir?: string;
-  commandCwd?: string;
-  currentCwd: string;
-}): string {
-  if (options.envWorkdir) return options.envWorkdir;
-  if (options.commandCwd) return options.commandCwd;
-  return findWorkspaceRoot(options.currentCwd) || options.currentCwd;
-}
-
-function validateWorkingDirectory(cwd: string): string | null {
-  try {
-    const stat = statSync(cwd);
-    if (!stat.isDirectory()) {
-      return `Working directory is not a directory: ${cwd}`;
-    }
-    return null;
-  } catch (error: unknown) {
-    const message = error instanceof Error ? error.message : String(error);
-    return `Working directory is not accessible: ${cwd} (${message})`;
-  }
-}
-
-export function appendStderrTail(current: string, chunk: string): string {
-  const next = `${current}${chunk}`;
-  if (next.length <= STDERR_TAIL_LIMIT) return next;
-  return next.slice(-STDERR_TAIL_LIMIT);
-}
-
-function formatExitMessage(exitReason: string | number, stderrTail: string): string {
-  const stderr = stderrTail.trim();
-  return stderr
-    ? `Exited before becoming ready (${exitReason}): ${stderr}`
-    : `Exited before becoming ready (${exitReason})`;
-}
-
-async function probeReady(
-  agentCardUrl: string,
-  healthUrl: string | undefined,
-  timeoutMs: number,
-): Promise<boolean> {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), timeoutMs);
-  try {
-    if (healthUrl) {
-      const resp = await fetch(healthUrl, { signal: controller.signal });
-      if (!resp.ok) return false;
-    }
-    const cardResp = await fetch(agentCardUrl, { signal: controller.signal });
-    return cardResp.ok;
-  } catch {
-    return false;
-  } finally {
-    clearTimeout(timer);
-  }
-}
-
-export class LocalA2AService {
-  private static instance: LocalA2AService | null = null;
-  private states: Map<string, LocalA2AProcessState> = new Map();
-  private autoStartEnabled: boolean;
-
-  constructor() {
-    this.autoStartEnabled = readBooleanEnv("LOCAL_A2A_AUTO_START", true);
-  }
-
-  public static getInstance(): LocalA2AService {
-    if (!LocalA2AService.instance) {
-      LocalA2AService.instance = new LocalA2AService();
-    }
-    return LocalA2AService.instance;
-  }
-
-  getOrCreateState(
-    runtimeId: string,
-    localRuntime: LocalRuntimeConfig,
-  ): LocalA2AProcessState {
-    let state = this.states.get(runtimeId);
-    if (!state) {
-      state = {
-        runtimeId,
-        label: localRuntime.label || runtimeId,
-        status: this.autoStartEnabled ? "stopped" : "disabled",
-        enabled: this.autoStartEnabled,
-        assistantId: runtimeId,
-        agentCardUrl: localRuntime.agentCardUrl,
-        healthUrl: localRuntime.healthUrl,
-        localRuntime,
-      };
-      this.states.set(runtimeId, state);
-    } else {
-      state.localRuntime = localRuntime;
-      state.label = localRuntime.label || state.label;
-      state.agentCardUrl = localRuntime.agentCardUrl;
-      state.healthUrl = localRuntime.healthUrl;
-    }
-    return state;
-  }
-
-  async getRegisteredState(
-    runtimeId: string,
-    localRuntime: LocalRuntimeConfig,
-  ): Promise<LocalA2AProviderState> {
-    const state = this.getOrCreateState(runtimeId, localRuntime);
-
-    if (state.process && state.status === "running") {
-      const alive = await probeReady(
-        localRuntime.agentCardUrl,
-        localRuntime.healthUrl,
-        2000,
-      );
-      if (!alive) {
-        state.process = undefined;
-        state.pid = undefined;
-        state.status = "stopped";
-        state.message = "Process exited unexpectedly";
-      }
-    }
-
-    return this.snapshotState(state);
-  }
-
-  async startProvider(
-    runtimeId: string,
-    localRuntime: LocalRuntimeConfig,
-  ): Promise<LocalA2AProviderState> {
-    const state = this.getOrCreateState(runtimeId, localRuntime);
-
-    if (!state.enabled) {
-      state.status = "disabled";
-      state.message = "Disabled by configuration";
-      return this.snapshotState(state);
-    }
-
-    if (state.process && state.status === "running") {
-      return this.snapshotState(state);
-    }
-
-    state.enabled = true;
-    const { executable, args, cwd, env: cmdEnv } = localRuntime.command;
-    const resolvedCwd = resolveLocalA2AWorkingDirectory({
-      envWorkdir: process.env.LOCAL_A2A_WORKDIR || undefined,
-      commandCwd: cwd,
-      currentCwd: process.cwd(),
-    });
-    const cwdError = validateWorkingDirectory(resolvedCwd);
-    if (cwdError) {
-      state.process = undefined;
-      state.pid = undefined;
-      state.status = "failed";
-      state.message = cwdError;
-      return this.snapshotState(state);
-    }
-
-    state.status = "starting";
-    state.message = "Starting";
-    let stderrTail = "";
-
-    const childEnv = cmdEnv ? { ...process.env, ...cmdEnv } : process.env;
-
-    const child = spawn(executable, args, {
-      cwd: resolvedCwd,
-      env: childEnv,
-      stdio: ["ignore", "pipe", "pipe"],
-      shell: false,
-    });
-
-    state.process = child;
-    state.pid = child.pid ?? undefined;
-
-    child.once("error", (error: Error) => {
-      const current = this.states.get(runtimeId);
-      if (!current) return;
-      current.process = undefined;
-      current.pid = undefined;
-      current.status = "failed";
-      const stderr = stderrTail.trim();
-      current.message = stderr
-        ? `Failed to start: ${error.message}: ${stderr}`
-        : `Failed to start: ${error.message}`;
-    });
-
-    child.stdout.on("data", (chunk: Buffer) => {
-      const text = chunk.toString("utf8").trim();
-      if (text) console.log(`[local-a2a:${runtimeId}] ${text}`);
-    });
-    child.stderr.on("data", (chunk: Buffer) => {
-      const rawText = chunk.toString("utf8");
-      stderrTail = appendStderrTail(stderrTail, rawText);
-      const text = rawText.trim();
-      if (text) console.error(`[local-a2a:${runtimeId}] ${text}`);
-    });
-    child.on("close", (code, signal) => {
-      const current = this.states.get(runtimeId);
-      if (!current) return;
-      current.process = undefined;
-      current.pid = undefined;
-      const exitReason = signal ?? code ?? "unknown";
-      if (current.status === "running") {
-        current.status = "stopped";
-        current.message = `Stopped (${exitReason})`;
-      } else if (current.status === "starting") {
-        current.status = "failed";
-        current.message = formatExitMessage(exitReason, stderrTail);
-      }
-    });
-
-    const ready = await this.waitUntilReady(localRuntime, state, 15000);
-    if (!ready) {
-      if (state.process) {
-        state.process.kill("SIGTERM");
-      }
-      state.process = undefined;
-      state.pid = undefined;
-      if ((state.status as string) !== "failed") {
-        state.status = "failed";
-        state.message = "Server did not become ready";
-      }
-      return this.snapshotState(state);
-    }
-
-    state.status = "running";
-    state.message = "Running";
-    state.lastStartedAt = new Date().toISOString();
-    return this.snapshotState(state);
-  }
-
-  async stopProvider(runtimeId: string): Promise<LocalA2AProviderState> {
-    const state = this.states.get(runtimeId);
-    if (!state) {
-      return this.emptyState(runtimeId);
-    }
-
-    const child = state.process;
-    if (child) {
-      child.kill("SIGTERM");
-      await delay(750);
-      if (child.exitCode === null) {
-        child.kill("SIGKILL");
-      }
-      state.process = undefined;
-    }
-    state.pid = undefined;
-    state.status = "stopped";
-    state.message = "Stopped";
-    state.lastStoppedAt = new Date().toISOString();
-    return this.snapshotState(state);
-  }
-
-  async restartProvider(
-    runtimeId: string,
-    localRuntime: LocalRuntimeConfig,
-  ): Promise<LocalA2AProviderState> {
-    await this.stopProvider(runtimeId);
-    return this.startProvider(runtimeId, localRuntime);
-  }
-
-  removeState(runtimeId: string): void {
-    const state = this.states.get(runtimeId);
-    if (state?.process) {
-      state.process.kill("SIGTERM");
-      state.process = undefined;
-    }
-    this.states.delete(runtimeId);
-  }
-
-  private async waitUntilReady(
-    localRuntime: LocalRuntimeConfig,
-    state: LocalA2AProcessState,
-    timeoutMs: number,
-  ): Promise<boolean> {
-    const start = Date.now();
-    while (Date.now() - start < timeoutMs) {
-      if (state.status === "failed") return false;
-      if (await probeReady(localRuntime.agentCardUrl, localRuntime.healthUrl, 1500)) {
-        return true;
-      }
-      await delay(500);
-    }
-    return false;
-  }
-
-  private emptyState(runtimeId: string): LocalA2AProviderState {
-    return {
-      runtimeId,
-      label: runtimeId,
-      status: "stopped",
-      enabled: false,
-    };
-  }
-
-  private snapshotState(state: LocalA2AProcessState): LocalA2AProviderState {
-    return {
-      runtimeId: state.runtimeId,
-      label: state.label,
-      status: state.status,
-      enabled: state.enabled,
-      assistantId: state.assistantId,
-      agentCardUrl: state.agentCardUrl,
-      healthUrl: state.healthUrl,
-      pid: state.pid,
-      message: state.message,
-      lastStartedAt: state.lastStartedAt,
-      lastStoppedAt: state.lastStoppedAt,
-    };
-  }
-}
-
-export const localA2AService = LocalA2AService.getInstance();