@axiom-lattice/gateway 3.0.0 → 3.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axiom-lattice/gateway",
-  "version": "3.0.0",
+  "version": "3.0.2",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
   "types": "dist/index.d.ts",
@@ -41,11 +41,11 @@
     "redis": "^5.0.1",
     "uuid": "^9.0.1",
     "zod": "3.25.76",
-    "@axiom-lattice/cli-a2a": "0.1.6",
-    "@axiom-lattice/core": "3.0.0",
-    "@axiom-lattice/pg-stores": "2.0.0",
-    "@axiom-lattice/protocols": "3.0.0",
-    "@axiom-lattice/queue-redis": "1.0.54"
+    "@axiom-lattice/cli-a2a": "0.1.7",
+    "@axiom-lattice/core": "3.0.2",
+    "@axiom-lattice/pg-stores": "2.0.2",
+    "@axiom-lattice/protocols": "3.0.1",
+    "@axiom-lattice/queue-redis": "1.0.55"
   },
   "devDependencies": {
     "@types/bcrypt": "^6.0.0",

package/src/controllers/eval.ts

@@ -394,11 +394,15 @@ export function registerEvalRoutes(app: FastifyInstance): void {
     try {
       const tenantId = getTenantId(request);
       const { pid } = request.params as { pid: string };
-      const runId = await evalRunner.startRun(tenantId, pid);
+      const body = (request.body ?? {}) as { suiteIds?: string[] };
+      const runId = await evalRunner.startRun(tenantId, pid, body?.suiteIds);
       reply.status(202).send({ success: true, message: "Run started", data: { run_id: runId } });
     } catch (err) {
       const msg = (err as Error).message;
-      const code = msg === "Project not found" ? 404 : msg.includes("already in progress") ? 409 : 500;
+      const code = msg === "Project not found" ? 404
+        : msg.includes("already in progress") ? 409
+        : msg.includes("No suites match") ? 400
+        : 500;
       reply.status(code).send({ success: false, message: msg });
     }
   });

package/src/services/__tests__/eval-runner.test.ts

@@ -0,0 +1,131 @@
+/**
+ * EvalRunner 孤儿 run 处理测试
+ *
+ * 网关重启后 DB 中 status=running 的 run 在内存中不存在（孤儿）。
+ * startRun 应先将孤儿标记为 failed，再允许新 run 启动。
+ */
+
+import { describe, expect, it, jest, beforeEach, afterEach } from "@jest/globals";
+
+const mockStore = {
+  getProjectById: jest.fn(),
+  getRunsByTenant: jest.fn(),
+  getSuitesByProject: jest.fn(),
+  getCasesBySuite: jest.fn(),
+  updateRunStatus: jest.fn(),
+  createRun: jest.fn(),
+  createRunResult: jest.fn(),
+  getResultsByRun: jest.fn(),
+};
+
+jest.mock("@axiom-lattice/core", () => {
+  const actual = jest.requireActual("@axiom-lattice/core");
+  return {
+    ...actual,
+    getStoreLattice: jest.fn(() => ({ store: mockStore })),
+    modelLatticeManager: {
+      hasLattice: jest.fn(() => true),
+      getAllLattices: jest.fn(() => [{ key: "mock-model" }]),
+    },
+    LatticeEvalProject: jest.fn().mockImplementation(() => ({
+      calibrateJudge: jest.fn().mockResolvedValue({ ok: true }),
+      runAllSuitesBatch: jest.fn().mockResolvedValue({
+        report: { summary: { total_cases: 0, passed_cases: 0, failed_cases: 0, pass_rate: 0 } },
+      }),
+    })),
+  };
+});
+
+import { evalRunner } from "../eval-runner";
+
+const runningRun = (id: string, createdAt = new Date()) => ({
+  id,
+  projectId: "p1",
+  tenantId: "t1",
+  status: "running" as const,
+  concurrency: 3,
+  totalCases: 0,
+  passedCases: 0,
+  failedCases: 0,
+  avgScore: 0,
+  createdAt,
+});
+
+describe("EvalRunner orphaned run handling", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockStore.getProjectById.mockResolvedValue({ id: "p1", name: "eval-test", tenantId: "t1" });
+    mockStore.getSuitesByProject.mockResolvedValue([]);
+    mockStore.getCasesBySuite.mockResolvedValue([]);
+    mockStore.updateRunStatus.mockResolvedValue(null);
+    mockStore.createRun.mockResolvedValue(runningRun("new-run-1"));
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("marks orphaned running runs as failed before starting a new run", async () => {
+    const orphan = runningRun("orphan-1", new Date(Date.now() - 60_000));
+    mockStore.getRunsByTenant
+      .mockResolvedValueOnce([orphan])                  // first call: orphan found
+      .mockResolvedValueOnce([]);                        // second call: all cleaned
+
+    const runId = await evalRunner.startRun("t1", "p1");
+
+    expect(mockStore.updateRunStatus).toHaveBeenCalledWith(
+      "t1",
+      "orphan-1",
+      expect.objectContaining({ status: "failed" })
+    );
+    expect(mockStore.createRun).toHaveBeenCalled();
+    expect(runId).toBeTruthy();
+  });
+
+  it("does not clean a run that is alive in memory", async () => {
+    mockStore.getRunsByTenant.mockResolvedValue([]);
+
+    const runId = await evalRunner.startRun("t1", "p1");
+    expect(runId).toBeTruthy();
+    expect(mockStore.updateRunStatus).not.toHaveBeenCalled();
+  });
+
+  it("does not clean a run created after process start (other instance)", async () => {
+    const live = runningRun("other-instance-1", new Date()); // created now → after process start
+    mockStore.getRunsByTenant
+      .mockResolvedValueOnce([live])
+      .mockResolvedValueOnce([live]); // still running after cleanup attempt
+
+    await expect(evalRunner.startRun("t1", "p1")).rejects.toThrow(/already in progress/);
+    expect(mockStore.updateRunStatus).not.toHaveBeenCalled();
+  });
+
+  it("runs only the requested suites when suiteIds is provided", async () => {
+    mockStore.getSuitesByProject.mockResolvedValue([
+      { id: "s-dev", name: "po-extraction-user-sample", tenantId: "t1", projectId: "p1", createdAt: new Date(), updatedAt: new Date() },
+      { id: "s-val", name: "po-extraction-validation", tenantId: "t1", projectId: "p1", createdAt: new Date(), updatedAt: new Date() },
+    ]);
+    const { LatticeEvalProject } = jest.requireMock("@axiom-lattice/core");
+    const runProjectConfigs: any[] = [];
+    LatticeEvalProject.mockImplementation((config: any) => {
+      runProjectConfigs.push(config);
+      return {
+        calibrateJudge: jest.fn().mockResolvedValue({ ok: true }),
+        runAllSuitesBatch: jest.fn().mockResolvedValue({
+          report: { summary: { total_cases: 0, passed_cases: 0, failed_cases: 0, pass_rate: 0 } },
+        }),
+      };
+    });
+
+    const runId = await evalRunner.startRun("t1", "p1", ["s-val"]);
+    expect(runId).toBeTruthy();
+    expect(runProjectConfigs[0].suites.map((s: any) => s.suiteName)).toEqual(["po-extraction-validation"]);
+  });
+
+  it("throws when suiteIds filter matches nothing", async () => {
+    mockStore.getSuitesByProject.mockResolvedValue([
+      { id: "s-dev", name: "dev", tenantId: "t1", projectId: "p1", createdAt: new Date(), updatedAt: new Date() },
+    ]);
+    await expect(evalRunner.startRun("t1", "p1", ["s-val"])).rejects.toThrow(/No suites match/);
+  });
+});

package/src/services/eval-runner.ts

@@ -37,15 +37,27 @@ function mapLogs(logs: LatticeEvalLogEvent[]): Array<{ timestamp: string; level:
   }));
 }
 
+/**
+ * EvalRunner — in-process evaluation run scheduler.
+ *
+ * @remarks
+ * Single-instance assumption: run state lives in an in-memory Map plus a
+ * per-instance EventEmitter. Deploying multiple gateway instances requires
+ * a distributed lease (runner instance id + heartbeat) before the orphan
+ * cleanup below is safe — without it, instance B could mark instance A's
+ * live run failed. The createdAt guard limits cleanup to runs created
+ * BEFORE this process started, which are true orphans of a restart.
+ */
 class EvalRunner {
   private runs = new Map<string, RunContext>();
   private eventEmitter = new EventEmitter();
+  private readonly processStartedAt = new Date();
 
   getEventEmitter(): EventEmitter {
     return this.eventEmitter;
   }
 
-  async startRun(tenantId: string, projectId: string): Promise<string> {
+  async startRun(tenantId: string, projectId: string, suiteIds?: string[]): Promise<string> {
     const store = this.getEvalStore();
     const project = await store.getProjectById(tenantId, projectId);
     if (!project) throw new Error("Project not found");
@@ -57,11 +69,31 @@ class EvalRunner {
     }
 
     const existingRuns = await store.getRunsByTenant(tenantId, { projectId, status: "running" });
-    if (existingRuns.length > 0) {
+    for (const run of existingRuns) {
+      // Only clean runs created before this process started: they are true
+      // orphans of a gateway restart. Runs created after startup but missing
+      // from memory belong to another live instance — leave them alone.
+      const orphaned = !this.runs.has(run.id) && run.createdAt < this.processStartedAt;
+      if (orphaned) {
+        await store.updateRunStatus(tenantId, run.id, {
+          status: "failed",
+          error: "Gateway restarted — run orphaned",
+          completedAt: new Date(),
+        });
+      }
+    }
+    const stillRunning = await store.getRunsByTenant(tenantId, { projectId, status: "running" });
+    if (stillRunning.length > 0) {
       throw new Error("A run is already in progress for this project");
     }
 
-    const suites = await store.getSuitesByProject(tenantId, projectId);
+    let suites = await store.getSuitesByProject(tenantId, projectId);
+    if (suiteIds && suiteIds.length > 0) {
+      suites = suites.filter((s) => suiteIds.includes(s.id));
+      if (suites.length === 0) {
+        throw new Error("No suites match the requested suiteIds filter");
+      }
+    }
     const evalSuites: LatticeEvalSuiteType[] = [];
     let totalCases = 0;
 
@@ -173,6 +205,15 @@ class EvalRunner {
     const runPromise = (async () => {
       try {
         const evalProject = new LatticeEvalProject(projectConfig, onCaseComplete);
+        const calibration = await evalProject.calibrateJudge();
+        if (!calibration.ok) {
+          await store.updateRunStatus(tenantId, runId, {
+            status: "failed",
+            error: `Judge calibration failed: ${calibration.reason}`,
+            completedAt: new Date(),
+          });
+          throw new Error(`Judge calibration failed: ${calibration.reason}`);
+        }
         const { report } = await evalProject.runAllSuitesBatch(concurrency, abortController.signal);
 
         if (!abortController.signal.aborted) {

package/src/services/local-a2a.ts

@@ -39,6 +39,8 @@ export function findWorkspaceRoot(startDir: string): string | null {
     if (parent === current) return null;
     current = parent;
   }
+
+  return null;
 }
 
 export function resolveLocalA2AWorkingDirectory(options: {