@axiom-lattice/gateway 2.1.111 → 2.1.113

package/src/services/__tests__/ExportImportService.test.ts

@@ -0,0 +1,130 @@
+import { ExportImportService } from '../ExportImportService';
+import { ExportableEntityRegistry } from '@axiom-lattice/core';
+import type { ExportableEntityDefinition, ExportableEntity } from '@axiom-lattice/core';
+
+function makeDef(
+  entityType: string,
+  entities: ExportableEntity[] = [],
+  dependsOn: string[] = [],
+): ExportableEntityDefinition {
+  return {
+    entityType,
+    label: entityType,
+    category: 'core',
+    dependsOn,
+    cascadeParents: [],
+    listForExport: jest.fn().mockResolvedValue(entities),
+    previewImport: jest.fn().mockResolvedValue({ conflicts: [], insertions: [], errors: [] }),
+    applyImport: jest.fn().mockResolvedValue({ newId: 'new-id' }),
+  };
+}
+
+describe('ExportImportService', () => {
+  beforeEach(() => {
+    (ExportableEntityRegistry as any).instance = undefined;
+    const registry = ExportableEntityRegistry.getInstance();
+    const s1 = { _exportId: '', data: { id: 's1', name: 'Skill 1' } };
+    const s2 = { _exportId: '', data: { id: 's2', name: 'Skill 2' } };
+    registry.register(makeDef('skill', [s1, s2]));
+    registry.register(makeDef('agent', [], ['skill']));
+    registry.register(makeDef('binding', [], ['agent']));
+  });
+
+  it('generates a valid ExportBundle', async () => {
+    const svc = new ExportImportService();
+    const { bundle } = await svc.export('tenant-1', ['skill']);
+
+    expect(bundle.version).toBe(1);
+    expect(bundle.sourceTenantId).toBe('tenant-1');
+    expect(bundle.entities.skill).toHaveLength(2);
+    expect(bundle.entities.skill[0]._exportId).toMatch(/^exp-skill-0/);
+    expect(bundle.entities.skill[0].data.id).toBe('s1');
+    expect(bundle.entities.skill[0].data).not.toHaveProperty('tenantId');
+    expect(bundle.entities.skill[0].data).not.toHaveProperty('createdAt');
+    expect(bundle.entities.skill[0].data).not.toHaveProperty('updatedAt');
+    expect(bundle.dependencyOrder).toEqual(['skill']);
+  });
+
+  it('filters entities by entityIds', async () => {
+    const svc = new ExportImportService();
+    const { bundle } = await svc.export('tenant-1', ['skill'], { skill: ['s1'] });
+    expect(bundle.entities.skill).toHaveLength(1);
+    expect(bundle.entities.skill[0].data.id).toBe('s1');
+  });
+
+  it('exportPreview detects missing dependencies', async () => {
+    const svc = new ExportImportService();
+    const result = await svc.exportPreview('tenant-1', ['agent']);
+    expect(result.missingDependencies).toContain('skill');
+    expect(result.cascadeAdditions).toEqual([]);
+  });
+
+  it('export includes transitive deps when explicitly requested', async () => {
+    const svc = new ExportImportService();
+    const { bundle } = await svc.export('tenant-1', ['skill', 'agent']);
+    expect(bundle.dependencyOrder).toContain('skill');
+    expect(bundle.dependencyOrder).toContain('agent');
+    expect(bundle.dependencyOrder.indexOf('skill')).toBeLessThan(bundle.dependencyOrder.indexOf('agent'));
+  });
+
+  it('previewImport returns insertions for new entities', async () => {
+    const svc = new ExportImportService();
+    const bundle = {
+      version: 1 as const,
+      exportedAt: new Date().toISOString(),
+      sourceTenantId: 't1',
+      entities: {
+        skill: [{ _exportId: 'exp-s-1', data: { id: 's1', name: 'Skill' } }],
+      },
+      dependencyOrder: ['skill'],
+    };
+
+    await svc.previewImport('tenant-2', bundle);
+    const registry = ExportableEntityRegistry.getInstance();
+    expect(registry.get('skill').previewImport).toHaveBeenCalled();
+  });
+
+  it('throws on invalid bundle version', async () => {
+    const svc = new ExportImportService();
+    await expect(
+      svc.previewImport('tenant-2', {
+        version: 99 as any,
+        entities: {},
+        dependencyOrder: [],
+        exportedAt: '',
+        sourceTenantId: '',
+      }),
+    ).rejects.toThrow(/version/);
+  });
+
+  it('throws on missing entities field', async () => {
+    const svc = new ExportImportService();
+    await expect(
+      svc.previewImport('tenant-2', {
+        version: 1,
+        exportedAt: '',
+        sourceTenantId: '',
+        dependencyOrder: [],
+      } as any),
+    ).rejects.toThrow(/entities/);
+  });
+
+  it('getExportJob returns undefined for unknown jobId', () => {
+    const svc = new ExportImportService();
+    expect(svc.getExportJob('nonexistent', 'tenant-1')).toBeUndefined();
+  });
+
+  it('getExportJob returns bundle after export', async () => {
+    const svc = new ExportImportService();
+    const { jobId, bundle } = await svc.export('tenant-1', ['skill']);
+    const retrieved = svc.getExportJob(jobId, 'tenant-1');
+    expect(retrieved).toEqual(bundle);
+  });
+
+  it('listExportableTypes returns registered types', () => {
+    const svc = new ExportImportService();
+    const types = svc.listExportableTypes();
+    expect(types).toHaveLength(3);
+    expect(types.map((t) => t.entityType)).toEqual(['skill', 'agent', 'binding']);
+  });
+});

package/src/services/eval-runner.ts

@@ -1,6 +1,9 @@
 import { EventEmitter } from "events";
-import { getStoreLattice, modelLatticeManager } from "@axiom-lattice/core";
-import { LatticeEvalProject } from "@axiom-lattice/agent-eval";
+import {
+  getStoreLattice,
+  modelLatticeManager,
+  LatticeEvalProject,
+} from "@axiom-lattice/core";
 import type { EvalStore } from "@axiom-lattice/protocols";
 import type {
   CaseRunResult,
@@ -8,8 +11,7 @@ import type {
   LatticeEvalProjectType,
   LatticeEvalSuiteType,
   LatticeEvalLogEvent,
-  OutputType,
-} from "@axiom-lattice/agent-eval";
+} from "@axiom-lattice/core";
 import { v4 as uuidv4 } from "uuid";
 
 export interface EvalStreamEvent {
@@ -48,6 +50,12 @@ class EvalRunner {
     const project = await store.getProjectById(tenantId, projectId);
     if (!project) throw new Error("Project not found");
 
+    for (const ctx of this.runs.values()) {
+      if (ctx.projectId === projectId) {
+        throw new Error("A run is already in progress for this project");
+      }
+    }
+
     const existingRuns = await store.getRunsByTenant(tenantId, { projectId, status: "running" });
     if (existingRuns.length > 0) {
       throw new Error("A run is already in progress for this project");
@@ -65,8 +73,8 @@ class EvalRunner {
         cases: cases.map((c) => ({
           caseId: c.id,
           input: { message: c.inputMessage, files: c.inputFiles },
-          steps: c.steps as Array<{ agent_id: string; override_message?: string }>,
-          output: { type: c.outputType } as OutputType,
+          steps: c.steps,
+          output: { type: "message_content" },
           eval: {
             content_assertion: c.contentAssertion,
             eval_rubrics: c.rubrics?.map((r) => ({
@@ -79,39 +87,35 @@ class EvalRunner {
       });
     }
 
-    const runId = uuidv4();
-    const concurrency = (project as unknown as Record<string, unknown>).concurrency as number || 3;
-    await store.createRun(tenantId, projectId, runId, { totalCases, concurrency });
-
-    const judgeCfg = project.judgeModelConfig as Record<string, unknown>;
-    const hasModelKey = Boolean(judgeCfg.modelKey);
-    const hasApiKey = Boolean(judgeCfg.apiKeyEnvName || judgeCfg.apiKey);
-    const hasCredentials = hasApiKey;
-
-    let judgeModelConfig: { modelKey?: string; model?: any } = {};
+    const judgeCfg = (project.judgeModelConfig ?? {}) as Record<string, unknown>;
+    const judgeModelKey = (judgeCfg.modelKey as string) || "";
+    let resolvedJudgeModelKey: string;
 
-    if (hasModelKey) {
-      judgeModelConfig = { modelKey: judgeCfg.modelKey as string };
-    } else if (!hasCredentials) {
-      const firstModel = modelLatticeManager.getAllLattices()[0];
-      if (firstModel) {
-        judgeModelConfig = { modelKey: firstModel.key };
-      } else {
-        judgeModelConfig = { model: judgeCfg as any };
+    if (judgeModelKey) {
+      if (!modelLatticeManager.hasLattice(judgeModelKey)) {
+        throw new Error(`Judge model "${judgeModelKey}" is not registered`);
       }
+      resolvedJudgeModelKey = judgeModelKey;
     } else {
-      judgeModelConfig = { model: judgeCfg as any };
+      const firstModel = modelLatticeManager.getAllLattices()[0];
+      if (!firstModel) {
+        throw new Error("No model registered — cannot run eval without a judge model");
+      }
+      resolvedJudgeModelKey = firstModel.key;
+      console.warn(`[eval-runner] Project "${project.name}" has no judge modelKey — falling back to first registered model "${firstModel.key}"`);
     }
 
+    const runId = uuidv4();
+    const concurrency = Math.max(1, Math.floor((project as unknown as Record<string, unknown>).concurrency as number) || 3);
+    await store.createRun(tenantId, projectId, runId, { totalCases, concurrency });
+
     const projectConfig: LatticeEvalProjectType = {
       projectName: project.name,
       version: project.version,
       description: project.description,
       suites: evalSuites,
-      judge_agent_config: judgeModelConfig,
+      judge_agent_config: { modelKey: resolvedJudgeModelKey },
       lattice_server_config: {
-        base_url: (project.targetServerConfig as Record<string, string>).base_url || "",
-        api_key: (project.targetServerConfig as Record<string, string>).api_key || "",
         tenant_id: tenantId,
       },
       concurrency,
@@ -163,37 +167,41 @@ class EvalRunner {
     const runPromise = (async () => {
       try {
         const evalProject = new LatticeEvalProject(projectConfig, onCaseComplete);
-        const { report } = await evalProject.runAllSuitesBatch(concurrency);
-
-        const completedCount = stats.passed + stats.failed;
-        await store.updateRunStatus(tenantId, runId, {
-          status: "completed",
-          completedAt: new Date(),
-          passedCases: stats.passed,
-          failedCases: stats.failed,
-          avgScore: completedCount > 0 ? stats.totalScore / completedCount : 0,
-        });
-
-        this.eventEmitter.emit(`run:${runId}`, {
-          type: "completed",
-          runId,
-          data: { passed: stats.passed, failed: stats.failed, avgScore: completedCount > 0 ? stats.totalScore / completedCount : 0 },
-        } satisfies EvalStreamEvent);
+        const { report } = await evalProject.runAllSuitesBatch(concurrency, abortController.signal);
+
+        if (!abortController.signal.aborted) {
+          const completedCount = stats.passed + stats.failed;
+          await store.updateRunStatus(tenantId, runId, {
+            status: "completed",
+            completedAt: new Date(),
+            passedCases: stats.passed,
+            failedCases: stats.failed,
+            avgScore: completedCount > 0 ? stats.totalScore / completedCount : 0,
+          });
+
+          this.eventEmitter.emit(`run:${runId}`, {
+            type: "completed",
+            runId,
+            data: { passed: stats.passed, failed: stats.failed, avgScore: completedCount > 0 ? stats.totalScore / completedCount : 0 },
+          } satisfies EvalStreamEvent);
+        }
 
         return { report };
       } catch (err) {
         const errorMsg = (err as Error).message;
-        await store.updateRunStatus(tenantId, runId, {
-          status: "failed",
-          error: errorMsg,
-          completedAt: new Date(),
-        });
-
-        this.eventEmitter.emit(`run:${runId}`, {
-          type: "error",
-          runId,
-          data: { message: errorMsg },
-        } satisfies EvalStreamEvent);
+        if (!abortController.signal.aborted) {
+          await store.updateRunStatus(tenantId, runId, {
+            status: "failed",
+            error: errorMsg,
+            completedAt: new Date(),
+          });
+
+          this.eventEmitter.emit(`run:${runId}`, {
+            type: "error",
+            runId,
+            data: { message: errorMsg },
+          } satisfies EvalStreamEvent);
+        }
 
         throw err;
       } finally {