@amodalai/amodal 0.1.2

package/src/commands/eval.ts

@@ -0,0 +1,268 @@
+/**
+ * @license
+ * Copyright 2025 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+import {readFile, writeFile, mkdir} from 'node:fs/promises';
+import * as path from 'node:path';
+import {
+  formatEvalTable,
+  formatComparisonTable,
+  formatEvalMarkdown,
+  diffEvalResults,
+} from '@amodalai/core';
+import type {
+  EvalSuiteResult,
+  EvalQueryProvider,
+  JudgeProvider,
+} from '@amodalai/core';
+import type {CommandModule} from 'yargs';
+import {createLocalServer} from '@amodalai/runtime';
+import {findRepoRoot} from '../shared/repo-discovery.js';
+
+export interface EvalOptions {
+  cwd?: string;
+  filter?: string;
+  save?: string;
+  diff?: string;
+  ci?: boolean;
+  port?: number;
+}
+
+/**
+ * Run eval suite against a local repo.
+ */
+export async function runEval(options: EvalOptions): Promise<void> {
+  let repoPath: string;
+  try {
+    repoPath = findRepoRoot(options.cwd);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`[eval] ${msg}\n`);
+    process.exit(1);
+  }
+
+  const port = options.port ?? 0;
+  process.stderr.write(`[eval] Loading repo from ${repoPath}\n`);
+
+  const server = await createLocalServer({
+    repoPath,
+    port,
+    host: '127.0.0.1',
+    hotReload: false,
+  });
+
+  const httpServer = await server.start();
+  const addr = httpServer.address();
+  const actualPort = typeof addr === 'object' && addr !== null ? addr.port : port;
+  const baseUrl = `http://127.0.0.1:${actualPort}`;
+
+  try {
+    // Load repo to get evals
+    const {loadRepoFromDisk} = await import('@amodalai/core');
+    const repo = await loadRepoFromDisk(repoPath);
+
+    if (repo.evals.length === 0) {
+      process.stderr.write('[eval] No evals found in evals/\n');
+      return;
+    }
+
+    // Create query provider that hits the local server
+    const queryProvider: EvalQueryProvider = {
+      query: async (message: string, tenantId?: string) => {
+        const response = await fetch(`${baseUrl}/chat`, {
+          method: 'POST',
+          headers: {'Content-Type': 'application/json'},
+          body: JSON.stringify({message, tenant_id: tenantId ?? 'eval-runner'}),
+        });
+
+        const text = await response.text();
+        const lines = text.split('\n');
+        let fullResponse = '';
+        const toolCalls: Array<{name: string; parameters: Record<string, unknown>}> = [];
+        let usage: {inputTokens: number; outputTokens: number} | undefined;
+
+        for (const line of lines) {
+          if (!line.startsWith('data: ')) continue;
+          try {
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- SSE parsing
+            const event = JSON.parse(line.substring(6)) as Record<string, unknown>;
+            if (event['type'] === 'text_delta') {
+              fullResponse += String(event['content'] ?? '');
+            } else if (event['type'] === 'tool_call_start') {
+              toolCalls.push({
+                name: String(event['tool_name'] ?? ''),
+                // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- SSE data
+                parameters: (event['parameters'] ?? {}) as Record<string, unknown>,
+              });
+            } else if (event['type'] === 'done') {
+              // Extract usage from done event if the runtime provides it
+              if (event['usage']) {
+                // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+                const u = event['usage'] as {input_tokens: number; output_tokens: number};
+                if (u.input_tokens > 0 || u.output_tokens > 0) {
+                  usage = {inputTokens: u.input_tokens, outputTokens: u.output_tokens};
+                }
+              }
+            } else if (event['type'] === 'tool_call_result') {
+              // Enrich the last matching tool call with result info
+              const toolId = String(event['tool_id'] ?? '');
+              const status = String(event['status'] ?? '');
+              const result = String(event['result'] ?? event['error'] ?? '');
+              const preview = result.length > 300 ? result.substring(0, 300) + '...' : result;
+              // Add as a separate entry so the eval runner can see outcomes
+              toolCalls.push({
+                name: `${String(event['tool_name'] ?? 'request')}_result`,
+                parameters: {tool_id: toolId, status, result: preview},
+              });
+            }
+          } catch {
+            // skip
+          }
+        }
+
+        // If runtime didn't report usage, estimate from content length
+        // (~4 chars per token is a reasonable approximation for English text)
+        if (!usage) {
+          const outputChars = fullResponse.length + toolCalls.reduce((n, tc) => n + JSON.stringify(tc.parameters).length, 0);
+          const estimatedOutput = Math.ceil(outputChars / 4);
+          // Input is harder to estimate — use 3x output as a rough proxy (prompt + context)
+          const estimatedInput = estimatedOutput * 3;
+          usage = {inputTokens: estimatedInput, outputTokens: estimatedOutput};
+        }
+
+        return {response: fullResponse, toolCalls, usage};
+      },
+    };
+
+    // Simple judge — use the same server's LLM
+    const judgeProvider: JudgeProvider = {
+      judge: async (prompt: string) => {
+        const response = await fetch(`${baseUrl}/chat`, {
+          method: 'POST',
+          headers: {'Content-Type': 'application/json'},
+          body: JSON.stringify({message: prompt, tenant_id: 'eval-judge', session_id: `judge-${Date.now()}`}),
+        });
+        const text = await response.text();
+        let result = '';
+        for (const line of text.split('\n')) {
+          if (!line.startsWith('data: ')) continue;
+          try {
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- SSE parsing
+            const event = JSON.parse(line.substring(6)) as Record<string, unknown>;
+            if (event['type'] === 'text_delta') {
+              result += String(event['content'] ?? '');
+            }
+          } catch {
+            // skip
+          }
+        }
+        return result;
+      },
+    };
+
+    // Get git SHA
+    let gitSha: string | undefined;
+    try {
+      const {execSync} = await import('node:child_process');
+      gitSha = execSync('git rev-parse HEAD', {cwd: repoPath}).toString().trim();
+    } catch {
+      // not a git repo
+    }
+
+    // Get model info from repo config for cost tracking
+    const modelConfig = repo.config.models.main;
+    const model = {provider: modelConfig.provider, model: modelConfig.model};
+
+    const {runEvalSuite} = await import('@amodalai/core');
+    const gen = runEvalSuite(repo, {queryProvider, judgeProvider, filter: options.filter, gitSha, model});
+
+    let suiteResult: EvalSuiteResult | undefined;
+    while (true) {
+      const next = await gen.next();
+      if (next.done) {
+        suiteResult = next.value;
+        break;
+      }
+      const progress = next.value;
+      if (progress.type === 'eval_start') {
+        process.stderr.write(`[${progress.current}/${progress.total}] Running ${progress.evalName}...\n`);
+      } else if (progress.type === 'eval_complete') {
+        const icon = progress.passed ? 'PASS' : 'FAIL';
+        process.stderr.write(`[${progress.current}/${progress.total}] ${icon} ${progress.evalName}\n`);
+      }
+    }
+
+    if (!suiteResult) return;
+
+    // Save results if --save <name>
+    if (options.save) {
+      const saveDir = path.join(repoPath, '.amodal', 'evals');
+      await mkdir(saveDir, {recursive: true});
+      const savePath = path.join(saveDir, `${options.save}.json`);
+      await writeFile(savePath, JSON.stringify(suiteResult, null, 2) + '\n');
+      process.stderr.write(`[eval] Saved results to ${savePath}\n`);
+    }
+
+    // Compare against saved baseline if --diff <name>
+    if (options.diff) {
+      const baselinePath = path.join(repoPath, '.amodal', 'evals', `${options.diff}.json`);
+      let baseline: EvalSuiteResult | undefined;
+      try {
+        const raw = await readFile(baselinePath, 'utf-8');
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+        baseline = JSON.parse(raw) as EvalSuiteResult;
+      } catch {
+        process.stderr.write(`[eval] Baseline "${options.diff}" not found at ${baselinePath}\n`);
+        process.stderr.write(`[eval] Run with --save ${options.diff} first to create it.\n`);
+      }
+
+      if (baseline) {
+        const diffs = diffEvalResults(suiteResult, baseline);
+        process.stdout.write(formatComparisonTable(suiteResult, baseline, diffs));
+      }
+    }
+
+    // Always show the current run table
+    if (options.ci) {
+      process.stdout.write(formatEvalMarkdown(suiteResult, undefined, model));
+    } else if (!options.diff) {
+      process.stdout.write(formatEvalTable(suiteResult, model));
+    }
+
+    // Exit with non-zero if any failures
+    if (suiteResult.totalFailed > 0) {
+      process.exitCode = 1;
+    }
+  } finally {
+    await server.stop();
+  }
+}
+
+export const evalCommand: CommandModule = {
+  command: 'eval',
+  describe: 'Run eval suite',
+  builder: (yargs) =>
+    yargs
+      .option('filter', {type: 'string', describe: 'Filter evals by name'})
+      .option('save', {type: 'string', describe: 'Save results with a name (e.g. --save baseline)'})
+      .option('diff', {type: 'string', describe: 'Compare against a saved run (e.g. --diff baseline)'})
+      .option('ci', {type: 'boolean', default: false, describe: 'Output in CI-friendly markdown format'})
+      .option('port', {type: 'number', describe: 'Port for local eval server'}),
+  handler: async (argv) => {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    const filter = argv['filter'] as string | undefined;
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    const save = argv['save'] as string | undefined;
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    const diff = argv['diff'] as string | undefined;
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    const ci = argv['ci'] as boolean;
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
+    const port = argv['port'] as number | undefined;
+
+    await runEval({filter, save, diff, ci, port});
+  },
+};
+

package/src/commands/experiment.test.ts

@@ -0,0 +1,125 @@
+/**
+ * @license
+ * Copyright 2025 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+import {describe, it, expect, vi, beforeEach} from 'vitest';
+
+describe('runExperimentCommand', () => {
+  let fetchSpy: ReturnType<typeof vi.fn>;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    fetchSpy = vi.fn();
+    vi.stubGlobal('fetch', fetchSpy);
+  });
+
+  it('lists experiments', async () => {
+    fetchSpy.mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve({
+        experiments: [
+          {id: 'exp-1', name: 'test-exp', status: 'draft'},
+        ],
+      }),
+    });
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+
+    const {runExperimentCommand} = await import('./experiment.js');
+    await runExperimentCommand({
+      action: 'list',
+      platformUrl: 'http://localhost:4000',
+      platformApiKey: 'key-123',
+    });
+
+    const output = stdoutSpy.mock.calls.map(([s]) => s).join('');
+    expect(output).toContain('exp-1');
+    expect(output).toContain('test-exp');
+    stdoutSpy.mockRestore();
+  });
+
+  it('creates an experiment', async () => {
+    fetchSpy.mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve({id: 'exp-new'}),
+    });
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+
+    const {runExperimentCommand} = await import('./experiment.js');
+    await runExperimentCommand({
+      action: 'create',
+      name: 'my-experiment',
+      controlConfig: '{"model":"claude-sonnet-4-20250514"}',
+      variantConfig: '{"model":"gpt-4o"}',
+      platformUrl: 'http://localhost:4000',
+      platformApiKey: 'key-123',
+    });
+
+    expect(fetchSpy).toHaveBeenCalledOnce();
+    const output = stdoutSpy.mock.calls.map(([s]) => s).join('');
+    expect(output).toContain('Created experiment: exp-new');
+    stdoutSpy.mockRestore();
+  });
+
+  it('deploys an experiment', async () => {
+    fetchSpy.mockResolvedValue({ok: true, json: () => Promise.resolve({status: 'ok'})});
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+
+    const {runExperimentCommand} = await import('./experiment.js');
+    await runExperimentCommand({
+      action: 'deploy',
+      id: 'exp-1',
+      platformUrl: 'http://localhost:4000',
+      platformApiKey: 'key-123',
+    });
+
+    const output = stdoutSpy.mock.calls.map(([s]) => s).join('');
+    expect(output).toContain('Deployed experiment: exp-1');
+    stdoutSpy.mockRestore();
+  });
+
+  it('watches an experiment', async () => {
+    fetchSpy.mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve({id: 'exp-1', name: 'test', status: 'deployed'}),
+    });
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+
+    const {runExperimentCommand} = await import('./experiment.js');
+    await runExperimentCommand({
+      action: 'watch',
+      id: 'exp-1',
+      platformUrl: 'http://localhost:4000',
+      platformApiKey: 'key-123',
+    });
+
+    const output = stdoutSpy.mock.calls.map(([s]) => s).join('');
+    expect(output).toContain('"id": "exp-1"');
+    stdoutSpy.mockRestore();
+  });
+
+  it('handles empty experiment list', async () => {
+    fetchSpy.mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve({experiments: []}),
+    });
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+
+    const {runExperimentCommand} = await import('./experiment.js');
+    await runExperimentCommand({
+      action: 'list',
+      platformUrl: 'http://localhost:4000',
+      platformApiKey: 'key-123',
+    });
+
+    const output = stdoutSpy.mock.calls.map(([s]) => s).join('');
+    expect(output).toContain('No experiments found');
+    stdoutSpy.mockRestore();
+  });
+});

package/src/commands/experiment.ts

@@ -0,0 +1,153 @@
+/**
+ * @license
+ * Copyright 2025 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+import type {CommandModule} from 'yargs';
+import {resolvePlatformConfig} from '../shared/platform-client.js';
+
+export interface ExperimentOptions {
+  action: 'create' | 'deploy' | 'watch' | 'list';
+  name?: string;
+  id?: string;
+  platformUrl?: string;
+  platformApiKey?: string;
+  controlConfig?: string;
+  variantConfig?: string;
+}
+
+/**
+ * Manage model experiments via the platform API.
+ */
+export async function runExperimentCommand(options: ExperimentOptions): Promise<void> {
+  let platformUrl: string;
+  let apiKey: string;
+  try {
+    const config = await resolvePlatformConfig({
+      url: options.platformUrl,
+      apiKey: options.platformApiKey,
+    });
+    platformUrl = config.url;
+    apiKey = config.apiKey;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`[experiment] ${msg}\n`);
+    process.exit(1);
+  }
+
+  const headers = {
+    'Content-Type': 'application/json',
+    'Authorization': `Bearer ${apiKey}`,
+  };
+
+  switch (options.action) {
+    case 'list': {
+      const res = await fetch(`${platformUrl}/api/experiments`, {headers});
+      if (!res.ok) {
+        process.stderr.write(`[experiment] HTTP ${res.status}\n`);
+        return;
+      }
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- platform response
+      const data = await res.json() as {experiments: Array<Record<string, unknown>>};
+      if (data.experiments.length === 0) {
+        process.stdout.write('No experiments found.\n');
+        return;
+      }
+      for (const exp of data.experiments) {
+        process.stdout.write(`${exp['id']}  ${exp['name']}  [${exp['status']}]\n`);
+      }
+      break;
+    }
+
+    case 'create': {
+      if (!options.name) {
+        process.stderr.write('[experiment] --name is required for create\n');
+        return;
+      }
+      const controlConfig: Record<string, unknown> = options.controlConfig
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- CLI JSON input
+        ? JSON.parse(options.controlConfig) as Record<string, unknown>
+        : {};
+      const variantConfig: Record<string, unknown> = options.variantConfig
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- CLI JSON input
+        ? JSON.parse(options.variantConfig) as Record<string, unknown>
+        : {};
+
+      const res = await fetch(`${platformUrl}/api/experiments`, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({name: options.name, controlConfig, variantConfig}),
+      });
+      if (!res.ok) {
+        process.stderr.write(`[experiment] HTTP ${res.status}\n`);
+        return;
+      }
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- platform response
+      const {id} = await res.json() as {id: string};
+      process.stdout.write(`Created experiment: ${id}\n`);
+      break;
+    }
+
+    case 'deploy': {
+      if (!options.id) {
+        process.stderr.write('[experiment] --id is required for deploy\n');
+        return;
+      }
+      const res = await fetch(`${platformUrl}/api/experiments/${options.id}`, {
+        method: 'PUT',
+        headers,
+        body: JSON.stringify({status: 'deployed'}),
+      });
+      if (!res.ok) {
+        process.stderr.write(`[experiment] HTTP ${res.status}\n`);
+        return;
+      }
+      process.stdout.write(`Deployed experiment: ${options.id}\n`);
+      break;
+    }
+
+    case 'watch': {
+      if (!options.id) {
+        process.stderr.write('[experiment] --id is required for watch\n');
+        return;
+      }
+      const res = await fetch(`${platformUrl}/api/experiments/${options.id}`, {headers});
+      if (!res.ok) {
+        process.stderr.write(`[experiment] HTTP ${res.status}\n`);
+        return;
+      }
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- platform response
+      const exp = await res.json() as Record<string, unknown>;
+      process.stdout.write(JSON.stringify(exp, null, 2) + '\n');
+      break;
+    }
+
+    default:
+      process.stderr.write(`[experiment] Unknown action: ${options.action}\n`);
+      break;
+  }
+}
+
+export const experimentCommand: CommandModule = {
+  command: 'experiment <action>',
+  describe: 'Manage model experiments',
+  builder: (yargs) =>
+    yargs
+      .positional('action', {
+        type: 'string',
+        demandOption: true,
+        choices: ['create', 'deploy', 'watch', 'list'] as const,
+      })
+      .option('name', {type: 'string'})
+      .option('id', {type: 'string'}),
+  handler: async (argv) => {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- yargs parsed argv
+    const opts = argv as unknown as ExperimentOptions;
+    await runExperimentCommand({
+      action: opts['action'],
+      name: opts['name'],
+      id: opts['id'],
+    });
+  },
+};

package/src/commands/index.ts

@@ -0,0 +1,76 @@
+/**
+ * @license
+ * Copyright 2025 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+
+import {initCommand} from './init.js';
+import {devCommand} from './dev.js';
+import {inspectCommand} from './inspect.js';
+import {validateCommand} from './validate.js';
+import {connectCommand} from './connect.js';
+import {installPkgCommand} from './install-pkg.js';
+import {uninstallCommand} from './uninstall.js';
+import {listCommand} from './list.js';
+import {updateCommand} from './update.js';
+import {diffCommand} from './diff.js';
+import {searchCommand} from './search.js';
+import {publishCommand} from './publish.js';
+import {loginCommand} from './login.js';
+import {linkCommand} from './link.js';
+import {syncCommand} from './sync.js';
+import {secretsCommand} from './secrets.js';
+import {deployCommand} from './deploy.js';
+import {buildCommand} from './build.js';
+import {dockerCommand} from './docker.js';
+import {rollbackCommand} from './rollback.js';
+import {deploymentsCommand} from './deployments.js';
+import {promoteCommand} from './promote.js';
+import {serveCommand} from './serve.js';
+import {statusCommand} from './status.js';
+import {chatCommand} from './chat.js';
+import {auditCommand} from './audit.js';
+import {evalCommand} from './eval.js';
+import {experimentCommand} from './experiment.js';
+import {testQueryCommand} from './test-query.js';
+import {automationsCommand} from './automations.js';
+
+/**
+ * All amodal subcommands for flat registration on the root yargs instance.
+ */
+export const amodalCommands = [
+  // Project
+  initCommand,
+  devCommand,
+  inspectCommand,
+  validateCommand,
+  // Package management
+  connectCommand,
+  installPkgCommand,
+  uninstallCommand,
+  listCommand,
+  updateCommand,
+  diffCommand,
+  searchCommand,
+  publishCommand,
+  loginCommand,
+  linkCommand,
+  syncCommand,
+  // Platform
+  secretsCommand,
+  deployCommand,
+  buildCommand,
+  dockerCommand,
+  rollbackCommand,
+  deploymentsCommand,
+  promoteCommand,
+  serveCommand,
+  statusCommand,
+  chatCommand,
+  auditCommand,
+  automationsCommand,
+  // Advanced
+  evalCommand,
+  experimentCommand,
+  testQueryCommand,
+];