@inneranimalmedia/agentsam-sdk 2.6.2 → 2.6.3

package/test/cli/eval-live.test.mjs

@@ -0,0 +1,219 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+
+import {
+  finishLiveEvalRun,
+  getLiveEvalStatus,
+  startLiveEvalRun,
+} from '../../src/eval/index.js';
+import { recordToolReceipt } from '../../src/mcp/index.js';
+import { runEval } from '../../src/commands/eval.js';
+
+function tempHome() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'agentsam-live-eval-test-'));
+}
+
+test('live eval lifecycle: start, track receipts, get status, finish', async () => {
+  const home = tempHome();
+
+  // Initially inactive
+  const initialStatus = getLiveEvalStatus({ home });
+  assert.equal(initialStatus.active, false);
+
+  // Start live eval run
+  const run = startLiveEvalRun(
+    {
+      suite: 'sdk-real-work-20260919',
+      case: 'cad-shell-unification',
+      model: 'Muse Spark 1.3',
+      provider: 'cursor',
+      client: 'cursor',
+      reasoning: 'high',
+    },
+    { home }
+  );
+
+  assert.ok(run.run_id.startsWith('evr_'));
+  assert.equal(run.suite_id, 'sdk-real-work-20260919');
+  assert.equal(run.case_id, 'cad-shell-unification');
+  assert.equal(run.model, 'Muse Spark 1.3');
+
+  // Verify starting again fails
+  assert.throws(() => startLiveEvalRun({}, { home }), /live_eval_run_already_active/);
+
+  // Record tool calls into the live session
+  recordToolReceipt({ tool_key: 'github_read', source_client: 'cursor', duration_ms: 50 }, { home });
+  recordToolReceipt({ tool_key: 'mcp:d1_query', source_client: 'cursor_mcp', duration_ms: 12 }, { home });
+  recordToolReceipt({ tool_key: 'terminal_exec', source_client: 'agentsam', duration_ms: 300 }, { home });
+
+  // Status check reflects active state and tool receipts
+  const activeStatus = getLiveEvalStatus({ home });
+  assert.equal(activeStatus.active, true);
+  assert.equal(activeStatus.receipts_summary.tool_call_count, 3);
+  assert.equal(activeStatus.receipts_summary.mcp_call_count, 1);
+
+  // Finish run
+  const result = await finishLiveEvalRun({ home, gate: 'PASS' });
+  assert.equal(result.evalRunRow.id, run.run_id);
+  assert.equal(result.evalRunRow.passed, 1);
+  assert.equal(result.evalRunRow.tool_calls_attempted, 3);
+  assert.equal(result.evalRunRow.tool_calls_succeeded, 3);
+  assert.equal(result.modelObservationRow.id, `obs_${run.run_id}`);
+  assert.equal(result.modelObservationRow.passed, 1);
+
+  // After finish, status is inactive again
+  const postStatus = getLiveEvalStatus({ home });
+  assert.equal(postStatus.active, false);
+});
+
+test('agentsam eval live CLI commands work end-to-end', async () => {
+  const home = tempHome();
+  const logs = [];
+  const write = (str) => logs.push(str);
+
+  // CLI start
+  const startResult = await runEval(
+    ['live', 'start', '--case', 'errors-v1-reconciliation', '--model', 'Muse Spark 1.3', '--json'],
+    { home, write }
+  );
+  assert.ok(startResult.run_id.startsWith('evr_'));
+
+  // Record receipt
+  recordToolReceipt({ tool_key: 'mcp:eval_query', source_client: 'cursor_mcp', duration_ms: 40 }, { home });
+
+  // CLI status
+  const statusResult = await runEval(['live', 'status', '--json'], { home, write });
+  assert.equal(statusResult.active, true);
+  assert.equal(statusResult.receipts_summary.tool_call_count, 1);
+
+  // CLI finish
+  const finishResult = await runEval(['live', 'finish', '--gate', 'PASS', '--json'], { home, write });
+  assert.equal(finishResult.evalRunRow.passed, 1);
+  assert.equal(finishResult.summary.gate_status, 'PASS');
+
+  // Verify non-JSON output formats cleanly
+  const emptyStatus = await runEval(['live', 'status'], { home, write });
+  assert.equal(emptyStatus.active, false);
+});
+
+test('finishLiveEvalRun archives run locally, generates SQL for both tables, and respects configurable tenant/database', async () => {
+  const home = tempHome();
+  const run = startLiveEvalRun(
+    {
+      case: 'cad-render-eval',
+      model: 'Muse Spark 1.3',
+      tenant: 'test-tenant',
+      workspace: 'cad-workspace',
+      user: 'test-user',
+      database: 'test-database',
+    },
+    { home }
+  );
+
+  assert.equal(run.tenant_id, 'test-tenant');
+  assert.equal(run.workspace_id, 'cad-workspace');
+  assert.equal(run.user_id, 'test-user');
+  assert.equal(run.database, 'test-database');
+
+  const result = await finishLiveEvalRun({ home, gate: 'PASS' });
+
+  // 1. Dual record construction
+  assert.equal(result.evalRunRow.tenant_id, 'test-tenant');
+  assert.equal(result.modelObservationRow.tenant_id, 'test-tenant');
+  assert.equal(result.modelObservationRow.workspace_id, 'cad-workspace');
+  assert.equal(result.modelObservationRow.user_id, 'test-user');
+  assert.equal(result.modelObservationRow.status, 'completed');
+  assert.equal(result.modelObservationRow.passed, 1);
+  assert.equal(result.modelObservationRow.expected_markers_total, 1);
+  assert.equal(typeof result.modelObservationRow.output_chars, 'number');
+
+  // 2. Both tables present in generated D1 SQL
+  assert.ok(result.summary.d1_sql.includes('INSERT OR REPLACE INTO agentsam_eval_runs'));
+  assert.ok(result.summary.d1_sql.includes('INSERT OR REPLACE INTO agentsam_model_eval_observations'));
+
+  // 3. Local archive created
+  const archivedRunPath = path.join(home, '.agentsam', 'eval', 'runs', `${run.run_id}.json`);
+  assert.ok(fs.existsSync(archivedRunPath), 'Archived run file must exist locally');
+  const archived = JSON.parse(fs.readFileSync(archivedRunPath, 'utf8'));
+  assert.equal(archived.evalRunRow.id, run.run_id);
+  assert.equal(archived.modelObservationRow.id, `obs_${run.run_id}`);
+});
+
+test('finishLiveEvalRun preserves active run state and receipts if remote persistence fails without --force', async () => {
+  const home = tempHome();
+  const run = startLiveEvalRun({ case: 'resilience-test' }, { home });
+
+  recordToolReceipt({ tool_key: 'git_status', duration_ms: 20 }, { home });
+
+  // Call finish with remote=true pointing to an unreachable endpoint and wrangler fallback disabled
+  const result = await finishLiveEvalRun({
+    home,
+    remote: true,
+    evalEndpoint: 'http://127.0.0.1:1/nonexistent',
+    wranglerFallback: false,
+    timeoutMs: 100,
+  });
+
+  // Verification: remote failure noted, evidence preserved, active run kept intact
+  assert.equal(result.summary.d1_executed, false);
+  assert.equal(result.summary.state_preserved, true);
+  assert.ok(result.summary.d1_error);
+
+  const activeStatus = getLiveEvalStatus({ home });
+  assert.equal(activeStatus.active, true, 'Active run state must be retained when remote persistence fails');
+  assert.equal(activeStatus.receipts_summary.tool_call_count, 1, 'Tool receipts must be preserved when remote persistence fails');
+
+  // Finish with force: true to finalize and clear active run
+  const forcedResult = await finishLiveEvalRun({
+    home,
+    remote: true,
+    evalEndpoint: 'http://127.0.0.1:1/nonexistent',
+    wranglerFallback: false,
+    timeoutMs: 100,
+    force: true,
+  });
+
+  assert.equal(forcedResult.summary.d1_executed, false);
+  const clearedStatus = getLiveEvalStatus({ home });
+  assert.equal(clearedStatus.active, false, 'Forced finish clears active run');
+});
+
+test('finishLiveEvalRun reuses the configured inneranimalmedia MCP bearer for remote persistence', { concurrency: false }, async () => {
+  const home = tempHome();
+  const mcpDir = path.join(home, '.agentsam', 'mcp');
+  fs.mkdirSync(mcpDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(mcpDir, 'inneranimalmedia.json'),
+    JSON.stringify({ auth: { type: 'bearer', token: 'fixture-eval-token' } }),
+  );
+
+  startLiveEvalRun({ case: 'authenticated-remote-eval' }, { home });
+
+  const originalFetch = globalThis.fetch;
+  let authorization = null;
+  try {
+    globalThis.fetch = async (_url, init = {}) => {
+      authorization = new Headers(init.headers).get('Authorization');
+      return new Response(JSON.stringify({ ok: true }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    };
+
+    const result = await finishLiveEvalRun({
+      home,
+      remote: true,
+      evalEndpoint: 'https://example.test/api/eval/record',
+      wranglerFallback: false,
+    });
+
+    assert.equal(authorization, 'Bearer fixture-eval-token');
+    assert.equal(result.summary.d1_executed, true);
+    assert.equal(result.summary.persistence_method, 'http_endpoint');
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});

package/test/cli/go.test.mjs

@@ -0,0 +1,71 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import { fileURLToPath } from 'node:url';
+import { discoverGoRuntime, resolveProductRoot, ensureProductContract, buildGoProduct } from '../../src/go/index.js';
+import { runGo } from '../../src/commands/go.js';
+
+const SDK_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+
+test('discoverGoRuntime finds agentsam-go-worker without inventing a sibling', () => {
+  const discovery = discoverGoRuntime(SDK_ROOT);
+  assert.equal(discovery.go.ok, true);
+  assert.ok(discovery.runtime, 'expected Go runtime');
+  assert.match(discovery.runtime.module, /agentsam-go-worker/);
+  assert.equal(path.basename(discovery.runtime.productRoot), 'agentsam-go-worker');
+  const productRoot = resolveProductRoot(discovery, 'agentsam-go-worker');
+  const contract = ensureProductContract(productRoot);
+  assert.equal(contract.product, 'agentsam-go-worker');
+  assert.equal(contract.existing, true);
+  assert.deepEqual(contract.created, []);
+});
+
+test('ensureProductContract refuses missing product roots instead of scaffolding siblings', () => {
+  const missing = path.join(os.tmpdir(), `agentsam-go-missing-${Date.now()}`);
+  assert.throws(() => ensureProductContract(missing), /product_root_missing/);
+});
+
+test('buildGoProduct emits build receipt after go test/vet/build', () => {
+  const discovery = discoverGoRuntime(SDK_ROOT);
+  const productRoot = resolveProductRoot(discovery);
+  const result = buildGoProduct({
+    productRoot,
+    runtimeRoot: discovery.runtime.runtimeRoot,
+    repositoryRoot: discovery.repository_root,
+  });
+  assert.equal(result.receipt.schema, 'agentsam.go-build-receipt.v1');
+  assert.equal(result.tests.ok, true);
+  assert.equal(result.vet.ok, true);
+  assert.ok(result.build.binary);
+  assert.ok(fs.existsSync(result.receiptPath));
+});
+
+test('agentsam go --cloudflare agentsam-go-worker --skip-deploy is idempotent', async () => {
+  const chunks = [];
+  const write = (v) => chunks.push(String(v));
+  const first = await runGo(['--cloudflare', 'agentsam-go-worker', '--skip-deploy', '--json'], { write });
+  assert.equal(first.ok, true);
+  assert.equal(first.product, 'agentsam-go-worker');
+  assert.equal(first.scaffold_changes_required, false);
+
+  const chunks2 = [];
+  const second = await runGo(['--cloudflare', 'agentsam-go-worker', '--skip-deploy', '--json'], {
+    write: (v) => chunks2.push(String(v)),
+  });
+  assert.equal(second.ok, true);
+  assert.equal(second.existing_product, true);
+  assert.equal(second.scaffold_changes_required, false);
+  assert.ok(second.deploy.receiptPath);
+  assert.ok(fs.existsSync(second.deploy.productPath));
+});
+
+test('agentsam go status --json reports discovery', async () => {
+  const chunks = [];
+  const result = await runGo(['status', '--json'], { write: (v) => chunks.push(String(v)) });
+  assert.equal(result.ok, true);
+  assert.ok(result.discovery.go.ok);
+  const parsed = JSON.parse(chunks.join(''));
+  assert.equal(parsed.product, 'agentsam-go-worker');
+});

package/test/cli/local-vault-and-provider-setup.test.mjs

@@ -0,0 +1,268 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+
+import {
+  getSecureProviderKey,
+  setSecureProviderKey,
+  deleteSecureProviderKey,
+  listSecureConfiguredProviders,
+  hydrateSecureCredentials,
+  localVaultPath,
+  localVaultSaltPath,
+} from '../../src/security/local-vault.js';
+import { resolveExecutableCommand } from '../../src/security/process.js';
+import {
+  resolveProviderCredential,
+  setProviderCredential,
+  removeProviderCredential,
+  exportProviderEnvProfile,
+} from '../../src/lib/provider-credentials.js';
+import {
+  validateAndSaveProviderCredential,
+  providerChoices,
+  runProviders,
+} from '../../src/commands/providers.js';
+import { renderModelsStatus } from '../../src/commands/models.js';
+
+// picocolors color-detection differs between local runs and CI runners (CI often
+// reports TTY-like support that enables ANSI output) - strip escape codes before
+// asserting on rendered text content so this test checks content, not incidental
+// terminal color state.
+// eslint-disable-next-line no-control-regex
+const ANSI_PATTERN = /\x1B\[[0-9;]*m/g;
+const stripAnsi = (s) => String(s).replace(ANSI_PATTERN, '');
+
+function createTempHome(t) {
+  const home = fs.mkdtempSync(path.join(os.tmpdir(), 'agentsam-vault-test-'));
+  t.after(() => fs.rmSync(home, { recursive: true, force: true }));
+  return home;
+}
+
+test('resolveExecutableCommand resolves Windows .cmd shims with shell: true on win32', () => {
+  const npxWin = resolveExecutableCommand('npx', 'win32');
+  assert.equal(npxWin.command, 'npx.cmd');
+  assert.equal(npxWin.shell, true);
+
+  const npmWin = resolveExecutableCommand('npm', 'win32');
+  assert.equal(npmWin.command, 'npm.cmd');
+  assert.equal(npmWin.shell, true);
+
+  const wranglerWin = resolveExecutableCommand('wrangler', 'win32');
+  assert.equal(wranglerWin.command, 'wrangler.cmd');
+  assert.equal(wranglerWin.shell, true);
+
+  const scriptWin = resolveExecutableCommand('build.bat', 'win32');
+  assert.equal(scriptWin.command, 'build.bat');
+  assert.equal(scriptWin.shell, true);
+
+  // Linux and macOS are untouched
+  const npxMac = resolveExecutableCommand('npx', 'darwin');
+  assert.equal(npxMac.command, 'npx');
+  assert.equal(npxMac.shell, false);
+
+  const npmLinux = resolveExecutableCommand('npm', 'linux');
+  assert.equal(npmLinux.command, 'npm');
+  assert.equal(npmLinux.shell, false);
+});
+
+test('local vault encrypts with AES-256-GCM and never writes plaintext secrets to disk', (t) => {
+  const home = createTempHome(t);
+  const opts = { home, disableOsStore: true };
+
+  const secret = 'sk-proj-testsecret1234567890abcdef';
+  setSecureProviderKey('openai', secret, opts);
+
+  const retrieved = getSecureProviderKey('openai', opts);
+  assert.equal(retrieved.value, secret);
+  assert.equal(retrieved.source, 'local_vault');
+
+  // Verify disk contents
+  const encFile = localVaultPath(opts);
+  assert.equal(fs.existsSync(encFile), true);
+  const rawText = fs.readFileSync(encFile, 'utf8');
+
+  // Plaintext MUST NOT appear anywhere in the file
+  assert.equal(rawText.includes(secret), false);
+
+  // Must have JSON with iv, tag, and data
+  const parsed = JSON.parse(rawText);
+  assert.ok(parsed.secrets.openai.iv);
+  assert.ok(parsed.secrets.openai.tag);
+  assert.ok(parsed.secrets.openai.data);
+
+  // Salt file must exist with restrictive permissions
+  const saltFile = localVaultSaltPath(opts);
+  assert.equal(fs.existsSync(saltFile), true);
+  if (process.platform !== 'win32') {
+    assert.equal(fs.statSync(saltFile).mode & 0o777, 0o600);
+    assert.equal(fs.statSync(encFile).mode & 0o777, 0o600);
+  }
+});
+
+test('local vault detects tampering of encrypted ciphertext', (t) => {
+  const home = createTempHome(t);
+  const opts = { home, disableOsStore: true };
+
+  setSecureProviderKey('anthropic', 'sk-ant-test-key-12345', opts);
+
+  // Tamper with the ciphertext
+  const encFile = localVaultPath(opts);
+  const parsed = JSON.parse(fs.readFileSync(encFile, 'utf8'));
+  const originalData = parsed.secrets.anthropic.data;
+  // Flip the last hex byte
+  const tamperedData = originalData.slice(0, -2) + (originalData.slice(-2) === 'aa' ? 'bb' : 'aa');
+  parsed.secrets.anthropic.data = tamperedData;
+  fs.writeFileSync(encFile, JSON.stringify(parsed));
+
+  // Decryption must fail and return null safely rather than corrupt data
+  const retrieved = getSecureProviderKey('anthropic', opts);
+  assert.equal(retrieved, null);
+});
+
+test('local vault stores and retrieves Cloudflare accountId alongside token', (t) => {
+  const home = createTempHome(t);
+  const opts = { home, disableOsStore: true };
+
+  setSecureProviderKey('cloudflare', { value: 'cf-token-123', accountId: 'abcdef0123456789abcdef0123456789' }, opts);
+  const retrieved = getSecureProviderKey('cloudflare', opts);
+
+  assert.equal(retrieved.value, 'cf-token-123');
+  assert.equal(retrieved.accountId, 'abcdef0123456789abcdef0123456789');
+});
+
+test('hydrateSecureCredentials populates process.env for unconfigured providers without overwriting', (t) => {
+  const home = createTempHome(t);
+  const opts = { home, disableOsStore: true };
+
+  setSecureProviderKey('gemini', 'gemini-key-val-789', opts);
+
+  const mockEnv = {
+    EXISTING_KEY: 'pre-existing',
+  };
+
+  const hydrated = hydrateSecureCredentials(mockEnv, opts);
+  assert.ok(hydrated.includes('gemini'));
+  assert.equal(mockEnv.GEMINI_API_KEY, 'gemini-key-val-789');
+
+  // Second run does not overwrite
+  mockEnv.GEMINI_API_KEY = 'user-override';
+  hydrateSecureCredentials(mockEnv, opts);
+  assert.equal(mockEnv.GEMINI_API_KEY, 'user-override');
+});
+
+test('resolveProviderCredential resolves keys from local secure vault across any directory', (t) => {
+  const home = createTempHome(t);
+  const opts = { home, disableOsStore: true };
+
+  setProviderCredential('cursor', 'cur-key-secret-456', opts);
+
+  const resolved = resolveProviderCredential('cursor', { env: {}, home, disableOsStore: true });
+  assert.equal(resolved.configured, true);
+  assert.equal(resolved.value, 'cur-key-secret-456');
+
+  // Deleting removes it
+  removeProviderCredential('cursor', opts);
+  const afterRemove = resolveProviderCredential('cursor', { env: {}, home, disableOsStore: true });
+  assert.equal(afterRemove.configured, false);
+});
+
+test('validateAndSaveProviderCredential rejects invalid key and does NOT persist it', async (t) => {
+  const home = createTempHome(t);
+  const opts = {
+    home,
+    disableOsStore: true,
+    fetchImpl: async () => ({
+      ok: false,
+      status: 401,
+      json: async () => ({ error: { message: 'Incorrect API key provided' } }),
+    }),
+  };
+
+  const result = await validateAndSaveProviderCredential('openai', 'sk-invalid-test-key', opts);
+  assert.equal(result.ok, false);
+  assert.equal(result.saved, false);
+  assert.match(result.error, /Incorrect API key/i);
+
+  // Verify it was NOT stored in the vault!
+  const retrieved = getSecureProviderKey('openai', opts);
+  assert.equal(retrieved, null);
+});
+
+test('validateAndSaveProviderCredential verifies valid key and saves ONLY to local vault without plaintext file', async (t) => {
+  const home = createTempHome(t);
+  const opts = {
+    home,
+    disableOsStore: true,
+    fetchImpl: async () => ({
+      ok: true,
+      status: 200,
+      json: async () => ({
+        data: [{ id: 'gpt-4o' }, { id: 'gpt-4o-mini' }],
+      }),
+    }),
+  };
+
+  const result = await validateAndSaveProviderCredential('openai', 'sk-valid-key-xyz', opts);
+  assert.equal(result.ok, true);
+  assert.equal(result.saved, true);
+  assert.equal(result.model_count, 2);
+
+  // Verify it was saved to local vault
+  const retrieved = getSecureProviderKey('openai', opts);
+  assert.equal(retrieved.value, 'sk-valid-key-xyz');
+
+  // CRITICAL: Plaintext .env profile file MUST NOT be created automatically!
+  const plaintextFile = path.join(home, '.agentsam', 'env.d', 'openai.env');
+  assert.equal(fs.existsSync(plaintextFile), false, 'Plaintext .env file must NOT exist after secure provider setup');
+});
+
+test('exportProviderEnvProfile and agentsam providers export create plaintext .env file ONLY on explicit opt-in', async (t) => {
+  const home = createTempHome(t);
+  const opts = { home, disableOsStore: true };
+
+  // Set up provider key in secure vault
+  setProviderCredential('anthropic', 'sk-ant-test-export-key', opts);
+
+  // Confirm NO plaintext file exists yet
+  const plaintextFile = path.join(home, '.agentsam', 'env.d', 'anthropic.env');
+  assert.equal(fs.existsSync(plaintextFile), false);
+
+  // Explicit opt-in export via CLI command
+  let output = '';
+  const exported = await runProviders(['export', 'anthropic'], {
+    ...opts,
+    write: (text) => { output += text; },
+  });
+
+  assert.equal(exported.provider, 'anthropic');
+  assert.equal(fs.existsSync(plaintextFile), true);
+  const content = fs.readFileSync(plaintextFile, 'utf8');
+  assert.match(content, /ANTHROPIC_API_KEY="sk-ant-test-export-key"/);
+  assert.match(output, /Exported anthropic profile/);
+});
+
+test('renderModelsStatus shows tip: Select your preferred provider when providers are unconfigured', () => {
+  const status = {
+    providers: [
+      { id: 'openai', label: 'OpenAI', configured: false, credential: 'OPENAI_API_KEY' },
+      { id: 'anthropic', label: 'Anthropic', configured: false, credential: 'ANTHROPIC_API_KEY' },
+    ],
+    discovery: {},
+    providerModels: {},
+    local: { online: false, models: [] },
+  };
+
+  const output = stripAnsi(renderModelsStatus(status));
+  assert.match(output, /Tip:\s*Select your preferred provider/i);
+});
+
+test('providerChoices returns valid choice list for select picker', () => {
+  const choices = providerChoices();
+  assert.ok(choices.length >= 6);
+  assert.ok(choices.some((c) => c.value === 'openai'));
+  assert.ok(choices.some((c) => c.value === 'anthropic'));
+  assert.ok(choices.some((c) => c.value === 'gemini'));
+});