kiku-agent 1.3.1 → 1.3.3

package/server/agent/tests/envBootstrap.test.js

@@ -0,0 +1,87 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { bootstrapEnv } from '../envBootstrap.js';
+import { isCookieFileValid } from '../youtubeAuth.js';
+
+function tmpDir() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'kiku-envboot-'));
+}
+
+function withCleanEnv(keys, fn) {
+  const saved = keys.map(k => [k, process.env[k]]);
+  for (const [k] of saved) delete process.env[k];
+  return Promise.resolve()
+    .then(fn)
+    .finally(() => {
+      for (const [k, v] of saved) {
+        if (v === undefined) delete process.env[k];
+        else process.env[k] = v;
+      }
+    });
+}
+
+test('bootstrapEnv: package file wins, user file fills unset keys, process.env always wins', async () => {
+  const dir = tmpDir();
+  const pkg = path.join(dir, 'package.env');
+  const user = path.join(dir, 'user.env');
+  fs.writeFileSync(pkg, 'BOOT_A=pkg\nBOOT_SHARED=pkg\n');
+  fs.writeFileSync(user, 'BOOT_SHARED=user\nBOOT_B=user\n# BOOT_COMMENTED=hidden\n');
+
+  await withCleanEnv(['BOOT_A', 'BOOT_SHARED', 'BOOT_B', 'BOOT_COMMENTED'], async () => {
+    process.env.BOOT_SHARED = 'already-set';
+    const result = bootstrapEnv({ packageEnvPath: pkg, userEnvPath: user, force: true });
+
+    assert.equal(result.applied, true);
+    assert.equal(process.env.BOOT_A, 'pkg');
+    assert.equal(process.env.BOOT_B, 'user');
+    assert.equal(process.env.BOOT_SHARED, 'already-set', 'existing process.env values must never be overwritten');
+    assert.equal(process.env.BOOT_COMMENTED, undefined, 'commented keys must stay inactive');
+    assert.equal(result.files.length, 2);
+  });
+});
+
+test('bootstrapEnv: is idempotent without force and forceable for tests', async () => {
+  const dir = tmpDir();
+  const first = bootstrapEnv({ force: true, packageEnvPath: path.join(dir, 'missing.env'), userEnvPath: path.join(dir, 'missing2.env') });
+  assert.equal(first.applied, true);
+
+  const second = bootstrapEnv({ packageEnvPath: path.join(dir, 'x.env'), userEnvPath: path.join(dir, 'y.env') });
+  assert.equal(second.files.length, 0, 'non-forced call must be a no-op after bootstrap');
+
+  const forced = bootstrapEnv({ force: true, packageEnvPath: path.join(dir, 'x.env'), userEnvPath: path.join(dir, 'y.env') });
+  assert.equal(forced.files.length, 0);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('bootstrapEnv: tolerates CRLF files and strips surrounding quotes', async () => {
+  const dir = tmpDir();
+  const pkg = path.join(dir, 'package.env');
+  fs.writeFileSync(pkg, 'BOOT_CRLF = "crlf value" \r\nBOOT_QUOTED=\'quoted\'\r\n');
+
+  await withCleanEnv(['BOOT_CRLF', 'BOOT_QUOTED'], async () => {
+    bootstrapEnv({ packageEnvPath: pkg, userEnvPath: path.join(dir, 'none.env'), force: true });
+    assert.equal(process.env.BOOT_CRLF, 'crlf value');
+    assert.equal(process.env.BOOT_QUOTED, 'quoted');
+  });
+});
+
+test('isCookieFileValid rejects empty, short, and non-Netscape cookie files', () => {
+  const dir = tmpDir();
+  assert.equal(isCookieFileValid(path.join(dir, 'missing.txt')), false);
+
+  const empty = path.join(dir, 'empty.txt');
+  fs.writeFileSync(empty, '');
+  assert.equal(isCookieFileValid(empty), false);
+
+  const garbage = path.join(dir, 'garbage.txt');
+  fs.writeFileSync(garbage, 'not a cookie file at all\njust words\n');
+  assert.equal(isCookieFileValid(garbage), false);
+
+  const netscape = path.join(dir, 'cookies.txt');
+  fs.writeFileSync(netscape, '# Netscape HTTP Cookie File\n.youtube.com\tTRUE\t/\tTRUE\t0\tKEY\tvalue\n');
+  assert.equal(isCookieFileValid(netscape), true);
+  fs.rmSync(dir, { recursive: true, force: true });
+});

package/server/agent/tests/envCommand.test.js

@@ -0,0 +1,284 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {
+  resolveEnvTargets,
+  readEnvFile,
+  readMergedEnvMap,
+  upsertEnvKey,
+  removeEnvKey,
+  maskValue,
+  isSecretKey,
+  buildEnvTemplate,
+  generateEnvFile,
+  detectAutoDefaults,
+} from '../envManager.js';
+import { routeCommand } from '../commands.js';
+
+function tmpDir() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'kiku-env-'));
+}
+
+test('envManager: upsert preserves comments and replaces active and commented keys', () => {
+  const dir = tmpDir();
+  const file = path.join(dir, '.env');
+  fs.writeFileSync(file, '# header comment\nLMSTUDIO_BASE_URL=http://a/\n# KIKU_VRAM_TOTAL_GB=\nKIKU_YTDLP=yt-dlp\n');
+
+  upsertEnvKey(file, 'KIKU_CONTEXT_LIMIT', '131072');
+  upsertEnvKey(file, 'LMSTUDIO_BASE_URL', 'http://127.0.0.1:1234/v1');
+  upsertEnvKey(file, 'KIKU_VRAM_TOTAL_GB', '6');
+
+  const map = readEnvFile(file);
+  assert.equal(map.LMSTUDIO_BASE_URL, 'http://127.0.0.1:1234/v1');
+  assert.equal(map.KIKU_CONTEXT_LIMIT, '131072');
+  assert.equal(map.KIKU_VRAM_TOTAL_GB, '6');
+  assert.equal(map.KIKU_YTDLP, 'yt-dlp');
+
+  const raw = fs.readFileSync(file, 'utf8');
+  assert.ok(raw.includes('# header comment'), 'comments must survive upserts');
+  assert.ok(!/^KIKU_VRAM_TOTAL_GB=$/m.test(raw), 'no leftover empty duplicate');
+
+  removeEnvKey(file, 'KIKU_CONTEXT_LIMIT');
+  assert.equal(readEnvFile(file).KIKU_CONTEXT_LIMIT, undefined);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('envManager: masks secrets but leaves plain configuration readable', () => {
+  assert.ok(isSecretKey('OPENAI_API_KEY'));
+  assert.ok(isSecretKey('KIKU_MASTER_KEYS'));
+  assert.ok(isSecretKey('KIKU_YT_COOKIES'));
+  assert.ok(!isSecretKey('LMSTUDIO_BASE_URL'));
+  assert.ok(!isSecretKey('KIKU_CONTEXT_LIMIT'));
+  assert.ok(maskValue('OPENAI_API_KEY', 'sk-abcdefghijklmnopqrs').includes('…'));
+  assert.equal(maskValue('LMSTUDIO_BASE_URL', 'http://127.0.0.1:1234/v1'), 'http://127.0.0.1:1234/v1');
+  assert.equal(maskValue('OPENAI_API_KEY', ''), '(not set)');
+});
+
+test('envManager: generated template sets only necessary values and comments the rest', () => {
+  const content = buildEnvTemplate({
+    values: { LMSTUDIO_BASE_URL: 'http://127.0.0.1:1234/v1', KIKU_CONTEXT_LIMIT: '131072' },
+    detected: { loadedModel: 'google/gemma-4-e4b', contextLength: 131072 },
+  });
+
+  assert.ok(/^LMSTUDIO_BASE_URL=http:\/\/127\.0\.0\.1:1234\/v1$/m.test(content));
+  assert.ok(/^KIKU_CONTEXT_LIMIT=131072$/m.test(content));
+  assert.ok(/^# LMSTUDIO_MODELS_DIR=/m.test(content), 'recommended vars without values stay commented');
+  assert.ok(/^# KIKU_YT_COOKIES=/m.test(content), 'optional vars stay commented');
+  assert.ok(/^# OPENAI_API_KEY=/m.test(content), 'cloud keys stay commented');
+  assert.ok(/never set by fans\/CLI users/.test(content), 'server-only group is explicitly excluded');
+  assert.ok(/already configured by the platform developer/.test(content), 'server credentials are attributed to the hosted deployment');
+  for (const serverKey of ['UPSTASH_REDIS_REST_URL', 'UPSTASH_REDIS_REST_TOKEN', 'SUPABASE_URL', 'SUPABASE_SERVICE_ROLE_KEY', 'SUPABASE_ANON_KEY']) {
+    assert.ok(!new RegExp(`^${serverKey}=`).test(content), `${serverKey} must never be set active by the template`);
+  }
+  assert.ok(/gemma-4-e4b/.test(content), 'detection provenance is recorded');
+});
+
+test('envManager: generateEnvFile refuses to overwrite silently', () => {
+  const dir = tmpDir();
+  const file = path.join(dir, '.env');
+  generateEnvFile({ targetPath: file, values: { LMSTUDIO_BASE_URL: 'http://x/' } });
+  assert.throws(
+    () => generateEnvFile({ targetPath: file, values: {}, allowOverwrite: false }),
+    /already exists/
+  );
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('envManager: resolveEnvTargets prefers the package file and falls back to the user file', () => {
+  const dir = tmpDir();
+  const pkg = path.join(dir, 'package.env');
+  const user = path.join(dir, 'user.env');
+
+  const without = resolveEnvTargets({ packageEnvPath: pkg, userEnvPath: user });
+  assert.equal(without.mode, 'user');
+  assert.equal(without.primaryPath, user);
+
+  fs.writeFileSync(pkg, 'A=1\n');
+  const withPkg = resolveEnvTargets({ packageEnvPath: pkg, userEnvPath: user });
+  assert.equal(withPkg.mode, 'repo');
+  assert.equal(withPkg.primaryPath, pkg);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('/env: bare status renders without touching files', async () => {
+  assert.equal(await routeCommand('/env', { envProbe: false }), 'HANDLED');
+});
+
+test('/env: path subcommand renders', async () => {
+  assert.equal(await routeCommand('/env path', {}), 'HANDLED');
+});
+
+test('/env: init writes defaults to an override target without prompting', async () => {
+  const dir = tmpDir();
+  const target = path.join(dir, '.env');
+  assert.equal(await routeCommand('/env init', { envOverridePath: target, envProbe: false }), 'HANDLED');
+  const map = readEnvFile(target);
+  assert.equal(map.LMSTUDIO_BASE_URL, 'http://127.0.0.1:1234/v1');
+  assert.equal(map.OPENAI_API_KEY, undefined, 'cloud keys are never set unless the user asks');
+  assert.equal(map.KIKU_MASTER_KEYS, undefined, 'server-only vars are never set by the wizard');
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+function withInteractiveStdin(fn) {
+  const previousTty = process.stdin.isTTY;
+  process.stdin.isTTY = true;
+  return Promise.resolve(fn()).finally(() => {
+    if (previousTty === undefined) delete process.stdin.isTTY;
+    else process.stdin.isTTY = previousTty;
+  });
+}
+
+test('/env: init routes wizard questions through ctx.readInput (TUI handoff)', async () => {
+  const dir = tmpDir();
+  const target = path.join(dir, '.env');
+
+  const asked = [];
+  const answers = ['', '32768', '', 'n'];
+  const ctx = {
+    sub: 'init',
+    parts: ['/env', 'init'],
+    extra: '',
+    envOverridePath: target,
+    envProbe: false,
+    readInput: async (options) => {
+      asked.push(options);
+      return answers.shift() ?? '';
+    },
+  };
+
+  await withInteractiveStdin(() => routeCommand('/env init', ctx));
+
+  assert.equal(asked.length, 4, 'four wizard questions — no answer leaks into a chat query');
+  assert.ok(asked.every(q => q.history === false), 'wizard answers never enter command history');
+  const map = readEnvFile(target);
+  assert.equal(map.LMSTUDIO_BASE_URL, 'http://127.0.0.1:1234/v1', 'empty answer accepts the default');
+  assert.equal(map.KIKU_CONTEXT_LIMIT, '32768');
+  assert.equal(map.LMSTUDIO_MODELS_DIR, undefined);
+  assert.equal(map.OPENAI_API_KEY, undefined);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('/env: init overwrite question goes through ctx.readInput and regenerates the file', async () => {
+  const dir = tmpDir();
+  const packageFile = path.join(dir, 'package.env');
+  const userFile = path.join(dir, 'user.env');
+  fs.writeFileSync(packageFile, 'LMSTUDIO_BASE_URL=http://127.0.0.1:9999/v1\n');
+
+  const asked = [];
+  const answers = ['y', 'http://127.0.0.1:1234/v1', '', '', 'n'];
+  const ctx = {
+    sub: 'init',
+    parts: ['/env', 'init'],
+    extra: '',
+    envTargets: { packageEnvPath: packageFile, userEnvPath: userFile },
+    envProbe: false,
+    readInput: async (options) => {
+      asked.push(options);
+      return answers.shift() ?? '';
+    },
+  };
+
+  await withInteractiveStdin(() => routeCommand('/env init', ctx));
+
+  assert.ok(asked.length >= 1);
+  assert.match(String(asked[0].prompt), /Overwrite/, 'first question is the overwrite confirmation');
+  const map = readEnvFile(packageFile);
+  assert.equal(map.LMSTUDIO_BASE_URL, 'http://127.0.0.1:1234/v1', 'existing file is regenerated after confirm');
+  assert.equal(readEnvFile(userFile), null, 'user env file stays untouched');
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('/env: set/get/unset round-trips through the override target', async () => {
+  const dir = tmpDir();
+  const target = path.join(dir, '.env');
+  assert.equal(await routeCommand('/env set KIKU_CONTEXT_LIMIT 32768', { envOverridePath: target }), 'HANDLED');
+  assert.equal(readEnvFile(target).KIKU_CONTEXT_LIMIT, '32768');
+  assert.equal(await routeCommand('/env get KIKU_CONTEXT_LIMIT', { envOverridePath: target }), 'HANDLED');
+  assert.equal(await routeCommand('/env unset KIKU_CONTEXT_LIMIT', { envOverridePath: target }), 'HANDLED');
+  assert.equal(readEnvFile(target).KIKU_CONTEXT_LIMIT, undefined);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('/env: unset with an override target never touches the real package or user files', async () => {
+  const dir = tmpDir();
+  const target = path.join(dir, 'override.env');
+  const pkg = path.join(dir, 'pkg.env');
+  const user = path.join(dir, 'user.env');
+  fs.writeFileSync(target, 'KIKU_CONTEXT_LIMIT=8192\n');
+  fs.writeFileSync(pkg, 'KIKU_CONTEXT_LIMIT=131072\n');
+  fs.writeFileSync(user, 'OTHER=1\n');
+
+  await routeCommand('/env unset KIKU_CONTEXT_LIMIT', {
+    envOverridePath: target,
+    envTargets: { packageEnvPath: pkg, userEnvPath: user },
+  });
+
+  assert.equal(readEnvFile(target).KIKU_CONTEXT_LIMIT, undefined, 'override file must be cleared');
+  assert.equal(readEnvFile(pkg).KIKU_CONTEXT_LIMIT, '131072', 'real package file must stay untouched');
+  assert.equal(readEnvFile(user).OTHER, '1', 'real user file must stay untouched');
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('envManager: readMergedEnvMap resolves package-first with user fallback and tracks sources', () => {
+  const dir = tmpDir();
+  const pkg = path.join(dir, 'package.env');
+  const user = path.join(dir, 'user.env');
+  fs.writeFileSync(pkg, 'LMSTUDIO_BASE_URL=http://pkg:1234/v1\nONLY_PKG=1\n');
+  fs.writeFileSync(user, 'LMSTUDIO_BASE_URL=http://user:1234/v1\nONLY_USER=2\n');
+
+  const { map, sources } = readMergedEnvMap({ packageEnvPath: pkg, userEnvPath: user });
+  assert.equal(map.LMSTUDIO_BASE_URL, 'http://pkg:1234/v1', 'package file must win');
+  assert.equal(map.ONLY_PKG, '1');
+  assert.equal(map.ONLY_USER, '2', 'user file must fill unset keys');
+  assert.equal(sources.LMSTUDIO_BASE_URL, pkg);
+  assert.equal(sources.ONLY_USER, user);
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('/env: get reports keys set only in the user file and unset clears both files', async () => {
+  const dir = tmpDir();
+  const pkg = path.join(dir, 'package.env');
+  const user = path.join(dir, 'user.env');
+  fs.writeFileSync(pkg, 'ONLY_PKG=1\nSHARED=pkg\n');
+  fs.writeFileSync(user, 'ONLY_USER=2\nSHARED=user\n');
+
+  const output = await captureConsoleOutput(() => routeCommand('/env get ONLY_USER', { envTargets: { packageEnvPath: pkg, userEnvPath: user } }));
+  assert.match(output, /ONLY_USER=2/);
+  assert.match(output, /user\.env/, 'must reveal which file the value comes from');
+
+  await routeCommand('/env unset SHARED', { envTargets: { packageEnvPath: pkg, userEnvPath: user } });
+  assert.equal(readEnvFile(pkg).SHARED, undefined, 'package copy must be removed');
+  assert.equal(readEnvFile(user).SHARED, undefined, 'user copy must be removed');
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('envManager: removeEnvKey rejects regex-unsafe keys instead of throwing opaque errors', () => {
+  const dir = tmpDir();
+  const file = path.join(dir, '.env');
+  fs.writeFileSync(file, 'A=1\n');
+  assert.throws(() => removeEnvKey(file, '*'), /Invalid environment variable name/);
+  assert.throws(() => removeEnvKey(file, '('), /Invalid environment variable name/);
+  assert.equal(readEnvFile(file).A, '1');
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test('envManager: detectAutoDefaults with probe:false performs no network access', async () => {
+  const detected = await detectAutoDefaults({ probe: false, baseUrl: 'http://127.0.0.1:1/v1', timeoutMs: 100 });
+  assert.equal(detected.lmStudioReachable, false);
+  assert.equal(detected.loadedModel, null);
+  assert.ok('modelsDir' in detected, 'filesystem-only model dir detection still runs');
+});
+
+async function captureConsoleOutput(fn) {
+  const chunks = [];
+  const original = console.log;
+  console.log = (...args) => chunks.push(args.map(a => String(a)).join(' '));
+  try {
+    await fn();
+  } finally {
+    console.log = original;
+  }
+  return chunks.join('\n').replace(/\x1b\[[0-9;]*m/g, '');
+}

package/server/agent/tests/errorHandling.test.js

@@ -0,0 +1,80 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import {
+  parseApiErrorText,
+  parseExecutionError,
+  formatExecutionErrorCard,
+  createAbortError,
+  isAbortError,
+} from '../errors.js';
+
+test('parseApiErrorText extracts clean human-readable error messages from JSON payloads', () => {
+  const openRouterRaw = JSON.stringify({
+    error: {
+      message: 'Insufficient credits. This account never purchased credits. Make sure your key is on the correct account or org, and if so, purchase more at https://openrouter.ai/settings/credits',
+      code: 402,
+      metadata: { limit_source: 'credits' }
+    }
+  });
+
+  const parsed = parseApiErrorText(openRouterRaw);
+  assert.equal(parsed.includes('Insufficient credits.'), true);
+  assert.equal(parsed.includes('{"error":'), false);
+
+  const stringError = JSON.stringify({ error: 'Rate limit exceeded: TPM limit 6000 reached' });
+  assert.equal(parseApiErrorText(stringError), 'Rate limit exceeded: TPM limit 6000 reached');
+
+  const plainText = 'Gateway Timeout 504';
+  assert.equal(parseApiErrorText(plainText), 'Gateway Timeout 504');
+});
+
+test('parseExecutionError generates structured diagnostics for HTTP 402 Insufficient Credits', () => {
+  const err = new Error('OpenRouter Cloud error (402): {"error":{"message":"Insufficient credits. Please add balance."}}');
+  err.status = 402;
+  const diag = parseExecutionError(err, { provider: 'openrouter', providerName: 'OpenRouter Cloud' });
+
+  assert.equal(diag.type, 'insufficient_credits');
+  assert.equal(diag.title.includes('402'), true);
+  assert.equal(diag.headline.includes('out of credits'), true);
+  assert.equal(diag.recoveryOptions.some(opt => opt.cmd === '/provider use lmstudio'), true);
+});
+
+test('parseExecutionError generates structured diagnostics for HTTP 429 Rate Limit', () => {
+  const err = new Error('Groq Cloud error (429): Rate limit reached for model');
+  err.status = 429;
+  const diag = parseExecutionError(err, { provider: 'groq', providerName: 'Groq Cloud', model: 'llama-3.3-70b' });
+
+  assert.equal(diag.type, 'rate_limit');
+  assert.equal(diag.title.includes('429'), true);
+  assert.equal(diag.recoveryOptions.some(opt => opt.cmd === '/provider use lmstudio'), true);
+});
+
+test('parseExecutionError handles connection refused and local vs cloud providers', () => {
+  const localErr = new Error('fetch failed: connect ECONNREFUSED 127.0.0.1:1234');
+  const localDiag = parseExecutionError(localErr, { provider: 'lmstudio', providerName: 'LM Studio' });
+  assert.equal(localDiag.type, 'network');
+  assert.equal(localDiag.title.includes('LM Studio'), true);
+  assert.equal(localDiag.recoveryOptions.some(opt => opt.cmd === '/model reconnect'), true);
+
+  const cloudErr = new Error('fetch failed: connect ENOTFOUND api.groq.com');
+  const cloudDiag = parseExecutionError(cloudErr, { provider: 'groq', providerName: 'Groq Cloud' });
+  assert.equal(cloudDiag.type, 'network');
+  assert.equal(cloudDiag.recoveryOptions.some(opt => opt.cmd === '/provider use lmstudio'), true);
+});
+
+test('formatExecutionErrorCard creates a clean rich card options object', () => {
+  const err = new Error('OpenRouter Cloud error (402): {"error":{"message":"Insufficient credits."}}');
+  const card = formatExecutionErrorCard(err, { provider: 'openrouter', providerName: 'OpenRouter Cloud' });
+
+  assert.equal(card.status, 'error');
+  assert.equal(card.title.includes('OpenRouter Cloud'), true);
+  assert.equal(card.lines.length > 0, true);
+  assert.equal(card.lines.some(l => l.includes('/provider use lmstudio')), true);
+});
+
+test('createAbortError and isAbortError work properly', () => {
+  const abortErr = createAbortError();
+  assert.equal(isAbortError(abortErr), true);
+  assert.equal(isAbortError(new Error('regular error')), false);
+});

package/server/agent/tests/errors.test.js

@@ -0,0 +1,13 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { createAbortError, isAbortError } from '../errors.js';
+
+test('cancellation preserves AbortError identity across layers', () => {
+  const error = createAbortError();
+
+  assert.equal(error.name, 'AbortError');
+  assert.equal(error.message, 'Query cancelled by user.');
+  assert.equal(isAbortError(error), true);
+  assert.equal(isAbortError(new Error('failure')), false);
+});

package/server/agent/tests/evalAnalyze.test.js

@@ -0,0 +1,54 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(here, '../../..');
+const script = path.join(repoRoot, 'server/agent/eval/analyze.py');
+
+function runPython(args) {
+  for (const bin of ['python', 'python3', 'py']) {
+    const extra = bin === 'py' ? ['-3'] : [];
+    const result = spawnSync(bin, [...extra, script, ...args], { cwd: repoRoot, encoding: 'utf8', timeout: 60000 });
+    if (result.error?.code === 'ENOENT') continue;
+    return result;
+  }
+  throw new Error('no python interpreter found');
+}
+
+test('analyze.py exists and keeps eval no-comment rule', () => {
+  assert.equal(fs.existsSync(script), true);
+  const text = fs.readFileSync(script, 'utf8');
+  const hashLines = text.split('\n').filter((l) => l.includes('#'));
+  assert.deepEqual(hashLines, []);
+});
+
+test('analyze codebase mode returns whole-repo inventory and drift', () => {
+  const result = runPython(['--mode', 'codebase', '--skip-corpus']);
+  assert.equal(result.status, 0, result.stderr);
+  const payload = JSON.parse(result.stdout);
+  assert.ok(payload.codebase.fileCount > 200);
+  assert.ok(payload.codebase.totalLines > 50000);
+  assert.ok(payload.catalogDrift.pairCount === 8);
+  assert.ok(Array.isArray(payload.catalogDrift.pairs));
+});
+
+test('analyze report mode handles live report and gates', () => {
+  const report = path.join(repoRoot, 'artifacts/eval/live.json');
+  if (!fs.existsSync(report)) return;
+  const result = runPython(['--mode', 'report', '--report', 'artifacts/eval/live.json', '--skip-corpus']);
+  assert.equal(result.status, 0, result.stderr);
+  const payload = JSON.parse(result.stdout);
+  assert.equal(payload.report.status, 'ok');
+  assert.equal(payload.gates.available, true);
+});
+
+test('eval CLI exposes analyze subcommand and npm shortcut', () => {
+  const cliText = fs.readFileSync(path.join(repoRoot, 'server/agent/eval/modules/core/cli.js'), 'utf8');
+  assert.match(cliText, /analyze/);
+  const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'));
+  assert.equal(pkg.scripts['eval:analyze'], 'node server/agent/eval/modules/core/cli.js analyze');
+});

package/server/agent/tests/evalCommand.test.js

@@ -0,0 +1,54 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { performance } from 'node:perf_hooks';
+import { handlers } from '../commands.js';
+
+function makeCtx(command) {
+  const parts = command.split(/\s+/);
+  return {
+    parts,
+    sub: parts[1] || '',
+    extra: parts.slice(2).join(' '),
+    client: { provider: 'lmstudio', providerName: 'LM Studio', model: 'google/gemma-4-e4b', baseUrl: 'http://127.0.0.1:1234/v1' },
+    agent: {},
+  };
+}
+
+async function captureConsole(fn) {
+  const chunks = [];
+  const original = console.log;
+  console.log = (...args) => chunks.push(args.map(a => String(a)).join(' '));
+  try {
+    const result = await fn();
+    return { result, text: chunks.join('\n') };
+  } finally {
+    console.log = original;
+  }
+}
+
+test('/eval status summarizes the live report in-process and never spawns a run', async () => {
+  const started = performance.now();
+  const { result, text } = await captureConsole(() => handlers['/eval'](makeCtx('/eval status')));
+  const elapsed = performance.now() - started;
+
+  assert.equal(result, 'HANDLED');
+  assert.ok(elapsed < 1000, `/eval status took ${elapsed.toFixed(0)}ms — a spawn-based regression would take seconds`);
+  assert.ok(
+    text.includes('Pass@1') || text.includes('No evaluation report found yet'),
+    'must show the live report summary or the empty-state card'
+  );
+});
+
+test('/eval with an unknown subcommand shows the help card instead of forwarding to the eval CLI', async () => {
+  const { result, text } = await captureConsole(() => handlers['/eval'](makeCtx('/eval foobar')));
+  assert.equal(result, 'HANDLED');
+  assert.ok(text.includes('Subcommands'), 'unknown subcommand must render the help card');
+  assert.ok(!text.includes('exited with code'), 'unknown subcommand must not spawn the eval CLI');
+});
+
+test('/eval validate verifies the frozen corpus in-process', async () => {
+  const { result, text } = await captureConsole(() => handlers['/eval'](makeCtx('/eval validate')));
+  assert.equal(result, 'HANDLED');
+  assert.ok(text.includes('Corpus Valid'), 'validation must succeed');
+  assert.ok(text.includes('catalog'), 'panel must include the catalog split');
+});