mod8-cli 0.2.0

package/dist/verify/runner.js

@@ -0,0 +1,437 @@
+import { spawn } from 'child_process';
+import { promises as fs } from 'fs';
+import { join, dirname, resolve } from 'path';
+import { tmpdir } from 'os';
+import { randomBytes } from 'crypto';
+import { fileURLToPath } from 'url';
+import yaml from 'js-yaml';
+import chalk from 'chalk';
+// Resolve mod8 root from this compiled file's location (dist/verify/runner.js → mod8)
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const MOD8_ROOT = resolve(__dirname, '..', '..');
+const MOD8_BIN_TARGET = join(MOD8_ROOT, 'bin', 'mod8.js');
+const ANSI_RE = /\x1B\[[0-9;]*[a-zA-Z]/g;
+const stripAnsi = (s) => s.replace(ANSI_RE, '');
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+async function setupSandbox() {
+    const id = randomBytes(4).toString('hex');
+    const dir = join(tmpdir(), `mod8-verify-${id}`);
+    await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+    const binDir = join(dir, 'bin');
+    await fs.mkdir(binDir, { recursive: true });
+    await fs.symlink(MOD8_BIN_TARGET, join(binDir, 'mod8'));
+    const env = {
+        ...process.env,
+        MOD8_CONFIG_DIR: dir,
+        PATH: `${binDir}:${process.env.PATH ?? ''}`,
+    };
+    // API key: explicit verify key only. Strip the user's real key so
+    // tests can never accidentally hit prod credentials.
+    delete env.ANTHROPIC_API_KEY;
+    delete env.OPENAI_API_KEY;
+    delete env.GOOGLE_API_KEY;
+    delete env.GEMINI_API_KEY;
+    if (process.env.MOD8_VERIFY_KEY) {
+        env.ANTHROPIC_API_KEY = process.env.MOD8_VERIFY_KEY;
+    }
+    // Ensure consent gate doesn't block --all tests.
+    env.MOD8_AUTO_CONFIRM = '1';
+    return { dir, binDir, env };
+}
+async function teardownSandbox(sandbox) {
+    await fs.rm(sandbox.dir, { recursive: true, force: true });
+}
+function runShell(command, stdin, env, timeoutMs = 15000) {
+    return new Promise((resolveP, rejectP) => {
+        const child = spawn('bash', ['-c', command], { env });
+        let stdout = '';
+        let stderr = '';
+        const timer = setTimeout(() => child.kill('SIGKILL'), timeoutMs);
+        child.stdout.on('data', (d) => {
+            stdout += d.toString();
+        });
+        child.stderr.on('data', (d) => {
+            stderr += d.toString();
+        });
+        child.on('error', (err) => {
+            clearTimeout(timer);
+            rejectP(err);
+        });
+        child.on('close', (code) => {
+            clearTimeout(timer);
+            resolveP({ stdout, stderr, exitCode: code ?? 0 });
+        });
+        if (stdin !== undefined)
+            child.stdin.write(stdin);
+        child.stdin.end();
+    });
+}
+async function runRepl(command, inputs, env, timeoutMs) {
+    // Spawn directly via bash; stdin is a regular pipe.  Ink-driven REPLs that
+    // require raw-mode TTY will crash on entry — that's expected, the runner just
+    // captures whatever stdout/stderr was emitted before the crash + SIGKILL on
+    // timeout, so spec assertions can still match against the early output.
+    const child = spawn('bash', ['-c', command], {
+        env,
+        detached: true, // own process group so we can SIGKILL grandchildren too
+    });
+    let stdout = '';
+    let stderr = '';
+    let exited = false;
+    let resolveClose;
+    const closePromise = new Promise((res) => {
+        resolveClose = res;
+    });
+    const killEverything = () => {
+        try {
+            if (child.pid)
+                process.kill(-child.pid, 'SIGKILL');
+        }
+        catch {
+            try {
+                child.kill('SIGKILL');
+            }
+            catch {
+                /* already gone */
+            }
+        }
+        try {
+            child.stdout?.destroy();
+        }
+        catch { /* ignore */ }
+        try {
+            child.stderr?.destroy();
+        }
+        catch { /* ignore */ }
+    };
+    const timer = setTimeout(() => {
+        killEverything();
+        // Fallback: even if 'close' never fires (orphaned grandchild holding
+        // pipes open elsewhere), resolve with -1 so the runner can move on.
+        setTimeout(() => resolveClose(-1), 500);
+    }, timeoutMs);
+    child.stdout.on('data', (d) => { stdout += d.toString(); });
+    child.stderr.on('data', (d) => { stderr += d.toString(); });
+    child.on('error', () => { });
+    child.stdin.on('error', () => { });
+    child.on('exit', () => { exited = true; });
+    child.on('close', (code) => resolveClose(code ?? 0));
+    await sleep(300);
+    for (const step of inputs) {
+        if (exited)
+            break;
+        if (step.send !== undefined) {
+            try {
+                child.stdin.write(step.send);
+            }
+            catch { /* ignore — child gone */ }
+        }
+        if (step.delay_ms !== undefined) {
+            await sleep(step.delay_ms);
+        }
+    }
+    try {
+        child.stdin.end();
+    }
+    catch { /* ignore */ }
+    const exitCode = await closePromise;
+    clearTimeout(timer);
+    // Best-effort cleanup if we returned via the fallback timer.
+    if (exitCode === -1)
+        killEverything();
+    return { stdout, stderr, exitCode };
+}
+function asArray(v) {
+    if (v === undefined)
+        return [];
+    return Array.isArray(v) ? v : [v];
+}
+async function matchExpect(expect, result, env) {
+    if (!expect)
+        return { ok: true };
+    const cleanStdout = stripAnsi(result.stdout);
+    const cleanStderr = stripAnsi(result.stderr);
+    for (const needle of asArray(expect.stdout_contains)) {
+        if (!cleanStdout.includes(needle)) {
+            return {
+                ok: false,
+                reason: `stdout missing ${JSON.stringify(needle)}\n--- actual stdout ---\n${cleanStdout || '(empty)'}\n--- actual stderr ---\n${cleanStderr || '(empty)'}`,
+            };
+        }
+    }
+    for (const pattern of asArray(expect.stdout_matches)) {
+        const re = compileRegex(pattern);
+        if (!re.test(cleanStdout)) {
+            return {
+                ok: false,
+                reason: `stdout doesn't match /${pattern}/\n--- actual stdout ---\n${cleanStdout || '(empty)'}`,
+            };
+        }
+    }
+    for (const needle of asArray(expect.stdout_omits)) {
+        if (cleanStdout.includes(needle)) {
+            return {
+                ok: false,
+                reason: `stdout should NOT contain ${JSON.stringify(needle)}\n--- actual stdout ---\n${cleanStdout}`,
+            };
+        }
+    }
+    for (const needle of asArray(expect.stderr_contains)) {
+        if (!cleanStderr.includes(needle)) {
+            return {
+                ok: false,
+                reason: `stderr missing ${JSON.stringify(needle)}\n--- actual stderr ---\n${cleanStderr || '(empty)'}`,
+            };
+        }
+    }
+    if (expect.exit_code !== undefined && result.exitCode !== expect.exit_code) {
+        return {
+            ok: false,
+            reason: `expected exit code ${expect.exit_code}, got ${result.exitCode}\n--- stdout ---\n${cleanStdout}\n--- stderr ---\n${cleanStderr}`,
+        };
+    }
+    if (expect.file_exists !== undefined) {
+        const path = expandEnv(expect.file_exists, env);
+        try {
+            await fs.access(path);
+        }
+        catch {
+            return { ok: false, reason: `file does not exist: ${path}` };
+        }
+    }
+    if (expect.file_mode !== undefined) {
+        const [pathRaw, expectedMode] = expect.file_mode.split(':');
+        const path = expandEnv(pathRaw, env);
+        try {
+            const stat = await fs.stat(path);
+            const mode = (stat.mode & 0o777).toString(8);
+            if (mode !== expectedMode) {
+                return { ok: false, reason: `file ${path} mode is ${mode}, expected ${expectedMode}` };
+            }
+        }
+        catch (err) {
+            return { ok: false, reason: `cannot stat ${path}: ${err.message}` };
+        }
+    }
+    return { ok: true };
+}
+/**
+ * Collect all *.yaml files under a directory, recursively.  Returns paths
+ * relative to the root so display lines stay short ("behavior/foo.yaml"
+ * rather than the full sandbox path).
+ */
+async function collectSpecFiles(root, relBase = '') {
+    const out = [];
+    const entries = await fs.readdir(root, { withFileTypes: true });
+    for (const e of entries) {
+        const rel = relBase ? `${relBase}/${e.name}` : e.name;
+        if (e.isDirectory()) {
+            out.push(...(await collectSpecFiles(join(root, e.name), rel)));
+        }
+        else if (e.isFile() && e.name.endsWith('.yaml')) {
+            out.push(rel);
+        }
+    }
+    return out;
+}
+/**
+ * Compile a regex from a spec string. Supports a leading `(?<flags>)` prefix
+ * (e.g. `(?i)…` for case-insensitive) — JS's RegExp doesn't accept inline
+ * flags by default, so we strip them and pass to the constructor.
+ */
+function compileRegex(pattern) {
+    const m = pattern.match(/^\(\?([imsuy]+)\)([\s\S]*)$/);
+    if (m)
+        return new RegExp(m[2], m[1]);
+    return new RegExp(pattern);
+}
+function expandEnv(s, env) {
+    return s.replace(/\$([A-Z_][A-Z0-9_]*)/g, (_, name) => env[name] ?? '');
+}
+async function runTest(test, spec) {
+    const start = Date.now();
+    const requiresKey = test.requires_api_key ?? spec.requires_api_key ?? false;
+    if (requiresKey && !process.env.MOD8_VERIFY_KEY) {
+        return {
+            name: test.name,
+            status: 'skipped',
+            durationMs: 0,
+            reason: 'no MOD8_VERIFY_KEY',
+        };
+    }
+    let sandbox = null;
+    try {
+        sandbox = await setupSandbox();
+        if (test.setup) {
+            for (let i = 0; i < test.setup.length; i++) {
+                const step = test.setup[i];
+                const cmd = step.run ?? step.shell;
+                if (!cmd) {
+                    return {
+                        name: test.name,
+                        status: 'fail',
+                        durationMs: Date.now() - start,
+                        reason: `setup step ${i + 1}: missing 'run' or 'shell'`,
+                    };
+                }
+                const result = await runShell(cmd, step.stdin, sandbox.env);
+                if (result.exitCode !== 0) {
+                    return {
+                        name: test.name,
+                        status: 'fail',
+                        durationMs: Date.now() - start,
+                        reason: `setup step ${i + 1} (${cmd}) exited ${result.exitCode}\n--- stdout ---\n${stripAnsi(result.stdout)}\n--- stderr ---\n${stripAnsi(result.stderr)}`,
+                    };
+                }
+            }
+        }
+        if (test.repl) {
+            const result = await runRepl(test.repl.run, test.repl.inputs ?? [], sandbox.env, test.repl.timeout_ms ?? 30000);
+            const match = await matchExpect(test.expect, result, sandbox.env);
+            return {
+                name: test.name,
+                status: match.ok ? 'pass' : 'fail',
+                durationMs: Date.now() - start,
+                reason: match.ok ? undefined : match.reason,
+            };
+        }
+        if (test.steps) {
+            for (let i = 0; i < test.steps.length; i++) {
+                const step = test.steps[i];
+                const cmd = step.run ?? step.shell;
+                if (!cmd) {
+                    return {
+                        name: test.name,
+                        status: 'fail',
+                        durationMs: Date.now() - start,
+                        reason: `step ${i + 1}: missing 'run' or 'shell'`,
+                    };
+                }
+                const result = await runShell(cmd, step.stdin, sandbox.env);
+                const match = await matchExpect(step.expect, result, sandbox.env);
+                if (!match.ok) {
+                    return {
+                        name: test.name,
+                        status: 'fail',
+                        durationMs: Date.now() - start,
+                        reason: `step ${i + 1} (${cmd}): ${match.reason}`,
+                    };
+                }
+            }
+            return { name: test.name, status: 'pass', durationMs: Date.now() - start };
+        }
+        if (test.run || test.shell) {
+            const cmd = test.run ?? test.shell;
+            const result = await runShell(cmd, test.stdin, sandbox.env);
+            const match = await matchExpect(test.expect, result, sandbox.env);
+            return {
+                name: test.name,
+                status: match.ok ? 'pass' : 'fail',
+                durationMs: Date.now() - start,
+                reason: match.ok ? undefined : match.reason,
+            };
+        }
+        return {
+            name: test.name,
+            status: 'fail',
+            durationMs: Date.now() - start,
+            reason: 'test has no run/shell/steps/repl',
+        };
+    }
+    catch (err) {
+        return {
+            name: test.name,
+            status: 'fail',
+            durationMs: Date.now() - start,
+            reason: `runner error: ${err.message}`,
+        };
+    }
+    finally {
+        if (sandbox)
+            await teardownSandbox(sandbox);
+    }
+}
+function printResult(result) {
+    const dur = `${(result.durationMs / 1000).toFixed(2)}s`;
+    if (result.status === 'pass') {
+        console.log(`  ${chalk.green('✓')} ${result.name}  ${chalk.dim(dur)}`);
+    }
+    else if (result.status === 'skipped') {
+        console.log(`  ${chalk.yellow('⊘')} ${chalk.dim(result.name)}  ${chalk.dim(`(skipped — ${result.reason})`)}`);
+    }
+    else {
+        console.log(`  ${chalk.red('✗')} ${result.name}  ${chalk.dim(dur)}`);
+        const indented = (result.reason ?? '').split('\n').map((l) => '      ' + l).join('\n');
+        console.log(chalk.red(indented));
+    }
+}
+export async function runVerify(opts = {}) {
+    const specsDir = opts.specsDir ?? join(MOD8_ROOT, 'specs');
+    const start = Date.now();
+    let files;
+    try {
+        files = (await collectSpecFiles(specsDir)).sort();
+    }
+    catch {
+        console.error(chalk.red(`mod8 verify: no specs/ directory found at ${specsDir}`));
+        return { pass: 0, fail: 0, skipped: 0, durationMs: 0, specs: [] };
+    }
+    if (files.length === 0) {
+        console.error(chalk.dim(`mod8 verify: no .yaml specs found in ${specsDir}`));
+        return { pass: 0, fail: 0, skipped: 0, durationMs: 0, specs: [] };
+    }
+    const hasKey = !!process.env.MOD8_VERIFY_KEY;
+    console.log();
+    console.log(chalk.bold('mod8 verify') +
+        chalk.dim(` · ${files.length} spec file${files.length === 1 ? '' : 's'}`) +
+        (hasKey ? chalk.dim(' · MOD8_VERIFY_KEY set') : chalk.dim(' · no MOD8_VERIFY_KEY (api tests will skip)')));
+    const specResults = [];
+    let totalPass = 0, totalFail = 0, totalSkipped = 0;
+    for (const relPath of files) {
+        const path = join(specsDir, relPath);
+        const data = await fs.readFile(path, 'utf8');
+        let spec;
+        try {
+            spec = yaml.load(data);
+        }
+        catch (err) {
+            console.log();
+            console.log(chalk.bold(relPath) + chalk.red(' (parse error)'));
+            console.log('  ' + chalk.red(err.message));
+            totalFail++;
+            continue;
+        }
+        console.log();
+        console.log(chalk.bold(relPath) + chalk.dim(` — ${spec.name}`));
+        const results = [];
+        for (const test of spec.tests ?? []) {
+            const result = await runTest(test, spec);
+            results.push(result);
+            printResult(result);
+            if (result.status === 'pass')
+                totalPass++;
+            else if (result.status === 'fail')
+                totalFail++;
+            else
+                totalSkipped++;
+        }
+        specResults.push({ file: relPath, name: spec.name, results });
+    }
+    const durationMs = Date.now() - start;
+    console.log();
+    console.log(chalk.dim('─'.repeat(60)));
+    const total = totalPass + totalFail + totalSkipped;
+    console.log(`${chalk.green(totalPass + ' pass')} · ` +
+        `${totalFail > 0 ? chalk.red(totalFail + ' fail') : chalk.dim(totalFail + ' fail')} · ` +
+        `${chalk.dim(totalSkipped + ' skipped')} · ` +
+        `${chalk.dim(total + ' total · ' + (durationMs / 1000).toFixed(2) + 's')}`);
+    console.log();
+    return {
+        pass: totalPass,
+        fail: totalFail,
+        skipped: totalSkipped,
+        durationMs,
+        specs: specResults,
+    };
+}

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "mod8-cli",
+  "version": "0.2.0",
+  "description": "Talk to any LLM from your terminal — Claude, GPT, Gemini, DeepSeek, Mistral, Groq, anything OpenAI-compatible. BYOK or `mod8 login` to use the hosted proxy.",
+  "type": "module",
+  "bin": {
+    "mod8": "./bin/mod8.js"
+  },
+  "files": [
+    "bin",
+    "dist",
+    "specs",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "verify": "node ./bin/mod8.js verify",
+    "prepublishOnly": "npm run build"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "keywords": [
+    "cli",
+    "llm",
+    "ai",
+    "chat",
+    "byok",
+    "claude",
+    "anthropic",
+    "openai",
+    "gpt",
+    "gemini",
+    "google",
+    "deepseek",
+    "mistral",
+    "groq",
+    "openrouter",
+    "xai",
+    "grok",
+    "together",
+    "compare",
+    "openai-compatible"
+  ],
+  "license": "MIT",
+  "author": "Yonatan Zlit",
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.94.0",
+    "@google/generative-ai": "^0.24.1",
+    "chalk": "^5.6.2",
+    "commander": "^14.0.3",
+    "ink": "^7.0.2",
+    "ink-text-input": "^6.0.0",
+    "js-yaml": "^4.1.1",
+    "openai": "^6.36.0",
+    "react": "^19.2.5"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^25.6.0",
+    "@types/react": "^19.2.14",
+    "ink-testing-library": "^4.0.0",
+    "tsx": "^4.21.0",
+    "typescript": "^6.0.3"
+  }
+}

package/specs/all-mode.yaml

@@ -0,0 +1,44 @@
+name: --all parallel mode
+description: --all renders three speaker blocks plus Total summary
+
+tests:
+  - name: all three providers respond, total line shows 3/3
+    shell: "MOD8_MOCK=1 mod8 --all 'compare in 5 words'"
+    expect:
+      stdout_contains:
+        - "Anthropic (Claude)"
+        - "OpenAI (GPT)"
+        - "Google (Gemini)"
+        - "Total:"
+        - "3/3 ok"
+      exit_code: 0
+
+  - name: partial failure renders friendly per-block error and partial total
+    shell: "MOD8_MOCK=1 MOD8_MOCK_FAIL=openai mod8 --all 'test'"
+    expect:
+      stdout_contains:
+        - "Anthropic (Claude)"
+        - "✗ failed"
+        - "Google (Gemini)"
+        - "2/3 ok"
+
+  - name: 401 error in --all is classified per-block
+    shell: "MOD8_MOCK=1 MOD8_MOCK_ERROR=401 MOD8_MOCK_ERROR_PROVIDER=anthropic mod8 --all 'test'"
+    expect:
+      stdout_contains:
+        - "✗ failed"
+        - "invalid API key"
+        - "2/3 ok"
+
+  - name: cost displayed in Total
+    shell: "MOD8_MOCK=1 mod8 --all 'hi'"
+    expect:
+      stdout_matches: "Total:.*tok.*\\$"
+
+  - name: consent gate refuses piped stdin without auto-confirm
+    shell: "echo 'n' | MOD8_AUTO_CONFIRM= MOD8_MOCK=1 mod8 --all 'test'"
+    expect:
+      stderr_contains:
+        - "first-run confirmation"
+        - "MOD8_AUTO_CONFIRM=1"
+      exit_code: 1

package/specs/behavior/auto-fallback.yaml

@@ -0,0 +1,49 @@
+name: behavior — auto-fallback after consecutive work errors
+description: |
+  When the work-mode provider keeps failing, mod8 should auto-fallback to
+  host so the user is never trapped.  The DECISION (warn vs fallback) is a
+  pure function in src/commands/intentRouting.ts: fallbackDecision(count).
+  These tests pin the threshold and the warn/fallback transition.
+
+  The runtime side of the fix (chat.tsx incrementing the counter on each
+  error, calling fallbackDecision, and switching modes) is covered by code
+  review — driving 3 consecutive turns through Ink would require a real PTY,
+  which the verify sandbox doesn't have.  The pure decision function is the
+  load-bearing piece and IS verifiable here.
+
+tests:
+  - name: 0 errors → ok (no banner, nothing to do)
+    shell: 'mod8 dev:check-fallback 0'
+    expect:
+      stdout_contains: "decision=ok"
+
+  - name: 1 error → warn (suggest switch-back, don't force)
+    shell: 'mod8 dev:check-fallback 1'
+    expect:
+      stdout_contains: "decision=warn"
+
+  - name: 2 errors → still warn (one shy of threshold)
+    shell: 'mod8 dev:check-fallback 2'
+    expect:
+      stdout_contains: "decision=warn"
+
+  - name: 3 errors → fallback (auto-switch to host)
+    shell: 'mod8 dev:check-fallback 3'
+    expect:
+      stdout_contains: "decision=fallback"
+
+  - name: 4+ errors → still fallback (decision is monotonic past threshold)
+    shell: 'mod8 dev:check-fallback 7'
+    expect:
+      stdout_contains: "decision=fallback"
+
+  - name: threshold is published as 3 (don't lower it without thinking)
+    shell: 'mod8 dev:check-fallback 0'
+    expect:
+      stdout_contains: "threshold=3"
+
+  - name: negative count is rejected
+    shell: 'mod8 dev:check-fallback -1'
+    expect:
+      stderr_contains: "non-negative"
+      exit_code: 1