@haystackeditor/cli 0.15.13 → 0.15.15

package/dist/verification/runner.js

@@ -0,0 +1,102 @@
+/**
+ * Command runner for verification contract commands.
+ *
+ * Every contract command (setup, preflight, scenario, safety check) runs
+ * through here so the evidence trail is uniform: output is streamed to the
+ * terminal AND teed to a log file under the artifact dir, and the result
+ * carries exit code + duration for the manifest's `commands_run` section.
+ */
+import { spawn, spawnSync } from 'node:child_process';
+import { createWriteStream } from 'node:fs';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+export async function runCommand(command, opts) {
+    const startedAt = Date.now();
+    let logStream;
+    if (opts.logFile) {
+        await fs.mkdir(path.dirname(opts.logFile), { recursive: true });
+        logStream = createWriteStream(opts.logFile);
+        logStream.write(`$ ${command}\n`);
+    }
+    const record = await new Promise((resolvePromise) => {
+        // detached puts the child in its own process group so a timeout kill
+        // reaches grandchildren (dev servers spawned by package-manager wrappers),
+        // not just the shell.
+        const child = spawn(command, {
+            cwd: opts.cwd,
+            shell: true,
+            detached: process.platform !== 'win32',
+            env: { ...process.env, ...opts.env, HAYSTACK_VERIFICATION: '1' },
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            killTree(child.pid);
+        }, opts.timeoutMs);
+        const onChunk = (chunk, stderr) => {
+            logStream?.write(chunk);
+            if (!opts.quiet) {
+                (stderr ? process.stderr : process.stdout).write(chunk);
+            }
+        };
+        child.stdout?.on('data', (c) => onChunk(c, false));
+        child.stderr?.on('data', (c) => onChunk(c, true));
+        const finish = (exitCode) => {
+            clearTimeout(timer);
+            const status = timedOut ? 'timeout' : exitCode === 0 ? 'passed' : 'failed';
+            if (timedOut) {
+                logStream?.write(`\n[haystack] command timed out after ${opts.timeoutMs}ms and was killed\n`);
+            }
+            resolvePromise({
+                command,
+                status,
+                exit_code: exitCode,
+                duration_ms: Date.now() - startedAt,
+            });
+        };
+        child.on('error', (err) => {
+            logStream?.write(`\n[haystack] failed to spawn: ${err.message}\n`);
+            finish(null);
+        });
+        child.on('close', (code) => finish(code));
+    });
+    if (logStream) {
+        await new Promise((res) => logStream.end(res));
+        record.log_path = path.relative(opts.cwd, opts.logFile);
+    }
+    return record;
+}
+function killTree(pid) {
+    if (pid === undefined)
+        return;
+    try {
+        if (process.platform === 'win32') {
+            const result = spawnSync('taskkill', ['/PID', String(pid), '/T', '/F'], {
+                windowsHide: true,
+                encoding: 'utf8',
+            });
+            if (result.error) {
+                throw result.error;
+            }
+            if (result.status !== 0) {
+                const detail = result.stderr.trim() || result.stdout.trim() || `taskkill exited with code ${result.status}`;
+                if (/not found|no running instance/i.test(detail)) {
+                    return;
+                }
+                throw new Error(detail);
+            }
+        }
+        else {
+            // Negative pid → the whole process group (see `detached` above).
+            process.kill(-pid, 'SIGKILL');
+        }
+    }
+    catch (error) {
+        if (error.code === 'ESRCH') {
+            return;
+        }
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`\n[haystack] failed to kill process tree for pid ${pid}: ${message}\n`);
+    }
+}

package/dist/verification/safety.d.ts

@@ -0,0 +1,10 @@
+export interface SafetyCheck {
+    name: string;
+    status: 'pass' | 'warn' | 'fail';
+    detail: string;
+}
+export interface SafetyReport {
+    ok: boolean;
+    checks: SafetyCheck[];
+}
+export declare function checkSafety(rootDir?: string): Promise<SafetyReport>;

package/dist/verification/safety.js

@@ -0,0 +1,252 @@
+/**
+ * `haystack verification check-safety` — verifies the verification setup
+ * itself can't hurt anyone:
+ *
+ *   1. The contract's safety policy forbids real side effects.
+ *   2. The repo's production-disabled check (if configured) actually passes.
+ *   3. Dev-only helper routes (/__dev/, log-me-in, dev-login) are gated
+ *      behind an environment check.
+ *   4. Neither the contract nor collected artifacts contain secrets.
+ *
+ * Checks are heuristic by design (static scans, not proofs) — the strongest
+ * guarantee remains the repo-owned `safety.production_disabled_check` command,
+ * which should be a real test asserting dev routes 404 in production mode.
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import fg from 'fast-glob';
+import { loadContract } from './contract.js';
+import { runCommand } from './runner.js';
+const SOURCE_GLOBS = ['**/*.{ts,tsx,js,jsx,mjs,cjs,py,rb,go,php}'];
+const SCAN_IGNORE = [
+    '**/node_modules/**',
+    '**/dist/**',
+    '**/build/**',
+    '**/.next/**',
+    '**/vendor/**',
+    '**/.git/**',
+    '**/*.test.*',
+    '**/*.spec.*',
+];
+const MAX_SCAN_FILES = 5000;
+const MAX_FILE_BYTES = 512 * 1024;
+/** Signatures of dev-only helpers that must never be reachable in production. */
+const DEV_HELPER_PATTERNS = [
+    /\/__dev\//,
+    /log[-_]?me[-_]?in/i,
+    /dev[-_]login/i,
+];
+/** Evidence that a file gates behavior on the runtime environment. */
+const ENV_GUARD_PATTERNS = [
+    /NODE_ENV/,
+    /import\.meta\.env\.(DEV|PROD|MODE)/,
+    /RAILS_ENV/,
+    /DJANGO_SETTINGS|settings\.DEBUG/,
+    /APP_ENV|FLASK_ENV|ENVIRONMENT/,
+];
+const SECRET_PATTERNS = [
+    { name: 'AWS access key', regex: /AKIA[0-9A-Z]{16}/ },
+    { name: 'GitHub token', regex: /gh[pousr]_[A-Za-z0-9]{36,}/ },
+    { name: 'GitHub fine-grained token', regex: /github_pat_[A-Za-z0-9_]{20,}/ },
+    { name: 'OpenAI-style key', regex: /sk-[A-Za-z0-9_-]{24,}/ },
+    { name: 'Anthropic key', regex: /sk-ant-[A-Za-z0-9_-]{24,}/ },
+    { name: 'Haystack CLI token', regex: /hsk_live_[A-Za-z0-9]{16,}/ },
+    { name: 'Slack token', regex: /xox[baprs]-[A-Za-z0-9-]{10,}/ },
+    { name: 'Private key block', regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
+];
+export async function checkSafety(rootDir = process.cwd()) {
+    const checks = [];
+    const loaded = await loadContract(rootDir);
+    if (loaded.status !== 'ok') {
+        checks.push({
+            name: 'contract',
+            status: 'fail',
+            detail: loaded.status === 'missing'
+                ? 'No verification contract — run `haystack verification init` first'
+                : `Contract is invalid: ${loaded.errors[0]}`,
+        });
+        return { ok: false, checks };
+    }
+    const contract = loaded.contract;
+    checkPolicy(contract, checks);
+    await runProductionDisabledCheck(contract, rootDir, checks);
+    await scanDevHelpers(rootDir, contract, checks);
+    await scanSecrets(rootDir, loaded.path, contract, checks);
+    return { ok: !checks.some((c) => c.status === 'fail'), checks };
+}
+function checkPolicy(contract, checks) {
+    const { safety } = contract;
+    const allowed = [
+        ['allow_real_email', safety.allow_real_email],
+        ['allow_real_payments', safety.allow_real_payments],
+        ['allow_customer_data', safety.allow_customer_data],
+    ].filter(([, v]) => v);
+    if (allowed.length > 0) {
+        for (const [flag] of allowed) {
+            checks.push({
+                name: 'safety-policy',
+                status: 'fail',
+                detail: `safety.${flag} is true — set it to false and use sandboxed/mocked services instead`,
+            });
+        }
+    }
+    else {
+        checks.push({
+            name: 'safety-policy',
+            status: 'pass',
+            detail: 'Policy forbids real email, real payments, and customer data',
+        });
+    }
+}
+async function runProductionDisabledCheck(contract, rootDir, checks) {
+    const command = contract.safety.production_disabled_check;
+    if (!command) {
+        checks.push({
+            name: 'production-disabled-check',
+            status: 'warn',
+            detail: 'No safety.production_disabled_check configured — add a test proving dev-only helpers are disabled in production builds',
+        });
+        return;
+    }
+    const logFile = path.resolve(rootDir, contract.environment.artifact_dir, 'logs', 'production-disabled-check.log');
+    const result = await runCommand(command, {
+        cwd: rootDir,
+        timeoutMs: 5 * 60 * 1000,
+        logFile,
+        quiet: true,
+    });
+    checks.push({
+        name: 'production-disabled-check',
+        status: result.status === 'passed' ? 'pass' : 'fail',
+        detail: result.status === 'passed'
+            ? `\`${command}\` passed (${(result.duration_ms / 1000).toFixed(1)}s)`
+            : `\`${command}\` ${result.status} (exit ${result.exit_code ?? 'n/a'}) — see ${result.log_path}`,
+    });
+}
+async function scanDevHelpers(rootDir, contract, checks) {
+    const files = await fg(SOURCE_GLOBS, {
+        cwd: rootDir,
+        ignore: [...SCAN_IGNORE, path.posix.join(contract.environment.artifact_dir, '**')],
+        dot: false,
+        onlyFiles: true,
+        suppressErrors: true,
+    });
+    const ungated = [];
+    let gatedCount = 0;
+    let skippedCount = 0;
+    for (const file of files.slice(0, MAX_SCAN_FILES)) {
+        const full = path.join(rootDir, file);
+        let content;
+        try {
+            const stat = await fs.stat(full);
+            if (stat.size > MAX_FILE_BYTES) {
+                skippedCount++;
+                continue;
+            }
+            content = await fs.readFile(full, 'utf-8');
+        }
+        catch {
+            skippedCount++;
+            continue;
+        }
+        if (!DEV_HELPER_PATTERNS.some((p) => p.test(content)))
+            continue;
+        if (ENV_GUARD_PATTERNS.some((p) => p.test(content))) {
+            gatedCount++;
+        }
+        else {
+            ungated.push(file);
+        }
+    }
+    if (ungated.length > 0) {
+        checks.push({
+            name: 'dev-helper-gating',
+            status: 'fail',
+            detail: `Dev-only helper code with no environment guard in: ${ungated.slice(0, 5).join(', ')}${ungated.length > 5 ? ` (+${ungated.length - 5} more)` : ''} — gate on NODE_ENV (or equivalent) so it cannot run in production`,
+        });
+    }
+    else {
+        checks.push({
+            name: 'dev-helper-gating',
+            status: 'pass',
+            detail: gatedCount > 0
+                ? `${gatedCount} file(s) with dev helpers all reference an environment guard`
+                : 'No dev-only helper routes detected',
+        });
+    }
+    if (files.length > MAX_SCAN_FILES) {
+        checks.push({
+            name: 'dev-helper-gating',
+            status: 'warn',
+            detail: `Scanned the first ${MAX_SCAN_FILES} of ${files.length} source files — coverage was truncated`,
+        });
+    }
+    if (skippedCount > 0) {
+        // Reduced coverage must be visible — silence reads as "covered".
+        checks.push({
+            name: 'dev-helper-gating',
+            status: 'warn',
+            detail: `${skippedCount} source file(s) were skipped (unreadable or >${MAX_FILE_BYTES / 1024}KB) and not scanned`,
+        });
+    }
+}
+async function scanSecrets(rootDir, contractPath, contract, checks) {
+    const targets = [contractPath];
+    const artifactDir = path.resolve(rootDir, contract.environment.artifact_dir);
+    const artifactFiles = await fg('**/*', {
+        cwd: artifactDir,
+        onlyFiles: true,
+        dot: true,
+        suppressErrors: true,
+    });
+    for (const file of artifactFiles)
+        targets.push(path.join(artifactDir, file));
+    const findings = [];
+    let skippedCount = 0;
+    for (const target of targets) {
+        let content;
+        try {
+            const stat = await fs.stat(target);
+            if (stat.size > 1024 * 1024) {
+                skippedCount++; // binary/huge artifacts — not scanned
+                continue;
+            }
+            content = await fs.readFile(target, 'utf-8');
+        }
+        catch {
+            skippedCount++;
+            continue;
+        }
+        for (const { name, regex } of SECRET_PATTERNS) {
+            const match = content.match(regex);
+            if (match) {
+                findings.push(`${name} in ${path.relative(rootDir, target)} (${redact(match[0])})`);
+            }
+        }
+    }
+    if (findings.length > 0) {
+        checks.push({
+            name: 'secret-scan',
+            status: 'fail',
+            detail: `Possible secrets found: ${findings.join('; ')} — redact logs/artifacts and rotate anything real`,
+        });
+    }
+    else {
+        checks.push({
+            name: 'secret-scan',
+            status: 'pass',
+            detail: `No secrets detected in the contract or ${artifactFiles.length - skippedCount} artifact file(s)`,
+        });
+    }
+    if (skippedCount > 0) {
+        // Reduced coverage must be visible — silence reads as "covered".
+        checks.push({
+            name: 'secret-scan',
+            status: 'warn',
+            detail: `${skippedCount} file(s) were skipped (unreadable or >1MB) and not scanned for secrets`,
+        });
+    }
+}
+function redact(secret) {
+    return `${secret.slice(0, 8)}…`;
+}

package/dist/verification/validate.d.ts

@@ -0,0 +1,12 @@
+export type Readiness = 'ready' | 'partially_ready' | 'not_ready' | 'unsafe';
+export type CheckLevel = 'pass' | 'warn' | 'fail' | 'unsafe';
+export interface ValidationItem {
+    level: CheckLevel;
+    message: string;
+}
+export interface ValidationReport {
+    readiness: Readiness;
+    contract_path: string;
+    items: ValidationItem[];
+}
+export declare function validateContract(rootDir?: string): Promise<ValidationReport>;

package/dist/verification/validate.js

@@ -0,0 +1,219 @@
+/**
+ * Contract validator — answers "can Haystack trust this repo's verification
+ * setup?" without running anything expensive.
+ *
+ * Readiness verdicts:
+ *   ready            all checks pass
+ *   partially_ready  usable, but with gaps (warnings)
+ *   not_ready        contract missing/invalid or references broken commands
+ *   unsafe           the contract permits real external side effects, or
+ *                    defines login helpers with no production-disabled proof
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { loadContract } from './contract.js';
+/** Package-manager subcommands that are built-ins, not package.json scripts. */
+const PM_BUILTINS = new Set([
+    'install', 'i', 'ci', 'add', 'remove', 'rm', 'uninstall', 'update', 'up', 'upgrade',
+    'exec', 'dlx', 'npx', 'create', 'init', 'link', 'unlink', 'publish', 'pack',
+    'audit', 'outdated', 'why', 'list', 'ls', 'prune', 'rebuild', 'config', 'store',
+]);
+export async function validateContract(rootDir = process.cwd()) {
+    const loaded = await loadContract(rootDir);
+    if (loaded.status === 'missing') {
+        return {
+            readiness: 'not_ready',
+            contract_path: loaded.path,
+            items: [
+                {
+                    level: 'fail',
+                    message: `No contract found at ${path.relative(rootDir, loaded.path)} — run \`haystack verification init\``,
+                },
+            ],
+        };
+    }
+    if (loaded.status === 'invalid') {
+        return {
+            readiness: 'not_ready',
+            contract_path: loaded.path,
+            items: loaded.errors.map((error) => ({ level: 'fail', message: error })),
+        };
+    }
+    const contract = loaded.contract;
+    const items = [];
+    items.push({ level: 'pass', message: 'Contract parses and matches the schema' });
+    // ── Command references ─────────────────────────────────────────────────────
+    const packageScripts = await readPackageScripts(rootDir);
+    const commands = [];
+    const env = contract.environment;
+    if (env.setup)
+        commands.push({ label: 'environment.setup', command: env.setup });
+    if (env.start)
+        commands.push({ label: 'environment.start', command: env.start });
+    commands.push({ label: 'environment.preflight', command: env.preflight });
+    if (env.reset)
+        commands.push({ label: 'environment.reset', command: env.reset });
+    if (contract.safety.production_disabled_check) {
+        commands.push({
+            label: 'safety.production_disabled_check',
+            command: contract.safety.production_disabled_check,
+        });
+    }
+    for (const [name, scenario] of Object.entries(contract.scenarios)) {
+        commands.push({ label: `scenarios.${name}`, command: scenario.command });
+    }
+    for (const [name, persona] of Object.entries(contract.personas)) {
+        commands.push({ label: `personas.${name}.login`, command: persona.login });
+    }
+    for (const { label, command } of commands) {
+        const issue = await checkCommandReference(command, rootDir, packageScripts);
+        if (issue) {
+            items.push({ level: issue.level, message: `${label}: ${issue.message}` });
+        }
+    }
+    // ── Coverage warnings ──────────────────────────────────────────────────────
+    const scenarioNames = Object.keys(contract.scenarios);
+    items.push({
+        level: 'pass',
+        message: `${scenarioNames.length} scenario${scenarioNames.length === 1 ? '' : 's'} defined (${scenarioNames.join(', ')})`,
+    });
+    if (!scenarioNames.includes('smoke')) {
+        items.push({
+            level: 'warn',
+            message: 'No `smoke` scenario — add a basic logged-in smoke test so every risky PR has a baseline check',
+        });
+    }
+    if (Object.keys(contract.personas).length === 0) {
+        items.push({
+            level: 'warn',
+            message: 'No personas defined — logged-in and role-based flows cannot be verified',
+        });
+    }
+    if (!env.start) {
+        items.push({
+            level: 'warn',
+            message: 'No start command — scenarios must boot the app themselves',
+        });
+    }
+    if (!env.reset) {
+        items.push({
+            level: 'warn',
+            message: 'No reset command — repeated runs may not start from a clean state',
+        });
+    }
+    const artifactDir = path.resolve(rootDir, env.artifact_dir);
+    if (await pathExists(artifactDir)) {
+        items.push({ level: 'pass', message: `Artifact directory exists (${env.artifact_dir})` });
+    }
+    else {
+        items.push({
+            level: 'pass',
+            message: `Artifact directory configured (${env.artifact_dir}) — will be created on first run`,
+        });
+    }
+    // ── Safety ─────────────────────────────────────────────────────────────────
+    const safety = contract.safety;
+    for (const [flag, value] of [
+        ['allow_real_email', safety.allow_real_email],
+        ['allow_real_payments', safety.allow_real_payments],
+        ['allow_customer_data', safety.allow_customer_data],
+    ]) {
+        if (value) {
+            items.push({
+                level: 'unsafe',
+                message: `safety.${flag} is true — verification must not have real external side effects`,
+            });
+        }
+    }
+    if (Object.keys(contract.personas).length > 0 && !safety.production_disabled_check) {
+        items.push({
+            level: 'unsafe',
+            message: 'Personas/login helpers are defined but safety.production_disabled_check is missing — there is no proof dev login is disabled in production builds',
+        });
+    }
+    if (safety.production_disabled_check) {
+        items.push({ level: 'pass', message: 'Production-disabled check configured' });
+    }
+    return {
+        readiness: computeReadiness(items),
+        contract_path: loaded.path,
+        items,
+    };
+}
+function computeReadiness(items) {
+    if (items.some((i) => i.level === 'unsafe'))
+        return 'unsafe';
+    if (items.some((i) => i.level === 'fail'))
+        return 'not_ready';
+    if (items.some((i) => i.level === 'warn'))
+        return 'partially_ready';
+    return 'ready';
+}
+async function readPackageScripts(rootDir) {
+    try {
+        const pkg = JSON.parse(await fs.readFile(path.join(rootDir, 'package.json'), 'utf-8'));
+        return pkg.scripts ?? {};
+    }
+    catch {
+        return null; // Not a Node project — script checks don't apply.
+    }
+}
+/**
+ * Best-effort static check that a contract command points at something real.
+ * Only definitive breakage is a `fail`; ambiguous cases (bare `pnpm foo` that
+ * could resolve to a node_modules binary) are warnings.
+ */
+async function checkCommandReference(command, rootDir, packageScripts) {
+    const tokens = command.trim().split(/\s+/);
+    // `bash scripts/x`, `sh ./scripts/x`, `node scripts/x.mjs`, or a direct
+    // `./scripts/x` — the referenced file must exist.
+    let scriptPath = null;
+    if (['bash', 'sh', 'node', 'python', 'python3', 'ruby'].includes(tokens[0]) && tokens[1]) {
+        scriptPath = tokens[1];
+    }
+    else if (tokens[0]?.startsWith('./') || tokens[0]?.startsWith('scripts/')) {
+        scriptPath = tokens[0];
+    }
+    if (scriptPath && !scriptPath.startsWith('-')) {
+        if (!(await pathExists(path.resolve(rootDir, scriptPath)))) {
+            return { level: 'fail', message: `references ${scriptPath}, which does not exist` };
+        }
+        return null;
+    }
+    // Package-manager script references.
+    if (packageScripts && ['pnpm', 'npm', 'yarn', 'bun'].includes(tokens[0])) {
+        let scriptName;
+        let explicitRun = false;
+        if (tokens[1] === 'run' && tokens[2]) {
+            scriptName = tokens[2];
+            explicitRun = true;
+        }
+        else if (tokens[1] && !tokens[1].startsWith('-')) {
+            scriptName = tokens[1];
+        }
+        if (!scriptName || PM_BUILTINS.has(scriptName))
+            return null;
+        if (packageScripts[scriptName])
+            return null;
+        if (explicitRun) {
+            return { level: 'fail', message: `script "${scriptName}" is not in package.json scripts` };
+        }
+        // Bare `pnpm foo` can also resolve to node_modules/.bin/foo.
+        if (await pathExists(path.join(rootDir, 'node_modules', '.bin', scriptName)))
+            return null;
+        return {
+            level: 'warn',
+            message: `"${scriptName}" is not in package.json scripts (and node_modules/.bin/${scriptName} was not found)`,
+        };
+    }
+    return null;
+}
+async function pathExists(p) {
+    try {
+        await fs.access(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.15.13",
+  "version": "0.15.15",
   "description": "Set up Haystack for your project — automated PR review, triage, and merge queue",
   "type": "module",
   "bin": {
@@ -40,12 +40,14 @@
     "fast-glob": "3.3.3",
     "glob": "10.5.0",
     "inquirer": "9.3.8",
+    "ws": "8.19.0",
     "yaml": "2.8.2",
     "zod": "3.25.76"
   },
   "devDependencies": {
     "@types/inquirer": "9.0.9",
     "@types/node": "20.19.30",
+    "@types/ws": "8.18.1",
     "typescript": "5.9.3"
   },
   "files": [