@haystackeditor/cli 0.15.17 → 0.15.19

package/dist/verification/manifest.js

@@ -1,186 +0,0 @@
-/**
- * Artifact manifest + review packet.
- *
- * The manifest is the standardized output of every verification run — it's
- * what turns messy command output into reviewable evidence. The review packet
- * is the same data rendered as markdown for humans (and for posting on PRs).
- *
- * Status vocabulary (distinct on purpose — "failed verification" is different
- * from "could not verify"):
- *   verified              scenario ran and passed
- *   failed                scenario ran and failed
- *   inconclusive          artifacts collected without a run (or mixed signal)
- *   environment_not_ready preflight/setup failed before the scenario ran
- *   unsafe_to_verify      safety policy forbids running (real side effects)
- *   missing_contract      no valid .haystack/verification.yml
- */
-import { spawnSync } from 'node:child_process';
-import * as fs from 'node:fs/promises';
-import * as path from 'node:path';
-const EXTENSION_TYPES = {
-    '.png': 'screenshot',
-    '.jpg': 'screenshot',
-    '.jpeg': 'screenshot',
-    '.webp': 'screenshot',
-    '.gif': 'screenshot',
-    '.webm': 'video',
-    '.mp4': 'video',
-    '.mov': 'video',
-    '.log': 'log',
-    '.txt': 'log',
-    '.json': 'json',
-    '.html': 'html',
-    '.zip': 'trace',
-};
-export function classifyArtifact(filePath) {
-    return EXTENSION_TYPES[path.extname(filePath).toLowerCase()] ?? 'file';
-}
-export function getHeadSha(cwd) {
-    const res = spawnSync('git', ['rev-parse', 'HEAD'], { cwd, encoding: 'utf-8' });
-    if (res.status !== 0)
-        return null;
-    return res.stdout.trim() || null;
-}
-/**
- * Walk the artifact directory (plus configured log paths) and build the
- * manifest's artifact list. Manifest/packet outputs from previous runs are
- * excluded so evidence doesn't nest.
- */
-export async function collectArtifacts(opts) {
-    const entries = [];
-    const seen = new Set();
-    const sinceMs = opts.since ? opts.since.getTime() - 1000 : null; // 1s slack for coarse mtimes
-    const addFile = async (absPath, description) => {
-        const relPath = path.relative(opts.rootDir, absPath);
-        if (seen.has(relPath))
-            return;
-        if (/(^|\/)(manifest[^/]*\.json|review-packet[^/]*\.md)$/.test(relPath))
-            return;
-        let stat;
-        try {
-            stat = await fs.stat(absPath);
-        }
-        catch {
-            return;
-        }
-        if (!stat.isFile())
-            return;
-        if (sinceMs !== null && stat.mtimeMs < sinceMs)
-            return;
-        seen.add(relPath);
-        entries.push({
-            type: classifyArtifact(absPath),
-            path: relPath,
-            ...(description ? { description } : {}),
-            size_bytes: stat.size,
-        });
-    };
-    const walk = async (dir) => {
-        let dirEntries;
-        try {
-            dirEntries = await fs.readdir(dir, { withFileTypes: true });
-        }
-        catch {
-            return;
-        }
-        for (const entry of dirEntries) {
-            const full = path.join(dir, entry.name);
-            if (entry.isDirectory()) {
-                await walk(full);
-            }
-            else if (entry.isFile()) {
-                await addFile(full);
-            }
-        }
-    };
-    await walk(opts.artifactDir);
-    for (const extra of opts.extraPaths ?? []) {
-        await addFile(path.resolve(opts.rootDir, extra), 'configured log path');
-    }
-    entries.sort((a, b) => a.path.localeCompare(b.path));
-    return entries;
-}
-/** Read scenario-declared gaps from `<artifact_dir>/gaps.txt` (one per line), if present. */
-export async function readGaps(artifactDir) {
-    try {
-        const raw = await fs.readFile(path.join(artifactDir, 'gaps.txt'), 'utf-8');
-        return raw
-            .split('\n')
-            .map((line) => line.trim())
-            .filter((line) => line.length > 0 && !line.startsWith('#'));
-    }
-    catch {
-        return [];
-    }
-}
-export async function writeManifest(manifest, artifactDir) {
-    await fs.mkdir(artifactDir, { recursive: true });
-    const manifestPath = path.join(artifactDir, `manifest-${manifest.scenario}.json`);
-    await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
-    return manifestPath;
-}
-const STATUS_HEADLINES = {
-    verified: '✅ Verified — evidence looks strong.',
-    failed: '❌ Blocked — verification failed.',
-    inconclusive: '🟡 Needs human review — verification is incomplete.',
-    environment_not_ready: '⚠️ Not ready — the environment could not be prepared, so nothing was verified.',
-    unsafe_to_verify: '🛑 Unsafe — the safety policy allows real external side effects; verification was not run.',
-    missing_contract: '⚠️ Not ready — repo verification contract is missing or broken.',
-};
-export function renderReviewPacket(manifest) {
-    const lines = [];
-    lines.push(`# Haystack Verification — \`${manifest.scenario}\``);
-    lines.push('');
-    lines.push(STATUS_HEADLINES[manifest.status]);
-    lines.push('');
-    lines.push(`- Commit: \`${manifest.commit_sha ?? 'unknown'}\``);
-    lines.push(`- Started: ${manifest.started_at}`);
-    lines.push(`- Completed: ${manifest.completed_at}`);
-    lines.push('');
-    if (manifest.commands_run.length > 0) {
-        lines.push('## Commands run');
-        lines.push('');
-        lines.push('| Command | Status | Duration |');
-        lines.push('|---------|--------|----------|');
-        for (const cmd of manifest.commands_run) {
-            const icon = cmd.status === 'passed' ? '✅' : cmd.status === 'timeout' ? '⏱ timeout' : '❌ failed';
-            lines.push(`| \`${cmd.command.replace(/\|/g, '\\|')}\` | ${icon} | ${(cmd.duration_ms / 1000).toFixed(1)}s |`);
-        }
-        lines.push('');
-    }
-    lines.push('## Artifacts');
-    lines.push('');
-    if (manifest.artifacts.length === 0) {
-        lines.push('_No artifacts were captured._');
-    }
-    else {
-        for (const artifact of manifest.artifacts) {
-            const desc = artifact.description ? ` — ${artifact.description}` : '';
-            lines.push(`- \`${artifact.path}\` (${artifact.type})${desc}`);
-        }
-    }
-    lines.push('');
-    lines.push('## Gaps');
-    lines.push('');
-    if (manifest.gaps.length === 0) {
-        lines.push('_No gaps declared by this scenario._');
-    }
-    else {
-        for (const gap of manifest.gaps) {
-            lines.push(`- ${gap}`);
-        }
-    }
-    lines.push('');
-    lines.push('## Safety');
-    lines.push('');
-    lines.push(`- Real email sent: ${manifest.safety.real_email_sent ? '⚠️ yes' : 'no'}`);
-    lines.push(`- Real payment attempted: ${manifest.safety.real_payment_attempted ? '⚠️ yes' : 'no'}`);
-    lines.push(`- Customer data accessed: ${manifest.safety.customer_data_accessed ? '⚠️ yes' : 'no'}`);
-    lines.push('');
-    return lines.join('\n');
-}
-export async function writeReviewPacket(manifest, artifactDir) {
-    const packetPath = path.join(artifactDir, `review-packet-${manifest.scenario}.md`);
-    await fs.writeFile(packetPath, renderReviewPacket(manifest), 'utf-8');
-    return packetPath;
-}

package/dist/verification/runner.d.ts

@@ -1,20 +0,0 @@
-export type CommandStatus = 'passed' | 'failed' | 'timeout';
-export interface CommandRunRecord {
-    command: string;
-    status: CommandStatus;
-    exit_code: number | null;
-    duration_ms: number;
-    /** Repo-relative path to the captured stdout+stderr log, when one was written. */
-    log_path?: string;
-}
-export interface RunCommandOptions {
-    cwd: string;
-    /** Wall-clock cap; the process group is killed when exceeded. */
-    timeoutMs: number;
-    /** Absolute path to tee combined stdout+stderr into (created if missing). */
-    logFile?: string;
-    env?: Record<string, string>;
-    /** Suppress streaming to the terminal (still captured to logFile). */
-    quiet?: boolean;
-}
-export declare function runCommand(command: string, opts: RunCommandOptions): Promise<CommandRunRecord>;

package/dist/verification/runner.js

@@ -1,102 +0,0 @@
-/**
- * Command runner for verification contract commands.
- *
- * Every contract command (setup, preflight, scenario, safety check) runs
- * through here so the evidence trail is uniform: output is streamed to the
- * terminal AND teed to a log file under the artifact dir, and the result
- * carries exit code + duration for the manifest's `commands_run` section.
- */
-import { spawn, spawnSync } from 'node:child_process';
-import { createWriteStream } from 'node:fs';
-import * as fs from 'node:fs/promises';
-import * as path from 'node:path';
-export async function runCommand(command, opts) {
-    const startedAt = Date.now();
-    let logStream;
-    if (opts.logFile) {
-        await fs.mkdir(path.dirname(opts.logFile), { recursive: true });
-        logStream = createWriteStream(opts.logFile);
-        logStream.write(`$ ${command}\n`);
-    }
-    const record = await new Promise((resolvePromise) => {
-        // detached puts the child in its own process group so a timeout kill
-        // reaches grandchildren (dev servers spawned by package-manager wrappers),
-        // not just the shell.
-        const child = spawn(command, {
-            cwd: opts.cwd,
-            shell: true,
-            detached: process.platform !== 'win32',
-            env: { ...process.env, ...opts.env, HAYSTACK_VERIFICATION: '1' },
-            stdio: ['ignore', 'pipe', 'pipe'],
-        });
-        let timedOut = false;
-        const timer = setTimeout(() => {
-            timedOut = true;
-            killTree(child.pid);
-        }, opts.timeoutMs);
-        const onChunk = (chunk, stderr) => {
-            logStream?.write(chunk);
-            if (!opts.quiet) {
-                (stderr ? process.stderr : process.stdout).write(chunk);
-            }
-        };
-        child.stdout?.on('data', (c) => onChunk(c, false));
-        child.stderr?.on('data', (c) => onChunk(c, true));
-        const finish = (exitCode) => {
-            clearTimeout(timer);
-            const status = timedOut ? 'timeout' : exitCode === 0 ? 'passed' : 'failed';
-            if (timedOut) {
-                logStream?.write(`\n[haystack] command timed out after ${opts.timeoutMs}ms and was killed\n`);
-            }
-            resolvePromise({
-                command,
-                status,
-                exit_code: exitCode,
-                duration_ms: Date.now() - startedAt,
-            });
-        };
-        child.on('error', (err) => {
-            logStream?.write(`\n[haystack] failed to spawn: ${err.message}\n`);
-            finish(null);
-        });
-        child.on('close', (code) => finish(code));
-    });
-    if (logStream) {
-        await new Promise((res) => logStream.end(res));
-        record.log_path = path.relative(opts.cwd, opts.logFile);
-    }
-    return record;
-}
-function killTree(pid) {
-    if (pid === undefined)
-        return;
-    try {
-        if (process.platform === 'win32') {
-            const result = spawnSync('taskkill', ['/PID', String(pid), '/T', '/F'], {
-                windowsHide: true,
-                encoding: 'utf8',
-            });
-            if (result.error) {
-                throw result.error;
-            }
-            if (result.status !== 0) {
-                const detail = result.stderr.trim() || result.stdout.trim() || `taskkill exited with code ${result.status}`;
-                if (/not found|no running instance/i.test(detail)) {
-                    return;
-                }
-                throw new Error(detail);
-            }
-        }
-        else {
-            // Negative pid → the whole process group (see `detached` above).
-            process.kill(-pid, 'SIGKILL');
-        }
-    }
-    catch (error) {
-        if (error.code === 'ESRCH') {
-            return;
-        }
-        const message = error instanceof Error ? error.message : String(error);
-        process.stderr.write(`\n[haystack] failed to kill process tree for pid ${pid}: ${message}\n`);
-    }
-}

package/dist/verification/safety.d.ts

@@ -1,10 +0,0 @@
-export interface SafetyCheck {
-    name: string;
-    status: 'pass' | 'warn' | 'fail';
-    detail: string;
-}
-export interface SafetyReport {
-    ok: boolean;
-    checks: SafetyCheck[];
-}
-export declare function checkSafety(rootDir?: string): Promise<SafetyReport>;

package/dist/verification/safety.js

@@ -1,252 +0,0 @@
-/**
- * `haystack verification check-safety` — verifies the verification setup
- * itself can't hurt anyone:
- *
- *   1. The contract's safety policy forbids real side effects.
- *   2. The repo's production-disabled check (if configured) actually passes.
- *   3. Dev-only helper routes (/__dev/, log-me-in, dev-login) are gated
- *      behind an environment check.
- *   4. Neither the contract nor collected artifacts contain secrets.
- *
- * Checks are heuristic by design (static scans, not proofs) — the strongest
- * guarantee remains the repo-owned `safety.production_disabled_check` command,
- * which should be a real test asserting dev routes 404 in production mode.
- */
-import * as fs from 'node:fs/promises';
-import * as path from 'node:path';
-import fg from 'fast-glob';
-import { loadContract } from './contract.js';
-import { runCommand } from './runner.js';
-const SOURCE_GLOBS = ['**/*.{ts,tsx,js,jsx,mjs,cjs,py,rb,go,php}'];
-const SCAN_IGNORE = [
-    '**/node_modules/**',
-    '**/dist/**',
-    '**/build/**',
-    '**/.next/**',
-    '**/vendor/**',
-    '**/.git/**',
-    '**/*.test.*',
-    '**/*.spec.*',
-];
-const MAX_SCAN_FILES = 5000;
-const MAX_FILE_BYTES = 512 * 1024;
-/** Signatures of dev-only helpers that must never be reachable in production. */
-const DEV_HELPER_PATTERNS = [
-    /\/__dev\//,
-    /log[-_]?me[-_]?in/i,
-    /dev[-_]login/i,
-];
-/** Evidence that a file gates behavior on the runtime environment. */
-const ENV_GUARD_PATTERNS = [
-    /NODE_ENV/,
-    /import\.meta\.env\.(DEV|PROD|MODE)/,
-    /RAILS_ENV/,
-    /DJANGO_SETTINGS|settings\.DEBUG/,
-    /APP_ENV|FLASK_ENV|ENVIRONMENT/,
-];
-const SECRET_PATTERNS = [
-    { name: 'AWS access key', regex: /AKIA[0-9A-Z]{16}/ },
-    { name: 'GitHub token', regex: /gh[pousr]_[A-Za-z0-9]{36,}/ },
-    { name: 'GitHub fine-grained token', regex: /github_pat_[A-Za-z0-9_]{20,}/ },
-    { name: 'OpenAI-style key', regex: /sk-[A-Za-z0-9_-]{24,}/ },
-    { name: 'Anthropic key', regex: /sk-ant-[A-Za-z0-9_-]{24,}/ },
-    { name: 'Haystack CLI token', regex: /hsk_live_[A-Za-z0-9]{16,}/ },
-    { name: 'Slack token', regex: /xox[baprs]-[A-Za-z0-9-]{10,}/ },
-    { name: 'Private key block', regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
-];
-export async function checkSafety(rootDir = process.cwd()) {
-    const checks = [];
-    const loaded = await loadContract(rootDir);
-    if (loaded.status !== 'ok') {
-        checks.push({
-            name: 'contract',
-            status: 'fail',
-            detail: loaded.status === 'missing'
-                ? 'No verification contract — run `haystack verification init` first'
-                : `Contract is invalid: ${loaded.errors[0]}`,
-        });
-        return { ok: false, checks };
-    }
-    const contract = loaded.contract;
-    checkPolicy(contract, checks);
-    await runProductionDisabledCheck(contract, rootDir, checks);
-    await scanDevHelpers(rootDir, contract, checks);
-    await scanSecrets(rootDir, loaded.path, contract, checks);
-    return { ok: !checks.some((c) => c.status === 'fail'), checks };
-}
-function checkPolicy(contract, checks) {
-    const { safety } = contract;
-    const allowed = [
-        ['allow_real_email', safety.allow_real_email],
-        ['allow_real_payments', safety.allow_real_payments],
-        ['allow_customer_data', safety.allow_customer_data],
-    ].filter(([, v]) => v);
-    if (allowed.length > 0) {
-        for (const [flag] of allowed) {
-            checks.push({
-                name: 'safety-policy',
-                status: 'fail',
-                detail: `safety.${flag} is true — set it to false and use sandboxed/mocked services instead`,
-            });
-        }
-    }
-    else {
-        checks.push({
-            name: 'safety-policy',
-            status: 'pass',
-            detail: 'Policy forbids real email, real payments, and customer data',
-        });
-    }
-}
-async function runProductionDisabledCheck(contract, rootDir, checks) {
-    const command = contract.safety.production_disabled_check;
-    if (!command) {
-        checks.push({
-            name: 'production-disabled-check',
-            status: 'warn',
-            detail: 'No safety.production_disabled_check configured — add a test proving dev-only helpers are disabled in production builds',
-        });
-        return;
-    }
-    const logFile = path.resolve(rootDir, contract.environment.artifact_dir, 'logs', 'production-disabled-check.log');
-    const result = await runCommand(command, {
-        cwd: rootDir,
-        timeoutMs: 5 * 60 * 1000,
-        logFile,
-        quiet: true,
-    });
-    checks.push({
-        name: 'production-disabled-check',
-        status: result.status === 'passed' ? 'pass' : 'fail',
-        detail: result.status === 'passed'
-            ? `\`${command}\` passed (${(result.duration_ms / 1000).toFixed(1)}s)`
-            : `\`${command}\` ${result.status} (exit ${result.exit_code ?? 'n/a'}) — see ${result.log_path}`,
-    });
-}
-async function scanDevHelpers(rootDir, contract, checks) {
-    const files = await fg(SOURCE_GLOBS, {
-        cwd: rootDir,
-        ignore: [...SCAN_IGNORE, path.posix.join(contract.environment.artifact_dir, '**')],
-        dot: false,
-        onlyFiles: true,
-        suppressErrors: true,
-    });
-    const ungated = [];
-    let gatedCount = 0;
-    let skippedCount = 0;
-    for (const file of files.slice(0, MAX_SCAN_FILES)) {
-        const full = path.join(rootDir, file);
-        let content;
-        try {
-            const stat = await fs.stat(full);
-            if (stat.size > MAX_FILE_BYTES) {
-                skippedCount++;
-                continue;
-            }
-            content = await fs.readFile(full, 'utf-8');
-        }
-        catch {
-            skippedCount++;
-            continue;
-        }
-        if (!DEV_HELPER_PATTERNS.some((p) => p.test(content)))
-            continue;
-        if (ENV_GUARD_PATTERNS.some((p) => p.test(content))) {
-            gatedCount++;
-        }
-        else {
-            ungated.push(file);
-        }
-    }
-    if (ungated.length > 0) {
-        checks.push({
-            name: 'dev-helper-gating',
-            status: 'fail',
-            detail: `Dev-only helper code with no environment guard in: ${ungated.slice(0, 5).join(', ')}${ungated.length > 5 ? ` (+${ungated.length - 5} more)` : ''} — gate on NODE_ENV (or equivalent) so it cannot run in production`,
-        });
-    }
-    else {
-        checks.push({
-            name: 'dev-helper-gating',
-            status: 'pass',
-            detail: gatedCount > 0
-                ? `${gatedCount} file(s) with dev helpers all reference an environment guard`
-                : 'No dev-only helper routes detected',
-        });
-    }
-    if (files.length > MAX_SCAN_FILES) {
-        checks.push({
-            name: 'dev-helper-gating',
-            status: 'warn',
-            detail: `Scanned the first ${MAX_SCAN_FILES} of ${files.length} source files — coverage was truncated`,
-        });
-    }
-    if (skippedCount > 0) {
-        // Reduced coverage must be visible — silence reads as "covered".
-        checks.push({
-            name: 'dev-helper-gating',
-            status: 'warn',
-            detail: `${skippedCount} source file(s) were skipped (unreadable or >${MAX_FILE_BYTES / 1024}KB) and not scanned`,
-        });
-    }
-}
-async function scanSecrets(rootDir, contractPath, contract, checks) {
-    const targets = [contractPath];
-    const artifactDir = path.resolve(rootDir, contract.environment.artifact_dir);
-    const artifactFiles = await fg('**/*', {
-        cwd: artifactDir,
-        onlyFiles: true,
-        dot: true,
-        suppressErrors: true,
-    });
-    for (const file of artifactFiles)
-        targets.push(path.join(artifactDir, file));
-    const findings = [];
-    let skippedCount = 0;
-    for (const target of targets) {
-        let content;
-        try {
-            const stat = await fs.stat(target);
-            if (stat.size > 1024 * 1024) {
-                skippedCount++; // binary/huge artifacts — not scanned
-                continue;
-            }
-            content = await fs.readFile(target, 'utf-8');
-        }
-        catch {
-            skippedCount++;
-            continue;
-        }
-        for (const { name, regex } of SECRET_PATTERNS) {
-            const match = content.match(regex);
-            if (match) {
-                findings.push(`${name} in ${path.relative(rootDir, target)} (${redact(match[0])})`);
-            }
-        }
-    }
-    if (findings.length > 0) {
-        checks.push({
-            name: 'secret-scan',
-            status: 'fail',
-            detail: `Possible secrets found: ${findings.join('; ')} — redact logs/artifacts and rotate anything real`,
-        });
-    }
-    else {
-        checks.push({
-            name: 'secret-scan',
-            status: 'pass',
-            detail: `No secrets detected in the contract or ${artifactFiles.length - skippedCount} artifact file(s)`,
-        });
-    }
-    if (skippedCount > 0) {
-        // Reduced coverage must be visible — silence reads as "covered".
-        checks.push({
-            name: 'secret-scan',
-            status: 'warn',
-            detail: `${skippedCount} file(s) were skipped (unreadable or >1MB) and not scanned for secrets`,
-        });
-    }
-}
-function redact(secret) {
-    return `${secret.slice(0, 8)}…`;
-}

package/dist/verification/validate.d.ts

@@ -1,12 +0,0 @@
-export type Readiness = 'ready' | 'partially_ready' | 'not_ready' | 'unsafe';
-export type CheckLevel = 'pass' | 'warn' | 'fail' | 'unsafe';
-export interface ValidationItem {
-    level: CheckLevel;
-    message: string;
-}
-export interface ValidationReport {
-    readiness: Readiness;
-    contract_path: string;
-    items: ValidationItem[];
-}
-export declare function validateContract(rootDir?: string): Promise<ValidationReport>;