@haystackeditor/cli 0.15.14 → 0.15.15

package/dist/index.js

@@ -41,6 +41,7 @@ import { setupCommand } from './commands/setup.js';
 import { schemaCommand, listSchemas } from './commands/schema-cmd.js';
 import { registerWebhook, listWebhooks, rotateWebhookSecret, setWebhookEnabled, listDeliveries, replayDelivery, } from './commands/webhooks.js';
 import { editRulesCommand, validateRulesCommand } from './commands/rules.js';
+import { verificationInitCommand, verificationValidateCommand, verificationPreflightCommand, verificationRunCommand, verificationCollectCommand, verificationCheckSafetyCommand, } from './commands/verification.js';
 import { headlessLoginCommand, headlessLogoutCommand, listTokensCommand, revokeTokenCommand, } from './commands/tokens.js';
 // `mcp` is imported lazily inside its command action (below), NOT at the top level.
 // It is the only module that pulls in `@modelcontextprotocol/sdk`; keeping it out of
@@ -817,6 +818,135 @@ rules
         process.exit(1);
     }
 });
+// ─── verification ────────────────────────────────────────────────────────────
+//
+// Repo-local verification contract (.haystack/verification.yml): how to boot
+// the app, prove it's ready, log in as test personas, run scenarios, and
+// collect evidence into a standard artifact manifest + review packet.
+// See docs/VERIFICATION-CONTRACT.md.
+const verification = program
+    .command('verification')
+    .description('Repo verification contract — validate, run scenarios, collect evidence');
+verification
+    .command('init')
+    .description('Scaffold .haystack/verification.yml, preflight/smoke scripts, and the agent skill')
+    .option('-f, --force', 'Overwrite existing files')
+    .option('--skill', 'Only write the install-verification agent skill (hand the integration to a coding agent)')
+    .addHelpText('after', `
+Creates:
+  .haystack/verification.yml                       the verification contract
+  scripts/haystack-preflight                       app-is-ready check
+  scripts/haystack-smoke                           boot + capture-evidence smoke scenario
+  .haystack/AGENTS.md                              how agents should use the contract
+  .haystack/skills/install-verification/SKILL.md   skill for completing the integration
+
+Detection fills in framework, package manager, dev command, port, and any
+auth-bypass env var found in the repo's own env examples. Review the output —
+detection is a starting point, not truth.
+`)
+    .action(async (opts) => {
+    try {
+        await verificationInitCommand(opts);
+    }
+    catch (err) {
+        console.error(chalk.red('init failed:'), err instanceof Error ? err.message : err);
+        process.exit(1);
+    }
+});
+verification
+    .command('validate')
+    .description('Check whether the repo satisfies the verification contract')
+    .option('--json', 'Machine-readable report')
+    .addHelpText('after', `
+Readiness verdicts:
+  ready            all checks pass (exit 0)
+  partially_ready  usable, with gaps (exit 0)
+  not_ready        contract missing/invalid or commands broken (exit 1)
+  unsafe           contract permits real side effects, or login helpers
+                   lack a production-disabled proof (exit 2)
+`)
+    .action(async (opts) => {
+    try {
+        await verificationValidateCommand(opts);
+    }
+    catch (err) {
+        console.error(chalk.red('validate failed:'), err instanceof Error ? err.message : err);
+        process.exit(1);
+    }
+});
+verification
+    .command('preflight')
+    .description('Run the contract preflight — is the app up and answering?')
+    .action(async () => {
+    try {
+        await verificationPreflightCommand();
+    }
+    catch (err) {
+        console.error(chalk.red('preflight failed:'), err instanceof Error ? err.message : err);
+        process.exit(1);
+    }
+});
+verification
+    .command('run <scenario>')
+    .description('Run a contract scenario and write the artifact manifest + review packet')
+    .option('--json', 'Print the manifest as JSON')
+    .option('--setup', 'Run environment.setup first (clean-clone mode)')
+    .option('--skip-preflight', 'Skip the preflight check')
+    .option('--timeout <seconds>', 'Override the scenario timeout', (v) => parseInt(v, 10))
+    .addHelpText('after', `
+Runs preflight, then the scenario command, capturing stdout/stderr to the
+artifact directory. Ends with a status:
+  verified               scenario passed (exit 0)
+  failed                 scenario failed (exit 1)
+  environment_not_ready  setup/preflight failed before the scenario ran (exit 1)
+  unsafe_to_verify       safety policy permits real side effects (exit 2)
+
+Evidence lands in the contract's artifact_dir:
+  manifest-<scenario>.json        machine-readable evidence manifest
+  review-packet-<scenario>.md     human-readable review packet
+  logs/                           captured command output
+
+Examples:
+  haystack verification run smoke
+  haystack verification run billing_upgrade --json
+  haystack verification run smoke --setup            # clean-clone mode (CI)
+`)
+    .action(async (scenario, opts) => {
+    try {
+        await verificationRunCommand(scenario, opts);
+    }
+    catch (err) {
+        console.error(chalk.red('run failed:'), err instanceof Error ? err.message : err);
+        process.exit(1);
+    }
+});
+verification
+    .command('collect-artifacts')
+    .description('Rebuild an evidence manifest from artifacts already on disk (status: inconclusive)')
+    .option('--scenario <name>', 'Scenario name to record in the manifest')
+    .option('--json', 'Print the manifest as JSON')
+    .action(async (opts) => {
+    try {
+        await verificationCollectCommand(opts);
+    }
+    catch (err) {
+        console.error(chalk.red('collect-artifacts failed:'), err instanceof Error ? err.message : err);
+        process.exit(1);
+    }
+});
+verification
+    .command('check-safety')
+    .description('Verify the verification setup is safe (policy, prod-disabled proof, dev-route gating, secret scan)')
+    .option('--json', 'Machine-readable report')
+    .action(async (opts) => {
+    try {
+        await verificationCheckSafetyCommand(opts);
+    }
+    catch (err) {
+        console.error(chalk.red('check-safety failed:'), err instanceof Error ? err.message : err);
+        process.exit(1);
+    }
+});
 // ─── mcp ─────────────────────────────────────────────────────────────────────
 program
     .command('mcp')

package/dist/verification/contract.d.ts

@@ -0,0 +1,240 @@
+import { z } from 'zod';
+export declare const CONTRACT_RELATIVE_PATH = ".haystack/verification.yml";
+export declare const DEFAULT_ARTIFACT_DIR = ".haystack/artifacts";
+declare const scenarioSchema: z.ZodObject<{
+    command: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    /** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
+    artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    /** Wall-clock cap for the scenario command. Default applied by the runner. */
+    timeout_seconds: z.ZodOptional<z.ZodNumber>;
+    /**
+     * Whether `run` gates this scenario on environment.preflight. Set false for
+     * self-contained scenarios that boot the app themselves (like the scaffolded
+     * smoke script) — preflight against a cold environment would always fail.
+     */
+    preflight: z.ZodDefault<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    command: string;
+    preflight: boolean;
+    description?: string | undefined;
+    artifacts?: string[] | undefined;
+    timeout_seconds?: number | undefined;
+}, {
+    command: string;
+    description?: string | undefined;
+    artifacts?: string[] | undefined;
+    timeout_seconds?: number | undefined;
+    preflight?: boolean | undefined;
+}>;
+declare const safetySchema: z.ZodObject<{
+    /** Command proving dev-only helpers are disabled in production builds. */
+    production_disabled_check: z.ZodOptional<z.ZodString>;
+    external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
+    allow_real_email: z.ZodDefault<z.ZodBoolean>;
+    allow_real_payments: z.ZodDefault<z.ZodBoolean>;
+    allow_customer_data: z.ZodDefault<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    allow_real_email: boolean;
+    allow_real_payments: boolean;
+    allow_customer_data: boolean;
+    production_disabled_check?: string | undefined;
+    external_services?: "none" | "sandbox_only" | "mocked" | undefined;
+}, {
+    production_disabled_check?: string | undefined;
+    external_services?: "none" | "sandbox_only" | "mocked" | undefined;
+    allow_real_email?: boolean | undefined;
+    allow_real_payments?: boolean | undefined;
+    allow_customer_data?: boolean | undefined;
+}>;
+export declare const verificationContractSchema: z.ZodObject<{
+    version: z.ZodEffects<z.ZodUnion<[z.ZodLiteral<1>, z.ZodLiteral<"1">]>, 1, 1 | "1">;
+    app: z.ZodObject<{
+        name: z.ZodString;
+        framework: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        framework?: string | undefined;
+    }, {
+        name: string;
+        framework?: string | undefined;
+    }>;
+    environment: z.ZodObject<{
+        setup: z.ZodOptional<z.ZodString>;
+        start: z.ZodOptional<z.ZodString>;
+        preflight: z.ZodString;
+        reset: z.ZodOptional<z.ZodString>;
+        artifact_dir: z.ZodDefault<z.ZodString>;
+        log_paths: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        preflight: string;
+        artifact_dir: string;
+        log_paths: string[];
+        start?: string | undefined;
+        setup?: string | undefined;
+        reset?: string | undefined;
+    }, {
+        preflight: string;
+        start?: string | undefined;
+        setup?: string | undefined;
+        reset?: string | undefined;
+        artifact_dir?: string | undefined;
+        log_paths?: string[] | undefined;
+    }>;
+    personas: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+        login: z.ZodString;
+        description: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        login: string;
+        description?: string | undefined;
+    }, {
+        login: string;
+        description?: string | undefined;
+    }>>>;
+    scenarios: z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodObject<{
+        command: z.ZodString;
+        description: z.ZodOptional<z.ZodString>;
+        /** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
+        artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        /** Wall-clock cap for the scenario command. Default applied by the runner. */
+        timeout_seconds: z.ZodOptional<z.ZodNumber>;
+        /**
+         * Whether `run` gates this scenario on environment.preflight. Set false for
+         * self-contained scenarios that boot the app themselves (like the scaffolded
+         * smoke script) — preflight against a cold environment would always fail.
+         */
+        preflight: z.ZodDefault<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        command: string;
+        preflight: boolean;
+        description?: string | undefined;
+        artifacts?: string[] | undefined;
+        timeout_seconds?: number | undefined;
+    }, {
+        command: string;
+        description?: string | undefined;
+        artifacts?: string[] | undefined;
+        timeout_seconds?: number | undefined;
+        preflight?: boolean | undefined;
+    }>>, Record<string, {
+        command: string;
+        preflight: boolean;
+        description?: string | undefined;
+        artifacts?: string[] | undefined;
+        timeout_seconds?: number | undefined;
+    }>, Record<string, {
+        command: string;
+        description?: string | undefined;
+        artifacts?: string[] | undefined;
+        timeout_seconds?: number | undefined;
+        preflight?: boolean | undefined;
+    }>>;
+    safety: z.ZodObject<{
+        /** Command proving dev-only helpers are disabled in production builds. */
+        production_disabled_check: z.ZodOptional<z.ZodString>;
+        external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
+        allow_real_email: z.ZodDefault<z.ZodBoolean>;
+        allow_real_payments: z.ZodDefault<z.ZodBoolean>;
+        allow_customer_data: z.ZodDefault<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        allow_real_email: boolean;
+        allow_real_payments: boolean;
+        allow_customer_data: boolean;
+        production_disabled_check?: string | undefined;
+        external_services?: "none" | "sandbox_only" | "mocked" | undefined;
+    }, {
+        production_disabled_check?: string | undefined;
+        external_services?: "none" | "sandbox_only" | "mocked" | undefined;
+        allow_real_email?: boolean | undefined;
+        allow_real_payments?: boolean | undefined;
+        allow_customer_data?: boolean | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    version: 1;
+    app: {
+        name: string;
+        framework?: string | undefined;
+    };
+    environment: {
+        preflight: string;
+        artifact_dir: string;
+        log_paths: string[];
+        start?: string | undefined;
+        setup?: string | undefined;
+        reset?: string | undefined;
+    };
+    personas: Record<string, {
+        login: string;
+        description?: string | undefined;
+    }>;
+    scenarios: Record<string, {
+        command: string;
+        preflight: boolean;
+        description?: string | undefined;
+        artifacts?: string[] | undefined;
+        timeout_seconds?: number | undefined;
+    }>;
+    safety: {
+        allow_real_email: boolean;
+        allow_real_payments: boolean;
+        allow_customer_data: boolean;
+        production_disabled_check?: string | undefined;
+        external_services?: "none" | "sandbox_only" | "mocked" | undefined;
+    };
+}, {
+    version: 1 | "1";
+    app: {
+        name: string;
+        framework?: string | undefined;
+    };
+    environment: {
+        preflight: string;
+        start?: string | undefined;
+        setup?: string | undefined;
+        reset?: string | undefined;
+        artifact_dir?: string | undefined;
+        log_paths?: string[] | undefined;
+    };
+    scenarios: Record<string, {
+        command: string;
+        description?: string | undefined;
+        artifacts?: string[] | undefined;
+        timeout_seconds?: number | undefined;
+        preflight?: boolean | undefined;
+    }>;
+    safety: {
+        production_disabled_check?: string | undefined;
+        external_services?: "none" | "sandbox_only" | "mocked" | undefined;
+        allow_real_email?: boolean | undefined;
+        allow_real_payments?: boolean | undefined;
+        allow_customer_data?: boolean | undefined;
+    };
+    personas?: Record<string, {
+        login: string;
+        description?: string | undefined;
+    }> | undefined;
+}>;
+export type VerificationContract = z.infer<typeof verificationContractSchema>;
+export type VerificationScenario = z.infer<typeof scenarioSchema>;
+export type VerificationSafety = z.infer<typeof safetySchema>;
+export type ContractLoadResult = {
+    status: 'ok';
+    contract: VerificationContract;
+    path: string;
+} | {
+    status: 'missing';
+    path: string;
+} | {
+    status: 'invalid';
+    path: string;
+    errors: string[];
+};
+/**
+ * Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
+ * Never throws — parse/schema failures come back as `status: 'invalid'` so
+ * callers can map them onto readiness / manifest statuses.
+ */
+export declare function loadContract(rootDir?: string): Promise<ContractLoadResult>;
+/** Resolve the artifact directory to an absolute path, creating it if needed. */
+export declare function ensureArtifactDir(contract: VerificationContract, rootDir?: string): Promise<string>;
+export {};

package/dist/verification/contract.js

@@ -0,0 +1,115 @@
+/**
+ * Haystack Verification Contract — `.haystack/verification.yml`
+ *
+ * The contract is the repo-local answer to five questions:
+ *   1. How do I set up and start the app?
+ *   2. How do I know the app is ready? (preflight)
+ *   3. How do I log in as useful test users? (personas)
+ *   4. Which verification flows can I run? (scenarios)
+ *   5. Where do logs, screenshots, and artifacts go?
+ *
+ * This is deliberately separate from `.haystack.json`: that file configures
+ * Haystack's own sandbox verification (services/flows/fixtures the sandbox
+ * drives). The contract instead describes commands the REPO owns — anything
+ * (Haystack, CI, a coding agent, a human) can run them and get the same
+ * evidence. See docs/VERIFICATION-CONTRACT.md.
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { parse as parseYaml } from 'yaml';
+import { z } from 'zod';
+export const CONTRACT_RELATIVE_PATH = '.haystack/verification.yml';
+export const DEFAULT_ARTIFACT_DIR = '.haystack/artifacts';
+const commandString = z.string().min(1, 'command must be a non-empty string');
+const personaSchema = z.object({
+    login: commandString.describe('Command that logs the given persona in (or prints credentials)'),
+    description: z.string().optional(),
+});
+const scenarioSchema = z.object({
+    command: commandString,
+    description: z.string().optional(),
+    /** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
+    artifacts: z.array(z.string()).optional(),
+    /** Wall-clock cap for the scenario command. Default applied by the runner. */
+    timeout_seconds: z.number().int().positive().max(7200).optional(),
+    /**
+     * Whether `run` gates this scenario on environment.preflight. Set false for
+     * self-contained scenarios that boot the app themselves (like the scaffolded
+     * smoke script) — preflight against a cold environment would always fail.
+     */
+    preflight: z.boolean().default(true),
+});
+const safetySchema = z.object({
+    /** Command proving dev-only helpers are disabled in production builds. */
+    production_disabled_check: commandString.optional(),
+    external_services: z.enum(['sandbox_only', 'mocked', 'none']).optional(),
+    allow_real_email: z.boolean().default(false),
+    allow_real_payments: z.boolean().default(false),
+    allow_customer_data: z.boolean().default(false),
+});
+export const verificationContractSchema = z.object({
+    // Accept 1 or "1" — YAML authors will write both.
+    version: z
+        .union([z.literal(1), z.literal('1')])
+        .transform(() => 1),
+    app: z.object({
+        name: commandString,
+        framework: z.string().optional(),
+    }),
+    environment: z.object({
+        setup: commandString.optional(),
+        start: commandString.optional(),
+        preflight: commandString,
+        reset: commandString.optional(),
+        artifact_dir: z.string().default(DEFAULT_ARTIFACT_DIR),
+        log_paths: z.array(z.string()).default([]),
+    }),
+    personas: z.record(personaSchema).default({}),
+    scenarios: z
+        .record(scenarioSchema)
+        .refine((s) => Object.keys(s).length > 0, {
+        message: 'at least one scenario is required',
+    }),
+    safety: safetySchema,
+});
+/**
+ * Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
+ * Never throws — parse/schema failures come back as `status: 'invalid'` so
+ * callers can map them onto readiness / manifest statuses.
+ */
+export async function loadContract(rootDir = process.cwd()) {
+    const contractPath = path.join(rootDir, CONTRACT_RELATIVE_PATH);
+    let raw;
+    try {
+        raw = await fs.readFile(contractPath, 'utf-8');
+    }
+    catch {
+        return { status: 'missing', path: contractPath };
+    }
+    let parsed;
+    try {
+        parsed = parseYaml(raw);
+    }
+    catch (err) {
+        return {
+            status: 'invalid',
+            path: contractPath,
+            errors: [`YAML parse error: ${err instanceof Error ? err.message : String(err)}`],
+        };
+    }
+    const result = verificationContractSchema.safeParse(parsed);
+    if (!result.success) {
+        return {
+            status: 'invalid',
+            path: contractPath,
+            errors: result.error.issues.map((issue) => `${issue.path.length ? issue.path.join('.') : '(root)'}: ${issue.message}`),
+        };
+    }
+    return { status: 'ok', contract: result.data, path: contractPath };
+}
+/** Resolve the artifact directory to an absolute path, creating it if needed. */
+export async function ensureArtifactDir(contract, rootDir = process.cwd()) {
+    const dir = path.resolve(rootDir, contract.environment.artifact_dir);
+    await fs.mkdir(path.join(dir, 'logs'), { recursive: true });
+    return dir;
+}

package/dist/verification/init.d.ts

@@ -0,0 +1,10 @@
+export interface InitResult {
+    created: string[];
+    skipped: string[];
+}
+export interface InitOptions {
+    force?: boolean;
+    /** Only write the agent skill file (for handing the integration to a coding agent). */
+    skillOnly?: boolean;
+}
+export declare function initVerification(rootDir?: string, options?: InitOptions): Promise<InitResult>;