@kirrosh/zond 0.22.0 → 0.26.0

package/src/core/output/run.ts

@@ -0,0 +1,91 @@
+/**
+ * ARV-116 (m-19): runner that turns an `OutputSpec` + CLI flags into
+ * a `ResolvedOutput`, then renders and writes the payload.
+ *
+ * Resolution order:
+ *   1. `--json` and `--report` are mutually exclusive — error.
+ *   2. If `--report` is set, look it up in `aliases` first, then in
+ *      `formats`. Unknown → error (consistent with ARV-97; never
+ *      silently swallow a typo).
+ *   3. If `--json` is set, pick the first envelope-wrapping format
+ *      from the spec. Specs without one fall back to the
+ *      `defaultFormat` (acceptable for commands like `run` whose
+ *      `--json` is special-cased — see ARV-117).
+ *   4. Otherwise use `defaultFormat`.
+ *   5. Channel: `--output` forces file. Without it, take the format's
+ *      `defaultChannel`. File channel without an explicit `--output`
+ *      uses `defaultFilename` (relative paths resolved against cwd).
+ */
+import { resolve as resolvePath } from "path";
+import {
+  OutputSpecError,
+  type OutputOptions,
+  type OutputSpec,
+  type ResolvedOutput,
+} from "./types.ts";
+
+export function resolveOutput<P>(
+  spec: OutputSpec<P>,
+  opts: OutputOptions,
+): ResolvedOutput {
+  if (opts.json && typeof opts.report === "string" && opts.report.length > 0) {
+    throw new OutputSpecError(
+      "--json and --report are mutually exclusive — pick one output channel",
+    );
+  }
+
+  // Step 1: resolve the format.
+  let format: string;
+  if (typeof opts.report === "string" && opts.report.length > 0) {
+    const alias = spec.aliases?.[opts.report];
+    format = alias ?? opts.report;
+  } else if (opts.json) {
+    format = pickEnvelopeFormat(spec) ?? spec.defaultFormat;
+  } else {
+    format = spec.defaultFormat;
+  }
+
+  const policy = spec.formats[format];
+  if (!policy) {
+    const known = Object.keys(spec.formats).sort().join(", ");
+    throw new OutputSpecError(
+      `Unknown --report format: "${format}". Available for ${spec.command}: ${known}`,
+    );
+  }
+
+  // Step 2: resolve channel + path.
+  const explicitOutput = typeof opts.output === "string" && opts.output.length > 0
+    ? opts.output
+    : undefined;
+  let channel: "stdout" | "file";
+  let path: string | undefined;
+  if (explicitOutput) {
+    channel = "file";
+    path = resolvePath(explicitOutput);
+  } else if (policy.defaultChannel === "file") {
+    channel = "file";
+    path = policy.defaultFilename ? resolvePath(policy.defaultFilename) : undefined;
+    if (!path) {
+      throw new OutputSpecError(
+        `Format "${format}" defaults to file but has no defaultFilename and --output was not set`,
+      );
+    }
+  } else {
+    channel = "stdout";
+  }
+
+  return {
+    format,
+    channel,
+    path,
+    envelopeWrap: policy.envelopeWrap === true,
+  };
+}
+
+function pickEnvelopeFormat<P>(spec: OutputSpec<P>): string | undefined {
+  for (const [name, policy] of Object.entries(spec.formats)) {
+    if (policy.envelopeWrap) return name;
+  }
+  return undefined;
+}
+

package/src/core/output/types.ts

@@ -0,0 +1,122 @@
+/**
+ * ARV-116 (m-19): typed declaration of every `--report` / `--output` /
+ * `--json` combination a command supports.
+ *
+ * Background. Lesson §E of strategy/lessons.md collects seven separate
+ * bug reports about output-flag plumbing:
+ *   - ARV-97 — `--report ndjson --output <path>` silently dropped events;
+ *   - ARV-50 — probe `--dry-run` JSON shape diverged from `--report json`;
+ *   - ARV-82 — `db runs --json` envelope `data` field shape disagreed
+ *     with sibling commands;
+ *   - …and four more along the same lines.
+ *
+ * Root cause: each command has its own ad-hoc flag parser. Some treat
+ * "ndjson" as an alias for `--ndjson`, some don't. Some honour
+ * `--output` for SARIF only, some for every format. Some wrap in a JSON
+ * envelope, some emit a raw stream. There is no single place to read
+ * "what does `command X` produce when I pass `--report Y --output Z`".
+ *
+ * This module gives that single place. A command declares an
+ * `OutputSpec<Payload>` once — listing the formats it supports, each
+ * format's default channel (stdout vs file), each format's default
+ * filename, and whether the format wraps in the standard `--json`
+ * envelope. A runner helper (`runCommandWithOutput`) consumes the spec
+ * plus the CLI flags and resolves them to a single `ResolvedOutput`
+ * decision, with consistent mutual-exclusion enforcement.
+ *
+ * The spec is the source-of-truth for ARV-120's build-time check:
+ * every `--json` command must declare an `OutputSpec` with an
+ * `envelopeWrap: true` entry, so the published schema and the runtime
+ * output cannot drift.
+ *
+ * This task only ships the infrastructure. Per-command migration
+ * (`run`, `checks`, `probe*`) lands in ARV-117/118/119.
+ */
+
+/** Where a rendered payload lands. */
+export type OutputChannel = "stdout" | "file";
+
+/** A format name — `sarif` / `ndjson` / `json` / `junit` etc. The set is
+ *  open: each command declares its own. */
+export type OutputFormat = string;
+
+/** Per-format policy: where the format writes by default, what filename
+ *  to use when it writes to a file, and whether the payload is wrapped
+ *  in the standard `{ ok, command, data, ... }` envelope. */
+export interface FormatPolicy {
+  /** Default destination when neither `--output` nor channel-overriding
+   *  flags are present. SARIF defaults to file (`zond-checks.sarif`);
+   *  NDJSON defaults to stdout (one event per line for piping); JSON
+   *  envelopes default to stdout. */
+  defaultChannel: OutputChannel;
+  /** Default filename when `defaultChannel === "file"`. Relative paths
+   *  are resolved against cwd by the runner. Ignored when
+   *  `defaultChannel === "stdout"` — the user has to opt in to a file
+   *  via `--output`. */
+  defaultFilename?: string;
+  /** True for formats that should be wrapped in the shared envelope
+   *  (`jsonOk` / `jsonError` from `cli/json-envelope.ts`). Streaming
+   *  formats (NDJSON) and bespoke serialisations (SARIF, JUnit) are
+   *  not envelope-wrapped — they have their own contracts. */
+  envelopeWrap?: boolean;
+  /** ARV-120: for `envelopeWrap` formats — basename of the JSON schema
+   *  under `docs/json-schema/` that describes the envelope's `data`
+   *  field. The build-time coverage test asserts the file exists, so
+   *  a renamed/deleted schema fails CI together with the spec. */
+  envelopeSchemaFile?: string;
+  /** Optional human description, surfaced by `--help` generators and
+   *  the README table. */
+  description?: string;
+}
+
+export interface OutputSpec<Payload = unknown> {
+  /** Command name — propagated into the JSON envelope's `command` field
+   *  and used by error messages. */
+  command: string;
+  /** Supported formats keyed by name. Unknown formats fall through to
+   *  an error (no silent acceptance, see ARV-97). */
+  formats: Record<OutputFormat, FormatPolicy>;
+  /** Default format when neither `--report` nor `--json` is set. */
+  defaultFormat: OutputFormat;
+  /** Optional alias map: `{ ndjson: 'ndjson' }` lets `--report ndjson`
+   *  fold into the `--ndjson` flag (ARV-63 alias, retained because
+   *  skill prompts ship it). Keys are flag values seen on the CLI;
+   *  values are the resolved format name. */
+  aliases?: Record<string, OutputFormat>;
+}
+
+/** Decision the runner makes after applying the spec to CLI flags. */
+export interface ResolvedOutput {
+  /** Resolved format name (always one of `spec.formats`). */
+  format: OutputFormat;
+  /** Resolved destination. */
+  channel: OutputChannel;
+  /** Filesystem path when `channel === "file"`. Absolute path expected
+   *  by callers — the runner resolves it against cwd. */
+  path?: string;
+  /** Whether the runner should wrap the payload in the standard
+   *  envelope before writing. Mirrors `FormatPolicy.envelopeWrap`. */
+  envelopeWrap: boolean;
+}
+
+/** Inputs the runner reads from the CLI layer. Names mirror commander
+ *  options so a command can pass `cmd.opts<OutputOptions>()` directly. */
+export interface OutputOptions {
+  /** `--report <format>`. */
+  report?: string;
+  /** `--output <path>`. */
+  output?: string;
+  /** `--json`. */
+  json?: boolean;
+}
+
+/** Thrown by `resolveOutput` when the user-supplied flags violate the
+ *  spec's policy (unknown format, mutually exclusive flags, …). The
+ *  CLI layer catches this and emits the standard `jsonError` or
+ *  human `printError`. */
+export class OutputSpecError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "OutputSpecError";
+  }
+}

package/src/core/parser/dynamic-values.ts

@@ -0,0 +1,160 @@
+/**
+ * `#(funcName)` / `#(funcName(args))` dynamic-value substitution for
+ * loaded YAML values (ARV-190, m-21).
+ *
+ * Lets a workspace avoid stale hardcoded data — UUIDs that get reused
+ * by the API and pin existing rows, dates that expire days after
+ * checkout, idempotency keys that race against past runs. Functions
+ * are evaluated at LOAD time (per-run, not per-request), so a single
+ * `#(uuid)` reference resolves to the same value across every step
+ * inside one run — the idempotency-replay scenarios only work if the
+ * key stays stable for the run's lifetime.
+ *
+ * Supported functions:
+ *   #(uuid)              — fresh UUID v4 (stable within one run via cache)
+ *   #(uuidStable(seed))  — deterministic UUID derived from seed (sha-256 → v4 shape)
+ *   #(today)             — YYYY-MM-DD now (UTC)
+ *   #(todayPlus(N))      — today + N days (N may be negative)
+ *   #(now)               — ISO 8601 timestamp
+ *   #(unix)              — seconds since epoch
+ *   #(alphanumeric(N))   — N random a-z0-9 chars
+ *   #(env:VAR)           — process.env.VAR (alias for ${VAR})
+ *
+ * Resolution order: env-interpolation (${VAR}) runs first because its
+ * default-value syntax can produce strings the dynamic resolver should
+ * see literally. Dynamic values run BEFORE @secret/@identity reference
+ * resolution so the function tokens never leak into a secret-typed
+ * value (and so secret-stored values that happen to look like `#(...)`
+ * stay opaque). Strings that have no `#(` substring short-circuit.
+ *
+ * Nested forms: a single value may carry multiple references mixed
+ * with literal text — `"req-#(uuid)-#(today)"` yields `req-<uuid>-2026-05-16`.
+ */
+
+import { createHash } from "node:crypto";
+
+export interface DynamicValueContext {
+  /** Per-run cache keyed by raw expression (`#(uuid)`, `#(today)`). */
+  cache: Map<string, string>;
+  /** Source of env vars; defaults to process.env. Override in tests. */
+  env?: Record<string, string | undefined>;
+  /** File path the value came from — surfaced in error messages. */
+  filePath?: string;
+}
+
+/** Build a fresh cache for one resolution pass. Exported so the caller
+ *  can share a cache across multiple `resolveDynamicValuesDeep` calls
+ *  that belong to the same run (e.g. workspace + per-API env files). */
+export function newDynamicCache(): Map<string, string> {
+  return new Map();
+}
+
+// Captures `#(funcName)` or `#(funcName(args))`. `args` may contain any
+// chars except an unescaped closing paren — keep it simple, the funcs we
+// support take string or integer args without nested parens.
+const FUNC_RE = /#\(([A-Za-z_][A-Za-z0-9_]*)(?::([^)]*))?(?:\(([^)]*)\))?\)/g;
+
+export function resolveDynamicValues(
+  text: string,
+  ctx: DynamicValueContext,
+): string {
+  if (typeof text !== "string" || text.indexOf("#(") === -1) return text;
+  return text.replace(FUNC_RE, (full, name: string, colonArg: string | undefined, parenArg: string | undefined) => {
+    const cached = ctx.cache.get(full);
+    if (cached !== undefined) return cached;
+    const value = evaluate(name, colonArg ?? parenArg, full, ctx);
+    ctx.cache.set(full, value);
+    return value;
+  });
+}
+
+export function resolveDynamicValuesDeep(
+  obj: Record<string, unknown>,
+  ctx: DynamicValueContext,
+): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const [k, v] of Object.entries(obj)) {
+    if (typeof v === "string") {
+      out[k] = resolveDynamicValues(v, ctx);
+    } else {
+      out[k] = String(v);
+    }
+  }
+  return out;
+}
+
+function evaluate(
+  name: string,
+  arg: string | undefined,
+  fullExpr: string,
+  ctx: DynamicValueContext,
+): string {
+  switch (name) {
+    case "uuid":
+      return crypto.randomUUID();
+    case "uuidStable": {
+      if (!arg) throw makeError(`#(uuidStable(seed)) requires a seed argument`, fullExpr, ctx);
+      return seedToUuid(arg);
+    }
+    case "today":
+      return new Date().toISOString().slice(0, 10);
+    case "todayPlus": {
+      if (!arg) throw makeError(`#(todayPlus(N)) requires an integer N`, fullExpr, ctx);
+      const n = Number.parseInt(arg, 10);
+      if (!Number.isFinite(n)) throw makeError(`#(todayPlus(${arg})) — N must be an integer`, fullExpr, ctx);
+      const d = new Date();
+      d.setUTCDate(d.getUTCDate() + n);
+      return d.toISOString().slice(0, 10);
+    }
+    case "now":
+      return new Date().toISOString();
+    case "unix":
+      return String(Math.floor(Date.now() / 1000));
+    case "alphanumeric": {
+      const n = arg ? Number.parseInt(arg, 10) : 8;
+      if (!Number.isFinite(n) || n <= 0 || n > 1024) {
+        throw makeError(`#(alphanumeric(${arg ?? ""})) — length must be 1..1024`, fullExpr, ctx);
+      }
+      return randomAlphanumeric(n);
+    }
+    case "env": {
+      if (!arg) throw makeError(`#(env:VAR) requires a variable name`, fullExpr, ctx);
+      const env = ctx.env ?? (process.env as Record<string, string | undefined>);
+      const v = env[arg];
+      if (v === undefined || v === "") {
+        throw makeError(`#(env:${arg}) is not set — define it in your shell or CI secret`, fullExpr, ctx);
+      }
+      return v;
+    }
+    default:
+      throw makeError(`unknown dynamic function "${name}" in ${fullExpr} — supported: uuid, uuidStable, today, todayPlus, now, unix, alphanumeric, env`, fullExpr, ctx);
+  }
+}
+
+function seedToUuid(seed: string): string {
+  // SHA-256 the seed, take first 16 bytes, format as UUID v4 shape so
+  // any consumer-side `format: uuid` validator accepts it. Variant +
+  // version bits are forced to match v4 — the value is still
+  // deterministic because the source bytes are stable per seed.
+  const hash = createHash("sha256").update(seed).digest();
+  const bytes = Buffer.from(hash.subarray(0, 16));
+  bytes[6] = (bytes[6]! & 0x0f) | 0x40;  // version 4
+  bytes[8] = (bytes[8]! & 0x3f) | 0x80;  // RFC 4122 variant
+  const hex = bytes.toString("hex");
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;
+}
+
+const ALPHANUM = "abcdefghijklmnopqrstuvwxyz0123456789";
+
+function randomAlphanumeric(n: number): string {
+  let out = "";
+  for (let i = 0; i < n; i++) {
+    out += ALPHANUM[Math.floor(Math.random() * ALPHANUM.length)];
+  }
+  return out;
+}
+
+function makeError(msg: string, expr: string, ctx: DynamicValueContext): Error {
+  const where = ctx.filePath ? ` (referenced from ${ctx.filePath})` : "";
+  return new Error(`Dynamic value ${expr}: ${msg}${where}`);
+}

package/src/core/parser/env-interpolation.ts

@@ -0,0 +1,104 @@
+/**
+ * `${VAR}` / `${VAR:-default}` substitution for `.env.yaml` (TASK-169, m-10).
+ *
+ * Lets a workspace commit `.env.yaml` without bare secrets:
+ *
+ *     auth_token: "${MYAPI_AUTH_TOKEN}"
+ *     base_url:   "${MYAPI_BASE_URL:-https://api.example.com}"
+ *
+ * Rules:
+ *   - `${VAR}`           → process.env.VAR (throws if missing).
+ *   - `${VAR:-default}`  → process.env.VAR ?? default. The default may
+ *     contain `:` (everything after `:-` up to the closing `}` is the
+ *     default).
+ *   - `\${LITERAL}`      → literal `${LITERAL}` (the backslash is
+ *     stripped). Matches the `dotenv-expand` / docker-compose convention.
+ *   - One level of resolution only — values pulled from env are NOT
+ *     re-scanned for further `${...}` (cycle-risk).
+ *   - Variable names matching /TOKEN|SECRET|PASSWORD|KEY|DSN/i are NOT
+ *     auto-registered with the redaction registry. Auto-registration is
+ *     opt-in via `@secret:` (TASK-170). We do print a one-line warning
+ *     suggesting the user mark them as a secret, but only the first time.
+ */
+
+const ENV_REF_RE = /(\\?)\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}/g;
+const SUSPICIOUS_NAME_RE = /TOKEN|SECRET|PASSWORD|API_KEY|^KEY$|_KEY$|DSN/i;
+
+export interface EnvInterpolationContext {
+  /** Absolute path of the file we're resolving — used for error messages. */
+  filePath: string;
+  /** YAML key whose value contains the reference — used for error messages. */
+  key: string;
+  /** Source of variables; defaults to `process.env`. Override in tests. */
+  env?: Record<string, string | undefined>;
+  /** Sink for human-facing warnings. Default: stderr write. */
+  warn?: (msg: string) => void;
+}
+
+/** Module-level set of variable names we've already warned about, so the
+ *  same `${MYAPI_AUTH_TOKEN}` reference doesn't yell once per env file. */
+const warned = new Set<string>();
+
+export function _resetEnvInterpolationWarnings(): void {
+  warned.clear();
+}
+
+/**
+ * Substitute every `${VAR}` / `${VAR:-default}` in `text`. Returns the
+ * fully-resolved string. Throws `Error` when an unresolved reference has
+ * no default.
+ */
+export function interpolateEnvRefs(text: string, ctx: EnvInterpolationContext): string {
+  if (typeof text !== "string" || text.length === 0) return text;
+  if (text.indexOf("${") === -1 && text.indexOf("\\$") === -1) return text;
+  const env = ctx.env ?? (process.env as Record<string, string | undefined>);
+  const warn = ctx.warn ?? ((m: string) => process.stderr.write(m + "\n"));
+
+  return text.replace(ENV_REF_RE, (full, escape: string, name: string, def: string | undefined) => {
+    if (escape === "\\") {
+      // Escaped reference → strip the backslash, keep the literal.
+      return full.slice(1);
+    }
+    const value = env[name];
+    if (value === undefined || value === "") {
+      if (def !== undefined) {
+        return def;
+      }
+      throw new Error(
+        `Environment variable \${${name}} is not set (referenced from "${ctx.filePath}", key "${ctx.key}"). ` +
+        `Provide it via your shell, CI secret, or use the \${${name}:-<default>} form to give it a fallback.`,
+      );
+    }
+    if (SUSPICIOUS_NAME_RE.test(name) && !warned.has(name)) {
+      warned.add(name);
+      warn(
+        `[zond] ${ctx.filePath}: variable \${${name}} looks like a secret. ` +
+        `Consider mapping it through @secret:${ctx.key} (TASK-170) so it is redacted in artifacts.`,
+      );
+    }
+    return value;
+  });
+}
+
+/**
+ * Apply interpolation to every string value in a flat env object. Other
+ * value types (numbers, booleans, nulls coming back from YAML) are
+ * stringified untouched, matching the existing loader behaviour.
+ */
+export function interpolateEnvObject(
+  obj: Record<string, unknown>,
+  filePath: string,
+  env?: Record<string, string | undefined>,
+): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const [k, v] of Object.entries(obj)) {
+    if (typeof v === "string") {
+      out[k] = interpolateEnvRefs(v, { filePath, key: k, env });
+    } else {
+      out[k] = String(v);
+    }
+  }
+  return out;
+}
+
+const SUSPICIOUS_ENV_NAME_RE = SUSPICIOUS_NAME_RE;

package/src/core/parser/filter.ts

@@ -1,4 +1,6 @@
 import type { TestSuite } from "./types.ts";
+import { compileOperationFilter } from "../selectors/operation-filter.ts";
+import type { EndpointInfo } from "../generator/types.ts";
 
 /**
  * Filter suites by tags (OR logic, case-insensitive).
@@ -38,3 +40,58 @@ export function filterSuitesByMethod(suites: TestSuite[], method: string): TestS
   }));
   return filtered.filter(s => s.tests.length > 0);
 }
+
+export interface SuiteFilterResult {
+  suites: TestSuite[];
+  errors: string[];
+}
+
+/**
+ * ARV-25: parity with `zond generate`/`zond checks run` — apply the unified
+ * `--include`/`--exclude` selector grammar (path/method/tag/operation-id)
+ * to a list of test suites. Step-level selectors (path, method, operation-id)
+ * filter steps within a suite; suite-level selectors (tag) borrow each
+ * step's parent suite tags. A suite drops out once it has no steps left.
+ *
+ * Reuses `compileOperationFilter` so semantics match generate/checks 1:1
+ * (multiple --include combine with OR; --exclude evaluated after includes).
+ *
+ * `operation-id` matches against `step.source?.endpoint` ("METHOD /path"),
+ * which is what the generator records; tests authored manually without
+ * `source.endpoint` simply never match operation-id selectors.
+ */
+export function filterSuitesByOperationFilter(
+  suites: TestSuite[],
+  includes: string[],
+  excludes: string[],
+): SuiteFilterResult {
+  if (includes.length === 0 && excludes.length === 0) {
+    return { suites, errors: [] };
+  }
+  const compiled = compileOperationFilter({ includes, excludes });
+  if (compiled.errors.length > 0) {
+    return { suites: [], errors: compiled.errors };
+  }
+  const filtered = suites.map(suite => ({
+    ...suite,
+    tests: suite.tests.filter(step => compiled.filter(stepToEndpoint(suite, step))),
+  }));
+  return { suites: filtered.filter(s => s.tests.length > 0), errors: [] };
+}
+
+function stepToEndpoint(suite: TestSuite, step: TestSuite["tests"][number]): EndpointInfo {
+  const sourceEndpoint = typeof step.source?.endpoint === "string" ? step.source.endpoint : undefined;
+  return {
+    path: step.path,
+    method: step.method,
+    operationId: sourceEndpoint,
+    summary: undefined,
+    tags: suite.tags ?? [],
+    parameters: [],
+    requestBodySchema: undefined,
+    requestBodyContentType: undefined,
+    responseContentTypes: [],
+    responses: [],
+    security: [],
+  };
+}