@elench/testkit 0.1.52 → 0.1.54

package/README.md

@@ -47,6 +47,14 @@ npx @elench/testkit --type int --write-status
 # Lifecycle
 npx @elench/testkit status
 npx @elench/testkit destroy
+npx @elench/testkit cleanup
+
+# Inspect the latest run artifact
+npx @elench/testkit show
+npx @elench/testkit show __testkit__/health/health.int.testkit.ts
+npx @elench/testkit artifacts __testkit__/health/health.int.testkit.ts
+npx @elench/testkit logs __testkit__/health/health.int.testkit.ts
+npx @elench/testkit watch
 
 # Known-failures tooling
 npx @elench/testkit known-failures validate --issue-mode error
@@ -56,6 +64,12 @@ npx @elench/testkit known-failures render --output KNOWN_FAILURES.md
 npx @elench/testkit db snapshot capture --service api --output scripts/testkit/schema-baseline.sql
 ```
 
+`testkit` now keeps the default terminal output intentionally short: one line
+per completed file, a concise failure block, and a final summary. Service logs,
+captured runtime output, emitted artifacts, and user-visible LLM responses are
+persisted under `.testkit/results/` and inspected on demand with `show`,
+`artifacts`, `logs`, or `watch`.
+
 ## Setup
 
 Create `testkit.setup.ts` at repo root:

package/bin/testkit.mjs

@@ -1,8 +1,6 @@
 #!/usr/bin/env node
-import { run } from "../lib/cli/index.mjs";
+import { execute } from "@oclif/core";
+import { normalizeCliArgs } from "../lib/cli/entrypoint.mjs";
 
-run().catch((error) => {
-  setImmediate(() => {
-    throw error;
-  });
-});
+const effectiveArgs = normalizeCliArgs(process.argv.slice(2));
+await execute({ args: effectiveArgs, dir: import.meta.url });

package/lib/cli/command-helpers.mjs

@@ -0,0 +1,170 @@
+import path from "path";
+import { Flags } from "@oclif/core";
+import { loadConfigs } from "../config/index.mjs";
+import {
+  parseFileTimeoutOption,
+  parseShardOption,
+  parseSuiteOption,
+  parseTypeOption,
+  parseWorkersOption,
+  resolveRequestedFiles,
+} from "./args.mjs";
+import * as runner from "../runner/index.mjs";
+import { createRunReporter } from "./presentation/run-reporter.mjs";
+
+export const sharedFlags = {
+  dir: Flags.string({
+    description: "Explicit product directory",
+  }),
+  service: Flags.string({
+    description: "Run or inspect only one service",
+  }),
+};
+
+export const runFlags = {
+  ...sharedFlags,
+  type: Flags.string({
+    char: "t",
+    multiple: true,
+    description: "Run specific suite type(s): int, e2e, dal, load, pw, all",
+  }),
+  suite: Flags.string({
+    char: "s",
+    multiple: true,
+    description: "Run specific suite(s)",
+  }),
+  file: Flags.string({
+    char: "f",
+    multiple: true,
+    description: "Run specific file(s)",
+  }),
+  workers: Flags.string({
+    description: "Number of test executors for the whole run",
+  }),
+  "file-timeout-seconds": Flags.string({
+    description: "Per-file wall-clock timeout in seconds",
+  }),
+  shard: Flags.string({
+    description: "Run only shard i of n at suite granularity",
+  }),
+  "write-status": Flags.boolean({
+    description: "Write a deterministic testkit.status.json snapshot",
+    default: false,
+  }),
+  "allow-partial-status": Flags.boolean({
+    description: "Allow --write-status for filtered runs",
+    default: false,
+  }),
+  "ignore-skip-rules": Flags.boolean({
+    description: "Run files even if testkit.setup.ts marks them skipped",
+    default: false,
+  }),
+  "output-mode": Flags.string({
+    description: "Reporter mode",
+    options: ["compact", "debug"],
+  }),
+  debug: Flags.boolean({
+    description: "Alias for --output-mode debug",
+    default: false,
+  }),
+};
+
+export async function resolveConfigsForCommand(flags) {
+  const allConfigs = await loadConfigs({ dir: flags.dir });
+  const configs = flags.service
+    ? allConfigs.filter((config) => config.name === flags.service)
+    : allConfigs;
+  if (flags.service && configs.length === 0) {
+    const available = allConfigs.map((config) => config.name).join(", ");
+    throw new Error(`Service "${flags.service}" not found. Available: ${available}`);
+  }
+  return { allConfigs, configs };
+}
+
+export async function executeRunCommand(command, flags, positionalType = null) {
+  const { allConfigs, configs } = await resolveConfigsForCommand(flags);
+  const workers = flags.workers == null ? null : parseWorkersOption(flags.workers);
+  const fileTimeoutSeconds =
+    flags["file-timeout-seconds"] == null
+      ? null
+      : parseFileTimeoutOption(flags["file-timeout-seconds"]);
+  const shard = parseShardOption(flags.shard);
+  const typeValues = parseTypeOption(flags.type, positionalType);
+  const suiteSelectors = parseSuiteOption(flags.suite);
+  const rawFileNames = Array.isArray(flags.file) ? flags.file : [flags.file].filter(Boolean);
+  const productDir = allConfigs[0]?.productDir || process.cwd();
+  const fileNames = resolveRequestedFiles(rawFileNames, productDir, process.cwd());
+  const outputMode = command.jsonEnabled()
+    ? "json"
+    : flags.debug
+      ? "debug"
+      : flags["output-mode"] || "compact";
+  const reporter = createRunReporter({ outputMode });
+  const result = await runner.runAll(
+    configs,
+    typeValues,
+    suiteSelectors,
+    {
+      ...flags,
+      typeValues,
+      fileNames,
+      workers,
+      fileTimeoutSeconds,
+      shard,
+      serviceFilter: flags.service || null,
+      reporter,
+      writeStatus: flags["write-status"],
+      allowPartialStatus: flags["allow-partial-status"],
+      ignoreSkipRules: flags["ignore-skip-rules"],
+    },
+    allConfigs
+  );
+  return {
+    outputMode,
+    ...result,
+  };
+}
+
+export async function runStatusLike(commandName, flags) {
+  const { allConfigs, configs } = await resolveConfigsForCommand(flags);
+
+  if (commandName === "cleanup") {
+    await runner.cleanup(allConfigs[0]?.productDir || process.cwd());
+    return { ok: true };
+  }
+
+  const productResults = [];
+  for (const config of configs) {
+    if (commandName === "status") {
+      productResults.push(runner.showStatus(config));
+      continue;
+    }
+    await runner.destroy(config);
+    productResults.push({ name: config.name, destroyed: true });
+  }
+
+  return { ok: true, results: productResults };
+}
+
+export function makeKnownFailuresFlags() {
+  return {
+    ...sharedFlags,
+    input: Flags.string({
+      description: "Known failures JSON path (repo-relative)",
+    }),
+    output: Flags.string({
+      description: "Output path",
+    }),
+    status: Flags.string({
+      description: "Status artifact path",
+    }),
+    "issue-mode": Flags.string({
+      description: "Issue validation mode override: off, warn, error",
+      options: ["off", "warn", "error"],
+    }),
+  };
+}
+
+export function relativeToProduct(productDir, targetPath) {
+  return path.relative(productDir, targetPath);
+}

package/lib/cli/commands/artifacts.mjs

@@ -0,0 +1,45 @@
+import { Args, Command } from "@oclif/core";
+import { sharedFlags } from "../command-helpers.mjs";
+import { collectArtifactEntries, loadLatestRunArtifact } from "../viewer.mjs";
+
+export default class ArtifactsCommand extends Command {
+  static summary = "List persisted artifacts from the latest run";
+
+  static enableJsonFlag = true;
+
+  static args = {
+    file: Args.string({
+      description: "Optional file path to filter artifacts",
+      required: false,
+    }),
+  };
+
+  static flags = sharedFlags;
+
+  async run() {
+    const { args, flags } = await this.parse(ArtifactsCommand);
+    const productDir = flags.dir || process.cwd();
+    const runArtifact = loadLatestRunArtifact(productDir);
+    const entries = collectArtifactEntries(productDir, runArtifact, args.file || null, flags.service || null)
+      .map((entry) => ({
+        service: entry.service.name,
+        suite: `${entry.suite.type}:${entry.suite.name}`,
+        file: entry.file.path,
+        name: entry.artifactRef.name,
+        kind: entry.artifactRef.kind,
+        summary: entry.artifactRef.summary,
+        path: entry.artifactRef.path,
+      }));
+
+    if (!this.jsonEnabled()) {
+      for (const entry of entries) {
+        this.log(`${entry.file}`);
+        this.log(`  ${entry.name}${entry.kind ? ` [${entry.kind}]` : ""}`);
+        if (entry.summary) this.log(`    ${entry.summary}`);
+        this.log(`    ${entry.path}`);
+      }
+    }
+
+    return entries;
+  }
+}

package/lib/cli/commands/cleanup.mjs

@@ -0,0 +1,15 @@
+import { Command } from "@oclif/core";
+import { runStatusLike, sharedFlags } from "../command-helpers.mjs";
+
+export default class CleanupCommand extends Command {
+  static summary = "Clean stale local testkit state";
+
+  static enableJsonFlag = true;
+
+  static flags = sharedFlags;
+
+  async run() {
+    const { flags } = await this.parse(CleanupCommand);
+    return runStatusLike("cleanup", flags);
+  }
+}

package/lib/cli/commands/db/snapshot/capture.mjs

@@ -0,0 +1,22 @@
+import { Command, Flags } from "@oclif/core";
+import { sharedFlags } from "../../../command-helpers.mjs";
+import { runDatabaseSnapshotCaptureCommand } from "../../../db.mjs";
+
+export default class DbSnapshotCaptureCommand extends Command {
+  static summary = "Capture a database schema snapshot";
+
+  static enableJsonFlag = true;
+
+  static flags = {
+    ...sharedFlags,
+    output: Flags.string({
+      description: "Output path for the snapshot",
+    }),
+  };
+
+  async run() {
+    const { flags } = await this.parse(DbSnapshotCaptureCommand);
+    await runDatabaseSnapshotCaptureCommand(flags);
+    return { ok: true };
+  }
+}

package/lib/cli/commands/destroy.mjs

@@ -0,0 +1,15 @@
+import { Command } from "@oclif/core";
+import { runStatusLike, sharedFlags } from "../command-helpers.mjs";
+
+export default class DestroyCommand extends Command {
+  static summary = "Destroy local testkit state";
+
+  static enableJsonFlag = true;
+
+  static flags = sharedFlags;
+
+  async run() {
+    const { flags } = await this.parse(DestroyCommand);
+    return runStatusLike("destroy", flags);
+  }
+}

package/lib/cli/commands/known-failures/render.mjs

@@ -0,0 +1,19 @@
+import { Command } from "@oclif/core";
+import { makeKnownFailuresFlags } from "../../command-helpers.mjs";
+import { runKnownFailuresRenderCommand } from "../../known-failures.mjs";
+
+export default class KnownFailuresRenderCommand extends Command {
+  static summary = "Render known failures markdown";
+
+  static enableJsonFlag = true;
+
+  static flags = makeKnownFailuresFlags();
+
+  async run() {
+    const { flags } = await this.parse(KnownFailuresRenderCommand);
+    await runKnownFailuresRenderCommand({
+      ...flags,
+    });
+    return { ok: true };
+  }
+}

package/lib/cli/commands/known-failures/validate.mjs

@@ -0,0 +1,20 @@
+import { Command } from "@oclif/core";
+import { makeKnownFailuresFlags } from "../../command-helpers.mjs";
+import { runKnownFailuresValidateCommand } from "../../known-failures.mjs";
+
+export default class KnownFailuresValidateCommand extends Command {
+  static summary = "Validate known failures against the latest status artifact";
+
+  static enableJsonFlag = true;
+
+  static flags = makeKnownFailuresFlags();
+
+  async run() {
+    const { flags } = await this.parse(KnownFailuresValidateCommand);
+    await runKnownFailuresValidateCommand({
+      ...flags,
+      issueMode: flags["issue-mode"] || null,
+    });
+    return { ok: true };
+  }
+}

package/lib/cli/commands/logs.mjs

@@ -0,0 +1,47 @@
+import { Args, Command, Flags } from "@oclif/core";
+import { sharedFlags } from "../command-helpers.mjs";
+import { readLogTail } from "../../runner/logs.mjs";
+import { getServiceLogRefs, loadLatestRunArtifact, resolveFileSubject } from "../viewer.mjs";
+import path from "path";
+
+export default class LogsCommand extends Command {
+  static summary = "Show backend log tails relevant to one file from the latest run";
+
+  static enableJsonFlag = true;
+
+  static args = {
+    file: Args.string({
+      description: "Optional file path; defaults to the first failed file",
+      required: false,
+    }),
+  };
+
+  static flags = {
+    ...sharedFlags,
+    tail: Flags.integer({
+      description: "Number of lines to show from each log",
+      default: 40,
+    }),
+  };
+
+  async run() {
+    const { args, flags } = await this.parse(LogsCommand);
+    const productDir = flags.dir || process.cwd();
+    const runArtifact = loadLatestRunArtifact(productDir);
+    const subject = resolveFileSubject(runArtifact, args.file || null, flags.service || null);
+    const logs = getServiceLogRefs(runArtifact, subject.service.name).map((entry) => ({
+      ...entry,
+      lines: readLogTail(path.join(productDir, entry.path), flags.tail),
+    }));
+
+    if (!this.jsonEnabled()) {
+      for (const entry of logs) {
+        this.log(`${entry.runtimeLabel}`);
+        this.log(`  ${entry.path}`);
+        for (const line of entry.lines) this.log(`  ${line}`);
+      }
+    }
+
+    return logs;
+  }
+}

package/lib/cli/commands/run.mjs

@@ -0,0 +1,23 @@
+import { Args, Command } from "@oclif/core";
+import { executeRunCommand, runFlags } from "../command-helpers.mjs";
+
+export default class RunCommand extends Command {
+  static summary = "Run test suites";
+
+  static enableJsonFlag = true;
+
+  static args = {
+    type: Args.string({
+      description: "Optional suite type shortcut: int, e2e, dal, load, pw, all",
+      required: false,
+      options: ["int", "e2e", "dal", "load", "pw", "all"],
+    }),
+  };
+
+  static flags = runFlags;
+
+  async run() {
+    const { args, flags } = await this.parse(RunCommand);
+    return executeRunCommand(this, flags, args.type || null);
+  }
+}

package/lib/cli/commands/show.mjs

@@ -0,0 +1,47 @@
+import { Args, Command, Flags } from "@oclif/core";
+import { sharedFlags } from "../command-helpers.mjs";
+import { formatFileDetail, loadLatestRunArtifact, resolveFileSubject } from "../viewer.mjs";
+
+export default class ShowCommand extends Command {
+  static summary = "Show the most useful details for one file from the latest run";
+
+  static enableJsonFlag = true;
+
+  static args = {
+    file: Args.string({
+      description: "File path to inspect; defaults to the first failed file",
+      required: false,
+    }),
+  };
+
+  static flags = {
+    ...sharedFlags,
+    "log-tail": Flags.integer({
+      description: "Number of backend log lines to include",
+      default: 12,
+    }),
+  };
+
+  async run() {
+    const { args, flags } = await this.parse(ShowCommand);
+    const productDir = flags.dir || process.cwd();
+    const runArtifact = loadLatestRunArtifact(productDir);
+    const subject = resolveFileSubject(runArtifact, args.file || null, flags.service || null);
+    const result = {
+      file: subject.file,
+      suite: {
+        name: subject.suite.name,
+        type: subject.suite.type,
+      },
+      service: {
+        name: subject.service.name,
+      },
+    };
+    if (!this.jsonEnabled()) {
+      for (const line of formatFileDetail(productDir, runArtifact, subject, { logTail: flags["log-tail"] })) {
+        this.log(line);
+      }
+    }
+    return result;
+  }
+}

package/lib/cli/commands/status.mjs

@@ -0,0 +1,15 @@
+import { Command } from "@oclif/core";
+import { runStatusLike, sharedFlags } from "../command-helpers.mjs";
+
+export default class StatusCommand extends Command {
+  static summary = "Show local testkit state";
+
+  static enableJsonFlag = true;
+
+  static flags = sharedFlags;
+
+  async run() {
+    const { flags } = await this.parse(StatusCommand);
+    return runStatusLike("status", flags);
+  }
+}

package/lib/cli/commands/watch.mjs

@@ -0,0 +1,23 @@
+import React, { createElement } from "react";
+import { Command } from "@oclif/core";
+import { render } from "ink";
+import { sharedFlags } from "../command-helpers.mjs";
+import { WatchApp } from "../tui/watch-app.mjs";
+
+export default class WatchCommand extends Command {
+  static summary = "Open an interactive viewer for the latest run artifact";
+
+  static flags = sharedFlags;
+
+  async run() {
+    const { flags } = await this.parse(WatchCommand);
+    const productDir = flags.dir || process.cwd();
+    const app = render(
+      createElement(WatchApp, {
+        productDir,
+        serviceFilter: flags.service || null,
+      })
+    );
+    await app.waitUntilExit();
+  }
+}

package/lib/cli/entrypoint.mjs

@@ -0,0 +1,83 @@
+export function normalizeCliArgs(argv) {
+  const topLevelCommands = new Set([
+    "run",
+    "status",
+    "destroy",
+    "cleanup",
+    "show",
+    "logs",
+    "artifacts",
+    "watch",
+    "known-failures",
+    "db",
+    "help",
+    "--help",
+    "-h",
+    "--version",
+    "-v",
+  ]);
+  const runTypeShortcuts = new Set(["int", "e2e", "dal", "load", "pw", "all"]);
+  const valueFlags = new Set([
+    "--dir",
+    "--service",
+    "--type",
+    "--suite",
+    "--file",
+    "--workers",
+    "--file-timeout-seconds",
+    "--shard",
+    "--input",
+    "--output",
+    "--status",
+    "--issue-mode",
+    "--output-mode",
+    "--tail",
+    "--log-tail",
+  ]);
+  const positionals = findPositionals(argv, valueFlags);
+  const firstPositional = positionals[0] || null;
+  const shouldPrefixRun =
+    !firstPositional ||
+    runTypeShortcuts.has(firstPositional.value) ||
+    !topLevelCommands.has(firstPositional.value);
+
+  if (shouldPrefixRun) {
+    return ["run", ...argv];
+  }
+
+  if (topLevelCommands.has(firstPositional.value) && argv[0] !== firstPositional.value) {
+    return reorderCommandArgs(argv, positionals);
+  }
+
+  return argv;
+}
+
+function findPositionals(args, flagsWithValues) {
+  const positionals = [];
+  for (let index = 0; index < args.length; index += 1) {
+    const value = args[index];
+    if (!value.startsWith("-")) {
+      positionals.push({ index, value });
+      continue;
+    }
+    if (flagsWithValues.has(value)) {
+      index += 1;
+    }
+  }
+  return positionals;
+}
+
+function reorderCommandArgs(args, positionals) {
+  const commandTokens = [positionals[0]];
+  if (positionals[0]?.value === "known-failures" && positionals[1]) {
+    commandTokens.push(positionals[1]);
+  }
+  if (positionals[0]?.value === "db" && positionals[1] && positionals[2]) {
+    commandTokens.push(positionals[1], positionals[2]);
+  }
+  const commandIndexes = new Set(commandTokens.map((token) => token.index));
+  return [
+    ...commandTokens.map((token) => token.value),
+    ...args.filter((_value, index) => !commandIndexes.has(index)),
+  ];
+}