@appthrust/kest 0.5.0 → 0.7.0

package/README.md

@@ -113,17 +113,14 @@ await ns.assert({
 Custom timeouts are supported per action:
 
 ```ts
-await ns.assert(
-  {
-    apiVersion: "apps/v1",
-    kind: "Deployment",
-    name: "my-app",
-    test() {
-      expect(this.status?.availableReplicas).toBe(3);
-    },
+await ns.assert({
+  apiVersion: "apps/v1",
+  kind: "Deployment",
+  name: "my-app",
+  test() {
+    expect(this.status?.availableReplicas).toBe(3);
   },
-  { timeout: "30s", interval: "1s" },
-);
+});
 ```
 
 ### Create Resources
@@ -252,7 +249,7 @@ await ns.assertOne<ConfigMap>({
   test() {
     expect(this.data?.mode).toBe("auto");
   },
-}, { timeout: "30s", interval: "1s" });
+});
 ```
 
 `assertOne` throws if zero or more than one resource matches, and retries until exactly one is found or the timeout expires.
@@ -269,19 +266,57 @@ await ns.assertAbsence({
 });
 ```
 
-With retry-based polling to wait for a resource to disappear:
+### Error Assertions
+
+Assert that applying or creating a resource produces an error (e.g. an admission webhook rejects the request, or a validation rule fails). The `test` callback inspects the error -- `this` is bound to the `Error`:
 
 ```ts
-await ns.assertAbsence(
-  {
-    apiVersion: "apps/v1",
-    kind: "Deployment",
-    name: "my-app",
+await ns.assertApplyError({
+  apply: {
+    apiVersion: "example.com/v1",
+    kind: "MyResource",
+    metadata: { name: "my-resource" },
+    spec: { immutableField: "changed" },
+  },
+  test() {
+    expect(this.message).toContain("field is immutable");
   },
-  { timeout: "30s", interval: "1s" },
-);
+});
+```
+
+The `test` callback participates in retry -- if it throws, the action is retried until the callback passes or the timeout expires. This is useful when a webhook is being set up asynchronously:
+
+```ts
+await ns.assertApplyError({
+  apply: {
+    apiVersion: "example.com/v1",
+    kind: "MyResource",
+    metadata: { name: "my-resource" },
+    spec: { immutableField: "changed" },
+  },
+  test(error) {
+    expect(error.message).toContain("field is immutable");
+  },
+});
 ```
 
+`assertCreateError` works identically for `kubectl create`:
+
+```ts
+await ns.assertCreateError({
+  create: {
+    apiVersion: "v1",
+    kind: "ConfigMap",
+    metadata: { name: "already-exists" },
+  },
+  test(error) {
+    expect(error.message).toContain("already exists");
+  },
+});
+```
+
+If the apply/create unexpectedly succeeds (e.g. the webhook is not yet active), the resource is immediately reverted and the action retries until the expected error occurs.
+
 ### Label Resources
 
 Add, update, or remove labels on Kubernetes resources using `kubectl label`:
@@ -470,6 +505,8 @@ The top-level API surface available in every test callback.
 | ----------------------------------------------------------------------- | ----------------------------------------------------------- |
 | `apply(manifest, options?)`                                             | Apply a Kubernetes manifest and register cleanup            |
 | `create(manifest, options?)`                                            | Create a Kubernetes resource and register cleanup           |
+| `assertApplyError(input, options?)`                                     | Assert that `kubectl apply` produces an error               |
+| `assertCreateError(input, options?)`                                    | Assert that `kubectl create` produces an error              |
 | `applyStatus(manifest, options?)`                                       | Apply a status subresource (server-side apply)              |
 | `delete(resource, options?)`                                            | Delete a resource by API version, kind, and name            |
 | `label(input, options?)`                                                | Add, update, or remove labels on a resource                 |
@@ -486,7 +523,7 @@ The top-level API surface available in every test callback.
 
 ### Namespace / Cluster
 
-Returned by `newNamespace()` and `useCluster()` respectively. They expose the same core methods (`apply`, `create`, `applyStatus`, `delete`, `label`, `get`, `assert`, `assertAbsence`, `assertList`, `assertOne`) scoped to their namespace or cluster context. `Cluster` additionally supports `newNamespace`.
+Returned by `newNamespace()` and `useCluster()` respectively. They expose the same core methods (`apply`, `create`, `assertApplyError`, `assertCreateError`, `applyStatus`, `delete`, `label`, `get`, `assert`, `assertAbsence`, `assertList`, `assertOne`) scoped to their namespace or cluster context. `Cluster` additionally supports `newNamespace`.
 
 `Namespace` also exposes a `name` property:
 

package/example/example.test.ts

@@ -60,6 +60,34 @@ test("Example: applies ConfigMap using YAML, file import, and object literal", a
   // 4. Namespace
 });
 
+test("Example: diff demo - ConfigMap data mismatch (expected to fail)", async (s) => {
+  s.given("a new namespace exists");
+  const ns = await s.newNamespace();
+
+  s.when("I apply a ConfigMap with actual data");
+  await ns.apply<ConfigMap>({
+    apiVersion: "v1",
+    kind: "ConfigMap",
+    metadata: { name: "diff-demo" },
+    data: { mode: "actual-value", env: "production" },
+  });
+
+  s.then("asserting with different expected data should produce a diff");
+  await ns.assert<ConfigMap>({
+    apiVersion: "v1",
+    kind: "ConfigMap",
+    name: "diff-demo",
+    test() {
+      expect(this).toMatchObject({
+        data: {
+          mode: "expected-value",
+          env: "staging",
+        },
+      });
+    },
+  });
+});
+
 test("Example: asserts a non-existent ConfigMap (expected to fail)", async (s) => {
   s.given("a new namespace exists");
   const ns = await s.newNamespace();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appthrust/kest",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Kubernetes E2E testing framework designed for humans and AI alike",
   "type": "module",
   "module": "ts/index.ts",

package/ts/actions/assert-apply-error.ts

@@ -0,0 +1,43 @@
+import type { AssertApplyErrorInput } from "../apis";
+import { apply } from "./apply";
+import type { MutateDef } from "./types";
+
+export const assertApplyError = {
+  type: "mutate",
+  name: "AssertApplyError",
+  mutate:
+    ({ kubectl }) =>
+    async (input) => {
+      const applyFn = apply.mutate({ kubectl });
+      let result: Awaited<ReturnType<typeof applyFn>> | undefined;
+      let rejection: Error | undefined;
+      try {
+        result = await applyFn(input.apply);
+      } catch (err) {
+        rejection = err as Error;
+      }
+
+      // Apply succeeded unexpectedly -- revert immediately and throw so that
+      // the scenario wrapper retries.
+      if (result !== undefined) {
+        await result.revert();
+        throw new Error(
+          `Expected ${apply.describe(input.apply)} to err, but it succeeded`
+        );
+      }
+
+      // Apply erred as expected -- run test callback.
+      // biome-ignore lint/style/noNonNullAssertion: rejection is guaranteed non-undefined when result is undefined
+      await input.test.call(rejection!, rejection!);
+
+      return {
+        async revert() {
+          // Nothing to clean up -- the resource was never created.
+        },
+        output: undefined,
+      };
+    },
+  describe: (input) => {
+    return `${apply.describe(input.apply)} (expected error)`;
+  },
+} satisfies MutateDef<AssertApplyErrorInput, void>;

package/ts/actions/assert-create-error.ts

@@ -0,0 +1,43 @@
+import type { AssertCreateErrorInput } from "../apis";
+import { create } from "./create";
+import type { MutateDef } from "./types";
+
+export const assertCreateError = {
+  type: "mutate",
+  name: "AssertCreateError",
+  mutate:
+    ({ kubectl }) =>
+    async (input) => {
+      const createFn = create.mutate({ kubectl });
+      let result: Awaited<ReturnType<typeof createFn>> | undefined;
+      let rejection: Error | undefined;
+      try {
+        result = await createFn(input.create);
+      } catch (err) {
+        rejection = err as Error;
+      }
+
+      // Create succeeded unexpectedly -- revert immediately and throw so that
+      // the scenario wrapper retries.
+      if (result !== undefined) {
+        await result.revert();
+        throw new Error(
+          `Expected ${create.describe(input.create)} to err, but it succeeded`
+        );
+      }
+
+      // Create erred as expected -- run test callback.
+      // biome-ignore lint/style/noNonNullAssertion: rejection is guaranteed non-undefined when result is undefined
+      await input.test.call(rejection!, rejection!);
+
+      return {
+        async revert() {
+          // Nothing to clean up -- the resource was never created.
+        },
+        output: undefined,
+      };
+    },
+  describe: (input) => {
+    return `${create.describe(input.create)} (expected error)`;
+  },
+} satisfies MutateDef<AssertCreateErrorInput, void>;

package/ts/actions/create-namespace.ts

@@ -1,6 +1,6 @@
+import { generateName } from "../naming";
 import { create } from "./create";
 import type { MutateDef } from "./types";
-import { generateName } from "../naming";
 
 /**
  * Input for namespace creation.

package/ts/apis/index.ts

@@ -87,6 +87,75 @@ export interface Scenario {
     options?: undefined | ActionOptions
   ): Promise<void>;
 
+  /**
+   * Asserts that `kubectl apply` produces an error.
+   *
+   * The manifest is applied, and the action succeeds when the API server
+   * returns an error (e.g. an admission webhook rejects the request). The
+   * `test` callback must also pass for the action to succeed.
+   *
+   * If the apply unexpectedly succeeds, the created resource is immediately
+   * reverted and the action is retried until the expected error occurs or the
+   * timeout expires.
+   *
+   * @template T - The expected Kubernetes resource shape.
+   * @param input - Manifest to apply and error assertion callback.
+   * @param options - Retry options such as timeout and polling interval.
+   *
+   * @example
+   * ```ts
+   * await s.assertApplyError({
+   *   apply: {
+   *     apiVersion: "example.com/v1",
+   *     kind: "MyResource",
+   *     metadata: { name: "my-resource" },
+   *     spec: { immutableField: "changed" },
+   *   },
+   *   test() {
+   *     expect(this.message).toContain("field is immutable");
+   *   },
+   * });
+   * ```
+   */
+  assertApplyError<T extends K8sResource>(
+    input: AssertApplyErrorInput<T>,
+    options?: undefined | ActionOptions
+  ): Promise<void>;
+
+  /**
+   * Asserts that `kubectl create` produces an error.
+   *
+   * The manifest is created, and the action succeeds when the API server
+   * returns an error. The `test` callback must also pass for the action to
+   * succeed.
+   *
+   * If the create unexpectedly succeeds, the created resource is immediately
+   * reverted and the action is retried until the expected error occurs or the
+   * timeout expires.
+   *
+   * @template T - The expected Kubernetes resource shape.
+   * @param input - Manifest to create and error assertion callback.
+   * @param options - Retry options such as timeout and polling interval.
+   *
+   * @example
+   * ```ts
+   * await s.assertCreateError({
+   *   create: {
+   *     apiVersion: "v1",
+   *     kind: "ConfigMap",
+   *     metadata: { name: "already-exists" },
+   *   },
+   *   test(error) {
+   *     expect(error.message).toContain("already exists");
+   *   },
+   * });
+   * ```
+   */
+  assertCreateError<T extends K8sResource>(
+    input: AssertCreateErrorInput<T>,
+    options?: undefined | ActionOptions
+  ): Promise<void>;
+
   /**
    * Applies the `status` subresource using server-side apply.
    *
@@ -573,6 +642,30 @@ export interface Cluster {
     options?: undefined | ActionOptions
   ): Promise<void>;
 
+  /**
+   * Asserts that `kubectl apply` produces an error.
+   *
+   * @template T - The expected Kubernetes resource shape.
+   * @param input - Manifest to apply and error assertion callback.
+   * @param options - Retry options such as timeout and polling interval.
+   */
+  assertApplyError<T extends K8sResource>(
+    input: AssertApplyErrorInput<T>,
+    options?: undefined | ActionOptions
+  ): Promise<void>;
+
+  /**
+   * Asserts that `kubectl create` produces an error.
+   *
+   * @template T - The expected Kubernetes resource shape.
+   * @param input - Manifest to create and error assertion callback.
+   * @param options - Retry options such as timeout and polling interval.
+   */
+  assertCreateError<T extends K8sResource>(
+    input: AssertCreateErrorInput<T>,
+    options?: undefined | ActionOptions
+  ): Promise<void>;
+
   /**
    * Applies the `status` subresource using server-side apply.
    *
@@ -871,6 +964,30 @@ export interface Namespace {
     options?: undefined | ActionOptions
   ): Promise<void>;
 
+  /**
+   * Asserts that `kubectl apply` produces an error in this namespace.
+   *
+   * @template T - The expected Kubernetes resource shape.
+   * @param input - Manifest to apply and error assertion callback.
+   * @param options - Retry options such as timeout and polling interval.
+   */
+  assertApplyError<T extends K8sResource>(
+    input: AssertApplyErrorInput<T>,
+    options?: undefined | ActionOptions
+  ): Promise<void>;
+
+  /**
+   * Asserts that `kubectl create` produces an error in this namespace.
+   *
+   * @template T - The expected Kubernetes resource shape.
+   * @param input - Manifest to create and error assertion callback.
+   * @param options - Retry options such as timeout and polling interval.
+   */
+  assertCreateError<T extends K8sResource>(
+    input: AssertCreateErrorInput<T>,
+    options?: undefined | ActionOptions
+  ): Promise<void>;
+
   /**
    * Applies the `status` subresource in this namespace using server-side apply.
    *
@@ -1279,6 +1396,56 @@ export interface ResourceOneTest<T extends K8sResource = K8sResource> {
   readonly test: (this: T, resource: T) => unknown | Promise<unknown>;
 }
 
+/**
+ * A test definition for {@link Scenario.assertApplyError},
+ * {@link Cluster.assertApplyError}, and {@link Namespace.assertApplyError}.
+ *
+ * Attempts `kubectl apply` and asserts that the API server returns an error.
+ * When the operation errors as expected, the `test` callback is invoked with
+ * `this` bound to the {@link Error}.
+ */
+export interface AssertApplyErrorInput<T extends K8sResource = K8sResource> {
+  /**
+   * The manifest to apply. Accepts the same formats as {@link Scenario.apply}:
+   * an object literal, a YAML string, or an imported YAML module.
+   */
+  readonly apply: ApplyingManifest<T>;
+
+  /**
+   * Assertion callback invoked when the apply errors as expected.
+   *
+   * `this` is bound to the {@link Error} returned by the API server.
+   * Throwing (or rejecting) signals that the error did not match expectations
+   * and triggers a retry (if timeout allows).
+   */
+  readonly test: (this: Error, error: Error) => unknown | Promise<unknown>;
+}
+
+/**
+ * A test definition for {@link Scenario.assertCreateError},
+ * {@link Cluster.assertCreateError}, and {@link Namespace.assertCreateError}.
+ *
+ * Attempts `kubectl create` and asserts that the API server returns an error.
+ * When the operation errors as expected, the `test` callback is invoked with
+ * `this` bound to the {@link Error}.
+ */
+export interface AssertCreateErrorInput<T extends K8sResource = K8sResource> {
+  /**
+   * The manifest to create. Accepts the same formats as {@link Scenario.create}:
+   * an object literal, a YAML string, or an imported YAML module.
+   */
+  readonly create: ApplyingManifest<T>;
+
+  /**
+   * Assertion callback invoked when the create errors as expected.
+   *
+   * `this` is bound to the {@link Error} returned by the API server.
+   * Throwing (or rejecting) signals that the error did not match expectations
+   * and triggers a retry (if timeout allows).
+   */
+  readonly test: (this: Error, error: Error) => unknown | Promise<unknown>;
+}
+
 /**
  * Kubernetes cluster selector for {@link Scenario.useCluster}.
  */

package/ts/naming/index.ts

@@ -8,7 +8,8 @@ const consonantDigits = "bcdfghjklmnpqrstvwxyz0123456789";
 export function randomConsonantDigits(length = 8): string {
   let result = "";
   for (let i = 0; i < length; i++) {
-    result += consonantDigits[Math.floor(Math.random() * consonantDigits.length)];
+    result +=
+      consonantDigits[Math.floor(Math.random() * consonantDigits.length)];
   }
   return result;
 }
@@ -24,4 +25,3 @@ export function randomConsonantDigits(length = 8): string {
 export function generateName(prefix: string, suffixLength = 5): string {
   return `${prefix}${randomConsonantDigits(suffixLength)}`;
 }
-

package/ts/recording/index.ts

@@ -53,6 +53,7 @@ type CommandEvent =
 
 type RetryEvent =
   | BaseEvent<"RetryStart", Record<string, never>>
+  | BaseEvent<"RetryAttempt", { readonly attempt: number }>
   | BaseEvent<
       "RetryEnd",
       | {

package/ts/reporter/markdown/model.ts

@@ -27,7 +27,7 @@ export interface BDDSection {
 export interface Action {
   name: string;
   attempts?: undefined | number;
-  command?: undefined | Command;
+  commands: Array<Command>;
   error?: undefined | Error;
 }
 

package/ts/reporter/markdown/parser/index.ts

@@ -7,6 +7,7 @@ import type {
   Report,
   Scenario,
 } from "../model";
+import { stripAnsi } from "../strip-ansi";
 
 const bddKeywordByKind = {
   BDDGiven: "given",
@@ -89,6 +90,9 @@ function handleNonBDDEvent(state: ParseState, event: Event): void {
     case "RetryEnd":
       handleRetryEnd(state, event);
       return;
+    case "RetryAttempt":
+      handleRetryAttempt(state);
+      return;
     case "RetryStart":
       return;
     default:
@@ -114,7 +118,7 @@ function handleActionStart(
     return;
   }
 
-  const action: Action = { name: event.data.description };
+  const action: Action = { name: event.data.description, commands: [] };
   scenario.overview.push({
     name: event.data.description,
     status: "pending",
@@ -158,7 +162,9 @@ function applyRegularActionEnd(
     currentAction.error = {
       message: {
         text: event.data.error.message,
-        language: isDiffLike(event.data.error.message) ? "diff" : "text",
+        language: isDiffLike(stripAnsi(event.data.error.message))
+          ? "diff"
+          : "text",
       },
     };
   }
@@ -180,17 +186,21 @@ function handleCommandResult(
   }
 
   const { currentAction } = state;
-  if (!currentAction?.command) {
+  if (!currentAction || currentAction.commands.length === 0) {
     return;
   }
 
-  currentAction.command.stdout = {
+  const command = currentAction.commands.at(-1);
+  if (!command) {
+    return;
+  }
+  command.stdout = {
     text: event.data.stdout,
     ...(event.data.stdoutLanguage
       ? { language: event.data.stdoutLanguage }
       : {}),
   };
-  currentAction.command.stderr = {
+  command.stderr = {
     text: event.data.stderr,
     ...(event.data.stderrLanguage
       ? { language: event.data.stderrLanguage }
@@ -216,7 +226,7 @@ function handleCommandRun(
   if (!state.currentAction) {
     return;
   }
-  state.currentAction.command = createCommandFromRun(event);
+  state.currentAction.commands.push(createCommandFromRun(event));
 }
 
 function handleScenarioStart(
@@ -259,6 +269,16 @@ function handleCleanupActionEnd(
   return true;
 }
 
+function handleRetryAttempt(state: ParseState): void {
+  if (state.inCleanup) {
+    return;
+  }
+  if (!state.currentAction) {
+    return;
+  }
+  state.currentAction.commands = [];
+}
+
 function handleRetryEnd(
   state: ParseState,
   event: Extract<Event, { kind: "RetryEnd" }>
@@ -338,7 +358,7 @@ function bddFromEvent(event: Event): BDDSection | undefined {
   return { keyword, description: event.data.description, actions: [] };
 }
 
-function isDiffLike(message: string): boolean {
+export function isDiffLike(message: string): boolean {
   const lines = message.split(/\r?\n/);
   let sawPlus = false;
   let sawMinus = false;

package/ts/reporter/markdown/renderer/index.ts

@@ -1,6 +1,7 @@
 import { codeToANSIForcedColors } from "../../shiki";
 import type { MarkdownReporterOptions } from "../index";
 import type { Action, Report } from "../model";
+import { stripAnsi } from "../strip-ansi";
 
 const markdownLang = "markdown";
 const markdownTheme = "catppuccin-mocha";
@@ -83,15 +84,6 @@ async function highlightMarkdown(
   }
 }
 
-function stripAnsi(input: string): string {
-  // Prefer Bun's built-in ANSI stripper when available.
-  if (typeof Bun !== "undefined" && typeof Bun.stripANSI === "function") {
-    return Bun.stripANSI(input);
-  }
-  // biome-ignore lint/suspicious/noControlCharactersInRegex: intended
-  return input.replace(/\u001b\[[0-9;]*m/g, "");
-}
-
 function trimFinalNewline(input: string): string {
   return input.replace(/\n$/, "");
 }
@@ -165,7 +157,7 @@ export function renderReport(
       if (!status) {
         if (action.error) {
           status = "failure";
-        } else if (action.command) {
+        } else if (action.commands.length > 0) {
           status = "success";
         } else {
           status = "pending";
@@ -180,8 +172,7 @@ export function renderReport(
       lines.push(`**${emoji} ${stripAnsi(action.name)}**${attemptsSuffix}`);
       lines.push("");
 
-      const cmd = action.command;
-      if (cmd) {
+      for (const cmd of action.commands) {
         const base = [cmd.cmd, ...cmd.args].join(" ").trim();
         const stdin = cmd.stdin?.text;
         const stdinLanguage = cmd.stdin?.language ?? "text";

package/ts/reporter/markdown/strip-ansi.ts

@@ -0,0 +1,8 @@
+export function stripAnsi(input: string): string {
+  // Prefer Bun's built-in ANSI stripper when available.
+  if (typeof Bun !== "undefined" && typeof Bun.stripANSI === "function") {
+    return Bun.stripANSI(input);
+  }
+  // biome-ignore lint/suspicious/noControlCharactersInRegex: intended
+  return input.replace(/\u001b\[[0-9;]*m/g, "");
+}

package/ts/retry.ts

@@ -77,6 +77,7 @@ export async function retryUntil<T>(
     }
 
     retries += 1;
+    recorder?.record("RetryAttempt", { attempt: retries });
 
     try {
       const value = await fn();

package/ts/scenario/index.ts

@@ -2,6 +2,8 @@ import { apply } from "../actions/apply";
 import { applyStatus } from "../actions/apply-status";
 import { assert } from "../actions/assert";
 import { assertAbsence } from "../actions/assert-absence";
+import { assertApplyError } from "../actions/assert-apply-error";
+import { assertCreateError } from "../actions/assert-create-error";
 import { assertList } from "../actions/assert-list";
 import { assertOne } from "../actions/assert-one";
 import { create } from "../actions/create";
@@ -39,6 +41,8 @@ export function createScenario(deps: CreateScenarioOptions): InternalScenario {
   return {
     apply: createMutateFn(deps, apply),
     create: createMutateFn(deps, create),
+    assertApplyError: createMutateFn(deps, assertApplyError),
+    assertCreateError: createMutateFn(deps, assertCreateError),
     applyStatus: createOneWayMutateFn(deps, applyStatus),
     delete: createOneWayMutateFn(deps, deleteResource),
     label: createOneWayMutateFn(deps, label),
@@ -205,6 +209,8 @@ const createNewNamespaceFn =
       name: namespaceName,
       apply: createMutateFn(namespacedDeps, apply),
       create: createMutateFn(namespacedDeps, create),
+      assertApplyError: createMutateFn(namespacedDeps, assertApplyError),
+      assertCreateError: createMutateFn(namespacedDeps, assertCreateError),
       applyStatus: createOneWayMutateFn(namespacedDeps, applyStatus),
       delete: createOneWayMutateFn(namespacedDeps, deleteResource),
       label: createOneWayMutateFn(namespacedDeps, label),
@@ -229,6 +235,8 @@ const createUseClusterFn =
     return {
       apply: createMutateFn(clusterDeps, apply),
       create: createMutateFn(clusterDeps, create),
+      assertApplyError: createMutateFn(clusterDeps, assertApplyError),
+      assertCreateError: createMutateFn(clusterDeps, assertCreateError),
       applyStatus: createOneWayMutateFn(clusterDeps, applyStatus),
       delete: createOneWayMutateFn(clusterDeps, deleteResource),
       label: createOneWayMutateFn(clusterDeps, label),