@homeflare/alchemy 0.48.1 → 0.50.0

package/src/kubernetes/ready-first-install.test.ts

@@ -0,0 +1,126 @@
+/**
+ * The FIRST install of a `HomeFlare.Kubernetes.Ready` row (round 1, findings 2 and 4): Alchemy runs
+ * a deferred adoption `read` before `reconcile`, and `reconcile` connects (the cluster-identity GET)
+ * before its first look. An apiserver restarting under a CNI rollout must not fail either.
+ */
+import { expect, test } from 'bun:test';
+import * as Cause from 'effect/Cause';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import type { FakeObject } from '../talos/fake-apiserver.ts';
+import { readHandler, readReady, reconcileReady } from './ready.ts';
+import {
+  CRD_PATH,
+  DEP_PATH,
+  DS_PATH,
+  HOST,
+  IDENTITY_PATH,
+  checksOf,
+  connection,
+  driven,
+  get,
+  readyCrd,
+  readyDep,
+  readyDs,
+  sequence,
+  withCluster,
+} from './ready.fixtures.ts';
+
+const props = { checks: checksOf(), connection, pollInterval: '5s', waitTimeout: '5m' };
+const allReady = (): Record<string, FakeObject> => ({
+  [get(CRD_PATH)]: { body: readyCrd },
+  [get(DEP_PATH)]: { body: readyDep },
+  [get(DS_PATH)]: { body: readyDs },
+});
+const identity = `GET ${HOST}${IDENTITY_PATH}`;
+const settle = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+  driven(effect, 400).pipe(
+    Effect.map((exit) =>
+      Exit.isSuccess(exit)
+        ? { ok: exit.value }
+        : { error: Cause.squash(exit.cause) as { _tag?: string } },
+    ),
+  );
+
+test('read on a first create (no prior output) adopts nothing and touches no cluster', async () => {
+  // every GET would fail: a read that looked would propagate it
+  const down = { ...allReady(), [identity]: { status: 503 } };
+  const out = await withCluster(down, settle(readHandler({ olds: props, output: undefined })));
+  expect(out).toEqual({ ok: undefined });
+});
+
+test('read of an existing row still looks (the live pass is what diff/refresh rely on)', async () => {
+  const out = await withCluster(
+    allReady(),
+    readHandler({ olds: props, output: { checks: '', connection, ready: false } }),
+  );
+  expect((out as { ready: boolean }).ready).toBe(true);
+});
+
+for (const [name, first] of [
+  ['a 503', { status: 503, body: { message: 'secret-needle' } }],
+  ['a 429', { status: 429 }],
+  ['a hung GET', { hang: true }],
+] as const) {
+  test(`first create: ${name} on the identity GET is tolerated, then the gate opens`, async () => {
+    const objects = sequence(allReady(), identity, [first, first, undefined]);
+    expect(await withCluster(objects, settle(reconcileReady(props)))).toMatchObject({
+      ok: { ready: true },
+    });
+  });
+
+  test(`first create: ${name} right after apply (first check GET) is tolerated`, async () => {
+    const objects = sequence(allReady(), get(DEP_PATH), [first, first, { body: readyDep }]);
+    expect(await withCluster(objects, settle(reconcileReady(props)))).toMatchObject({
+      ok: { ready: true },
+    });
+  });
+}
+
+test('an identity GET that never recovers times out as the gate, naming the transient', async () => {
+  const out = (await withCluster(
+    { ...allReady(), [identity]: { status: 503 } },
+    settle(reconcileReady({ ...props, waitTimeout: '30s' })),
+  )) as { error: { _tag: string; lastTransient?: unknown; failing: string[] } };
+  expect(out.error._tag).toBe('KubernetesReadyTimeout');
+  expect(out.error.lastTransient).toMatchObject({ status: 503, tag: 'KubernetesApiError' });
+  expect(out.error.failing).toHaveLength(3);
+});
+
+test('401/403 on the identity GET propagate at once, scrubbed of the response body', async () => {
+  for (const status of [401, 403]) {
+    const out = (await withCluster(
+      { ...allReady(), [identity]: { body: { message: 'identity-body-needle' }, status } },
+      settle(reconcileReady(props)),
+    )) as { error: { _tag: string; statusCode: number } };
+    expect(out.error).toMatchObject({ _tag: 'KubernetesReadyApiError', statusCode: status });
+    expect(JSON.stringify(out.error) + String(out.error)).not.toContain('identity-body-needle');
+  }
+});
+
+test('a one-pass read/diff connect scrubs the identity GET body too (5xx and 4xx)', async () => {
+  for (const status of [503, 401, 403]) {
+    const out = (await withCluster(
+      { ...allReady(), [identity]: { body: { message: 'identity-body-needle' }, status } },
+      settle(readReady(props)),
+    )) as { error: { _tag: string; statusCode: number } };
+    expect(out.error).toMatchObject({ _tag: 'KubernetesReadyApiError', statusCode: status });
+    expect(JSON.stringify(out.error) + String(out.error)).not.toContain('identity-body-needle');
+  }
+});
+
+test('a vault failure and a uid mismatch still propagate at once (never waited out)', async () => {
+  const down = () => ({ exitCode: 2, stderr: 'permission denied' });
+  const out = (await withCluster(allReady(), settle(reconcileReady(props)), down)) as {
+    error: { _tag?: string };
+  };
+  expect(out.error._tag).not.toBe('KubernetesReadyTimeout');
+  const other = {
+    ...allReady(),
+    [identity]: { body: { metadata: { uid: 'uid-other' } } },
+  };
+  const mismatch = (await withCluster(other, settle(reconcileReady(props)))) as {
+    error: { _tag: string };
+  };
+  expect(mismatch.error._tag).toBe('TalosClusterIdentityMismatch');
+});

package/src/kubernetes/ready-poll-states.test.ts

@@ -0,0 +1,92 @@
+/**
+ * Round 1, findings 3 and 7 through the poll: what the timeout reports, defects that propagate, a
+ * cleared transient that is not blamed, and the strategy failure's typed reason.
+ */
+import { expect, test } from 'bun:test';
+import * as Cause from 'effect/Cause';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import { connectCluster } from 'alchemy/Kubernetes/internal/client';
+import type { FakeObject } from '../talos/fake-apiserver.ts';
+import { poll } from './ready-poll.ts';
+import {
+  CRD_PATH,
+  DEP_PATH,
+  DS_PATH,
+  checksOf,
+  connection,
+  driven,
+  get,
+  readyCrd,
+  readyDep,
+  readyDs,
+  sequence,
+  withCluster,
+} from './ready.fixtures.ts';
+
+const WAIT = Duration.minutes(10);
+const EVERY = Duration.seconds(10);
+const allReady = (): Record<string, FakeObject> => ({
+  [get(CRD_PATH)]: { body: readyCrd },
+  [get(DEP_PATH)]: { body: readyDep },
+  [get(DS_PATH)]: { body: readyDs },
+});
+const run = (objects: Record<string, FakeObject>, checks = checksOf()) =>
+  withCluster(
+    objects,
+    driven(poll(() => connectCluster(connection), checks, WAIT, EVERY)).pipe(
+      Effect.map((exit) => (Exit.isSuccess(exit) ? 'ok' : Cause.squash(exit.cause))),
+    ),
+  ) as Promise<Record<string, unknown> & Error>;
+
+test('the timeout records per key what the last pass saw: notFound, or the counts (no body)', async () => {
+  const objects = allReady();
+  objects[get(CRD_PATH)] = { body: { message: 'state-body-needle' }, status: 404 };
+  const stuck = { ...readyDs, status: { ...readyDs.status, updatedNumberScheduled: 1 } };
+  objects[get(DS_PATH)] = { body: stuck };
+  const error = await run(objects);
+  expect(error['_tag']).toBe('KubernetesReadyTimeout');
+  expect(error['states']).toEqual({
+    'CustomResourceDefinition/x.cilium.io': { notFound: true },
+    'DaemonSet/kube-system/cilium': {
+      counts: {
+        desiredNumberScheduled: 3,
+        generation: 2,
+        numberAvailable: 3,
+        observedGeneration: 2,
+        updatedNumberScheduled: 1,
+      },
+    },
+  });
+  expect(error.message).toContain('x.cilium.io (not found)');
+  expect(error.message).toContain('updatedNumberScheduled=1');
+  expect(JSON.stringify(error) + error.message).not.toContain('state-body-needle');
+});
+
+test('an untagged defect (upstream path-builder throw) propagates instead of waiting ten minutes', async () => {
+  // a namespaced kind with an empty namespace makes upstream's path builder throw a plain Error
+  const bad = [{ kind: 'Deployment', name: 'x', namespace: '' }] as never;
+  const error = await run(allReady(), bad);
+  expect(error).toBeInstanceOf(Error);
+  expect(error['_tag']).toBeUndefined();
+  expect(error.message).toContain('requires a namespace');
+});
+
+test('a transient that has cleared is not blamed by the timeout', async () => {
+  const stuck = { ...readyDs, status: { ...readyDs.status, updatedNumberScheduled: 0 } };
+  // 503 on the first look, then answered (and never ready)
+  const flaky = sequence(allReady(), get(DS_PATH), [{ status: 503 }, { body: stuck }]);
+  const error = await run(flaky);
+  expect(error['_tag']).toBe('KubernetesReadyTimeout');
+  expect(error['lastTransient']).toBeUndefined();
+});
+
+test('OnDelete ends the poll with KubernetesRolloutFailed and its own reason', async () => {
+  const objects = allReady();
+  objects[get(DS_PATH)] = { body: { ...readyDs, spec: { updateStrategy: { type: 'OnDelete' } } } };
+  const error = await run(objects);
+  expect(error['_tag']).toBe('KubernetesRolloutFailed');
+  expect(error['reason']).toBe('UpdateStrategyNotRollingUpdate');
+  expect(error['check']).toBe('DaemonSet/kube-system/cilium');
+});

package/src/kubernetes/ready-poll.test.ts

@@ -0,0 +1,194 @@
+/**
+ * Error classes of the poll and of a single pass, through the real adapter and `readObject` over
+ * the fake apiserver. Time is TestClock's: the 10 m deadline costs no real seconds.
+ */
+import { expect, test } from 'bun:test';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import * as Cause from 'effect/Cause';
+import { connectCluster } from 'alchemy/Kubernetes/internal/client';
+import { type FakeObject } from '../talos/fake-apiserver.ts';
+import { isNotFound, pass, poll, transientOf } from './ready-poll.ts';
+import {
+  CRD_PATH,
+  DEP_PATH,
+  DS_PATH,
+  connection,
+  crdCheck,
+  depCheck,
+  driven,
+  dsCheck,
+  get,
+  readyCrd,
+  readyDep,
+  readyDs,
+  withCluster,
+} from './ready.fixtures.ts';
+
+const WAIT = Duration.minutes(10);
+const EVERY = Duration.seconds(10);
+const checks = [dsCheck, depCheck, crdCheck];
+const allReady = (): Record<string, FakeObject> => ({
+  [get(CRD_PATH)]: { body: readyCrd },
+  [get(DEP_PATH)]: { body: readyDep },
+  [get(DS_PATH)]: { body: readyDs },
+});
+
+/** The failure of a polled run, or `ok`. */
+const pollOutcome = (objects: Record<string, FakeObject>, mutate?: () => void) =>
+  withCluster(
+    objects,
+    driven(
+      Effect.gen(function* () {
+        mutate?.();
+        yield* poll(() => connectCluster(connection), checks, WAIT, EVERY);
+      }),
+    ).pipe(
+      Effect.map((exit) =>
+        Exit.isSuccess(exit)
+          ? { ok: true }
+          : { error: Cause.squash(exit.cause) as { _tag?: string } },
+      ),
+    ),
+  );
+
+test('an all-ready cluster passes on the first look', async () => {
+  const outcome = await pollOutcome(allReady());
+  expect(outcome).toEqual({ ok: true });
+});
+
+test('404 in either shape is pending: the poll times out naming the keys, no transient', async () => {
+  const objects = allReady();
+  objects[get(CRD_PATH)] = { body: { kind: 'Status' }, status: 404 };
+  const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+  expect(outcome.error['_tag']).toBe('KubernetesReadyTimeout');
+  expect(outcome.error['failing']).toEqual(['CustomResourceDefinition/x.cilium.io']);
+  expect(outcome.error['seconds']).toBe(600);
+  expect(outcome.error['lastTransient']).toBeUndefined();
+  // the second shape (a later alchemy raises KubernetesNotFound), matched by tag string alone
+  expect(isNotFound({ _tag: 'KubernetesNotFound' })).toBe(true);
+  expect(isNotFound({ _tag: 'KubernetesApiError', statusCode: 404 })).toBe(true);
+  expect(isNotFound({ _tag: 'KubernetesApiError', statusCode: 403 })).toBe(false);
+  expect(isNotFound(new Error('404'))).toBe(false);
+});
+
+for (const [name, entry, expected] of [
+  ['5xx', { status: 503, body: { secret: 'x' } }, { status: 503, tag: 'KubernetesApiError' }],
+  ['429', { status: 429 }, { status: 429, tag: 'KubernetesApiError' }],
+  ['a per-GET timeout', { hang: true }, { tag: 'KubernetesReadyGetTimeout' }],
+  // ★ upstream retries a transport error every 5 s up to 8 times, about 40 s
+  //   (`client.ts:158-161,166-172`), longer than the 5 s per-GET bound, so through the fake socket it
+  //   surfaces as the GET timeout; the classifier below is what a plain Error from a later upstream
+  //   (no retry) would reach.
+  ['a transport error', { error: 'ECONNREFUSED 10.0.0.1' }, { tag: 'KubernetesReadyGetTimeout' }],
+] as const) {
+  test(`${name} is pending in a poll and named as lastTransient (tag + status, no body)`, async () => {
+    const objects = allReady();
+    objects[get(DEP_PATH)] = entry;
+    const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+    expect(outcome.error['_tag']).toBe('KubernetesReadyTimeout');
+    // ⚠️ after a timed-out GET the rest of the pass is pending unread (socket rationing)
+    const unread = 'tag' in expected && expected.tag === 'KubernetesReadyGetTimeout';
+    expect(outcome.error['failing']).toEqual([
+      'Deployment/kube-system/cilium-operator',
+      ...(unread ? ['CustomResourceDefinition/x.cilium.io'] : []),
+    ]);
+    expect(outcome.error['lastTransient']).toEqual(expected);
+    expect(JSON.stringify(outcome.error)).not.toContain('ECONNREFUSED');
+    expect(JSON.stringify(outcome.error)).not.toContain('secret');
+  });
+}
+
+test('transientOf: tolerable classes only; 4xx, tagged adapter errors and non-errors are not', () => {
+  expect(transientOf(new Error('Failed Kubernetes GET /x: ECONNRESET'))).toEqual({
+    tag: 'TransportError',
+  });
+  expect(transientOf({ _tag: 'KubernetesApiError', statusCode: 502 })).toEqual({
+    status: 502,
+    tag: 'KubernetesApiError',
+  });
+  expect(transientOf({ _tag: 'KubernetesApiError', statusCode: 429 })?.status).toBe(429);
+  for (const error of [
+    { _tag: 'KubernetesApiError', statusCode: 401 },
+    { _tag: 'KubernetesApiError', statusCode: 403 },
+    { _tag: 'KubernetesApiError', statusCode: 404 },
+    { _tag: 'TalosVaultKeyMissing' },
+    // an untagged Error that is not upstream's transport wrapper is a defect (path-builder throw)
+    new Error('Kubernetes object apps/v1/Deployment/x requires a namespace'),
+    new Error('ECONNRESET'),
+    Object.assign(new Error('tagged'), { _tag: 'TalosOpenBaoConnectTimeout' }),
+    'a string',
+  ]) {
+    expect(transientOf(error)).toBeUndefined();
+  }
+});
+
+test('the poll recovers when a transient clears (the cluster answers on a later pass)', async () => {
+  const objects = allReady();
+  objects[get(DEP_PATH)] = { status: 503 };
+  const outcome = await pollOutcome(objects, () => {
+    // flip to ready after the first pass has looked: mutate on the live table
+    setTimeout(() => (objects[get(DEP_PATH)] = { body: readyDep }), 0);
+  });
+  // the timer runs on real time; the poll sleeps virtual seconds, so the next pass sees it ready
+  expect(outcome).toEqual({ ok: true });
+});
+
+for (const status of [401, 403, 400, 409]) {
+  test(`${status} propagates from a poll instead of waiting out the deadline`, async () => {
+    const objects = allReady();
+    objects[get(DS_PATH)] = { status, body: { message: 'nope' } };
+    const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+    expect(outcome.error['_tag']).toBe('KubernetesReadyApiError');
+    expect(outcome.error['statusCode']).toBe(status);
+  });
+}
+
+test('ProgressDeadlineExceeded ends the poll with the typed failure, immediately', async () => {
+  const objects = allReady();
+  objects[get(DEP_PATH)] = {
+    body: {
+      ...readyDep,
+      status: {
+        ...readyDep.status,
+        conditions: [{ reason: 'ProgressDeadlineExceeded', type: 'Progressing' }],
+      },
+    },
+  };
+  const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+  expect(outcome.error['_tag']).toBe('KubernetesRolloutFailed');
+  expect(outcome.error['check']).toBe('Deployment/kube-system/cilium-operator');
+});
+
+test('a single pass tolerates 404 only: 5xx, 429, timeout and transport errors propagate', async () => {
+  const run = (entry: FakeObject) =>
+    withCluster(
+      { ...allReady(), [get(DEP_PATH)]: entry },
+      driven(
+        Effect.gen(function* () {
+          const transport = yield* connectCluster(connection);
+          return yield* pass(transport, checks, false);
+        }),
+        30,
+      ).pipe(
+        Effect.map((exit) =>
+          Exit.isSuccess(exit)
+            ? exit.value
+            : { error: Cause.squash(exit.cause) as { _tag?: string } },
+        ),
+      ),
+    );
+  expect(await run({ status: 404 })).toMatchObject({
+    failed: [],
+    pending: ['Deployment/kube-system/cilium-operator'],
+  });
+  for (const [entry, tag] of [
+    [{ status: 503 }, 'KubernetesReadyApiError'],
+    [{ status: 429 }, 'KubernetesReadyApiError'],
+    [{ hang: true }, 'KubernetesReadyGetTimeout'],
+  ] as const) {
+    expect(((await run(entry)) as { error: { _tag: string } }).error._tag).toBe(tag);
+  }
+  expect(((await run({ error: 'ECONNRESET' })) as { error: unknown }).error).toBeInstanceOf(Error);
+});

package/src/kubernetes/ready-poll.ts

@@ -0,0 +1,215 @@
+/**
+ * The poll loop of `HomeFlare.Kubernetes.Ready`: one bounded GET per check, and the deadline.
+ * Reads go through alchemy's own `readObject` (TLS, CA and client-cert handling stay upstream's),
+ * over the transport `connectCluster` proved against the pinned uid.
+ *
+ * ⛔ Error classes live in `ready-classify.ts` (404 pending; transients tolerated ONLY in a poll;
+ *   everything else propagates). A single pass (`tolerant: false`, used by `read`/`diff`)
+ *   tolerates ONLY 404. The poll covers the CONNECT too (`ready-connect.ts`): the identity GET of
+ *   a cluster whose apiserver is restarting under a CNI rollout is the same transient.
+ * ⚠️ upstream `readObject` takes no signal: a timed-out GET ABANDONS its socket rather than
+ *   aborting it (`client.ts:71-74`); nothing here can close it, and process exit is NOT a bound
+ *   for a long apply. So a timeout is rationed: once one GET of a pass times out the rest of that
+ *   pass is pending without a GET, and the poll doubles its pause per consecutive timeout (cap
+ *   30 s), resetting on a clean pass. A hung apiserver then abandons a few dozen sockets in a 10 m
+ *   wait, not ~120. ★ STANDING WATCH: drop this rationing when upstream's `readObject` takes an
+ *   AbortSignal (alchemy PR 1948, per-attempt deadlines).
+ * ⚠️ upstream also retries a transport error itself, for about 40 s (`Schedule.max([spaced 5 s,
+ *   recurs(8)])`, `client.ts:158-161`): longer than `GET_TIMEOUT`, so a dead socket surfaces here
+ *   as the GET timeout, not as the transport error.
+ */
+import type { ClusterTransport } from 'alchemy/Kubernetes/ClusterAdapter';
+import { readObject } from 'alchemy/Kubernetes/internal/client';
+import * as Data from 'effect/Data';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Ref from 'effect/Ref';
+import { isNotFound, scrubbed, transientOf } from './ready-classify.ts';
+import { type ReadyCheck, checkKey, countsOf, evaluate } from './ready-checks.ts';
+import {
+  type KeyState,
+  KubernetesReadyTimeout,
+  KubernetesRolloutFailed,
+  type LastTransient,
+} from './ready-errors.ts';
+
+export { KubernetesReadyApiError, isNotFound, transientOf } from './ready-classify.ts';
+
+/** Each GET is bounded like `readClusterUid` (`talos/cluster-identity.ts`). */
+export const GET_TIMEOUT = Duration.seconds(5);
+
+/** One GET exceeded `GET_TIMEOUT`. Pending inside a poll, propagated by a single pass. */
+export class KubernetesReadyGetTimeout extends Data.TaggedError('KubernetesReadyGetTimeout')<{
+  readonly check: string;
+  readonly seconds: number;
+}> {
+  override get message(): string {
+    return `Kubernetes.Ready: GET for ${this.check} timed out after ${this.seconds}s`;
+  }
+}
+
+const REFS = {
+  CustomResourceDefinition: {
+    apiVersion: 'apiextensions.k8s.io/v1',
+    kind: 'CustomResourceDefinition',
+  },
+  DaemonSet: { apiVersion: 'apps/v1', kind: 'DaemonSet' },
+  Deployment: { apiVersion: 'apps/v1', kind: 'Deployment' },
+} as const;
+
+const refOf = (check: ReadyCheck) => ({
+  ...REFS[check.kind],
+  name: check.name,
+  ...('namespace' in check ? { namespace: check.namespace } : {}),
+});
+
+/** Longest pause the poll backs off to after consecutive GET timeouts. */
+export const BACKOFF_CAP = Duration.seconds(30);
+
+export interface PassResult {
+  /** Keys of checks that are not ready yet. */
+  readonly pending: readonly string[];
+  /** Keys of checks that are terminally failed (see `KubernetesRolloutFailed`). */
+  readonly failed: readonly string[];
+  /** What this pass saw per key it reached; a key that hit a transient has none. */
+  readonly states: Readonly<Record<string, KeyState>>;
+  readonly lastTransient?: LastTransient;
+  /** A GET timed out (its socket is abandoned): the poll backs off. */
+  readonly timedOut: boolean;
+}
+
+const getOne = (transport: ClusterTransport, check: ReadyCheck) =>
+  readObject({ object: refOf(check), transport }).pipe(
+    Effect.timeoutOrElse({
+      duration: GET_TIMEOUT,
+      orElse: () =>
+        Effect.fail(
+          new KubernetesReadyGetTimeout({
+            check: checkKey(check),
+            seconds: Duration.toSeconds(GET_TIMEOUT),
+          }),
+        ),
+    }),
+  );
+
+/** One look at every check. `tolerant` selects the poll rules; see the header. */
+export const pass = (
+  transport: ClusterTransport,
+  checks: readonly ReadyCheck[],
+  tolerant: boolean,
+): Effect.Effect<PassResult, unknown> =>
+  Effect.gen(function* () {
+    const pending: string[] = [];
+    const failed: string[] = [];
+    const states: Record<string, KeyState> = {};
+    let lastTransient: LastTransient | undefined;
+    let timedOut = false;
+    for (const check of checks) {
+      const key = checkKey(check);
+      // ⛔ After a timeout its socket is abandoned: issue no further GET this pass.
+      if (timedOut) {
+        pending.push(key);
+        continue;
+      }
+      const outcome = yield* getOne(transport, check).pipe(
+        Effect.map((body) => {
+          states[key] = countsOf(check, body);
+          return evaluate(check, body);
+        }),
+        Effect.catch((error: unknown) => {
+          if (isNotFound(error)) {
+            states[key] = { notFound: true };
+            return Effect.succeed('pending' as const);
+          }
+          const transient = tolerant ? transientOf(error) : undefined;
+          if (transient === undefined) return Effect.fail(scrubbed(error));
+          lastTransient = transient;
+          timedOut = timedOut || transient.tag === 'KubernetesReadyGetTimeout';
+          return Effect.succeed('pending' as const);
+        }),
+      );
+      if (outcome === 'pending') pending.push(key);
+      if (outcome === 'failed') failed.push(key);
+    }
+    return {
+      failed,
+      pending,
+      states,
+      timedOut,
+      ...(lastTransient === undefined ? {} : { lastTransient }),
+    };
+  });
+
+/**
+ * Poll until every check is ready. A terminal failure ends it at once; otherwise the deadline
+ * fails with the check keys that were pending on the LAST pass, what that pass saw of each, and
+ * the transient error of the last pass only (⚠️ a transient that cleared is not to be blamed).
+ * `connect` runs inside the deadline (it retries its own transients, `ready-connect.ts`).
+ */
+export const poll = <R>(
+  connect: (
+    note: (transient: LastTransient) => Effect.Effect<void>,
+  ) => Effect.Effect<ClusterTransport, unknown, R>,
+  checks: readonly ReadyCheck[],
+  waitTimeout: Duration.Duration,
+  pollInterval: Duration.Duration,
+) =>
+  Effect.gen(function* () {
+    const pendingNow = checks.map(checkKey);
+    const last = yield* Ref.make<{ pending: readonly string[]; states: Record<string, KeyState> }>({
+      pending: pendingNow,
+      states: {},
+    });
+    const transient = yield* Ref.make<LastTransient | undefined>(undefined);
+    const loop = Effect.gen(function* () {
+      const transport = yield* connect((t) => Ref.set(transient, t));
+      let pause = pollInterval;
+      while (true) {
+        const result = yield* pass(transport, checks, true);
+        yield* Ref.update(last, (prior) => ({
+          pending: result.pending,
+          states: { ...prior.states, ...result.states },
+        }));
+        // ★ Overwritten every pass, so a transient that has since cleared is not blamed.
+        yield* Ref.set(transient, result.lastTransient);
+        const [firstFailed] = result.failed;
+        if (firstFailed !== undefined) {
+          return yield* Effect.fail(
+            new KubernetesRolloutFailed({
+              check: firstFailed,
+              reason: firstFailed.startsWith('DaemonSet/')
+                ? 'UpdateStrategyNotRollingUpdate'
+                : 'ProgressDeadlineExceeded',
+            }),
+          );
+        }
+        if (result.pending.length === 0) return;
+        // ★ Double per consecutive timeout (cap 30 s, or the interval if longer); a clean pass resets.
+        pause = result.timedOut
+          ? Duration.min(Duration.times(pause, 2), Duration.max(BACKOFF_CAP, pollInterval))
+          : pollInterval;
+        yield* Effect.sleep(pause);
+      }
+    });
+    return yield* loop.pipe(
+      Effect.timeoutOrElse({
+        duration: waitTimeout,
+        orElse: () =>
+          Effect.gen(function* () {
+            const { pending, states } = yield* Ref.get(last);
+            const lastTransient = yield* Ref.get(transient);
+            const seen = Object.fromEntries(
+              pending.flatMap((k) => (states[k] ? [[k, states[k]]] : [])),
+            );
+            return yield* Effect.fail(
+              new KubernetesReadyTimeout({
+                failing: pending,
+                seconds: Duration.toSeconds(waitTimeout),
+                states: seen,
+                ...(lastTransient === undefined ? {} : { lastTransient }),
+              }),
+            );
+          }),
+      }),
+    );
+  });

package/src/kubernetes/ready-rollout.test.ts

@@ -0,0 +1,58 @@
+/** Round 1, findings 1 and 7: the lower bound on updated-and-available pods, and the strategy rule. */
+import { expect, test } from 'bun:test';
+import { type ReadyCheck, evaluate } from './ready-checks.ts';
+
+const check = (minReady?: number): ReadyCheck => ({
+  kind: 'DaemonSet',
+  name: 'cilium',
+  namespace: 'kube-system',
+  ...(minReady === undefined ? {} : { minReady }),
+});
+const ds = (status: object, minReady?: number, spec: object = {}) =>
+  evaluate(check(minReady), { metadata: { generation: 2 }, spec, status });
+const status = (desired: number, updated: number, available: number) => ({
+  desiredNumberScheduled: desired,
+  numberAvailable: available,
+  numberReady: available,
+  observedGeneration: 2,
+  updatedNumberScheduled: updated,
+});
+
+// `numberAvailable` counts AVAILABLE OLD-template pods too (kubectl demands updated == desired).
+test('(a) 4 nodes, minReady 2: 2 new pods crashloop, 2 old pods available is pending', () => {
+  expect(ds(status(4, 2, 2), 2)).toBe('pending');
+});
+
+test('(b) 3 nodes: new pod on A works, new pod on B crashloops, C runs the old pod: pending', () => {
+  expect(ds(status(3, 2, 2), 2)).toBe('pending');
+});
+
+test('once the new pods really are available the same shapes are ready', () => {
+  expect(ds(status(4, 2, 4), 2)).toBe('ready'); // 2 updated, 2 old: 4 - 2 = 2 updated-available
+  expect(ds(status(3, 2, 3), 2)).toBe('ready');
+});
+
+test('the bound is a lower bound, not an exact count', () => {
+  // all 3 on the new template, 2 available: 2 - 0 >= 2
+  expect(ds(status(3, 3, 2), 2)).toBe('ready');
+  // 3 on the new template and 1 available is not
+  expect(ds(status(3, 3, 1), 2)).toBe('pending');
+  // updated below the bar fails even when everything is available
+  expect(ds(status(4, 1, 4), 2)).toBe('pending');
+});
+
+test('the default minReady (every node) is unchanged by the bound', () => {
+  expect(ds(status(3, 3, 3))).toBe('ready');
+  expect(ds(status(3, 2, 3))).toBe('pending');
+});
+
+test('a non-RollingUpdate strategy is a terminal failure; absent means RollingUpdate', () => {
+  const rolled = status(3, 3, 3);
+  expect(ds(rolled, undefined, { updateStrategy: { type: 'OnDelete' } })).toBe('failed');
+  expect(ds(rolled, undefined, { updateStrategy: { type: 'RollingUpdate' } })).toBe('ready');
+  expect(ds(rolled, undefined, {})).toBe('ready');
+  // checked before the generation rule, as kubectl does
+  expect(
+    ds({ ...rolled, observedGeneration: 1 }, undefined, { updateStrategy: { type: 'x' } }),
+  ).toBe('failed');
+});