@homeflare/alchemy 0.49.0 → 0.51.0

package/src/kubernetes/ready-errors.ts

@@ -0,0 +1,102 @@
+/**
+ * The typed errors of `HomeFlare.Kubernetes.Ready` that carry no logic, so a stack can `catchTag`
+ * them. ⛔ Every message names check keys, counts, durations and statuses only: never a response
+ * body, which is the server's text and not ours to republish.
+ */
+import * as Data from 'effect/Data';
+
+/** What the poll recorded when a pass last tolerated a transient error: tag and status, no body. */
+export interface LastTransient {
+  readonly tag: string;
+  readonly status?: number;
+}
+
+/**
+ * What the last pass saw of one check: the object was absent (404), or the numbers it reported.
+ * ★ A 404 that never resolves (a typo'd name, a chart that never creates it) must read differently
+ *   from a slow rollout, without republishing anything the server said (round 1, finding 3).
+ */
+export type KeyState =
+  | { readonly notFound: true }
+  | { readonly counts: Readonly<Record<string, number>> };
+
+const describeState = (state: KeyState | undefined): string => {
+  if (state === undefined) return '';
+  if ('notFound' in state) return ' (not found)';
+  const counts = Object.entries(state.counts).map(([name, value]) => `${name}=${value}`);
+  return counts.length === 0 ? '' : ` (${counts.join(' ')})`;
+};
+
+/** The gate never opened within `waitTimeout`. `failing` holds check keys only. */
+export class KubernetesReadyTimeout extends Data.TaggedError('KubernetesReadyTimeout')<{
+  readonly failing: readonly string[];
+  readonly seconds: number;
+  readonly lastTransient?: LastTransient;
+  /** Per failing key, from the last pass that reached it: `notFound`, or the counts seen. */
+  readonly states?: Readonly<Record<string, KeyState>>;
+}> {
+  override get message(): string {
+    const failing = this.failing.map((key) => `${key}${describeState(this.states?.[key])}`);
+    const last =
+      this.lastTransient === undefined
+        ? ''
+        : `; last transient error ${this.lastTransient.tag}${
+            this.lastTransient.status === undefined ? '' : ` ${this.lastTransient.status}`
+          }`;
+    return `Kubernetes.Ready: not ready within ${this.seconds}s: ${failing.join(', ')}${last}`;
+  }
+}
+
+/**
+ * A rollout that will not finish, which is not pending: a Deployment reported
+ * `ProgressDeadlineExceeded`, or a DaemonSet's `updateStrategy` is not `RollingUpdate` (kubectl
+ * `rollout_status.go:104-106` errors the same way: `OnDelete` pods never roll by themselves).
+ */
+export class KubernetesRolloutFailed extends Data.TaggedError('KubernetesRolloutFailed')<{
+  readonly check: string;
+  readonly reason?: 'ProgressDeadlineExceeded' | 'UpdateStrategyNotRollingUpdate';
+}> {
+  override get message(): string {
+    return this.reason === 'UpdateStrategyNotRollingUpdate'
+      ? `Kubernetes.Ready: ${this.check} does not use the RollingUpdate update strategy`
+      : `Kubernetes.Ready: ${this.check} exceeded its progress deadline (ProgressDeadlineExceeded)`;
+  }
+}
+
+/** A `waitTimeout`/`pollInterval` that is not a Go-style duration (`10m0s`, `90s`, `1h30m`). */
+export class KubernetesReadyBadDuration extends Data.TaggedError('KubernetesReadyBadDuration')<{
+  readonly field: string;
+  readonly value: string;
+  /** The resource whose prop it is; `Kubernetes.Ready` unless the HelmChart bound says otherwise. */
+  readonly resource?: string;
+}> {
+  override get message(): string {
+    // A hostile or typo'd value can be long: the message quotes a bounded prefix only.
+    const shown = this.value.length > 40 ? `${this.value.slice(0, 40)}...` : this.value;
+    return `${this.resource ?? 'Kubernetes.Ready'}: ${this.field} '${shown}' is not a duration like 10m0s`;
+  }
+}
+
+/** A check that can never be satisfied, refused at declaration instead of timing out in 10 m. */
+export class KubernetesReadyBadCheck extends Data.TaggedError('KubernetesReadyBadCheck')<{
+  /** Index into `checks`. */
+  readonly index: number;
+  readonly problem: string;
+}> {
+  override get message(): string {
+    return `Kubernetes.Ready: checks[${this.index}] ${this.problem}`;
+  }
+}
+
+/**
+ * `after` must hold Outputs of the chart (`chart.objects`). ⛔ A plain value, or a stable such as
+ * `chart.connection`, resolves at plan time, so the row would plan `noop` before the chart's
+ * update and the gate would be skipped.
+ */
+export class KubernetesReadyBadAfter extends Data.TaggedError('KubernetesReadyBadAfter')<{
+  readonly index: number;
+}> {
+  override get message(): string {
+    return `Kubernetes.Ready: after[${this.index}] is not chart.objects (the only shape that waits for the chart's update)`;
+  }
+}

package/src/kubernetes/ready-first-install.test.ts

@@ -0,0 +1,126 @@
+/**
+ * The FIRST install of a `HomeFlare.Kubernetes.Ready` row (round 1, findings 2 and 4): Alchemy runs
+ * a deferred adoption `read` before `reconcile`, and `reconcile` connects (the cluster-identity GET)
+ * before its first look. An apiserver restarting under a CNI rollout must not fail either.
+ */
+import { expect, test } from 'bun:test';
+import * as Cause from 'effect/Cause';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import type { FakeObject } from '../talos/fake-apiserver.ts';
+import { readHandler, readReady, reconcileReady } from './ready.ts';
+import {
+  CRD_PATH,
+  DEP_PATH,
+  DS_PATH,
+  HOST,
+  IDENTITY_PATH,
+  checksOf,
+  connection,
+  driven,
+  get,
+  readyCrd,
+  readyDep,
+  readyDs,
+  sequence,
+  withCluster,
+} from './ready.fixtures.ts';
+
+const props = { checks: checksOf(), connection, pollInterval: '5s', waitTimeout: '5m' };
+const allReady = (): Record<string, FakeObject> => ({
+  [get(CRD_PATH)]: { body: readyCrd },
+  [get(DEP_PATH)]: { body: readyDep },
+  [get(DS_PATH)]: { body: readyDs },
+});
+const identity = `GET ${HOST}${IDENTITY_PATH}`;
+const settle = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+  driven(effect, 400).pipe(
+    Effect.map((exit) =>
+      Exit.isSuccess(exit)
+        ? { ok: exit.value }
+        : { error: Cause.squash(exit.cause) as { _tag?: string } },
+    ),
+  );
+
+test('read on a first create (no prior output) adopts nothing and touches no cluster', async () => {
+  // every GET would fail: a read that looked would propagate it
+  const down = { ...allReady(), [identity]: { status: 503 } };
+  const out = await withCluster(down, settle(readHandler({ olds: props, output: undefined })));
+  expect(out).toEqual({ ok: undefined });
+});
+
+test('read of an existing row still looks (the live pass is what diff/refresh rely on)', async () => {
+  const out = await withCluster(
+    allReady(),
+    readHandler({ olds: props, output: { checks: '', connection, ready: false } }),
+  );
+  expect((out as { ready: boolean }).ready).toBe(true);
+});
+
+for (const [name, first] of [
+  ['a 503', { status: 503, body: { message: 'secret-needle' } }],
+  ['a 429', { status: 429 }],
+  ['a hung GET', { hang: true }],
+] as const) {
+  test(`first create: ${name} on the identity GET is tolerated, then the gate opens`, async () => {
+    const objects = sequence(allReady(), identity, [first, first, undefined]);
+    expect(await withCluster(objects, settle(reconcileReady(props)))).toMatchObject({
+      ok: { ready: true },
+    });
+  });
+
+  test(`first create: ${name} right after apply (first check GET) is tolerated`, async () => {
+    const objects = sequence(allReady(), get(DEP_PATH), [first, first, { body: readyDep }]);
+    expect(await withCluster(objects, settle(reconcileReady(props)))).toMatchObject({
+      ok: { ready: true },
+    });
+  });
+}
+
+test('an identity GET that never recovers times out as the gate, naming the transient', async () => {
+  const out = (await withCluster(
+    { ...allReady(), [identity]: { status: 503 } },
+    settle(reconcileReady({ ...props, waitTimeout: '30s' })),
+  )) as { error: { _tag: string; lastTransient?: unknown; failing: string[] } };
+  expect(out.error._tag).toBe('KubernetesReadyTimeout');
+  expect(out.error.lastTransient).toMatchObject({ status: 503, tag: 'KubernetesApiError' });
+  expect(out.error.failing).toHaveLength(3);
+});
+
+test('401/403 on the identity GET propagate at once, scrubbed of the response body', async () => {
+  for (const status of [401, 403]) {
+    const out = (await withCluster(
+      { ...allReady(), [identity]: { body: { message: 'identity-body-needle' }, status } },
+      settle(reconcileReady(props)),
+    )) as { error: { _tag: string; statusCode: number } };
+    expect(out.error).toMatchObject({ _tag: 'KubernetesReadyApiError', statusCode: status });
+    expect(JSON.stringify(out.error) + String(out.error)).not.toContain('identity-body-needle');
+  }
+});
+
+test('a one-pass read/diff connect scrubs the identity GET body too (5xx and 4xx)', async () => {
+  for (const status of [503, 401, 403]) {
+    const out = (await withCluster(
+      { ...allReady(), [identity]: { body: { message: 'identity-body-needle' }, status } },
+      settle(readReady(props)),
+    )) as { error: { _tag: string; statusCode: number } };
+    expect(out.error).toMatchObject({ _tag: 'KubernetesReadyApiError', statusCode: status });
+    expect(JSON.stringify(out.error) + String(out.error)).not.toContain('identity-body-needle');
+  }
+});
+
+test('a vault failure and a uid mismatch still propagate at once (never waited out)', async () => {
+  const down = () => ({ exitCode: 2, stderr: 'permission denied' });
+  const out = (await withCluster(allReady(), settle(reconcileReady(props)), down)) as {
+    error: { _tag?: string };
+  };
+  expect(out.error._tag).not.toBe('KubernetesReadyTimeout');
+  const other = {
+    ...allReady(),
+    [identity]: { body: { metadata: { uid: 'uid-other' } } },
+  };
+  const mismatch = (await withCluster(other, settle(reconcileReady(props)))) as {
+    error: { _tag: string };
+  };
+  expect(mismatch.error._tag).toBe('TalosClusterIdentityMismatch');
+});

package/src/kubernetes/ready-poll-states.test.ts

@@ -0,0 +1,92 @@
+/**
+ * Round 1, findings 3 and 7 through the poll: what the timeout reports, defects that propagate, a
+ * cleared transient that is not blamed, and the strategy failure's typed reason.
+ */
+import { expect, test } from 'bun:test';
+import * as Cause from 'effect/Cause';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import { connectCluster } from 'alchemy/Kubernetes/internal/client';
+import type { FakeObject } from '../talos/fake-apiserver.ts';
+import { poll } from './ready-poll.ts';
+import {
+  CRD_PATH,
+  DEP_PATH,
+  DS_PATH,
+  checksOf,
+  connection,
+  driven,
+  get,
+  readyCrd,
+  readyDep,
+  readyDs,
+  sequence,
+  withCluster,
+} from './ready.fixtures.ts';
+
+const WAIT = Duration.minutes(10);
+const EVERY = Duration.seconds(10);
+const allReady = (): Record<string, FakeObject> => ({
+  [get(CRD_PATH)]: { body: readyCrd },
+  [get(DEP_PATH)]: { body: readyDep },
+  [get(DS_PATH)]: { body: readyDs },
+});
+const run = (objects: Record<string, FakeObject>, checks = checksOf()) =>
+  withCluster(
+    objects,
+    driven(poll(() => connectCluster(connection), checks, WAIT, EVERY)).pipe(
+      Effect.map((exit) => (Exit.isSuccess(exit) ? 'ok' : Cause.squash(exit.cause))),
+    ),
+  ) as Promise<Record<string, unknown> & Error>;
+
+test('the timeout records per key what the last pass saw: notFound, or the counts (no body)', async () => {
+  const objects = allReady();
+  objects[get(CRD_PATH)] = { body: { message: 'state-body-needle' }, status: 404 };
+  const stuck = { ...readyDs, status: { ...readyDs.status, updatedNumberScheduled: 1 } };
+  objects[get(DS_PATH)] = { body: stuck };
+  const error = await run(objects);
+  expect(error['_tag']).toBe('KubernetesReadyTimeout');
+  expect(error['states']).toEqual({
+    'CustomResourceDefinition/x.cilium.io': { notFound: true },
+    'DaemonSet/kube-system/cilium': {
+      counts: {
+        desiredNumberScheduled: 3,
+        generation: 2,
+        numberAvailable: 3,
+        observedGeneration: 2,
+        updatedNumberScheduled: 1,
+      },
+    },
+  });
+  expect(error.message).toContain('x.cilium.io (not found)');
+  expect(error.message).toContain('updatedNumberScheduled=1');
+  expect(JSON.stringify(error) + error.message).not.toContain('state-body-needle');
+});
+
+test('an untagged defect (upstream path-builder throw) propagates instead of waiting ten minutes', async () => {
+  // a namespaced kind with an empty namespace makes upstream's path builder throw a plain Error
+  const bad = [{ kind: 'Deployment', name: 'x', namespace: '' }] as never;
+  const error = await run(allReady(), bad);
+  expect(error).toBeInstanceOf(Error);
+  expect(error['_tag']).toBeUndefined();
+  expect(error.message).toContain('requires a namespace');
+});
+
+test('a transient that has cleared is not blamed by the timeout', async () => {
+  const stuck = { ...readyDs, status: { ...readyDs.status, updatedNumberScheduled: 0 } };
+  // 503 on the first look, then answered (and never ready)
+  const flaky = sequence(allReady(), get(DS_PATH), [{ status: 503 }, { body: stuck }]);
+  const error = await run(flaky);
+  expect(error['_tag']).toBe('KubernetesReadyTimeout');
+  expect(error['lastTransient']).toBeUndefined();
+});
+
+test('OnDelete ends the poll with KubernetesRolloutFailed and its own reason', async () => {
+  const objects = allReady();
+  objects[get(DS_PATH)] = { body: { ...readyDs, spec: { updateStrategy: { type: 'OnDelete' } } } };
+  const error = await run(objects);
+  expect(error['_tag']).toBe('KubernetesRolloutFailed');
+  expect(error['reason']).toBe('UpdateStrategyNotRollingUpdate');
+  expect(error['check']).toBe('DaemonSet/kube-system/cilium');
+});

package/src/kubernetes/ready-poll.test.ts

@@ -0,0 +1,194 @@
+/**
+ * Error classes of the poll and of a single pass, through the real adapter and `readObject` over
+ * the fake apiserver. Time is TestClock's: the 10 m deadline costs no real seconds.
+ */
+import { expect, test } from 'bun:test';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import * as Cause from 'effect/Cause';
+import { connectCluster } from 'alchemy/Kubernetes/internal/client';
+import { type FakeObject } from '../talos/fake-apiserver.ts';
+import { isNotFound, pass, poll, transientOf } from './ready-poll.ts';
+import {
+  CRD_PATH,
+  DEP_PATH,
+  DS_PATH,
+  connection,
+  crdCheck,
+  depCheck,
+  driven,
+  dsCheck,
+  get,
+  readyCrd,
+  readyDep,
+  readyDs,
+  withCluster,
+} from './ready.fixtures.ts';
+
+const WAIT = Duration.minutes(10);
+const EVERY = Duration.seconds(10);
+const checks = [dsCheck, depCheck, crdCheck];
+const allReady = (): Record<string, FakeObject> => ({
+  [get(CRD_PATH)]: { body: readyCrd },
+  [get(DEP_PATH)]: { body: readyDep },
+  [get(DS_PATH)]: { body: readyDs },
+});
+
+/** The failure of a polled run, or `ok`. */
+const pollOutcome = (objects: Record<string, FakeObject>, mutate?: () => void) =>
+  withCluster(
+    objects,
+    driven(
+      Effect.gen(function* () {
+        mutate?.();
+        yield* poll(() => connectCluster(connection), checks, WAIT, EVERY);
+      }),
+    ).pipe(
+      Effect.map((exit) =>
+        Exit.isSuccess(exit)
+          ? { ok: true }
+          : { error: Cause.squash(exit.cause) as { _tag?: string } },
+      ),
+    ),
+  );
+
+test('an all-ready cluster passes on the first look', async () => {
+  const outcome = await pollOutcome(allReady());
+  expect(outcome).toEqual({ ok: true });
+});
+
+test('404 in either shape is pending: the poll times out naming the keys, no transient', async () => {
+  const objects = allReady();
+  objects[get(CRD_PATH)] = { body: { kind: 'Status' }, status: 404 };
+  const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+  expect(outcome.error['_tag']).toBe('KubernetesReadyTimeout');
+  expect(outcome.error['failing']).toEqual(['CustomResourceDefinition/x.cilium.io']);
+  expect(outcome.error['seconds']).toBe(600);
+  expect(outcome.error['lastTransient']).toBeUndefined();
+  // the second shape (a later alchemy raises KubernetesNotFound), matched by tag string alone
+  expect(isNotFound({ _tag: 'KubernetesNotFound' })).toBe(true);
+  expect(isNotFound({ _tag: 'KubernetesApiError', statusCode: 404 })).toBe(true);
+  expect(isNotFound({ _tag: 'KubernetesApiError', statusCode: 403 })).toBe(false);
+  expect(isNotFound(new Error('404'))).toBe(false);
+});
+
+for (const [name, entry, expected] of [
+  ['5xx', { status: 503, body: { secret: 'x' } }, { status: 503, tag: 'KubernetesApiError' }],
+  ['429', { status: 429 }, { status: 429, tag: 'KubernetesApiError' }],
+  ['a per-GET timeout', { hang: true }, { tag: 'KubernetesReadyGetTimeout' }],
+  // ★ upstream retries a transport error every 5 s up to 8 times, about 40 s
+  //   (`client.ts:158-161,166-172`), longer than the 5 s per-GET bound, so through the fake socket it
+  //   surfaces as the GET timeout; the classifier below is what a plain Error from a later upstream
+  //   (no retry) would reach.
+  ['a transport error', { error: 'ECONNREFUSED 10.0.0.1' }, { tag: 'KubernetesReadyGetTimeout' }],
+] as const) {
+  test(`${name} is pending in a poll and named as lastTransient (tag + status, no body)`, async () => {
+    const objects = allReady();
+    objects[get(DEP_PATH)] = entry;
+    const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+    expect(outcome.error['_tag']).toBe('KubernetesReadyTimeout');
+    // ⚠️ after a timed-out GET the rest of the pass is pending unread (socket rationing)
+    const unread = 'tag' in expected && expected.tag === 'KubernetesReadyGetTimeout';
+    expect(outcome.error['failing']).toEqual([
+      'Deployment/kube-system/cilium-operator',
+      ...(unread ? ['CustomResourceDefinition/x.cilium.io'] : []),
+    ]);
+    expect(outcome.error['lastTransient']).toEqual(expected);
+    expect(JSON.stringify(outcome.error)).not.toContain('ECONNREFUSED');
+    expect(JSON.stringify(outcome.error)).not.toContain('secret');
+  });
+}
+
+test('transientOf: tolerable classes only; 4xx, tagged adapter errors and non-errors are not', () => {
+  expect(transientOf(new Error('Failed Kubernetes GET /x: ECONNRESET'))).toEqual({
+    tag: 'TransportError',
+  });
+  expect(transientOf({ _tag: 'KubernetesApiError', statusCode: 502 })).toEqual({
+    status: 502,
+    tag: 'KubernetesApiError',
+  });
+  expect(transientOf({ _tag: 'KubernetesApiError', statusCode: 429 })?.status).toBe(429);
+  for (const error of [
+    { _tag: 'KubernetesApiError', statusCode: 401 },
+    { _tag: 'KubernetesApiError', statusCode: 403 },
+    { _tag: 'KubernetesApiError', statusCode: 404 },
+    { _tag: 'TalosVaultKeyMissing' },
+    // an untagged Error that is not upstream's transport wrapper is a defect (path-builder throw)
+    new Error('Kubernetes object apps/v1/Deployment/x requires a namespace'),
+    new Error('ECONNRESET'),
+    Object.assign(new Error('tagged'), { _tag: 'TalosOpenBaoConnectTimeout' }),
+    'a string',
+  ]) {
+    expect(transientOf(error)).toBeUndefined();
+  }
+});
+
+test('the poll recovers when a transient clears (the cluster answers on a later pass)', async () => {
+  const objects = allReady();
+  objects[get(DEP_PATH)] = { status: 503 };
+  const outcome = await pollOutcome(objects, () => {
+    // flip to ready after the first pass has looked: mutate on the live table
+    setTimeout(() => (objects[get(DEP_PATH)] = { body: readyDep }), 0);
+  });
+  // the timer runs on real time; the poll sleeps virtual seconds, so the next pass sees it ready
+  expect(outcome).toEqual({ ok: true });
+});
+
+for (const status of [401, 403, 400, 409]) {
+  test(`${status} propagates from a poll instead of waiting out the deadline`, async () => {
+    const objects = allReady();
+    objects[get(DS_PATH)] = { status, body: { message: 'nope' } };
+    const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+    expect(outcome.error['_tag']).toBe('KubernetesReadyApiError');
+    expect(outcome.error['statusCode']).toBe(status);
+  });
+}
+
+test('ProgressDeadlineExceeded ends the poll with the typed failure, immediately', async () => {
+  const objects = allReady();
+  objects[get(DEP_PATH)] = {
+    body: {
+      ...readyDep,
+      status: {
+        ...readyDep.status,
+        conditions: [{ reason: 'ProgressDeadlineExceeded', type: 'Progressing' }],
+      },
+    },
+  };
+  const outcome = (await pollOutcome(objects)) as { error: Record<string, unknown> };
+  expect(outcome.error['_tag']).toBe('KubernetesRolloutFailed');
+  expect(outcome.error['check']).toBe('Deployment/kube-system/cilium-operator');
+});
+
+test('a single pass tolerates 404 only: 5xx, 429, timeout and transport errors propagate', async () => {
+  const run = (entry: FakeObject) =>
+    withCluster(
+      { ...allReady(), [get(DEP_PATH)]: entry },
+      driven(
+        Effect.gen(function* () {
+          const transport = yield* connectCluster(connection);
+          return yield* pass(transport, checks, false);
+        }),
+        30,
+      ).pipe(
+        Effect.map((exit) =>
+          Exit.isSuccess(exit)
+            ? exit.value
+            : { error: Cause.squash(exit.cause) as { _tag?: string } },
+        ),
+      ),
+    );
+  expect(await run({ status: 404 })).toMatchObject({
+    failed: [],
+    pending: ['Deployment/kube-system/cilium-operator'],
+  });
+  for (const [entry, tag] of [
+    [{ status: 503 }, 'KubernetesReadyApiError'],
+    [{ status: 429 }, 'KubernetesReadyApiError'],
+    [{ hang: true }, 'KubernetesReadyGetTimeout'],
+  ] as const) {
+    expect(((await run(entry)) as { error: { _tag: string } }).error._tag).toBe(tag);
+  }
+  expect(((await run({ error: 'ECONNRESET' })) as { error: unknown }).error).toBeInstanceOf(Error);
+});