@homeflare/alchemy 0.49.0 → 0.50.0

package/src/kubernetes/ready-poll.ts

@@ -0,0 +1,215 @@
+/**
+ * The poll loop of `HomeFlare.Kubernetes.Ready`: one bounded GET per check, and the deadline.
+ * Reads go through alchemy's own `readObject` (TLS, CA and client-cert handling stay upstream's),
+ * over the transport `connectCluster` proved against the pinned uid.
+ *
+ * ⛔ Error classes live in `ready-classify.ts` (404 pending; transients tolerated ONLY in a poll;
+ *   everything else propagates). A single pass (`tolerant: false`, used by `read`/`diff`)
+ *   tolerates ONLY 404. The poll covers the CONNECT too (`ready-connect.ts`): the identity GET of
+ *   a cluster whose apiserver is restarting under a CNI rollout is the same transient.
+ * ⚠️ upstream `readObject` takes no signal: a timed-out GET ABANDONS its socket rather than
+ *   aborting it (`client.ts:71-74`); nothing here can close it, and process exit is NOT a bound
+ *   for a long apply. So a timeout is rationed: once one GET of a pass times out the rest of that
+ *   pass is pending without a GET, and the poll doubles its pause per consecutive timeout (cap
+ *   30 s), resetting on a clean pass. A hung apiserver then abandons a few dozen sockets in a 10 m
+ *   wait, not ~120. ★ STANDING WATCH: drop this rationing when upstream's `readObject` takes an
+ *   AbortSignal (alchemy PR 1948, per-attempt deadlines).
+ * ⚠️ upstream also retries a transport error itself, for about 40 s (`Schedule.max([spaced 5 s,
+ *   recurs(8)])`, `client.ts:158-161`): longer than `GET_TIMEOUT`, so a dead socket surfaces here
+ *   as the GET timeout, not as the transport error.
+ */
+import type { ClusterTransport } from 'alchemy/Kubernetes/ClusterAdapter';
+import { readObject } from 'alchemy/Kubernetes/internal/client';
+import * as Data from 'effect/Data';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Ref from 'effect/Ref';
+import { isNotFound, scrubbed, transientOf } from './ready-classify.ts';
+import { type ReadyCheck, checkKey, countsOf, evaluate } from './ready-checks.ts';
+import {
+  type KeyState,
+  KubernetesReadyTimeout,
+  KubernetesRolloutFailed,
+  type LastTransient,
+} from './ready-errors.ts';
+
+export { KubernetesReadyApiError, isNotFound, transientOf } from './ready-classify.ts';
+
+/** Each GET is bounded like `readClusterUid` (`talos/cluster-identity.ts`). */
+export const GET_TIMEOUT = Duration.seconds(5);
+
+/** One GET exceeded `GET_TIMEOUT`. Pending inside a poll, propagated by a single pass. */
+export class KubernetesReadyGetTimeout extends Data.TaggedError('KubernetesReadyGetTimeout')<{
+  readonly check: string;
+  readonly seconds: number;
+}> {
+  override get message(): string {
+    return `Kubernetes.Ready: GET for ${this.check} timed out after ${this.seconds}s`;
+  }
+}
+
+const REFS = {
+  CustomResourceDefinition: {
+    apiVersion: 'apiextensions.k8s.io/v1',
+    kind: 'CustomResourceDefinition',
+  },
+  DaemonSet: { apiVersion: 'apps/v1', kind: 'DaemonSet' },
+  Deployment: { apiVersion: 'apps/v1', kind: 'Deployment' },
+} as const;
+
+const refOf = (check: ReadyCheck) => ({
+  ...REFS[check.kind],
+  name: check.name,
+  ...('namespace' in check ? { namespace: check.namespace } : {}),
+});
+
+/** Longest pause the poll backs off to after consecutive GET timeouts. */
+export const BACKOFF_CAP = Duration.seconds(30);
+
+export interface PassResult {
+  /** Keys of checks that are not ready yet. */
+  readonly pending: readonly string[];
+  /** Keys of checks that are terminally failed (see `KubernetesRolloutFailed`). */
+  readonly failed: readonly string[];
+  /** What this pass saw per key it reached; a key that hit a transient has none. */
+  readonly states: Readonly<Record<string, KeyState>>;
+  readonly lastTransient?: LastTransient;
+  /** A GET timed out (its socket is abandoned): the poll backs off. */
+  readonly timedOut: boolean;
+}
+
+const getOne = (transport: ClusterTransport, check: ReadyCheck) =>
+  readObject({ object: refOf(check), transport }).pipe(
+    Effect.timeoutOrElse({
+      duration: GET_TIMEOUT,
+      orElse: () =>
+        Effect.fail(
+          new KubernetesReadyGetTimeout({
+            check: checkKey(check),
+            seconds: Duration.toSeconds(GET_TIMEOUT),
+          }),
+        ),
+    }),
+  );
+
+/** One look at every check. `tolerant` selects the poll rules; see the header. */
+export const pass = (
+  transport: ClusterTransport,
+  checks: readonly ReadyCheck[],
+  tolerant: boolean,
+): Effect.Effect<PassResult, unknown> =>
+  Effect.gen(function* () {
+    const pending: string[] = [];
+    const failed: string[] = [];
+    const states: Record<string, KeyState> = {};
+    let lastTransient: LastTransient | undefined;
+    let timedOut = false;
+    for (const check of checks) {
+      const key = checkKey(check);
+      // ⛔ After a timeout its socket is abandoned: issue no further GET this pass.
+      if (timedOut) {
+        pending.push(key);
+        continue;
+      }
+      const outcome = yield* getOne(transport, check).pipe(
+        Effect.map((body) => {
+          states[key] = countsOf(check, body);
+          return evaluate(check, body);
+        }),
+        Effect.catch((error: unknown) => {
+          if (isNotFound(error)) {
+            states[key] = { notFound: true };
+            return Effect.succeed('pending' as const);
+          }
+          const transient = tolerant ? transientOf(error) : undefined;
+          if (transient === undefined) return Effect.fail(scrubbed(error));
+          lastTransient = transient;
+          timedOut = timedOut || transient.tag === 'KubernetesReadyGetTimeout';
+          return Effect.succeed('pending' as const);
+        }),
+      );
+      if (outcome === 'pending') pending.push(key);
+      if (outcome === 'failed') failed.push(key);
+    }
+    return {
+      failed,
+      pending,
+      states,
+      timedOut,
+      ...(lastTransient === undefined ? {} : { lastTransient }),
+    };
+  });
+
+/**
+ * Poll until every check is ready. A terminal failure ends it at once; otherwise the deadline
+ * fails with the check keys that were pending on the LAST pass, what that pass saw of each, and
+ * the transient error of the last pass only (⚠️ a transient that cleared is not to be blamed).
+ * `connect` runs inside the deadline (it retries its own transients, `ready-connect.ts`).
+ */
+export const poll = <R>(
+  connect: (
+    note: (transient: LastTransient) => Effect.Effect<void>,
+  ) => Effect.Effect<ClusterTransport, unknown, R>,
+  checks: readonly ReadyCheck[],
+  waitTimeout: Duration.Duration,
+  pollInterval: Duration.Duration,
+) =>
+  Effect.gen(function* () {
+    const pendingNow = checks.map(checkKey);
+    const last = yield* Ref.make<{ pending: readonly string[]; states: Record<string, KeyState> }>({
+      pending: pendingNow,
+      states: {},
+    });
+    const transient = yield* Ref.make<LastTransient | undefined>(undefined);
+    const loop = Effect.gen(function* () {
+      const transport = yield* connect((t) => Ref.set(transient, t));
+      let pause = pollInterval;
+      while (true) {
+        const result = yield* pass(transport, checks, true);
+        yield* Ref.update(last, (prior) => ({
+          pending: result.pending,
+          states: { ...prior.states, ...result.states },
+        }));
+        // ★ Overwritten every pass, so a transient that has since cleared is not blamed.
+        yield* Ref.set(transient, result.lastTransient);
+        const [firstFailed] = result.failed;
+        if (firstFailed !== undefined) {
+          return yield* Effect.fail(
+            new KubernetesRolloutFailed({
+              check: firstFailed,
+              reason: firstFailed.startsWith('DaemonSet/')
+                ? 'UpdateStrategyNotRollingUpdate'
+                : 'ProgressDeadlineExceeded',
+            }),
+          );
+        }
+        if (result.pending.length === 0) return;
+        // ★ Double per consecutive timeout (cap 30 s, or the interval if longer); a clean pass resets.
+        pause = result.timedOut
+          ? Duration.min(Duration.times(pause, 2), Duration.max(BACKOFF_CAP, pollInterval))
+          : pollInterval;
+        yield* Effect.sleep(pause);
+      }
+    });
+    return yield* loop.pipe(
+      Effect.timeoutOrElse({
+        duration: waitTimeout,
+        orElse: () =>
+          Effect.gen(function* () {
+            const { pending, states } = yield* Ref.get(last);
+            const lastTransient = yield* Ref.get(transient);
+            const seen = Object.fromEntries(
+              pending.flatMap((k) => (states[k] ? [[k, states[k]]] : [])),
+            );
+            return yield* Effect.fail(
+              new KubernetesReadyTimeout({
+                failing: pending,
+                seconds: Duration.toSeconds(waitTimeout),
+                states: seen,
+                ...(lastTransient === undefined ? {} : { lastTransient }),
+              }),
+            );
+          }),
+      }),
+    );
+  });

package/src/kubernetes/ready-rollout.test.ts

@@ -0,0 +1,58 @@
+/** Round 1, findings 1 and 7: the lower bound on updated-and-available pods, and the strategy rule. */
+import { expect, test } from 'bun:test';
+import { type ReadyCheck, evaluate } from './ready-checks.ts';
+
+const check = (minReady?: number): ReadyCheck => ({
+  kind: 'DaemonSet',
+  name: 'cilium',
+  namespace: 'kube-system',
+  ...(minReady === undefined ? {} : { minReady }),
+});
+const ds = (status: object, minReady?: number, spec: object = {}) =>
+  evaluate(check(minReady), { metadata: { generation: 2 }, spec, status });
+const status = (desired: number, updated: number, available: number) => ({
+  desiredNumberScheduled: desired,
+  numberAvailable: available,
+  numberReady: available,
+  observedGeneration: 2,
+  updatedNumberScheduled: updated,
+});
+
+// `numberAvailable` counts AVAILABLE OLD-template pods too (kubectl demands updated == desired).
+test('(a) 4 nodes, minReady 2: 2 new pods crashloop, 2 old pods available is pending', () => {
+  expect(ds(status(4, 2, 2), 2)).toBe('pending');
+});
+
+test('(b) 3 nodes: new pod on A works, new pod on B crashloops, C runs the old pod: pending', () => {
+  expect(ds(status(3, 2, 2), 2)).toBe('pending');
+});
+
+test('once the new pods really are available the same shapes are ready', () => {
+  expect(ds(status(4, 2, 4), 2)).toBe('ready'); // 2 updated, 2 old: 4 - 2 = 2 updated-available
+  expect(ds(status(3, 2, 3), 2)).toBe('ready');
+});
+
+test('the bound is a lower bound, not an exact count', () => {
+  // all 3 on the new template, 2 available: 2 - 0 >= 2
+  expect(ds(status(3, 3, 2), 2)).toBe('ready');
+  // 3 on the new template and 1 available is not
+  expect(ds(status(3, 3, 1), 2)).toBe('pending');
+  // updated below the bar fails even when everything is available
+  expect(ds(status(4, 1, 4), 2)).toBe('pending');
+});
+
+test('the default minReady (every node) is unchanged by the bound', () => {
+  expect(ds(status(3, 3, 3))).toBe('ready');
+  expect(ds(status(3, 2, 3))).toBe('pending');
+});
+
+test('a non-RollingUpdate strategy is a terminal failure; absent means RollingUpdate', () => {
+  const rolled = status(3, 3, 3);
+  expect(ds(rolled, undefined, { updateStrategy: { type: 'OnDelete' } })).toBe('failed');
+  expect(ds(rolled, undefined, { updateStrategy: { type: 'RollingUpdate' } })).toBe('ready');
+  expect(ds(rolled, undefined, {})).toBe('ready');
+  // checked before the generation rule, as kubectl does
+  expect(
+    ds({ ...rolled, observedGeneration: 1 }, undefined, { updateStrategy: { type: 'x' } }),
+  ).toBe('failed');
+});

package/src/kubernetes/ready-secrecy.test.ts

@@ -0,0 +1,131 @@
+/**
+ * Secrecy of `HomeFlare.Kubernetes.Ready`: no PEM marker, kubeconfig byte or apiserver response
+ * body reaches an attribute, an error, a log line, or a file. The vault is the fake `bao` of the
+ * adapter fixtures (it serves a kubeconfig with PEM blocks); the apiserver bodies carry a needle.
+ */
+import { expect, spyOn, test } from 'bun:test';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import * as Cause from 'effect/Cause';
+import * as Logger from 'effect/Logger';
+import * as References from 'effect/References';
+import type { FakeObject } from '../talos/fake-apiserver.ts';
+import { vault } from '../talos/cluster-adapter.fixtures.ts';
+import { diffReady, readReady, reconcileReady } from './ready.ts';
+import {
+  CRD_PATH,
+  DEP_PATH,
+  DS_PATH,
+  connection,
+  crdCheck,
+  depCheck,
+  driven,
+  dsCheck,
+  get,
+  readyCrd,
+  readyDep,
+  readyDs,
+  withCluster,
+} from './ready.fixtures.ts';
+
+const BODY_NEEDLE = 'apiserver-body-needle-7f3a';
+const pemMarker = `-----${'BEGIN'}`;
+const b64 = (text: string) => Buffer.from(text).toString('base64');
+const props = {
+  checks: [dsCheck, depCheck, crdCheck],
+  connection,
+  waitTimeout: '20s',
+  pollInterval: '5s',
+};
+
+// The vault's own document is the needle source: every field value the fake `bao` serves.
+const served = JSON.parse(
+  vault({ args: ['talos-c1/data/kubeconfig'], command: 'bao' }).stdout ?? '{}',
+) as {
+  data: { data: { kubeconfig: string } };
+};
+const needles = [
+  BODY_NEEDLE,
+  pemMarker,
+  served.data.data.kubeconfig,
+  b64(`${pemMarker} CERTIFICATE-----`),
+  b64(`${pemMarker} PRIVATE KEY-----`),
+  'client-key-data',
+  'certificate-authority-data',
+];
+const leaks = (text: string) => needles.filter((needle) => text.includes(needle));
+const dump = (value: unknown) =>
+  JSON.stringify(value, value instanceof Object ? Object.getOwnPropertyNames(value) : undefined) +
+  String(value);
+
+const ready: Record<string, FakeObject> = {
+  [get(CRD_PATH)]: { body: readyCrd },
+  [get(DEP_PATH)]: { body: readyDep },
+  [get(DS_PATH)]: { body: readyDs },
+};
+
+test('attributes, errors and logs of every path carry no vault or response byte', async () => {
+  const logged: string[] = [];
+  const capture = Logger.layer([
+    Logger.make(({ message }) => void logged.push(JSON.stringify(message))),
+  ]);
+  const saved = { ...console };
+  for (const level of ['log', 'debug', 'info', 'warn', 'error'] as const) {
+    console[level] = (...args: unknown[]) => void logged.push(args.map(String).join(' '));
+  }
+  const writes = [spyOn(Bun, 'write'), spyOn(Bun, 'spawn')];
+  const outputs: unknown[] = [];
+  try {
+    const attempt = async (
+      objects: Record<string, FakeObject>,
+      run: Effect.Effect<unknown, unknown, never>,
+    ) => {
+      const exit = await withCluster(
+        objects,
+        driven(
+          run.pipe(
+            Effect.provideService(References.MinimumLogLevel, 'All'),
+            Effect.provide(capture),
+          ),
+          100,
+        ) as never,
+      );
+      const result = exit as Exit.Exit<unknown, unknown>;
+      outputs.push(Exit.isSuccess(result) ? result.value : Cause.squash(result.cause));
+    };
+    const failing = (entry: FakeObject): Record<string, FakeObject> => ({
+      ...ready,
+      [get(DEP_PATH)]: { ...entry, body: { message: BODY_NEEDLE } },
+    });
+    for (const entry of [
+      { status: 200 },
+      { status: 404 },
+      { status: 503 },
+      { status: 403 },
+      { status: 429 },
+    ]) {
+      for (const run of [
+        readReady(props),
+        reconcileReady(props),
+        diffReady(props as never, { checks: '', connection, ready: false }),
+      ]) {
+        await attempt(failing(entry), run as never);
+      }
+    }
+    await attempt(ready, reconcileReady(props) as never);
+  } finally {
+    Object.assign(console, saved);
+    for (const spy of writes) {
+      expect(spy).not.toHaveBeenCalled();
+      spy.mockRestore();
+    }
+  }
+  expect(outputs.length).toBeGreaterThan(10);
+  expect(leaks(outputs.map(dump).join('\n'))).toEqual([]);
+  expect(leaks(logged.join('\n'))).toEqual([]);
+});
+
+test('the probe can fail: a response body quoted into an error is caught by the needle check', () => {
+  const quoted = new Error(`GET responded 503: ${JSON.stringify({ message: BODY_NEEDLE })}`);
+  expect(leaks(dump(quoted))).toEqual([BODY_NEEDLE]);
+});

package/src/kubernetes/ready.fixtures.ts

@@ -0,0 +1,103 @@
+/**
+ * Shared harness for the `HomeFlare.Kubernetes.Ready` tests: the REAL `talos-openbao` adapter over a
+ * fake `bao` and the fake apiserver (`talos/fake-apiserver.ts`), so uid proof, `readObject` and the
+ * TLS-free request path all run for real. Nothing leaves the process.
+ */
+import * as Effect from 'effect/Effect';
+import * as Fiber from 'effect/Fiber';
+import * as Layer from 'effect/Layer';
+import * as TestClock from 'effect/testing/TestClock';
+import * as ChildProcessSpawner from 'effect/process/ChildProcessSpawner';
+import { TalosOpenBaoAdapter, talosOpenBaoConnection } from '../talos/cluster-adapter.ts';
+import { UIDS, config, vault } from '../talos/cluster-adapter.fixtures.ts';
+import { type FakeHandler, fakeSpawner } from '../talos/fake-process.ts';
+import { type FakeObject, fakeApiServer } from '../talos/fake-apiserver.ts';
+import type { ReadyCheck } from './ready-checks.ts';
+
+export const HOST = 'c1.cluster.invalid';
+export const connection = talosOpenBaoConnection('uid-c1');
+
+export const DS_PATH = `/apis/apps/v1/namespaces/kube-system/daemonsets/cilium`;
+export const DEP_PATH = `/apis/apps/v1/namespaces/kube-system/deployments/cilium-operator`;
+export const CRD_PATH = `/apis/apiextensions.k8s.io/v1/customresourcedefinitions/x.cilium.io`;
+
+export const dsCheck: ReadyCheck = { kind: 'DaemonSet', name: 'cilium', namespace: 'kube-system' };
+export const depCheck: ReadyCheck = {
+  kind: 'Deployment',
+  name: 'cilium-operator',
+  namespace: 'kube-system',
+};
+export const crdCheck: ReadyCheck = { kind: 'CustomResourceDefinition', name: 'x.cilium.io' };
+
+export const checksOf = (): ReadyCheck[] => [dsCheck, depCheck, crdCheck];
+
+export const readyDs = {
+  metadata: { generation: 2 },
+  status: {
+    desiredNumberScheduled: 3,
+    numberAvailable: 3,
+    numberReady: 3,
+    observedGeneration: 2,
+    updatedNumberScheduled: 3,
+  },
+};
+export const readyDep = {
+  metadata: { generation: 4 },
+  spec: { replicas: 2 },
+  status: { availableReplicas: 2, observedGeneration: 4, replicas: 2, updatedReplicas: 2 },
+};
+export const readyCrd = { status: { conditions: [{ status: 'True', type: 'Established' }] } };
+
+export const get = (path: string) => `GET ${HOST}${path}`;
+
+export const IDENTITY_PATH = '/api/v1/namespaces/kube-system';
+
+/**
+ * A live table whose `key` answers `entries[n]` on the n-th request (the last one thereafter):
+ * a cluster that is flaky for a while and then recovers, without real timers.
+ */
+export const sequence = (
+  objects: Record<string, FakeObject>,
+  key: string,
+  entries: readonly (FakeObject | undefined)[],
+): Record<string, FakeObject> => {
+  let calls = 0;
+  return new Proxy(objects, {
+    get: (target, prop) => {
+      if (prop !== key) return target[prop as string];
+      return entries[Math.min(calls++, entries.length - 1)];
+    },
+  });
+};
+
+/**
+ * Run `effect` against the fake cluster. `objects` is the live table: a test may mutate it from
+ * inside `effect`. The fake `https` module is restored on every exit path.
+ */
+export const withCluster = async <A, E>(
+  objects: Record<string, FakeObject>,
+  effect: Effect.Effect<A, E, ChildProcessSpawner.ChildProcessSpawner | never>,
+  handler: FakeHandler = vault,
+) => {
+  const api = fakeApiServer(UIDS, [], objects);
+  const spawner = Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, fakeSpawner(handler));
+  try {
+    const layer = Layer.mergeAll(TalosOpenBaoAdapter(config).pipe(Layer.provide(spawner)), spawner);
+    return await Effect.runPromise(
+      effect.pipe(Effect.provide(layer), Effect.provide(TestClock.layer())) as Effect.Effect<A, E>,
+    );
+  } finally {
+    api.restore();
+  }
+};
+
+/** Drive TestClock until `effect` settles (virtual seconds, no real waiting). */
+export const driven = <A, E, R>(effect: Effect.Effect<A, E, R>, seconds = 1000) =>
+  Effect.gen(function* () {
+    const fiber = yield* Effect.forkChild(Effect.exit(effect));
+    for (let i = 0; i < seconds; i++) {
+      yield* Effect.yieldNow;
+      yield* TestClock.adjust('1 second');
+    }
+    return yield* Fiber.join(fiber);
+  });