@homeflare/alchemy 0.1.0

package/src/proxmox/metric-server.ts

@@ -0,0 +1,214 @@
+/**
+ * `Proxmox.MetricServer` — where the cluster ships its metrics, declared.
+ *
+ * ★ THE FAILURE THIS PREVENTS IS A SILENT ONE, WHICH IS THE WHOLE ARGUMENT FOR DECLARING IT.
+ *   Undeclared, the target exists only as a section of `/etc/pve/status.cfg`: rebuild a node,
+ *   re-add one to the cluster, or restore a config from before it was configured, and the cluster
+ *   stops shipping. Nothing fails, no task errors, no alert fires — the graphs go flat, and the
+ *   first to find out is whoever went looking for a number that was not there.
+ *
+ * ⛔ CREATE IS `POST cluster/metrics/server/{id}` — THE OBJECT'S OWN PATH — SO `collection()`
+ *   RETURNS THE SAME STRING AS `path()`, AND THAT IS NOT A TYPO. MEASURED: a POST to the
+ *   collection answers "Method 'POST /cluster/metrics/server' not implemented", while the id path
+ *   gets as far as a permission check (`/`, Sys.Modify); the published schema agrees, giving the
+ *   collection a GET and nothing else. `createForm` omits `id` for the same reason — it is already
+ *   the last segment of the path being POSTed to, and a second copy can only disagree with it.
+ *
+ * ⛔ THE `token` IS WRITE-ONLY AND MUST NEVER BECOME AN ATTRIBUTE. Alchemy persists attributes
+ *   UNENCRYPTED, so one that reached state would outlive the change that set it, in the state
+ *   store and in every backup of it. It is `never` in the props below — declaring one is a COMPILE
+ *   ERROR rather than a leak found months later — and absent from `optional`, for the reason
+ *   given there. Set it out of band; PVE never returns it on a read anyway. `otel-headers`, where an
+ *   opentelemetry server's bearer token lives, gets the same treatment in metric-server-otel.ts.
+ *
+ * ⚠️ PRIVILEGES, AND THIS IS A BIGGER ASK THAN IT LOOKS. Read and diff are fine under an auditor
+ *   lease — the single-object GET checks `Sys.Audit` on `/` — while create, update and delete all
+ *   check `Sys.Modify` on `/` and answer "Permission check failed (/, Sys.Modify)" until the role
+ *   is widened. Widen it knowingly: the ACL path is the ROOT and there is no `/metrics` object to
+ *   scope to, so `Sys.Modify` also buys datacenter options, notification targets and every other
+ *   cluster-wide config write. A separate role for metric writes is the narrower answer.
+ */
+import { Resource } from 'alchemy';
+import * as Provider from 'alchemy/Provider';
+import * as Effect from 'effect/Effect';
+import { UNSET, body, update } from './metric-server-form.ts';
+import {
+  type MetricServerOtelAttributes,
+  type MetricServerOtelProps,
+  otelAttributes,
+  otelMatches,
+} from './metric-server-otel.ts';
+import { type PveRequirements, type WithTarget, pveHandlers } from './resource.ts';
+import { bool, int, text } from './values.ts';
+
+/**
+ * PVE's status plugins. Which one a server is decides which fields it will even accept.
+ *
+ * ★ `opentelemetry` IS OFFERED SINCE 2026-09-14, AND THE REASON IT WAS NOT STILL HOLDS FOR ANY FIELD
+ *   LEFT OUT. Its whole configuration is the `otel-*` family; declared without managing that family
+ *   it would build a target whose every meaningful setting sat outside the graph, and report `noop`
+ *   over it. metric-server-otel.ts manages every `otel-*` field but two, and says why for each.
+ */
+export type MetricServerType = 'graphite' | 'influxdb' | 'opentelemetry';
+
+export interface MetricServerProps extends WithTarget, MetricServerOtelProps {
+  /** ⚠️ A `pve-configid` — a letter, then letters, digits, `-` and `_`. A rejected one fails with
+   *   "invalid configuration ID", which reads as a broken request rather than as a naming rule. */
+  id: string;
+  /**
+   * ⛔ CREATE-TIME ONLY. The PUT schema has no `type` parameter at all: a section's type is fixed
+   *   when it is written, so the ⛔ in `attributes` turns a changed type into a refusal rather than
+   *   a silent no-op over the wrong target.
+   */
+  type: MetricServerType;
+  /** ⚠️ REQUIRED ON EVERY WRITE, UPDATE INCLUDED — see `required`. */
+  server: string;
+  port: number;
+  /** Keep the section but stop shipping. Unset is an ENABLED target, not an absent one. */
+  disable?: boolean;
+  /**
+   * graphite and influxdb only: UDP MTU (512-65536) and socket/HTTP timeout. Unset leaves PVE's
+   * defaults, 1500 and 1. ⚠️ opentelemetry has neither; its timeout is `otel-timeout`.
+   */
+  mtu?: number;
+  timeout?: number;
+  /** graphite only: the root path metrics are published under, e.g. `proxmox.mycluster`. */
+  path?: string;
+  /**
+   * influxdb only. `udp` is PVE's default and writes line protocol; `http`/`https` use the v2 API,
+   * the only place `organization` and `bucket` mean anything. An unset `verify-certificate` means
+   * VERIFY — PVE's default, and the safe one.
+   * ⚠️ PVE's hyphenated names are kept rather than camelCased, so nothing between here and the form
+   *   body has to remember a mapping.
+   */
+  influxdbproto?: 'udp' | 'http' | 'https';
+  organization?: string;
+  bucket?: string;
+  'api-path-prefix'?: string;
+  'max-body-size'?: number;
+  'verify-certificate'?: boolean;
+  /** ⛔ `never` ON PURPOSE — see the ⛔ in the header. The compile error is the feature. */
+  token?: never;
+}
+
+/**
+ * ⛔ NO `digest`, ON PURPOSE. PVE returns one with every read, but it is the digest of the WHOLE
+ *   `status.cfg` rather than of this section — declaring a SECOND metric server would rewrite this
+ *   one's stored attributes, and comparing it would report an update on a target nobody touched.
+ *   ⛔ And no `token`, for the reason the header gives at length.
+ * ⚠️ `mtu`, `timeout` and `max-body-size` report `UNSET` when the section carries none, i.e. when
+ *   PVE's own default is what the target will use; `verify-certificate` reports that default
+ *   directly, because "absent" there means the target verifies. `port` reports 0 only for a
+ *   malformed section — PVE requires a port — so a plan should say so loudly. The `otel-*` half
+ *   follows the same rules; see metric-server-otel.ts.
+ */
+export interface MetricServerAttributes extends MetricServerOtelAttributes {
+  id: string;
+  type: MetricServerType;
+  server: string;
+  port: number;
+  disable: boolean;
+  mtu: number;
+  timeout: number;
+  path: string;
+  influxdbproto: string;
+  organization: string;
+  bucket: string;
+  'api-path-prefix': string;
+  'max-body-size': number;
+  'verify-certificate': boolean;
+}
+
+export interface ProxmoxMetricServer extends Resource<
+  'Proxmox.MetricServer',
+  MetricServerProps,
+  MetricServerAttributes,
+  never,
+  PveRequirements
+> {}
+
+export const ProxmoxMetricServer = Resource<ProxmoxMetricServer>('Proxmox.MetricServer');
+
+/**
+ * The type-specific half of `matches`.
+ *
+ * ⛔ ONLY FIELDS THIS TYPE ACTUALLY SENDS ARE COMPARED. `verify-certificate` is not in graphite's
+ *   option set and `path` is not in influxdb's, so neither is ever sent to the wrong plugin — and
+ *   diffing one anyway would report an update that the PUT it triggers cannot satisfy, forever.
+ *   `mtu` and `timeout` are compared for graphite and influxdb only, because the opentelemetry
+ *   plugin has neither and `optional` never sends them to it.
+ */
+const matchesType = (attributes: MetricServerAttributes, props: MetricServerProps): boolean => {
+  if (props.type === 'opentelemetry') return otelMatches(attributes, props);
+  const socket =
+    attributes.mtu === (props.mtu ?? UNSET) && attributes.timeout === (props.timeout ?? UNSET);
+  if (props.type === 'graphite') return socket && attributes.path === (props.path ?? '');
+  return (
+    socket &&
+    attributes.influxdbproto === (props.influxdbproto ?? '') &&
+    attributes.organization === (props.organization ?? '') &&
+    attributes.bucket === (props.bucket ?? '') &&
+    attributes['api-path-prefix'] === (props['api-path-prefix'] ?? '') &&
+    attributes['max-body-size'] === (props['max-body-size'] ?? UNSET) &&
+    attributes['verify-certificate'] === (props['verify-certificate'] !== false)
+  );
+};
+
+const handlers = pveHandlers<MetricServerProps, MetricServerAttributes>({
+  attributes: (live, props) => {
+    /**
+     * ⛔ A SERVER OF ANOTHER TYPE IS ANOTHER OBJECT, AND "ABSENT" IS THE HONEST ANSWER. `type`
+     *   cannot be changed by a PUT, so reporting the foreign section as missing makes reconcile
+     *   POST instead and PVE refuses because the id is taken — a loud, accurate error rather than a
+     *   PUT pushing influxdb fields at a graphite section. An older PVE that omits `type` from the
+     *   read falls through and is trusted.
+     */
+    const liveType = text(live['type']);
+    if (liveType !== '' && liveType !== props.type) return undefined;
+    return {
+      ...otelAttributes(live),
+      'api-path-prefix': text(live['api-path-prefix']),
+      bucket: text(live['bucket']),
+      disable: bool(live['disable'], false),
+      id: props.id,
+      influxdbproto: text(live['influxdbproto']),
+      'max-body-size': int(live['max-body-size'], UNSET),
+      mtu: int(live['mtu'], UNSET),
+      organization: text(live['organization']),
+      path: text(live['path']),
+      port: int(live['port'], 0),
+      server: text(live['server']),
+      timeout: int(live['timeout'], UNSET),
+      type: props.type,
+      'verify-certificate': bool(live['verify-certificate'], true),
+    };
+  },
+  /** ⛔ THE SAME STRING AS `path`, DELIBERATELY — see the ⛔ in the header. */
+  collection: (props) => `cluster/metrics/server/${props.id}`,
+  /** ⚠️ `type` IS REQUIRED ON CREATE; `id` IS NOT SENT, because the id is the path. */
+  createForm: (props) => ({ ...body(props), type: props.type }),
+  /**
+   * ⛔ `id` and `type` are absent: one is the key the object was read by, the other is refused
+   *   rather than updated. Everything type-specific is `matchesType`'s, above.
+   */
+  matches: (attributes, props) =>
+    attributes.server === props.server &&
+    attributes.port === props.port &&
+    attributes.disable === (props.disable === true) &&
+    matchesType(attributes, props),
+  path: (props) => `cluster/metrics/server/${props.id}`,
+  /** The form and its clear-list, from the one table in metric-server-form.ts. */
+  updateForm: update,
+});
+
+/**
+ * ⛔ THE EMPTY `list` FROM `pveHandlers` EARNS ITS KEEP HERE. `GET cluster/metrics/server` answers
+ *   with every target the cluster already ships to, including the one somebody configured by hand
+ *   years ago; adopting that is how a later `alchemy destroy` silences a graph nobody declared.
+ * ⚠️ AND NOTHING BRAKES THIS DESTROY. A pool refuses while it holds guests, a container while it
+ *   runs; a metric server just leaves `status.cfg`, the cluster stops shipping on the next
+ *   interval, and no guest or task is affected. The plan diff is the only warning anyone gets —
+ *   the top of this file again, this time caused by a deploy.
+ */
+export const ProxmoxMetricServerProvider = () =>
+  Provider.effect(ProxmoxMetricServer, Effect.succeed(ProxmoxMetricServer.Provider.of(handlers)));

package/src/proxmox/network-apply-read.ts

@@ -0,0 +1,228 @@
+/**
+ * How this package finds out what a node has STAGED, and whether the cluster survived an apply.
+ *
+ * ★ SPLIT OUT OF network-apply.ts TO KEEP BOTH FILES UNDER THE 250-LINE CAP, and the seam is a real
+ *   one rather than a convenient line number: this file answers "what is true on the cluster right
+ *   now", network-apply.ts answers "what does the resource do about it". Nothing here decides a
+ *   diff, and the only write in either file is the apply itself.
+ *
+ * ⛔ THE PENDING STATE IS NOT IN `data`, WHICH IS WHY THIS FILE DOES ITS OWN FETCH. MEASURED in the
+ *   cluster's own source on n2, 2026-09-13 — not inferred from documentation:
+ *     PVE/API2/Network.pm   my $tmp = PVE::INotify::read_file('interfaces', 1);
+ *                           $rpcenv->set_result_attrib('changes', $changes) if $changes;
+ *     PVE/HTTPServer.pm     if (my $diff = $rpcenv->get_result_attrib('changes')) {
+ *                               $resp->{changes} = $diff;
+ *                           }
+ *   So the answer is `{"data":[…interfaces…],"changes":"<unified diff>"}`: `changes` is a SIBLING
+ *   of `data`, set only when the diff is non-empty. `client.ts`'s `pve()` returns `body.data` and
+ *   drops the envelope, so the one readable pending signal is invisible to every other resource in
+ *   this package. Without the fetch below, `Proxmox.NetworkApply` could not diff honestly at all.
+ *   ★ `pveEnvelopeWith` in client.ts now carries the same member failover as every other call.
+ *
+ * ⛔ AND THE INTERFACE ROWS CANNOT TELL YOU EITHER. MEASURED in PVE/INotify.pm `read_file`: when
+ *   `/etc/network/interfaces.new` exists, THAT is the file that gets parsed, so
+ *   `GET /nodes/{node}/network` answers with the STAGED config and a node about to be reconfigured
+ *   looks identical to one already running it. Same trap `sdn-apply.ts` documents for `?running=1`.
+ *
+ * ⚠️ `active` IS NOT A PENDING SIGNAL, AND THE OBVIOUS SHORTCUT IS MEASURABLY WRONG. On n2 and n3,
+ *   with nothing staged anywhere (VERIFIED 2026-09-13: `/etc/network/interfaces.new` absent on n2,
+ *   n3 and n4), `wlp91s0` already comes back with no `active` key at all. Counting interfaces that
+ *   are not up as "pending" would report an update on two of the three nodes forever.
+ *
+ * ⚠️ ONE THING GENUINELY CANNOT BE SEEN FROM HERE, AND IT IS SMALL. The diff is computed as
+ *   `diff -b -N -u`, so a staged file differing from the live one ONLY in whitespace produces no
+ *   `changes` at all. This module reports that node as settled while `interfaces.new` still sits
+ *   there waiting for the next reload anybody triggers. The file's existence is not exposed by the
+ *   API in any other form, so the alternative is not a better read — it is SSH.
+ */
+import * as Effect from 'effect/Effect';
+import { pve, pveEnvelopeWith, pveWith } from './client.ts';
+import { type PveCredential, type PveTarget } from './credentials.ts';
+import { leased } from './lease-cache.ts';
+import { bool, text } from './values.ts';
+
+/** The `GET /nodes/{node}/network` envelope. ⛔ `changes` is beside `data`, never inside it. */
+type NetworkEnvelope = { changes?: string };
+
+/** One row of `GET /cluster/status` — a `cluster` row, or one `node` row per member. */
+type StatusRow = { type?: string; name?: string; quorate?: unknown; online?: unknown };
+
+/** One `GET /nodes/{node}/tasks/{upid}/status` answer. `status` is `running` or `stopped`. */
+type TaskStatus = { status?: string; exitstatus?: string };
+
+/** What the cluster looked like the moment it was asked. Never persisted — see network-apply.ts. */
+export type ClusterHealth = { quorate: boolean; offline: readonly string[] };
+
+/**
+ * The unified diff between a node's applied interfaces file and its staged one, or `''`.
+ *
+ * ⛔ THIS STRING NEVER LEAVES THIS MODULE, AND THAT IS A SECRET RULE, NOT A STYLE ONE. Alchemy
+ *   persists attributes UNENCRYPTED (credentials.ts spells out why), and `/etc/network/interfaces`
+ *   is a file that can legally contain `wpa-psk` — n2 and n3 both carry a `wlp91s0`, MEASURED — as
+ *   well as any `pre-up` command somebody wrote. A diff of that file in a state store is a secret
+ *   in a state store. Only the COUNT below ever escapes.
+ */
+const stagedDiff = (target: PveTarget, node: string) =>
+  Effect.gen(function* () {
+    const path = `nodes/${node}/network`;
+    // ⚠️ THE `read` ROLE IS ENOUGH, AND DELIBERATELY SO. The schema's permission for this GET is
+    //   `"user": "all"` — no privilege check at all — so the pending read never needs the
+    //   provisioning lease, and a plan stays a plan even if the write role is missing entirely.
+    // ⛔ THROUGH THE LEASE CACHE, NOT `mint`. This read needs no particular identity — only the
+    //   apply and its task poll do, and network-apply.ts mints for those itself. Minting here left
+    //   SIX extra `hf-read@pve` tokens on n2 per TB4 plan (one per node, for read and for diff),
+    //   MEASURED 2026-09-14 while the cache itself made one read mint for the whole run.
+    const credential = yield* leased(target, 'read');
+    const envelope = (yield* pveEnvelopeWith(target, credential, 'GET', path)) as NetworkEnvelope;
+    return text(envelope.changes);
+  });
+
+/**
+ * Changed lines in a unified diff.
+ *
+ * ⚠️ A NON-EMPTY DIFF IS NEVER REPORTED AS ZERO, WHICH IS WHAT `Math.max` IS FOR. The `+++`/`---`
+ *   headers have to be skipped, and a removed line whose own text starts with `---` is skipped with
+ *   them; a staged change consisting of nothing else would otherwise count 0 and be reported as
+ *   "settled", which is the exact lie this resource exists to prevent. The count is informational —
+ *   the resource only ever compares it to zero — so an occasional undercount of 1 is harmless while
+ *   an undercount to zero is not.
+ */
+export const pendingLines = (diff: string) => {
+  if (diff === '') return 0;
+  const changed = diff
+    .split('\n')
+    .filter(
+      (line) =>
+        (line.startsWith('+') || line.startsWith('-')) &&
+        !line.startsWith('+++') &&
+        !line.startsWith('---'),
+    ).length;
+  return Math.max(changed, 1);
+};
+
+/** How many staged lines this node is carrying. Zero is the only settled value. */
+export const pendingCount = (target: PveTarget, node: string) =>
+  stagedDiff(target, node).pipe(Effect.map(pendingLines));
+
+/**
+ * Quorum, and every member that is not online.
+ *
+ * ⛔ QUORUM ALONE IS NOT THE CHECK, AND ON A THREE-NODE CLUSTER IT WOULD MISS THE ACCIDENT THIS
+ *   RESOURCE IS AFRAID OF. Losing ONE of three leaves `quorate: 1` — MEASURED shape on TB4:
+ *   `{"id":"cluster","name":"HF-TB4","nodes":3,"quorate":1,…}` plus one row per node with
+ *   `online: 1`. So a reload that drops n2 off the network is invisible to `quorate` and visible
+ *   only as `online: 0` on that row. Both are checked, and the node names come back so the failure
+ *   message can say WHICH member went away.
+ *
+ * ⚠️ AN UNCLUSTERED NODE IS QUORATE BY DEFINITION. A standalone PVE answers with node rows and no
+ *   `cluster` row; reading that absence as "not quorate" would make this resource undeclarable on
+ *   every single-node install.
+ *
+ * ⚠️ THIS ONE NEEDS `Sys.Audit` ON `/` — the only privilege in this family beyond the apply itself.
+ *   It is asked of the `read` role, which is auditor-shaped, rather than of `provision`.
+ */
+const clusterHealth = (target: PveTarget) =>
+  pve<StatusRow[]>(target, 'read', 'GET', 'cluster/status').pipe(
+    Effect.map((rows) => {
+      const all = rows ?? [];
+      const cluster = all.find((row) => row.type === 'cluster');
+      return {
+        offline: all
+          .filter((row) => row.type === 'node' && !bool(row.online))
+          .map((row) => text(row.name, '<unnamed>')),
+        quorate: cluster === undefined ? true : bool(cluster.quorate),
+      } satisfies ClusterHealth;
+    }),
+  );
+
+/**
+ * Why the cluster is not healthy, or `undefined` when it is.
+ *
+ * ★ A REASON RATHER THAN A BOOLEAN, BECAUSE THE CALLER'S ONLY USE FOR IT IS A REFUSAL MESSAGE.
+ *   "The deploy stopped because the cluster was degraded" is not actionable at three in the
+ *   morning; "quorate=true offline=[n3]" is. The two callers in network-apply.ts ask this before
+ *   the reload and again after it, and the second answer is the one that stops the chain from
+ *   reaching the next node.
+ *
+ * ⚠️ A MEMBER OFFLINE FOR AN UNRELATED REASON BLOCKS EVERY APPLY, AND THAT IS THE INTENDED
+ *   BEHAVIOUR RATHER THAN AN OVERSIGHT. A node down for maintenance is one failure away from the
+ *   pools losing min_size; reloading another node's networking while it is out is the move that
+ *   completes the outage. Bring it back, or apply by hand with the cluster in front of you.
+ */
+export const degradedReason = (target: PveTarget) =>
+  clusterHealth(target).pipe(
+    Effect.map((health) =>
+      health.quorate && health.offline.length === 0
+        ? undefined
+        : `quorate=${String(health.quorate)} offline=[${health.offline.join(', ')}]`,
+    ),
+  );
+
+/**
+ * ⛔ NOT AN EXITSTATUS PVE CAN RETURN, AND THAT IS THE POINT. The caller compares against `'OK'`;
+ *   these two say WHY the answer is not OK without needing a second branch for "we do not know".
+ */
+export const UNREACHABLE =
+  'the task could not be read -- the node stopped answering, or this lease cannot see it';
+const STILL_RUNNING = 'still running when this provider stopped waiting';
+
+/**
+ * How long to wait on `ifreload`, in one-second polls.
+ *
+ * ⚠️ THIRTY SECONDS IS A CEILING, NOT AN EXPECTATION. A reload of a handful of interfaces finishes
+ *   in a second or two; past thirty, something is wrong in a way a deploy cannot fix by waiting,
+ *   and the honest move is to stop and name the task rather than park the stack on it.
+ */
+const ATTEMPTS = 30;
+
+/**
+ * Wait for one `srvreload` task and answer its exitstatus.
+ *
+ * ⛔ THE APPLY IS ASYNCHRONOUS AND ITS 200 MEANS NOTHING. MEASURED: `reload_network_config` ends
+ *   `return $rpcenv->fork_worker('srvreload', 'networking', $authuser, $worker)`, so the PUT
+ *   returns a UPID the instant the worker is forked — before `ifreload -a` has run, let alone
+ *   succeeded. Waiting for the task is the only way to learn the outcome.
+ *
+ * ⛔ AND IT MUST BE POLLED WITH THE CREDENTIAL THAT STARTED IT. The task-status schema says
+ *   `"user": "all"` with the note "needs 'Sys.Audit' on '/nodes/<node>' if they are not the owner
+ *   of the task". Every mint returns a NEW token id, so a second mint is a different identity and
+ *   is not the owner — which is why this takes a `PveCredential` and the caller keeps one lease
+ *   across both calls, instead of widening the provisioning role to read its own tasks.
+ */
+export const awaitTask = (
+  target: PveTarget,
+  credential: PveCredential,
+  node: string,
+  upid: string,
+) =>
+  Effect.gen(function* () {
+    /**
+     * ⚠️ THE UPID GOES IN RAW, NOT PERCENT-ENCODED, AND THE CHOICE IS DELIBERATE. A UPID is
+     *   `UPID:<node>:<hex>:<hex>:<hex>:<type>:<id>:<user>:` — its only punctuation is `:`, `@`, `!`
+     *   and `-`, every one of which is a legal path character, so raw needs nothing from the
+     *   router. Encoding them instead RELIES on PVE unescaping each path segment, which is
+     *   plausible and was NOT measured here (measuring it means starting a task, i.e. writing). If
+     *   that assumption were wrong the poll would 404, this would report the outcome as unknown,
+     *   and a perfectly good reload would stop the deploy. Raw removes the assumption.
+     */
+    const path = `nodes/${node}/tasks/${upid}/status`;
+    for (let attempt = 0; attempt < ATTEMPTS; attempt += 1) {
+      /**
+       * ⛔ A FAILED POLL IS NOT A FAILED RELOAD, AND CALLING IT ONE WOULD BE THE WORST KIND OF
+       *   WRONG. If the member serving the poll is the node being reloaded, `ifreload -a` can drop
+       *   the very connection this poll rides on — see network-apply.ts.
+       *   The reload may well have succeeded. So the call is collapsed to "we cannot see the task"
+       *   and the caller reports the outcome as UNKNOWN, with the UPID, rather than guessing.
+       * ⚠️ A 403 LANDS HERE TOO, AND IT READS THE SAME. If the polling identity is somehow not the
+       *   task's owner, the endpoint wants `Sys.Audit` on the node and this sees only "no answer" —
+       *   which is why `UNREACHABLE` names both possibilities rather than blaming the network.
+       */
+      const status = yield* pveWith<TaskStatus>(target, credential, 'GET', path).pipe(
+        Effect.orElseSucceed(() => undefined),
+      );
+      if (status === undefined) return UNREACHABLE;
+      if (status.status === 'stopped') return text(status.exitstatus, 'stopped with no exitstatus');
+      yield* Effect.sleep('1 second');
+    }
+    return STILL_RUNNING;
+  });