@homeflare/alchemy 0.1.0

package/src/proxmox/ceph-flag.ts

@@ -0,0 +1,249 @@
+/**
+ * `Proxmox.CephFlag` — one cluster-wide Ceph OSD flag, declared: `noout`, `pause`, `norebalance`.
+ *
+ * ★ ONE RESOURCE PER FLAG, NOT ONE CARRYING ALL ELEVEN, AND THE CLUSTER'S OWN API DECIDED IT.
+ *   MEASURED on TB4 2026-09-13 (pve-manager 9.2.11, ceph tentacle 20.2.2) by reading
+ *   `/usr/share/perl5/PVE/API2/Cluster/Ceph.pm` and the published schema in
+ *   `/usr/share/pve-docs/api-viewer/apidoc.js`:
+ *     · `PUT /cluster/ceph/flags` ends in `fork_worker('cephsetflags', ...)` and returns a STRING
+ *       — a UPID. The bulk write is ASYNCHRONOUS, and this package has no task-polling machinery.
+ *     · `PUT /cluster/ceph/flags/{flag}` calls `$rados->mon_command` inline and returns null. The
+ *       per-flag write is SYNCHRONOUS.
+ *   `pveOperations.reconcile` READS BACK after every write and refuses to record a value it did
+ *   not see. Against the bulk endpoint that read-back races a worker which has not run yet, so it
+ *   would record the OLD value as the new attributes — the one honesty mechanism in this package
+ *   turned into a lie generator. Per-flag, the read-back means what it says.
+ *   ★ THE SECOND REASON IS SMALLER AND STILL REAL. The bulk GET answers rows of
+ *     `{name, description, value}`, and `description` is PVE's own English prose that no write
+ *     accepts: compare it and you have a forever-update, report it and this provider is pretending
+ *     to manage prose. `GET /cluster/ceph/flags/{flag}` answers a BARE BOOLEAN, so that trap
+ *     cannot be written here at all. The brief's plural `Proxmox.CephFlags` became singular for
+ *     those two reasons; the bulk endpoint is read by nothing in this package.
+ *
+ * ⛔ THESE ARE NOT CONFIGURATION. THEY ARE A HAND ON THE BRAKE, AND A DECLARATION CAN PULL IT OFF.
+ *   `noout` is what a person sets before pulling a disk. A stack saying `value: false` reasserts
+ *   that off on EVERY deploy — including a deploy somebody else runs, for an unrelated resource,
+ *   twenty minutes into a maintenance window — and the plan line reads `Proxmox.CephFlag noout
+ *   update`, which nobody reads as "ceph is about to start rebalancing 128 PGs across three nodes
+ *   while a disk is out of the chassis".
+ *   ★ SO `value` IS REQUIRED AND HAS NO DEFAULT, AND THAT IS THE MITIGATION. An optional `value`
+ *     defaulting to false would make `CephFlag('noout', { flag: 'noout', target })` mean "clear
+ *     it" — the dangerous direction, chosen by OMISSION. Here the dangerous sentence has to be
+ *     typed out by a person. Do not add a default, and do not declare a flag merely to document
+ *     that it is off: MEASURED 2026-09-13, all eleven flags on TB4 read 0, so eleven
+ *     `value: false` lines would plan green forever and do nothing except take the brake off
+ *     whenever somebody happens to apply them.
+ *
+ * ⛔ `delete` DOES NOTHING, ON PURPOSE, AND THE ASYMMETRY OF THE TWO FAILURES IS THE ARGUMENT.
+ *   `flags/{flag}` has exactly two methods, GET and PUT, so the factory's `destroy` would answer
+ *   "Method 'DELETE /cluster/ceph/flags/noout' not implemented" on every teardown — whatever this
+ *   handler does is INVENTED. The obvious invention, "clear the flag", is refused: removing a line
+ *   from a stack file, or Alchemy collecting an old generation after a replace, would silently
+ *   take the brake off and nothing would warn. Leaving a flag set that nothing declares any more
+ *   is the opposite kind of failure — ceph reports `HEALTH_WARN ... flag(s) set` for every one of
+ *   these, on `ceph -s`, in the PVE UI and in the `pve_*` metrics this estate already scrapes.
+ *   A silent destructive failure loses to a loud inert one.
+ *   ⚠️ THE RESIDUAL RISK, NAMED RATHER THAN DENIED: deleting a `value: true` declaration LEAVES
+ *     THE FLAG SET. Retire a maintenance window by flipping the line to `value: false` and
+ *     deploying BEFORE the line is removed, or a `noout` outlives the window that needed it.
+ *   ⚠️ REASONED, NOT MEASURED: that health warning was not observed, because observing it means
+ *     setting a flag on a live cluster and this file was researched read-only. Undeclaring stops
+ *     MANAGING a flag; to clear one, declare `value: false` and deploy, where a plan says so.
+ *
+ * ⛔ `pause` IS TWO CEPH FLAGS AND THIS RESOURCE READS ONLY ONE OF THEM. MEASURED in
+ *   `PVE::Ceph::Tools::get_real_flag_name`, whose own comment reads "the 'pause' flag gets always
+ *   set to both 'pauserd' and 'pausewr'": PVE writes both and then decides the flag is set by
+ *   looking at `pauserd` alone. A cluster where somebody ran `ceph osd unset pauserd` by hand
+ *   still has `pausewr` — writes blocked, guests hung — and this resource reports `value: false`
+ *   and plans `noop` straight past it. It cannot express a half-pause, so never use it to prove
+ *   one is gone; `ceph osd dump | head -1` can.
+ *   ⛔ `pause` AND `noup` ARE ALSO THE TWO THAT TAKE THE CLUSTER DOWN FROM A TYPO. `pause: true`
+ *     stops all reads and writes, so every guest on `cephtb4` — the cluster's only rbd pool,
+ *     MEASURED 2026-09-13 — freezes on its root disk, and so does `cephfs-tb4`. `noup: true`
+ *     stops a rebooted OSD from ever rejoining. Neither is undone by deleting the line.
+ *
+ * ⚠️ PRIVILEGES, AND THE WRITE ONE IS A BIG ASK. Both GETs check `Sys.Audit` on `/`; both PUTs
+ *   check `Sys.Modify` on `/`, and all four carry `allowtoken => 1`, so an API token may make the
+ *   call at all. There is no `/ceph` ACL object to scope to — the check names the ROOT path
+ *   literally — so granting the provision role `Sys.Modify` here also buys datacenter options,
+ *   metric servers, notification targets and every other cluster-wide write, exactly as
+ *   metric-server.ts warns about its own family.
+ *   ⚠️ REASONED, NOT MEASURED: nothing here was exercised under a minted lease, because a mint
+ *     writes an API token into the cluster and the brief for this file was reads only. Expect
+ *     "Permission check failed (/, Sys.Modify)" on the first deploy, and widen deliberately.
+ */
+import { Resource } from 'alchemy';
+import * as Provider from 'alchemy/Provider';
+import * as Effect from 'effect/Effect';
+import { type PveRequirements, type WithTarget, pveHandlers } from './resource.ts';
+import { bool, flag } from './values.ts';
+
+/**
+ * PVE's eleven flags, spelled as `PVE::Ceph::Tools::get_possible_osd_flags` spells them.
+ *
+ * ⚠️ THE ENUM IS CLOSED AND A TYPO IS A 400, WHICH IS WHY THIS IS A UNION AND NOT `string`. Both
+ *   the GET and the PUT declare `additionalProperties => 0` over exactly this list.
+ * ⚠️ CEPH HAS FLAGS PVE DOES NOT MODEL, and four of them are always on. MEASURED on TB4:
+ *   `ceph osd dump` reports `flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit`.
+ *   PVE reads that same string and answers only about its own eleven, so the others can neither
+ *   leak in here nor be set from here — `noautoscale` and `nosnaptrim` included.
+ *   ⛔ IF SOMEBODY LATER "IMPROVES" THE READ BY PARSING `osd dump` DIRECTLY, those four become
+ *     permanently-set flags that nothing declares: a forever-diff on a brand new cluster.
+ */
+export type CephFlagName =
+  | 'nobackfill'
+  | 'nodeep-scrub'
+  | 'nodown'
+  | 'noin'
+  | 'noout'
+  | 'norebalance'
+  | 'norecover'
+  | 'noscrub'
+  | 'notieragent'
+  | 'noup'
+  | 'pause';
+
+export interface CephFlagProps extends WithTarget {
+  /**
+   * Which flag. PVE's primary key here, and the last segment of the path.
+   *
+   * ⚠️ EDITING IT IN PLACE ORPHANS THE OLD FLAG RATHER THAN MOVING ANYTHING. `diff` reads the NEW
+   *   path and reconcile writes it, while the old flag keeps whatever this stack last put there.
+   *   There is deliberately no replace override for it — acl.ts needs one because its `delete`
+   *   removes a real grant, and `delete` here is inert by design, so a replace would do exactly
+   *   what an update already does. Declare a second resource and set the old one to `value: false`
+   *   rather than renaming this one.
+   * ⛔ TWO RESOURCES DECLARING THE SAME FLAG ARE ONE CLUSTER OBJECT, and Alchemy sees two ids
+   *   rather than a collision — the acl.ts hazard exactly. Disagreeing, they take turns winning
+   *   and BOTH plan `update` for ever; agreeing, deleting either leaves the flag where the other
+   *   put it. One declaration per flag per cluster.
+   */
+  flag: CephFlagName;
+  /**
+   * Set the flag (`true`) or clear it (`false`).
+   *
+   * ⛔ REQUIRED, WITH NO DEFAULT — see the second ⛔ in the header, which is the whole safety
+   *   argument for this family. It is also the only field PVE accepts on this endpoint, and it is
+   *   not optional in the schema either: omitting it from the form is a 400, not an untouched
+   *   flag. (The BULK endpoint is the one where omission means "leave it alone"; this is not it.)
+   */
+  value: boolean;
+}
+
+export interface CephFlagAttributes {
+  /**
+   * ⚠️ REPORTED, NEVER COMPARED. It is the path key the read was made WITH, copied back out of
+   *   props, so comparing it against props would be true by construction — the same reasoning
+   *   acl.ts gives for the four identity fields it also declines to diff.
+   */
+  flag: CephFlagName;
+  /** Whether ceph has the flag set right now. The only field `matches` looks at. */
+  value: boolean;
+}
+
+export interface ProxmoxCephFlag extends Resource<
+  'Proxmox.CephFlag',
+  CephFlagProps,
+  CephFlagAttributes,
+  never,
+  PveRequirements
+> {}
+
+export const ProxmoxCephFlag = Resource<ProxmoxCephFlag>('Proxmox.CephFlag');
+
+const handlers = pveHandlers<CephFlagProps, CephFlagAttributes>({
+  /**
+   * ⚠️ `live` IS A BARE BOOLEAN HERE, NOT AN OBJECT, AND IT IS HANDED TO `bool` WHOLE ON PURPOSE.
+   *   `get_flag` returns perl `1` or `0` under a `type => 'boolean'` return schema, so the wire is
+   *   `{"data":0}` or `{"data":false}` depending on how the REST layer renders it. `bool` accepts
+   *   every one of those spellings, which is exactly why it is a shared coercion. The factory
+   *   types this parameter `Record<string, unknown>` because every OTHER PVE read is an object;
+   *   reaching for `live['value']` here would answer undefined on every plan and diff `false`
+   *   forever against a set flag.
+   *   ⚠️ THE SCALAR IS A MEASURED PROPERTY OF THIS PVE, NOT A PROMISE. A later version wrapping
+   *     the answer in an object would make `bool` read a SET flag as false, and `value: false`
+   *     would then plan `noop` over a live flag — silently. Re-measure
+   *     `GET /cluster/ceph/flags/noout` after a major upgrade; cheaper than a second parse path.
+   *
+   * ⛔ IT NEVER ANSWERS undefined, WHICH MAKES THE CREATE BRANCH UNREACHABLE, AND ITS ERROR WILL
+   *   MISLEAD YOU. All eleven flags always exist; there is nothing for "absent" to mean. So a POST
+   *   to `collection` means the READ failed — an expired 300s lease, n2 down, or ceph simply not
+   *   configured on the cluster (every one of these four handlers opens with
+   *   `check_ceph_configured()`, which dies). Read the resulting "Method 'POST /cluster/ceph/flags'
+   *   not implemented" as "the read failed" and go and look at the credential, not at ceph. Same
+   *   shape as the unreachable POST in acl.ts, same advice.
+   */
+  attributes: (live, props) => ({ flag: props.flag, value: bool(live) }),
+  collection: () => 'cluster/ceph/flags',
+  /**
+   * ⛔ UNREACHABLE — see the second ⛔ in `attributes`. It is written in the BULK endpoint's shape
+   *   (the flag name is the parameter, not `value`) rather than left empty, because that is the
+   *   only body `cluster/ceph/flags` has ever documented. Nothing sends it today.
+   */
+  createForm: (props) => ({ [props.flag]: flag(props.value) ?? '0' }),
+  /**
+   * ⚠️ `value` IS THE ONLY THING COMPARED, AND THE LIST OF THINGS DELIBERATELY NOT COMPARED IS THE
+   *   point of this resource. `flag` is props-derived (see `CephFlagAttributes`). `description`
+   *   and `name` exist only on the bulk GET, which this file does not read, so neither can be
+   *   compared by accident. Nothing PVE returns here is rewritten, re-ordered or re-typed by the
+   *   cluster, because all PVE returns here is one boolean.
+   *   ★ MEASURED: on TB4, `GET /cluster/ceph/flags` answers value 0 for all eleven flags, so a
+   *     declaration of `value: false` reads back false and plans `noop`.
+   */
+  matches: (attributes, props) => attributes.value === props.value,
+  path: (props) => `cluster/ceph/flags/${props.flag}`,
+  /**
+   * ⚠️ ONLY `value` GOES IN THE BODY. `flag` is already bound by the path and the PUT declares
+   *   `additionalProperties => 0`, so a second copy in the form could only disagree with the URL.
+   * ⚠️ THE `?? '0'` IS UNREACHABLE, AND IS WRITTEN RATHER THAN CAST AWAY. `flag()` types its
+   *   answer `string | undefined` because it serves OPTIONAL props, and `value` here is required,
+   *   so the branch cannot be taken; a cast would claim a proof the type system has not made.
+   */
+  updateForm: (props) => ({ value: flag(props.value) ?? '0' }),
+});
+
+export const ProxmoxCephFlagProvider = () =>
+  Provider.effect(
+    ProxmoxCephFlag,
+    Effect.succeed(
+      ProxmoxCephFlag.Provider.of({
+        /**
+         * ⚠️ NOT THE ONE-LINE `Provider.of(handlers)` EVERY OTHER FILE HERE USES, AND THE TWO
+         *   REASONS ARE BOTH acl.ts's. PVE implements no DELETE on this path, and the factory's
+         *   read-back guard is dead whenever `attributes` cannot answer undefined. Unlike acl.ts
+         *   this SPREADS the factory's handlers instead of retyping them, so `list`, `read` and
+         *   `diff` cannot drift from the shared ones — only the two that genuinely differ are
+         *   written out below.
+         */
+        ...handlers,
+        /** ⛔ Inert, deliberately. The third ⛔ in the header is the whole argument; read it. */
+        delete: () => Effect.void,
+        /**
+         * ⚠️ THE READ-BACK GUARD, RESTORED FOR A FAMILY WHOSE "ABSENT" DOES NOT EXIST.
+         *   `ops.reconcile` refuses when the object is still missing after a write, and a flag is
+         *   never missing, so that check can never fire here — exactly the hole acl.ts patches
+         *   with `bound`. PVE answers 200 with `{"data":null}` on calls that did nothing, and this
+         *   PUT's documented return IS null, so the status code carries no evidence at all.
+         * ⚠️ THE PER-FLAG PUT IS SYNCHRONOUS (`$rados->mon_command` inline, MEASURED in Ceph.pm),
+         *   which is what makes this check fair rather than a race against a worker task. If it
+         *   ever fires spuriously the honest fix is to look at the mon, not to delete the guard.
+         */
+        reconcile: Effect.fn(function* ({ news }) {
+          const after = yield* handlers.reconcile({ news });
+          if (after.value !== news.value) {
+            return yield* Effect.die(
+              new Error(
+                `cluster/ceph/flags/${news.flag}: the PUT returned no error but ceph still ` +
+                  `reports the flag ${after.value ? 'set' : 'clear'} rather than ` +
+                  `${news.value ? 'set' : 'clear'}. This endpoint's success is documented as a ` +
+                  'null body, so the status code proves nothing -- check `ceph osd dump | ' +
+                  'head -1` and the cluster log. `pause` in particular is two ceph flags read ' +
+                  'as one; see the header.',
+              ),
+            );
+          }
+          return after;
+        }),
+      }),
+    ),
+  );

package/src/proxmox/ceph-fs-wire.ts

@@ -0,0 +1,246 @@
+/**
+ * The wire lane of `Proxmox.CephFs`: how a PVE answer becomes attributes, how a declaration becomes
+ * a PVE call, and how that call becomes a fact.
+ *
+ * ★ SPLIT OUT OF ceph-fs.ts TO KEEP BOTH FILES UNDER THE 250-LINE CAP, and the seam is a real one
+ *   rather than a convenient line number. ceph-fs.ts answers "what is a CephFS, and when has it
+ *   changed"; this file answers "what does PVE say, what does it want, and has it finished". On
+ *   this family that is where all the strangeness lives — the read is an index and not an object,
+ *   the paths are inverted, the delete takes its arguments in the query string, and both writes
+ *   fork a worker — so keeping it in one place is what stops the next reader assuming the ordinary
+ *   shape from the other nineteen resources. ⚠️ NOTHING HERE DECIDES A DIFF.
+ *
+ * ⚠️ THE `import type` BACK TO ceph-fs.ts IS A CYCLE ON PAPER ONLY. It is type-only and erased
+ *   before anything runs, exactly as metric-server-form.ts imports `MetricServerProps`. The props
+ *   stay the resource's public shape, declared in the file that declares the resource.
+ *
+ * ⛔ POST AND DELETE ANSWER WITH A UPID, NOT A RESULT — THE WRITE IS ASYNCHRONOUS AND A FAILED ONE
+ *   IS STILL HTTP 200. Read from the cluster's own source, /usr/share/perl5/PVE/API2/Ceph/FS.pm on
+ *   n2: `createfs` and `destroyfs` both end `return $rpcenv->fork_worker(...)`. Two different lies
+ *   come out of trusting that status code. A read-back straight after the POST reports "still
+ *   absent" about a filesystem that is being built as it says so — which is precisely the message
+ *   `pveOperations.reconcile` would print, pointing at the wrong cause. And a worker that dies
+ *   INSIDE — the pool create, or the add-storage step, both of which are wrapped in `eval` — is
+ *   never reported to the caller at all. `settle` below is the answer to both.
+ *   ⚠️ REASONED FROM THAT SOURCE, NOT MEASURED. Nothing was written to this cluster, so the race
+ *     has not been observed; only the `fork_worker` that guarantees it is there.
+ *
+ * ⛔ A DELETE BODY IS SILENTLY DISCARDED BY PVE, SO `remove-pools` GOES IN THE QUERY STRING.
+ *   MEASURED in /usr/share/perl5/PVE/APIServer/AnyEvent.pm on n2: line 928 reads the request
+ *   content into params only `if ($method eq 'PUT' || $method eq 'POST')`, and line 1653 sends
+ *   every other method down a branch that parses `$request->url->query()` and nothing else. So
+ *   `pve(target, role, 'DELETE', path, form)` would send a body PVE never looks at: the flags would
+ *   read as unset, the pools would quietly survive a delete that asked for them, and no error would
+ *   be raised anywhere. sdn-apply.ts already puts `?pending=1` in the path for the same reason.
+ *
+ * ⚠️ THE TASK POLL RUNS ON THE `read` ROLE THOUGH IT SITS INSIDE reconcile, FOR TWO REASONS. It is
+ *   a read, and resource.ts already mints `read` for its own read-back inside reconcile. And the
+ *   leases differ where it matters: `provision` is 300s and explicitly non-renewable, so a create
+ *   that waits out a slow worker on that lease can expire mid-poll, while `read` is 3600s.
+ *
+ * ⛔ THE POLLER IS NEVER THE TASK'S OWNER, WHICH COSTS A PRIVILEGE, AND THIS ONE IS MEASURED
+ *   RATHER THAN FEARED. client.ts mints a fresh credential per call and the mount vends a NEW
+ *   token id each time, so the token reading the status is never the token recorded in the UPID —
+ *   and PVE compares them exactly. /usr/share/perl5/PVE/API2/Tasks.pm on n2, `$check_task_user`:
+ *   `return $user eq $fulltoken || $user eq $task->{user};`, above it the comment "token only sees
+ *   token tasks, user sees user + token tasks". Token B of the same user matches neither branch.
+ *   So the fallback is the schema's other clause for `GET /nodes/{node}/tasks/{upid}/status`: "The
+ *   user needs 'Sys.Audit' permissions on '/nodes/<node>' if they are not the owner of the task."
+ *   Grant it on the `read` role, or every create burns the full cap below and reports the wrong
+ *   cause.
+ */
+import * as Effect from 'effect/Effect';
+import type { CephFsAttributes, CephFsProps } from './ceph-fs.ts';
+import { pve } from './client.ts';
+import type { PveTarget } from './credentials.ts';
+import { csv, flag, int, text } from './values.ts';
+
+/**
+ * This filesystem's row of the index, as attributes.
+ *
+ * ⚠️ `GET /nodes/{node}/ceph/fs` ANSWERS AN ARRAY, while the factory hands `attributes` the
+ *   `Record<string, unknown>` every other PVE read is shaped like. Rows are narrowed rather than
+ *   trusted, the same way acl.ts narrows its one flat grant list.
+ *
+ * ⛔ RETURNING undefined IS WHAT MAKES "ABSENT" MEAN ABSENT HERE, unlike acl.ts where the object is
+ *   always present-but-unbound. A filesystem missing from the index has not been created, and
+ *   `reconcile` takes the create branch on exactly that signal.
+ */
+export const readRow = (
+  live: Record<string, unknown>,
+  props: CephFsProps,
+): CephFsAttributes | undefined => {
+  const row = (Array.isArray(live) ? live : [])
+    .filter(
+      (entry): entry is Record<string, unknown> => typeof entry === 'object' && entry !== null,
+    )
+    .find((entry) => entry['name'] === props.name);
+  if (row === undefined) return undefined;
+  return {
+    data_pool: text(row['data_pool']),
+    data_pools: csv(
+      Array.isArray(row['data_pools'])
+        ? row['data_pools'].map((entry: unknown) => String(entry))
+        : [],
+    ),
+    metadata_pool: text(row['metadata_pool']),
+    metadata_pool_id: int(row['metadata_pool_id'], -1),
+    name: props.name,
+  };
+};
+
+/** ⚠️ `exitstatus` is ABSENT while a task runs, and is the exact string `OK` only on success. */
+type TaskStatus = { readonly status?: string; readonly exitstatus?: string };
+
+/**
+ * ⚠️ SHORT POLLS, LOW CAP. Ninety seconds covers two pool creates plus the ten seconds `createfs`
+ *   itself spends waiting for an MDS to go active, and is short enough that a stuck task fails the
+ *   deploy rather than parking it for the afternoon.
+ */
+const POLL_SECONDS = 2;
+const POLL_ATTEMPTS = 45;
+
+/** ⚠️ `nodes/{n}/ceph/fs/{name}` — the POST and DELETE target, and NOT where a read goes. */
+export const objectPath = (props: CephFsProps) => `nodes/${props.node}/ceph/fs/${props.name}`;
+
+/** Only the declared flags. ⚠️ An omitted one is PVE's own default, not a value to send. */
+const field = (name: string, value: string | undefined): Record<string, string> =>
+  value === undefined ? {} : { [name]: value };
+
+/**
+ * ⚠️ `name` IS DELIBERATELY ABSENT FROM THE FORM. It is already the last segment of the path being
+ *   POSTed to — the metric-server.ts case again — and a second copy can only ever disagree with it.
+ */
+export const createForm = (props: CephFsProps): Record<string, string> => ({
+  ...field('add-storage', flag(props['add-storage'])),
+  ...field('pg_num', props.pg_num === undefined ? undefined : String(props.pg_num)),
+});
+
+/** The delete target with its flags in the query string — see the ⛔ in the header. */
+const destroyPath = (props: CephFsProps) => {
+  const query = [
+    props['remove-pools'] === true ? 'remove-pools=1' : undefined,
+    props['remove-storages'] === true ? 'remove-storages=1' : undefined,
+  ].filter((part) => part !== undefined);
+  return query.length === 0 ? objectPath(props) : `${objectPath(props)}?${query.join('&')}`;
+};
+
+/**
+ * Wait for a forked PVE task, and fail loudly rather than quietly.
+ *
+ * ⚠️ THE UPID IS PERCENT-ENCODED BECAUSE IT IS FULL OF COLONS — `UPID:n2:00396D3A:…:root@pam:`
+ *   (MEASURED shape, from `GET /nodes/n2/tasks`) is ONE path segment, not seven. Encoding is what
+ *   guarantees it arrives as one; PVE's router decodes each segment before matching. REASONED from
+ *   the URI handling rather than measured — no task of this provider's has been polled yet.
+ */
+const settle = (target: PveTarget, node: string, upid: string, what: string) =>
+  Effect.gen(function* () {
+    for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt += 1) {
+      const status = yield* pve<TaskStatus>(
+        target,
+        'read',
+        'GET',
+        `nodes/${node}/tasks/${encodeURIComponent(upid)}/status`,
+      ).pipe(
+        // ⚠️ AN UNREADABLE STATUS IS "NOT SETTLED YET", NOT A FAILURE. It is a missing Sys.Audit
+        //   (see the header) or a node mid-restart; either way the task is still the cluster's
+        //   business, and the cap is what ends the wait rather than one unlucky GET.
+        Effect.orElseSucceed(() => undefined),
+      );
+      if (status?.status === 'stopped') {
+        const exit = text(status.exitstatus);
+        if (exit === 'OK') return;
+        return yield* Effect.die(
+          new Error(
+            `${what}: PVE task ${upid} finished with "${exit === '' ? 'no exit status' : exit}". ` +
+              'The write returned HTTP 200 because it only forked the worker -- the real error is ' +
+              `in \`pvesh get /nodes/${node}/tasks/${upid}/log\`.`,
+          ),
+        );
+      }
+      // ⚠️ A BARE NUMBER IS MILLISECONDS to Effect's Duration, not seconds. The cap in the
+      //   message below is derived from the same constant so the two cannot drift apart.
+      yield* Effect.sleep(POLL_SECONDS * 1000);
+    }
+    return yield* Effect.die(
+      new Error(
+        `${what}: PVE task ${upid} was still running after ` +
+          `${String(POLL_ATTEMPTS * POLL_SECONDS)}s, or its ` +
+          'status could not be read. Check the task log, and check that the `read` role holds ' +
+          `Sys.Audit on /nodes/${node} -- each call mints a new token, so the poller is never the ` +
+          'task owner and is answered 403 rather than told to wait.',
+      ),
+    );
+  });
+
+/**
+ * POST the create, then wait for the worker. ⛔ The caller still reads back; see ceph-fs.ts.
+ *
+ * ⚠️ A CREATE NEEDS A RUNNING *AND* A STANDBY MDS, and refuses BEFORE it forks -- "no running
+ *   Metadata Server (MDS) found!" / "no standby Metadata Server (MDS) found!" are synchronous, so
+ *   they surface as a failed POST rather than as a silent worker. TB4 has three (MEASURED: n3
+ *   `up:active` for cephfs-tb4, n2 and n4 `up:standby`), so this is a note for a smaller cluster.
+ * ⚠️ BOTH WRITES CHECK `Sys.Modify` ON `/` -- the root, with the breadth metric-server.ts warns
+ *   about: granting it buys datacenter options and every other cluster-wide config write too.
+ */
+export const createFs = (props: CephFsProps) =>
+  Effect.gen(function* () {
+    const upid = text(
+      yield* pve<string>(props.target, 'provision', 'POST', objectPath(props), createForm(props)),
+    );
+    // ⛔ NO UPID MEANS NO WORKER. PVE answers 200 with `{"data":null}` on calls that did nothing,
+    //   and here that is indistinguishable from success by status code alone. Refusing now is the
+    //   difference between a clear failure and a state entry for a filesystem nobody built.
+    if (upid === '') {
+      return yield* Effect.die(
+        new Error(
+          `${objectPath(props)}: POST returned no task id. PVE wraps every answer in ` +
+            '{"data":...} and can report success on a call that did nothing.',
+        ),
+      );
+    }
+    yield* settle(props.target, props.node, upid, objectPath(props));
+  });
+
+/**
+ * DELETE the filesystem, then wait for the worker.
+ *
+ * ⛔ WHAT THIS TAKES AWAY DEPENDS ENTIRELY ON `remove-pools`, AND THE DEFAULT IS THE SAFE ONE. With
+ *   it unset PVE removes the filesystem from the MDS map and LEAVES the two pools on disk: every
+ *   mount breaks, but the bytes are still there for an operator to recover. With it set the pools
+ *   go too and nothing is recoverable. Leaving it unset also means a later create of the SAME name
+ *   fails with "ceph pools '<name>_data' and/or '<name>_metadata' already exist" -- that refusal is
+ *   a BRAKE, not a bug, and it is the last thing standing between a mis-typed rename and the data.
+ */
+export const destroyFs = (props: CephFsProps) =>
+  Effect.gen(function* () {
+    const upid = text(yield* pve<string>(props.target, 'provision', 'DELETE', destroyPath(props)));
+    // ⚠️ A DESTROY THAT ANSWERS WITHOUT A UPID IS NOT REFUSED HERE. `destroyfs` dies synchronously
+    //   with "no such cephfs" when the filesystem is already gone, which surfaces as a failed
+    //   DELETE; an empty answer that is not an error is best treated as already-absent, and the
+    //   caller's read-back is what proves it either way.
+    if (upid !== '') yield* settle(props.target, props.node, upid, destroyPath(props));
+  });
+
+/**
+ * The two read-back failures, as messages rather than as guesses.
+ *
+ * ⛔ WAITING PROVES THE TASK ENDED; ONLY A READ PROVES THE FILESYSTEM DID OR DID NOT. `createfs`
+ *   wraps its pool creates in an `eval` and CLEANS THEM UP on error before dying, so a worker can
+ *   plausibly end having built and then removed everything it made. `destroyfs` refuses outright
+ *   while a non-disabled `cephfs` storage still references the filesystem. Neither outcome is
+ *   visible in a status code, which is why ceph-fs.ts reads back after both and dies with these.
+ */
+export const notCreated = (props: CephFsProps) =>
+  new Error(
+    `${objectPath(props)}: the create task finished but the filesystem is not in ` +
+      `GET nodes/${props.node}/ceph/fs. Check the task log -- a PVE worker can exit having ` +
+      'rolled back the pools it had just made.',
+  );
+
+export const notDestroyed = (props: CephFsProps) =>
+  new Error(
+    `${objectPath(props)}: the destroy task finished but the filesystem is still in ` +
+      `GET nodes/${props.node}/ceph/fs. PVE refuses to destroy a CephFS while a PVE storage entry ` +
+      'of type cephfs still references it and is not disabled.',
+  );