@homeflare/alchemy 0.1.0

package/src/proxmox/ha-rule.ts

@@ -0,0 +1,240 @@
+/**
+ * `Proxmox.HaRule` — where the CRM may place a guest, and which guests must not share a node.
+ *
+ * ★ THIS IS THE PVE 9 SUCCESSOR TO HA GROUPS, AND THE GROUPS ENDPOINT IS GONE. MEASURED on n2
+ *   (pve-manager/9.2.11, 2026-09-13): `pvesh get /cluster/ha/groups` answers `cannot index groups:
+ *   ha groups have been migrated to rules`, and `/etc/pve/ha/` holds `rules.cfg` with no
+ *   `groups.cfg` beside it. The published schema still documents groups and still offers a `group`
+ *   parameter on `Proxmox.HaResource`; both are dead. Do not port them back.
+ *
+ * ★ ONE RESOURCE FOR BOTH RULE TYPES, ON THE ceph-daemon.ts ARGUMENT AND AFTER READING BOTH
+ *   PLUGINS. `PVE::HA::Rules::NodeAffinity` and `::ResourceAffinity` share `resources`, `affinity`,
+ *   `disable` and `comment`; node-affinity adds `nodes` and `strict` and nothing else diverges. The
+ *   endpoint, the path, the create/update/delete calls, the `order` and `digest` traps below and
+ *   the list normalisation are identical for both. Two files would have carried every one of those
+ *   traps twice and let them drift.
+ *   ⛔ `type` IS CREATE-ONLY. MEASURED in `/usr/share/perl5/PVE/API2/HA/Rules.pm`: `update_rule`
+ *     takes the plugin from the STORED rule (`my $type = $rule->{type};`) and never rewrites it. So
+ *     `attributes` reports a rule of the other type as ABSENT, which sends reconcile down the
+ *     create branch and makes PVE refuse with `HA rule '<id>' already defined` — a loud, accurate
+ *     error rather than a PUT pushing node-affinity fields at a resource-affinity rule.
+ *     ⛔ AND YET IT IS SENT ON THE UPDATE ANYWAY, because the PUT is validated against a `oneOf`
+ *       keyed on it and refuses the whole body without it. That one is worth reading in full: it
+ *       is the ⛔ on `required` in ha-rule-form.ts.
+ *
+ * ⛔ THREE FIELDS PVE RETURNS AND NO WRITE ACCEPTS, EACH ONE A FOREVER-UPDATE IF COMPARED.
+ *   `order` is an ORDINAL PVE ASSIGNS ITSELF (`$rules->{order}->{$ruleid} =
+ *   PVE::HA::Rules::get_next_ordinal($rules)`) and it appears in no create or update schema at all;
+ *   it is reported in attributes and never compared. `digest` is the digest of the WHOLE rules
+ *   file, so it moves when a rule written for somebody else's guests moves — it is not even an
+ *   attribute, for the reason ha-resource.ts gives. `errors` is PVE's own feasibility verdict,
+ *   recomputed across every rule on every read; reported, never compared. See `problems` in
+ *   ha-rule-form.ts.
+ *
+ * ⚠️ IT COMPOSES WITH `Proxmox.HaResource`, AND THE EDGE IS REAL RATHER THAN TIDY. MEASURED:
+ *   `create_rule` runs `$assert_valid_resources_param`, which calls
+ *   `PVE::HA::Config::service_is_configured($resource)` for every sid and dies `cannot use
+ *   unmanaged resource(s) <sid>` if one is not already an HA resource. A rule naming a sid nothing
+ *   has put under HA is a HARD 400, not a warning — so `resources` should be written as the
+ *   HaResource's own attribute (`resources: [web.sid, db.sid]`) and let Alchemy order the deploy,
+ *   not as a bare string that happens to match. `$assert_valid_nodes_param` refuses a non-existent
+ *   node the same way.
+ *   ⛔ AND THE EDGE RUNS BACK THE OTHER WAY, WHERE ALCHEMY CANNOT SEE IT. Destroying an
+ *     `HaResource` deletes with PVE's default `purge=1`, which strips that sid out of every rule
+ *     and DELETES any rule left with no members. So a plan that only removes a guest can remove
+ *     this resource's object from under it; the next plan then reads absent and recreates it, or
+ *     fails on the `unmanaged resource` check if the guest is gone for good.
+ *
+ * ⚠️ BOTH LANES ALREADY HAVE THE PRIVILEGES, WHICH IS NEW. MEASURED 2026-09-13:
+ *   GET (collection and item) checks `Sys.Audit` on `/` and the built-in `PVEAuditor` holds it, so
+ *   `readRole` stays the default `read` lease — unlike storage.ts and the SDN families. POST, PUT
+ *   and DELETE all check `Sys.Console` on `/`, and `pvesh get /access/roles/LXCProvisioner` now
+ *   returns `Sys.Console` among its 27 privileges. The warning in ha-resource.ts's header — that
+ *   `Sys.Console` is missing — is stale; the role was widened, and the price it named still stands:
+ *   `Sys.Console` is also what opens a root shell on every node.
+ *
+ * ⚠️ REMOVAL IS THE PLAIN DEFAULT, DELIBERATELY, AND `delete` IS THE FACTORY'S. An HA rule is
+ *   exactly a line of TypeScript — nothing in it is irreplaceable the way a Ceph pool or a ZFS
+ *   vdev is — so the `retain` default that resource.ts explains does not apply. Read the plan for
+ *   what it is, though: while a negative resource-affinity rule is absent, nothing stops the CRM
+ *   putting both halves of a pair on one node.
+ *
+ * ⚠️ A READ GOES THROUGH THE GROUPS MIGRATION. Every GET calls `migrate_groups_to_rules`, so on a
+ *   cluster that still has `groups.cfg` the answer includes SYNTHETIC rules nobody declared. This
+ *   cluster has none, so nothing is injected here; a rule id colliding with an old group name on a
+ *   cluster that does would read back something this provider never wrote.
+ *
+ * ⚠️ A CHANGED `type` READS AS `update` IN THE PLAN AND FAILS AS "already defined" ON THE APPLY,
+ *   AND THOSE TWO WORDS POINT AWAY FROM THE CAUSE. `attributes` answers undefined when the live
+ *   rule's type disagrees, so `diff` takes resource.ts's `live === undefined` branch and says
+ *   update; reconcile then reads undefined, POSTs, and PVE refuses because the rule is plainly
+ *   there. The cause is that `type` is create-only. metric-server.ts has the same shape, so this
+ *   is the package's consistent behaviour rather than a quirk of this file — but it is worth
+ *   knowing before reading the error. Delete the declaration and write a new one to change a type.
+ */
+import { Resource } from 'alchemy';
+import * as Provider from 'alchemy/Provider';
+import * as Effect from 'effect/Effect';
+import { body, clearList, commentText, nodeList, problems } from './ha-rule-form.ts';
+import { type PveRequirements, type WithTarget, pveHandlers } from './resource.ts';
+import { bool, csv, int, text, withClears } from './values.ts';
+
+export type HaRuleType = 'node-affinity' | 'resource-affinity';
+
+/**
+ * ⛔ REQUIRED ON BOTH TYPES, AND THAT IS A DELIBERATE NARROWING OF PVE'S SCHEMA. PVE makes it
+ *   optional for node-affinity (defaulting to `positive`) and mandatory for resource-affinity. A
+ *   provider that mirrored that has two bad options when a declaration omits it: guess `positive`,
+ *   which silently flips a live `negative` rule — the one keeping two guests APART — into one that
+ *   pins them together; or leave it out of the write, which makes `matches` disagree with a live
+ *   `negative` rule forever while every PUT changes nothing. Requiring one word from the caller
+ *   removes both.
+ */
+export type HaAffinity = 'positive' | 'negative';
+
+interface HaRuleBase extends WithTarget {
+  /** A `pve-configid`: a letter, then letters, digits, `-` and `_`. Identity — renaming is a new
+   *  rule, and the old one stays behind exactly as `sid` does in ha-resource.ts. */
+  rule: string;
+  /**
+   * The HA resources this rule constrains, as prefixed sids — `ct:101`, `vm:100`.
+   * ⚠️ EVERY ONE MUST ALREADY BE UNDER HA. See the composition ⚠️ in the header: this is an
+   *   ordering edge, and the honest way to express it is the HaResource's own `sid` attribute.
+   */
+  resources: readonly string[] | string;
+  affinity: HaAffinity;
+  /** Keep the rule in the file but stop the CRM applying it. ⛔ Clearing it needs `delete=disable`
+   *  — see the ⛔ in `optional` in ha-rule-form.ts. */
+  disable?: boolean;
+  /** Free text shown in the HA panel. Max 4096 characters. */
+  comment?: string;
+}
+
+export interface NodeAffinityRuleProps extends HaRuleBase {
+  type: 'node-affinity';
+  /**
+   * Cluster nodes, each optionally `<node>:<priority>`. Higher priority wins; the numbers are
+   * relative and nothing else. ⚠️ REQUIRED BY THE TYPE, not merely by PVE — an optional `nodes`
+   * would let a declaration that cannot describe any live rule sit in a permanent update loop.
+   */
+  nodes: readonly string[] | string;
+  /** `false` (PVE's default) makes the nodes a PREFERENCE; `true` makes them the only nodes the
+   *  resources may run on, and the resources STOP when none of them is available. */
+  strict?: boolean;
+}
+
+export interface ResourceAffinityRuleProps extends HaRuleBase {
+  type: 'resource-affinity';
+}
+
+/**
+ * ⚠️ A UNION, NOT ONE FLAT INTERFACE WITH OPTIONAL EXTRAS, because `nodes` and `strict` are not
+ *   merely unused on a resource-affinity rule — PVE's oneOf declares `additionalProperties: 0`, so
+ *   sending either is a 400, and comparing either would be a diff no write could ever settle. The
+ *   union makes both unwritable rather than ignored.
+ */
+export type HaRuleProps = NodeAffinityRuleProps | ResourceAffinityRuleProps;
+
+export interface HaRuleAttributes {
+  rule: string;
+  type: HaRuleType;
+  /** ⚠️ NORMALISED, NOT RAW. PVE returns `join(',', sort keys %$hash)` — a re-sorted SET. */
+  resources: string;
+  /** ⚠️ Normalised by `nodeList`; always `''` on a resource-affinity rule, which has no nodes. */
+  nodes: string;
+  affinity: string;
+  strict: boolean;
+  disable: boolean;
+  comment: string;
+  /** ⛔ PVE-ASSIGNED AND UNWRITABLE. Reported so a plan can show it; never compared. */
+  order: number;
+  /** ⛔ PVE's feasibility verdict for this rule. Reported; never compared. */
+  errors: string;
+}
+
+export interface ProxmoxHaRule extends Resource<
+  'Proxmox.HaRule',
+  HaRuleProps,
+  HaRuleAttributes,
+  never,
+  PveRequirements
+> {}
+
+export const ProxmoxHaRule = Resource<ProxmoxHaRule>('Proxmox.HaRule');
+
+const handlers = pveHandlers<HaRuleProps, HaRuleAttributes>({
+  attributes: (live, props) => {
+    /**
+     * ⛔ A RULE OF THE OTHER TYPE IS ANOTHER OBJECT, AND "ABSENT" IS THE HONEST ANSWER — the same
+     *   move metric-server.ts makes, for the same reason: `type` cannot be changed by a PUT, so
+     *   reporting the foreign rule as missing makes reconcile POST and PVE refuse because the id
+     *   is taken. An older PVE that omitted `type` from the read would fall through and be trusted.
+     */
+    const liveType = text(live['type']);
+    if (liveType !== '' && liveType !== props.type) return undefined;
+    return {
+      affinity: text(live['affinity']),
+      comment: text(live['comment']),
+      /**
+       * ⚠️ ABSENT WHEN OFF, AND THAT IS NOT THE SAME AS `false` BY ACCIDENT. MEASURED in
+       *   `PVE::HA::Rules::set_rule_defaults`: a default is materialised into the read only when it
+       *   is TRUTHY, so `disable` (no default) and `strict` (default 0) are simply missing from an
+       *   enabled, non-strict rule while `affinity` (default `positive`) is always present.
+       */
+      disable: bool(live['disable'], false),
+      errors: problems(live['errors']),
+      nodes: nodeList(typeof live['nodes'] === 'string' ? live['nodes'] : undefined),
+      order: int(live['order'], 0),
+      resources: csv(typeof live['resources'] === 'string' ? live['resources'] : undefined),
+      rule: text(live['rule'], props.rule),
+      strict: bool(live['strict'], false),
+      type: props.type,
+    };
+  },
+  collection: () => 'cluster/ha/rules',
+  /** ⚠️ ONLY `rule` IS ADDED HERE: the POST is to the collection, so the id is not in the path.
+   *   `type` is already in `body` — the PUT needs it too, for the reason ha-rule-form.ts gives. */
+  createForm: (props) => ({ ...body(props), rule: props.rule }),
+  /**
+   * ⛔ ONLY FIELDS A WRITE CAN ACTUALLY SET ARE COMPARED. `order`, `errors` and `digest` are out
+   *   for the reason the header gives at length — each would report an update that the PUT it
+   *   triggers cannot satisfy, on every plan, forever. `rule` and `type` are out too: one is the
+   *   key the object was read by, the other is refused rather than updated.
+   * ⚠️ `nodes` AND `strict` ARE COMPARED ONLY ON A NODE-AFFINITY RULE. They do not exist on the
+   *   other plugin, so `attributes.nodes` is `''` and `attributes.strict` is `false` there by
+   *   construction — comparing them would be comparing a declaration against nothing.
+   * ⚠️ THE COMMENT IS COMPARED AS PVE WILL STORE IT, not as it was declared — `commentText` and
+   *   its ⛔ explain why a `%XX` in a comment does not survive the round trip.
+   */
+  matches: (attributes, props) =>
+    attributes.resources === csv(props.resources) &&
+    attributes.affinity === props.affinity &&
+    attributes.comment === commentText(props.comment ?? '') &&
+    attributes.disable === (props.disable === true) &&
+    (props.type !== 'node-affinity' ||
+      (attributes.nodes === nodeList(props.nodes) &&
+        attributes.strict === (props.strict === true))),
+  /** ⚠️ A `pve-configid` has no `/` or `:` in it, so there is nothing here to encode. */
+  path: (props) => `cluster/ha/rules/${props.rule}`,
+  /**
+   * ⛔ THE `delete=` HALF IS WHAT MAKES THIS SETTLE. A PUT merges into the stored rule and a
+   *   `disable=0` is thrown away before the plugin sees it, so without the clear list a rule
+   *   disabled by hand could never be re-enabled by a declaration and every plan would report the
+   *   same update. `clearList` and `body` come off one table — see ha-rule-form.ts.
+   */
+  updateForm: (props) => {
+    const clear = clearList(props);
+    const fields = body(props);
+    return withClears(fields, clear);
+  },
+});
+
+/**
+ * ⛔ THE EMPTY `list` FROM `pveHandlers` STANDS, AND IT COSTS NOTHING HERE TODAY.
+ *   `GET /cluster/ha/rules` MEASURED `[]` on this cluster on 2026-09-13 — but the same call on a
+ *   cluster somebody has used returns every rule an operator ever clicked, plus the synthetic ones
+ *   the groups migration invents on read. Adopting either is how a later `alchemy destroy` removes
+ *   a separation rule nobody declared. Adoption stays an explicit act.
+ */
+export const ProxmoxHaRuleProvider = () =>
+  Provider.effect(ProxmoxHaRule, Effect.succeed(ProxmoxHaRule.Provider.of(handlers)));

package/src/proxmox/index.ts

@@ -0,0 +1,42 @@
+/**
+ * Proxmox providers for Alchemy.
+ *
+ * ⛔ THIS BARREL IS THE PUBLIC API, AND IT IS DELIBERATELY SMALLER THAN THE DIRECTORY.
+ *   These are the symbols a real stack consumes; the rest of the files are internals a
+ *   provider needs but a consumer should not depend on. An `export *` here would publish
+ *   every helper as API and make the next refactor a breaking change.
+ * ★ Anything unlisted is still reachable by path if you genuinely need it — that is a
+ *   deliberate, visible act rather than an accident of barrelling.
+ */
+export { ProxmoxAclProvider } from './acl.ts';
+export { ProxmoxApiTokenProvider } from './api-token.ts';
+export { ProxmoxBackupJobProvider } from './backup-job.ts';
+export { ProxmoxCephDaemonProvider } from './ceph-daemon.ts';
+export { ProxmoxCephFlagProvider } from './ceph-flag.ts';
+export { ProxmoxCephFsProvider } from './ceph-fs.ts';
+export { ProxmoxCephOsdProvider } from './ceph-osd.ts';
+export { ProxmoxCephPool, ProxmoxCephPoolProvider } from './ceph-pool.ts';
+export { ProxmoxFirewallAliasProvider } from './firewall-alias.ts';
+export { ProxmoxGroupProvider } from './group.ts';
+export { ProxmoxHaResourceProvider } from './ha-resource.ts';
+export { ProxmoxHaRuleProvider } from './ha-rule.ts';
+export { ProxmoxLxcProvider } from './lxc.ts';
+export { ProxmoxMetricServerProvider } from './metric-server.ts';
+export { ProxmoxNetworkApplyProvider } from './network-apply.ts';
+export { ProxmoxNodeNetworkProvider } from './node-network.ts';
+export { ProxmoxNotificationTargetProvider } from './notification-target.ts';
+export { PbsDatastoreProvider } from './pbs-datastore.ts';
+export { PbsPruneJobProvider } from './pbs-prune-job.ts';
+export { PbsSyncJobProvider } from './pbs-sync-job.ts';
+export { PbsVerifyJobProvider } from './pbs-verify-job.ts';
+export { ProxmoxPool, ProxmoxPoolProvider } from './pool.ts';
+export { ProxmoxVmProvider } from './qemu.ts';
+export { ProxmoxReplicationJobProvider } from './replication-job.ts';
+export { ProxmoxRoleProvider } from './role.ts';
+export { ProxmoxSdnApplyProvider } from './sdn-apply.ts';
+export { ProxmoxSdnSubnetProvider } from './sdn-subnet.ts';
+export { ProxmoxSdnVnetProvider } from './sdn-vnet.ts';
+export { ProxmoxSdnZoneProvider } from './sdn-zone.ts';
+export { ProxmoxStorageProvider } from './storage.ts';
+export { ProxmoxUserProvider } from './user.ts';
+export { ProxmoxZfsPoolProvider } from './zfs-pool.ts';

package/src/proxmox/lease-cache.test.ts

@@ -0,0 +1,163 @@
+/**
+ * ★ BOTH DIRECTIONS ARE PINNED. A cache that never serves a hit is a slow no-op nobody notices; a
+ *   cache that serves an expiring credential turns a 401 into "the object is absent", which is how
+ *   a plan comes to say CREATE for something that exists. The margin is the whole safety property,
+ *   so it is tested from both sides of its edge.
+ *
+ * ⛔ AND THE CONCURRENCY IS PINNED, BECAUSE THAT IS WHAT THE MEASUREMENT WAS ABOUT. 98 resources
+ *   reached an empty cache in the same instant, and the hand-written cache this replaced was wrong
+ *   only on the path where a mint fails — so that path is tested, not assumed.
+ */
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import * as Cache from 'effect/Cache';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import type { PveCredential } from './credentials.ts';
+import { type LeaseKey, makeLeases, timeToLive } from './lease-cache.ts';
+
+const TB4: LeaseKey = {
+  mount: 'proxmox-tb4',
+  role: 'read',
+  scheme: 'pve',
+};
+/** ⚠️ Same role as TB4, DIFFERENT mount — the collision the key has to survive. */
+const OPS: LeaseKey = { ...TB4, mount: 'proxmox-ops' };
+
+const cred = (tokenId: string, leaseSeconds: number): PveCredential => ({
+  leaseSeconds,
+  secret: 'not-a-real-secret',
+  tokenId,
+});
+
+const keptSeconds = (exit: Exit.Exit<PveCredential, unknown>) =>
+  Duration.toMillis(timeToLive(exit)) / 1000;
+
+/** A mint that counts itself, takes a moment — so callers genuinely overlap — and does as told. */
+const countingMint = (outcome: (call: number) => PveCredential | Error) => {
+  let calls = 0;
+  const mintFor = () =>
+    Effect.gen(function* () {
+      calls += 1;
+      const result = outcome(calls);
+      yield* Effect.sleep('10 millis');
+      if (result instanceof Error) return yield* Effect.fail(result);
+      return result;
+    });
+  return { calls: () => calls, mintFor };
+};
+
+const concurrently = <A, E>(times: number, effect: Effect.Effect<A, E>) =>
+  Effect.all(
+    Array.from({ length: times }, () => Effect.exit(effect)),
+    { concurrency: 'unbounded' },
+  );
+
+describe('how long a lease is kept', () => {
+  it('keeps a 300s provision lease for 240s — the last minute is the margin', () => {
+    assert.equal(keptSeconds(Exit.succeed(cred('hf-provision@pve!a', 300))), 240);
+  });
+
+  it('keeps a lease one second past the margin for one second', () => {
+    assert.equal(keptSeconds(Exit.succeed(cred('hf-read@pve!a', 61))), 1);
+  });
+
+  it('never keeps a lease inside the margin', () => {
+    assert.equal(keptSeconds(Exit.succeed(cred('hf-read@pve!brief', 60))), 0);
+    assert.equal(keptSeconds(Exit.succeed(cred('hf-read@pve!brief', 30))), 0);
+  });
+
+  it('never keeps a lease with no reported duration', () => {
+    // ⛔ `lease_duration` absent reads as 0. Keeping a credential of unknown lifetime is the one
+    //    thing this file must not do — not keeping is always safe.
+    assert.equal(keptSeconds(Exit.succeed(cred('hf-read@pve!unknown', 0))), 0);
+  });
+
+  it('never keeps a failure', () => {
+    assert.equal(keptSeconds(Exit.fail(new Error('vault sealed'))), 0);
+  });
+});
+
+describe('lease cache under concurrency', () => {
+  it('mints once for many concurrent callers, and serves the next from the cache', async () => {
+    const mint = countingMint(() => cred('hf-read@pve!a', 3600));
+    await Effect.runPromise(
+      Effect.gen(function* () {
+        const leases = yield* makeLeases(mint.mintFor);
+        const exits = yield* concurrently(20, Cache.get(leases, TB4));
+        assert.equal(exits.filter(Exit.isSuccess).length, 20);
+        // An equal but separately built key is the same entry.
+        yield* Cache.get(leases, { ...TB4 });
+      }),
+    );
+    assert.equal(mint.calls(), 1);
+  });
+
+  // ⛔ THE HAND-WRITTEN CACHE'S BUG LIVED HERE: a failed mint let every waiter claim at once.
+  it('mints ONCE more after a failure, not once per waiter', async () => {
+    const mint = countingMint((call) =>
+      call === 1 ? new Error('vault sealed') : cred('hf-read@pve!b', 3600),
+    );
+    await Effect.runPromise(
+      Effect.gen(function* () {
+        const leases = yield* makeLeases(mint.mintFor);
+        const first = yield* concurrently(20, Cache.get(leases, TB4));
+        assert.equal(first.filter(Exit.isFailure).length, 20, 'the waiters share the failure');
+        const second = yield* concurrently(20, Cache.get(leases, TB4));
+        assert.equal(second.filter(Exit.isSuccess).length, 20);
+      }),
+    );
+    assert.equal(mint.calls(), 2);
+  });
+
+  it('shares an uncacheable lease with its waiters and mints again for the next caller', async () => {
+    const mint = countingMint((call) => cred(`hf-read@pve!brief${String(call)}`, 30));
+    await Effect.runPromise(
+      Effect.gen(function* () {
+        const leases = yield* makeLeases(mint.mintFor);
+        yield* concurrently(20, Cache.get(leases, TB4));
+        yield* Cache.get(leases, TB4);
+      }),
+    );
+    assert.equal(mint.calls(), 2);
+  });
+
+  it('does not serve one cluster the credential of another with the same mount and role', async () => {
+    const mint = countingMint((call) => cred(`hf-read@pve!${String(call)}`, 3600));
+    const [tb4, ops] = await Effect.runPromise(
+      Effect.gen(function* () {
+        const leases = yield* makeLeases(mint.mintFor);
+        return [yield* Cache.get(leases, TB4), yield* Cache.get(leases, OPS)] as const;
+      }),
+    );
+    assert.notEqual(tb4.tokenId, ops.tokenId);
+    assert.equal(mint.calls(), 2);
+  });
+
+  it('shares one lease across two members of the same cluster mount', async () => {
+    const mint = countingMint((call) => cred(`hf-read@pve!${String(call)}`, 3600));
+    const N2: LeaseKey = { mount: 'proxmox-tb4', role: 'read', scheme: 'pve' };
+    const N3: LeaseKey = { mount: 'proxmox-tb4', role: 'read', scheme: 'pve' };
+    await Effect.runPromise(
+      Effect.gen(function* () {
+        const leases = yield* makeLeases(mint.mintFor);
+        yield* Cache.get(leases, N2);
+        yield* Cache.get(leases, N3);
+      }),
+    );
+    assert.equal(mint.calls(), 1);
+  });
+
+  it('keeps read and provision apart', async () => {
+    const mint = countingMint((call) => cred(`hf@pve!${String(call)}`, 3600));
+    const PROVISION: LeaseKey = { ...TB4, role: 'provision' };
+    const [read, provision] = await Effect.runPromise(
+      Effect.gen(function* () {
+        const leases = yield* makeLeases(mint.mintFor);
+        return [yield* Cache.get(leases, TB4), yield* Cache.get(leases, PROVISION)] as const;
+      }),
+    );
+    assert.notEqual(read.tokenId, provision.tokenId);
+  });
+});

package/src/proxmox/lease-cache.ts

@@ -0,0 +1,131 @@
+/**
+ * One credential per (target, role) for the life of a run, instead of one per API call.
+ *
+ * 🔴 THE MEASUREMENT THAT MADE THIS NECESSARY. A single `alchemy plan` plus `deploy` of the TB4
+ *   stack — 98 resources — left **760 API token entries in `/etc/pve/user.cfg`**, counted on n2
+ *   immediately afterwards. `mint()` asked OpenBao for a new credential on EVERY call, and each
+ *   mint creates a real Proxmox token: read, diff, reconcile and read-back each got their own. They
+ *   drain as the leases expire (760 down to 712 over five minutes), so nothing was broken — but
+ *   `/etc/pve` is pmxcfs, a REPLICATED cluster filesystem, and every one of those mints is a write
+ *   to it that all three nodes must agree on. Two plans in quick succession stack thousands.
+ *
+ * ⛔ AND ON PBS THE COST IS SECONDS, NOT BYTES. `pbs_grant.go` waits for PBS to honour a freshly
+ *   granted ACL before handing the credential over — measured at about 4.5s, and unavoidable,
+ *   because PBS caches its parsed ACL config. Per mint. The ten PBS resources in pbs.ts would pay
+ *   that ten times over for credentials identical to each other.
+ *
+ * ★ SO THE FIX IS TO STOP THROWING AWAY A CREDENTIAL THAT IS STILL VALID. This is strictly FEWER
+ *   credentials in existence at once, not more: the lease TTL is untouched, and a token that would
+ *   have been minted and abandoned is simply never minted. It is the same reasoning
+ *   `network-apply.ts` already applies by hand when it threads one lease through `pveWith` for its
+ *   apply and its task poll.
+ *
+ * ⚠️ THE CACHE IS PROCESS-SCOPED AND THAT IS THE INTENDED LIFETIME. A plan or a deploy is one
+ *   process; the next one starts cold. Nothing is written to disk, and a credential never outlives
+ *   the run that minted it.
+ *
+ * ⛔ IT IS NOT A PERFORMANCE CACHE, SO IT MUST NEVER SERVE A CREDENTIAL THAT COULD DIE MID-CALL.
+ *   The provision lease is 300s and a deploy can run longer than that. A credential handed out at
+ *   t+299 would fail the call it was handed to, and the failure surfaces as a 401 — which `read`
+ *   folds into "absent", which is how a plan comes to say CREATE for an object that exists. That is
+ *   the exact class of bug this package found three times in one day, so the margin below is
+ *   deliberately generous rather than tuned.
+ *
+ * ★ THE MECHANISM IS EFFECT'S OWN `Cache`, AND THE HAND-WRITTEN ONE IT REPLACED WAS WRONG IN
+ *   EXACTLY THE PLACE A LIBRARY IS NOT. A TTL map alone still minted 89 credentials for 98
+ *   resources, because Alchemy plans concurrently and every resource missed in the same instant;
+ *   the in-flight map added to fix that got the first miss right and the failure path wrong. A
+ *   waiter woken by a FAILED or uncacheable mint claimed the slot without looking again, the
+ *   waiters overwrote each other's claims, and each one's cleanup deleted somebody else's — the
+ *   thundering herd back through the error door (found in review 2026-09-14, confirmed by
+ *   reading). `Cache.get` shares one pending lookup among concurrent callers by design. MEASURED
+ *   with a probe before this was written: ten concurrent gets on equal-but-distinct key objects ran
+ *   the lookup once; a zero time-to-live ran it again for the next caller; a failure reached all
+ *   ten waiters and was not kept.
+ */
+import * as Cache from 'effect/Cache';
+import * as Duration from 'effect/Duration';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import { type ApiTarget, type PveCredential, type PveRole, mint } from './credentials.ts';
+
+/**
+ * Re-mint once a lease is inside this much of its end, in seconds.
+ *
+ * ⚠️ 60s AGAINST A 300s PROVISION LEASE MEANS A CACHED CREDENTIAL IS ONLY EVER HANDED OUT WITH AT
+ *   LEAST A MINUTE LEFT. The slowest single call in this package is `network-apply`'s reload poll,
+ *   and that mints its own lease deliberately — see the ⚠️ on `pveWith` there — so nothing using
+ *   this cache comes near a minute.
+ * ⚠️ A MOUNT REPORTING NO `lease_duration` IS NEVER CACHED: `leaseSeconds` is `0` and
+ *   `timeToLive` answers zero. Not caching is always safe; caching a credential whose lifetime is
+ *   unknown is not.
+ */
+const REMINT_MARGIN_SECONDS = 60;
+
+/**
+ * What a credential is cached under.
+ *
+ * ⛔ KEYED ON THE OPENBAO MOUNT AND ROLE, NOT ON A MEMBER HOSTNAME. A PVE API token is
+ *   cluster-wide — `user.cfg` is replicated — so n2, n3 and n4 share one credential. Keying on
+ *   `api` would mint three identical tokens when failover rotates members.
+ * ★ THE MOUNT CANNOT COLLIDE ACROSS CLUSTERS: `proxmox-tb4`, `proxmox-ops` and `pbs-tb4` are
+ *   distinct OpenBao mounts with distinct policies; two estates never share a mount name.
+ * ★ A PLAIN OBJECT IS A SAFE KEY BECAUSE EFFECT COMPARES KEYS STRUCTURALLY — two separately built
+ *   literals with the same fields are one entry, which the probe above measured.
+ */
+export type LeaseKey = {
+  readonly mount: string;
+  readonly role: PveRole;
+  readonly scheme: ApiTarget['scheme'];
+};
+
+const keyOf = (target: ApiTarget, role: PveRole): LeaseKey => ({
+  mount: target.mount,
+  role,
+  scheme: target.scheme,
+});
+
+/**
+ * How long a freshly minted credential may be handed out again.
+ *
+ * ⛔ ZERO FOR A FAILURE, FOR AN UNKNOWN LIFETIME AND FOR A LEASE INSIDE THE MARGIN. Zero means "do
+ *   not keep": the callers already waiting on THIS mint still receive its result, and the next
+ *   caller mints again. Keeping a failure would turn one refused mint into a refused run.
+ * ⚠️ EXPORTED FOR THE TEST. The margin is the whole safety property, so it is pinned from both
+ *   sides of its edge without waiting four minutes of wall clock.
+ */
+export const timeToLive = (exit: Exit.Exit<PveCredential, unknown>): Duration.Duration =>
+  Exit.isSuccess(exit) && exit.value.leaseSeconds > REMINT_MARGIN_SECONDS
+    ? Duration.seconds(exit.value.leaseSeconds - REMINT_MARGIN_SECONDS)
+    : Duration.zero;
+
+/**
+ * A lease cache over any mint.
+ *
+ * ⚠️ ONE BEHAVIOUR CHANGED WITH THE MOVE TO `Cache`, DELIBERATELY. Callers waiting on a mint that
+ *   FAILS now receive that failure, where the hand-written version had each of them mint again.
+ *   They asked in the same instant for the same credential, and a plan with one failed resource
+ *   fails either way — so the change is in how many errors the output lists, not in the outcome,
+ *   and a sealed vault is asked once rather than ninety-eight times.
+ * ★ `requireServicesAt: 'lookup'` IS WHAT LETS IT BE BUILT ONCE, AT IMPORT. The mint needs an
+ *   `HttpClient`; that requirement moves to each `get`, where every caller already has one.
+ * ⚠️ EXPORTED FOR THE TEST, which hands it a counting mint instead of calling OpenBao.
+ */
+export const makeLeases = <E, R>(mintFor: (key: LeaseKey) => Effect.Effect<PveCredential, E, R>) =>
+  Cache.makeWith(mintFor, { capacity: 64, requireServicesAt: 'lookup', timeToLive });
+
+const targetForKey = (key: LeaseKey): ApiTarget =>
+  key.scheme === 'pbs'
+    ? { api: 'https://pbs.invalid/api2/json', mount: key.mount, scheme: 'pbs' }
+    : { members: [], mount: key.mount, scheme: 'pve' };
+
+const leases = Effect.runSync(makeLeases((key) => mint(targetForKey(key), key.role)));
+
+/**
+ * A credential for `role` on `target`: a still-valid lease when one is kept, one shared mint when
+ * not.
+ *
+ * ★ client.ts is the one place that decides to use the cache. `mint` stays exported and unchanged
+ *   for the caller that reasons about identity rather than cost — network-apply.ts.
+ */
+export const leased = (target: ApiTarget, role: PveRole) => Cache.get(leases, keyOf(target, role));