@marinoscar/platform-api 0.0.0-stage → 0.1.0-next.10

package/dist/nodes/node-lifecycle.service.js

@@ -0,0 +1,217 @@
+"use strict";
+// =============================================================================
+// Fleet lifecycle: one notion of liveness, shared by everything (issue #270,
+// epic #254)
+// =============================================================================
+//
+// `deregister` is a GRACEFUL SHUTDOWN, and graceful shutdowns are not how
+// workers usually stop. A node that is OOM-killed, that segfaults in a native
+// dependency, that is a spot instance reclaimed mid-job, or that is simply on
+// the wrong side of a network partition never calls it. Nothing else writes
+// to that row either — a node has no database access — so its `status` sits
+// at `online` forever, and the admin fleet page renders a green "online" chip
+// above a three-week-old heartbeat. That is WORSE than showing no status at
+// all: an operator who can see there is no data goes and looks; an operator
+// shown a confident green chip does not.
+//
+// This file is the answer to "what does this fleet actually look like right
+// now", and it is deliberately SMALL: a policy read that cannot throw, and a
+// pure function that turns a row plus a clock into a health verdict. The two
+// crons and the admin plane all go through it, which is the entire point —
+// see below.
+//
+// -----------------------------------------------------------------------------
+// THE FAILURE THIS FILE'S EXISTENCE PREVENTS: TWO NOTIONS OF LIVENESS
+// -----------------------------------------------------------------------------
+//
+// There are three consumers of "is this node alive":
+//
+//   1. `NodeStaleOfflineTask` — flips a silent node's STATUS to `offline`.
+//   2. `NodeOfflinePruneTask` — deletes `offline` rows past retention.
+//   3. `NodesAdminService` — renders a health verdict on the fleet page.
+//
+// If (1) and (3) computed the window separately, the page would show a "stale"
+// pill for a node the sweep still counts as fine, or the reverse — and which
+// one an operator believed would depend on which screen they happened to be
+// looking at. So the window is read ONCE, here, and the verdict is computed by
+// ONE function, here.
+//
+// The same argument is what makes `offlineStaleMultiplier` a MULTIPLIER rather
+// than an independent "declare a node offline after N minutes" setting. See
+// `NodeStaleOfflineTask`'s header: a second duration is a second definition of
+// liveness, and the two can be configured into contradicting each other by an
+// administrator who has no way to know they are related.
+//
+// -----------------------------------------------------------------------------
+// WHY THIS IS A SERVICE AND NOT A FUNCTION IN EACH TASK
+// -----------------------------------------------------------------------------
+//
+// REJECTED: each cron reading `SystemSettingsService` itself. It is one line
+// per task, and it is how the fallback behaviour diverges: the first task to
+// be given a `try`/`catch` gets one, the second does not, and a settings read
+// that failed then stops the prune while the sweep carries on — which is
+// precisely the ordering that silently voids retention (again, see
+// `NodeOfflinePruneTask`). One accessor, one fallback, one log line.
+//
+// REJECTED: caching the policy in memory. The values are read at most 145
+// times a day between the two crons; a cache would buy nothing and would mean
+// an administrator changing the stale window has to guess how long until it
+// takes effect.
+// =============================================================================
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var NodeLifecycleService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NodeLifecycleService = void 0;
+exports.deriveNodeHealth = deriveNodeHealth;
+const common_1 = require("@nestjs/common");
+const index_1 = require("../settings/index");
+const index_2 = require("../jobs/index");
+/**
+ * `offline` when the status already says so, `healthy` when the last heartbeat
+ * is inside the stale window, `stale` otherwise.
+ *
+ * THE ORDER OF THE THREE BRANCHES IS THE WHOLE DEFINITION, so it is worth
+ * spelling out what each one means:
+ *
+ *   - `offline` FIRST, from `status`. Either the node said so (`deregister`,
+ *     a graceful shutdown) or the stale sweep concluded it. Recomputing a
+ *     verdict from the heartbeat for a row that is already `offline` would let
+ *     a node that was gracefully deregistered five seconds ago render as
+ *     `healthy`, which is a straight contradiction of the chip beside it.
+ *
+ *   - `healthy` when `now - lastHeartbeatAt < staleHeartbeatSeconds`. Note it
+ *     is `lastHeartbeatAt` and NOT `status`: a node whose row says `online` is
+ *     making a claim about itself that only the heartbeat can corroborate.
+ *
+ *   - `stale` for everything else, INCLUDING `lastHeartbeatAt === null`. A
+ *     node that registered and never phoned home is not healthy — it is a
+ *     process that started and never got as far as its first heartbeat, which
+ *     is a real and common failure (a bad credential, a firewall, a crash in
+ *     startup). Reading `null` as healthy is how such a node hides.
+ *
+ * `disabled` and `draining` are deliberately NOT special-cased. Both are
+ * operator intent about what a node may CLAIM, and neither says anything about
+ * whether the process is still running: a disabled node that is still
+ * heartbeating is disabled AND healthy, and an operator re-enabling it wants
+ * to know that before they do. The two facts are rendered as two fields
+ * (`status` and `health`) precisely so they cannot overwrite each other.
+ *
+ * PURE, and takes `now` explicitly, so the fleet page and the crons can be
+ * pinned to one instant in a test rather than racing the wall clock.
+ */
+function deriveNodeHealth(node, staleHeartbeatSeconds, now = new Date()) {
+    if (node.status === 'offline') {
+        return 'offline';
+    }
+    if (!node.lastHeartbeatAt) {
+        return 'stale';
+    }
+    const staleAfter = now.getTime() - staleHeartbeatSeconds * 1000;
+    return node.lastHeartbeatAt.getTime() > staleAfter ? 'healthy' : 'stale';
+}
+/**
+ * The fleet's lifecycle policy: staleness, offline and retention cut-offs from the `nodes` system settings.
+ *
+ * @stability experimental
+ */
+let NodeLifecycleService = NodeLifecycleService_1 = class NodeLifecycleService {
+    systemSettings;
+    logger = new common_1.Logger(NodeLifecycleService_1.name);
+    constructor(systemSettings) {
+        this.systemSettings = systemSettings;
+    }
+    /**
+     * The fleet policy, from the `nodes` system-settings namespace.
+     *
+     * READ THROUGH THE NARROW SETTINGS ACCESSOR (`getNodesPolicy`), not with a
+     * `system_settings` query of its own — the same rule
+     * `JobStuckService.getStuckThresholdMinutes()` follows, and for the same
+     * reason: a second read path is how "the sweep uses a different window than
+     * the fleet page shows" starts.
+     *
+     * NEVER THROWS. Two of the three callers are cron ticks with nobody to
+     * report to, and a settings read that failed is not a reason to stop
+     * sweeping — it is a reason to sweep on the shipped defaults, loudly.
+     * Each field is validated INDEPENDENTLY rather than the block being
+     * accepted or rejected whole: a stored row that somehow carries a good
+     * `staleHeartbeatSeconds` and a garbage `offlineRetentionDays` should keep
+     * the good half, because falling back on both would silently change the
+     * threshold an operator is watching.
+     */
+    async getPolicy() {
+        const defaults = index_2.DEFAULT_NODES_POLICY;
+        try {
+            const policy = (await this.systemSettings.getNodesPolicy());
+            return {
+                staleHeartbeatSeconds: positive(policy?.staleHeartbeatSeconds, defaults.staleHeartbeatSeconds),
+                offlineStaleMultiplier: positive(policy?.offlineStaleMultiplier, defaults.offlineStaleMultiplier),
+                offlineRetentionDays: positive(policy?.offlineRetentionDays, defaults.offlineRetentionDays),
+                // ⚠ FAIL-CLOSED, UNLIKE ITS THREE NEIGHBOURS (#349, epic #345). The
+                // other three degrade to the SHIPPED value, because sweeping on the
+                // default window beats not sweeping. This one degrades to `false`,
+                // because the safe answer to "may a node hold a credential to this
+                // database?" when the stored setting is unreadable is no. Only a
+                // literal `true` enables it; anything else — a missing key, a string
+                // `"true"`, a number — is off.
+                //
+                // ⚠ THE RULE ITSELF MOVED TO `jobs/node-offload.service.ts` (#352) and
+                // is CALLED here rather than repeated. `NodeOffloadService` needs the
+                // same answer and cannot import this file (nodes imports jobs; the
+                // reverse is a cycle), and two literal `=== true` checks are how "the
+                // fleet page says brokering is off while the claim thinks it is on"
+                // starts.
+                jobSecretBrokerEnabled: (0, index_2.readJobSecretBrokerEnabled)(policy),
+            };
+        }
+        catch (error) {
+            this.logger.warn(`Could not read the nodes system settings; falling back to the shipped fleet policy ` +
+                `(stale ${defaults.staleHeartbeatSeconds}s x${defaults.offlineStaleMultiplier}, ` +
+                `retention ${defaults.offlineRetentionDays}d): ` +
+                `${error instanceof Error ? error.message : String(error)}`);
+            return { ...defaults };
+        }
+    }
+    /**
+     * The instant before which a heartbeat means "this node is gone", from
+     * `staleHeartbeatSeconds x offlineStaleMultiplier`.
+     *
+     * Derived here rather than in the sweep so that the number the sweep acts on
+     * and the number the fleet page explains are computed once. See
+     * `NodeStaleOfflineTask` for why this is a multiple of the stale window and
+     * not a setting of its own.
+     */
+    staleCutoff(policy, now) {
+        return new Date(now.getTime() - policy.staleHeartbeatSeconds * policy.offlineStaleMultiplier * 1000);
+    }
+    /** The instant before which an `offline` node's record has outlived its retention. */
+    retentionCutoff(policy, now) {
+        return new Date(now.getTime() - policy.offlineRetentionDays * 24 * 60 * 60 * 1000);
+    }
+};
+exports.NodeLifecycleService = NodeLifecycleService;
+exports.NodeLifecycleService = NodeLifecycleService = NodeLifecycleService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [index_1.SystemSettingsService])
+], NodeLifecycleService);
+/**
+ * A stored number, or the shipped default when it is missing, non-finite or
+ * not positive.
+ *
+ * `NaN` is the case that matters: every comparison against it is false, so a
+ * `NaN` window would produce a cutoff of `Invalid Date` and an `updateMany`
+ * that silently matches NOTHING — a sweep that runs 144 times a day and does
+ * nothing, with no error anywhere. Falling back to the default keeps the
+ * fleet honest on a damaged settings row.
+ */
+function positive(value, fallback) {
+    return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : fallback;
+}
+//# sourceMappingURL=node-lifecycle.service.js.map

package/dist/nodes/node-lifecycle.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"node-lifecycle.service.js","sourceRoot":"","sources":["../../src/nodes/node-lifecycle.service.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,6EAA6E;AAC7E,aAAa;AACb,gFAAgF;AAChF,EAAE;AACF,0EAA0E;AAC1E,8EAA8E;AAC9E,8EAA8E;AAC9E,4EAA4E;AAC5E,4EAA4E;AAC5E,8EAA8E;AAC9E,4EAA4E;AAC5E,4EAA4E;AAC5E,yCAAyC;AACzC,EAAE;AACF,4EAA4E;AAC5E,6EAA6E;AAC7E,6EAA6E;AAC7E,2EAA2E;AAC3E,aAAa;AACb,EAAE;AACF,gFAAgF;AAChF,sEAAsE;AACtE,gFAAgF;AAChF,EAAE;AACF,qDAAqD;AACrD,EAAE;AACF,2EAA2E;AAC3E,uEAAuE;AACvE,yEAAyE;AACzE,EAAE;AACF,+EAA+E;AAC/E,6EAA6E;AAC7E,4EAA4E;AAC5E,+EAA+E;AAC/E,sBAAsB;AACtB,EAAE;AACF,+EAA+E;AAC/E,4EAA4E;AAC5E,+EAA+E;AAC/E,8EAA8E;AAC9E,yDAAyD;AACzD,EAAE;AACF,gFAAgF;AAChF,wDAAwD;AACxD,gFAAgF;AAChF,EAAE;AACF,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,mEAAmE;AACnE,qEAAqE;AACrE,EAAE;AACF,0EAA0E;AAC1E,8EAA8E;AAC9E,4EAA4E;AAC5E,gBAAgB;AAChB,gFAAgF;;;;;;;;;;;;;AAyDhF,4CAgBC;AAvED,2CAAoD;AAEpD,6CAA0D;AAC1D,yCAAmG;AAmBnG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,SAAgB,gBAAgB,CAC9B,IAAqB,EACrB,qBAA6B,EAC7B,MAAY,IAAI,IAAI,EAAE;IAEtB,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,qBAAqB,GAAG,IAAI,CAAC;IAEhE,OAAO,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC;AAC3E,CAAC;AAED;;;;GAIG;AAEI,IAAM,oBAAoB,4BAA1B,MAAM,oBAAoB;IAGF;IAFZ,MAAM,GAAG,IAAI,eAAM,CAAC,sBAAoB,CAAC,IAAI,CAAC,CAAC;IAEhE,YAA6B,cAAqC;QAArC,mBAAc,GAAd,cAAc,CAAuB;IAAG,CAAC;IAEtE;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,SAAS;QACb,MAAM,QAAQ,GAAG,4BAAoB,CAAC;QAEtC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,CAA4C,CAAC;YAEvG,OAAO;gBACL,qBAAqB,EAAE,QAAQ,CAAC,MAAM,EAAE,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB,CAAC;gBAC9F,sBAAsB,EAAE,QAAQ,CAAC,MAAM,EAAE,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB,CAAC;gBACjG,oBAAoB,EAAE,QAAQ,CAAC,MAAM,EAAE,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB,CAAC;gBAC3F,oEAAoE;gBACpE,oEAAoE;gBACpE,mEAAmE;gBACnE,mEAAmE;gBACnE,iEAAiE;gBACjE,qEAAqE;gBACrE,+BAA+B;gBAC/B,EAAE;gBACF,uEAAuE;gBACvE,sEAAsE;gBACtE,mEAAmE;gBACnE,sEAAsE;gBACtE,oEAAoE;gBACpE,UAAU;gBACV,sBAAsB,EAAE,IAAA,kCAA0B,EAAC,MAAM,CAAC;aAC3D,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qFAAqF;gBACnF,UAAU,QAAQ,CAAC,qBAAqB,MAAM,QAAQ,CAAC,sBAAsB,IAAI;gBACjF,aAAa,QAAQ,CAAC,oBAAoB,MAAM;gBAChD,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9D,CAAC;YAEF,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,WAAW,CAAC,MAAmB,EAAE,GAAS;QACxC,OAAO,IAAI,IAAI,CACb,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,qBAAqB,GAAG,MAAM,CAAC,sBAAsB,GAAG,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,sFAAsF;IACtF,eAAe,CAAC,MAAmB,EAAE,GAAS;QAC5C,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACrF,CAAC;CACF,CAAA;AAhFY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;qCAIkC,6BAAqB;GAHvD,oBAAoB,CAgFhC;AAED;;;;;;;;;GASG;AACH,SAAS,QAAQ,CAAC,KAAc,EAAE,QAAgB;IAChD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC7F,CAAC"}

package/dist/nodes/node-secret-broker.service.d.ts

@@ -0,0 +1,281 @@
+import { type JobsPrisma } from '../jobs/index';
+import { JobHandlerRegistry } from '../jobs/index';
+import { NodeJobSecretRequestDto, NodeJobSecretResponseDto } from './dto/node-job-secret.dto';
+import { NodeLifecycleService } from './node-lifecycle.service';
+import { NodesService } from './nodes.service';
+/**
+ * How many unrevoked grants one sweep looks at.
+ *
+ * Bounded so a deployment that somehow accumulated thousands of orphans does
+ * not spend one cron tick issuing thousands of `DROP ROLE`s inside a single
+ * promise chain — the next tick continues, oldest first. In a healthy
+ * deployment this number is never reached, because the settle-event path has
+ * already revoked almost everything before the sweep runs.
+ */
+/**
+ * How far past the job's lease a credential is asked to stay valid, in
+ * milliseconds — a CLOCK-SKEW ALLOWANCE, and not a grace period on the node's
+ * authorization.
+ *
+ * ⚠ THE TWO DEADLINES ARE EVALUATED BY DIFFERENT CLOCKS, AND THAT IS THE WHOLE
+ * REASON THIS NUMBER EXISTS. The lease is computed and enforced here, on this
+ * process's clock; a credential's expiry is enforced by the BACKEND that issued
+ * it — PostgreSQL checks `VALID UNTIL` against the database server's own clock
+ * at authentication time. A credential set to expire at exactly the lease
+ * deadline therefore dies EARLY on any deployment whose database clock runs
+ * even slightly ahead, and it dies part-way through the work rather than
+ * before it: an authentication failure in the middle of a multi-hour dump,
+ * which is both the most expensive place to fail and the hardest to attribute.
+ *
+ * Sixty seconds is far beyond any plausible NTP skew and far below anything an
+ * attacker could use. It does not extend a node's authority: the node is
+ * renewing its lease continuously while it works (#347), the settle listener
+ * revokes on the terminal write, and the sweeper's predicate is the hold
+ * guard's complement — so the practical lifetime of a credential remains
+ * "until the node stops", plus a minute during which nothing will accept the
+ * job as held anyway.
+ *
+ * ⚠ IT LIVES HERE, AND NOT IN ANY BROKER, ON PURPOSE. `JobSecretBroker.issue`
+ * is contracted to honour `until` EXACTLY — "a broker may grant LESS ... it
+ * must not grant more" — and a broker that quietly added its own overhang would
+ * make that sentence false for the first implementation that shipped, which is
+ * how a contract stops being load-bearing. Worse, every future broker would
+ * then pick its own unspecified overhang. How long a credential should live is
+ * one decision, and this is the one funnel every broker is reached through, so
+ * it is taken once, here, and every broker gets the allowance for free.
+ *
+ * `IssuedJobSecret.expiresAt` still reports what the broker actually managed to
+ * set, so the sweeper's own `row.expiresAt > now` check sees the truth rather
+ * than the request.
+  *
+  * @stability experimental
+ */
+export declare const SECRET_CLOCK_SKEW_ALLOWANCE_MS = 60000;
+/**
+ * What one sweep did, for the cron's log line.
+ *
+ * @stability experimental
+ */
+export interface NodeSecretSweepResult {
+    /** Grants examined this tick. */
+    examined: number;
+    /** Grants successfully revoked and marked. */
+    revoked: number;
+    /** Grants that should have been revoked and could not be (logged, retried next tick). */
+    failed: number;
+}
+/**
+ * Issues, and revokes, the per-job credential of a handler's `nodeSecretBroker`, bounded by the job's lease; stores only its handle.
+ *
+ * @stability experimental
+ */
+export declare class NodeSecretBrokerService {
+    private readonly prisma;
+    private readonly nodes;
+    private readonly registry;
+    private readonly lifecycle;
+    private readonly logger;
+    constructor(prisma: JobsPrisma, nodes: NodesService, registry: JobHandlerRegistry, lifecycle: NodeLifecycleService);
+    /**
+     * The one credential for the job this node is holding — minted, or the
+     * existing grant extended.
+     *
+     * ⚠ RE-CALLABLE WHILE THE LEASE IS LIVE, AND THAT IS A REQUIREMENT RATHER
+     * THAN A TOLERANCE. A node asks again as a matter of course: a process
+     * restarted while still holding the lease, a response lost on the way back,
+     * a retry of the same job by the same node. Every one of those calls lands on
+     * THE SAME GRANT — `@@unique([jobId, kind])` makes a second row
+     * unrepresentable, and `JobSecretBroker.issue` is contracted to extend rather
+     * than mint. ONE CREDENTIAL PER JOB, EVER; a broker that minted per call
+     * would leak one role per restart into the very database it is protecting,
+     * and revocation — which knows exactly one handle — would clean up exactly
+     * one of them.
+     *
+     * The credential's validity is the JOB'S LEASE EXPIRY, plus
+     * {@link SECRET_CLOCK_SKEW_ALLOWANCE_MS}. Not a duration configured by a
+     * broker, not one the node asked for: the node is already renewing that lease
+     * (#347), so the credential rides a clock that is maintained for other
+     * reasons and that stops the instant the node stops. A second clock could
+     * only disagree with it, and the disagreement would be silent. The allowance
+     * is not a second clock either — it is a tolerance on the fact that the
+     * credential's deadline is enforced by the BACKEND's clock and the lease by
+     * ours; see the constant for why it is taken here rather than by each broker.
+     */
+    issueForJob(userId: string, nodeId: string, jobId: string, dto: NodeJobSecretRequestDto): Promise<NodeJobSecretResponseDto>;
+    /**
+     * Destroy every live grant for a job that has just settled.
+     *
+     * THE FAST PATH. Called from `NodeSecretRevoker` milliseconds after the
+     * terminal write, so in the overwhelming majority of cases the credential is
+     * gone long before anything else notices the job finished.
+     *
+     * NEVER THROWS — every failure is logged and left for the sweeper, which is
+     * exactly what the unrevoked row is for. The caller runs inside
+     * `EventEmitter2`'s synchronous dispatch, where a rejection would surface as
+     * an `unhandledRejection` with a stack pointing at a worker that did nothing
+     * wrong.
+     *
+     * Returns how many grants it revoked, for the caller's log line.
+     */
+    revokeForJob(jobId: string): Promise<number>;
+    /**
+     * Whether a job type could have a grant outstanding at all.
+     *
+     * A MAP LOOKUP, and it is what keeps the settle listener off the database for
+     * the ~100% of settles in a deployment where no type declares a broker. If
+     * nothing in this process can mint a `job_node_secrets` row for this type,
+     * nothing in this process needs to look for one. The sweeper is the backstop
+     * for the one case this misses — a fork that REMOVED a broker while grants of
+     * its kind were still live — and that case is a deployment change, not a
+     * per-job event.
+     */
+    couldHoldGrant(type: string): boolean;
+    /**
+     * Revoke every grant whose job is no longer held by the node it was minted
+     * for.
+     *
+     * SEE THE FILE HEADER FOR WHY THIS IS NOT REDUNDANT with the settle listener:
+     * three real cases emit no event at all, and one of them — a job reaped after
+     * its executor died — is precisely the case where an outstanding credential is
+     * most likely.
+     *
+     * THE PREDICATE IS THE COMPLEMENT OF THE HOLD GUARD (`stillHeld` below), not a
+     * list of job statuses. A list would have to be kept in step with the queue's
+     * state machine by hand, and the failure of forgetting an arm is a credential
+     * that is never revoked — invisible until somebody audits roles.
+     *
+     * NEVER THROWS for one bad grant: a broker that cannot be reached leaves its
+     * row unrevoked, which is exactly the state that makes the next tick retry it.
+     */
+    sweep(now?: Date): Promise<NodeSecretSweepResult>;
+    /**
+     * Whether a grant is still legitimate — the EXACT complement of what the
+     * sweeper revokes.
+     *
+     * The four job conditions are `assertJobHeldByNode`'s four, stated once more
+     * because this is the only other place in the system that has to answer the
+     * same question without an HTTP request in hand. The fifth condition is the
+     * grant's own clock: a credential past its `expiresAt` no longer works
+     * anyway, so leaving its role in place buys nothing and costs a role.
+     *
+     * ⚠ NOT the claim-token check `assertJobHeldByNode` gained in #364, even
+     * though `issueForJob` above now makes that check on every request. The two
+     * are asking different questions, and this one has no way to ask the other's.
+     * The guard asks "is this REQUEST from the claim the row currently carries",
+     * which only the caller can answer, by quoting a token it was handed. A
+     * sweep asks "is this GRANT still attached to held work", and there is no
+     * caller in a cron tick to quote anything — comparing the grant against the
+     * row's current token would be the tautology the node plane was stuck with
+     * before the token crossed the wire, since the sweeper would be reading both
+     * sides from the same row.
+     *
+     * Nor is it needed. A grant outlives one claim of a job no more than it
+     * outlives the job: if the row was reaped and re-claimed, its lease moved,
+     * and this grant's `expiresAt` — bounded by the lease that minted it — has
+     * already passed or is about to, so the clock condition below catches
+     * exactly the case a token check would have. Adding one here would buy
+     * nothing and would couple credential revocation to a value only a live node
+     * holds.
+     */
+    private stillHeld;
+    /**
+     * Revoke one grant and mark the row, or log why not.
+     *
+     * ⚠ THE MARK IS AN `updateMany` GUARDED ON `revokedAt: null`, not an
+     * `update`. Both revocation paths can be acting on the same row at the same
+     * instant — a job settling while a sweep is mid-batch is not exotic — and
+     * this makes the race produce one winner and one no-op rather than two
+     * writers disagreeing about when the grant ended.
+     *
+     * THE BROKER CALL COMES FIRST AND THE MARK SECOND. If the order were
+     * reversed, a `revoke` that threw would leave a row marked revoked and a
+     * credential that still works, which is the one state nothing would ever
+     * clean up — the sweeper only looks at unmarked rows. Failing before the mark
+     * leaves the row exactly as the next tick needs to find it.
+     */
+    private revokeGrant;
+    /** Best-effort revoke used to undo a grant this server failed to record. */
+    private revokeQuietly;
+    /** The broker a job type declares, if it declares one. */
+    private brokerForType;
+    /**
+     * The registered broker that mints grants of `kind`.
+     *
+     * ⚠ RESOLVED BY `kind`, NOT BY THE JOB'S `type`, AND THAT IS DELIBERATE. The
+     * sweeper meets rows whose `jobs` row has already been deleted by the history
+     * purge — retention on job history has nothing to do with a grant's lifetime
+     * — and a lookup that needed the job's type could not revoke those at all.
+     * `kind` is recorded on the row precisely so a grant stays revocable after
+     * everything else about its job is gone.
+     *
+     * Two brokers declaring the same `kind` is a fork's configuration error, not
+     * a supported arrangement: they would be two different authorities behind one
+     * name, and revoking through the wrong one silently does nothing. It is
+     * logged and the first registration wins, which is the same "loud, last
+     * writer takes it" posture `JobHandlerRegistry.register` takes for a
+     * duplicate type.
+     */
+    private brokerForKind;
+    /**
+     * 403 unless an administrator has switched brokering on for this deployment.
+     *
+     * ⚠ THIS IS A SECOND FENCE, NOT THE ONLY ONE. The same setting removes every
+     * broker-carrying type from `NodesService.nodeEligibleTypes`, so a node in a
+     * deployment with this off never claims such a job and never reaches this
+     * check. That is the fence that matters operationally; this one exists
+     * because a node can be holding a job claimed BEFORE an administrator turned
+     * the setting off, and the answer to "may I have the credential now" must be
+     * no from the instant the switch moves rather than from the next claim.
+     *
+     * A NAMED REASON, in `details`, because "403" on its own sends a node
+     * operator to look at permissions — and their credential is fine. The thing
+     * that is wrong is a system setting they may not even be able to see.
+     */
+    private assertBrokerEnabled;
+    /**
+     * 503 unless the broker says it can mint right now.
+     *
+     * WHY 503 AND NOT 422, which is what the data plane answers for an
+     * unresolvable input. The distinction is whether a retry can EVER help, and
+     * here it can: the ordinary `usable()` failure is a privilege the deployment's
+     * database user does not hold (managed PostgreSQL denying `CREATEROLE` is the
+     * standard case), and an administrator granting it makes the very same job
+     * succeed with nothing about the job having changed. `422` would tell the node
+     * "this can never work — fail the job", which is false, and a node that
+     * believed it would burn the whole attempt budget in the minute before
+     * somebody fixed the grant. `503` says "not now", which is the truth.
+     *
+     * The broker's `remedy` is carried through untouched. It is the part a person
+     * pastes into a terminal, and burying it in a server log while the node sees a
+     * bare 503 is how a fixable refusal becomes an outage nobody can explain.
+     */
+    private assertUsable;
+    /**
+     * Refuses any field that ASKS for something — which is every field this
+     * body can carry except `claimToken`.
+     *
+     * The same net `NodeDataPlaneService.rejectCallerSuppliedFields` casts, with
+     * an allowlist of exactly one, because a node may not request a secret it was
+     * not assigned: a `kind`, a `scope`, a `database` or a `ttl` here would each
+     * be a node choosing part of a credential's shape, and every one of those is
+     * the server's choice derived from the job it is holding.
+     *
+     * ⚠ `claimToken` IS THAT ONE, AND IT IS NOT A CRACK IN THE RULE (#364). The
+     * rule is "a node may not ask for anything"; the token asks for nothing. It
+     * answers which claim is speaking, it is checked against the row by
+     * `assertJobHeldByNode` before this method runs, and the only outcome it can
+     * produce is the caller being REFUSED. Nothing about the credential — kind,
+     * scope, lifetime — moves because of it. The allowlist and the DTO schema
+     * must agree: this set is the authority, and a field described in
+     * `nodeJobSecretRequestSchema` but absent here is a 400 on every request
+     * that sends it.
+     *
+     * REFUSED RATHER THAN IGNORED, for the reason the upload route already
+     * records: ignoring means the node's author never learns their field had no
+     * effect, and the bug is found days later by a person instead of minutes later
+     * by a machine. It costs a correct client exactly nothing — no legitimate node
+     * ever sends anything else here.
+     */
+    private rejectCallerSuppliedFields;
+}
+//# sourceMappingURL=node-secret-broker.service.d.ts.map

package/dist/nodes/node-secret-broker.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"node-secret-broker.service.d.ts","sourceRoot":"","sources":["../../src/nodes/node-secret-broker.service.ts"],"names":[],"mappings":"AAyIA,OAAO,EAAgC,KAAK,UAAU,EAAE,MAAM,eAAe,CAAC;AAG9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAC9F,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C;;;;;;;;GAQG;AACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,eAAO,MAAM,8BAA8B,QAAS,CAAC;AAIrD;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACpC,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,yFAAyF;IACzF,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,qBACa,uBAAuB;IAIP,OAAO,CAAC,QAAQ,CAAC,MAAM;IAGhD,OAAO,CAAC,QAAQ,CAAC,KAAK;IAItB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAIzB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAd5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA4C;gBAGvB,MAAM,EAAE,UAAU,EAG3C,KAAK,EAAE,YAAY,EAInB,QAAQ,EAAE,kBAAkB,EAI5B,SAAS,EAAE,oBAAoB;IAOlD;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,WAAW,CACf,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,uBAAuB,GAC3B,OAAO,CAAC,wBAAwB,CAAC;IAoHpC;;;;;;;;;;;;;;OAcG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBlD;;;;;;;;;;OAUG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAQrC;;;;;;;;;;;;;;;;OAgBG;IACG,KAAK,CAAC,GAAG,GAAE,IAAiB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA+CnE;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,OAAO,CAAC,SAAS;IAkBjB;;;;;;;;;;;;;;OAcG;YACW,WAAW;IAsDzB,4EAA4E;YAC9D,aAAa;IAiB3B,0DAA0D;IAC1D,OAAO,CAAC,aAAa;IAIrB;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,aAAa;IAiBrB;;;;;;;;;;;;;;OAcG;YACW,mBAAmB;IA0BjC;;;;;;;;;;;;;;;;OAgBG;YACW,YAAY;IA6B1B;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,OAAO,CAAC,0BAA0B;CAsBnC"}