@marinoscar/platform-api 0.0.0-stage → 0.1.0-next.4

package/dist/jobs/job-execution-profile.js

@@ -0,0 +1,333 @@
+"use strict";
+// =============================================================================
+// Per-type execution profiles (issue #346, epic #345)
+// =============================================================================
+//
+// Until now every job in the queue was governed by the SAME two deployment-wide
+// numbers: `JOBS_JOB_TIMEOUT_MS` (how long any one attempt may run) and
+// `JOBS_MAX_ATTEMPTS` (how many attempts any job gets). That is exactly right
+// for a queue whose jobs resemble each other, and exactly wrong the moment one
+// does not: a six-second thumbnail and a six-hour database dump cannot share a
+// timeout, and a type that must NEVER be automatically retried cannot share a
+// budget with one that should be retried three times. Tuning the globals to
+// suit the outlier ruins them for everything else — a ten-minute default
+// stretched to six hours means a wedged thumbnail now holds its slot until
+// lunchtime.
+//
+// A profile is that pair of numbers, declared BY THE HANDLER, for its own type.
+// It is optional: a handler that declares none behaves exactly as it did
+// before, on the globals, and every resolver below is written so that path is
+// not merely equivalent but literally the same code it always ran.
+//
+// -----------------------------------------------------------------------------
+// ⚠ EXACTLY TWO NUMBERS. THE LEASE AND THE RENEWAL INTERVAL ARE DERIVED.
+// -----------------------------------------------------------------------------
+//
+// This is the decision the whole file exists to defend, so it is worth being
+// blunt about what is NOT here. A profile may not declare:
+//
+//   - a LEASE LENGTH. The lease is how long a claimer may hold a row before
+//     the reaper (#263) decides its executor died. A lease SHORTER than the
+//     runtime ceiling is a job that reaps itself: it runs for its permitted
+//     time, the reaper finds the lease expired, requeues the row, a second
+//     executor claims it, and the same work runs twice — concurrently, on a
+//     job the author explicitly told us takes a long time. `resolveJobLeaseMs`
+//     derives it as `maxRuntimeMs + LEASE_GRACE_MS`, so it is longer than the
+//     ceiling BY CONSTRUCTION and there is no way to write down the state
+//     where it is not.
+//
+//   - a RENEWAL INTERVAL. A remote node extends its lease on a schedule
+//     (`resolveRenewIntervalMs`). An interval at or above the lease is a
+//     renewal that ALWAYS arrives after the lease it was meant to extend has
+//     already expired — the node is doing the work, sending the renewals, and
+//     losing the job anyway, which reads as a queue bug and is really an
+//     arithmetic one. Dividing the lease by three leaves room for two missed
+//     renewals before anything is lost.
+//
+// Both are values that can DISAGREE with `maxRuntimeMs`, and the disagreement
+// is silent, remote and intermittent — the worst kind. This is the same
+// argument `job-handler.interface.ts` already makes against a
+// `nodeEligible: boolean` flag ("deriving eligibility from the members makes
+// that wrong state unrepresentable: there is nothing to set inconsistently"),
+// applied to durations rather than to capabilities. There is nothing to set
+// inconsistently here because there is nothing to set.
+//
+// REJECTED for the same reason: a `retryBackoffMs`, a `priority`, a
+// `concurrencyLimit`. Each is a real thing somebody will want, and none of
+// them belongs in a struct whose promise is "these two numbers cannot
+// contradict each other". Adding a third that CAN contradict them turns the
+// promise into a convention, and a convention is what this file is replacing.
+//
+// -----------------------------------------------------------------------------
+// EVERY RESOLVER IS TOTAL, BOUNDS-CHECKED, AND NEVER THROWS
+// -----------------------------------------------------------------------------
+//
+// The same defensive posture `resolveWorkerConcurrency` and `resolveJobLeaseMs`
+// already take in `job.worker.ts`, for the same two reasons: these are
+// reachable from a directly-constructed test double holding a stub
+// `ConfigService`, and they are reachable from a fork's handler whose profile
+// is whatever that fork typed. A missing key, a `NaN`, a negative ceiling or a
+// budget of zero must degrade to the SHIPPED DEFAULT — never to `NaN`, which
+// downstream becomes an unwritable `lease_expires_at`, a `setTimeout(NaN)`
+// that fires immediately, and an attempt comparison that is false for every
+// row (silently disabling the reaper's give-up phase).
+//
+// An unusable profile is dropped WHOLE rather than field by field. The two
+// numbers are one declaration: honouring the attempt budget an author mistyped
+// while inventing a runtime ceiling they never chose is a stranger state than
+// putting the type back where it was before anyone wrote a profile. It warns,
+// naming the type, so the typo is fixable rather than merely survivable.
+//
+// -----------------------------------------------------------------------------
+// THE IMPORT CYCLE WITH `job.worker.ts` IS REAL, AND INERT
+// -----------------------------------------------------------------------------
+//
+// This module imports `resolveJobLeaseMs` from `job.worker.ts`, and that file
+// imports the profile type and the readers below back. Neither module touches
+// the other AT EVALUATION TIME — every use on both sides is inside a function
+// body — so the cycle never observes a half-initialised module.
+//
+// It is deliberate rather than tolerated. The alternative is a second lease
+// derivation living here, which is precisely the drift `resolveJobLeaseMs`'s
+// own comment exists to forbid: one function, one number, both executors.
+// =============================================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resetJobProfileWarnings = resetJobProfileWarnings;
+exports.resolveJobProfile = resolveJobProfile;
+exports.resolveMaxAttempts = resolveMaxAttempts;
+exports.resolveRenewIntervalMs = resolveRenewIntervalMs;
+exports.resolveLeaseHorizonMs = resolveLeaseHorizonMs;
+exports.buildClaimLeases = buildClaimLeases;
+const common_1 = require("@nestjs/common");
+const job_worker_1 = require("./job.worker");
+/**
+ * Shipped default attempt budget, repeated from `config/configuration.ts` for
+ * the reason every `configNumber` fallback in this module repeats one: a stub
+ * `ConfigService` must degrade to the shipped behaviour, not to `NaN`.
+ */
+const DEFAULT_MAX_ATTEMPTS = 3;
+/**
+ * The renewal interval is the lease divided by this, then clamped.
+ *
+ * THREE, so a node may miss TWO consecutive renewals — a GC pause, a dropped
+ * request, a control-plane restart — and still hold its job. Two would leave
+ * no margin at all; ten would spend nine times the requests to buy margin
+ * nobody needs.
+ */
+const RENEW_INTERVAL_DIVISOR = 3;
+/**
+ * Floor and ceiling for the derived renewal interval.
+ *
+ * The floor stops a short-lease type from turning a fleet into a renewal
+ * storm; the ceiling stops a very long lease (a six-hour dump) from producing
+ * an interval so long that a node which died in minute two keeps its job for
+ * an hour. Neither bound is configurable, because a bound that can be set
+ * wrong is the thing this file exists to remove.
+ */
+const MIN_RENEW_INTERVAL_MS = 5_000;
+const MAX_RENEW_INTERVAL_MS = 60_000;
+const logger = new common_1.Logger('JobExecutionProfile');
+/**
+ * Types already warned about, so a bad profile costs ONE log line rather than
+ * one per claim.
+ *
+ * MODULE-LEVEL FOR THE SAME REASON `unknownModeWarned` IS in `job.worker.ts`:
+ * these readers run on every claim and every settle, so an unlatched warning
+ * would be several lines a second, forever, burying the one line an operator
+ * needs to see.
+ */
+const warnedProfiles = new Set();
+/**
+ * Clears the warn-once latch above.
+ *
+ * FOR TESTS ONLY, exactly as `resetUnknownWorkerModeWarning` is: a latch that
+ * survived between cases would make "warns exactly once" pass for the first
+ * case and vacuously for every case after it. Nothing in the application calls
+ * this.
+  *
+  * @stability experimental
+ */
+function resetJobProfileWarnings() {
+    warnedProfiles.clear();
+}
+/**
+ * A numeric setting with a defensive fallback — the same shape, and for the
+ * same reason, as `JobWorker.configNumber` and `JobTerminalService.configNumber`.
+ */
+function configNumber(config, key, fallback) {
+    const value = config.get(key);
+    return typeof value === 'number' && Number.isFinite(value) ? value : fallback;
+}
+/** A finite number at or above `min` — the only shape either field may take. */
+function usable(value, min) {
+    return typeof value === 'number' && Number.isFinite(value) && value >= min;
+}
+/**
+ * The profile `handler` declares, or `undefined` when it declares none — or
+ * declares one this process refuses to act on.
+ *
+ * THE SINGLE VALIDATION POINT. Every consumer of a profile goes through here,
+ * so "is this profile usable" is answered once and identically for the lease,
+ * the timeout and the attempt budget; three separate opinions about a
+ * half-valid profile is how a type ends up with a lease derived from a ceiling
+ * nothing else honours.
+ *
+ * `maxRuntimeMs` must be finite and non-negative (`0` is the legitimate "no
+ * ceiling" value, so the floor is zero, not one); `maxAttempts` must be finite
+ * and at least `1`, because a budget of zero describes a job that may never be
+ * claimed — which is not a retry policy, it is a way to delete work silently.
+ *
+ * Returns `undefined` rather than throwing, for the reason
+ * `JobHandlerRegistry.get` returns `undefined`: the caller is on a claim path
+ * with a job to run, and a fork's typo in a profile must degrade that job to
+ * the deployment defaults, not take the worker down.
+  *
+  * @stability experimental
+ */
+function resolveJobProfile(handler) {
+    const profile = handler?.profile;
+    if (!profile) {
+        return undefined;
+    }
+    if (usable(profile.maxRuntimeMs, 0) && usable(profile.maxAttempts, 1)) {
+        return profile;
+    }
+    if (!warnedProfiles.has(handler.type)) {
+        warnedProfiles.add(handler.type);
+        logger.warn(`Job type "${handler.type}" declares an unusable execution profile ` +
+            `(maxRuntimeMs=${String(profile.maxRuntimeMs)}, ` +
+            `maxAttempts=${String(profile.maxAttempts)}); ignoring it and falling back to the ` +
+            'deployment-wide JOBS_JOB_TIMEOUT_MS and JOBS_MAX_ATTEMPTS. maxRuntimeMs must be a ' +
+            'finite number >= 0 (0 means no ceiling) and maxAttempts a finite number >= 1.');
+    }
+    return undefined;
+}
+/**
+ * The attempt budget for whatever type `handler` handles: its own, or the
+ * deployment's.
+ *
+ * READ BY BOTH GIVE-UP PATHS, and it has to be, which is the point of it
+ * being a function rather than a field read at either. `JobTerminalService`
+ * decides retry-or-fail for a job that reported back; `JobStuckService`
+ * decides requeue-or-fail for a job whose executor died without reporting
+ * anything. If only the first learned about per-type budgets, a
+ * `maxAttempts: 1` job that the terminal path correctly refused to retry would
+ * be found by the reaper, judged against the global 3, and REQUEUED —
+ * resurrecting the exact automatic retry the profile exists to forbid, on the
+ * one path nobody is watching. Both call this.
+ *
+ * `handler` is `JobHandler | undefined` rather than a type string because both
+ * callers already hold the handler (or the registry lookup that produced
+ * `undefined` for a type this process cannot run), and an unknown type has no
+ * profile to consult — it takes the deployment default, which is what it took
+ * before profiles existed.
+  *
+  * @stability experimental
+ */
+function resolveMaxAttempts(config, handler) {
+    const profile = resolveJobProfile(handler);
+    if (profile) {
+        return profile.maxAttempts;
+    }
+    return configNumber(config, 'jobs.maxAttempts', DEFAULT_MAX_ATTEMPTS);
+}
+/**
+ * How often a claimer should extend its lease, derived from the lease itself.
+ *
+ * DERIVED, NEVER DECLARED — the file header explains why a declared interval
+ * is the one number that can be wrong in a way nobody notices until a node
+ * starts losing jobs it is actively running. The clamp keeps both extremes
+ * survivable: a very short lease cannot produce a renewal storm, and a very
+ * long one cannot produce an interval that hides a dead node for an hour.
+ *
+ * Defensive against a nonsense lease for the same reason everything else here
+ * is: this is the divisor of a number that came out of a config read, and a
+ * `NaN` interval becomes a `setInterval(NaN)` firing every tick. An unusable
+ * lease degrades to the FLOOR rather than the ceiling — renewing too often
+ * costs a few requests, renewing too rarely loses the job.
+  *
+  * @stability experimental
+ */
+function resolveRenewIntervalMs(leaseMs) {
+    if (!usable(leaseMs, 1)) {
+        return MIN_RENEW_INTERVAL_MS;
+    }
+    return Math.min(MAX_RENEW_INTERVAL_MS, Math.max(MIN_RENEW_INTERVAL_MS, Math.floor(leaseMs / RENEW_INTERVAL_DIVISOR)));
+}
+/**
+ * How far into the future a lease may legitimately point, as a duration from
+ * "now": the LONGEST lease any registered handler could ask for, plus one
+ * grace.
+ *
+ * THIS IS THE REAPER'S FOURTH SIGNAL (#347), and it exists because the other
+ * three stopped covering a case they used to cover by accident. Until in-process
+ * renewal existed, `stuckRunningWhere`'s aged-claim clause reaped ANY row that
+ * had been `running` too long, lease or no lease — which caught a corrupt
+ * `lease_expires_at` for free, while also (this is the bug #347 fixes) reaping
+ * jobs that were running perfectly well. Narrowing that clause to unleased rows
+ * fixes the second problem and would silently drop the first: a lease pushed
+ * absurdly far out — a bug in a fork's own claim path, a clock jump on the
+ * writer, a hostile write — would then match NO signal at all and hold its
+ * dedup key forever.
+ *
+ * A lease further out than the longest lease ANY REGISTERED HANDLER COULD
+ * LEGITIMATELY ASK FOR is, by construction, not a live executor's promise.
+ * That is a fact this process can compute from what it already knows, without
+ * a new column, a new setting, or a per-row handler lookup inside the sweep.
+ *
+ * ⚠ THE DEPLOYMENT-WIDE LEASE IS ALWAYS IN THE MAXIMUM, even when the registry
+ * is empty. A `JOBS_WORKER_MODE=off` control plane may register no handlers at
+ * all and must still reap for its fleet; a `jobs` row may name a type this
+ * process does not register, and such a row was claimed on the global lease.
+ * Taking the max over registered handlers ALONE would give an empty registry a
+ * horizon of one grace period and reap every live row on the next sweep — the
+ * exact opposite of what this function is for.
+ *
+ * ⚠ THE TRADE-OFF, STATED HONESTLY: this ceiling MOVES WITH THE PROCESS
+ * READING IT. A deployment that removes a handler (or lowers its
+ * `maxRuntimeMs`, or lowers `JOBS_JOB_TIMEOUT_MS`) shortens the horizon, and
+ * in-flight rows of that type carrying the older, longer lease may be reaped
+ * on the next sweep. That is correct rather than merely acceptable: no process
+ * in that deployment can run those rows to completion any more, so requeueing
+ * them is the only outcome that is not "stuck until a human notices". It is
+ * also why the horizon is computed per sweep rather than cached — see
+ * `JobStuckService.leaseHorizon`.
+  *
+  * @stability experimental
+ */
+function resolveLeaseHorizonMs(config, registry) {
+    const longest = registry
+        .types()
+        .reduce((max, type) => Math.max(max, (0, job_worker_1.resolveJobLeaseMs)(config, resolveJobProfile(registry.get(type)))), 
+    // The floor is the lease an UNPROFILED type takes, which is also the
+    // lease every unregistered type takes. See the warning above.
+    (0, job_worker_1.resolveJobLeaseMs)(config, undefined));
+    return longest + job_worker_1.LEASE_GRACE_MS;
+}
+/**
+ * One `{ type, leaseMs }` pair per eligible type, for `ClaimOptions.leases`.
+ *
+ * THE REASON `ClaimOptions` TAKES A LIST RATHER THAN A NUMBER. A claim is not
+ * necessarily homogeneous: the in-process worker offers every registered type
+ * and takes whichever row is most urgent, and a worker node claims up to its
+ * concurrency ACROSS SEVERAL TYPES IN ONE STATEMENT. With per-type ceilings
+ * those rows no longer share a lease, and a single batch-wide number would
+ * have to be either the shortest (reaping the long job mid-run) or the longest
+ * (leaving a dead executor's short job held for hours). Neither is a lease;
+ * both are a compromise the caller should not have to make.
+ *
+ * BOTH CLAIMERS CALL THIS, which is the same "one function, one number, both
+ * executors" rule `resolveJobLeaseMs` states for the scalar case. A node and
+ * the API server must derive the SAME lease for the same type, because the
+ * lease is not a private detail of a claimer — it is the contract the reaper
+ * reads to decide a claim is dead.
+  *
+  * @stability experimental
+ */
+function buildClaimLeases(config, registry, eligibleTypes) {
+    return eligibleTypes.map((type) => ({
+        type,
+        leaseMs: (0, job_worker_1.resolveJobLeaseMs)(config, resolveJobProfile(registry.get(type))),
+    }));
+}
+//# sourceMappingURL=job-execution-profile.js.map

package/dist/jobs/job-execution-profile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-execution-profile.js","sourceRoot":"","sources":["../../src/jobs/job-execution-profile.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,sDAAsD;AACtD,gFAAgF;AAChF,EAAE;AACF,gFAAgF;AAChF,wEAAwE;AACxE,8EAA8E;AAC9E,+EAA+E;AAC/E,+EAA+E;AAC/E,8EAA8E;AAC9E,4EAA4E;AAC5E,yEAAyE;AACzE,2EAA2E;AAC3E,aAAa;AACb,EAAE;AACF,gFAAgF;AAChF,yEAAyE;AACzE,8EAA8E;AAC9E,mEAAmE;AACnE,EAAE;AACF,gFAAgF;AAChF,yEAAyE;AACzE,gFAAgF;AAChF,EAAE;AACF,6EAA6E;AAC7E,2DAA2D;AAC3D,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,4EAA4E;AAC5E,2EAA2E;AAC3E,4EAA4E;AAC5E,+EAA+E;AAC/E,8EAA8E;AAC9E,0EAA0E;AAC1E,uBAAuB;AACvB,EAAE;AACF,wEAAwE;AACxE,yEAAyE;AACzE,6EAA6E;AAC7E,8EAA8E;AAC9E,yEAAyE;AACzE,6EAA6E;AAC7E,wCAAwC;AACxC,EAAE;AACF,8EAA8E;AAC9E,wEAAwE;AACxE,8DAA8D;AAC9D,6EAA6E;AAC7E,8EAA8E;AAC9E,4EAA4E;AAC5E,uDAAuD;AACvD,EAAE;AACF,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,4EAA4E;AAC5E,8EAA8E;AAC9E,EAAE;AACF,gFAAgF;AAChF,4DAA4D;AAC5D,gFAAgF;AAChF,EAAE;AACF,gFAAgF;AAChF,uEAAuE;AACvE,mEAAmE;AACnE,8EAA8E;AAC9E,+EAA+E;AAC/E,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,uDAAuD;AACvD,EAAE;AACF,2EAA2E;AAC3E,+EAA+E;AAC/E,8EAA8E;AAC9E,8EAA8E;AAC9E,yEAAyE;AACzE,EAAE;AACF,gFAAgF;AAChF,2DAA2D;AAC3D,gFAAgF;AAChF,EAAE;AACF,8EAA8E;AAC9E,8EAA8E;AAC9E,8EAA8E;AAC9E,gEAAgE;AAChE,EAAE;AACF,4EAA4E;AAC5E,6EAA6E;AAC7E,0EAA0E;AAC1E,gFAAgF;;AA+FhF,0DAEC;AAuCD,8CAuBC;AAwBD,gDAQC;AAmBD,wDASC;AA2CD,sDAeC;AAkCD,4CASC;AA9TD,2CAAwC;AAKxC,6CAAiE;AAoCjE;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAEjC;;;;;;;;GAQG;AACH,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC,MAAM,MAAM,GAAG,IAAI,eAAM,CAAC,qBAAqB,CAAC,CAAC;AAEjD;;;;;;;;GAQG;AACH,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;AAEzC;;;;;;;;;GASG;AACH,SAAgB,uBAAuB;IACrC,cAAc,CAAC,KAAK,EAAE,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,MAAqB,EAAE,GAAW,EAAE,QAAgB;IACxE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAS,GAAG,CAAC,CAAC;IAEtC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAChF,CAAC;AAED,gFAAgF;AAChF,SAAS,MAAM,CAAC,KAAc,EAAE,GAAW;IACzC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,CAAC;AAC7E,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAgB,iBAAiB,CAAC,OAA+B;IAC/D,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAEjC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,CAAC,IAAI,CACT,aAAa,OAAO,CAAC,IAAI,2CAA2C;YAClE,iBAAiB,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI;YACjD,eAAe,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,yCAAyC;YACnF,oFAAoF;YACpF,+EAA+E,CAClF,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAgB,kBAAkB,CAAC,MAAqB,EAAE,OAA+B;IACvF,MAAM,OAAO,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE3C,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,OAAO,CAAC,WAAW,CAAC;IAC7B,CAAC;IAED,OAAO,YAAY,CAAC,MAAM,EAAE,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;AACxE,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAgB,sBAAsB,CAAC,OAAe;IACpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC;QACxB,OAAO,qBAAqB,CAAC;IAC/B,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CACb,qBAAqB,EACrB,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,sBAAsB,CAAC,CAAC,CAC9E,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,SAAgB,qBAAqB,CACnC,MAAqB,EACrB,QAA4B;IAE5B,MAAM,OAAO,GAAG,QAAQ;SACrB,KAAK,EAAE;SACP,MAAM,CACL,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CACZ,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAA,8BAAiB,EAAC,MAAM,EAAE,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjF,qEAAqE;IACrE,8DAA8D;IAC9D,IAAA,8BAAiB,EAAC,MAAM,EAAE,SAAS,CAAC,CACrC,CAAC;IAEJ,OAAO,OAAO,GAAG,2BAAc,CAAC;AAClC,CAAC;AAcD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAgB,gBAAgB,CAC9B,MAAqB,EACrB,QAA4B,EAC5B,aAAuB;IAEvB,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI;QACJ,OAAO,EAAE,IAAA,8BAAiB,EAAC,MAAM,EAAE,iBAAiB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;KAC1E,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/jobs/job-handler.interface.d.ts

@@ -0,0 +1,291 @@
+import { type Job } from './data/jobs-db';
+import type { z } from 'zod';
+import type { JobExecutionProfile } from './job-execution-profile';
+import type { JobSecretBroker } from './job-secret-broker';
+/**
+ * DI token for job handlers.
+ *
+ * Present for symmetry with `OBJECT_PROCESSOR` and
+ * `NOTIFICATION_CHANNEL_SENDERS`, and so a fork that prefers to collect
+ * handlers with a `multi`-style provider array has a token to collect them
+ * under. Note that the registration path this epic actually uses is
+ * SELF-REGISTRATION from each handler's own `OnModuleInit` — see
+ * `job-handler.registry.ts` for why, and `handlers/README.md` for the recipe.
+  *
+  * @stability experimental
+ */
+export declare const JOB_HANDLER: unique symbol;
+/**
+ * One job type: the contract a feature implements so the queue can run its
+ * work. Two members are required (`type`, `process`); every other member is
+ * optional and its PRESENCE is the declaration:
+ *
+ * - `label`: the admin list's display phrase (`jobTypeLabel`).
+ * - `profile`: `{ maxRuntimeMs, maxAttempts }`, exactly; the lease, renewal
+ *   and reaper patience are derived from `maxRuntimeMs`.
+ * - `nodeResultSchema` + `persistNodeResult`: both present makes the type
+ *   node-eligible (derived; there is no flag). Neither: server-only.
+ * - `nodeSecretBroker`: the type needs a per-job credential, brokered through
+ *   `POST /api/nodes/:id/jobs/:jobId/secret`, never persisted by a node.
+ * - `nodeOffloadEnabled`: a deployment's runtime switch for offering an
+ *   eligible type to nodes (read at claim time).
+ * - `deriveOutputKey`: where a node's output upload lands.
+ * - `canDelete`: veto an admin delete of a row of this type.
+ *
+ * Register with `registry.register(this)` from the handler's `onModuleInit`.
+ *
+ * @example
+ * ```ts
+ * @Injectable()
+ * export class ExportCsvHandler implements JobHandler, OnModuleInit {
+ *   readonly type = 'export.csv';
+ *   readonly label = 'CSV export';
+ *   constructor(private readonly registry: JobHandlerRegistry) {}
+ *   onModuleInit(): void { this.registry.register(this); }
+ *   async process(job: Job): Promise<void> { ... }
+ * }
+ * ```
+ *
+ * @extensionPoint hook
+ * @stability stable
+ */
+export interface JobHandler {
+    /**
+     * The `Job.type` value this handler is responsible for.
+     *
+     * The registry keys on this string and the worker dispatches on it, so it
+     * must be unique across every handler in the process — a duplicate
+     * overwrites, loudly (see `JobHandlerRegistry.register`). Use a dotted,
+     * lowercase, product-neutral key (`'email.send'`, `'export.csv'`), and
+     * treat it as PERMANENT once jobs of that type exist: rows outlive the
+     * handler that produced them, and renaming the key orphans every historical
+     * row and every pending job already queued under the old name.
+     */
+    readonly type: string;
+    /**
+     * The type's display label (`'Fleet sweep'`): a short phrase in sentence
+     * case, sized for a table cell, shown next to `type` in the admin job list,
+     * the insights page and the fleet's job types (`jobTypeLabel`). Optional:
+     * an unlabelled type shows its type string. A type whose handler is not
+     * loaded in a process registers its label with `registerJobTypeLabel`.
+     * Unlike `type`, a label may change at any time.
+     */
+    readonly label?: string;
+    /**
+     * Runs the job.
+     *
+     * THROW TO FAIL — the worker turns a rejection into `Job.lastError` plus a
+     * retry (or a terminal `failed` status once the attempt budget is spent).
+     * Returning normally means the work is done and durable; do not return
+     * before the writes this job is responsible for have committed.
+     *
+     * Should be IDEMPOTENT wherever the underlying operation allows it. A job
+     * can legitimately run more than once: a retry after a partial failure, or
+     * a lease that expired because the executing process was killed mid-run and
+     * another worker reclaimed it. The queue guarantees at-least-once, never
+     * exactly-once.
+     */
+    process(job: Job): Promise<void>;
+    /**
+     * How this type is allowed to run, when the deployment-wide defaults are
+     * wrong for it.
+     *
+     * OPTIONAL, AND OMITTING IT IS THE NORMAL ANSWER. A handler with no profile
+     * runs on `JOBS_JOB_TIMEOUT_MS` and `JOBS_MAX_ATTEMPTS` exactly as every
+     * handler did before profiles existed. Declare one only when this type is
+     * genuinely unlike the rest of the queue — a job that legitimately runs for
+     * hours, or one that must never be automatically retried.
+     *
+     * ⚠ TWO NUMBERS, AND THERE WILL ONLY EVER BE TWO. You may not declare a
+     * lease length here, and you may not declare a renewal interval: those are
+     * DERIVED from `maxRuntimeMs` (`resolveJobLeaseMs`,
+     * `resolveRenewIntervalMs`) precisely because they are the values that can
+     * DISAGREE with it. A lease shorter than the runtime ceiling is a job that
+     * reaps itself into duplicate execution; a renewal interval at or above the
+     * lease is a renewal that always arrives too late. Deriving them makes both
+     * states unrepresentable — the same argument this file's header already
+     * makes against a `nodeEligible` flag, applied to durations. See
+     * `job-execution-profile.ts` for the full version, and do not add a third
+     * field to that interface.
+     */
+    readonly profile?: JobExecutionProfile;
+    /**
+     * Validates the result a remote worker node posts back for this job type.
+     *
+     * PRESENT ONLY ON NODE-ELIGIBLE HANDLERS, and only ever together with
+     * `persistNodeResult` — see the file header: both members or neither, and
+     * exactly one of the two means server-only.
+     *
+     * The value it parses arrives from a machine the API server does not
+     * control, so this schema is a trust boundary, not a convenience: it is the
+     * only thing standing between an arbitrary remote body and
+     * `persistNodeResult`'s writes.
+     */
+    readonly nodeResultSchema?: z.ZodType;
+    /**
+     * Writes down a node-computed result that `nodeResultSchema` has already
+     * validated.
+     *
+     * PRESENT ONLY ON NODE-ELIGIBLE HANDLERS (see `nodeResultSchema`).
+     *
+     * PERSIST ONLY. No recomputation, no re-downloading the input, no second
+     * call to whatever provider the node used — the file header explains why
+     * that rule is what keeps a node from needing database access at all. If
+     * this method cannot do its job without redoing the work, the type is not
+     * node-eligible.
+     *
+     * Throwing here fails the job exactly as throwing from `process` does.
+     */
+    persistNodeResult?(job: Job, result: unknown): Promise<void>;
+    /**
+     * Where this type's node-written output must land, when
+     * `node-outputs/<jobId>/<uuid>` is the wrong answer.
+     *
+     * OPTIONAL, AND OMITTING IT IS THE NORMAL ANSWER. A handler that does not
+     * implement this gets the data plane's default key, unchanged — a fresh,
+     * job-attributable, never-reused location under `node-outputs/`, which is
+     * the right shape for any artifact nothing outside the job ever names.
+     * Implement it only when the artifact's location is part of its contract:
+     * a row somewhere records the key, a retention sweep lists a prefix, or a
+     * download endpoint reconstructs it. See the file header for the full
+     * argument and the two rejected alternatives.
+     *
+     * ⚠ THIS IS STILL THE SERVER CHOOSING. It runs in the API process, from the
+     * handler that owns the artifact, with the `Job` row as its only input —
+     * nothing from the node's request reaches it, and a node-supplied `key` is
+     * refused with a 400 before this is ever called.
+     *
+     * ⚠ IT MUST BE IDEMPOTENT PER JOB, and that is this member's one hard
+     * requirement. A node asks for an upload URL more than once as a matter of
+     * course: a transfer that timed out, a response lost on the way back, a
+     * process restarted while holding the lease. Every one of those calls must
+     * yield THE SAME KEY. A derivation that mints something new each time —
+     * inserting an artifact row, or interpolating `randomUUID()`/`Date.now()` —
+     * produces a second artifact per retry, and the row the rest of the system
+     * reads then points at bytes the node never finished writing. Derive from
+     * values already fixed on the job, or re-read the artifact row this job
+     * already created and return its recorded key. #351 makes that structural
+     * for the database backup with a `@unique` `jobId` on its run row, so
+     * "re-read by `jobId`, return the existing key" is enforced by the database
+     * rather than by the handler remembering to.
+     *
+     * The returned key must satisfy the data plane's `SAFE_STORAGE_KEY`; one
+     * that does not is refused server-side with a 500 and the node gets no URL
+     * at all. Throwing here fails the request the same way — no URL is minted,
+     * and the node's correct response is the same as for any other refusal.
+     */
+    deriveOutputKey?(job: Job): Promise<string>;
+    /**
+     * May a node run THIS type in THIS deployment, right now?
+     *
+     * OPTIONAL, AND OMITTING IT IS THE NORMAL ANSWER — a node-eligible type with
+     * no gate is offered to nodes, which is what every type before #352 did.
+     * Implement it only when a deployment must be able to say "not this
+     * workload" about a type that is structurally perfectly capable of running
+     * remotely.
+     *
+     * ⚠ IT IS A POLICY READ, NOT A DECLARATION, and that is why it is a METHOD
+     * rather than a `readonly nodeOffloadEnabled: boolean`. Everything else on
+     * this interface states a fact about the TYPE that is fixed at build time;
+     * this one asks a question whose answer an administrator changes at 3pm on a
+     * Tuesday. A boolean field would be read once at registration and be wrong
+     * from then on. `NodesService.nodeEligibleTypes` awaits this at CLAIM time,
+     * beside the two settings intersections it already performs, so a switch
+     * flipped in the admin UI takes effect on the next claim and not at the next
+     * deploy.
+     *
+     * ⚠ IT DOES NOT CHANGE NODE ELIGIBILITY. Eligibility is derived from
+     * `nodeResultSchema` + `persistNodeResult` and stays derived from them: this
+     * gate cannot make a server-only type runnable on a node, and a `false` here
+     * does not remove the type from `serverOnlyTypes()`'s complement. It decides
+     * only what THIS deployment OFFERS today — the same runtime intersection the
+     * secret-broker filter performs, and the same reason it is not a registry
+     * mutation.
+     *
+     * REJECTED: putting the deployment's answer in `NodesService` itself — a
+     * `if (type === BACKUP_JOB_TYPE) …` reading the `databaseBackup` namespace.
+     * That is the central dispatch table this file's header exists to abolish,
+     * one arm long, and it makes the nodes module depend on a feature module's
+     * settings shape. Asking the handler keeps the knowledge where the feature
+     * is: `DatabaseBackupRunHandler` reads its own
+     * `databaseBackup .nodeOffloadEnabled`, and a fork's handler reads whatever its own feature
+     * calls the same idea.
+     *
+     * Throwing is not a way to say "no": it fails the whole claim, which is a
+     * far bigger hammer than withholding one type. Report the settings read's
+     * failure as `false` if it can fail at all.
+     */
+    nodeOffloadEnabled?(): Promise<boolean>;
+    /**
+     * Mints the short-lived credential a REMOTE executor of this type needs, and
+     * destroys it again.
+     *
+     * OPTIONAL, AND OMITTING IT IS THE NORMAL ANSWER — almost every job type is
+     * pure compute over presigned bytes and needs no credential at all. Its
+     * PRESENCE is the declaration that this type does, exactly as the presence of
+     * `nodeResultSchema` + `persistNodeResult` is the declaration that the type is
+     * node-eligible; there is no `requiresSecret` string and no switch keyed on
+     * one (see the file header, and `job-secret-broker.ts` for the whole
+     * argument).
+     *
+     * ⚠ A BROKER IS NOT A PERMISSION TO USE ONE. Whether a node in THIS
+     * deployment may hold a credential to THIS database is a trust-boundary
+     * decision an administrator makes, not one a handler makes: it is the
+     * `nodes.jobSecretBrokerEnabled` system setting, default OFF. With it off,
+     * `POST /api/nodes/:id/jobs/:jobId/secret` refuses with a named reason AND
+     * the type is filtered out of the node claim, so a node never even sees the
+     * job. That filter is a runtime intersection, never a mutation of the
+     * registry — see `NodesService.nodeEligibleTypes`.
+     *
+     * ⚠ NOTHING IT RETURNS MAY BE PERSISTED EXCEPT THE HANDLE. The material is
+     * serialised into one HTTP response and dropped; `job_node_secrets` has no
+     * column that could hold it. See `IssuedJobSecret`.
+     */
+    readonly nodeSecretBroker?: JobSecretBroker;
+    /**
+     * May an administrator delete THIS job row, right now? (#480)
+     *
+     * Returns a human-readable REASON TO REFUSE, or `null` to allow. The reason
+     * is shown to the operator verbatim (it becomes the 409's `message`), so
+     * write it as an instruction: what is wrong and what to do instead.
+     *
+     * OPTIONAL, AND OMITTING IT IS THE NORMAL ANSWER. A handler without it gets
+     * exactly what every type had before #480: any non-running row may be
+     * deleted. Implement it only when a NON-TERMINAL row of this type is load-
+     * bearing for state the feature owns outside the `jobs` table — a row whose
+     * deletion would strand that state with nothing left to advance it.
+     *
+     * ⚠ WHY THE HANDLER ANSWERS AND NOT THE ADMIN SERVICE. A delete is not a
+     * settlement: no `job.settled` fires, so no listener a feature has attached
+     * to the queue's lifecycle ever learns the row is gone. The only moment the
+     * feature can object is BEFORE the delete, and the only party that knows
+     * whether it should is the feature. The rejected alternative — an
+     * `if (type === BROADCAST_CHUNK_TYPE) …` in `JobAdminService` reading
+     * `notification_broadcasts` — is the central dispatch table this file's
+     * header exists to abolish, and it would make the jobs module depend on a
+     * feature's schema. Same argument as `nodeOffloadEnabled`: a POLICY question
+     * the owning feature answers, so the queue stays free of feature knowledge.
+     *
+     * ⚠ CONSULTED BY ONE PATH ONLY: `JobAdminService.remove`
+     * (`DELETE /api/admin/jobs/:id`). The history purge never asks, because it
+     * deletes only terminal rows, and a terminal row is history, never
+     * machinery. An implementation should therefore answer `null` for a job
+     * whose own status is terminal (`succeeded`/`failed`) — deleting history is
+     * always fine — and object only to a row that could still run.
+     *
+     * ⚠ A CHEAP READ, NEVER A WRITE. It runs inside an admin HTTP request,
+     * before the delete and outside any transaction; it must not mutate
+     * anything (in particular, it must not "helpfully" cancel the feature's own
+     * state — the operator asked to delete a job, not to cancel a broadcast).
+     * It is advisory against a concurrent state change by construction: the
+     * delete that follows is guarded only against the row becoming `running`.
+     *
+     * ⚠ A THROW REFUSES THE DELETE (fail closed). The admin service logs the
+     * error and answers the same 409 with a generic reason
+     * (`details.reason: 'owner_check_failed'`). Deleting is irreversible and a
+     * refusal is not — an operator can simply try again — so when the owner
+     * cannot say "yes", the answer is "no".
+     */
+    canDelete?(job: Job): Promise<string | null>;
+}
+//# sourceMappingURL=job-handler.interface.d.ts.map

package/dist/jobs/job-handler.interface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-handler.interface.d.ts","sourceRoot":"","sources":["../../src/jobs/job-handler.interface.ts"],"names":[],"mappings":"AA+KA,OAAO,EAAE,KAAK,GAAG,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE3D;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,WAAW,eAAwB,CAAC;AAEjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,UAAU;IACzB;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;;;;;OAOG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAExB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjC;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAEvC;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC;IAEtC;;;;;;;;;;;;;OAaG;IACH,iBAAiB,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoCG;IACH,eAAe,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,kBAAkB,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAExC;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,eAAe,CAAC;IAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2CG;IACH,SAAS,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC9C"}