@cat-factory/worker 0.175.0 → 0.177.0

package/dist/infrastructure/containers/RunContainer.d.ts

@@ -0,0 +1,150 @@
+import { Container } from '@cloudflare/containers';
+import type { StopParams } from '@cloudflare/containers';
+import type { Env } from '../env';
+import { type StopObservation } from './stopCause';
+/**
+ * The behaviour every per-run Cloudflare Container shares: one Durable Object instance per run
+ * id hosts that run's sequence of jobs, the harness listens on 8080, and the base
+ * `Container.fetch` proxies inbound requests there once it has booted.
+ *
+ * The two concrete classes ({@link import('./ExecutionContainer').ExecutionContainer} and
+ * {@link import('./DeployContainer').DeployContainer}) exist only because a Cloudflare
+ * Container's IMAGE is pinned per container class by the wrangler `[[containers]]` block. They
+ * are two bindings onto one behaviour, which is why that behaviour lives here rather than being
+ * kept in step by hand across both.
+ *
+ * No long-lived secrets are configured: the image carries none, and every per-job credential
+ * (the VCS token, the LLM session token, an apiserver token) arrives in the `/jobs` request
+ * body at dispatch time. The one exception is `HARNESS_SHARED_SECRET` below, an inbound-auth
+ * shared secret rather than a tenant credential.
+ */
+export declare abstract class RunContainer extends Container<Env> {
+    /** The harness HTTP server port (matches each image's Dockerfile ENTRYPOINT/EXPOSE). */
+    defaultPort: number;
+    envVars: Record<string, string>;
+    sleepAfter: string;
+    /**
+     * Whether the stop this container is about to observe is one WE asked for: the idle reclaim in
+     * {@link onActivityExpired} and the deliberate teardown in {@link shutdown}, both of which stop
+     * the container by signalling it.
+     *
+     * It exists because the resulting exit state is evidence about nothing. We sent the signal, so
+     * the code that comes back describes our own request (and escalates to a SIGKILL 137 whenever
+     * the harness does not exit inside the platform's grace period), while the account the run
+     * actually needs is the cause already recorded beside it. Left recorded, that exit is read back
+     * as the container's cause of death and reported as "most often an out-of-memory kill" under a
+     * verdict that says the platform reclaimed an idle container.
+     *
+     * In-memory rather than persisted, and reset by {@link onStart}: it describes ONE stop of ONE
+     * container life, and the isolate cannot outlive the stop it is set for. A container that comes
+     * back up has a real death ahead of it again.
+     */
+    private selfInitiatedStop;
+    /**
+     * A container life is starting, so nothing from here on is a stop we asked for.
+     *
+     * The base class flushes any deferred `onStop` from the PREVIOUS life before it calls this, so
+     * the reset can never land between a self-initiated stop and the hook that observes it.
+     */
+    onStart(): Promise<void>;
+    /**
+     * Record that THIS run's container was drained by a new-version rollout (a deploy, exit 143)
+     * rather than crashing. The transport's next job poll (which 404s once the container restarts
+     * empty) reads this back through {@link recentStopObservation}, so the engine
+     * recovers it on the larger transient budget instead of failing the run as a crash.
+     * Persisted to DO storage (not in-memory) so it survives the isolate reset a combined
+     * worker+container deploy causes.
+     */
+    onError(error: unknown): Promise<unknown>;
+    /**
+     * Record EVERY stop, with whatever this hook can say about it.
+     *
+     * Two things ride the same write. A `runtime_signal` SIGTERM (143) is a rollout drain, which
+     * depending on the runtime version surfaces here instead of (or as well as) through `onError`,
+     * so it is recorded as that cause the same way. And the `{ exitCode, reason }` pair itself is
+     * recorded for every stop, cause or not, because it is the ONLY account of a death this
+     * runtime can keep: a Cloudflare Container's stdout goes to the deployment's Workers logs and
+     * nothing here can read it back, so without this an OOM-killed agent reaches the operator as
+     * "container evicted or crashed" and nothing else (finding D1).
+     *
+     * Recording an exit changes no verdict: the transient/crash classification still comes from
+     * the cause alone, so a plain crash keeps spending the crash budget and merely says why.
+     *
+     * A stop WE asked for is the exception, and records nothing. Its exit state is the echo of our
+     * own signal rather than an observation, and the cause that explains it has already been
+     * recorded by whoever asked (see {@link selfInitiatedStop}).
+     *
+     * Both rules live in {@link observationForStop}, beside the other half of "what does this stop
+     * mean" (`isRolloutSignal`) and where a plain unit test can reach them: nothing in this class
+     * can be exercised without a real Durable Object.
+     */
+    onStop(params: StopParams): Promise<void>;
+    /**
+     * The idle window elapsed. Record it as the reclaim cause it is, then reclaim as the base
+     * class would.
+     *
+     * This container is kept warm ONLY by the driver's job polls, so an idle expiry with a job
+     * still outstanding means the backend stopped polling for longer than `sleepAfter`: a
+     * poll-scheduling hiccup, not the workload dying. Unrecorded, the resulting 404 poll is
+     * indistinguishable from an OOM: it spends the single crash-eviction budget, and a second
+     * hiccup in the same step then fails a healthy run (stuck-run audit F12).
+     *
+     * This hook cannot tell that case from the ROUTINE one (the run parked on a human decision and
+     * nothing is running), because only the harness knows whether a job is still live and asking
+     * it would itself be activity. So the marker is minted for both, and what keeps that honest is
+     * the boundary in {@link fetch}: the routine marker is dropped by the next dispatch, so it
+     * cannot still be lying around to excuse the following step's crash.
+     *
+     * Nothing is recorded when the container is already gone: the base class stops nothing in
+     * that case, so a marker would be attributing a reclaim that never happened.
+     */
+    onActivityExpired(): Promise<void>;
+    /**
+     * Proxy an inbound harness call, and take a JOB ACCEPTANCE as the end of whatever this
+     * container previously observed about itself.
+     *
+     * A stop cause explains the death of a container that was serving the jobs outstanding when it
+     * was recorded. `POST /jobs` answering 2xx says a new job starts here, so nothing recorded
+     * before it can account for that job's death. Without the boundary the routine case poisons
+     * the rare one: a run parks on a human decision, its container idles out with nothing running,
+     * and the `idle` marker that leaves behind is still inside its (deliberately wide) window to
+     * excuse a genuine OOM in the NEXT step as transient churn.
+     *
+     * It has to be the acceptance rather than the container starting: a 404 poll BOOTS the
+     * container on its way to discovering the job is gone, so clearing on start would drop the
+     * record moments before the read that exists to consume it.
+     */
+    fetch(request: Request): Promise<Response>;
+    /**
+     * What this run's container observed about its own stop, for the transport to read over RPC
+     * after `jobId`'s poll 404s: the `cause` that tells a reclaim apart from a crash, and the
+     * `exit` state that is the only surviving account of the death itself. An EMPTY observation
+     * means nothing this container saw explains that 404, so the caller reports a bare crash.
+     *
+     * Claimed by the polling job rather than deleted, so a retried durable poll step re-reads the
+     * same answer while a different job still finds it spent. See {@link takeStopCause}.
+     */
+    recentStopObservation(jobId: string): Promise<StopObservation>;
+    /**
+     * Reclaim this container now (SIGKILL via the base class), rather than waiting for the
+     * `sleepAfter` idle timer. Called over RPC when a run settles or faults, so a leaked instance
+     * isn't billed while idle. Best-effort and idempotent: destroying an already-stopped
+     * container is a no-op, and we swallow any error so the caller's failure handling is never
+     * derailed by cleanup.
+     *
+     * It is also where the stop record's life ENDS. A record is written on every stop and deleted
+     * only when a NEW job is accepted, so without this the last one a run ever observed sits in
+     * that run's Durable Object for good: the run is over, no dispatch is coming to clear it, and
+     * the value is a diagnostic nobody will read again. Deleting it here is what keeps the whole
+     * mechanism transient rather than a per-run key that accumulates for the lifetime of the
+     * deployment.
+     *
+     * Ordered after the teardown so the destroy's own `onStop` cannot land behind the delete,
+     * belt-and-braces with {@link selfInitiatedStop}, which stops that hook writing at all.
+     */
+    shutdown(): Promise<void>;
+    private record;
+    /** The DO storage, narrowed to what the stop-cause bookkeeping uses. */
+    private get storage();
+}
+//# sourceMappingURL=RunContainer.d.ts.map

package/dist/infrastructure/containers/RunContainer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RunContainer.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/containers/RunContainer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAExD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,QAAQ,CAAA;AAEjC,OAAO,EAML,KAAK,eAAe,EAErB,MAAM,aAAa,CAAA;AAEpB;;;;;;;;;;;;;;;GAeG;AACH,8BAAsB,YAAa,SAAQ,SAAS,CAAC,GAAG,CAAC;IACvD,wFAAwF;IAC/E,WAAW,SAAO;IAKlB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAElC;IAMG,UAAU,SAAQ;IAE3B;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,iBAAiB,CAAQ;IAEjC;;;;;OAKG;IACY,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAGtC;IAED;;;;;;;OAOG;IACY,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAIvD;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACY,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAGvD;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACY,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAShD;IAED;;;;;;;;;;;;;;OAcG;IACY,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAWxD;IAED;;;;;;;;OAQG;IACG,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAEnE;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAc9B;IAED,OAAO,CAAC,MAAM;IAId,wEAAwE;IACxE,OAAO,KAAK,OAAO,GAElB;CACF"}

package/dist/infrastructure/containers/RunContainer.js

@@ -0,0 +1,229 @@
+import { Container } from '@cloudflare/containers';
+import { runBestEffort } from '@cat-factory/kernel';
+import { logger } from '../observability/logger';
+import { clearStopCause, isRolloutSignal, observationForStop, recordStopCause, takeStopCause, } from './stopCause';
+/**
+ * The behaviour every per-run Cloudflare Container shares: one Durable Object instance per run
+ * id hosts that run's sequence of jobs, the harness listens on 8080, and the base
+ * `Container.fetch` proxies inbound requests there once it has booted.
+ *
+ * The two concrete classes ({@link import('./ExecutionContainer').ExecutionContainer} and
+ * {@link import('./DeployContainer').DeployContainer}) exist only because a Cloudflare
+ * Container's IMAGE is pinned per container class by the wrangler `[[containers]]` block. They
+ * are two bindings onto one behaviour, which is why that behaviour lives here rather than being
+ * kept in step by hand across both.
+ *
+ * No long-lived secrets are configured: the image carries none, and every per-job credential
+ * (the VCS token, the LLM session token, an apiserver token) arrives in the `/jobs` request
+ * body at dispatch time. The one exception is `HARNESS_SHARED_SECRET` below, an inbound-auth
+ * shared secret rather than a tenant credential.
+ */
+export class RunContainer extends Container {
+    /** The harness HTTP server port (matches each image's Dockerfile ENTRYPOINT/EXPOSE). */
+    defaultPort = 8080;
+    // When configured, hand the inbound-auth shared secret to the harness so it rejects any /jobs
+    // call that doesn't present the matching `x-harness-secret` header (which the transport
+    // sends). Omitted when unset, leaving the harness open as before.
+    envVars = this.env.HARNESS_SHARED_SECRET
+        ? { HARNESS_SHARED_SECRET: this.env.HARNESS_SHARED_SECRET }
+        : {};
+    // A job is dispatched, then polled every ~15s while it runs, so the instance stays warm for
+    // the job's duration without holding a single request open. Polling is the ONLY thing that
+    // keeps it warm, which is why an elapsed window is recorded as a reclaim cause rather than
+    // left to read as a crash. See `onActivityExpired`.
+    sleepAfter = '10m';
+    /**
+     * Whether the stop this container is about to observe is one WE asked for: the idle reclaim in
+     * {@link onActivityExpired} and the deliberate teardown in {@link shutdown}, both of which stop
+     * the container by signalling it.
+     *
+     * It exists because the resulting exit state is evidence about nothing. We sent the signal, so
+     * the code that comes back describes our own request (and escalates to a SIGKILL 137 whenever
+     * the harness does not exit inside the platform's grace period), while the account the run
+     * actually needs is the cause already recorded beside it. Left recorded, that exit is read back
+     * as the container's cause of death and reported as "most often an out-of-memory kill" under a
+     * verdict that says the platform reclaimed an idle container.
+     *
+     * In-memory rather than persisted, and reset by {@link onStart}: it describes ONE stop of ONE
+     * container life, and the isolate cannot outlive the stop it is set for. A container that comes
+     * back up has a real death ahead of it again.
+     */
+    selfInitiatedStop = false;
+    /**
+     * A container life is starting, so nothing from here on is a stop we asked for.
+     *
+     * The base class flushes any deferred `onStop` from the PREVIOUS life before it calls this, so
+     * the reset can never land between a self-initiated stop and the hook that observes it.
+     */
+    async onStart() {
+        this.selfInitiatedStop = false;
+        await super.onStart();
+    }
+    /**
+     * Record that THIS run's container was drained by a new-version rollout (a deploy, exit 143)
+     * rather than crashing. The transport's next job poll (which 404s once the container restarts
+     * empty) reads this back through {@link recentStopObservation}, so the engine
+     * recovers it on the larger transient budget instead of failing the run as a crash.
+     * Persisted to DO storage (not in-memory) so it survives the isolate reset a combined
+     * worker+container deploy causes.
+     */
+    async onError(error) {
+        if (isRolloutSignal(error))
+            await this.record({ cause: 'rollout' });
+        // Preserve the base behaviour (log + rethrow) so nothing else changes.
+        return super.onError(error);
+    }
+    /**
+     * Record EVERY stop, with whatever this hook can say about it.
+     *
+     * Two things ride the same write. A `runtime_signal` SIGTERM (143) is a rollout drain, which
+     * depending on the runtime version surfaces here instead of (or as well as) through `onError`,
+     * so it is recorded as that cause the same way. And the `{ exitCode, reason }` pair itself is
+     * recorded for every stop, cause or not, because it is the ONLY account of a death this
+     * runtime can keep: a Cloudflare Container's stdout goes to the deployment's Workers logs and
+     * nothing here can read it back, so without this an OOM-killed agent reaches the operator as
+     * "container evicted or crashed" and nothing else (finding D1).
+     *
+     * Recording an exit changes no verdict: the transient/crash classification still comes from
+     * the cause alone, so a plain crash keeps spending the crash budget and merely says why.
+     *
+     * A stop WE asked for is the exception, and records nothing. Its exit state is the echo of our
+     * own signal rather than an observation, and the cause that explains it has already been
+     * recorded by whoever asked (see {@link selfInitiatedStop}).
+     *
+     * Both rules live in {@link observationForStop}, beside the other half of "what does this stop
+     * mean" (`isRolloutSignal`) and where a plain unit test can reach them: nothing in this class
+     * can be exercised without a real Durable Object.
+     */
+    async onStop(params) {
+        const observed = observationForStop(params, this.selfInitiatedStop);
+        if (observed)
+            await this.record(observed);
+    }
+    /**
+     * The idle window elapsed. Record it as the reclaim cause it is, then reclaim as the base
+     * class would.
+     *
+     * This container is kept warm ONLY by the driver's job polls, so an idle expiry with a job
+     * still outstanding means the backend stopped polling for longer than `sleepAfter`: a
+     * poll-scheduling hiccup, not the workload dying. Unrecorded, the resulting 404 poll is
+     * indistinguishable from an OOM: it spends the single crash-eviction budget, and a second
+     * hiccup in the same step then fails a healthy run (stuck-run audit F12).
+     *
+     * This hook cannot tell that case from the ROUTINE one (the run parked on a human decision and
+     * nothing is running), because only the harness knows whether a job is still live and asking
+     * it would itself be activity. So the marker is minted for both, and what keeps that honest is
+     * the boundary in {@link fetch}: the routine marker is dropped by the next dispatch, so it
+     * cannot still be lying around to excuse the following step's crash.
+     *
+     * Nothing is recorded when the container is already gone: the base class stops nothing in
+     * that case, so a marker would be attributing a reclaim that never happened.
+     */
+    async onActivityExpired() {
+        if (this.ctx.container?.running) {
+            await this.record({ cause: 'idle' });
+            // The reclaim below is us signalling the container, so the exit it reports is our own
+            // request coming back (a SIGKILL 137 whenever the harness does not exit inside the grace
+            // period) and not a death to attribute. The cause just recorded is the whole account.
+            this.selfInitiatedStop = true;
+        }
+        await super.onActivityExpired();
+    }
+    /**
+     * Proxy an inbound harness call, and take a JOB ACCEPTANCE as the end of whatever this
+     * container previously observed about itself.
+     *
+     * A stop cause explains the death of a container that was serving the jobs outstanding when it
+     * was recorded. `POST /jobs` answering 2xx says a new job starts here, so nothing recorded
+     * before it can account for that job's death. Without the boundary the routine case poisons
+     * the rare one: a run parks on a human decision, its container idles out with nothing running,
+     * and the `idle` marker that leaves behind is still inside its (deliberately wide) window to
+     * excuse a genuine OOM in the NEXT step as transient churn.
+     *
+     * It has to be the acceptance rather than the container starting: a 404 poll BOOTS the
+     * container on its way to discovering the job is gone, so clearing on start would drop the
+     * record moments before the read that exists to consume it.
+     */
+    async fetch(request) {
+        const res = await super.fetch(request);
+        if (isJobDispatch(request) && res.ok) {
+            // Best-effort: the job is accepted and running by now, so bookkeeping must never turn a
+            // live dispatch into a failure. A drop is reported rather than swallowed, because the only
+            // symptom otherwise is a crash quietly misread as churn some minutes later.
+            await runBestEffort(logger, 'clear container stop cause on dispatch', () => clearStopCause(this.storage));
+        }
+        return res;
+    }
+    /**
+     * What this run's container observed about its own stop, for the transport to read over RPC
+     * after `jobId`'s poll 404s: the `cause` that tells a reclaim apart from a crash, and the
+     * `exit` state that is the only surviving account of the death itself. An EMPTY observation
+     * means nothing this container saw explains that 404, so the caller reports a bare crash.
+     *
+     * Claimed by the polling job rather than deleted, so a retried durable poll step re-reads the
+     * same answer while a different job still finds it spent. See {@link takeStopCause}.
+     */
+    async recentStopObservation(jobId) {
+        return takeStopCause(this.storage, Date.now(), jobId);
+    }
+    /**
+     * Reclaim this container now (SIGKILL via the base class), rather than waiting for the
+     * `sleepAfter` idle timer. Called over RPC when a run settles or faults, so a leaked instance
+     * isn't billed while idle. Best-effort and idempotent: destroying an already-stopped
+     * container is a no-op, and we swallow any error so the caller's failure handling is never
+     * derailed by cleanup.
+     *
+     * It is also where the stop record's life ENDS. A record is written on every stop and deleted
+     * only when a NEW job is accepted, so without this the last one a run ever observed sits in
+     * that run's Durable Object for good: the run is over, no dispatch is coming to clear it, and
+     * the value is a diagnostic nobody will read again. Deleting it here is what keeps the whole
+     * mechanism transient rather than a per-run key that accumulates for the lifetime of the
+     * deployment.
+     *
+     * Ordered after the teardown so the destroy's own `onStop` cannot land behind the delete,
+     * belt-and-braces with {@link selfInitiatedStop}, which stops that hook writing at all.
+     */
+    async shutdown() {
+        this.selfInitiatedStop = true;
+        try {
+            await this.destroy();
+        }
+        catch {
+            // silent-catch-ok: already gone / not running, so there is nothing to reclaim and nothing
+            // for an operator to do about it.
+        }
+        // Best-effort: the container is reclaimed either way, and a retained key is a tidiness
+        // problem, never a correctness one. Reported rather than swallowed so a storage backend
+        // failing every delete does not stay invisible.
+        await runBestEffort(logger, 'clear container stop cause on shutdown', () => clearStopCause(this.storage));
+    }
+    record(observed) {
+        return recordStopCause(this.storage, observed, Date.now());
+    }
+    /** The DO storage, narrowed to what the stop-cause bookkeeping uses. */
+    get storage() {
+        return this.ctx.storage;
+    }
+}
+/**
+ * Whether an inbound request is the transport starting a job (`POST /jobs`), as opposed to
+ * polling or stopping one (`GET`/`DELETE /jobs/{id}`).
+ *
+ * The harness route is matched here rather than announced over a separate RPC because the
+ * dispatch already passes through this object, and an extra round trip would be one more thing
+ * that can fail between "job accepted" and "record cleared". The URL is parsed rather than
+ * string-compared so a query string or a trailing slash cannot make a dispatch look like
+ * something else.
+ */
+function isJobDispatch(request) {
+    if (request.method !== 'POST')
+        return false;
+    try {
+        return new URL(request.url).pathname.replace(/\/+$/, '') === '/jobs';
+    }
+    catch {
+        // silent-catch-ok: an unparseable URL is not a dispatch, which is the whole question here.
+        return false;
+    }
+}
+//# sourceMappingURL=RunContainer.js.map

package/dist/infrastructure/containers/RunContainer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RunContainer.js","sourceRoot":"","sources":["../../../src/infrastructure/containers/RunContainer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAElD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAEnD,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAA;AAChD,OAAO,EACL,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,eAAe,EAGf,aAAa,GACd,MAAM,aAAa,CAAA;AAEpB;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAgB,YAAa,SAAQ,SAAc;IACvD,wFAAwF;IAC/E,WAAW,GAAG,IAAI,CAAA;IAE3B,8FAA8F;IAC9F,wFAAwF;IACxF,kEAAkE;IACzD,OAAO,GAA2B,IAAI,CAAC,GAAG,CAAC,qBAAqB;QACvE,CAAC,CAAC,EAAE,qBAAqB,EAAE,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE;QAC3D,CAAC,CAAC,EAAE,CAAA;IAEN,4FAA4F;IAC5F,2FAA2F;IAC3F,2FAA2F;IAC3F,oDAAoD;IAC3C,UAAU,GAAG,KAAK,CAAA;IAE3B;;;;;;;;;;;;;;;OAeG;IACK,iBAAiB,GAAG,KAAK,CAAA;IAEjC;;;;;OAKG;IACM,KAAK,CAAC,OAAO;QACpB,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAA;QAC9B,MAAM,KAAK,CAAC,OAAO,EAAE,CAAA;IACvB,CAAC;IAED;;;;;;;OAOG;IACM,KAAK,CAAC,OAAO,CAAC,KAAc;QACnC,IAAI,eAAe,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QACnE,uEAAuE;QACvE,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACM,KAAK,CAAC,MAAM,CAAC,MAAkB;QACtC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACnE,IAAI,QAAQ;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAC3C,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACM,KAAK,CAAC,iBAAiB;QAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;YACpC,sFAAsF;YACtF,yFAAyF;YACzF,sFAAsF;YACtF,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAA;QAC/B,CAAC;QACD,MAAM,KAAK,CAAC,iBAAiB,EAAE,CAAA;IACjC,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACM,KAAK,CAAC,KAAK,CAAC,OAAgB;QACnC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QACtC,IAAI,aAAa,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YACrC,wFAAwF;YACxF,2FAA2F;YAC3F,4EAA4E;YAC5E,MAAM,aAAa,CAAC,MAAM,EAAE,wCAAwC,EAAE,GAAG,EAAE,CACzE,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAC7B,CAAA;QACH,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,qBAAqB,CAAC,KAAa;QACvC,OAAO,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IACvD,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAA;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAA;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,0FAA0F;YAC1F,kCAAkC;QACpC,CAAC;QACD,uFAAuF;QACvF,wFAAwF;QACxF,gDAAgD;QAChD,MAAM,aAAa,CAAC,MAAM,EAAE,wCAAwC,EAAE,GAAG,EAAE,CACzE,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAC7B,CAAA;IACH,CAAC;IAEO,MAAM,CAAC,QAAyB;QACtC,OAAO,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED,wEAAwE;IACxE,IAAY,OAAO;QACjB,OAAO,IAAI,CAAC,GAAG,CAAC,OAAsC,CAAA;IACxD,CAAC;CACF;AAED;;;;;;;;;GASG;AACH,SAAS,aAAa,CAAC,OAAgB;IACrC,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;QAAE,OAAO,KAAK,CAAA;IAC3C,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,OAAO,CAAA;IACtE,CAAC;IAAC,MAAM,CAAC;QACP,2FAA2F;QAC3F,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC"}

package/dist/infrastructure/containers/stopCause.d.ts

@@ -0,0 +1,240 @@
+/**
+ * The reclaim causes a container can observe about itself.
+ *
+ * - `rollout`: a deploy drained it (exit 143). Expected churn during a release.
+ * - `idle`: its own `sleepAfter` window elapsed with nothing talking to it. The container is
+ *   kept warm ONLY by the driver's job polls, so this means the backend stopped polling for
+ *   longer than that window: a poll-scheduling hiccup (a Workflows instance evicted and
+ *   re-driven by the cron sweeper), not the workload failing. Left unrecorded it reads as a
+ *   crash, and two hiccups in one step then exhaust the single crash-eviction budget and fail
+ *   a healthy run (stuck-run audit F12).
+ */
+export type ContainerStopCause = 'rollout' | 'idle';
+/** DO-storage key holding the {@link StopCauseRecord} of the most recent self-observed stop. */
+export declare const STOP_CAUSE_KEY = "containerStopCause";
+/**
+ * What the runtime reported when the workload stopped, mirroring the container base class's
+ * `StopParams`. Recorded for EVERY stop, including the ones no {@link ContainerStopCause}
+ * explains, because it answers a different question: the cause decides the recovery BUDGET, the
+ * exit state is the only account of the death anyone gets afterwards.
+ *
+ * A Cloudflare Container's stdout is delivered to the deployment's Workers logs and is not
+ * readable back from inside the Durable Object, so this is the whole of the post-mortem this
+ * runtime can mint. It is still the difference between "container evicted or crashed" and "exit
+ * code 137", i.e. between a run nobody can diagnose and an out-of-memory kill.
+ */
+export interface ContainerExitState {
+    code: number;
+    reason: 'exit' | 'runtime_signal';
+}
+/** What the container persists when it observes its own stop. */
+export interface StopCauseRecord {
+    /**
+     * The infrastructure-churn cause, when the container recognised one. Absent for a stop it
+     * cannot explain (a crash, an OOM kill), which is exactly the case {@link exit} exists for.
+     */
+    cause?: ContainerStopCause;
+    /** What the runtime reported about the stop, when a hook that carries it fired. */
+    exit?: ContainerExitState;
+    /** Epoch ms the stop was observed. */
+    at: number;
+    /**
+     * The job id this record has already been spent explaining, once one has claimed it.
+     *
+     * A claim rather than a delete, because the caller runs inside a RETRYING durable step. A
+     * destructive read is not replay-safe: a `step.do` that reads the record and then throws (a
+     * contended persist, a failed emit) re-runs with the attribution already gone and reports the
+     * `crash` this whole mechanism exists to spare. Keyed by the claimant so a REPLAY of the same
+     * poll re-reads the same answer, while a different job's poll finds it spent — which is the
+     * "one reclaim explains exactly one eviction" rule, now stated in a way a retry cannot break.
+     */
+    claimedBy?: string;
+}
+/**
+ * How long after each cause a 404 poll may still be attributed to it.
+ *
+ * They differ because the two causes are observed at different distances from the poll that
+ * finds the container gone. A rollout drain interrupts an IN-FLIGHT poll, so the very next one
+ * lands seconds later. An idle reclaim happens precisely because polling stopped, so the poll
+ * that discovers it arrives however long the gap outran the idle window: minutes, not seconds.
+ * A window sized for the rollout case would therefore read every real idle reclaim as a crash,
+ * which is the finding itself.
+ *
+ * The window is a BACKSTOP, not the primary bound on what a record may excuse. What actually
+ * scopes it is the pair of rules around it: {@link clearStopCause} drops the record the moment a
+ * new job is accepted, and {@link takeStopCause} lets exactly one job spend it. So the wide
+ * `idle` window buys the poll gap it exists for without also handing the next step's crash an
+ * alibi.
+ */
+export declare const ATTRIBUTION_WINDOW_MS: {
+    rollout: number;
+    idle: number;
+};
+/**
+ * How long a stop this container could NOT attribute (a crash, an OOM kill) may still explain a
+ * 404 poll, i.e. the window governing a record that carries only an {@link ContainerExitState}.
+ *
+ * The same reasoning as the `idle` window, and deliberately the same number: what separates the
+ * death from the poll that discovers it is the POLL GAP, and a crash is discovered by the same
+ * ~15s tick an idle reclaim is, stretched by whatever delayed the driver. Sizing this to the
+ * rollout case instead would read every crash discovered after a re-drive as unexplained, which
+ * is the class this record exists to explain.
+ *
+ * Wide is affordable here for a reason the cause windows do not share: an exit state changes no
+ * verdict. It is attached to the failure `detail`, so the cost of over-attributing is a
+ * misleading sentence, never a wrongly-extended recovery budget.
+ */
+export declare const EXIT_ATTRIBUTION_WINDOW_MS: number;
+/**
+ * How far apart two hook calls may be and still be treated as observations of ONE stop, i.e. the
+ * bound on {@link recordStopCause}'s merge.
+ *
+ * The merge exists for a single reason: `onError` and `onStop` are two views of the same death,
+ * fired by the runtime within moments of each other. Anything older is a DIFFERENT, earlier
+ * stop, and merging onto it is not conservative, it is destructive twice over: the new
+ * observation inherits the old record's `at`, so it is born already aged, and both halves then
+ * fall out of their own attribution windows and explain nothing. A record that is 40 minutes old
+ * would silently swallow the crash that just happened.
+ *
+ * Sized generously against what it models (two hooks, milliseconds apart) and far below the
+ * narrowest attribution window, so it cannot be the thing that decides an attribution: what a
+ * merge may do is complete an account of one stop, never date it.
+ */
+export declare const STOP_MERGE_WINDOW_MS = 60000;
+/** What a container observed about its own stop, once attributed to the job that reads it. */
+export interface StopObservation {
+    cause?: ContainerStopCause;
+    exit?: ContainerExitState;
+}
+/**
+ * The message the base container class surfaces when the RUNTIME (not the workload) stopped a
+ * container: a deploy draining the old version. Its own exit-code parser is keyed on the plain
+ * "runtime signalled the container to exit:" form and does not recognise the rollout wording,
+ * which is why that case reaches `onError` at all.
+ */
+export declare function isRolloutSignal(error: unknown): boolean;
+/**
+ * What a container should record about a stop the runtime just reported, or undefined when it
+ * should record nothing.
+ *
+ * Two rules, both about what the stop MEANS rather than what it says:
+ *
+ * - A `runtime_signal` 143 is the rollout drain surfacing through this hook instead of (or as
+ *   well as) `onError`, depending on the runtime version, so it names the same cause.
+ * - A stop the container ASKED for records nothing at all. Its idle reclaim and its shutdown RPC
+ *   both work by signalling the container, so the exit that comes back is this container's own
+ *   request echoed at it (escalating to a SIGKILL 137 when the workload does not exit inside the
+ *   grace period), not an observation of how anything died. Recorded anyway, it is read back as
+ *   the container's cause of death and reported to the operator as "most often an out-of-memory
+ *   kill" underneath a verdict that says the platform reclaimed an idle container. The cause the
+ *   caller recorded when it decided to stop is the whole account, and it is already stored.
+ */
+export declare function observationForStop(params: {
+    exitCode: number;
+    reason: 'exit' | 'runtime_signal';
+}, selfInitiated: boolean): StopObservation | undefined;
+/**
+ * What a stored record still explains at `now`, or an EMPTY observation when it explains
+ * nothing.
+ *
+ * An empty observation covers three things that all mean the same to the caller: nothing was
+ * recorded, the record outlived its window (the container recovered and ran on, so a later death
+ * is its own), or the record names a cause this build no longer has. The last is possible
+ * because the record is PERSISTED and the vocabulary is closed: a deploy that retires a member
+ * leaves rows behind. All three land on the caller's existing default, reporting the eviction
+ * as a `crash`, which is the honest reading of "no attribution" and the conservative one:
+ * it costs a run one restart of its recovery budget, never a wrongly-extended one.
+ *
+ * The two halves are attributed INDEPENDENTLY, on their own windows: a record can be too old to
+ * excuse the eviction as churn while still being the only account of how the container died, and
+ * dropping the exit state with the cause would throw away the diagnostic to protect a budget the
+ * diagnostic never touches.
+ */
+export declare function attributeStopCause(record: unknown, now: number, claimant: string): StopObservation;
+/**
+ * The slice of DO storage the bookkeeping below needs. Narrowed to three methods so the rules
+ * can be exercised against a plain fake. The alternative is asserting them through a real
+ * Durable Object, which means reaching into the container base class's private fields.
+ */
+export interface StopCauseStorage {
+    get: (key: string) => Promise<unknown>;
+    put: (key: string, value: StopCauseRecord) => Promise<void>;
+    delete: (key: string) => Promise<unknown>;
+}
+/**
+ * Persist what this container just observed about its own stop, MERGING onto an unclaimed record
+ * of the SAME stop rather than replacing it.
+ *
+ * Merging is the point. One stop reaches this container through up to two hooks carrying
+ * different halves of the answer: `onError` recognises a rollout drain and knows no exit state,
+ * `onStop` carries `{ exitCode, reason }` and cannot name the churn. They fire in either order
+ * (and, on some runtime versions, both), so a plain overwrite means whichever landed second
+ * silently discarded the other's half: either the recovery budget or the only account of the
+ * death, depending on the ordering that day.
+ *
+ * Two records are never merged:
+ *
+ * - a CLAIMED one, which has already explained somebody's eviction, so anything observed after
+ *   it belongs to the next one;
+ * - one older than {@link STOP_MERGE_WINDOW_MS}, which is a different stop entirely. This
+ *   container's records are not reliably cleared between stops (`takeStopCause` deliberately
+ *   leaves an expired record in place, and only an accepted dispatch deletes one), so a stale
+ *   record is the ORDINARY state of a container that idled out and was re-driven, not an edge
+ *   case. Merging onto it would back-date the new observation to the old stop's `at` and age it
+ *   out of its own attribution window on arrival: the crash that just happened would be
+ *   recorded, read back, and dropped as too old to explain the eviction it caused.
+ *
+ * The kept `at` is the EARLIEST of the two observations, which within the merge window is a
+ * choice between timestamps moments apart: both describe one stop, and the window measures how
+ * long ago that stop happened.
+ */
+export declare function recordStopCause(storage: StopCauseStorage, observed: StopObservation, now: number): Promise<void>;
+/**
+ * The one-line account of a container's stop for the eviction `detail`, or undefined when
+ * nothing was recorded.
+ *
+ * `cause` is the account the run has ALREADY been given, and passing it is what keeps the detail
+ * from arguing with the verdict beside it. The two halves are recorded independently and the
+ * exit state is attached whether or not a cause was recognised, so they routinely describe one
+ * event twice: an idle reclaim is performed BY the platform with a SIGKILL, so a run correctly
+ * reported as "idle container reclaimed between polls" carries exit 137, and read as an
+ * unexplained death that code says "most often an out-of-memory kill". The operator is then
+ * holding a verdict and a detail that cannot both be true, which is worse than either alone,
+ * because there is no way to tell which one to act on.
+ *
+ * So the interpretation is offered only where it is the only account there is. Where a cause was
+ * named, the exit code is reported as the MECHANICS of that stop: still the difference between
+ * "reclaimed" and "reclaimed with a SIGKILL", and never a second, competing cause of death.
+ *
+ * Either way it states the platform's own limit rather than leaving it to be inferred: a
+ * Cloudflare Container writes its stdout to the deployment's Workers logs, and nothing inside
+ * the Durable Object can read it back, so an operator who reads this and goes looking for a log
+ * tail here should be told where the tail actually is.
+ */
+export declare function describeContainerExit(exit: ContainerExitState | undefined, cause?: ContainerStopCause): string | undefined;
+/**
+ * Forget whatever this container observed before now.
+ *
+ * Called when a NEW job is accepted, which is the moment the record stops being able to explain
+ * anything: a stop cause accounts for the death of a container that was serving the jobs
+ * outstanding when it was observed, and a job dispatched afterwards is not one of them. Without
+ * this the common benign case poisons the rare dangerous one — a run parks on a human decision,
+ * its container idles out with nothing running, and the `idle` marker that leaves behind is
+ * still there half an hour later to excuse a genuine OOM in the NEXT step as transient churn.
+ */
+export declare function clearStopCause(storage: StopCauseStorage): Promise<void>;
+/**
+ * Read what explains `claimant`'s 404 poll, CLAIMING it for that job.
+ *
+ * One stop explains exactly one job's eviction: the engine answers one by re-dispatching onto
+ * a fresh container under the same DO id, so an unclaimed record would still be sitting there to
+ * excuse the next death, whenever and for whatever reason it came. The claim is written rather
+ * than the record deleted so a REPLAYED poll for the same job reads back the same answer (see
+ * {@link StopCauseRecord.claimedBy}).
+ *
+ * The claim covers the record, not one half of it: an exit state read here is spent as surely as
+ * a cause is, because attaching one container's death to two runs' failures is the same lie
+ * either way.
+ */
+export declare function takeStopCause(storage: StopCauseStorage, now: number, claimant: string): Promise<StopObservation>;
+//# sourceMappingURL=stopCause.d.ts.map

package/dist/infrastructure/containers/stopCause.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopCause.d.ts","sourceRoot":"","sources":["../../../src/infrastructure/containers/stopCause.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;GAUG;AACH,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,MAAM,CAAA;AAEnD,gGAAgG;AAChG,eAAO,MAAM,cAAc,uBAAuB,CAAA;AAElD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAAA;CAClC;AAED,iEAAiE;AACjE,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,KAAK,CAAC,EAAE,kBAAkB,CAAA;IAC1B,mFAAmF;IACnF,IAAI,CAAC,EAAE,kBAAkB,CAAA;IACzB,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAA;IACV;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,qBAAqB;;;CAGY,CAAA;AAE9C;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,0BAA0B,QAAc,CAAA;AAErD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,oBAAoB,QAAS,CAAA;AAE1C,8FAA8F;AAC9F,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,EAAE,kBAAkB,CAAA;IAC1B,IAAI,CAAC,EAAE,kBAAkB,CAAA;CAC1B;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAIvD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAAA;CAAE,EAC/D,aAAa,EAAE,OAAO,GACrB,eAAe,GAAG,SAAS,CAO7B;AAwBD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,OAAO,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,GACf,eAAe,CAYjB;AAOD;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IACtC,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAC1C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,eAAe,EACzB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAUf;AAmDD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,kBAAkB,GAAG,SAAS,EACpC,KAAK,CAAC,EAAE,kBAAkB,GACzB,MAAM,GAAG,SAAS,CAgBpB;AAED;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAE7E;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,CAAC,CAS1B"}