@earendil-works/chord 1.4.0 → 1.5.1

package/dist/jobs/job.js

@@ -0,0 +1,387 @@
+// Jobs (§15), in memory: the handle `self.job` returns, one per attempt, over the job's records on its object. It
+// holds the implementation and the invocations: `run` runs outside transactions, `finish` in one with the cleanup,
+// both on the attempt that is live; the attempt's seal cuts its invocations off (slice 6a §1.3). Each phase of a run
+// (its start's intent, its run, its finish) is a `chord.job` span, under the run's stored carrier or its starter's span.
+import { withAbortSignal, withoutAbortSignal, withTelemetry } from "../context/index.js";
+import { ChordError, isChordError, PluginError } from "../data/index.js";
+import { sleep } from "../host/index.js";
+import { CallContext, nextDelay, SpanCtx } from "../host/internal.js";
+import { capped, chordSpans, ended, OpenSpan, parentOf, spanFailed, untraced } from "../telemetry.js";
+/** Failed finishes in a row after which a run is blocked. */
+const FINISH_ATTEMPTS = 5;
+/**
+ * A job as one attempt declares it: its implementation and the invocations and finishes running on the attempt. Starts
+ * are taken while the attempt is live; the object's records keep the runs.
+ */
+export class Job {
+    #records;
+    /** The definition its attempt declared: its invocations get args in its version, whatever the records' is now. */
+    #definition;
+    #implementation;
+    #owner;
+    /** `declared` until the attempt is live (starts are refused), `sealed` once it is sealed. */
+    #state = "declared";
+    /** Runs whose `run` runs on this attempt. */
+    #invocations = new Set();
+    /** The seal's error and cut reason, once sealed: a reservation committing after it cuts its run off with them. */
+    #sealed;
+    /** Runs whose finish runs on this attempt. */
+    #finishing = new Set();
+    constructor(records, implementation, owner) {
+        this.#records = records;
+        // Its attempt's declaration redefined the records just before (`ChordObject.jobRecords`).
+        this.#definition = records.definition;
+        this.#implementation = implementation;
+        this.#owner = owner;
+    }
+    get running() {
+        // Its value is the list of entries `JobRun.entryFor` builds.
+        return this.#records.running;
+    }
+    async start(args, context, options = {}) {
+        const { documents, token } = this.#owner;
+        const parent = context.telemetry;
+        if (untraced(parent))
+            return documents.tx((tx) => this.stage(tx, args, context, options), token, context);
+        // The intent: the start's transaction, in its span (the run keeps it: its later phases are its children).
+        const attributes = {
+            "chord.host": this.#owner.host.spanHost,
+            job: this.#definition.name,
+            phase: "intent",
+            object: capped(documents.label),
+        };
+        return chordSpans(parent)("chord.job", attributes, async (span) => {
+            const traced = withTelemetry(span, context);
+            try {
+                const run = await documents.tx((tx) => this.stage(tx, args, traced, options), token, traced);
+                span.setAttributes({ outcome: "ok", "job.id": run.id });
+                return run;
+            }
+            catch (error) {
+                span.setAttributes({ outcome: ended(span, "job_failed", error, context.abortSignal) });
+                throw error;
+            }
+        });
+    }
+    outcome(id, context) {
+        return this.#records.outcome(id, context);
+    }
+    abort(id, context) {
+        return this.#records.abort(id, context);
+    }
+    /** A persisted job's runs, loaded once per object at its first activation (before it goes live). */
+    load(context) {
+        return this.#records.load(context);
+    }
+    /** The close writer of this job's runs, after the attempt's drain (slice 6b §4.2). */
+    writeAtClose(context) {
+        return this.#records.writeAtClose(context);
+    }
+    /** The attempt went live: starts are taken, and the records' waiting runs are invoked or finished here. */
+    live() {
+        if (this.#state !== "declared")
+            return;
+        this.#state = "live";
+        this.#records.attach(this);
+    }
+    /**
+     * Stages a run in `tx` (`start`, `tx.start`), in order: the args checked, dedupe by key (runs started earlier in
+     * `tx` included), the concurrency check, then the record and the state document, created in the commit. The run
+     * whose finish `tx` is counts for neither: `finish` may start the same job again, with the same key too. Returns
+     * the run's handle at once; a start that only dedupes stages nothing.
+     */
+    stage(tx, args, context, options) {
+        // Declaration policy: a handle belongs to its object (types cannot say whose transaction this is).
+        if (tx.documents !== this.#owner.documents) {
+            throw new Error(`${this.#owner.documents.label}: a transaction starts only its own object's jobs`);
+        }
+        if (this.#state === "sealed") {
+            throw new ChordError({
+                code: "released",
+                reason: "ended",
+                message: `${this.#records.name}: its attempt ended`,
+            });
+        }
+        if (this.#state === "declared") {
+            throw new ChordError({ code: "not_ready", message: `${this.#records.name}: its object is not live yet` });
+        }
+        return this.#records.stage(tx, args, context, options);
+    }
+    /**
+     * The attempt is sealed with `error`: starts are refused, running invocations are cut off (their signals fire, in
+     * memory they end `aborted` with `reason` and finish on the next live attempt, persisted ones are stopped and
+     * resumed, R3), and the runs finishing here wait for it.
+     */
+    seal(error, reason) {
+        if (this.#state === "sealed")
+            return;
+        this.#state = "sealed";
+        this.#sealed = { error, reason };
+        this.#records.detach(this, this.#finishing);
+        this.#finishing.clear();
+        for (const run of this.#invocations)
+            this.#records.cut(run, error, reason);
+        this.#invocations.clear();
+    }
+    /**
+     * Runs `run` for a committed run (with its signal aborted already after an early abort); a persisted run's
+     * reservation commits first (`running`: a crash from then on counts, D6). The reservation, the invocation and the
+     * run's finish are one piece of work in the drain, admitted before plugin code runs: `run` may end its own object
+     * before its first await.
+     */
+    invoke(run) {
+        const invocation = run.begin();
+        if (!this.#records.persisted || run.phase === "running") {
+            this.#start(run, invocation);
+            return;
+        }
+        const { documents, token, context } = this.#owner;
+        const reserving = documents
+            .tx((tx) => this.#records.reserve(tx, run), token, context)
+            .then(() => {
+            if (this.#state !== "sealed") {
+                this.#start(run, invocation);
+                return;
+            }
+            this.#records.cut(run, this.#sealed.error, this.#sealed.reason);
+            // No `run` started: nothing else ends it, and an abort joined to it returns.
+            invocation.ended();
+        }, () => {
+            invocation.settle();
+            invocation.ended();
+            // Refused because this attempt was sealed (at admission, or queued past the seal): the run is pending still,
+            // for the next live attempt (this one is detached already). Any other failure (the store closed or failed,
+            // the object ended) is not retried here: retrying on this attempt would only fail again.
+            if (this.#state === "sealed")
+                this.#records.requeue(run);
+        });
+        this.#owner.admit(reserving);
+    }
+    /**
+     * Starts `invocation` of `run` on this attempt, in its run phase's span: its ctx and its settling act for this
+     * invocation only.
+     */
+    #start(run, invocation) {
+        const { context, caller, host } = this.#owner;
+        const signal = invocation.abort.signal;
+        const span = this.#phaseSpan(run, "run");
+        const scope = withAbortSignal(signal, context);
+        const ctx = new JobContext(span === undefined ? scope : withTelemetry(span.span, scope), caller, host, run.id, this.#stateOf(run, invocation));
+        const work = Promise.withResolvers();
+        this.#owner.admitRun(work.promise, () => ctx.settle());
+        this.#invocations.add(run);
+        const running = (async () => {
+            try {
+                // Its own version's args: a run started by old work during a restart runs on the old code (its definition).
+                return await this.#implementation.run(this.#records.argsOf(run, this.#definition), ctx);
+            }
+            finally {
+                // The fence closes as `run` settles: a change made later is refused.
+                invocation.settle();
+            }
+        })();
+        running
+            .then((result) => this.#settled(run, invocation, ctx, { kind: "done", result }), (error) => this.#settled(run, invocation, ctx, this.#failed(run, invocation, error)))
+            .finally(() => invocation.ended())
+            .then(work.resolve);
+        span?.endWhen(running, "job_failed", signal);
+    }
+    /**
+     * The span of `run`'s phase `phase`, under its start's carrier (a persisted run resumed in a new attempt or process
+     * too: the same trace; without one the adapter's root); none when nothing records.
+     */
+    #phaseSpan(run, phase) {
+        const { host, documents } = this.#owner;
+        const parent = parentOf(run.trace, host.telemetry);
+        if (untraced(parent))
+            return undefined;
+        return new OpenSpan(parent, "chord.job", {
+            "chord.host": host.spanHost,
+            job: this.#definition.name,
+            phase,
+            object: capped(documents.label),
+            "job.id": run.id,
+        });
+    }
+    /**
+     * Runs the finish of `run`, whose outcome is recorded: `finish` with the cleanup in one transaction under the
+     * attempt's token, retried with backoff after a throw (reported), blocked after five in a row. The attempt's seal
+     * stops it without a report or a count; the next live attempt finishes the run.
+     */
+    finish(run) {
+        const work = this.#finish(run);
+        this.#owner.admit(work);
+    }
+    /** How a throwing `run` ended: `aborted` after its invocation's signal fired, otherwise `threw` (reported). */
+    #failed(run, invocation, error) {
+        if (invocation.abort.signal.aborted)
+            return { kind: "failed", error: { kind: "aborted", reason: "abort" } };
+        this.#records.report({ kind: "job_failed", id: run.id, error });
+        const thrown = isChordError(error) || error instanceof PluginError ? error : internal(error);
+        return { kind: "failed", error: { kind: "threw", error: thrown } };
+    }
+    /**
+     * `invocation` of `run` settled: its `ctx.get` references go; its outcome is recorded unless the attempt's seal cut
+     * it off (which recorded its own, or stopped the run for a later invocation), then the run finishes here while this
+     * attempt is live.
+     */
+    async #settled(run, invocation, ctx, ended) {
+        ctx.settle();
+        if (invocation.token.sealed || run.invocation !== invocation)
+            return;
+        this.#invocations.delete(run);
+        this.#records.settled(run, ended);
+    }
+    /**
+     * `run`'s finish phase in its span: each failed try a `chord.retry` event with its backoff; ends `ok`, `error`
+     * (blocked) or `aborted` (the attempt's seal).
+     */
+    async #finish(run) {
+        const span = this.#phaseSpan(run, "finish");
+        const { context } = this.#owner;
+        let outcome = "error";
+        try {
+            outcome = await this.#finishTries(run, span, span === undefined ? context : withTelemetry(span.span, context));
+        }
+        finally {
+            span?.span.setAttributes({ outcome });
+            span?.end();
+        }
+    }
+    /** The tries of `run`'s finish under `context`, retried with backoff; how the phase ended. */
+    async #finishTries(run, span, context) {
+        const { documents, token, signal } = this.#owner;
+        this.#finishing.add(run);
+        let delay;
+        try {
+            for (;;) {
+                try {
+                    await documents.tx(async (tx) => {
+                        const staging = this.#records.staging(tx);
+                        staging.finishing = run;
+                        await this.#implementation.finish?.(this.#records.outcomeOf(run, context), tx);
+                        this.#records.change(tx, { kind: "delete", run });
+                    }, token, context);
+                    return "ok";
+                }
+                catch (error) {
+                    if (this.#state === "sealed")
+                        return "aborted";
+                    this.#records.report({ kind: "job_finish_failed", id: run.id, error });
+                    // The count is the run's (stored for a persisted job): failures count across a crash.
+                    const finishFailures = run.finishFailures + 1;
+                    if (finishFailures >= FINISH_ATTEMPTS) {
+                        if (span !== undefined)
+                            spanFailed(span.span, "job_failed", error);
+                        await this.#records.record(run, { kind: "block", run, reason: "finish_failed" });
+                        return "error";
+                    }
+                    await this.#records.record(run, { kind: "update", run, fields: { finishFailures } });
+                    delay = nextDelay(delay);
+                    span?.span.addEvent("chord.retry", { backoff: delay });
+                    try {
+                        await sleep(delay, withAbortSignal(signal, context));
+                    }
+                    catch {
+                        return "aborted";
+                    }
+                }
+            }
+        }
+        finally {
+            this.#finishing.delete(run);
+        }
+    }
+    /** `ctx.state` of `invocation` of `run`, when the job has a state: changes under its token while it runs. */
+    #stateOf(run, invocation) {
+        const records = this.#records;
+        const states = records.states;
+        if (states === undefined)
+            return undefined;
+        const { documents } = this.#owner;
+        const refused = () => new ChordError({
+            code: "released",
+            reason: "settled",
+            message: `${records.name}: run ${run.id} settled; its state is closed`,
+        });
+        return {
+            get value() {
+                const value = records.stateOf(run.id);
+                if (value === undefined)
+                    throw refused();
+                return value;
+            },
+            change: (fn, context, options = {}) => {
+                if (invocation.state !== "running")
+                    return Promise.reject(refused());
+                let persistence;
+                try {
+                    persistence = checkedPersistence(options.persistence, records.name);
+                }
+                catch (error) {
+                    return Promise.reject(error);
+                }
+                // Never the abort signal: a run records its progress also after an abort.
+                const unaborted = withoutAbortSignal(context);
+                // A change resetting the restart count is stored at once: the reset must be durable (D8).
+                const now = run.restarts === 0 ? persistence : IMMEDIATE;
+                return documents.tx(async (tx) => {
+                    const value = fn(await tx.draftSlot(states, run.id));
+                    records.progressed(tx, run);
+                    return value;
+                }, invocation.token, unaborted, now);
+            },
+        };
+    }
+}
+const IMMEDIATE = { kind: "immediate" };
+/**
+ * A state change's `persistence` (a trust boundary: plugin code): `immediate` when omitted; a deferred one's `withinMs`
+ * a number from 0 (`Infinity` waits for the others), `invalid/args` otherwise.
+ */
+function checkedPersistence(persistence, what) {
+    if (persistence === undefined || persistence.kind === "immediate")
+        return IMMEDIATE;
+    const { withinMs } = persistence;
+    if (persistence.kind !== "deferred" ||
+        (withinMs !== undefined && !(typeof withinMs === "number" && withinMs >= 0))) {
+        throw new ChordError({
+            code: "invalid",
+            reason: "args",
+            message: `${what}: persistence is immediate or deferred with withinMs >= 0`,
+        });
+    }
+    return persistence;
+}
+/** A plugin's throw that is neither a ChordError nor a PluginError, as a run's outcome keeps it. */
+function internal(error) {
+    return new PluginError("internal", error instanceof Error ? error.message : String(error));
+}
+/** A run's ctx: the handler's ctx of its object, its id and, with a state, its state document. */
+export class JobContext extends CallContext {
+    jobId;
+    constructor(context, caller, host, jobId, state) {
+        super(context, caller, host);
+        this.jobId = jobId;
+        if (state !== undefined)
+            this.state = state;
+    }
+    /** Its child under `span` (`ctx.trace`): a job's ctx still, its id and state kept. */
+    traced(span) {
+        return new JobSpanContext(this, span, this.jobId, this.state);
+    }
+}
+/** A run's ctx under a child span (`ctx.trace`): its parent's members, its run's id and state. */
+class JobSpanContext extends SpanCtx {
+    jobId;
+    constructor(parent, span, jobId, state) {
+        super(parent, span);
+        this.jobId = jobId;
+        if (state !== undefined)
+            this.state = state;
+    }
+    traced(span) {
+        return new JobSpanContext(this, span, this.jobId, this.state);
+    }
+}
+//# sourceMappingURL=job.js.map

package/dist/jobs/job.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"job.js","sourceRoot":"","sources":["../../src/jobs/job.ts"],"names":[],"mappings":"AAAA,kHAAkH;AAClH,mHAAmH;AACnH,qHAAqH;AACrH,yHAAyH;AAGzH,OAAO,EAAgB,eAAe,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACvG,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAkB,MAAM,kBAAkB,CAAC;AAIzF,OAAO,EAAY,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAgB,WAAW,EAAe,SAAS,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEjG,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAiGtG,6DAA6D;AAC7D,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B;;;GAGG;AACH,MAAM,OAAO,GAAG;IACN,QAAQ,CAAa;IAC9B,kHAAkH;IACzG,WAAW,CAA2C;IACtD,eAAe,CAAuB;IACtC,MAAM,CAAW;IAC1B,6FAA6F;IAC7F,MAAM,GAAmC,UAAU,CAAC;IACpD,6CAA6C;IACpC,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,kHAAkH;IAClH,OAAO,CAAyE;IAChF,8CAA8C;IACrC,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IAExC,YAAY,OAAmB,EAAE,cAAoC,EAAE,KAAe;QACrF,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,0FAA0F;QAC1F,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,IAAI,OAAO;QACV,6DAA6D;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,OAA0F,CAAC;IACjH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAe,EAAE,OAAgB,EAAE,OAAO,GAAiB,EAAE;QACxE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACzC,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC1G,0GAA0G;QAC1G,MAAM,UAAU,GAAG;YAClB,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ;YACvC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YAC1B,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;SACtB,CAAC;QACX,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACjE,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC;gBACJ,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC7F,IAAI,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxD,OAAO,GAAG,CAAC;YACZ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;gBACvF,MAAM,KAAK,CAAC;YACb,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,EAAU,EAAE,OAAgB;QACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,EAAU,EAAE,OAAgB;QACjC,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,oGAAoG;IACpG,IAAI,CAAC,OAAgB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,sFAAsF;IACtF,YAAY,CAAC,OAAgB;QAC5B,OAAO,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,2GAA2G;IAC3G,IAAI;QACH,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU;YAAE,OAAO;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,EAAe,EAAE,IAAa,EAAE,OAAgB,EAAE,OAAqB;QAC5E,mGAAmG;QACnG,IAAI,EAAE,CAAC,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,mDAAmD,CAAC,CAAC;QACpG,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,UAAU,CAAC;gBACpB,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,qBAAqB;aACnD,CAAC,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,UAAU,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,8BAA8B,EAAE,CAAC,CAAC;QAC3G,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,KAAiB,EAAE,MAAiB;QACxC,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ;YAAE,OAAO;QACrC,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAC3E,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,GAAW;QACjB,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YAC7B,OAAO;QACR,CAAC;QACD,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAClD,MAAM,SAAS,GAAG,SAAS;aACzB,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC;aAC1D,IAAI,CACJ,GAAG,EAAE;YACJ,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC7B,OAAO;YACR,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,OAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,OAAQ,CAAC,MAAM,CAAC,CAAC;YAClE,6EAA6E;YAC7E,UAAU,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC,EACD,GAAG,EAAE;YACJ,UAAU,CAAC,MAAM,EAAE,CAAC;YACpB,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,6GAA6G;YAC7G,2GAA2G;YAC3G,yFAAyF;YACzF,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1D,CAAC,CACD,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,GAAW,EAAE,UAAsB;QACzC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC9C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,IAAI,UAAU,CACzB,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAC5D,MAAM,EACN,IAAI,EACJ,GAAG,CAAC,EAAE,EACN,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC,CAC9B,CAAC;QACF,MAAM,IAAI,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;QAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;YAC3B,IAAI,CAAC;gBACJ,4GAA4G;gBAC5G,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,CAAC;YACzF,CAAC;oBAAS,CAAC;gBACV,qEAAqE;gBACrE,UAAU,CAAC,MAAM,EAAE,CAAC;YACrB,CAAC;QACF,CAAC,CAAC,EAAE,CAAC;QACL,OAAO;aACL,IAAI,CACJ,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EACzE,CAAC,KAAc,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAC7F;aACA,OAAO,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;aACjC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrB,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW,EAAE,KAAuB;QAC9C,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,IAAI,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,SAAS,CAAC;QACvC,OAAO,IAAI,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE;YACxC,YAAY,EAAE,IAAI,CAAC,QAAQ;YAC3B,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;YAC1B,KAAK;YACL,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;YAC/B,QAAQ,EAAE,GAAG,CAAC,EAAE;SAChB,CAAC,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,GAAW;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,+GAA+G;IAC/G,OAAO,CAAC,GAAW,EAAE,UAAsB,EAAE,KAAc;QAC1D,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC;QAC5G,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC7F,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IACpE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,UAAsB,EAAE,GAAe,EAAE,KAAc;QAClF,GAAG,CAAC,MAAM,EAAE,CAAC;QACb,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,KAAK,UAAU;YAAE,OAAO;QACrE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,GAAW;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC5C,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAChC,IAAI,OAAO,GAA+B,OAAO,CAAC;QAClD,IAAI,CAAC;YACJ,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QAChH,CAAC;gBAAS,CAAC;YACV,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;YACtC,IAAI,EAAE,GAAG,EAAE,CAAC;QACb,CAAC;IACF,CAAC;IAED,8FAA8F;IAC9F,KAAK,CAAC,YAAY,CACjB,GAAW,EACX,IAAuC,EACvC,OAAgB;QAEhB,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,KAAyB,CAAC;QAC9B,IAAI,CAAC;YACJ,SAAS,CAAC;gBACT,IAAI,CAAC;oBACJ,MAAM,SAAS,CAAC,EAAE,CACjB,KAAK,EAAE,EAAE,EAAE,EAAE;wBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;wBAC1C,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC;wBACxB,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;wBAC/E,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;oBACnD,CAAC,EACD,KAAK,EACL,OAAO,CACP,CAAC;oBACF,OAAO,IAAI,CAAC;gBACb,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ;wBAAE,OAAO,SAAS,CAAC;oBAC/C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;oBACvE,sFAAsF;oBACtF,MAAM,cAAc,GAAG,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC;oBAC9C,IAAI,cAAc,IAAI,eAAe,EAAE,CAAC;wBACvC,IAAI,IAAI,KAAK,SAAS;4BAAE,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;wBACnE,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;wBACjF,OAAO,OAAO,CAAC;oBAChB,CAAC;oBACD,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,CAAC;oBACrF,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACzB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC;wBACJ,MAAM,KAAK,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;oBACtD,CAAC;oBAAC,MAAM,CAAC;wBACR,OAAO,SAAS,CAAC;oBAClB,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACF,CAAC;IAED,6GAA6G;IAC7G,QAAQ,CAAC,GAAW,EAAE,UAAsB;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC3C,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAClC,MAAM,OAAO,GAAG,GAAe,EAAE,CAChC,IAAI,UAAU,CAAC;YACd,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,GAAG,OAAO,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,+BAA+B;SACtE,CAAC,CAAC;QACJ,OAAO;YACN,IAAI,KAAK;gBACR,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtC,IAAI,KAAK,KAAK,SAAS;oBAAE,MAAM,OAAO,EAAE,CAAC;gBACzC,OAAO,KAAK,CAAC;YACd,CAAC;YACD,MAAM,EAAE,CACP,EAAmC,EACnC,OAAgB,EAChB,OAAO,GAAiD,EAAE,EAC7C,EAAE;gBACf,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;oBAAE,OAAO,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;gBACrE,IAAI,WAA8B,CAAC;gBACnC,IAAI,CAAC;oBACJ,WAAW,GAAG,kBAAkB,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;gBACrE,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC9B,CAAC;gBACD,0EAA0E;gBAC1E,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAC9C,0FAA0F;gBAC1F,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,OAAO,SAAS,CAAC,EAAE,CAClB,KAAK,EAAE,EAAE,EAAE,EAAE;oBACZ,MAAM,KAAK,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;oBACrD,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;oBAC5B,OAAO,KAAK,CAAC;gBACd,CAAC,EACD,UAAU,CAAC,KAAK,EAChB,SAAS,EACT,GAAG,CACH,CAAC;YACH,CAAC;SACD,CAAC;IACH,CAAC;CACD;AAED,MAAM,SAAS,GAAsB,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AAE3D;;;GAGG;AACH,SAAS,kBAAkB,CAAC,WAA0C,EAAE,IAAY;IACnF,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,CAAC,IAAI,KAAK,WAAW;QAAE,OAAO,SAAS,CAAC;IACpF,MAAM,EAAE,QAAQ,EAAE,GAAG,WAAW,CAAC;IACjC,IACC,WAAW,CAAC,IAAI,KAAK,UAAU;QAC/B,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,IAAI,CAAC,CAAC,CAAC,EAC3E,CAAC;QACF,MAAM,IAAI,UAAU,CAAC;YACpB,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,GAAG,IAAI,2DAA2D;SAC3E,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,WAAW,CAAC;AACpB,CAAC;AAED,oGAAoG;AACpG,SAAS,QAAQ,CAAC,KAAc;IAC/B,OAAO,IAAI,WAAW,CAAC,UAAU,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,kGAAkG;AAClG,MAAM,OAAO,UAAW,SAAQ,WAAW;IACjC,KAAK,CAAS;IAGvB,YACC,OAAgB,EAChB,MAAc,EACd,IAAa,EACb,KAAa,EACb,KAAuC;QAEvC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,KAAK,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IAC7C,CAAC;IAED,sFAAsF;IAC7E,MAAM,CAAC,IAAmB;QAClC,OAAO,IAAI,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/D,CAAC;CACD;AAED,kGAAkG;AAClG,MAAM,cAAe,SAAQ,OAAO;IAC1B,KAAK,CAAS;IAGvB,YACC,MAAmC,EACnC,IAAmB,EACnB,KAAa,EACb,KAAuC;QAEvC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACpB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,KAAK,KAAK,SAAS;YAAE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IAC7C,CAAC;IAEQ,MAAM,CAAC,IAAmB;QAClC,OAAO,IAAI,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/D,CAAC;CACD","sourcesContent":["// Jobs (§15), in memory: the handle `self.job` returns, one per attempt, over the job's records on its object. It\n// holds the implementation and the invocations: `run` runs outside transactions, `finish` in one with the cleanup,\n// both on the attempt that is live; the attempt's seal cuts its invocations off (slice 6a §1.3). Each phase of a run\n// (its start's intent, its run, its finish) is a `chord.job` span, under the run's stored carrier or its starter's span.\n\nimport type { TelemetrySpan } from \"@earendil-works/pi-telemetry\";\nimport { type Context, withAbortSignal, withoutAbortSignal, withTelemetry } from \"../context/index.ts\";\nimport { ChordError, isChordError, PluginError, type Principal } from \"../data/index.ts\";\nimport type { Draft } from \"../delta/index.ts\";\nimport type { CommitPersistence, JsonObject, Tx } from \"../documents/index.ts\";\nimport type { Documents, SealToken, Transaction } from \"../documents/internal.ts\";\nimport { type Ctx, sleep } from \"../host/index.ts\";\nimport { type AnyHost, CallContext, type Caller, nextDelay, SpanCtx } from \"../host/internal.ts\";\nimport type { JsonValue } from \"../json.ts\";\nimport { capped, chordSpans, ended, OpenSpan, parentOf, spanFailed, untraced } from \"../telemetry.ts\";\nimport type { Watchable } from \"../watch.ts\";\nimport type { JobDefinition, Outcome } from \"./definition.ts\";\nimport type { JobRecords, Settled } from \"./records.ts\";\nimport type { AnyOutcome, Invocation, JobRun, Run } from \"./run.ts\";\nimport type { RunningEntry } from \"./running.ts\";\n\n/** A job's handle, `self.job(Job, { run, finish? })`; `A`, `R` and `S` are its args, result and state. */\nexport interface JobHandle<A, R, S> {\n\t/**\n\t * Starts a run in one transaction on the object's line and resolves with its `Run` after the commit: the args\n\t * checked (`invalid/args`); a `key` of an unsettled run returns that run; at the `concurrency` limit (running plus\n\t * finishing runs) it rejects `unavailable/busy`. `run` starts after the commit.\n\t */\n\tstart(args: A, context: Context, options?: StartOptions): Promise<Run<Outcome<A, R, S>>>;\n\t/** Waits for the outcome of the unsettled run `id`; a settled or unknown id rejects `gone`. */\n\toutcome(id: string, context: Context): Promise<Outcome<A, R, S>>;\n\t/**\n\t * Aborts run `id` (D7, as Durable's `abortTask`): marks it (stored for a persisted job) and fires its `run`'s\n\t * signal, resolving `\"marked\"` once its `run` settled (a run not invoked yet ends `aborted` without one); a\n\t * finishing run is only marked, a blocked one dropped (no `finish`; its waiters reject `aborted`). `\"terminal\"` when\n\t * no unsettled run has that id. `context` ends only this wait.\n\t */\n\tabort(id: string, context: Context): Promise<\"marked\" | \"terminal\">;\n\t/** The unsettled runs, one entry each in start order, tagged by phase (a JSON watchable, a state member). */\n\treadonly running: Watchable<readonly RunningEntry<A, S>[]>;\n}\n\n/** What `start` and `tx.start` take besides the args. */\nexport type StartOptions = {\n\t/** Names one run: a start with the key of an unsettled run returns that run (args not compared). */\n\treadonly key?: string;\n};\n\n/** What `self.job` takes: the run and the optional finish. */\nexport type JobImplementation<A, R, S, P extends Principal = Principal> = {\n\t/** The work, outside transactions; its ctx's abort signal fires on `abort(id)` and on its object's end. */\n\treadonly run: (args: A, ctx: JobCtx<P, S>) => Promise<R>;\n\t/** Records the outcome in one transaction (only `await tx.*`); the run's record and state document go with it. */\n\treadonly finish?: (outcome: Outcome<A, R, S>, tx: Tx) => void | Promise<void>;\n};\n\n/** A run's ctx: a handler's ctx (the object's principal or the host's), its run's id and, with a state, the state. */\nexport type JobCtx<P extends Principal, S> = Ctx<P> & { readonly jobId: string } & ([S] extends [undefined]\n\t\t? unknown\n\t\t: { readonly state: RunState<S> });\n\n/** A run's state document. */\nexport interface RunState<S> {\n\t/** The document's value. */\n\treadonly value: S;\n\t/**\n\t * One transaction on the object's line drafting only this document, under `context` without its abort signal:\n\t * admitted while `run` runs (also after an abort, with the run's own ctx), refused `released` once it settled. A\n\t * persisted job's change may be `deferred` (D8): adopted and published at once, stored within `withinMs` (1,000 ms by\n\t * default) or earlier; a crash loses it. In memory the option changes nothing.\n\t */\n\tchange<T>(\n\t\tfn: (draft: Draft<S>) => T,\n\t\tcontext: Context,\n\t\toptions?: { readonly persistence?: CommitPersistence },\n\t): Promise<T>;\n}\n\n/** Where a job's handle runs its work: its attempt, as the job sees it. */\nexport type JobOwner = {\n\treadonly host: AnyHost;\n\t/** Whom runs act for: the object's principal (remote) or the host's (local), from this host. */\n\treadonly caller: Caller;\n\t/** The object's context, without an abort signal: runs, state changes and finishes run under it. */\n\treadonly context: Context;\n\t/** Aborted when the attempt is sealed: a finish's backoff waits under it. */\n\treadonly signal: AbortSignal;\n\t/** The attempt's seal token: its starts and finishes are transactions under it. */\n\treadonly token: SealToken;\n\treadonly documents: Documents;\n\t/**\n\t * Adds running work (a finish, a reservation) to the attempt's calls drain; `cleanup` (its bookkeeping) runs once,\n\t * when it settles or when the attempt ends with it still running.\n\t */\n\tadmit(running: Promise<unknown>, cleanup?: () => void): void;\n\t/** Adds a run's invocation to the attempt's drain at its end, as `admit`. */\n\tadmitRun(running: Promise<unknown>, cleanup: () => void): void;\n};\n\n/**\n * Why a seal cuts a running in-memory run off: `restart` (its attempt ended while the object continues) or `reload`\n * (a native restart, slice 7b §8); its outcome is `failed { aborted }` with that reason.\n */\nexport type CutReason = \"restart\" | \"reload\";\n\n/** A job's implementation as Chord calls it, whatever the job's types. */\nexport type AnyJobImplementation = {\n\treadonly run: (args: JsonValue, ctx: JobContext) => Promise<unknown>;\n\treadonly finish?: (outcome: AnyOutcome, tx: Tx) => void | Promise<void>;\n};\n\n/** Failed finishes in a row after which a run is blocked. */\nconst FINISH_ATTEMPTS = 5;\n\n/**\n * A job as one attempt declares it: its implementation and the invocations and finishes running on the attempt. Starts\n * are taken while the attempt is live; the object's records keep the runs.\n */\nexport class Job implements JobHandle<JsonValue, unknown, JsonValue | undefined> {\n\treadonly #records: JobRecords;\n\t/** The definition its attempt declared: its invocations get args in its version, whatever the records' is now. */\n\treadonly #definition: JobDefinition<unknown, unknown, unknown>;\n\treadonly #implementation: AnyJobImplementation;\n\treadonly #owner: JobOwner;\n\t/** `declared` until the attempt is live (starts are refused), `sealed` once it is sealed. */\n\t#state: \"declared\" | \"live\" | \"sealed\" = \"declared\";\n\t/** Runs whose `run` runs on this attempt. */\n\treadonly #invocations = new Set<JobRun>();\n\t/** The seal's error and cut reason, once sealed: a reservation committing after it cuts its run off with them. */\n\t#sealed: { readonly error: ChordError; readonly reason: CutReason } | undefined;\n\t/** Runs whose finish runs on this attempt. */\n\treadonly #finishing = new Set<JobRun>();\n\n\tconstructor(records: JobRecords, implementation: AnyJobImplementation, owner: JobOwner) {\n\t\tthis.#records = records;\n\t\t// Its attempt's declaration redefined the records just before (`ChordObject.jobRecords`).\n\t\tthis.#definition = records.definition;\n\t\tthis.#implementation = implementation;\n\t\tthis.#owner = owner;\n\t}\n\n\tget running(): Watchable<readonly RunningEntry<JsonValue, JsonValue | undefined>[]> {\n\t\t// Its value is the list of entries `JobRun.entryFor` builds.\n\t\treturn this.#records.running as unknown as Watchable<readonly RunningEntry<JsonValue, JsonValue | undefined>[]>;\n\t}\n\n\tasync start(args: JsonValue, context: Context, options: StartOptions = {}): Promise<Run<AnyOutcome>> {\n\t\tconst { documents, token } = this.#owner;\n\t\tconst parent = context.telemetry;\n\t\tif (untraced(parent)) return documents.tx((tx) => this.stage(tx, args, context, options), token, context);\n\t\t// The intent: the start's transaction, in its span (the run keeps it: its later phases are its children).\n\t\tconst attributes = {\n\t\t\t\"chord.host\": this.#owner.host.spanHost,\n\t\t\tjob: this.#definition.name,\n\t\t\tphase: \"intent\",\n\t\t\tobject: capped(documents.label),\n\t\t} as const;\n\t\treturn chordSpans(parent)(\"chord.job\", attributes, async (span) => {\n\t\t\tconst traced = withTelemetry(span, context);\n\t\t\ttry {\n\t\t\t\tconst run = await documents.tx((tx) => this.stage(tx, args, traced, options), token, traced);\n\t\t\t\tspan.setAttributes({ outcome: \"ok\", \"job.id\": run.id });\n\t\t\t\treturn run;\n\t\t\t} catch (error) {\n\t\t\t\tspan.setAttributes({ outcome: ended(span, \"job_failed\", error, context.abortSignal) });\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t});\n\t}\n\n\toutcome(id: string, context: Context): Promise<AnyOutcome> {\n\t\treturn this.#records.outcome(id, context);\n\t}\n\n\tabort(id: string, context: Context): Promise<\"marked\" | \"terminal\"> {\n\t\treturn this.#records.abort(id, context);\n\t}\n\n\t/** A persisted job's runs, loaded once per object at its first activation (before it goes live). */\n\tload(context: Context): Promise<void> {\n\t\treturn this.#records.load(context);\n\t}\n\n\t/** The close writer of this job's runs, after the attempt's drain (slice 6b §4.2). */\n\twriteAtClose(context: Context): Promise<void> {\n\t\treturn this.#records.writeAtClose(context);\n\t}\n\n\t/** The attempt went live: starts are taken, and the records' waiting runs are invoked or finished here. */\n\tlive(): void {\n\t\tif (this.#state !== \"declared\") return;\n\t\tthis.#state = \"live\";\n\t\tthis.#records.attach(this);\n\t}\n\n\t/**\n\t * Stages a run in `tx` (`start`, `tx.start`), in order: the args checked, dedupe by key (runs started earlier in\n\t * `tx` included), the concurrency check, then the record and the state document, created in the commit. The run\n\t * whose finish `tx` is counts for neither: `finish` may start the same job again, with the same key too. Returns\n\t * the run's handle at once; a start that only dedupes stages nothing.\n\t */\n\tstage(tx: Transaction, args: unknown, context: Context, options: StartOptions): Run<AnyOutcome> {\n\t\t// Declaration policy: a handle belongs to its object (types cannot say whose transaction this is).\n\t\tif (tx.documents !== this.#owner.documents) {\n\t\t\tthrow new Error(`${this.#owner.documents.label}: a transaction starts only its own object's jobs`);\n\t\t}\n\t\tif (this.#state === \"sealed\") {\n\t\t\tthrow new ChordError({\n\t\t\t\tcode: \"released\",\n\t\t\t\treason: \"ended\",\n\t\t\t\tmessage: `${this.#records.name}: its attempt ended`,\n\t\t\t});\n\t\t}\n\t\tif (this.#state === \"declared\") {\n\t\t\tthrow new ChordError({ code: \"not_ready\", message: `${this.#records.name}: its object is not live yet` });\n\t\t}\n\t\treturn this.#records.stage(tx, args, context, options);\n\t}\n\n\t/**\n\t * The attempt is sealed with `error`: starts are refused, running invocations are cut off (their signals fire, in\n\t * memory they end `aborted` with `reason` and finish on the next live attempt, persisted ones are stopped and\n\t * resumed, R3), and the runs finishing here wait for it.\n\t */\n\tseal(error: ChordError, reason: CutReason): void {\n\t\tif (this.#state === \"sealed\") return;\n\t\tthis.#state = \"sealed\";\n\t\tthis.#sealed = { error, reason };\n\t\tthis.#records.detach(this, this.#finishing);\n\t\tthis.#finishing.clear();\n\t\tfor (const run of this.#invocations) this.#records.cut(run, error, reason);\n\t\tthis.#invocations.clear();\n\t}\n\n\t/**\n\t * Runs `run` for a committed run (with its signal aborted already after an early abort); a persisted run's\n\t * reservation commits first (`running`: a crash from then on counts, D6). The reservation, the invocation and the\n\t * run's finish are one piece of work in the drain, admitted before plugin code runs: `run` may end its own object\n\t * before its first await.\n\t */\n\tinvoke(run: JobRun): void {\n\t\tconst invocation = run.begin();\n\t\tif (!this.#records.persisted || run.phase === \"running\") {\n\t\t\tthis.#start(run, invocation);\n\t\t\treturn;\n\t\t}\n\t\tconst { documents, token, context } = this.#owner;\n\t\tconst reserving = documents\n\t\t\t.tx((tx) => this.#records.reserve(tx, run), token, context)\n\t\t\t.then(\n\t\t\t\t() => {\n\t\t\t\t\tif (this.#state !== \"sealed\") {\n\t\t\t\t\t\tthis.#start(run, invocation);\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tthis.#records.cut(run, this.#sealed!.error, this.#sealed!.reason);\n\t\t\t\t\t// No `run` started: nothing else ends it, and an abort joined to it returns.\n\t\t\t\t\tinvocation.ended();\n\t\t\t\t},\n\t\t\t\t() => {\n\t\t\t\t\tinvocation.settle();\n\t\t\t\t\tinvocation.ended();\n\t\t\t\t\t// Refused because this attempt was sealed (at admission, or queued past the seal): the run is pending still,\n\t\t\t\t\t// for the next live attempt (this one is detached already). Any other failure (the store closed or failed,\n\t\t\t\t\t// the object ended) is not retried here: retrying on this attempt would only fail again.\n\t\t\t\t\tif (this.#state === \"sealed\") this.#records.requeue(run);\n\t\t\t\t},\n\t\t\t);\n\t\tthis.#owner.admit(reserving);\n\t}\n\n\t/**\n\t * Starts `invocation` of `run` on this attempt, in its run phase's span: its ctx and its settling act for this\n\t * invocation only.\n\t */\n\t#start(run: JobRun, invocation: Invocation): void {\n\t\tconst { context, caller, host } = this.#owner;\n\t\tconst signal = invocation.abort.signal;\n\t\tconst span = this.#phaseSpan(run, \"run\");\n\t\tconst scope = withAbortSignal(signal, context);\n\t\tconst ctx = new JobContext(\n\t\t\tspan === undefined ? scope : withTelemetry(span.span, scope),\n\t\t\tcaller,\n\t\t\thost,\n\t\t\trun.id,\n\t\t\tthis.#stateOf(run, invocation),\n\t\t);\n\t\tconst work = Promise.withResolvers<void>();\n\t\tthis.#owner.admitRun(work.promise, () => ctx.settle());\n\t\tthis.#invocations.add(run);\n\t\tconst running = (async () => {\n\t\t\ttry {\n\t\t\t\t// Its own version's args: a run started by old work during a restart runs on the old code (its definition).\n\t\t\t\treturn await this.#implementation.run(this.#records.argsOf(run, this.#definition), ctx);\n\t\t\t} finally {\n\t\t\t\t// The fence closes as `run` settles: a change made later is refused.\n\t\t\t\tinvocation.settle();\n\t\t\t}\n\t\t})();\n\t\trunning\n\t\t\t.then(\n\t\t\t\t(result) => this.#settled(run, invocation, ctx, { kind: \"done\", result }),\n\t\t\t\t(error: unknown) => this.#settled(run, invocation, ctx, this.#failed(run, invocation, error)),\n\t\t\t)\n\t\t\t.finally(() => invocation.ended())\n\t\t\t.then(work.resolve);\n\t\tspan?.endWhen(running, \"job_failed\", signal);\n\t}\n\n\t/**\n\t * The span of `run`'s phase `phase`, under its start's carrier (a persisted run resumed in a new attempt or process\n\t * too: the same trace; without one the adapter's root); none when nothing records.\n\t */\n\t#phaseSpan(run: JobRun, phase: \"run\" | \"finish\"): OpenSpan<\"chord.job\"> | undefined {\n\t\tconst { host, documents } = this.#owner;\n\t\tconst parent = parentOf(run.trace, host.telemetry);\n\t\tif (untraced(parent)) return undefined;\n\t\treturn new OpenSpan(parent, \"chord.job\", {\n\t\t\t\"chord.host\": host.spanHost,\n\t\t\tjob: this.#definition.name,\n\t\t\tphase,\n\t\t\tobject: capped(documents.label),\n\t\t\t\"job.id\": run.id,\n\t\t});\n\t}\n\n\t/**\n\t * Runs the finish of `run`, whose outcome is recorded: `finish` with the cleanup in one transaction under the\n\t * attempt's token, retried with backoff after a throw (reported), blocked after five in a row. The attempt's seal\n\t * stops it without a report or a count; the next live attempt finishes the run.\n\t */\n\tfinish(run: JobRun): void {\n\t\tconst work = this.#finish(run);\n\t\tthis.#owner.admit(work);\n\t}\n\n\t/** How a throwing `run` ended: `aborted` after its invocation's signal fired, otherwise `threw` (reported). */\n\t#failed(run: JobRun, invocation: Invocation, error: unknown): Settled {\n\t\tif (invocation.abort.signal.aborted) return { kind: \"failed\", error: { kind: \"aborted\", reason: \"abort\" } };\n\t\tthis.#records.report({ kind: \"job_failed\", id: run.id, error });\n\t\tconst thrown = isChordError(error) || error instanceof PluginError ? error : internal(error);\n\t\treturn { kind: \"failed\", error: { kind: \"threw\", error: thrown } };\n\t}\n\n\t/**\n\t * `invocation` of `run` settled: its `ctx.get` references go; its outcome is recorded unless the attempt's seal cut\n\t * it off (which recorded its own, or stopped the run for a later invocation), then the run finishes here while this\n\t * attempt is live.\n\t */\n\tasync #settled(run: JobRun, invocation: Invocation, ctx: JobContext, ended: Settled): Promise<void> {\n\t\tctx.settle();\n\t\tif (invocation.token.sealed || run.invocation !== invocation) return;\n\t\tthis.#invocations.delete(run);\n\t\tthis.#records.settled(run, ended);\n\t}\n\n\t/**\n\t * `run`'s finish phase in its span: each failed try a `chord.retry` event with its backoff; ends `ok`, `error`\n\t * (blocked) or `aborted` (the attempt's seal).\n\t */\n\tasync #finish(run: JobRun): Promise<void> {\n\t\tconst span = this.#phaseSpan(run, \"finish\");\n\t\tconst { context } = this.#owner;\n\t\tlet outcome: \"ok\" | \"error\" | \"aborted\" = \"error\";\n\t\ttry {\n\t\t\toutcome = await this.#finishTries(run, span, span === undefined ? context : withTelemetry(span.span, context));\n\t\t} finally {\n\t\t\tspan?.span.setAttributes({ outcome });\n\t\t\tspan?.end();\n\t\t}\n\t}\n\n\t/** The tries of `run`'s finish under `context`, retried with backoff; how the phase ended. */\n\tasync #finishTries(\n\t\trun: JobRun,\n\t\tspan: OpenSpan<\"chord.job\"> | undefined,\n\t\tcontext: Context,\n\t): Promise<\"ok\" | \"error\" | \"aborted\"> {\n\t\tconst { documents, token, signal } = this.#owner;\n\t\tthis.#finishing.add(run);\n\t\tlet delay: number | undefined;\n\t\ttry {\n\t\t\tfor (;;) {\n\t\t\t\ttry {\n\t\t\t\t\tawait documents.tx(\n\t\t\t\t\t\tasync (tx) => {\n\t\t\t\t\t\t\tconst staging = this.#records.staging(tx);\n\t\t\t\t\t\t\tstaging.finishing = run;\n\t\t\t\t\t\t\tawait this.#implementation.finish?.(this.#records.outcomeOf(run, context), tx);\n\t\t\t\t\t\t\tthis.#records.change(tx, { kind: \"delete\", run });\n\t\t\t\t\t\t},\n\t\t\t\t\t\ttoken,\n\t\t\t\t\t\tcontext,\n\t\t\t\t\t);\n\t\t\t\t\treturn \"ok\";\n\t\t\t\t} catch (error) {\n\t\t\t\t\tif (this.#state === \"sealed\") return \"aborted\";\n\t\t\t\t\tthis.#records.report({ kind: \"job_finish_failed\", id: run.id, error });\n\t\t\t\t\t// The count is the run's (stored for a persisted job): failures count across a crash.\n\t\t\t\t\tconst finishFailures = run.finishFailures + 1;\n\t\t\t\t\tif (finishFailures >= FINISH_ATTEMPTS) {\n\t\t\t\t\t\tif (span !== undefined) spanFailed(span.span, \"job_failed\", error);\n\t\t\t\t\t\tawait this.#records.record(run, { kind: \"block\", run, reason: \"finish_failed\" });\n\t\t\t\t\t\treturn \"error\";\n\t\t\t\t\t}\n\t\t\t\t\tawait this.#records.record(run, { kind: \"update\", run, fields: { finishFailures } });\n\t\t\t\t\tdelay = nextDelay(delay);\n\t\t\t\t\tspan?.span.addEvent(\"chord.retry\", { backoff: delay });\n\t\t\t\t\ttry {\n\t\t\t\t\t\tawait sleep(delay, withAbortSignal(signal, context));\n\t\t\t\t\t} catch {\n\t\t\t\t\t\treturn \"aborted\";\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t} finally {\n\t\t\tthis.#finishing.delete(run);\n\t\t}\n\t}\n\n\t/** `ctx.state` of `invocation` of `run`, when the job has a state: changes under its token while it runs. */\n\t#stateOf(run: JobRun, invocation: Invocation): RunState<JsonObject> | undefined {\n\t\tconst records = this.#records;\n\t\tconst states = records.states;\n\t\tif (states === undefined) return undefined;\n\t\tconst { documents } = this.#owner;\n\t\tconst refused = (): ChordError =>\n\t\t\tnew ChordError({\n\t\t\t\tcode: \"released\",\n\t\t\t\treason: \"settled\",\n\t\t\t\tmessage: `${records.name}: run ${run.id} settled; its state is closed`,\n\t\t\t});\n\t\treturn {\n\t\t\tget value(): JsonObject {\n\t\t\t\tconst value = records.stateOf(run.id);\n\t\t\t\tif (value === undefined) throw refused();\n\t\t\t\treturn value;\n\t\t\t},\n\t\t\tchange: <T>(\n\t\t\t\tfn: (draft: Draft<JsonObject>) => T,\n\t\t\t\tcontext: Context,\n\t\t\t\toptions: { readonly persistence?: CommitPersistence } = {},\n\t\t\t): Promise<T> => {\n\t\t\t\tif (invocation.state !== \"running\") return Promise.reject(refused());\n\t\t\t\tlet persistence: CommitPersistence;\n\t\t\t\ttry {\n\t\t\t\t\tpersistence = checkedPersistence(options.persistence, records.name);\n\t\t\t\t} catch (error) {\n\t\t\t\t\treturn Promise.reject(error);\n\t\t\t\t}\n\t\t\t\t// Never the abort signal: a run records its progress also after an abort.\n\t\t\t\tconst unaborted = withoutAbortSignal(context);\n\t\t\t\t// A change resetting the restart count is stored at once: the reset must be durable (D8).\n\t\t\t\tconst now = run.restarts === 0 ? persistence : IMMEDIATE;\n\t\t\t\treturn documents.tx(\n\t\t\t\t\tasync (tx) => {\n\t\t\t\t\t\tconst value = fn(await tx.draftSlot(states, run.id));\n\t\t\t\t\t\trecords.progressed(tx, run);\n\t\t\t\t\t\treturn value;\n\t\t\t\t\t},\n\t\t\t\t\tinvocation.token,\n\t\t\t\t\tunaborted,\n\t\t\t\t\tnow,\n\t\t\t\t);\n\t\t\t},\n\t\t};\n\t}\n}\n\nconst IMMEDIATE: CommitPersistence = { kind: \"immediate\" };\n\n/**\n * A state change's `persistence` (a trust boundary: plugin code): `immediate` when omitted; a deferred one's `withinMs`\n * a number from 0 (`Infinity` waits for the others), `invalid/args` otherwise.\n */\nfunction checkedPersistence(persistence: CommitPersistence | undefined, what: string): CommitPersistence {\n\tif (persistence === undefined || persistence.kind === \"immediate\") return IMMEDIATE;\n\tconst { withinMs } = persistence;\n\tif (\n\t\tpersistence.kind !== \"deferred\" ||\n\t\t(withinMs !== undefined && !(typeof withinMs === \"number\" && withinMs >= 0))\n\t) {\n\t\tthrow new ChordError({\n\t\t\tcode: \"invalid\",\n\t\t\treason: \"args\",\n\t\t\tmessage: `${what}: persistence is immediate or deferred with withinMs >= 0`,\n\t\t});\n\t}\n\treturn persistence;\n}\n\n/** A plugin's throw that is neither a ChordError nor a PluginError, as a run's outcome keeps it. */\nfunction internal(error: unknown): PluginError {\n\treturn new PluginError(\"internal\", error instanceof Error ? error.message : String(error));\n}\n\n/** A run's ctx: the handler's ctx of its object, its id and, with a state, its state document. */\nexport class JobContext extends CallContext {\n\treadonly jobId: string;\n\tdeclare readonly state: RunState<JsonObject> | undefined;\n\n\tconstructor(\n\t\tcontext: Context,\n\t\tcaller: Caller,\n\t\thost: AnyHost,\n\t\tjobId: string,\n\t\tstate: RunState<JsonObject> | undefined,\n\t) {\n\t\tsuper(context, caller, host);\n\t\tthis.jobId = jobId;\n\t\tif (state !== undefined) this.state = state;\n\t}\n\n\t/** Its child under `span` (`ctx.trace`): a job's ctx still, its id and state kept. */\n\toverride traced(span: TelemetrySpan): Ctx {\n\t\treturn new JobSpanContext(this, span, this.jobId, this.state);\n\t}\n}\n\n/** A run's ctx under a child span (`ctx.trace`): its parent's members, its run's id and state. */\nclass JobSpanContext extends SpanCtx {\n\treadonly jobId: string;\n\tdeclare readonly state: RunState<JsonObject> | undefined;\n\n\tconstructor(\n\t\tparent: JobContext | JobSpanContext,\n\t\tspan: TelemetrySpan,\n\t\tjobId: string,\n\t\tstate: RunState<JsonObject> | undefined,\n\t) {\n\t\tsuper(parent, span);\n\t\tthis.jobId = jobId;\n\t\tif (state !== undefined) this.state = state;\n\t}\n\n\toverride traced(span: TelemetrySpan): Ctx {\n\t\treturn new JobSpanContext(this, span, this.jobId, this.state);\n\t}\n}\n"]}

package/dist/jobs/persisted.d.ts

@@ -0,0 +1,38 @@
+import type { Context } from "../context/index.ts";
+import type { Persistence, Stored } from "../documents/internal.ts";
+import type { StagedWrite } from "../store/step.ts";
+import { type StoredRun } from "./stored.ts";
+/** A run record as a load found it: its run id, the record and what is stored of it. */
+export type LoadedRecord = {
+    readonly id: string;
+    readonly record: StoredRun;
+    readonly stored: Stored;
+};
+/** The record documents of one persisted job's runs, and what is stored of each. */
+export declare class PersistedRuns {
+    #private;
+    /** `records` and `states` are the job's record and state documents' store sides (`chord.jobs`, `chord.job-states`). */
+    constructor(records: Persistence, states: Persistence);
+    /**
+     * Stages run `id`'s record `record` into `writes` (rule 8: serialized now): its creation, or a new base of what is
+     * stored; returns the adoption, which takes what the commit stored.
+     */
+    stage(id: string, record: StoredRun, writes: StagedWrite[]): () => void;
+    /**
+     * Stages the retirement of run `id`'s record into `writes` (the run settled or was dropped), and of its state
+     * document when the run is of another job version; the adoption forgets them.
+     */
+    stageRetire(id: string, writes: StagedWrite[]): () => void;
+    /** Finds what is stored of run `id`'s state document (a run of another job version), keeping it for the drop. */
+    findBlockedState(id: string, context: Context): Promise<void>;
+    /**
+     * A run of another job version that a reload's code now takes (`retryBlocked`): its state document is the job's
+     * again (read, decoded and retired with its record as any run's), no longer one to retire undecoded.
+     */
+    forgetBlockedState(id: string): void;
+    /** Every stored record of the job, in run id order, page by page with plain reads (activation holds the line). */
+    load(context: Context): Promise<LoadedRecord[]>;
+    /** The object's documents are dropped: nothing is known of the store any more. */
+    drop(): void;
+}
+//# sourceMappingURL=persisted.d.ts.map

package/dist/jobs/persisted.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"persisted.d.ts","sourceRoot":"","sources":["../../src/jobs/persisted.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAEpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAwB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAEnE,wFAAwF;AACxF,MAAM,MAAM,YAAY,GAAG;IAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAKxG,oFAAoF;AACpF,qBAAa,aAAa;;IAOzB,uHAAuH;IACvH,YAAY,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAGpD;IAED;;;OAGG;IACH,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAgBtE;IAED;;;OAGG;IACH,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CASzD;IAED,iHAAiH;IAC3G,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;IAED;;;OAGG;IACH,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAEnC;IAED,kHAAkH;IAC5G,IAAI,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAgBpD;IAED,kFAAkF;IAClF,IAAI,IAAI,IAAI,CAGX;CACD"}

package/dist/jobs/persisted.js

@@ -0,0 +1,95 @@
+// A persisted job's run records in its object's store (slice 6b §4.1, D5): the record document of each unsettled run
+// (namespace `chord.jobs`, kind the job name, key the run id, `history: none`), written whole in the transactions that
+// change the run, and read once per object when it first activates; for a run of another job version, also what is
+// stored of its state document (`chord.job-states`), never decoded, so its drop retires it whatever the code declares.
+import { changeDocument, createDocument, retireDocument } from "../store/engine.js";
+import { RUN_RECORD, recordOf } from "./stored.js";
+/** The keys a load reads per page. */
+const KEYS_PAGE = 100;
+/** The record documents of one persisted job's runs, and what is stored of each. */
+export class PersistedRuns {
+    #records;
+    #states;
+    #stored = new Map();
+    /** What is stored of the state documents of runs of another job version, by run id. */
+    #blockedStates = new Map();
+    /** `records` and `states` are the job's record and state documents' store sides (`chord.jobs`, `chord.job-states`). */
+    constructor(records, states) {
+        this.#records = records;
+        this.#states = states;
+    }
+    /**
+     * Stages run `id`'s record `record` into `writes` (rule 8: serialized now): its creation, or a new base of what is
+     * stored; returns the adoption, which takes what the commit stored.
+     */
+    stage(id, record, writes) {
+        const text = JSON.stringify(record);
+        const stored = this.#stored.get(id);
+        const { version, fingerprint } = RUN_RECORD;
+        const value = record;
+        if (stored === undefined) {
+            const created = { id: 0 };
+            writes.push(createDocument(this.#records.at(id), { version, fingerprint, value }, text, (made) => {
+                created.id = made;
+            }));
+            return () => this.#stored.set(id, { id: created.id, revision: 1, version, deltasSinceBase: 0 });
+        }
+        writes.push(changeDocument(stored.id, stored.revision, { kind: "base", version, fingerprint, value }, text));
+        return () => this.#stored.set(id, { ...stored, revision: stored.revision + 1 });
+    }
+    /**
+     * Stages the retirement of run `id`'s record into `writes` (the run settled or was dropped), and of its state
+     * document when the run is of another job version; the adoption forgets them.
+     */
+    stageRetire(id, writes) {
+        const stored = this.#stored.get(id);
+        if (stored !== undefined)
+            writes.push(retireDocument(stored.id, stored.revision));
+        const state = this.#blockedStates.get(id);
+        if (state !== undefined)
+            writes.push(retireDocument(state.id, state.revision));
+        return () => {
+            this.#stored.delete(id);
+            this.#blockedStates.delete(id);
+        };
+    }
+    /** Finds what is stored of run `id`'s state document (a run of another job version), keeping it for the drop. */
+    async findBlockedState(id, context) {
+        const found = await this.#states.find(id, context);
+        if (found !== undefined)
+            this.#blockedStates.set(id, found.stored);
+    }
+    /**
+     * A run of another job version that a reload's code now takes (`retryBlocked`): its state document is the job's
+     * again (read, decoded and retired with its record as any run's), no longer one to retire undecoded.
+     */
+    forgetBlockedState(id) {
+        this.#blockedStates.delete(id);
+    }
+    /** Every stored record of the job, in run id order, page by page with plain reads (activation holds the line). */
+    async load(context) {
+        const loaded = [];
+        let after;
+        for (;;) {
+            const keys = await this.#records.keys(after, KEYS_PAGE, context);
+            for (const id of keys) {
+                const document = await this.#records.read(id, context);
+                if (document === undefined)
+                    continue;
+                const { record: stored, value, deltasSinceBase } = document;
+                const found = { id: stored.id, revision: stored.revision, version: stored.version, deltasSinceBase };
+                this.#stored.set(id, found);
+                loaded.push({ id, record: recordOf(value), stored: found });
+            }
+            if (keys.length < KEYS_PAGE)
+                return loaded;
+            after = keys.at(-1);
+        }
+    }
+    /** The object's documents are dropped: nothing is known of the store any more. */
+    drop() {
+        this.#stored.clear();
+        this.#blockedStates.clear();
+    }
+}
+//# sourceMappingURL=persisted.js.map

package/dist/jobs/persisted.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"persisted.js","sourceRoot":"","sources":["../../src/jobs/persisted.ts"],"names":[],"mappings":"AAAA,qHAAqH;AACrH,uHAAuH;AACvH,mHAAmH;AACnH,uHAAuH;AAKvH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpF,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAkB,MAAM,aAAa,CAAC;AAKnE,sCAAsC;AACtC,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,oFAAoF;AACpF,MAAM,OAAO,aAAa;IAChB,QAAQ,CAAc;IACtB,OAAO,CAAc;IACrB,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,uFAAuF;IAC9E,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEpD,uHAAuH;IACvH,YAAY,OAAoB,EAAE,MAAmB;QACpD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,EAAU,EAAE,MAAiB,EAAE,MAAqB;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,UAAU,CAAC;QAC5C,MAAM,KAAK,GAAG,MAA+B,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CACV,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE;gBACpF,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC;YACnB,CAAC,CAAC,CACF,CAAC;YACF,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;QACjG,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7G,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,CAAC;IACjF,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,EAAU,EAAE,MAAqB;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpC,IAAI,MAAM,KAAK,SAAS;YAAE,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC/E,OAAO,GAAG,EAAE;YACX,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxB,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC,CAAC;IACH,CAAC;IAED,iHAAiH;IACjH,KAAK,CAAC,gBAAgB,CAAC,EAAU,EAAE,OAAgB;QAClD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACnD,IAAI,KAAK,KAAK,SAAS;YAAE,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,EAAU;QAC5B,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,kHAAkH;IAClH,KAAK,CAAC,IAAI,CAAC,OAAgB;QAC1B,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,IAAI,KAAyB,CAAC;QAC9B,SAAS,CAAC;YACT,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YACjE,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBACvD,IAAI,QAAQ,KAAK,SAAS;oBAAE,SAAS;gBACrC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,QAAQ,CAAC;gBAC5D,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,CAAC;gBACrG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,KAAmB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3E,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,GAAG,SAAS;gBAAE,OAAO,MAAM,CAAC;YAC3C,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,IAAI;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;CACD","sourcesContent":["// A persisted job's run records in its object's store (slice 6b §4.1, D5): the record document of each unsettled run\n// (namespace `chord.jobs`, kind the job name, key the run id, `history: none`), written whole in the transactions that\n// change the run, and read once per object when it first activates; for a run of another job version, also what is\n// stored of its state document (`chord.job-states`), never decoded, so its drop retires it whatever the code declares.\n\nimport type { Context } from \"../context/index.ts\";\nimport type { JsonObject } from \"../documents/index.ts\";\nimport type { Persistence, Stored } from \"../documents/internal.ts\";\nimport { changeDocument, createDocument, retireDocument } from \"../store/engine.ts\";\nimport type { StagedWrite } from \"../store/step.ts\";\nimport { RUN_RECORD, recordOf, type StoredRun } from \"./stored.ts\";\n\n/** A run record as a load found it: its run id, the record and what is stored of it. */\nexport type LoadedRecord = { readonly id: string; readonly record: StoredRun; readonly stored: Stored };\n\n/** The keys a load reads per page. */\nconst KEYS_PAGE = 100;\n\n/** The record documents of one persisted job's runs, and what is stored of each. */\nexport class PersistedRuns {\n\treadonly #records: Persistence;\n\treadonly #states: Persistence;\n\treadonly #stored = new Map<string, Stored>();\n\t/** What is stored of the state documents of runs of another job version, by run id. */\n\treadonly #blockedStates = new Map<string, Stored>();\n\n\t/** `records` and `states` are the job's record and state documents' store sides (`chord.jobs`, `chord.job-states`). */\n\tconstructor(records: Persistence, states: Persistence) {\n\t\tthis.#records = records;\n\t\tthis.#states = states;\n\t}\n\n\t/**\n\t * Stages run `id`'s record `record` into `writes` (rule 8: serialized now): its creation, or a new base of what is\n\t * stored; returns the adoption, which takes what the commit stored.\n\t */\n\tstage(id: string, record: StoredRun, writes: StagedWrite[]): () => void {\n\t\tconst text = JSON.stringify(record);\n\t\tconst stored = this.#stored.get(id);\n\t\tconst { version, fingerprint } = RUN_RECORD;\n\t\tconst value = record as unknown as JsonObject;\n\t\tif (stored === undefined) {\n\t\t\tconst created = { id: 0 };\n\t\t\twrites.push(\n\t\t\t\tcreateDocument(this.#records.at(id), { version, fingerprint, value }, text, (made) => {\n\t\t\t\t\tcreated.id = made;\n\t\t\t\t}),\n\t\t\t);\n\t\t\treturn () => this.#stored.set(id, { id: created.id, revision: 1, version, deltasSinceBase: 0 });\n\t\t}\n\t\twrites.push(changeDocument(stored.id, stored.revision, { kind: \"base\", version, fingerprint, value }, text));\n\t\treturn () => this.#stored.set(id, { ...stored, revision: stored.revision + 1 });\n\t}\n\n\t/**\n\t * Stages the retirement of run `id`'s record into `writes` (the run settled or was dropped), and of its state\n\t * document when the run is of another job version; the adoption forgets them.\n\t */\n\tstageRetire(id: string, writes: StagedWrite[]): () => void {\n\t\tconst stored = this.#stored.get(id);\n\t\tif (stored !== undefined) writes.push(retireDocument(stored.id, stored.revision));\n\t\tconst state = this.#blockedStates.get(id);\n\t\tif (state !== undefined) writes.push(retireDocument(state.id, state.revision));\n\t\treturn () => {\n\t\t\tthis.#stored.delete(id);\n\t\t\tthis.#blockedStates.delete(id);\n\t\t};\n\t}\n\n\t/** Finds what is stored of run `id`'s state document (a run of another job version), keeping it for the drop. */\n\tasync findBlockedState(id: string, context: Context): Promise<void> {\n\t\tconst found = await this.#states.find(id, context);\n\t\tif (found !== undefined) this.#blockedStates.set(id, found.stored);\n\t}\n\n\t/**\n\t * A run of another job version that a reload's code now takes (`retryBlocked`): its state document is the job's\n\t * again (read, decoded and retired with its record as any run's), no longer one to retire undecoded.\n\t */\n\tforgetBlockedState(id: string): void {\n\t\tthis.#blockedStates.delete(id);\n\t}\n\n\t/** Every stored record of the job, in run id order, page by page with plain reads (activation holds the line). */\n\tasync load(context: Context): Promise<LoadedRecord[]> {\n\t\tconst loaded: LoadedRecord[] = [];\n\t\tlet after: string | undefined;\n\t\tfor (;;) {\n\t\t\tconst keys = await this.#records.keys(after, KEYS_PAGE, context);\n\t\t\tfor (const id of keys) {\n\t\t\t\tconst document = await this.#records.read(id, context);\n\t\t\t\tif (document === undefined) continue;\n\t\t\t\tconst { record: stored, value, deltasSinceBase } = document;\n\t\t\t\tconst found = { id: stored.id, revision: stored.revision, version: stored.version, deltasSinceBase };\n\t\t\t\tthis.#stored.set(id, found);\n\t\t\t\tloaded.push({ id, record: recordOf(value as JsonObject), stored: found });\n\t\t\t}\n\t\t\tif (keys.length < KEYS_PAGE) return loaded;\n\t\t\tafter = keys.at(-1);\n\t\t}\n\t}\n\n\t/** The object's documents are dropped: nothing is known of the store any more. */\n\tdrop(): void {\n\t\tthis.#stored.clear();\n\t\tthis.#blockedStates.clear();\n\t}\n}\n"]}