@hotmeshio/long-tail 0.28.0 → 0.28.1

package/build/services/machines/gateway/api-escalations.js

@@ -42,6 +42,8 @@ const escalationApi = __importStar(require("../../../api/escalations"));
 const uuid_1 = require("../../../lib/uuid");
 const escalations_1 = require("./escalations");
 const actions_1 = require("./actions");
+/** A row that was never given items (one parked by hand) answers 400; it holds none. */
+const NO_ITEMS_STATUS = 400;
 class ApiLineEscalations {
     constructor(userFor) {
         this.userFor = userFor;
@@ -75,6 +77,8 @@ class ApiLineEscalations {
     }
     async items(id) {
         const result = await escalationApi.getEscalationItems({ id }, this.auth(actions_1.CONTROLLER));
+        if (result.status === NO_ITEMS_STATUS)
+            return [];
         if (result.status >= 300)
             throw new Error(`items failed ${result.status}: ${result.error}`);
         return result.data.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));

package/build/services/machines/gateway/dispatcher.d.ts

@@ -52,12 +52,15 @@ export declare class Dispatcher {
     /** Work still running sends nothing more and changes no row. */
     stop(): void;
     private live;
-    /** This send's attempt number, and whether the machine may have the command already. */
+    /**
+     * This send's attempt number, whether the machine may have the command already, and when it was
+     * issued: the time given (a round's start, the same for every gateway), else the first send's.
+     */
     private countSend;
     /** False when the codec has no message for the command. `since` is when the command's row was created. */
-    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<boolean>;
+    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number, issuedAt?: number): Promise<boolean>;
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<CommandReply | null>;
+    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number, issuedAt?: number): Promise<CommandReply | null>;
     command(a: CommandAction): Promise<void>;
     /**
      * A machine-command row: send it, then resolve the row with the reply, as the
@@ -68,7 +71,9 @@ export declare class Dispatcher {
      */
     control(a: ControlAction): Promise<void>;
     dispatch(a: DispatchAction): Promise<void>;
-    /** The units were already at the machine: its acceptance completes the handoff. */
+    /** A job sent again after the machine restarted, for a row that already ended: the reply is only logged. */
+    private resent;
+    /** The units were already at the machine: a job it has taken and is ready for completes the handoff. */
     private received;
     /** Ask a silent machine what it holds; null when it cannot say or does not answer in time. */
     private heldJobs;

package/build/services/machines/gateway/dispatcher.js

@@ -57,20 +57,25 @@ class Dispatcher {
         if (this.stopping.signal.aborted)
             throw new GatewayStopped();
     }
-    /** This send's attempt number, and whether the machine may have the command already. */
-    countSend(cmdId, since) {
-        const attempt = (this.sent.get(cmdId) ?? 0) + 1;
+    /**
+     * This send's attempt number, whether the machine may have the command already, and when it was
+     * issued: the time given (a round's start, the same for every gateway), else the first send's.
+     */
+    countSend(cmdId, since, issuedAt) {
+        const before = this.sent.get(cmdId);
+        const record = { attempt: (before?.attempt ?? 0) + 1, issuedAt: issuedAt ?? before?.issuedAt ?? Date.now() };
         this.sent.delete(cmdId);
-        this.sent.set(cmdId, attempt);
+        this.sent.set(cmdId, record);
         if (this.sent.size > SENT_REMEMBERED)
             this.sent.delete(this.sent.keys().next().value);
-        return { attempt, retry: attempt > 1 || (since !== undefined && since < this.d.state.takenAt) };
+        const retry = record.attempt > 1 || (since !== undefined && since < this.d.state.takenAt);
+        return { attempt: record.attempt, retry, issuedAt: (0, protocol_1.isoTs)(record.issuedAt) };
     }
     /** False when the codec has no message for the command. `since` is when the command's row was created. */
-    async publish(machineId, op, cmdId, args, since) {
+    async publish(machineId, op, cmdId, args, since, issuedAt) {
         this.live();
         const ident = (0, registration_1.identFor)(this.d.state, this.d.line, machineId);
-        const command = { cmdId, op, args, ...this.countSend(cmdId, since) };
+        const command = { cmdId, op, args, ...this.countSend(cmdId, since, issuedAt) };
         const wires = this.d.codec.encode(ident, command);
         for (const wire of wires) {
             await this.d.bus.publish(wire.topic, wire.payload);
@@ -79,11 +84,11 @@ class Dispatcher {
         return wires.length > 0;
     }
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    async ask(machineId, op, cmdId, args, since) {
+    async ask(machineId, op, cmdId, args, since, issuedAt) {
         for (let attempt = 1; attempt <= this.d.dispatchAttempts; attempt++) {
             this.live();
             const waiting = this.d.replies.wait(cmdId, this.d.replyTimeoutMs);
-            if (!await this.publish(machineId, op, cmdId, args, since)) {
+            if (!await this.publish(machineId, op, cmdId, args, since, issuedAt)) {
                 this.d.replies.settle({ cmdId, status: contracts_1.COMMAND_STATUS.UNSUPPORTED, reason: `the line's codec has no ${op} command` });
             }
             const reply = await waiting;
@@ -112,8 +117,9 @@ class Dispatcher {
      */
     async control(a) {
         const now = Date.now();
-        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, this.d.replyTimeoutMs * this.d.dispatchAttempts);
-        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt);
+        const ttlMs = this.d.replyTimeoutMs * this.d.dispatchAttempts;
+        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, ttlMs);
+        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt, expiresAt - ttlMs);
         // Timers may fire a little early; the command must have expired before the row says no-reply.
         if (!reply && Date.now() <= expiresAt)
             await (0, promises_1.setTimeout)(expiresAt - Date.now() + 1, undefined, { signal: this.stopping.signal }).catch(() => undefined);
@@ -142,22 +148,36 @@ class Dispatcher {
         for (const job of jobs)
             state.rememberDispatch(`${a.cmdId}${job.suffix}`, a.rowId);
         const since = state.get(a.rowId)?.createdAt;
-        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since)));
+        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since, a.issuedAt)));
         const refused = replies.find((r) => r && !took(r));
         const silent = jobs.filter((_, i) => !replies[i]);
         const held = !refused && silent.length ? await this.heldJobs(a.machineId, a.cmdId) : null;
         this.live();
         const reconciled = !!held && silent.every((j) => held.has(j.jobId));
+        if (a.ended) {
+            this.resent(a, jobs, refused ?? null, !refused && (!silent.length || reconciled));
+            return;
+        }
         const row = state.get(a.rowId);
         if (!refused && (!silent.length || reconciled)) {
             if (row)
                 row.dispatch = state_1.DISPATCH_STATE.ACCEPTED;
+            for (const job of jobs)
+                state.rememberSent(a.machineId, job.jobId, a.rowId);
             if (reconciled)
                 log.info(log_1.GATEWAY_LOG.DISPATCH_RECONCILED, { row: a.rowId, machine: a.machineId, jobs: silent.map((j) => j.jobId) });
             log.info(log_1.GATEWAY_LOG.DISPATCHED, { row: a.rowId, machine: a.machineId, reason: replies.map((r) => r?.reason ?? 'held').join('; ') });
             this.check(a.rowId, 'claim', await lt.claim(a.machineId, a.rowId));
-            if (a.receivedOnAccept)
+            if (!a.receivedOnAccept)
+                return;
+            const preparing = jobs.filter((job, i) => replies[i]?.status === protocol_1.REPLY_STATUS.ACCEPTED && replies[i]?.preparing).map((job) => `${a.cmdId}${job.suffix}`);
+            if (!preparing.length) {
                 await this.received(a);
+                return;
+            }
+            if (row)
+                row.awaitingReady = new Set(preparing);
+            log.info(log_1.GATEWAY_LOG.DISPATCH_PREPARING, { row: a.rowId, machine: a.machineId, jobs: preparing });
             return;
         }
         // The handoff may have ended meanwhile: a machine that ran the job anyway reports the arrival.
@@ -176,7 +196,15 @@ class Dispatcher {
             return;
         await this.withdraw(a, jobs.filter((_, i) => !replies[i] || took(replies[i])));
     }
-    /** The units were already at the machine: its acceptance completes the handoff. */
+    /** A job sent again after the machine restarted, for a row that already ended: the reply is only logged. */
+    resent(a, jobs, refused, taken) {
+        const context = { row: a.rowId, machine: a.machineId, jobs: jobs.map((j) => j.jobId), resent: true };
+        if (taken)
+            this.d.log.info(log_1.GATEWAY_LOG.DISPATCHED, { ...context, reason: 'sent again after a restart' });
+        else
+            this.d.log.warn(log_1.GATEWAY_LOG.DISPATCH_REFUSED, { ...context, reason: refused ? `${refused.status}: ${refused.reason}` : 'no reply' });
+    }
+    /** The units were already at the machine: a job it has taken and is ready for completes the handoff. */
     async received(a) {
         this.live();
         const units = a.args.units.map((u) => u.unitId);

package/build/services/machines/gateway/executor.d.ts

@@ -57,6 +57,8 @@ export declare class Executor {
      * carrying the units out of it.
      */
     private lateReply;
+    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    private readyForArrival;
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     private accumulate;
     private check;

package/build/services/machines/gateway/executor.js

@@ -109,7 +109,7 @@ class Executor {
                 this.watchOffline(a.machineId);
                 return [(0, wire_1.plannedAction)(a)];
             case 'settleReply':
-                if (replies.settle({ cmdId: a.cmdId, status: a.status, reason: a.reason, ...(a.code ? { code: a.code } : {}) }))
+                if (replies.settle({ cmdId: a.cmdId, status: a.status, reason: a.reason, ...(a.code ? { code: a.code } : {}), ...(a.preparing ? { preparing: true } : {}) }))
                     return [(0, wire_1.plannedAction)(a)];
                 return [{ verb: 'ack', detail: `late ${a.status} ${a.cmdId}` }, ...await this.lateReply(a)];
             case 'settleSnapshot':
@@ -123,6 +123,9 @@ class Executor {
             case 'refuseBinding':
                 this.d.log.warn(log_1.GATEWAY_LOG.ADDRESS_REFUSED, { machine: a.machineId, address: a.address, reason: a.reason });
                 return [(0, wire_1.plannedAction)(a)];
+            case 'unrecoverable':
+                this.d.log.warn(log_1.GATEWAY_LOG.DISPATCH_UNRECOVERABLE, { machine: a.machineId, jobs: a.jobs });
+                return [(0, wire_1.plannedAction)(a)];
         }
     }
     /**
@@ -140,7 +143,7 @@ class Executor {
         const context = { row: into.id, machine: m, cmdId: a.cmdId, reason: a.reason };
         if (a.status === protocol_1.REPLY_STATUS.COMPLETED) {
             log.info(log_1.GATEWAY_LOG.DISPATCH_READY, context);
-            return [];
+            return this.readyForArrival(into, m, a.cmdId);
         }
         if (a.status !== protocol_1.REPLY_STATUS.FAILED)
             return [];
@@ -152,6 +155,19 @@ class Executor {
             received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_FAILED, detail: `${m} could not prepare the job: ${a.reason}`, ...(a.code ? { code: a.code } : {}),
         }));
     }
+    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    async readyForArrival(into, m, cmdId) {
+        const { lt, state } = this.d;
+        if (into.ended || !into.awaitingReady?.has(cmdId))
+            return [];
+        into.awaitingReady.delete(cmdId);
+        if (into.awaitingReady.size)
+            return [];
+        const units = into.units.map((u) => u.unitId);
+        const resolved = await lt.resolve(m, into.id, { received: true, receiver: m, units });
+        state.close(into.id);
+        return [outcomeOf('resolve', into.id, resolved)];
+    }
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     async accumulate(a) {
         const { lt, state } = this.d;

package/build/services/machines/gateway/gateway.d.ts

@@ -15,6 +15,7 @@ import type { LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import { GatewayState } from './state';
 import { type WireSink } from './wire';
+import { WireWindow } from './wire-window';
 export interface GatewaySettings {
     topicRoot: string;
     replyTimeoutMs: number;
@@ -32,6 +33,8 @@ export interface GatewaySettings {
     progressStep?: number;
     /** Record telemetry messages on the wire too; off unless set. */
     wireTelemetry?: boolean;
+    /** How many of the latest wire events the gateway keeps for reading over HTTP; 500 by default. */
+    wireWindow?: number;
 }
 export interface GatewayDeps {
     bus: MachineBus;
@@ -60,6 +63,8 @@ export declare class MachineGateway {
     private readonly addresses;
     private readonly admission;
     private readonly wire;
+    /** The line's last wire events, newest first on read. */
+    readonly window: WireWindow;
     private quietTimer?;
     constructor(deps: GatewayDeps);
     start(): Promise<void>;

package/build/services/machines/gateway/gateway.js

@@ -28,6 +28,7 @@ const plan_machine_1 = require("./plan-machine");
 const state_1 = require("./state");
 const receipts_1 = require("./receipts");
 const wire_1 = require("./wire");
+const wire_window_1 = require("./wire-window");
 class MachineGateway {
     constructor(deps) {
         this.deps = deps;
@@ -42,14 +43,16 @@ class MachineGateway {
         this.state.retainedAlarms = this.codec.retainedAlarms === true;
         this.state.progressStep = deps.settings.progressStep ?? this.state.progressStep;
         this.addresses = deps.addresses ?? new address_book_1.MemoryAddressBook();
-        this.wire = new wire_1.WireRecorder(deps.line, deps.wire ?? publish_1.publishMachineWireEvent, { telemetry: deps.settings.wireTelemetry === true });
+        this.window = new wire_window_1.WireWindow(deps.settings.wireWindow);
+        const sink = deps.wire ?? publish_1.publishMachineWireEvent;
+        this.wire = new wire_1.WireRecorder(deps.line, (event) => { this.window.push(event); sink(event); }, { telemetry: deps.settings.wireTelemetry === true });
         this.executor = new executor_1.Executor({
             addresses: this.addresses, wire: this.wire,
             bus: deps.bus, lt: deps.lt, replies: new reply_waiter_1.ReplyWaiter(), state: this.state, line: deps.line,
             codec: this.codec, replyTimeoutMs: deps.settings.replyTimeoutMs,
             dispatchAttempts: deps.settings.dispatchAttempts, offlineDebounceMs: deps.settings.offlineDebounceMs, log: this.log,
         });
-        this.admission = { state: this.state, line: deps.line, executor: this.executor, log: this.log, pace: () => this.pace() };
+        this.admission = { state: this.state, line: deps.line, executor: this.executor, log: this.log, pace: () => this.pace(), openRows: (q) => deps.lt.listOpen(q) };
         this.receipts = new receipts_1.ReceiptBatcher({ bus: deps.bus, codec: this.codec, delayMs: deps.settings.receiptDelayMs ?? DEFAULT_RECEIPT_DELAY_MS, log: this.log, wire: this.wire });
     }
     async start() {
@@ -231,9 +234,14 @@ class MachineGateway {
             if (seen.violations.length && msg.deliveries === 1) {
                 this.log.warn(log_1.GATEWAY_LOG.CONTRACT_VIOLATION, { topic: msg.topic, violations: seen.violations });
             }
-            const applied = seen.violations.length && this.codec.strict
+            let applied = seen.violations.length && this.codec.strict
                 ? await (0, admission_1.holdViolation)(this.admission, msg, seen)
                 : await this.applyMachine(msg.topic, seen.parsed, seen.body);
+            // An arrival still waiting for its row after every delivery is shown to a person instead; one that read no id at once.
+            const waited = msg.deliveries >= wire_1.MAX_DELIVERIES || (0, admission_1.isUnreadableArrival)(seen);
+            if (applied.disposition === transport_1.DISPOSITION.NAK && !applied.failed && waited && (0, admission_1.isArrival)(seen)) {
+                applied = await (0, admission_1.holdArrival)(this.admission, msg, seen);
+            }
             failed ||= applied.failed;
             outcome = worse(outcome, applied.disposition);
             this.wire.inbound(msg, seen, { ...applied, actions: [...admitted.actions, ...applied.actions] }, msg.deliveries);

package/build/services/machines/gateway/log.d.ts

@@ -5,8 +5,10 @@ export declare const GATEWAY_LOG: {
     readonly DISPATCHED: "machine_gateway.dispatch.accepted";
     readonly DISPATCH_REFUSED: "machine_gateway.dispatch.refused";
     readonly DISPATCH_READY: "machine_gateway.dispatch.ready";
+    readonly DISPATCH_PREPARING: "machine_gateway.dispatch.preparing";
     readonly DISPATCH_FAILED: "machine_gateway.dispatch.failed";
     readonly DISPATCH_RECONCILED: "machine_gateway.dispatch.reconciled";
+    readonly DISPATCH_UNRECOVERABLE: "machine_gateway.dispatch.unrecoverable";
     readonly VERB_INVALID: "machine_gateway.verb.invalid";
     readonly VERB_FAILED: "machine_gateway.verb.failed";
     readonly ROW_NOT_VISIBLE: "machine_gateway.verb.row_not_visible";

package/build/services/machines/gateway/log.js

@@ -10,8 +10,10 @@ exports.GATEWAY_LOG = {
     DISPATCHED: 'machine_gateway.dispatch.accepted',
     DISPATCH_REFUSED: 'machine_gateway.dispatch.refused',
     DISPATCH_READY: 'machine_gateway.dispatch.ready',
+    DISPATCH_PREPARING: 'machine_gateway.dispatch.preparing',
     DISPATCH_FAILED: 'machine_gateway.dispatch.failed',
     DISPATCH_RECONCILED: 'machine_gateway.dispatch.reconciled',
+    DISPATCH_UNRECOVERABLE: 'machine_gateway.dispatch.unrecoverable',
     VERB_INVALID: 'machine_gateway.verb.invalid',
     VERB_FAILED: 'machine_gateway.verb.failed',
     ROW_NOT_VISIBLE: 'machine_gateway.verb.row_not_visible',

package/build/services/machines/gateway/lt-http.js

@@ -56,6 +56,8 @@ class LtHttp {
     }
 }
 exports.LtHttp = LtHttp;
+/** A row that was never given items (one parked by hand) answers 400; it holds none. */
+const NO_ITEMS_STATUS = 400;
 /** Escalation verbs over HTTP, each run with the bearer token of the actor it runs as. */
 class HttpLineEscalations {
     constructor(http, tokenFor, claimMinutes = 24 * 60) {
@@ -80,8 +82,15 @@ class HttpLineEscalations {
         return (await this.items(id)).map((i) => i.itemKey);
     }
     async items(id) {
-        const view = await this.http.request('GET', `/escalations/${id}/items`, this.tokenFor(actions_1.CONTROLLER));
-        return view.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));
+        try {
+            const view = await this.http.request('GET', `/escalations/${id}/items`, this.tokenFor(actions_1.CONTROLLER));
+            return view.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));
+        }
+        catch (err) {
+            if (err instanceof LtHttpError && err.status === NO_ITEMS_STATUS)
+                return [];
+            throw err;
+        }
     }
     async get(id) {
         if (!(0, uuid_1.isUuid)(id))

package/build/services/machines/gateway/pacing.js

@@ -66,7 +66,7 @@ function dispatchAction(row, machineId, dispatchTtlMs) {
     row.createdAt ??= now;
     const { round, expiresAt } = dispatchWindow(row.dueAt ?? row.createdAt, now, dispatchTtlMs);
     return {
-        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt),
+        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt), issuedAt: expiresAt - dispatchTtlMs,
         ...(row.dispatch === state_1.DISPATCH_STATE.ARRIVING ? { receivedOnAccept: true } : {}),
     };
 }

package/build/services/machines/gateway/plan-machine.js

@@ -36,7 +36,10 @@ function planMachineMessage(state, parsed, env, dispatchTtlMs) {
 function planByKind(state, m, parsed, env, b, ttl) {
     switch (parsed.kind) {
         case protocol_1.MESSAGE_KIND.REPLY:
-            return ack([{ kind: 'settleReply', cmdId: b.cmdId, status: b.status, reason: b.reason, ...(typeof b.code === 'string' ? { code: b.code } : {}) }]);
+            return ack([{
+                    kind: 'settleReply', cmdId: b.cmdId, status: b.status, reason: b.reason,
+                    ...(typeof b.code === 'string' ? { code: b.code } : {}), ...(b.preparing === true ? { preparing: true } : {}),
+                }]);
         case protocol_1.MESSAGE_KIND.JOB:
             return planJob(state, m, b, env?.at, ttl);
         case protocol_1.MESSAGE_KIND.ALARM:
@@ -191,7 +194,9 @@ function planAlarm(state, parsed, alarmId, env) {
                 payload: { decision: contracts_1.DECISION.CLEARED, clearedBy: b.clearedBy ?? null, note: `cleared at ${m}${b.clearedBy ? ` by ${b.clearedBy}` : ''}` },
             }]);
     }
-    if (!protocol_1.STOPPING_LEVELS.includes(b.level))
+    // A stopping level needs a person; so does any alarm the machine flags, at the level it said.
+    const stopping = protocol_1.STOPPING_LEVELS.includes(b.level);
+    if (!stopping && b.needsPerson !== true)
         return ack();
     const known = state.alarms.get(key);
     if (known && (!known.raisedAt || !b.raisedAt || known.raisedAt === b.raisedAt))
@@ -201,7 +206,7 @@ function planAlarm(state, parsed, alarmId, env) {
         : [];
     const track = { machineId: m, alarmId, raisedAt: b.raisedAt, level: b.level, open: true };
     state.alarms.set(key, track);
-    return ack([...superseded, ...(0, lifecycle_1.notePause)(state, track, b), {
+    return ack([...superseded, ...(stopping ? (0, lifecycle_1.notePause)(state, track, b) : []), {
             kind: 'createAlarmRow', machineId: m, alarmId,
             row: {
                 type: contracts_1.ROW_TYPE.ALARM, subtype: b.level, role: contracts_1.LINE_ROLE.OPERATOR, priority: b.level === protocol_1.LEVEL.EMERGENCY ? 1 : 2,
@@ -254,27 +259,37 @@ function planPresence(state, m, b) {
     }
     return ack(actions);
 }
-/** A new boot id means the machine restarted and lost its queue: send its open dispatches again. */
+/**
+ * A new boot id means the machine restarted and lost its queue: send its open
+ * dispatches again, and the jobs it had ready for units it still holds, from
+ * the rows that sent them. A lost job nothing is remembered for is logged.
+ */
 function planSnapshot(state, m, b, ttl) {
     const link = state.link(m);
     if (b.software)
         link.software = b.software;
     const jobIds = Array.isArray(b.jobs) ? b.jobs.map((j) => j?.jobId).filter((id) => typeof id === 'string') : [];
+    // A snapshot without a boot id says nothing about restarts.
+    const rebooted = typeof b.bootId === 'string' && !!link.bootId && link.bootId !== b.bootId;
+    const held = new Set(jobIds);
+    const lost = rebooted ? state.lostAfterReboot(m, held) : { resend: [], unknown: [] };
     if (Array.isArray(b.jobs)) {
-        state.keepHeld(m, new Set(jobIds));
+        state.keepHeld(m, new Set([...jobIds, ...lost.resend.map((l) => l.jobId)]));
         link.jobs = b.jobs;
     }
+    if (typeof b.bootId === 'string')
+        link.bootId = b.bootId;
     const fresh = !state.retainedAlarms && !state.readingRetained && Array.isArray(b.alarms);
     const cleared = fresh ? alarmsNoLongerListed(state, m, b.alarms) : [];
     const answer = { kind: 'settleSnapshot', machineId: m, jobIds };
-    const previous = link.bootId;
-    link.bootId = b.bootId;
-    if (!previous || previous === b.bootId)
+    if (!rebooted)
         return ack([...cleared, answer]);
-    const held = new Set(jobIds);
     const missing = (r) => (0, jobs_1.dispatchJobs)(r.jobId, r.units, state.unitsPerDispatch).some((j) => !held.has(j.jobId));
-    const lost = state.open().filter((r) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && missing(r));
-    return ack([...cleared, answer, ...lost.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl))]);
+    const open = state.open().filter((r) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && missing(r));
+    const rows = new Map(lost.resend.map((l) => [l.row.id, l.row]));
+    const again = [...rows.values()].map((r) => ({ ...(0, pacing_1.dispatchAction)(r, m, ttl), ended: true }));
+    const unknown = lost.unknown.length ? [{ kind: 'unrecoverable', machineId: m, jobs: lost.unknown }] : [];
+    return ack([...cleared, answer, ...open.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl)), ...again, ...unknown]);
 }
 /** Alarm rows for `m` whose alarm its snapshot no longer lists: they cleared at the machine. */
 function alarmsNoLongerListed(state, m, listed) {

package/build/services/machines/gateway/plan-shared.d.ts

@@ -5,9 +5,11 @@ import { type GatewayState, type HandoffRow } from './state';
 /** A cancel for each job a handoff's units became at `machineId`, or for those `keep` leaves out. */
 export declare function cancelJobs(state: GatewayState, row: HandoffRow, machineId: string, jobId: string, tag?: string, keep?: (job: DispatchJob) => boolean): Action[];
 /**
- * Withdraw a dispatch that may still sit in the receiver's queue. A job some
- * of whose units already reached the receiver is under way: it stays, and
- * what happens to it is the workflow's next decision (a machine-command row).
+ * Withdraw a dispatch that may still sit in the receiver's queue: one sent or
+ * accepted. A job that never went out (the row waiting for room, or for its
+ * units to arrive) leaves the receiver nothing to withdraw. A job some of whose
+ * units already reached the receiver is under way: it stays, and what happens
+ * to it is the workflow's next decision (a machine-command row).
  */
 export declare function cancelDispatch(state: GatewayState, row: HandoffRow): Action[];
 /**

package/build/services/machines/gateway/plan-shared.js

@@ -14,13 +14,15 @@ function cancelJobs(state, row, machineId, jobId, tag = '', keep = () => false)
     }));
 }
 /**
- * Withdraw a dispatch that may still sit in the receiver's queue. A job some
- * of whose units already reached the receiver is under way: it stays, and
- * what happens to it is the workflow's next decision (a machine-command row).
+ * Withdraw a dispatch that may still sit in the receiver's queue: one sent or
+ * accepted. A job that never went out (the row waiting for room, or for its
+ * units to arrive) leaves the receiver nothing to withdraw. A job some of whose
+ * units already reached the receiver is under way: it stays, and what happens
+ * to it is the workflow's next decision (a machine-command row).
  */
 function cancelDispatch(state, row) {
     const to = row.toMachine;
-    if (!to || row.dispatch === state_1.DISPATCH_STATE.NONE || row.dispatch === state_1.DISPATCH_STATE.REFUSED)
+    if (!to || (row.dispatch !== state_1.DISPATCH_STATE.PENDING && row.dispatch !== state_1.DISPATCH_STATE.ACCEPTED))
         return [];
     return cancelJobs(state, row, to, row.jobId, '', (job) => job.units.some((u) => state.heldJob(to, u.unitId) === job.jobId));
 }

package/build/services/machines/gateway/reply-waiter.d.ts

@@ -10,6 +10,8 @@ export interface CommandReply {
     reason: string;
     /** The machine's structured code for the outcome, when its reply carried one. */
     code?: string;
+    /** An acceptance the machine will follow with `completed` or `failed` once the job is prepared. */
+    preparing?: boolean;
 }
 /** The jobs a machine reports holding, queued or in work. */
 export type HeldJobs = Set<string>;

package/build/services/machines/gateway/state.d.ts

@@ -35,6 +35,8 @@ export interface HandoffRow extends HandoffFacets {
     trace?: TraceEntry[];
     /** Each unit's arrival event, as its receiver reported it (dispatch on arrival). */
     arrivals?: Record<string, Record<string, any>>;
+    /** Dispatch commands accepted as preparing; on arrival dispatch the row is received once each is ready. */
+    awaitingReady?: Set<string>;
     /** Lifecycle item keys already on the row (lifecycle.ts). */
     noted?: string[];
     /** Parked with a limit on its items, which lifecycle notes must never use up. */
@@ -98,6 +100,10 @@ export declare class GatewayState {
     holdDispatch: boolean;
     /** Units at each machine that arrived and have not left: machine → unit → job. */
     private held;
+    /** Jobs sent and taken, by machine: job → the row that sent it, kept while the machine holds the job's units. */
+    private sentJobs;
+    /** Jobs whose units left each machine since its last snapshot: they ended, a snapshot without them lost nothing. */
+    private endedJobs;
     private openRows;
     private closedRows;
     readonly alarms: Map<string, AlarmTrack>;
@@ -122,6 +128,8 @@ export declare class GatewayState {
     private strangers;
     index(row: HandoffRow): void;
     get(id: string): HandoffRow | undefined;
+    /** A row that ended, while the gateway still remembers it. */
+    closedRow(id: string): HandoffRow | undefined;
     close(id: string, ended?: RowEnd): HandoffRow | undefined;
     isClosed(id: string): boolean;
     open(): HandoffRow[];
@@ -130,11 +138,26 @@ export declare class GatewayState {
     closedFor(unitId: string, pred: (r: HandoffRow) => boolean): HandoffRow | undefined;
     hold(machineId: string, unitId: string, jobId: string): void;
     letGo(machineId: string, unitId: string): void;
+    /** A job the machine took, remembered by its row while the machine holds the job's units. */
+    rememberSent(machineId: string, jobId: string, rowId: string): void;
+    /**
+     * After a restart, the jobs the machine lost: those its units are held under
+     * or its last snapshot listed, that the new snapshot lacks and no open row
+     * will send again. `resend` names the rows remembered for them; `unknown`
+     * the jobs nothing is remembered for.
+     */
+    lostAfterReboot(machineId: string, listed: Set<string>): {
+        resend: {
+            jobId: string;
+            row: HandoffRow;
+        }[];
+        unknown: string[];
+    };
     /** The job a unit arrived under at `machineId`, while it is still there. */
     heldJob(machineId: string, unitId: string): string | undefined;
     heldAt(machineId: string): string[];
     heldUnitsAt(machineId: string): HeldUnit[];
-    /** Keep only units whose job the machine still lists (its snapshot). */
+    /** Keep only units whose job the machine still lists (its snapshot); a job it no longer lists is over. */
     keepHeld(machineId: string, jobIds: Set<string>): void;
     /** Keep a sender's finished record until its row is known, so an arrival cannot close the row without it. */
     rememberFinish(machineId: string, unitId: string, body: Record<string, any>): void;