@hotmeshio/long-tail 0.27.0 → 0.28.1

package/build/services/machines/gateway/gateway.js

@@ -4,6 +4,7 @@
  * reads escalation events (a new handoff row means "dispatch"), reads every
  * machine message on the line (a receipt means "resolve"), and performs the
  * verbs a person would, as the machine. Neither side knows about the other.
+ * Every message read or sent is recorded on the wire (wire.ts).
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MachineGateway = void 0;
@@ -15,14 +16,19 @@ const address_book_1 = require("./address-book");
 const admission_1 = require("./admission");
 const protocol_1 = require("../../../lib/machines/protocol");
 const transport_1 = require("../../../lib/machines/transport");
+const publish_1 = require("../../../lib/events/publish");
 const reply_waiter_1 = require("./reply-waiter");
 const executor_1 = require("./executor");
+const dispatcher_1 = require("./dispatcher");
+const inspect_1 = require("./inspect");
 const log_1 = require("./log");
 const plan_escalation_1 = require("./plan-escalation");
 const pacing_1 = require("./pacing");
 const plan_machine_1 = require("./plan-machine");
 const state_1 = require("./state");
 const receipts_1 = require("./receipts");
+const wire_1 = require("./wire");
+const wire_window_1 = require("./wire-window");
 class MachineGateway {
     constructor(deps) {
         this.deps = deps;
@@ -32,18 +38,22 @@ class MachineGateway {
         this.codec = deps.codec ?? (0, protocol_1.genericCodec)({ root: deps.settings.topicRoot, line: deps.line });
         this.state.unitsPerDispatch = this.codec.unitsPerDispatch;
         this.state.dispatchOnArrival = this.codec.dispatchOn === protocol_1.DISPATCH_ON.ARRIVAL;
+        this.state.rowOps = (0, protocol_1.declaredRowOps)(this.codec);
         this.state.lifecycle = deps.settings.lifecycle === true;
         this.state.retainedAlarms = this.codec.retainedAlarms === true;
         this.state.progressStep = deps.settings.progressStep ?? this.state.progressStep;
         this.addresses = deps.addresses ?? new address_book_1.MemoryAddressBook();
+        this.window = new wire_window_1.WireWindow(deps.settings.wireWindow);
+        const sink = deps.wire ?? publish_1.publishMachineWireEvent;
+        this.wire = new wire_1.WireRecorder(deps.line, (event) => { this.window.push(event); sink(event); }, { telemetry: deps.settings.wireTelemetry === true });
         this.executor = new executor_1.Executor({
-            addresses: this.addresses,
+            addresses: this.addresses, wire: this.wire,
             bus: deps.bus, lt: deps.lt, replies: new reply_waiter_1.ReplyWaiter(), state: this.state, line: deps.line,
             codec: this.codec, replyTimeoutMs: deps.settings.replyTimeoutMs,
             dispatchAttempts: deps.settings.dispatchAttempts, offlineDebounceMs: deps.settings.offlineDebounceMs, log: this.log,
         });
-        this.admission = { state: this.state, line: deps.line, executor: this.executor, log: this.log, pace: () => this.pace() };
-        this.receipts = new receipts_1.ReceiptBatcher({ bus: deps.bus, codec: this.codec, delayMs: deps.settings.receiptDelayMs ?? DEFAULT_RECEIPT_DELAY_MS, log: this.log });
+        this.admission = { state: this.state, line: deps.line, executor: this.executor, log: this.log, pace: () => this.pace(), openRows: (q) => deps.lt.listOpen(q) };
+        this.receipts = new receipts_1.ReceiptBatcher({ bus: deps.bus, codec: this.codec, delayMs: deps.settings.receiptDelayMs ?? DEFAULT_RECEIPT_DELAY_MS, log: this.log, wire: this.wire });
     }
     async start() {
         await this.deps.bus.connect();
@@ -61,10 +71,17 @@ class MachineGateway {
         }
         this.log.info(log_1.GATEWAY_LOG.STARTED, { site: this.deps.line.siteId, line: this.deps.line.lineId });
     }
+    /**
+     * Hand the line over cleanly: nothing new is raised or sent, a message
+     * still being handled is nak'd for the next leader if it would make a row,
+     * receipts for what was applied go out, then the sessions close.
+     */
     async stop() {
         if (this.quietTimer)
             clearInterval(this.quietTimer);
         this.executor.stop();
+        await this.deps.feed.unsubscribe();
+        await this.deps.bus.unsubscribe();
         await this.executor.idle();
         await this.receipts.flush();
         await this.deps.feed.close();
@@ -81,6 +98,10 @@ class MachineGateway {
     idle() {
         return this.executor.idle();
     }
+    /** A dry run of one message on a copy of the state: nothing is applied (inspect.ts). */
+    inspect(msg) {
+        return (0, inspect_1.dryRun)({ state: this.state, line: this.deps.line, codec: this.codec, dispatchTtlMs: this.deps.settings.dispatchTtlMs }, { ...msg, deliveries: 1 });
+    }
     /**
      * On taking the line: open handoff and command rows (a dispatch or command
      * interrupted by a restart is sent again with the same cmdId), the gateway's
@@ -138,16 +159,21 @@ class MachineGateway {
         const active = new Set();
         this.state.readingRetained = true;
         try {
-            for (const msg of await this.deps.bus.retained((0, protocol_1.lineFilter)(this.codec.root, line.siteId, line.lineId))) {
-                for (const raw of this.decode(msg.topic, msg.payload, true)) {
+            for (const read of await this.deps.bus.retained((0, protocol_1.lineFilter)(this.codec.root, line.siteId, line.lineId))) {
+                const msg = { ...read, retained: true };
+                for (const raw of this.decode(msg)) {
                     if (!raw.parsed)
                         continue;
-                    const seen = await (0, admission_1.admitSender)(this.admission, msg.topic, raw);
-                    if (!('parsed' in seen) || !seen.parsed)
+                    const admitted = await (0, admission_1.admitSender)(this.admission, msg.topic, raw);
+                    if (!('seen' in admitted)) {
+                        this.wire.inbound(msg, raw, admitted, 1, admitted.sender);
                         continue;
+                    }
+                    const { seen } = admitted;
                     if (seen.parsed.kind === protocol_1.MESSAGE_KIND.ALARM && seen.body?.body?.active)
                         active.add(this.state.alarmKey(seen.parsed.ident.machineId, seen.parsed.key));
-                    await this.applyMachine(msg.topic, seen.parsed, seen.body);
+                    const applied = await this.applyMachine(msg.topic, seen.parsed, seen.body);
+                    this.wire.inbound(msg, seen, { ...applied, actions: [...admitted.actions, ...applied.actions] }, 1);
                 }
             }
         }
@@ -170,7 +196,8 @@ class MachineGateway {
             return transport_1.DISPOSITION.ACK;
         }
         catch (err) {
-            this.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { event: event.type, error: err.message });
+            if (!(err instanceof dispatcher_1.GatewayStopped))
+                this.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { event: event.type, error: err.message });
             return transport_1.DISPOSITION.NAK;
         }
     }
@@ -188,31 +215,39 @@ class MachineGateway {
         return { ...event, role: row.role, data: row };
     }
     async onMachine(msg) {
-        const decoded = this.decode(msg.topic, msg.payload, msg.retained);
+        const decoded = this.decode(msg);
         const acted = [];
         let outcome = transport_1.DISPOSITION.TERM;
         let failed = false;
         for (const raw of decoded) {
             if (!raw.parsed)
                 continue;
-            const seen = await (0, admission_1.admitSender)(this.admission, msg.topic, raw);
-            if (!('parsed' in seen)) {
-                failed ||= seen.failed;
-                outcome = worse(outcome, seen.disposition);
+            const admitted = await (0, admission_1.admitSender)(this.admission, msg.topic, raw);
+            if (!('seen' in admitted)) {
+                failed ||= admitted.failed;
+                outcome = worse(outcome, admitted.disposition);
+                this.wire.inbound(msg, raw, admitted, msg.deliveries, admitted.sender);
                 continue;
             }
+            const { seen } = admitted;
             acted.push(seen);
             if (seen.violations.length && msg.deliveries === 1) {
                 this.log.warn(log_1.GATEWAY_LOG.CONTRACT_VIOLATION, { topic: msg.topic, violations: seen.violations });
             }
-            const applied = seen.violations.length && this.codec.strict
+            let applied = seen.violations.length && this.codec.strict
                 ? await (0, admission_1.holdViolation)(this.admission, msg, seen)
                 : await this.applyMachine(msg.topic, seen.parsed, seen.body);
+            // An arrival still waiting for its row after every delivery is shown to a person instead; one that read no id at once.
+            const waited = msg.deliveries >= wire_1.MAX_DELIVERIES || (0, admission_1.isUnreadableArrival)(seen);
+            if (applied.disposition === transport_1.DISPOSITION.NAK && !applied.failed && waited && (0, admission_1.isArrival)(seen)) {
+                applied = await (0, admission_1.holdArrival)(this.admission, msg, seen);
+            }
             failed ||= applied.failed;
             outcome = worse(outcome, applied.disposition);
+            this.wire.inbound(msg, seen, { ...applied, actions: [...admitted.actions, ...applied.actions] }, msg.deliveries);
         }
         // No row has claimed it after this many tries, so it is not racing its row: stop. A failed verb always retries.
-        if (outcome === transport_1.DISPOSITION.NAK && !failed && msg.deliveries >= MAX_DELIVERIES) {
+        if (outcome === transport_1.DISPOSITION.NAK && !failed && msg.deliveries >= wire_1.MAX_DELIVERIES) {
             this.log.warn(log_1.GATEWAY_LOG.UNMATCHED, { topic: msg.topic, deliveries: msg.deliveries });
             outcome = transport_1.DISPOSITION.TERM;
         }
@@ -221,14 +256,19 @@ class MachineGateway {
         return outcome;
     }
     /** A message the codec cannot read is not retried: it is logged and has no effect. */
-    decode(topic, payload, retained) {
+    decode(msg) {
+        let decoded;
         try {
-            return this.codec.decode(topic, payload, retained);
+            decoded = this.codec.decode(msg.topic, msg.payload, msg.retained);
         }
         catch (err) {
-            this.log.warn(log_1.GATEWAY_LOG.UNREADABLE, { topic, error: err.message });
+            this.log.warn(log_1.GATEWAY_LOG.UNREADABLE, { topic: msg.topic, error: err.message });
+            this.wire.unreadable(msg, err.message);
             return [];
         }
+        if (!decoded.length)
+            this.wire.unreadable(msg, 'not a machine message', 'none');
+        return decoded;
     }
     /** Done with the message: confirm each id its codec asked to confirm. */
     confirm(decoded) {
@@ -247,21 +287,21 @@ class MachineGateway {
         }
         catch (err) {
             this.log.warn(log_1.GATEWAY_LOG.UNREADABLE, { topic, error: err.message });
-            return { disposition: transport_1.DISPOSITION.TERM, failed: false };
+            return { disposition: transport_1.DISPOSITION.TERM, failed: false, actions: [{ verb: 'unreadable', detail: err.message }] };
         }
         try {
-            await this.executor.apply(plan.actions);
+            const actions = await this.executor.apply(plan.actions);
             await this.pace();
-            return { disposition: plan.disposition, failed: false };
+            return { disposition: plan.disposition, failed: false, actions };
         }
         catch (err) {
-            this.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { topic, error: err.message });
-            return { disposition: transport_1.DISPOSITION.NAK, failed: true };
+            if (!(err instanceof dispatcher_1.GatewayStopped))
+                this.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { topic, error: err.message });
+            return { disposition: transport_1.DISPOSITION.NAK, failed: true, actions: [] };
         }
     }
 }
 exports.MachineGateway = MachineGateway;
-const MAX_DELIVERIES = 30;
 const DEFAULT_RECEIPT_DELAY_MS = 250;
 const RANK = { [transport_1.DISPOSITION.TERM]: 0, [transport_1.DISPOSITION.ACK]: 1, [transport_1.DISPOSITION.NAK]: 2 };
 /** One wire message may decode to several; any that must be retried retries the message. */

package/build/services/machines/gateway/index.d.ts

@@ -8,3 +8,6 @@ export * from './provision';
 export * from './state';
 export * from './address-book';
 export * from './registration';
+export * from './wire';
+export * from './project';
+export * from './inspect';

package/build/services/machines/gateway/index.js

@@ -24,3 +24,6 @@ __exportStar(require("./provision"), exports);
 __exportStar(require("./state"), exports);
 __exportStar(require("./address-book"), exports);
 __exportStar(require("./registration"), exports);
+__exportStar(require("./wire"), exports);
+__exportStar(require("./project"), exports);
+__exportStar(require("./inspect"), exports);

package/build/services/machines/gateway/inspect.d.ts

@@ -0,0 +1,17 @@
+/**
+ * A dry run of one wire message: what the codec reads and what the gateway
+ * would do, planned on a copy of the state so the line is untouched and no
+ * row is created or changed.
+ */
+import type { LineDef } from '../../../lib/machines/line';
+import type { LineCodec } from '../../../lib/machines/protocol';
+import type { InboundMessage } from '../../../lib/machines/transport';
+import type { InspectResult } from '../../../types/machines-view';
+import type { GatewayState } from './state';
+export interface DryRunContext {
+    state: GatewayState;
+    line: LineDef;
+    codec: LineCodec;
+    dispatchTtlMs: number;
+}
+export declare function dryRun(ctx: DryRunContext, msg: InboundMessage): InspectResult;

package/build/services/machines/gateway/inspect.js

@@ -0,0 +1,68 @@
+"use strict";
+/**
+ * A dry run of one wire message: what the codec reads and what the gateway
+ * would do, planned on a copy of the state so the line is untouched and no
+ * row is created or changed.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dryRun = dryRun;
+const plan_machine_1 = require("./plan-machine");
+const registration_1 = require("./registration");
+const wire_1 = require("./wire");
+/** What admitting the sender would do, or null when the message would be acted on. */
+function admission(state, line, raw) {
+    const sender = (0, registration_1.resolveSender)(state, line, raw);
+    if (sender.status === registration_1.SENDER.BOUND)
+        return { seen: sender.seen, actions: [{ verb: 'ack', detail: `bind ${sender.machineId} to ${sender.address}` }] };
+    if (sender.seen)
+        return { seen: sender.seen, actions: [] };
+    if (sender.status === registration_1.SENDER.UNKNOWN)
+        return { seen: null, actions: [{ verb: 'none', detail: `${sender.address} claims no station on this line` }] };
+    const raised = state.claimRaised(sender.address, sender.machineId);
+    return {
+        seen: null,
+        actions: [raised
+                ? { verb: 'none', detail: `${sender.address} says it is ${sender.machineId}; waiting for a person` }
+                : { verb: 'create-row', detail: `${sender.address} says it is ${sender.machineId}; a person decides` }],
+    };
+}
+function planOne(ctx, state, raw) {
+    const admitted = admission(state, ctx.line, raw);
+    if (!admitted.seen)
+        return admitted.actions;
+    if (raw.violations.length && ctx.codec.strict)
+        return [...admitted.actions, { verb: 'hold', detail: raw.violations.join('; ') }];
+    try {
+        const plan = (0, plan_machine_1.planMachineMessage)(state, admitted.seen.parsed, admitted.seen.body, ctx.dispatchTtlMs);
+        const actions = [...admitted.actions, ...plan.actions.map(wire_1.plannedAction)];
+        return (0, wire_1.appliedActions)({ disposition: plan.disposition, failed: false, actions }, 1);
+    }
+    catch (err) {
+        return [...admitted.actions, { verb: 'unreadable', detail: err.message }];
+    }
+}
+function dryRun(ctx, msg) {
+    let decoded;
+    try {
+        decoded = ctx.codec.decode(msg.topic, msg.payload, msg.retained);
+    }
+    catch (err) {
+        return { kind: null, event: null, summary: err.message, inspections: [], violations: [], actions: [{ verb: 'unreadable', detail: err.message }] };
+    }
+    if (!decoded.length) {
+        return { kind: null, event: null, summary: 'not a machine message', inspections: [], violations: [], actions: [{ verb: 'none', detail: 'not a machine message' }] };
+    }
+    const state = ctx.state.clone();
+    const actions = [];
+    const violations = [];
+    for (const raw of decoded) {
+        violations.push(...raw.violations);
+        if (raw.parsed)
+            actions.push(...planOne(ctx, state, raw));
+    }
+    const [first] = decoded;
+    return {
+        kind: first.parsed?.kind ?? null, event: (0, wire_1.eventOf)(first), summary: (0, wire_1.summarize)(first),
+        inspections: decoded, violations, actions: actions.length ? actions : [{ verb: 'unreadable', detail: 'unknown topic' }],
+    };
+}

package/build/services/machines/gateway/log.d.ts

@@ -1,11 +1,14 @@
+import { publishMachineLogEvent } from '../../../lib/events/publish';
 export declare const GATEWAY_LOG: {
     readonly STARTED: "machine_gateway.started";
     readonly RECONCILED: "machine_gateway.reconciled";
     readonly DISPATCHED: "machine_gateway.dispatch.accepted";
     readonly DISPATCH_REFUSED: "machine_gateway.dispatch.refused";
     readonly DISPATCH_READY: "machine_gateway.dispatch.ready";
+    readonly DISPATCH_PREPARING: "machine_gateway.dispatch.preparing";
     readonly DISPATCH_FAILED: "machine_gateway.dispatch.failed";
     readonly DISPATCH_RECONCILED: "machine_gateway.dispatch.reconciled";
+    readonly DISPATCH_UNRECOVERABLE: "machine_gateway.dispatch.unrecoverable";
     readonly VERB_INVALID: "machine_gateway.verb.invalid";
     readonly VERB_FAILED: "machine_gateway.verb.failed";
     readonly ROW_NOT_VISIBLE: "machine_gateway.verb.row_not_visible";
@@ -39,3 +42,10 @@ export interface GatewayLog {
 }
 export declare const consoleLog: GatewayLog;
 export declare const silentLog: GatewayLog;
+/** Each entry published as `machine.log.<site>.<line>.<event>` with its context, for the line's dashboard view. */
+export declare function eventsLog(line: {
+    siteId: string;
+    lineId: string;
+}, publish?: typeof publishMachineLogEvent): GatewayLog;
+/** Every entry goes to each log in turn. */
+export declare function composeLogs(...logs: GatewayLog[]): GatewayLog;

package/build/services/machines/gateway/log.js

@@ -1,14 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.silentLog = exports.consoleLog = exports.GATEWAY_LOG = void 0;
+exports.eventsLog = eventsLog;
+exports.composeLogs = composeLogs;
+const publish_1 = require("../../../lib/events/publish");
 exports.GATEWAY_LOG = {
     STARTED: 'machine_gateway.started',
     RECONCILED: 'machine_gateway.reconciled',
     DISPATCHED: 'machine_gateway.dispatch.accepted',
     DISPATCH_REFUSED: 'machine_gateway.dispatch.refused',
     DISPATCH_READY: 'machine_gateway.dispatch.ready',
+    DISPATCH_PREPARING: 'machine_gateway.dispatch.preparing',
     DISPATCH_FAILED: 'machine_gateway.dispatch.failed',
     DISPATCH_RECONCILED: 'machine_gateway.dispatch.reconciled',
+    DISPATCH_UNRECOVERABLE: 'machine_gateway.dispatch.unrecoverable',
     VERB_INVALID: 'machine_gateway.verb.invalid',
     VERB_FAILED: 'machine_gateway.verb.failed',
     ROW_NOT_VISIBLE: 'machine_gateway.verb.row_not_visible',
@@ -40,3 +45,18 @@ exports.consoleLog = {
     error: (event, context) => console.error(JSON.stringify({ level: 'error', event, ...context })),
 };
 exports.silentLog = { info: () => undefined, warn: () => undefined, error: () => undefined };
+/** Each entry published as `machine.log.<site>.<line>.<event>` with its context, for the line's dashboard view. */
+function eventsLog(line, publish = publish_1.publishMachineLogEvent) {
+    const emit = (level) => (event, context) => {
+        publish({ siteId: line.siteId, lineId: line.lineId, level, event, context }).catch(() => undefined);
+    };
+    return { info: emit('info'), warn: emit('warn'), error: emit('error') };
+}
+/** Every entry goes to each log in turn. */
+function composeLogs(...logs) {
+    const each = (level) => (event, context) => {
+        for (const log of logs)
+            log[level](event, context);
+    };
+    return { info: each('info'), warn: each('warn'), error: each('error') };
+}

package/build/services/machines/gateway/lt-http.js

@@ -56,6 +56,8 @@ class LtHttp {
     }
 }
 exports.LtHttp = LtHttp;
+/** A row that was never given items (one parked by hand) answers 400; it holds none. */
+const NO_ITEMS_STATUS = 400;
 /** Escalation verbs over HTTP, each run with the bearer token of the actor it runs as. */
 class HttpLineEscalations {
     constructor(http, tokenFor, claimMinutes = 24 * 60) {
@@ -80,8 +82,15 @@ class HttpLineEscalations {
         return (await this.items(id)).map((i) => i.itemKey);
     }
     async items(id) {
-        const view = await this.http.request('GET', `/escalations/${id}/items`, this.tokenFor(actions_1.CONTROLLER));
-        return view.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));
+        try {
+            const view = await this.http.request('GET', `/escalations/${id}/items`, this.tokenFor(actions_1.CONTROLLER));
+            return view.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));
+        }
+        catch (err) {
+            if (err instanceof LtHttpError && err.status === NO_ITEMS_STATUS)
+                return [];
+            throw err;
+        }
     }
     async get(id) {
         if (!(0, uuid_1.isUuid)(id))

package/build/services/machines/gateway/pacing.js

@@ -39,6 +39,7 @@ function dispatchArgs(row, expiresAt) {
         expiresAt: (0, protocol_1.isoTs)(expiresAt),
         ...(row.artifactUrl ? { artifact: { url: row.artifactUrl, sha256: row.artifactSha256 } } : {}),
         ...(row.parameters ? { parameters: row.parameters } : {}),
+        ...(row.refs ? { refs: row.refs } : {}),
     };
 }
 /**
@@ -65,7 +66,7 @@ function dispatchAction(row, machineId, dispatchTtlMs) {
     row.createdAt ??= now;
     const { round, expiresAt } = dispatchWindow(row.dueAt ?? row.createdAt, now, dispatchTtlMs);
     return {
-        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt),
+        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt), issuedAt: expiresAt - dispatchTtlMs,
         ...(row.dispatch === state_1.DISPATCH_STATE.ARRIVING ? { receivedOnAccept: true } : {}),
     };
 }

package/build/services/machines/gateway/plan-escalation.js

@@ -93,8 +93,9 @@ function commandRefusal(state, line, c, role) {
     const own = (0, line_1.machineRole)((0, line_1.machineById)(line, c.machineId));
     if (role !== own)
         return `commands for ${c.machineId} are parked on ${own}, not ${role}`;
-    if (!protocol_1.CONTROL_OPS.includes(c.op))
-        return `${c.op} is not a command a row may ask for; one of ${protocol_1.CONTROL_OPS.join(', ')}`;
+    const allowed = rowOps(state);
+    if (!allowed.includes(c.op))
+        return `${c.op} is not a command a row may ask for; one of ${allowed.join(', ')}`;
     if (c.op === protocol_1.OP.CANCEL && typeof c.args.jobId !== 'string')
         return `${protocol_1.OP.CANCEL} needs args.jobId, the job to withdraw or stop`;
     if (!(0, registration_1.reachable)(state, line, c.machineId))
@@ -110,7 +111,7 @@ function planCommandRow(state, line, event) {
     if (event.verb !== VERB.CREATED && event.verb !== VERB.CLAIMED)
         return [];
     const c = (0, commands_1.readCommandFacets)(event.data?.metadata);
-    if (!c || !state.trackCommand(id))
+    if (!c || !state.trackCommand(id, c.machineId))
         return [];
     const refusal = commandRefusal(state, line, c, event.role);
     if (refusal) {
@@ -119,6 +120,10 @@ function planCommandRow(state, line, event) {
     const createdAt = event.data?.created_at ? new Date(event.data.created_at).getTime() : NaN;
     return [{ kind: 'control', rowId: id, machineId: c.machineId, op: c.op, args: c.args, ...(Number.isFinite(createdAt) ? { createdAt } : {}) }];
 }
+/** The ops a machine-command row may ask for: the control ops, a snapshot, and the codec's own. */
+function rowOps(state) {
+    return [...protocol_1.CONTROL_OPS, protocol_1.OP.SNAPSHOT, ...state.rowOps];
+}
 function onLine(line, machineId) {
     return !!machineId && line.machines.some((m) => m.machineId === machineId);
 }

package/build/services/machines/gateway/plan-machine.js

@@ -36,7 +36,10 @@ function planMachineMessage(state, parsed, env, dispatchTtlMs) {
 function planByKind(state, m, parsed, env, b, ttl) {
     switch (parsed.kind) {
         case protocol_1.MESSAGE_KIND.REPLY:
-            return ack([{ kind: 'settleReply', cmdId: b.cmdId, status: b.status, reason: b.reason }]);
+            return ack([{
+                    kind: 'settleReply', cmdId: b.cmdId, status: b.status, reason: b.reason,
+                    ...(typeof b.code === 'string' ? { code: b.code } : {}), ...(b.preparing === true ? { preparing: true } : {}),
+                }]);
         case protocol_1.MESSAGE_KIND.JOB:
             return planJob(state, m, b, env?.at, ttl);
         case protocol_1.MESSAGE_KIND.ALARM:
@@ -139,7 +142,8 @@ function planJob(state, m, b, at, ttl) {
         if (b.result === protocol_1.JOB_RESULT.OK)
             return ack([released]);
         return ack([released, ...(0, plan_shared_1.failHandoff)(state, row, m, {
-                received: false, unitId, errorCode: contracts_1.HANDOFF_ERROR.SENDER_REJECTED, reason: b.reason ?? null, detail: `${m} finished ${unitId}: ${b.result}${b.reason ? ` (${b.reason})` : ''}`,
+                received: false, unitId, errorCode: contracts_1.HANDOFF_ERROR.SENDER_REJECTED, result: b.result, reason: b.reason ?? null,
+                detail: `${m} finished ${unitId}: ${b.result}${b.reason ? ` (${b.reason})` : ''}`,
             })]);
     }
     if (b.event === protocol_1.JOB_EVENT.TRANSFER_FAILED) {
@@ -190,7 +194,9 @@ function planAlarm(state, parsed, alarmId, env) {
                 payload: { decision: contracts_1.DECISION.CLEARED, clearedBy: b.clearedBy ?? null, note: `cleared at ${m}${b.clearedBy ? ` by ${b.clearedBy}` : ''}` },
             }]);
     }
-    if (!protocol_1.STOPPING_LEVELS.includes(b.level))
+    // A stopping level needs a person; so does any alarm the machine flags, at the level it said.
+    const stopping = protocol_1.STOPPING_LEVELS.includes(b.level);
+    if (!stopping && b.needsPerson !== true)
         return ack();
     const known = state.alarms.get(key);
     if (known && (!known.raisedAt || !b.raisedAt || known.raisedAt === b.raisedAt))
@@ -198,9 +204,9 @@ function planAlarm(state, parsed, alarmId, env) {
     const superseded = known?.rowId && known.open
         ? [{ kind: 'resolve', as: actions_1.CONTROLLER, rowId: known.rowId, payload: { decision: contracts_1.DECISION.CLEARED, note: `${m} raised ${alarmId} again; this one ended` } }]
         : [];
-    const track = { machineId: m, alarmId, raisedAt: b.raisedAt, open: true };
+    const track = { machineId: m, alarmId, raisedAt: b.raisedAt, level: b.level, open: true };
     state.alarms.set(key, track);
-    return ack([...superseded, ...(0, lifecycle_1.notePause)(state, track, b), {
+    return ack([...superseded, ...(stopping ? (0, lifecycle_1.notePause)(state, track, b) : []), {
             kind: 'createAlarmRow', machineId: m, alarmId,
             row: {
                 type: contracts_1.ROW_TYPE.ALARM, subtype: b.level, role: contracts_1.LINE_ROLE.OPERATOR, priority: b.level === protocol_1.LEVEL.EMERGENCY ? 1 : 2,
@@ -222,10 +228,12 @@ function catalogEnvelope(catalog) {
 }
 function planPresence(state, m, b) {
     const link = state.link(m);
+    const session = typeof b.sessionId === 'string' ? b.sessionId : undefined;
     if (b.online !== true) {
         if (link.online)
             link.since = Date.now();
         link.online = false;
+        link.reason = typeof b.reason === 'string' ? b.reason : undefined;
         if (link.rowId || link.watching)
             return ack();
         link.watching = true;
@@ -234,37 +242,54 @@ function planPresence(state, m, b) {
     if (b.software)
         link.software = b.software;
     const wasOffline = !link.online;
+    // A new session while online: the machine reconnected before its offline presence was heard.
+    const reconnected = !!session && !!link.sessionId && session !== link.sessionId;
     link.online = true;
     link.watching = false;
+    link.reason = undefined;
+    if (session)
+        link.sessionId = session;
     const actions = [];
     // Back from a drop: ask for the full state rather than wait for the next heartbeat.
-    if (wasOffline)
-        actions.push({ kind: 'command', machineId: m, op: protocol_1.OP.SNAPSHOT, cmdId: `${m}:resync:${link.since}`, args: {} });
+    if (wasOffline || reconnected)
+        actions.push({ kind: 'command', machineId: m, op: protocol_1.OP.SNAPSHOT, cmdId: `${m}:resync:${session ?? link.since}`, args: {} });
     if (link.rowId) {
         actions.push({ kind: 'resolve', as: actions_1.CONTROLLER, rowId: link.rowId, payload: { decision: contracts_1.DECISION.RESTORED, note: `${m} back online` } });
         link.rowId = undefined;
     }
     return ack(actions);
 }
-/** A new boot id means the machine restarted and lost its queue: send its open dispatches again. */
+/**
+ * A new boot id means the machine restarted and lost its queue: send its open
+ * dispatches again, and the jobs it had ready for units it still holds, from
+ * the rows that sent them. A lost job nothing is remembered for is logged.
+ */
 function planSnapshot(state, m, b, ttl) {
     const link = state.link(m);
     if (b.software)
         link.software = b.software;
     const jobIds = Array.isArray(b.jobs) ? b.jobs.map((j) => j?.jobId).filter((id) => typeof id === 'string') : [];
-    if (Array.isArray(b.jobs))
-        state.keepHeld(m, new Set(jobIds));
+    // A snapshot without a boot id says nothing about restarts.
+    const rebooted = typeof b.bootId === 'string' && !!link.bootId && link.bootId !== b.bootId;
+    const held = new Set(jobIds);
+    const lost = rebooted ? state.lostAfterReboot(m, held) : { resend: [], unknown: [] };
+    if (Array.isArray(b.jobs)) {
+        state.keepHeld(m, new Set([...jobIds, ...lost.resend.map((l) => l.jobId)]));
+        link.jobs = b.jobs;
+    }
+    if (typeof b.bootId === 'string')
+        link.bootId = b.bootId;
     const fresh = !state.retainedAlarms && !state.readingRetained && Array.isArray(b.alarms);
     const cleared = fresh ? alarmsNoLongerListed(state, m, b.alarms) : [];
     const answer = { kind: 'settleSnapshot', machineId: m, jobIds };
-    const previous = link.bootId;
-    link.bootId = b.bootId;
-    if (!previous || previous === b.bootId)
+    if (!rebooted)
         return ack([...cleared, answer]);
-    const held = new Set(jobIds);
     const missing = (r) => (0, jobs_1.dispatchJobs)(r.jobId, r.units, state.unitsPerDispatch).some((j) => !held.has(j.jobId));
-    const lost = state.open().filter((r) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && missing(r));
-    return ack([...cleared, answer, ...lost.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl))]);
+    const open = state.open().filter((r) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && missing(r));
+    const rows = new Map(lost.resend.map((l) => [l.row.id, l.row]));
+    const again = [...rows.values()].map((r) => ({ ...(0, pacing_1.dispatchAction)(r, m, ttl), ended: true }));
+    const unknown = lost.unknown.length ? [{ kind: 'unrecoverable', machineId: m, jobs: lost.unknown }] : [];
+    return ack([...cleared, answer, ...open.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl)), ...again, ...unknown]);
 }
 /** Alarm rows for `m` whose alarm its snapshot no longer lists: they cleared at the machine. */
 function alarmsNoLongerListed(state, m, listed) {

package/build/services/machines/gateway/plan-shared.d.ts

@@ -5,9 +5,11 @@ import { type GatewayState, type HandoffRow } from './state';
 /** A cancel for each job a handoff's units became at `machineId`, or for those `keep` leaves out. */
 export declare function cancelJobs(state: GatewayState, row: HandoffRow, machineId: string, jobId: string, tag?: string, keep?: (job: DispatchJob) => boolean): Action[];
 /**
- * Withdraw a dispatch that may still sit in the receiver's queue. A job some
- * of whose units already reached the receiver is under way: it stays, and
- * what happens to it is the workflow's next decision (a machine-command row).
+ * Withdraw a dispatch that may still sit in the receiver's queue: one sent or
+ * accepted. A job that never went out (the row waiting for room, or for its
+ * units to arrive) leaves the receiver nothing to withdraw. A job some of whose
+ * units already reached the receiver is under way: it stays, and what happens
+ * to it is the workflow's next decision (a machine-command row).
  */
 export declare function cancelDispatch(state: GatewayState, row: HandoffRow): Action[];
 /**

package/build/services/machines/gateway/plan-shared.js

@@ -14,13 +14,15 @@ function cancelJobs(state, row, machineId, jobId, tag = '', keep = () => false)
     }));
 }
 /**
- * Withdraw a dispatch that may still sit in the receiver's queue. A job some
- * of whose units already reached the receiver is under way: it stays, and
- * what happens to it is the workflow's next decision (a machine-command row).
+ * Withdraw a dispatch that may still sit in the receiver's queue: one sent or
+ * accepted. A job that never went out (the row waiting for room, or for its
+ * units to arrive) leaves the receiver nothing to withdraw. A job some of whose
+ * units already reached the receiver is under way: it stays, and what happens
+ * to it is the workflow's next decision (a machine-command row).
  */
 function cancelDispatch(state, row) {
     const to = row.toMachine;
-    if (!to || row.dispatch === state_1.DISPATCH_STATE.NONE || row.dispatch === state_1.DISPATCH_STATE.REFUSED)
+    if (!to || (row.dispatch !== state_1.DISPATCH_STATE.PENDING && row.dispatch !== state_1.DISPATCH_STATE.ACCEPTED))
         return [];
     return cancelJobs(state, row, to, row.jobId, '', (job) => job.units.some((u) => state.heldJob(to, u.unitId) === job.jobId));
 }

package/build/services/machines/gateway/project.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The line as its leader sees it, for the dashboard: each machine's link,
+ * held units, open rows and alarms from the gateway state, and the claims
+ * waiting for a person. Pure: reads the state and the line, changes nothing.
+ */
+import { type LineDef } from '../../../lib/machines/line';
+import type { LineView } from '../../../types/machines-view';
+import type { GatewayState } from './state';
+export declare function projectLine(state: GatewayState, line: LineDef): LineView;