@hotmeshio/long-tail 0.26.0 → 0.28.0

package/build/services/machines/config.js

@@ -4,11 +4,14 @@
  * an unknown or incomplete selection throws; nothing falls back silently.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ESCALATION_FEED = exports.MACHINE_TRANSPORT = void 0;
+exports.DEFAULT_TOPIC_ROOT = exports.ESCALATION_FEED = exports.MACHINE_TRANSPORT = void 0;
 exports.loadMachineLineConfig = loadMachineLineConfig;
 const protocol_1 = require("../../lib/machines/protocol");
 exports.MACHINE_TRANSPORT = { MEMORY: 'memory', NATS: 'nats', MQTT: 'mqtt' };
 exports.ESCALATION_FEED = { MEMORY: 'memory', CORE: 'core', IN_PROCESS: 'in-process' };
+/** The generic line protocol's topic root when MACHINE_TOPIC_ROOT is unset. */
+exports.DEFAULT_TOPIC_ROOT = 'line/v1';
+const SWITCH = { ON: 'on', OFF: 'off' };
 function oneOf(name, value, allowed) {
     if (!allowed.includes(value)) {
         throw new Error(`${name}="${value}" is not one of ${allowed.join(', ')}`);
@@ -24,6 +27,12 @@ function int(env, name, fallback) {
         throw new Error(`${name}="${raw}" must be a non-negative integer`);
     return n;
 }
+function percentStep(env, name, fallback) {
+    const n = int(env, name, fallback);
+    if (n > 100)
+        throw new Error(`${name}="${n}" must be at most 100`);
+    return n;
+}
 function required(env, name, mode) {
     const value = env[name];
     if (!value)
@@ -55,19 +64,22 @@ function loadMachineLineConfig(env = process.env) {
     const config = {
         transport,
         feed,
-        topicRoot: env.MACHINE_TOPIC_ROOT ?? 'line/v1',
+        topicRoot: env.MACHINE_TOPIC_ROOT ?? exports.DEFAULT_TOPIC_ROOT,
         lineConfigPath: env.MACHINE_LINE_CONFIG || undefined,
         replyTimeoutMs: int(env, 'MACHINE_REPLY_TIMEOUT_MS', protocol_1.REPLY_DEADLINE_MS),
         dispatchAttempts: Math.max(1, int(env, 'MACHINE_DISPATCH_ATTEMPTS', 3)),
-        handoffTimeout: env.MACHINE_HANDOFF_TIMEOUT ?? '10m',
         offlineDebounceMs: int(env, 'MACHINE_OFFLINE_DEBOUNCE_MS', 15_000),
         quietMs: int(env, 'MACHINE_QUIET_MS', 90_000),
         dispatchTtlMs: int(env, 'MACHINE_DISPATCH_TTL_MS', 600_000),
         receiptDelayMs: int(env, 'MACHINE_RECEIPT_DELAY_MS', 250),
+        lifecycle: oneOf('MACHINE_LIFECYCLE', env.MACHINE_LIFECYCLE || SWITCH.OFF, Object.values(SWITCH)) === SWITCH.ON,
+        progressStep: percentStep(env, 'MACHINE_PROGRESS_STEP', 25),
+        wireTelemetry: oneOf('MACHINE_WIRE_TELEMETRY', env.MACHINE_WIRE_TELEMETRY || SWITCH.OFF, Object.values(SWITCH)) === SWITCH.ON,
         ltBaseUrl: env.LT_BASE_URL ?? 'http://localhost:3000',
     };
     if (transport === exports.MACHINE_TRANSPORT.NATS) {
-        config.siteNatsUrl = required(env, 'MACHINE_SITE_NATS_URL', transport);
+        // A site NATS of its own; without one, the streams are Long Tail's messaging.
+        config.siteNatsUrl = env.MACHINE_SITE_NATS_URL || undefined;
         config.siteNatsUser = env.MACHINE_SITE_NATS_USER;
         config.siteNatsPassword = env.MACHINE_SITE_NATS_PASSWORD;
         config.mqtt = mqttEndpoint(env, transport, 5);
@@ -75,10 +87,8 @@ function loadMachineLineConfig(env = process.env) {
     if (transport === exports.MACHINE_TRANSPORT.MQTT)
         config.mqtt = mqttEndpoint(env, transport, 5);
     if (feed === exports.ESCALATION_FEED.CORE) {
-        config.ltNatsUrl = env.NATS_URL;
+        config.ltNatsUrl = env.NATS_URL || undefined;
         config.ltNatsToken = env.NATS_TOKEN;
-        if (!config.ltNatsUrl)
-            throw new Error('MACHINE_ESCALATION_FEED=core needs NATS_URL (Long Tail NATS)');
     }
     return config;
 }

package/build/services/machines/contracts.d.ts

@@ -29,14 +29,22 @@ export declare const LINE_ROLE: {
 export interface ArtifactRef {
     url: string;
     sha256: string;
+    /** The host's id for the file. */
+    id?: string;
+    contentType?: string;
+    sizeBytes?: number;
 }
 export interface HandoffUnit {
     unitId: string;
     carrierId: string | null;
     /** This unit's own file, when each unit has one (a per-unit model or toolpath). */
     artifact?: ArtifactRef;
+    /** This unit's files, when it has several. */
+    artifacts?: ArtifactRef[];
     /** Parameters for this unit alone, sent to the machine as given. */
     parameters?: Record<string, unknown>;
+    /** The host's references for this unit, by name; sent to the machine as given and searchable on the row. */
+    refs?: Record<string, string>;
 }
 /** Metadata on a handoff row. Everything a dispatch needs rides here. */
 export interface HandoffFacets {
@@ -60,6 +68,8 @@ export interface HandoffFacets {
     attempt: number;
     /** Parameters for the job, sent to the machine as given; what they mean is between the host and its codec. */
     parameters?: Record<string, unknown>;
+    /** The host's references for the job, by name (an order, a batch, a ticket); sent to the machine as given and searchable on the row. */
+    refs?: Record<string, string>;
 }
 /**
  * Items a handoff row accumulates, one per unit: the sender's finished-job
@@ -69,6 +79,11 @@ export interface HandoffFacets {
 export declare const itemKeys: {
     released: (unitId: string) => string;
     arrived: (unitId: string) => string;
+    started: (unitId: string) => string;
+    progress: (unitId: string, step: number) => string;
+    stage: (unitId: string, stage: string) => string;
+    paused: (alarmId: string, raisedAt: string) => string;
+    resumed: (alarmId: string, raisedAt: string) => string;
 };
 /** The unit an arrival item names; null for any other item. */
 export declare function arrivedUnit(itemKey: string): string | null;
@@ -125,6 +140,10 @@ export declare const TRACE_FACET: {
     readonly TRACE: "trace";
     readonly LOTS: "lots";
 };
+/** The facet a row holding a job carries while it runs: its latest noted progress. */
+export declare const LIFECYCLE_FACET: {
+    readonly PROGRESS: "progress";
+};
 /** What resolves a handoff row, from a machine or from a person's form. */
 export interface HandoffResolution {
     received: boolean;
@@ -136,8 +155,12 @@ export interface HandoffResolution {
     /** The machine that reported the error, when it is not the one resolving. */
     reportedBy?: string;
     errorCode?: HandoffErrorCode;
+    /** On `sender-rejected`, how the sender finished the unit: reject, hold, scrap or rework. */
+    result?: string;
     reason?: string | null;
     detail?: string;
+    /** The machine's structured code for a refusal or failure, when its reply carried one. */
+    code?: string;
 }
 /** Decisions on line-operator rows: failed handoffs, alarms, dark machines, contested registrations. */
 export declare const DECISION: {
@@ -187,7 +210,11 @@ export interface CommandResolution {
     op: string;
     status: CommandStatus;
     reason?: string;
+    /** The machine's structured code for the outcome, when its reply carried one. */
+    code?: string;
 }
-export declare const dispatchCommandId: (escalationId: string) => string;
-export declare const controlCommandId: (escalationId: string) => string;
+/** A row's dispatch command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+export declare const dispatchCommandId: (escalationId: string, round?: number) => string;
+/** A command row's command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+export declare const controlCommandId: (escalationId: string, round?: number) => string;
 export declare const cancelCommandId: (escalationId: string) => string;

package/build/services/machines/contracts.js

@@ -6,7 +6,7 @@
  * protocol that resolves it differs.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cancelCommandId = exports.controlCommandId = exports.dispatchCommandId = exports.COMMAND_STATUS = exports.signalKeys = exports.DECISION = exports.TRACE_FACET = exports.HANDOFF_ERROR = exports.itemKeys = exports.LINE_ROLE = exports.ROW_TYPE = void 0;
+exports.cancelCommandId = exports.controlCommandId = exports.dispatchCommandId = exports.COMMAND_STATUS = exports.signalKeys = exports.DECISION = exports.LIFECYCLE_FACET = exports.TRACE_FACET = exports.HANDOFF_ERROR = exports.itemKeys = exports.LINE_ROLE = exports.ROW_TYPE = void 0;
 exports.arrivedUnit = arrivedUnit;
 exports.handoffProgress = handoffProgress;
 exports.handoffDecisions = handoffDecisions;
@@ -43,6 +43,11 @@ const ARRIVED_PREFIX = 'arrived:';
 exports.itemKeys = {
     released: (unitId) => `${RELEASED_PREFIX}${unitId}`,
     arrived: (unitId) => `${ARRIVED_PREFIX}${unitId}`,
+    started: (unitId) => `started:${unitId}`,
+    progress: (unitId, step) => `progress:${unitId}:${step}`,
+    stage: (unitId, stage) => `stage:${unitId}:${stage}`,
+    paused: (alarmId, raisedAt) => `paused:${alarmId}:${raisedAt}`,
+    resumed: (alarmId, raisedAt) => `resumed:${alarmId}:${raisedAt}`,
 };
 /** The unit an arrival item names; null for any other item. */
 function arrivedUnit(itemKey) {
@@ -78,6 +83,8 @@ exports.HANDOFF_ERROR = {
  * containment-searchable, e.g. `{ lots: ['LOT-7'] }`.
  */
 exports.TRACE_FACET = { TRACE: 'trace', LOTS: 'lots' };
+/** The facet a row holding a job carries while it runs: its latest noted progress. */
+exports.LIFECYCLE_FACET = { PROGRESS: 'progress' };
 /** Decisions on line-operator rows: failed handoffs, alarms, dark machines, contested registrations. */
 exports.DECISION = {
     RETRY: 'retry',
@@ -107,9 +114,11 @@ exports.signalKeys = {
     jobStop: (jobKey, step, attempt) => `job-stop-${jobKey}-s${step}-a${attempt}`,
 };
 exports.COMMAND_STATUS = { ...protocol_1.REPLY_STATUS, NO_REPLY: 'no-reply', UNSUPPORTED: 'unsupported' };
-const dispatchCommandId = (escalationId) => `${escalationId}:dispatch`;
+/** A row's dispatch command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+const dispatchCommandId = (escalationId, round = 1) => (round > 1 ? `${escalationId}:dispatch:r${round}` : `${escalationId}:dispatch`);
 exports.dispatchCommandId = dispatchCommandId;
-const controlCommandId = (escalationId) => `${escalationId}:command`;
+/** A command row's command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+const controlCommandId = (escalationId, round = 1) => (round > 1 ? `${escalationId}:command:r${round}` : `${escalationId}:command`);
 exports.controlCommandId = controlCommandId;
 const cancelCommandId = (escalationId) => `${escalationId}:cancel`;
 exports.cancelCommandId = cancelCommandId;

package/build/services/machines/gateway/actions.d.ts

@@ -39,8 +39,8 @@ export type Action =
     rowId: string;
     machineId: string;
     op: string;
-    cmdId: string;
     args: Record<string, any>;
+    createdAt?: number;
 } | {
     kind: 'claim';
     as: Actor;
@@ -78,6 +78,7 @@ export type Action =
     cmdId: string;
     status: string;
     reason: string;
+    code?: string;
 }
 /** A machine reported the jobs it holds: answers a status query in flight. */
  | {

package/build/services/machines/gateway/admission.d.ts

@@ -7,19 +7,31 @@
 import { type LineDef } from '../../../lib/machines/line';
 import type { Inspection } from '../../../lib/machines/protocol';
 import { type Disposition, type InboundMessage } from '../../../lib/machines/transport';
+import type { WireAction } from '../../../types/machines-view';
 import type { Action } from './actions';
 import { type GatewayLog } from './log';
 import type { GatewayState } from './state';
-/** What handling one decoded message came to. */
+/** What handling one decoded message came to, with what the gateway did for the wire record. */
 export interface Applied {
     disposition: Disposition;
     failed: boolean;
+    actions: WireAction[];
+    /** The address the message came from, when it was not admitted as a station's. */
+    sender?: {
+        machineId: string | null;
+        address: string | null;
+    };
+}
+/** A message admitted as a station's, with what admitting it did (a binding, say). */
+export interface Admitted {
+    seen: Inspection;
+    actions: WireAction[];
 }
 export interface AdmissionContext {
     state: GatewayState;
     line: LineDef;
     executor: {
-        apply(actions: Action[]): Promise<void>;
+        apply(actions: Action[]): Promise<WireAction[]>;
     };
     log: GatewayLog;
     /** Send waiting handoffs that may now go out. */
@@ -30,6 +42,6 @@ export interface AdmissionContext {
  * bound and its waiting work goes out; a contested claim raises one row for a
  * person; either of those, and an address no station claims, is not acted on.
  */
-export declare function admitSender(ctx: AdmissionContext, topic: string, raw: Inspection): Promise<Inspection | Applied>;
+export declare function admitSender(ctx: AdmissionContext, topic: string, raw: Inspection): Promise<Admitted | Applied>;
 /** Under a strict codec a broken message is not acted on: a person sees it on a row, once. */
 export declare function holdViolation(ctx: AdmissionContext, msg: InboundMessage, seen: Inspection): Promise<Applied>;

package/build/services/machines/gateway/admission.js

@@ -11,8 +11,12 @@ exports.holdViolation = holdViolation;
 const contracts_1 = require("../contracts");
 const line_1 = require("../../../lib/machines/line");
 const transport_1 = require("../../../lib/machines/transport");
+const dispatcher_1 = require("./dispatcher");
 const log_1 = require("./log");
 const registration_1 = require("./registration");
+function from(sender) {
+    return { machineId: sender.machineId ?? null, address: sender.address };
+}
 /**
  * The station a message is from (registration.ts). A newly claimed station is
  * bound and its waiting work goes out; a contested claim raises one row for a
@@ -20,47 +24,58 @@ const registration_1 = require("./registration");
  */
 async function admitSender(ctx, topic, raw) {
     const sender = (0, registration_1.resolveSender)(ctx.state, ctx.line, raw);
+    const actions = [];
     try {
         if (sender.status === registration_1.SENDER.BOUND) {
-            await ctx.executor.apply([{ kind: 'bindAddress', machineId: sender.machineId, address: sender.address }]);
+            actions.push(...await ctx.executor.apply([{ kind: 'bindAddress', machineId: sender.machineId, address: sender.address }]));
             await ctx.pace();
         }
         if (sender.seen)
-            return sender.seen;
+            return { seen: sender.seen, actions };
         if (sender.status === registration_1.SENDER.CONTESTED)
             ctx.state.hearClaim(sender.address, sender.machineId);
         if (sender.status === registration_1.SENDER.CONTESTED && !ctx.state.claimRaised(sender.address, sender.machineId)) {
-            await ctx.executor.apply([{ kind: 'createRow', row: (0, registration_1.registrationRow)(ctx.state, ctx.line, sender.address, sender.machineId) }]);
-            ctx.state.raiseClaim(sender.address, sender.machineId);
+            const [made] = await ctx.executor.apply([{ kind: 'createRow', row: (0, registration_1.registrationRow)(ctx.state, ctx.line, sender.address, sender.machineId) }]);
+            ctx.state.raiseClaim(sender.address, sender.machineId, made?.rowId);
             ctx.log.warn(log_1.GATEWAY_LOG.ADDRESS_CONTESTED, { topic, address: sender.address, machine: sender.machineId });
+            actions.push({ ...made, detail: `${sender.address} says it is ${sender.machineId}; a person decides` });
+        }
+        else if (sender.status === registration_1.SENDER.CONTESTED) {
+            actions.push({ verb: 'none', detail: `${sender.address} says it is ${sender.machineId}; waiting for a person` });
+        }
+        if (sender.status === registration_1.SENDER.UNKNOWN) {
+            if (ctx.state.firstHeardStranger(sender.address))
+                ctx.log.warn(log_1.GATEWAY_LOG.ADDRESS_UNKNOWN, { topic, address: sender.address });
+            actions.push({ verb: 'none', detail: `${sender.address} claims no station on this line` });
         }
-        if (sender.status === registration_1.SENDER.UNKNOWN && ctx.state.firstHeardStranger(sender.address))
-            ctx.log.warn(log_1.GATEWAY_LOG.ADDRESS_UNKNOWN, { topic, address: sender.address });
-        return { disposition: transport_1.DISPOSITION.ACK, failed: false };
+        return { disposition: transport_1.DISPOSITION.ACK, failed: false, actions, sender: from(sender) };
     }
     catch (err) {
-        ctx.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { topic, error: err.message });
-        return { disposition: transport_1.DISPOSITION.NAK, failed: true };
+        if (!(err instanceof dispatcher_1.GatewayStopped))
+            ctx.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { topic, error: err.message });
+        return { disposition: transport_1.DISPOSITION.NAK, failed: true, actions, sender: from(sender) };
     }
 }
 /** Under a strict codec a broken message is not acted on: a person sees it on a row, once. */
 async function holdViolation(ctx, msg, seen) {
     const { ident, kind } = seen.parsed;
     const key = seen.receiptId ?? seen.body?.id ?? `${msg.topic}:${msg.payload}`;
+    const detail = seen.violations.join('; ');
     if (ctx.state.violationRaised(key))
-        return { disposition: transport_1.DISPOSITION.ACK, failed: false };
+        return { disposition: transport_1.DISPOSITION.ACK, failed: false, actions: [{ verb: 'hold', detail: `${detail} (already shown to a person)` }] };
     try {
-        await ctx.executor.apply([{ kind: 'createRow', row: {
+        const [made] = await ctx.executor.apply([{ kind: 'createRow', row: {
                     type: contracts_1.ROW_TYPE.CONTRACT_VIOLATION, role: (0, line_1.operatorRole)(ctx.line), priority: 2,
                     description: `${ident.machineId} sent a ${kind} message that breaks the contract: ${seen.violations.join('; ')}`,
                     metadata: { siteId: ident.siteId, lineId: ident.lineId, machineId: ident.machineId, kind, topic: msg.topic, messageId: seen.body?.id ?? null, violations: seen.violations },
                     escalation_payload: msg.payload,
                 } }]);
         ctx.state.raiseViolation(key);
-        return { disposition: transport_1.DISPOSITION.ACK, failed: false };
+        return { disposition: transport_1.DISPOSITION.ACK, failed: false, actions: [{ verb: 'hold', rowId: made?.rowId, detail }] };
     }
     catch (err) {
-        ctx.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { topic: msg.topic, error: err.message });
-        return { disposition: transport_1.DISPOSITION.NAK, failed: true };
+        if (!(err instanceof dispatcher_1.GatewayStopped))
+            ctx.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { topic: msg.topic, error: err.message });
+        return { disposition: transport_1.DISPOSITION.NAK, failed: true, actions: [] };
     }
 }

package/build/services/machines/gateway/dispatcher.d.ts

@@ -17,6 +17,7 @@ import { type LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import type { CommandReply, ReplyWaiter } from './reply-waiter';
 import { type GatewayState } from './state';
+import type { WireRecorder } from './wire';
 export interface DispatcherDeps {
     bus: MachineBus;
     codec: LineCodec;
@@ -27,6 +28,8 @@ export interface DispatcherDeps {
     replyTimeoutMs: number;
     dispatchAttempts: number;
     log: GatewayLog;
+    /** Records each command the gateway sends, for the line's wire view. */
+    wire?: Pick<WireRecorder, 'command'>;
 }
 type DispatchAction = Extract<Action, {
     kind: 'dispatch';
@@ -37,18 +40,31 @@ type CommandAction = Extract<Action, {
 type ControlAction = Extract<Action, {
     kind: 'control';
 }>;
+/** Ends a dispatch or command still running when the gateway stops; the next leader takes it up from the row. */
+export declare class GatewayStopped extends Error {
+    constructor();
+}
 export declare class Dispatcher {
     private readonly d;
+    private readonly sent;
+    private readonly stopping;
     constructor(d: DispatcherDeps);
-    /** False when the codec has no message for the command. */
-    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>): Promise<boolean>;
+    /** Work still running sends nothing more and changes no row. */
+    stop(): void;
+    private live;
+    /** This send's attempt number, and whether the machine may have the command already. */
+    private countSend;
+    /** False when the codec has no message for the command. `since` is when the command's row was created. */
+    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<boolean>;
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>): Promise<CommandReply | null>;
+    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<CommandReply | null>;
     command(a: CommandAction): Promise<void>;
     /**
      * A machine-command row: send it, then resolve the row with the reply, as the
      * machine that gave it. The command expires when the asking ends, so one that
      * reaches a machine after its row resolved no-reply is refused, never run.
+     * Rounds of the asking count from the row's creation, so the command sent
+     * again by a new gateway is the same command with the same arguments.
      */
     control(a: ControlAction): Promise<void>;
     dispatch(a: DispatchAction): Promise<void>;

package/build/services/machines/gateway/dispatcher.js

@@ -11,7 +11,7 @@
  * codec has no message for is unsupported: nothing is sent or awaited.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Dispatcher = void 0;
+exports.Dispatcher = exports.GatewayStopped = void 0;
 const promises_1 = require("timers/promises");
 const contracts_1 = require("../contracts");
 const registration_1 = require("./registration");
@@ -19,6 +19,7 @@ const protocol_1 = require("../../../lib/machines/protocol");
 const actions_1 = require("./actions");
 const escalations_1 = require("./escalations");
 const jobs_1 = require("./jobs");
+const pacing_1 = require("./pacing");
 const log_1 = require("./log");
 const state_1 = require("./state");
 /** A reply the machine sent, as opposed to none or an unsupported command. */
@@ -29,25 +30,64 @@ function fromMachine(reply) {
 function took(reply) {
     return reply?.status === protocol_1.REPLY_STATUS.ACCEPTED || reply?.status === protocol_1.REPLY_STATUS.COMPLETED;
 }
+/** The reply's structured code for a resolution, when it carried one. */
+function withCode(reply) {
+    return reply.code ? { code: reply.code } : {};
+}
+/** Most command ids whose sends are counted; the oldest are forgotten first. */
+const SENT_REMEMBERED = 10_000;
+/** Ends a dispatch or command still running when the gateway stops; the next leader takes it up from the row. */
+class GatewayStopped extends Error {
+    constructor() {
+        super('the gateway stopped');
+    }
+}
+exports.GatewayStopped = GatewayStopped;
 class Dispatcher {
     constructor(d) {
         this.d = d;
+        this.sent = new Map();
+        this.stopping = new AbortController();
+    }
+    /** Work still running sends nothing more and changes no row. */
+    stop() {
+        this.stopping.abort();
     }
-    /** False when the codec has no message for the command. */
-    async publish(machineId, op, cmdId, args) {
-        const wires = this.d.codec.encode((0, registration_1.identFor)(this.d.state, this.d.line, machineId), { cmdId, op, args });
-        for (const wire of wires)
+    live() {
+        if (this.stopping.signal.aborted)
+            throw new GatewayStopped();
+    }
+    /** This send's attempt number, and whether the machine may have the command already. */
+    countSend(cmdId, since) {
+        const attempt = (this.sent.get(cmdId) ?? 0) + 1;
+        this.sent.delete(cmdId);
+        this.sent.set(cmdId, attempt);
+        if (this.sent.size > SENT_REMEMBERED)
+            this.sent.delete(this.sent.keys().next().value);
+        return { attempt, retry: attempt > 1 || (since !== undefined && since < this.d.state.takenAt) };
+    }
+    /** False when the codec has no message for the command. `since` is when the command's row was created. */
+    async publish(machineId, op, cmdId, args, since) {
+        this.live();
+        const ident = (0, registration_1.identFor)(this.d.state, this.d.line, machineId);
+        const command = { cmdId, op, args, ...this.countSend(cmdId, since) };
+        const wires = this.d.codec.encode(ident, command);
+        for (const wire of wires) {
             await this.d.bus.publish(wire.topic, wire.payload);
+            this.d.wire?.command(ident, command, wire);
+        }
         return wires.length > 0;
     }
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    async ask(machineId, op, cmdId, args) {
+    async ask(machineId, op, cmdId, args, since) {
         for (let attempt = 1; attempt <= this.d.dispatchAttempts; attempt++) {
+            this.live();
             const waiting = this.d.replies.wait(cmdId, this.d.replyTimeoutMs);
-            if (!await this.publish(machineId, op, cmdId, args)) {
+            if (!await this.publish(machineId, op, cmdId, args, since)) {
                 this.d.replies.settle({ cmdId, status: contracts_1.COMMAND_STATUS.UNSUPPORTED, reason: `the line's codec has no ${op} command` });
             }
             const reply = await waiting;
+            this.live();
             if (reply)
                 return reply;
         }
@@ -67,15 +107,19 @@ class Dispatcher {
      * A machine-command row: send it, then resolve the row with the reply, as the
      * machine that gave it. The command expires when the asking ends, so one that
      * reaches a machine after its row resolved no-reply is refused, never run.
+     * Rounds of the asking count from the row's creation, so the command sent
+     * again by a new gateway is the same command with the same arguments.
      */
     async control(a) {
-        const expiresAt = Date.now() + this.d.replyTimeoutMs * this.d.dispatchAttempts;
-        const reply = await this.ask(a.machineId, a.op, a.cmdId, { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) });
+        const now = Date.now();
+        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, this.d.replyTimeoutMs * this.d.dispatchAttempts);
+        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt);
         // Timers may fire a little early; the command must have expired before the row says no-reply.
         if (!reply && Date.now() <= expiresAt)
-            await (0, promises_1.setTimeout)(expiresAt - Date.now() + 1);
+            await (0, promises_1.setTimeout)(expiresAt - Date.now() + 1, undefined, { signal: this.stopping.signal }).catch(() => undefined);
+        this.live();
         const outcome = reply
-            ? { accepted: reply.status === protocol_1.REPLY_STATUS.ACCEPTED, machineId: a.machineId, op: a.op, status: reply.status, reason: reply.reason }
+            ? { accepted: reply.status === protocol_1.REPLY_STATUS.ACCEPTED, machineId: a.machineId, op: a.op, status: reply.status, reason: reply.reason, ...withCode(reply) }
             : { accepted: false, machineId: a.machineId, op: a.op, status: contracts_1.COMMAND_STATUS.NO_REPLY, reason: `no reply after ${this.d.dispatchAttempts} attempts` };
         const context = { row: a.rowId, machine: a.machineId, op: a.op, status: outcome.status, reason: outcome.reason };
         if (outcome.accepted)
@@ -97,10 +141,12 @@ class Dispatcher {
         // Known before any reply, so a failure that follows an acceptance finds its row.
         for (const job of jobs)
             state.rememberDispatch(`${a.cmdId}${job.suffix}`, a.rowId);
-        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units })));
+        const since = state.get(a.rowId)?.createdAt;
+        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since)));
         const refused = replies.find((r) => r && !took(r));
         const silent = jobs.filter((_, i) => !replies[i]);
         const held = !refused && silent.length ? await this.heldJobs(a.machineId, a.cmdId) : null;
+        this.live();
         const reconciled = !!held && silent.every((j) => held.has(j.jobId));
         const row = state.get(a.rowId);
         if (!refused && (!silent.length || reconciled)) {
@@ -120,7 +166,7 @@ class Dispatcher {
         row.dispatch = state_1.DISPATCH_STATE.REFUSED;
         const unanswered = `${a.machineId} sent no reply after ${this.d.dispatchAttempts} attempts`;
         const resolution = refused
-            ? { received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_REFUSED, detail: `${a.machineId} ${refused.status}: ${refused.reason}` }
+            ? { received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_REFUSED, detail: `${a.machineId} ${refused.status}: ${refused.reason}`, ...withCode(refused) }
             : { received: false, errorCode: contracts_1.HANDOFF_ERROR.NO_REPLY, detail: held ? `${unanswered} and does not hold the job` : `${unanswered}; its state is unknown` };
         log.warn(log_1.GATEWAY_LOG.DISPATCH_REFUSED, { row: a.rowId, machine: a.machineId, ...resolution });
         const ended = await lt.resolve(fromMachine(refused ?? null) ? a.machineId : actions_1.CONTROLLER, a.rowId, resolution);
@@ -132,12 +178,14 @@ class Dispatcher {
     }
     /** The units were already at the machine: its acceptance completes the handoff. */
     async received(a) {
+        this.live();
         const units = a.args.units.map((u) => u.unitId);
         this.check(a.rowId, 'resolve', await this.d.lt.resolve(a.machineId, a.rowId, { received: true, receiver: a.machineId, units }));
         this.d.state.close(a.rowId);
     }
     /** Ask a silent machine what it holds; null when it cannot say or does not answer in time. */
     async heldJobs(machineId, cmdId) {
+        this.live();
         const waiting = this.d.replies.waitSnapshot(machineId, this.d.replyTimeoutMs);
         if (!await this.publish(machineId, protocol_1.OP.SNAPSHOT, `${cmdId}:status`, {}))
             this.d.replies.settleSnapshot(machineId, null);

package/build/services/machines/gateway/executor.d.ts

@@ -2,17 +2,20 @@
  * Runs the planner's actions: escalation verbs through LineEscalations,
  * dispatches and commands through the Dispatcher, alarm and machine rows,
  * and the offline debounce. A verb that fails for real throws, so the message that
- * caused it is delivered again.
+ * caused it is delivered again. What ran comes back as wire actions, for the
+ * record of the message that caused it.
  */
 import { type LineDef } from '../../../lib/machines/line';
 import { type LineCodec } from '../../../lib/machines/protocol';
 import type { MachineBus } from '../../../lib/machines/transport';
+import type { WireAction } from '../../../types/machines-view';
 import { type Action } from './actions';
 import type { AddressBook } from './address-book';
 import { type LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import type { ReplyWaiter } from './reply-waiter';
-import type { GatewayState } from './state';
+import { type GatewayState } from './state';
+import { type WireRecorder } from './wire';
 export interface ExecutorDeps {
     bus: MachineBus;
     lt: LineEscalations;
@@ -26,22 +29,33 @@ export interface ExecutorDeps {
     log: GatewayLog;
     /** Where learned addresses are kept (registration.ts). */
     addresses: AddressBook;
+    /** Records each command sent, for the line's wire view. */
+    wire?: Pick<WireRecorder, 'command'>;
 }
 export declare class Executor {
     private readonly d;
     private inflight;
     private quietPending;
     private timers;
+    private stopped;
     private readonly dispatcher;
     constructor(d: ExecutorDeps);
     /** Dispatches and commands run detached so one slow reply never holds up the feed. */
-    apply(actions: Action[]): Promise<void>;
+    apply(actions: Action[]): Promise<WireAction[]>;
     /** Resolves when every detached dispatch has finished. */
     idle(): Promise<void>;
+    /** Work still running sends nothing more and raises no row; the next leader takes it up from the message or the row. */
     stop(): void;
+    /** A row raised after stop() would be raised again by the next leader, so the message is nak'd instead. */
+    private live;
     private track;
     private run;
-    /** A dispatch's second reply: ready, or a failure to prepare that ends the handoff. */
+    /**
+     * A dispatch's second reply: ready, or a failure to prepare that ends the
+     * handoff. When the handoff into the machine has already ended, as one
+     * dispatched on arrival does at acceptance, the failure ends the handoff
+     * carrying the units out of it.
+     */
     private lateReply;
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     private accumulate;