@hotmeshio/long-tail 0.27.0 → 0.28.1

package/build/services/machines/gateway/dispatcher.d.ts

@@ -17,6 +17,7 @@ import { type LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import type { CommandReply, ReplyWaiter } from './reply-waiter';
 import { type GatewayState } from './state';
+import type { WireRecorder } from './wire';
 export interface DispatcherDeps {
     bus: MachineBus;
     codec: LineCodec;
@@ -27,6 +28,8 @@ export interface DispatcherDeps {
     replyTimeoutMs: number;
     dispatchAttempts: number;
     log: GatewayLog;
+    /** Records each command the gateway sends, for the line's wire view. */
+    wire?: Pick<WireRecorder, 'command'>;
 }
 type DispatchAction = Extract<Action, {
     kind: 'dispatch';
@@ -49,12 +52,15 @@ export declare class Dispatcher {
     /** Work still running sends nothing more and changes no row. */
     stop(): void;
     private live;
-    /** This send's attempt number, and whether the machine may have the command already. */
+    /**
+     * This send's attempt number, whether the machine may have the command already, and when it was
+     * issued: the time given (a round's start, the same for every gateway), else the first send's.
+     */
     private countSend;
     /** False when the codec has no message for the command. `since` is when the command's row was created. */
-    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<boolean>;
+    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number, issuedAt?: number): Promise<boolean>;
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<CommandReply | null>;
+    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number, issuedAt?: number): Promise<CommandReply | null>;
     command(a: CommandAction): Promise<void>;
     /**
      * A machine-command row: send it, then resolve the row with the reply, as the
@@ -65,7 +71,9 @@ export declare class Dispatcher {
      */
     control(a: ControlAction): Promise<void>;
     dispatch(a: DispatchAction): Promise<void>;
-    /** The units were already at the machine: its acceptance completes the handoff. */
+    /** A job sent again after the machine restarted, for a row that already ended: the reply is only logged. */
+    private resent;
+    /** The units were already at the machine: a job it has taken and is ready for completes the handoff. */
     private received;
     /** Ask a silent machine what it holds; null when it cannot say or does not answer in time. */
     private heldJobs;

package/build/services/machines/gateway/dispatcher.js

@@ -30,6 +30,10 @@ function fromMachine(reply) {
 function took(reply) {
     return reply?.status === protocol_1.REPLY_STATUS.ACCEPTED || reply?.status === protocol_1.REPLY_STATUS.COMPLETED;
 }
+/** The reply's structured code for a resolution, when it carried one. */
+function withCode(reply) {
+    return reply.code ? { code: reply.code } : {};
+}
 /** Most command ids whose sends are counted; the oldest are forgotten first. */
 const SENT_REMEMBERED = 10_000;
 /** Ends a dispatch or command still running when the gateway stops; the next leader takes it up from the row. */
@@ -53,29 +57,38 @@ class Dispatcher {
         if (this.stopping.signal.aborted)
             throw new GatewayStopped();
     }
-    /** This send's attempt number, and whether the machine may have the command already. */
-    countSend(cmdId, since) {
-        const attempt = (this.sent.get(cmdId) ?? 0) + 1;
+    /**
+     * This send's attempt number, whether the machine may have the command already, and when it was
+     * issued: the time given (a round's start, the same for every gateway), else the first send's.
+     */
+    countSend(cmdId, since, issuedAt) {
+        const before = this.sent.get(cmdId);
+        const record = { attempt: (before?.attempt ?? 0) + 1, issuedAt: issuedAt ?? before?.issuedAt ?? Date.now() };
         this.sent.delete(cmdId);
-        this.sent.set(cmdId, attempt);
+        this.sent.set(cmdId, record);
         if (this.sent.size > SENT_REMEMBERED)
             this.sent.delete(this.sent.keys().next().value);
-        return { attempt, retry: attempt > 1 || (since !== undefined && since < this.d.state.takenAt) };
+        const retry = record.attempt > 1 || (since !== undefined && since < this.d.state.takenAt);
+        return { attempt: record.attempt, retry, issuedAt: (0, protocol_1.isoTs)(record.issuedAt) };
     }
     /** False when the codec has no message for the command. `since` is when the command's row was created. */
-    async publish(machineId, op, cmdId, args, since) {
+    async publish(machineId, op, cmdId, args, since, issuedAt) {
         this.live();
-        const wires = this.d.codec.encode((0, registration_1.identFor)(this.d.state, this.d.line, machineId), { cmdId, op, args, ...this.countSend(cmdId, since) });
-        for (const wire of wires)
+        const ident = (0, registration_1.identFor)(this.d.state, this.d.line, machineId);
+        const command = { cmdId, op, args, ...this.countSend(cmdId, since, issuedAt) };
+        const wires = this.d.codec.encode(ident, command);
+        for (const wire of wires) {
             await this.d.bus.publish(wire.topic, wire.payload);
+            this.d.wire?.command(ident, command, wire);
+        }
         return wires.length > 0;
     }
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    async ask(machineId, op, cmdId, args, since) {
+    async ask(machineId, op, cmdId, args, since, issuedAt) {
         for (let attempt = 1; attempt <= this.d.dispatchAttempts; attempt++) {
             this.live();
             const waiting = this.d.replies.wait(cmdId, this.d.replyTimeoutMs);
-            if (!await this.publish(machineId, op, cmdId, args, since)) {
+            if (!await this.publish(machineId, op, cmdId, args, since, issuedAt)) {
                 this.d.replies.settle({ cmdId, status: contracts_1.COMMAND_STATUS.UNSUPPORTED, reason: `the line's codec has no ${op} command` });
             }
             const reply = await waiting;
@@ -104,14 +117,15 @@ class Dispatcher {
      */
     async control(a) {
         const now = Date.now();
-        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, this.d.replyTimeoutMs * this.d.dispatchAttempts);
-        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt);
+        const ttlMs = this.d.replyTimeoutMs * this.d.dispatchAttempts;
+        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, ttlMs);
+        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt, expiresAt - ttlMs);
         // Timers may fire a little early; the command must have expired before the row says no-reply.
         if (!reply && Date.now() <= expiresAt)
             await (0, promises_1.setTimeout)(expiresAt - Date.now() + 1, undefined, { signal: this.stopping.signal }).catch(() => undefined);
         this.live();
         const outcome = reply
-            ? { accepted: reply.status === protocol_1.REPLY_STATUS.ACCEPTED, machineId: a.machineId, op: a.op, status: reply.status, reason: reply.reason }
+            ? { accepted: reply.status === protocol_1.REPLY_STATUS.ACCEPTED, machineId: a.machineId, op: a.op, status: reply.status, reason: reply.reason, ...withCode(reply) }
             : { accepted: false, machineId: a.machineId, op: a.op, status: contracts_1.COMMAND_STATUS.NO_REPLY, reason: `no reply after ${this.d.dispatchAttempts} attempts` };
         const context = { row: a.rowId, machine: a.machineId, op: a.op, status: outcome.status, reason: outcome.reason };
         if (outcome.accepted)
@@ -134,22 +148,36 @@ class Dispatcher {
         for (const job of jobs)
             state.rememberDispatch(`${a.cmdId}${job.suffix}`, a.rowId);
         const since = state.get(a.rowId)?.createdAt;
-        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since)));
+        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since, a.issuedAt)));
         const refused = replies.find((r) => r && !took(r));
         const silent = jobs.filter((_, i) => !replies[i]);
         const held = !refused && silent.length ? await this.heldJobs(a.machineId, a.cmdId) : null;
         this.live();
         const reconciled = !!held && silent.every((j) => held.has(j.jobId));
+        if (a.ended) {
+            this.resent(a, jobs, refused ?? null, !refused && (!silent.length || reconciled));
+            return;
+        }
         const row = state.get(a.rowId);
         if (!refused && (!silent.length || reconciled)) {
             if (row)
                 row.dispatch = state_1.DISPATCH_STATE.ACCEPTED;
+            for (const job of jobs)
+                state.rememberSent(a.machineId, job.jobId, a.rowId);
             if (reconciled)
                 log.info(log_1.GATEWAY_LOG.DISPATCH_RECONCILED, { row: a.rowId, machine: a.machineId, jobs: silent.map((j) => j.jobId) });
             log.info(log_1.GATEWAY_LOG.DISPATCHED, { row: a.rowId, machine: a.machineId, reason: replies.map((r) => r?.reason ?? 'held').join('; ') });
             this.check(a.rowId, 'claim', await lt.claim(a.machineId, a.rowId));
-            if (a.receivedOnAccept)
+            if (!a.receivedOnAccept)
+                return;
+            const preparing = jobs.filter((job, i) => replies[i]?.status === protocol_1.REPLY_STATUS.ACCEPTED && replies[i]?.preparing).map((job) => `${a.cmdId}${job.suffix}`);
+            if (!preparing.length) {
                 await this.received(a);
+                return;
+            }
+            if (row)
+                row.awaitingReady = new Set(preparing);
+            log.info(log_1.GATEWAY_LOG.DISPATCH_PREPARING, { row: a.rowId, machine: a.machineId, jobs: preparing });
             return;
         }
         // The handoff may have ended meanwhile: a machine that ran the job anyway reports the arrival.
@@ -158,7 +186,7 @@ class Dispatcher {
         row.dispatch = state_1.DISPATCH_STATE.REFUSED;
         const unanswered = `${a.machineId} sent no reply after ${this.d.dispatchAttempts} attempts`;
         const resolution = refused
-            ? { received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_REFUSED, detail: `${a.machineId} ${refused.status}: ${refused.reason}` }
+            ? { received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_REFUSED, detail: `${a.machineId} ${refused.status}: ${refused.reason}`, ...withCode(refused) }
             : { received: false, errorCode: contracts_1.HANDOFF_ERROR.NO_REPLY, detail: held ? `${unanswered} and does not hold the job` : `${unanswered}; its state is unknown` };
         log.warn(log_1.GATEWAY_LOG.DISPATCH_REFUSED, { row: a.rowId, machine: a.machineId, ...resolution });
         const ended = await lt.resolve(fromMachine(refused ?? null) ? a.machineId : actions_1.CONTROLLER, a.rowId, resolution);
@@ -168,7 +196,15 @@ class Dispatcher {
             return;
         await this.withdraw(a, jobs.filter((_, i) => !replies[i] || took(replies[i])));
     }
-    /** The units were already at the machine: its acceptance completes the handoff. */
+    /** A job sent again after the machine restarted, for a row that already ended: the reply is only logged. */
+    resent(a, jobs, refused, taken) {
+        const context = { row: a.rowId, machine: a.machineId, jobs: jobs.map((j) => j.jobId), resent: true };
+        if (taken)
+            this.d.log.info(log_1.GATEWAY_LOG.DISPATCHED, { ...context, reason: 'sent again after a restart' });
+        else
+            this.d.log.warn(log_1.GATEWAY_LOG.DISPATCH_REFUSED, { ...context, reason: refused ? `${refused.status}: ${refused.reason}` : 'no reply' });
+    }
+    /** The units were already at the machine: a job it has taken and is ready for completes the handoff. */
     async received(a) {
         this.live();
         const units = a.args.units.map((u) => u.unitId);

package/build/services/machines/gateway/executor.d.ts

@@ -2,17 +2,20 @@
  * Runs the planner's actions: escalation verbs through LineEscalations,
  * dispatches and commands through the Dispatcher, alarm and machine rows,
  * and the offline debounce. A verb that fails for real throws, so the message that
- * caused it is delivered again.
+ * caused it is delivered again. What ran comes back as wire actions, for the
+ * record of the message that caused it.
  */
 import { type LineDef } from '../../../lib/machines/line';
 import { type LineCodec } from '../../../lib/machines/protocol';
 import type { MachineBus } from '../../../lib/machines/transport';
+import type { WireAction } from '../../../types/machines-view';
 import { type Action } from './actions';
 import type { AddressBook } from './address-book';
 import { type LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import type { ReplyWaiter } from './reply-waiter';
 import { type GatewayState } from './state';
+import { type WireRecorder } from './wire';
 export interface ExecutorDeps {
     bus: MachineBus;
     lt: LineEscalations;
@@ -26,19 +29,25 @@ export interface ExecutorDeps {
     log: GatewayLog;
     /** Where learned addresses are kept (registration.ts). */
     addresses: AddressBook;
+    /** Records each command sent, for the line's wire view. */
+    wire?: Pick<WireRecorder, 'command'>;
 }
 export declare class Executor {
     private readonly d;
     private inflight;
     private quietPending;
     private timers;
+    private stopped;
     private readonly dispatcher;
     constructor(d: ExecutorDeps);
     /** Dispatches and commands run detached so one slow reply never holds up the feed. */
-    apply(actions: Action[]): Promise<void>;
+    apply(actions: Action[]): Promise<WireAction[]>;
     /** Resolves when every detached dispatch has finished. */
     idle(): Promise<void>;
+    /** Work still running sends nothing more and raises no row; the next leader takes it up from the message or the row. */
     stop(): void;
+    /** A row raised after stop() would be raised again by the next leader, so the message is nak'd instead. */
+    private live;
     private track;
     private run;
     /**
@@ -48,6 +57,8 @@ export declare class Executor {
      * carrying the units out of it.
      */
     private lateReply;
+    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    private readyForArrival;
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     private accumulate;
     private check;

package/build/services/machines/gateway/executor.js

@@ -3,7 +3,8 @@
  * Runs the planner's actions: escalation verbs through LineEscalations,
  * dispatches and commands through the Dispatcher, alarm and machine rows,
  * and the offline debounce. A verb that fails for real throws, so the message that
- * caused it is delivered again.
+ * caused it is delivered again. What ran comes back as wire actions, for the
+ * record of the message that caused it.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Executor = void 0;
@@ -16,18 +17,28 @@ const log_1 = require("./log");
 const dispatcher_1 = require("./dispatcher");
 const plan_shared_1 = require("./plan-shared");
 const state_1 = require("./state");
+const wire_1 = require("./wire");
 const NOT_FOUND = 404;
 const CONFLICT = 409;
+/** A verb's outcome as the wire records it: done, already done, or refused by Long Tail. */
+function outcomeOf(verb, rowId, result) {
+    const refused = result.outcome === escalations_1.VERB_OUTCOME.INVALID || result.status === NOT_FOUND || result.status === CONFLICT;
+    if (refused)
+        return { verb: 'refuse', rowId, detail: `${verb}: ${result.detail ?? result.outcome}` };
+    return { verb, rowId, ...(result.outcome === escalations_1.VERB_OUTCOME.ALREADY ? { detail: result.detail ?? 'already ended' } : {}) };
+}
 class Executor {
     constructor(d) {
         this.d = d;
         this.inflight = new Set();
         this.quietPending = new Set();
         this.timers = new Set();
+        this.stopped = false;
         this.dispatcher = new dispatcher_1.Dispatcher(d);
     }
     /** Dispatches and commands run detached so one slow reply never holds up the feed. */
     async apply(actions) {
+        const done = [];
         for (const action of actions) {
             if (action.kind === 'dispatch')
                 this.track(this.dispatcher.dispatch(action));
@@ -35,22 +46,33 @@ class Executor {
                 this.track(this.dispatcher.command(action));
             else if (action.kind === 'control')
                 this.track(this.dispatcher.control(action));
-            else
-                await this.run(action);
+            else {
+                done.push(...await this.run(action));
+                continue;
+            }
+            done.push((0, wire_1.plannedAction)(action));
         }
+        return done;
     }
     /** Resolves when every detached dispatch has finished. */
     async idle() {
         while (this.inflight.size)
             await Promise.all([...this.inflight]);
     }
+    /** Work still running sends nothing more and raises no row; the next leader takes it up from the message or the row. */
     stop() {
+        this.stopped = true;
         for (const t of this.timers)
             clearTimeout(t);
         this.timers.clear();
         this.dispatcher.stop();
         this.d.replies.cancelAll();
     }
+    /** A row raised after stop() would be raised again by the next leader, so the message is nak'd instead. */
+    live() {
+        if (this.stopped)
+            throw new dispatcher_1.GatewayStopped();
+    }
     track(work) {
         const tracked = work.catch((err) => {
             if (!(err instanceof dispatcher_1.GatewayStopped))
@@ -62,37 +84,48 @@ class Executor {
     async run(a) {
         const { lt, state, replies } = this.d;
         switch (a.kind) {
-            case 'claim':
-                return this.check(a, await lt.claim(a.as, a.rowId));
+            case 'claim': {
+                const result = await lt.claim(a.as, a.rowId);
+                this.check(a, result);
+                return [outcomeOf('claim', a.rowId, result)];
+            }
             case 'accumulate':
                 return this.accumulate(a);
-            case 'resolve':
-                this.check(a, await lt.resolve(a.as, a.rowId, a.payload));
+            case 'resolve': {
+                const result = await lt.resolve(a.as, a.rowId, a.payload);
+                this.check(a, result);
                 state.close(a.rowId);
-                return;
+                return [outcomeOf('resolve', a.rowId, result)];
+            }
             case 'createAlarmRow':
-                return this.createAlarmRow(a);
+                return [{ verb: 'create-row', rowId: await this.createAlarmRow(a), detail: `${a.row.type} ${a.alarmId} at ${a.machineId}` }];
             case 'createRow': {
+                this.live();
                 const { id } = await lt.create(actions_1.CONTROLLER, a.row);
                 this.d.log.info(log_1.GATEWAY_LOG.VIOLATION_ROW, { row: id, type: a.row.type });
-                return;
+                return [{ verb: 'create-row', rowId: id, detail: a.row.type }];
             }
             case 'watchOffline':
-                return this.watchOffline(a.machineId);
+                this.watchOffline(a.machineId);
+                return [(0, wire_1.plannedAction)(a)];
             case 'settleReply':
-                if (!replies.settle({ cmdId: a.cmdId, status: a.status, reason: a.reason }))
-                    await this.lateReply(a);
-                return;
+                if (replies.settle({ cmdId: a.cmdId, status: a.status, reason: a.reason, ...(a.code ? { code: a.code } : {}), ...(a.preparing ? { preparing: true } : {}) }))
+                    return [(0, wire_1.plannedAction)(a)];
+                return [{ verb: 'ack', detail: `late ${a.status} ${a.cmdId}` }, ...await this.lateReply(a)];
             case 'settleSnapshot':
                 replies.settleSnapshot(a.machineId, new Set(a.jobIds));
-                return;
+                return [(0, wire_1.plannedAction)(a)];
             case 'bindAddress':
                 await this.d.addresses.bind(a.machineId, a.address);
                 state.bindAddress(a.machineId, a.address);
                 this.d.log.info(log_1.GATEWAY_LOG.ADDRESS_BOUND, { machine: a.machineId, address: a.address, ...(a.by ? { approvedBy: a.by } : {}) });
-                return;
+                return [{ verb: 'ack', detail: `bound ${a.machineId} to ${a.address}` }];
             case 'refuseBinding':
                 this.d.log.warn(log_1.GATEWAY_LOG.ADDRESS_REFUSED, { machine: a.machineId, address: a.address, reason: a.reason });
+                return [(0, wire_1.plannedAction)(a)];
+            case 'unrecoverable':
+                this.d.log.warn(log_1.GATEWAY_LOG.DISPATCH_UNRECOVERABLE, { machine: a.machineId, jobs: a.jobs });
+                return [(0, wire_1.plannedAction)(a)];
         }
     }
     /**
@@ -106,34 +139,52 @@ class Executor {
         const into = state.rowForDispatch(a.cmdId);
         const m = into?.toMachine;
         if (!into || !m)
-            return;
+            return [];
         const context = { row: into.id, machine: m, cmdId: a.cmdId, reason: a.reason };
-        if (a.status === protocol_1.REPLY_STATUS.COMPLETED)
-            return log.info(log_1.GATEWAY_LOG.DISPATCH_READY, context);
+        if (a.status === protocol_1.REPLY_STATUS.COMPLETED) {
+            log.info(log_1.GATEWAY_LOG.DISPATCH_READY, context);
+            return this.readyForArrival(into, m, a.cmdId);
+        }
         if (a.status !== protocol_1.REPLY_STATUS.FAILED)
-            return;
+            return [];
         const row = into.ended ? onwardRow(state, into, m) : into;
         log.warn(log_1.GATEWAY_LOG.DISPATCH_FAILED, { ...context, ...(into.ended ? { onward: row?.id ?? null } : {}) });
         if (!row)
-            return;
-        await this.apply((0, plan_shared_1.failHandoff)(state, row, m, {
-            received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_FAILED, detail: `${m} could not prepare the job: ${a.reason}`,
+            return [];
+        return this.apply((0, plan_shared_1.failHandoff)(state, row, m, {
+            received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_FAILED, detail: `${m} could not prepare the job: ${a.reason}`, ...(a.code ? { code: a.code } : {}),
         }));
     }
+    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    async readyForArrival(into, m, cmdId) {
+        const { lt, state } = this.d;
+        if (into.ended || !into.awaitingReady?.has(cmdId))
+            return [];
+        into.awaitingReady.delete(cmdId);
+        if (into.awaitingReady.size)
+            return [];
+        const units = into.units.map((u) => u.unitId);
+        const resolved = await lt.resolve(m, into.id, { received: true, receiver: m, units });
+        state.close(into.id);
+        return [outcomeOf('resolve', into.id, resolved)];
+    }
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     async accumulate(a) {
         const { lt, state } = this.d;
         const result = await lt.accumulate(a.as, a.rowId, a.itemKey, a.payload, a.metadata);
         this.check(a, result);
+        const added = { ...outcomeOf('accumulate', a.rowId, result), itemKey: a.itemKey };
         if (!a.complete || result.outcome === escalations_1.VERB_OUTCOME.INVALID)
-            return;
+            return [added];
         const held = new Set(result.itemKeys ?? await lt.itemKeys(a.rowId));
         if (!a.complete.keys.every((k) => held.has(k)))
-            return;
+            return [added];
         if (a.complete.then)
-            return this.apply(a.complete.then);
-        this.check(a, await lt.resolve(a.as, a.rowId, a.complete.payload ?? {}));
+            return [added, ...await this.apply(a.complete.then)];
+        const resolved = await lt.resolve(a.as, a.rowId, a.complete.payload ?? {});
+        this.check(a, resolved);
         state.close(a.rowId);
+        return [added, outcomeOf('resolve', a.rowId, resolved)];
     }
     check(a, result) {
         const rowId = a.rowId;
@@ -148,17 +199,19 @@ class Executor {
         }
     }
     async createAlarmRow(a) {
+        this.live();
         const { id } = await this.d.lt.create(actions_1.CONTROLLER, { ...a.row, role: (0, line_1.operatorRole)(this.d.line) });
         this.d.log.info(log_1.GATEWAY_LOG.ALARM_ROW, { row: id, machine: a.machineId, alarm: a.alarmId });
         const track = this.d.state.alarms.get(this.d.state.alarmKey(a.machineId, a.alarmId));
         const raisedAt = a.row.metadata?.raisedAt;
         if (track && (!track.raisedAt || !raisedAt || track.raisedAt === raisedAt)) {
             track.rowId = id;
-            return;
+            return id;
         }
         // Cleared at the machine, or raised again, while the row was being created.
         const note = track ? `${a.machineId} raised ${a.alarmId} again; this one ended` : `cleared at ${a.machineId}`;
         await this.d.lt.resolve(actions_1.CONTROLLER, id, { decision: contracts_1.DECISION.CLEARED, note });
+        return id;
     }
     /** Raise a quiet row for each connected machine silent longer than `quietMs`. */
     checkQuiet(machineIds, quietMs) {
@@ -172,6 +225,7 @@ class Executor {
         }
     }
     async raiseQuiet(machineId, silentMs) {
+        this.live();
         const { id } = await this.d.lt.create(actions_1.CONTROLLER, {
             type: contracts_1.ROW_TYPE.MACHINE_QUIET, role: (0, line_1.operatorRole)(this.d.line), priority: 2,
             description: `${machineId} is connected but has sent nothing for ${Math.round(silentMs / 1000)} s`,
@@ -194,11 +248,12 @@ class Executor {
         link.watching = false;
         if (link.online || link.rowId)
             return;
+        this.live();
         const since = new Date(link.since).toISOString();
         const { id } = await this.d.lt.create(actions_1.CONTROLLER, {
             type: contracts_1.ROW_TYPE.MACHINE_OFFLINE, role: (0, line_1.operatorRole)(this.d.line), priority: 2,
             description: `${machineId} has been offline since ${since}`,
-            metadata: { siteId: this.d.line.siteId, lineId: this.d.line.lineId, machineId, offlineSince: since },
+            metadata: { siteId: this.d.line.siteId, lineId: this.d.line.lineId, machineId, offlineSince: since, ...(link.reason ? { reason: link.reason } : {}) },
         });
         this.d.log.info(log_1.GATEWAY_LOG.OFFLINE_ROW, { row: id, machine: machineId });
         if (!link.online)

package/build/services/machines/gateway/gateway.d.ts

@@ -3,15 +3,19 @@
  * reads escalation events (a new handoff row means "dispatch"), reads every
  * machine message on the line (a receipt means "resolve"), and performs the
  * verbs a person would, as the machine. Neither side knows about the other.
+ * Every message read or sent is recorded on the wire (wire.ts).
  */
 import { type HandoffRowAdapter } from '../rows';
 import { type AddressBook } from './address-book';
 import type { LineDef } from '../../../lib/machines/line';
 import { type LineCodec } from '../../../lib/machines/protocol';
-import { type EscalationFeed, type MachineBus } from '../../../lib/machines/transport';
+import { type EscalationFeed, type InboundMessage, type MachineBus } from '../../../lib/machines/transport';
+import type { InspectResult } from '../../../types/machines-view';
 import type { LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import { GatewayState } from './state';
+import { type WireSink } from './wire';
+import { WireWindow } from './wire-window';
 export interface GatewaySettings {
     topicRoot: string;
     replyTimeoutMs: number;
@@ -27,6 +31,10 @@ export interface GatewaySettings {
     lifecycle?: boolean;
     /** With lifecycle notes, progress is noted each time it crosses this many percent; 0 notes none. */
     progressStep?: number;
+    /** Record telemetry messages on the wire too; off unless set. */
+    wireTelemetry?: boolean;
+    /** How many of the latest wire events the gateway keeps for reading over HTTP; 500 by default. */
+    wireWindow?: number;
 }
 export interface GatewayDeps {
     bus: MachineBus;
@@ -41,6 +49,8 @@ export interface GatewayDeps {
     codec?: LineCodec;
     /** Where a line that learns addresses keeps them; in memory by default. */
     addresses?: AddressBook;
+    /** Where wire records go; Long Tail's events by default. */
+    wire?: WireSink;
 }
 export declare class MachineGateway {
     private readonly deps;
@@ -52,14 +62,24 @@ export declare class MachineGateway {
     private readonly log;
     private readonly addresses;
     private readonly admission;
+    private readonly wire;
+    /** The line's last wire events, newest first on read. */
+    readonly window: WireWindow;
     private quietTimer?;
     constructor(deps: GatewayDeps);
     start(): Promise<void>;
+    /**
+     * Hand the line over cleanly: nothing new is raised or sent, a message
+     * still being handled is nak'd for the next leader if it would make a row,
+     * receipts for what was applied go out, then the sessions close.
+     */
     stop(): Promise<void>;
     /** Machines that report only changes are asked for their full state once the gateway listens. */
     private askEveryMachine;
     /** Resolves when no dispatch is in flight (tests). */
     idle(): Promise<void>;
+    /** A dry run of one message on a copy of the state: nothing is applied (inspect.ts). */
+    inspect(msg: Pick<InboundMessage, 'topic' | 'payload' | 'retained'>): InspectResult;
     /**
      * On taking the line: open handoff and command rows (a dispatch or command
      * interrupted by a restart is sent again with the same cmdId), the gateway's