@hotmeshio/long-tail 0.28.0 → 0.28.2

package/build/services/machines/contracts.js

@@ -6,7 +6,7 @@
  * protocol that resolves it differs.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cancelCommandId = exports.controlCommandId = exports.dispatchCommandId = exports.COMMAND_STATUS = exports.signalKeys = exports.DECISION = exports.LIFECYCLE_FACET = exports.TRACE_FACET = exports.HANDOFF_ERROR = exports.itemKeys = exports.LINE_ROLE = exports.ROW_TYPE = void 0;
+exports.cancelCommandId = exports.controlCommandId = exports.dispatchCommandId = exports.COMMAND_STATUS = exports.signalKeys = exports.HOLD_DECISIONS = exports.DECISION = exports.LIFECYCLE_FACET = exports.TRACE_FACET = exports.HANDOFF_ERROR = exports.itemKeys = exports.LINE_ROLE = exports.ROW_TYPE = void 0;
 exports.arrivedUnit = arrivedUnit;
 exports.handoffProgress = handoffProgress;
 exports.handoffDecisions = handoffDecisions;
@@ -24,12 +24,16 @@ exports.ROW_TYPE = {
     CONTRACT_VIOLATION: 'contract-violation',
     /** An address claims a station the line already knows by another address, or the reverse: a person approves or rejects. */
     MACHINE_REGISTRATION: 'machine-registration',
+    /** Units a machine finished on hold wait for a quality decision: release, rework or scrap. */
+    QUALITY_HOLD: 'quality-hold',
 };
 exports.LINE_ROLE = {
     /** People who clear alarms, bring machines back and decide failed handoffs. */
     OPERATOR: line_1.DEFAULT_OPERATOR_ROLE,
     /** People who take finished units off the line: the final handoff. */
     EXIT: line_1.DEFAULT_EXIT_ROLE,
+    /** People who decide units a machine finished on hold. */
+    QUALITY: line_1.DEFAULT_QUALITY_ROLE,
     /** The gateway's own identity. */
     CONTROLLER: 'line-controller',
 };
@@ -75,6 +79,8 @@ exports.HANDOFF_ERROR = {
     SOFTWARE_UNSUPPORTED: 'software-unsupported',
     WITHDRAWN: 'withdrawn',
     CANCELLED: 'cancelled',
+    /** An arrival no open handoff into the machine expects, shown to the operator role once per machine and carrier. */
+    UNMATCHED_ARRIVAL: 'unmatched-arrival',
     ...protocol_1.TRANSFER_FAILURE,
 };
 /**
@@ -95,6 +101,10 @@ exports.DECISION = {
     RESTORED: 'restored',
     APPROVE: 'approve',
     REJECT: 'reject',
+    /** A quality decision on held units: they continue from where they are. */
+    RELEASE: 'release',
+    /** A quality decision on held units: they run again from a station on the route. */
+    REWORK: 'rework',
 };
 /**
  * Failures a retry cannot repair. Units a sender finished as not ok leave the
@@ -107,11 +117,15 @@ function handoffDecisions(errorCode) {
     const noRetry = !!errorCode && NO_RETRY.includes(errorCode);
     return noRetry ? [exports.DECISION.MANUAL, exports.DECISION.SCRAP] : [exports.DECISION.RETRY, exports.DECISION.MANUAL, exports.DECISION.SCRAP];
 }
+/** The decisions a quality role may take on units a machine finished on hold. */
+exports.HOLD_DECISIONS = [exports.DECISION.RELEASE, exports.DECISION.REWORK, exports.DECISION.SCRAP];
 /** Signal keys for the unit workflow; every attempt mints a fresh key. */
 exports.signalKeys = {
     handoff: (jobKey, step, attempt) => `handoff-${jobKey}-s${step}-a${attempt}`,
     handoffError: (jobKey, step, attempt, round = 1) => `handoff-error-${jobKey}-s${step}-a${attempt}${round > 1 ? `-r${round}` : ''}`,
     jobStop: (jobKey, step, attempt) => `job-stop-${jobKey}-s${step}-a${attempt}`,
+    hold: (jobKey, step, attempt, round = 1) => `hold-${jobKey}-s${step}-a${attempt}${round > 1 ? `-r${round}` : ''}`,
+    disposition: (jobKey, step, attempt) => `disposition-${jobKey}-s${step}-a${attempt}`,
 };
 exports.COMMAND_STATUS = { ...protocol_1.REPLY_STATUS, NO_REPLY: 'no-reply', UNSUPPORTED: 'unsupported' };
 /** A row's dispatch command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */

package/build/services/machines/forms.d.ts

@@ -36,7 +36,7 @@ export declare const OPERATOR_FORM_SCHEMA: {
     properties: {
         decision: {
             type: string;
-            enum: ("reject" | "scrap" | "retry" | "manual" | "acknowledge" | "cleared" | "restored" | "approve")[];
+            enum: ("release" | "reject" | "scrap" | "rework" | "retry" | "manual" | "acknowledge" | "cleared" | "restored" | "approve")[];
             default: "acknowledge";
             description: string;
         };
@@ -53,3 +53,28 @@ export declare const OPERATOR_FORM_SCHEMA: {
         };
     };
 };
+export declare const QUALITY_FORM_SCHEMA: {
+    title: string;
+    description: string;
+    'x-lt-order': string[];
+    required: string[];
+    properties: {
+        decision: {
+            type: string;
+            enum: ("release" | "scrap" | "rework")[];
+            default: "release";
+            description: string;
+        };
+        station: {
+            type: string;
+            default: string;
+            description: string;
+        };
+        note: {
+            type: string;
+            format: string;
+            default: string;
+            description: string;
+        };
+    };
+};

package/build/services/machines/forms.js

@@ -6,7 +6,7 @@
  * handoffs and close alarm and machine-state rows.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.OPERATOR_FORM_SCHEMA = exports.HANDOFF_FORM_SCHEMA = void 0;
+exports.QUALITY_FORM_SCHEMA = exports.OPERATOR_FORM_SCHEMA = exports.HANDOFF_FORM_SCHEMA = void 0;
 const contracts_1 = require("./contracts");
 exports.HANDOFF_FORM_SCHEMA = {
     title: 'Receive units',
@@ -35,3 +35,19 @@ exports.OPERATOR_FORM_SCHEMA = {
         clearedBy: { type: 'string', default: '', description: 'Who cleared it at the machine, when the machine reports it' },
     },
 };
+exports.QUALITY_FORM_SCHEMA = {
+    title: 'Quality decision',
+    description: 'Units a machine finished on hold: release them to continue, send them back to a station for rework, or scrap them.',
+    'x-lt-order': ['decision', 'station', 'note'],
+    required: ['decision'],
+    properties: {
+        decision: {
+            type: 'string',
+            enum: [contracts_1.DECISION.RELEASE, contracts_1.DECISION.REWORK, contracts_1.DECISION.SCRAP],
+            default: contracts_1.DECISION.RELEASE,
+            description: 'Release, rework or scrap',
+        },
+        station: { type: 'string', default: '', description: 'For rework: the station the units run again from; the holding station when empty' },
+        note: { type: 'string', format: 'textarea', default: '', description: 'What was found' },
+    },
+};

package/build/services/machines/gateway/actions.d.ts

@@ -18,14 +18,19 @@ export interface CreateRowInput {
     escalation_payload?: string;
 }
 export type Action = 
-/** `receivedOnAccept`: the units are already at the machine, so its acceptance completes the handoff. */
+/**
+ * `receivedOnAccept`: the units are already at the machine, so its acceptance completes the handoff.
+ * `ended`: the row already resolved; the job is sent again for units the machine still holds, and no verb touches the row.
+ */
 {
     kind: 'dispatch';
     rowId: string;
     machineId: string;
     cmdId: string;
     args: Record<string, any>;
+    issuedAt?: number;
     receivedOnAccept?: boolean;
+    ended?: boolean;
 } | {
     kind: 'command';
     machineId: string;
@@ -79,6 +84,7 @@ export type Action =
     status: string;
     reason: string;
     code?: string;
+    preparing?: boolean;
 }
 /** A machine reported the jobs it holds: answers a status query in flight. */
  | {
@@ -99,6 +105,12 @@ export type Action =
     machineId: string;
     address: string;
     reason: string;
+}
+/** Jobs a restarted machine lost that this gateway cannot send again: nothing of them is remembered. */
+ | {
+    kind: 'unrecoverable';
+    machineId: string;
+    jobs: string[];
 };
 /** Once the row holds every one of `keys`: resolve it with `payload`, or run `then` instead. */
 export interface Completion {

package/build/services/machines/gateway/admission.d.ts

@@ -4,8 +4,9 @@
  * and under a strict codec a message that breaks the contract is held for a
  * person instead of acted on.
  */
+import type { OpenRowQuery } from '../rows';
 import { type LineDef } from '../../../lib/machines/line';
-import type { Inspection } from '../../../lib/machines/protocol';
+import { type Inspection } from '../../../lib/machines/protocol';
 import { type Disposition, type InboundMessage } from '../../../lib/machines/transport';
 import type { WireAction } from '../../../types/machines-view';
 import type { Action } from './actions';
@@ -36,6 +37,8 @@ export interface AdmissionContext {
     log: GatewayLog;
     /** Send waiting handoffs that may now go out. */
     pace(): Promise<void>;
+    /** The controller's pending rows of a type, for the rows shown to people once. */
+    openRows(query: OpenRowQuery): Promise<Record<string, any>[]>;
 }
 /**
  * The station a message is from (registration.ts). A newly claimed station is
@@ -45,3 +48,14 @@ export interface AdmissionContext {
 export declare function admitSender(ctx: AdmissionContext, topic: string, raw: Inspection): Promise<Admitted | Applied>;
 /** Under a strict codec a broken message is not acted on: a person sees it on a row, once. */
 export declare function holdViolation(ctx: AdmissionContext, msg: InboundMessage, seen: Inspection): Promise<Applied>;
+/** A job arrival: a scan, or a transfer that failed on the carrier read. */
+export declare function isArrival(seen: Inspection): boolean;
+/** An arrival that read no id at all: no row can ever claim it, so it goes to a person on its first delivery. */
+export declare function isUnreadableArrival(seen: Inspection): boolean;
+/**
+ * An arrival no open handoff into the machine expects, after every delivery:
+ * a person sees it on one handoff-error row per machine and carrier, with the
+ * raw arrival, so the unit can be re-scanned or routed. Resolving the row lets
+ * the next such arrival raise one again.
+ */
+export declare function holdArrival(ctx: AdmissionContext, msg: InboundMessage, seen: Inspection): Promise<Applied>;

package/build/services/machines/gateway/admission.js

@@ -8,8 +8,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.admitSender = admitSender;
 exports.holdViolation = holdViolation;
+exports.isArrival = isArrival;
+exports.isUnreadableArrival = isUnreadableArrival;
+exports.holdArrival = holdArrival;
 const contracts_1 = require("../contracts");
 const line_1 = require("../../../lib/machines/line");
+const protocol_1 = require("../../../lib/machines/protocol");
 const transport_1 = require("../../../lib/machines/transport");
 const dispatcher_1 = require("./dispatcher");
 const log_1 = require("./log");
@@ -79,3 +83,46 @@ async function holdViolation(ctx, msg, seen) {
         return { disposition: transport_1.DISPOSITION.NAK, failed: true, actions: [] };
     }
 }
+/** A job arrival: a scan, or a transfer that failed on the carrier read. */
+function isArrival(seen) {
+    const event = seen.body?.body?.event;
+    return seen.parsed?.kind === protocol_1.MESSAGE_KIND.JOB && (event === protocol_1.JOB_EVENT.ARRIVED || event === protocol_1.JOB_EVENT.TRANSFER_FAILED);
+}
+/** An arrival that read no id at all: no row can ever claim it, so it goes to a person on its first delivery. */
+function isUnreadableArrival(seen) {
+    return seen.parsed?.kind === protocol_1.MESSAGE_KIND.JOB && (0, protocol_1.isUnreadableRead)(seen.body?.body ?? {});
+}
+/**
+ * An arrival no open handoff into the machine expects, after every delivery:
+ * a person sees it on one handoff-error row per machine and carrier, with the
+ * raw arrival, so the unit can be re-scanned or routed. Resolving the row lets
+ * the next such arrival raise one again.
+ */
+async function holdArrival(ctx, msg, seen) {
+    const { ident } = seen.parsed;
+    const body = seen.body?.body ?? {};
+    const carrierId = typeof body.carrierId === 'string' ? body.carrierId : null;
+    const unitId = typeof body.unitId === 'string' ? body.unitId : null;
+    const detail = `no handoff into ${ident.machineId} expects ${carrierId ? `carrier ${carrierId}` : 'an unreadable carrier'}`;
+    try {
+        const open = await ctx.openRows({ type: contracts_1.ROW_TYPE.HANDOFF_ERROR, role: (0, line_1.operatorRole)(ctx.line) });
+        const shown = open.find((r) => r.metadata?.errorCode === contracts_1.HANDOFF_ERROR.UNMATCHED_ARRIVAL && r.metadata?.machineId === ident.machineId && (r.metadata?.carrierId ?? null) === carrierId);
+        if (shown)
+            return { disposition: transport_1.DISPOSITION.ACK, failed: false, actions: [{ verb: 'unmatched', rowId: shown.id, detail: `${detail} (already shown to a person)` }] };
+        const [made] = await ctx.executor.apply([{ kind: 'createRow', row: {
+                    type: contracts_1.ROW_TYPE.HANDOFF_ERROR, role: (0, line_1.operatorRole)(ctx.line), priority: 2,
+                    description: `${ident.machineId} reports an arrival no handoff expects: ${carrierId ? `carrier ${carrierId}` : 'the carrier could not be read'}`,
+                    metadata: {
+                        siteId: ident.siteId, lineId: ident.lineId, machineId: ident.machineId, errorCode: contracts_1.HANDOFF_ERROR.UNMATCHED_ARRIVAL,
+                        carrierId, unitId, event: body.event, topic: msg.topic, messageId: seen.body?.id ?? null, arrival: body,
+                    },
+                    escalation_payload: msg.payload,
+                } }]);
+        return { disposition: transport_1.DISPOSITION.ACK, failed: false, actions: [{ verb: 'create-row', rowId: made?.rowId, detail }] };
+    }
+    catch (err) {
+        if (!(err instanceof dispatcher_1.GatewayStopped))
+            ctx.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { topic: msg.topic, error: err.message });
+        return { disposition: transport_1.DISPOSITION.NAK, failed: true, actions: [] };
+    }
+}

package/build/services/machines/gateway/api-escalations.js

@@ -42,6 +42,8 @@ const escalationApi = __importStar(require("../../../api/escalations"));
 const uuid_1 = require("../../../lib/uuid");
 const escalations_1 = require("./escalations");
 const actions_1 = require("./actions");
+/** A row that was never given items (one parked by hand) answers 400; it holds none. */
+const NO_ITEMS_STATUS = 400;
 class ApiLineEscalations {
     constructor(userFor) {
         this.userFor = userFor;
@@ -75,6 +77,8 @@ class ApiLineEscalations {
     }
     async items(id) {
         const result = await escalationApi.getEscalationItems({ id }, this.auth(actions_1.CONTROLLER));
+        if (result.status === NO_ITEMS_STATUS)
+            return [];
         if (result.status >= 300)
             throw new Error(`items failed ${result.status}: ${result.error}`);
         return result.data.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));

package/build/services/machines/gateway/dispatcher.d.ts

@@ -52,12 +52,15 @@ export declare class Dispatcher {
     /** Work still running sends nothing more and changes no row. */
     stop(): void;
     private live;
-    /** This send's attempt number, and whether the machine may have the command already. */
+    /**
+     * This send's attempt number, whether the machine may have the command already, and when it was
+     * issued: the time given (a round's start, the same for every gateway), else the first send's.
+     */
     private countSend;
     /** False when the codec has no message for the command. `since` is when the command's row was created. */
-    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<boolean>;
+    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number, issuedAt?: number): Promise<boolean>;
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<CommandReply | null>;
+    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number, issuedAt?: number): Promise<CommandReply | null>;
     command(a: CommandAction): Promise<void>;
     /**
      * A machine-command row: send it, then resolve the row with the reply, as the
@@ -68,7 +71,9 @@ export declare class Dispatcher {
      */
     control(a: ControlAction): Promise<void>;
     dispatch(a: DispatchAction): Promise<void>;
-    /** The units were already at the machine: its acceptance completes the handoff. */
+    /** A job sent again after the machine restarted, for a row that already ended: the reply is only logged. */
+    private resent;
+    /** The units were already at the machine: a job it has taken and is ready for completes the handoff. */
     private received;
     /** Ask a silent machine what it holds; null when it cannot say or does not answer in time. */
     private heldJobs;

package/build/services/machines/gateway/dispatcher.js

@@ -57,20 +57,25 @@ class Dispatcher {
         if (this.stopping.signal.aborted)
             throw new GatewayStopped();
     }
-    /** This send's attempt number, and whether the machine may have the command already. */
-    countSend(cmdId, since) {
-        const attempt = (this.sent.get(cmdId) ?? 0) + 1;
+    /**
+     * This send's attempt number, whether the machine may have the command already, and when it was
+     * issued: the time given (a round's start, the same for every gateway), else the first send's.
+     */
+    countSend(cmdId, since, issuedAt) {
+        const before = this.sent.get(cmdId);
+        const record = { attempt: (before?.attempt ?? 0) + 1, issuedAt: issuedAt ?? before?.issuedAt ?? Date.now() };
         this.sent.delete(cmdId);
-        this.sent.set(cmdId, attempt);
+        this.sent.set(cmdId, record);
         if (this.sent.size > SENT_REMEMBERED)
             this.sent.delete(this.sent.keys().next().value);
-        return { attempt, retry: attempt > 1 || (since !== undefined && since < this.d.state.takenAt) };
+        const retry = record.attempt > 1 || (since !== undefined && since < this.d.state.takenAt);
+        return { attempt: record.attempt, retry, issuedAt: (0, protocol_1.isoTs)(record.issuedAt) };
     }
     /** False when the codec has no message for the command. `since` is when the command's row was created. */
-    async publish(machineId, op, cmdId, args, since) {
+    async publish(machineId, op, cmdId, args, since, issuedAt) {
         this.live();
         const ident = (0, registration_1.identFor)(this.d.state, this.d.line, machineId);
-        const command = { cmdId, op, args, ...this.countSend(cmdId, since) };
+        const command = { cmdId, op, args, ...this.countSend(cmdId, since, issuedAt) };
         const wires = this.d.codec.encode(ident, command);
         for (const wire of wires) {
             await this.d.bus.publish(wire.topic, wire.payload);
@@ -79,11 +84,11 @@ class Dispatcher {
         return wires.length > 0;
     }
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    async ask(machineId, op, cmdId, args, since) {
+    async ask(machineId, op, cmdId, args, since, issuedAt) {
         for (let attempt = 1; attempt <= this.d.dispatchAttempts; attempt++) {
             this.live();
             const waiting = this.d.replies.wait(cmdId, this.d.replyTimeoutMs);
-            if (!await this.publish(machineId, op, cmdId, args, since)) {
+            if (!await this.publish(machineId, op, cmdId, args, since, issuedAt)) {
                 this.d.replies.settle({ cmdId, status: contracts_1.COMMAND_STATUS.UNSUPPORTED, reason: `the line's codec has no ${op} command` });
             }
             const reply = await waiting;
@@ -112,8 +117,9 @@ class Dispatcher {
      */
     async control(a) {
         const now = Date.now();
-        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, this.d.replyTimeoutMs * this.d.dispatchAttempts);
-        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt);
+        const ttlMs = this.d.replyTimeoutMs * this.d.dispatchAttempts;
+        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, ttlMs);
+        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt, expiresAt - ttlMs);
         // Timers may fire a little early; the command must have expired before the row says no-reply.
         if (!reply && Date.now() <= expiresAt)
             await (0, promises_1.setTimeout)(expiresAt - Date.now() + 1, undefined, { signal: this.stopping.signal }).catch(() => undefined);
@@ -142,22 +148,36 @@ class Dispatcher {
         for (const job of jobs)
             state.rememberDispatch(`${a.cmdId}${job.suffix}`, a.rowId);
         const since = state.get(a.rowId)?.createdAt;
-        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since)));
+        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since, a.issuedAt)));
         const refused = replies.find((r) => r && !took(r));
         const silent = jobs.filter((_, i) => !replies[i]);
         const held = !refused && silent.length ? await this.heldJobs(a.machineId, a.cmdId) : null;
         this.live();
         const reconciled = !!held && silent.every((j) => held.has(j.jobId));
+        if (a.ended) {
+            this.resent(a, jobs, refused ?? null, !refused && (!silent.length || reconciled));
+            return;
+        }
         const row = state.get(a.rowId);
         if (!refused && (!silent.length || reconciled)) {
             if (row)
                 row.dispatch = state_1.DISPATCH_STATE.ACCEPTED;
+            for (const job of jobs)
+                state.rememberSent(a.machineId, job.jobId, a.rowId);
             if (reconciled)
                 log.info(log_1.GATEWAY_LOG.DISPATCH_RECONCILED, { row: a.rowId, machine: a.machineId, jobs: silent.map((j) => j.jobId) });
             log.info(log_1.GATEWAY_LOG.DISPATCHED, { row: a.rowId, machine: a.machineId, reason: replies.map((r) => r?.reason ?? 'held').join('; ') });
             this.check(a.rowId, 'claim', await lt.claim(a.machineId, a.rowId));
-            if (a.receivedOnAccept)
+            if (!a.receivedOnAccept)
+                return;
+            const preparing = jobs.filter((job, i) => replies[i]?.status === protocol_1.REPLY_STATUS.ACCEPTED && replies[i]?.preparing).map((job) => `${a.cmdId}${job.suffix}`);
+            if (!preparing.length) {
                 await this.received(a);
+                return;
+            }
+            if (row)
+                row.awaitingReady = new Set(preparing);
+            log.info(log_1.GATEWAY_LOG.DISPATCH_PREPARING, { row: a.rowId, machine: a.machineId, jobs: preparing });
             return;
         }
         // The handoff may have ended meanwhile: a machine that ran the job anyway reports the arrival.
@@ -176,7 +196,15 @@ class Dispatcher {
             return;
         await this.withdraw(a, jobs.filter((_, i) => !replies[i] || took(replies[i])));
     }
-    /** The units were already at the machine: its acceptance completes the handoff. */
+    /** A job sent again after the machine restarted, for a row that already ended: the reply is only logged. */
+    resent(a, jobs, refused, taken) {
+        const context = { row: a.rowId, machine: a.machineId, jobs: jobs.map((j) => j.jobId), resent: true };
+        if (taken)
+            this.d.log.info(log_1.GATEWAY_LOG.DISPATCHED, { ...context, reason: 'sent again after a restart' });
+        else
+            this.d.log.warn(log_1.GATEWAY_LOG.DISPATCH_REFUSED, { ...context, reason: refused ? `${refused.status}: ${refused.reason}` : 'no reply' });
+    }
+    /** The units were already at the machine: a job it has taken and is ready for completes the handoff. */
     async received(a) {
         this.live();
         const units = a.args.units.map((u) => u.unitId);

package/build/services/machines/gateway/executor.d.ts

@@ -57,6 +57,8 @@ export declare class Executor {
      * carrying the units out of it.
      */
     private lateReply;
+    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    private readyForArrival;
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     private accumulate;
     private check;

package/build/services/machines/gateway/executor.js

@@ -109,7 +109,7 @@ class Executor {
                 this.watchOffline(a.machineId);
                 return [(0, wire_1.plannedAction)(a)];
             case 'settleReply':
-                if (replies.settle({ cmdId: a.cmdId, status: a.status, reason: a.reason, ...(a.code ? { code: a.code } : {}) }))
+                if (replies.settle({ cmdId: a.cmdId, status: a.status, reason: a.reason, ...(a.code ? { code: a.code } : {}), ...(a.preparing ? { preparing: true } : {}) }))
                     return [(0, wire_1.plannedAction)(a)];
                 return [{ verb: 'ack', detail: `late ${a.status} ${a.cmdId}` }, ...await this.lateReply(a)];
             case 'settleSnapshot':
@@ -123,6 +123,9 @@ class Executor {
             case 'refuseBinding':
                 this.d.log.warn(log_1.GATEWAY_LOG.ADDRESS_REFUSED, { machine: a.machineId, address: a.address, reason: a.reason });
                 return [(0, wire_1.plannedAction)(a)];
+            case 'unrecoverable':
+                this.d.log.warn(log_1.GATEWAY_LOG.DISPATCH_UNRECOVERABLE, { machine: a.machineId, jobs: a.jobs });
+                return [(0, wire_1.plannedAction)(a)];
         }
     }
     /**
@@ -140,7 +143,7 @@ class Executor {
         const context = { row: into.id, machine: m, cmdId: a.cmdId, reason: a.reason };
         if (a.status === protocol_1.REPLY_STATUS.COMPLETED) {
             log.info(log_1.GATEWAY_LOG.DISPATCH_READY, context);
-            return [];
+            return this.readyForArrival(into, m, a.cmdId);
         }
         if (a.status !== protocol_1.REPLY_STATUS.FAILED)
             return [];
@@ -152,6 +155,19 @@ class Executor {
             received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_FAILED, detail: `${m} could not prepare the job: ${a.reason}`, ...(a.code ? { code: a.code } : {}),
         }));
     }
+    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    async readyForArrival(into, m, cmdId) {
+        const { lt, state } = this.d;
+        if (into.ended || !into.awaitingReady?.has(cmdId))
+            return [];
+        into.awaitingReady.delete(cmdId);
+        if (into.awaitingReady.size)
+            return [];
+        const units = into.units.map((u) => u.unitId);
+        const resolved = await lt.resolve(m, into.id, { received: true, receiver: m, units });
+        state.close(into.id);
+        return [outcomeOf('resolve', into.id, resolved)];
+    }
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     async accumulate(a) {
         const { lt, state } = this.d;

package/build/services/machines/gateway/gateway.d.ts

@@ -15,6 +15,7 @@ import type { LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import { GatewayState } from './state';
 import { type WireSink } from './wire';
+import { WireWindow } from './wire-window';
 export interface GatewaySettings {
     topicRoot: string;
     replyTimeoutMs: number;
@@ -32,6 +33,8 @@ export interface GatewaySettings {
     progressStep?: number;
     /** Record telemetry messages on the wire too; off unless set. */
     wireTelemetry?: boolean;
+    /** How many of the latest wire events the gateway keeps for reading over HTTP; 500 by default. */
+    wireWindow?: number;
 }
 export interface GatewayDeps {
     bus: MachineBus;
@@ -60,6 +63,8 @@ export declare class MachineGateway {
     private readonly addresses;
     private readonly admission;
     private readonly wire;
+    /** The line's last wire events, newest first on read. */
+    readonly window: WireWindow;
     private quietTimer?;
     constructor(deps: GatewayDeps);
     start(): Promise<void>;

package/build/services/machines/gateway/gateway.js

@@ -28,6 +28,7 @@ const plan_machine_1 = require("./plan-machine");
 const state_1 = require("./state");
 const receipts_1 = require("./receipts");
 const wire_1 = require("./wire");
+const wire_window_1 = require("./wire-window");
 class MachineGateway {
     constructor(deps) {
         this.deps = deps;
@@ -42,14 +43,16 @@ class MachineGateway {
         this.state.retainedAlarms = this.codec.retainedAlarms === true;
         this.state.progressStep = deps.settings.progressStep ?? this.state.progressStep;
         this.addresses = deps.addresses ?? new address_book_1.MemoryAddressBook();
-        this.wire = new wire_1.WireRecorder(deps.line, deps.wire ?? publish_1.publishMachineWireEvent, { telemetry: deps.settings.wireTelemetry === true });
+        this.window = new wire_window_1.WireWindow(deps.settings.wireWindow);
+        const sink = deps.wire ?? publish_1.publishMachineWireEvent;
+        this.wire = new wire_1.WireRecorder(deps.line, (event) => { this.window.push(event); sink(event); }, { telemetry: deps.settings.wireTelemetry === true });
         this.executor = new executor_1.Executor({
             addresses: this.addresses, wire: this.wire,
             bus: deps.bus, lt: deps.lt, replies: new reply_waiter_1.ReplyWaiter(), state: this.state, line: deps.line,
             codec: this.codec, replyTimeoutMs: deps.settings.replyTimeoutMs,
             dispatchAttempts: deps.settings.dispatchAttempts, offlineDebounceMs: deps.settings.offlineDebounceMs, log: this.log,
         });
-        this.admission = { state: this.state, line: deps.line, executor: this.executor, log: this.log, pace: () => this.pace() };
+        this.admission = { state: this.state, line: deps.line, executor: this.executor, log: this.log, pace: () => this.pace(), openRows: (q) => deps.lt.listOpen(q) };
         this.receipts = new receipts_1.ReceiptBatcher({ bus: deps.bus, codec: this.codec, delayMs: deps.settings.receiptDelayMs ?? DEFAULT_RECEIPT_DELAY_MS, log: this.log, wire: this.wire });
     }
     async start() {
@@ -231,9 +234,14 @@ class MachineGateway {
             if (seen.violations.length && msg.deliveries === 1) {
                 this.log.warn(log_1.GATEWAY_LOG.CONTRACT_VIOLATION, { topic: msg.topic, violations: seen.violations });
             }
-            const applied = seen.violations.length && this.codec.strict
+            let applied = seen.violations.length && this.codec.strict
                 ? await (0, admission_1.holdViolation)(this.admission, msg, seen)
                 : await this.applyMachine(msg.topic, seen.parsed, seen.body);
+            // An arrival still waiting for its row after every delivery is shown to a person instead; one that read no id at once.
+            const waited = msg.deliveries >= wire_1.MAX_DELIVERIES || (0, admission_1.isUnreadableArrival)(seen);
+            if (applied.disposition === transport_1.DISPOSITION.NAK && !applied.failed && waited && (0, admission_1.isArrival)(seen)) {
+                applied = await (0, admission_1.holdArrival)(this.admission, msg, seen);
+            }
             failed ||= applied.failed;
             outcome = worse(outcome, applied.disposition);
             this.wire.inbound(msg, seen, { ...applied, actions: [...admitted.actions, ...applied.actions] }, msg.deliveries);

package/build/services/machines/gateway/log.d.ts

@@ -5,8 +5,10 @@ export declare const GATEWAY_LOG: {
     readonly DISPATCHED: "machine_gateway.dispatch.accepted";
     readonly DISPATCH_REFUSED: "machine_gateway.dispatch.refused";
     readonly DISPATCH_READY: "machine_gateway.dispatch.ready";
+    readonly DISPATCH_PREPARING: "machine_gateway.dispatch.preparing";
     readonly DISPATCH_FAILED: "machine_gateway.dispatch.failed";
     readonly DISPATCH_RECONCILED: "machine_gateway.dispatch.reconciled";
+    readonly DISPATCH_UNRECOVERABLE: "machine_gateway.dispatch.unrecoverable";
     readonly VERB_INVALID: "machine_gateway.verb.invalid";
     readonly VERB_FAILED: "machine_gateway.verb.failed";
     readonly ROW_NOT_VISIBLE: "machine_gateway.verb.row_not_visible";
@@ -30,6 +32,7 @@ export declare const GATEWAY_LOG: {
     readonly LEADING: "machine_gateway.line.leading";
     readonly STANDBY: "machine_gateway.line.standby";
     readonly LEADERSHIP_LOST: "machine_gateway.line.leadership_lost";
+    readonly WIRE_LOST: "machine_gateway.line.wire_lost";
     readonly START_FAILED: "machine_gateway.line.start_failed";
 };
 export type GatewayLogEvent = typeof GATEWAY_LOG[keyof typeof GATEWAY_LOG];

package/build/services/machines/gateway/log.js

@@ -10,8 +10,10 @@ exports.GATEWAY_LOG = {
     DISPATCHED: 'machine_gateway.dispatch.accepted',
     DISPATCH_REFUSED: 'machine_gateway.dispatch.refused',
     DISPATCH_READY: 'machine_gateway.dispatch.ready',
+    DISPATCH_PREPARING: 'machine_gateway.dispatch.preparing',
     DISPATCH_FAILED: 'machine_gateway.dispatch.failed',
     DISPATCH_RECONCILED: 'machine_gateway.dispatch.reconciled',
+    DISPATCH_UNRECOVERABLE: 'machine_gateway.dispatch.unrecoverable',
     VERB_INVALID: 'machine_gateway.verb.invalid',
     VERB_FAILED: 'machine_gateway.verb.failed',
     ROW_NOT_VISIBLE: 'machine_gateway.verb.row_not_visible',
@@ -35,6 +37,7 @@ exports.GATEWAY_LOG = {
     LEADING: 'machine_gateway.line.leading',
     STANDBY: 'machine_gateway.line.standby',
     LEADERSHIP_LOST: 'machine_gateway.line.leadership_lost',
+    WIRE_LOST: 'machine_gateway.line.wire_lost',
     START_FAILED: 'machine_gateway.line.start_failed',
 };
 exports.consoleLog = {

package/build/services/machines/gateway/lt-http.js

@@ -56,6 +56,8 @@ class LtHttp {
     }
 }
 exports.LtHttp = LtHttp;
+/** A row that was never given items (one parked by hand) answers 400; it holds none. */
+const NO_ITEMS_STATUS = 400;
 /** Escalation verbs over HTTP, each run with the bearer token of the actor it runs as. */
 class HttpLineEscalations {
     constructor(http, tokenFor, claimMinutes = 24 * 60) {
@@ -80,8 +82,15 @@ class HttpLineEscalations {
         return (await this.items(id)).map((i) => i.itemKey);
     }
     async items(id) {
-        const view = await this.http.request('GET', `/escalations/${id}/items`, this.tokenFor(actions_1.CONTROLLER));
-        return view.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));
+        try {
+            const view = await this.http.request('GET', `/escalations/${id}/items`, this.tokenFor(actions_1.CONTROLLER));
+            return view.items.map((i) => ({ itemKey: i.itemKey, payload: i.payload ?? {} }));
+        }
+        catch (err) {
+            if (err instanceof LtHttpError && err.status === NO_ITEMS_STATUS)
+                return [];
+            throw err;
+        }
     }
     async get(id) {
         if (!(0, uuid_1.isUuid)(id))

package/build/services/machines/gateway/pacing.js

@@ -66,7 +66,7 @@ function dispatchAction(row, machineId, dispatchTtlMs) {
     row.createdAt ??= now;
     const { round, expiresAt } = dispatchWindow(row.dueAt ?? row.createdAt, now, dispatchTtlMs);
     return {
-        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt),
+        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt), issuedAt: expiresAt - dispatchTtlMs,
         ...(row.dispatch === state_1.DISPATCH_STATE.ARRIVING ? { receivedOnAccept: true } : {}),
     };
 }