@hotmeshio/long-tail 0.26.0 → 0.28.0

package/build/services/machines/gateway/inspect.js

@@ -0,0 +1,68 @@
+"use strict";
+/**
+ * A dry run of one wire message: what the codec reads and what the gateway
+ * would do, planned on a copy of the state so the line is untouched and no
+ * row is created or changed.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dryRun = dryRun;
+const plan_machine_1 = require("./plan-machine");
+const registration_1 = require("./registration");
+const wire_1 = require("./wire");
+/** What admitting the sender would do, or null when the message would be acted on. */
+function admission(state, line, raw) {
+    const sender = (0, registration_1.resolveSender)(state, line, raw);
+    if (sender.status === registration_1.SENDER.BOUND)
+        return { seen: sender.seen, actions: [{ verb: 'ack', detail: `bind ${sender.machineId} to ${sender.address}` }] };
+    if (sender.seen)
+        return { seen: sender.seen, actions: [] };
+    if (sender.status === registration_1.SENDER.UNKNOWN)
+        return { seen: null, actions: [{ verb: 'none', detail: `${sender.address} claims no station on this line` }] };
+    const raised = state.claimRaised(sender.address, sender.machineId);
+    return {
+        seen: null,
+        actions: [raised
+                ? { verb: 'none', detail: `${sender.address} says it is ${sender.machineId}; waiting for a person` }
+                : { verb: 'create-row', detail: `${sender.address} says it is ${sender.machineId}; a person decides` }],
+    };
+}
+function planOne(ctx, state, raw) {
+    const admitted = admission(state, ctx.line, raw);
+    if (!admitted.seen)
+        return admitted.actions;
+    if (raw.violations.length && ctx.codec.strict)
+        return [...admitted.actions, { verb: 'hold', detail: raw.violations.join('; ') }];
+    try {
+        const plan = (0, plan_machine_1.planMachineMessage)(state, admitted.seen.parsed, admitted.seen.body, ctx.dispatchTtlMs);
+        const actions = [...admitted.actions, ...plan.actions.map(wire_1.plannedAction)];
+        return (0, wire_1.appliedActions)({ disposition: plan.disposition, failed: false, actions }, 1);
+    }
+    catch (err) {
+        return [...admitted.actions, { verb: 'unreadable', detail: err.message }];
+    }
+}
+function dryRun(ctx, msg) {
+    let decoded;
+    try {
+        decoded = ctx.codec.decode(msg.topic, msg.payload, msg.retained);
+    }
+    catch (err) {
+        return { kind: null, event: null, summary: err.message, inspections: [], violations: [], actions: [{ verb: 'unreadable', detail: err.message }] };
+    }
+    if (!decoded.length) {
+        return { kind: null, event: null, summary: 'not a machine message', inspections: [], violations: [], actions: [{ verb: 'none', detail: 'not a machine message' }] };
+    }
+    const state = ctx.state.clone();
+    const actions = [];
+    const violations = [];
+    for (const raw of decoded) {
+        violations.push(...raw.violations);
+        if (raw.parsed)
+            actions.push(...planOne(ctx, state, raw));
+    }
+    const [first] = decoded;
+    return {
+        kind: first.parsed?.kind ?? null, event: (0, wire_1.eventOf)(first), summary: (0, wire_1.summarize)(first),
+        inspections: decoded, violations, actions: actions.length ? actions : [{ verb: 'unreadable', detail: 'unknown topic' }],
+    };
+}

package/build/services/machines/gateway/lifecycle.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Job lifecycle on the row. A machine's started and progress events, and the
+ * alarms that stop it, are noted as items on the open row that holds the job:
+ * the accepted dispatch into the machine until its units arrive, then the
+ * handoff out of it. Progress is noted only when it crosses a step, or reaches
+ * a named stage it has not reached before, so a row carries a handful of items
+ * while the stream keeps every report.
+ *
+ * Off unless the line turns it on (MACHINE_LIFECYCLE=on). A row parked with a
+ * limit on its items gets no notes, so they never use a slot its units need.
+ */
+import type { Action } from './actions';
+import { type AlarmTrack, type GatewayState } from './state';
+/** The progress step `percent` has reached; null below the first step, at 100, or when steps are off. */
+export declare function progressStepOf(percent: unknown, step: number): number | null;
+/** A started or progress event, noted on the row that holds the job. */
+export declare function noteJobEvent(state: GatewayState, m: string, b: Record<string, any>, at: string | undefined): Action[];
+/** Rows a stopping alarm pauses: those running or leaving with its job, or every row the machine holds. */
+export declare function notePause(state: GatewayState, track: AlarmTrack, b: Record<string, any>): Action[];
+/** The machine cleared an alarm: the rows it paused, still open, note the resume. */
+export declare function noteResume(state: GatewayState, track: AlarmTrack, b: Record<string, any>): Action[];

package/build/services/machines/gateway/lifecycle.js

@@ -0,0 +1,89 @@
+"use strict";
+/**
+ * Job lifecycle on the row. A machine's started and progress events, and the
+ * alarms that stop it, are noted as items on the open row that holds the job:
+ * the accepted dispatch into the machine until its units arrive, then the
+ * handoff out of it. Progress is noted only when it crosses a step, or reaches
+ * a named stage it has not reached before, so a row carries a handful of items
+ * while the stream keeps every report.
+ *
+ * Off unless the line turns it on (MACHINE_LIFECYCLE=on). A row parked with a
+ * limit on its items gets no notes, so they never use a slot its units need.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.progressStepOf = progressStepOf;
+exports.noteJobEvent = noteJobEvent;
+exports.notePause = notePause;
+exports.noteResume = noteResume;
+const contracts_1 = require("../contracts");
+const protocol_1 = require("../../../lib/machines/protocol");
+const jobs_1 = require("./jobs");
+const state_1 = require("./state");
+const owns = (jobId, reported) => !jobId || typeof reported !== 'string' || (0, jobs_1.jobOwns)(jobId, reported);
+const runningAt = (r, m, jobId) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && owns(r.jobId, jobId);
+const leavingFrom = (r, m, jobId) => r.fromMachine === m && !!r.senderJobId && owns(r.senderJobId, jobId);
+/** The open row holding `unitId` at `m`: its accepted dispatch, else the handoff out of `m`. */
+function holderRow(state, m, unitId, jobId) {
+    return state.openFor(unitId, (r) => runningAt(r, m, jobId)) ?? state.openFor(unitId, (r) => leavingFrom(r, m, jobId));
+}
+/** The progress step `percent` has reached; null below the first step, at 100, or when steps are off. */
+function progressStepOf(percent, step) {
+    if (step <= 0 || typeof percent !== 'number' || !Number.isFinite(percent))
+        return null;
+    const reached = Math.floor(percent / step) * step;
+    return reached > 0 && reached < 100 ? reached : null;
+}
+/** An item the row does not hold yet, added as the machine. */
+function note(row, m, itemKey, payload, metadata) {
+    if (row.capped || row.noted?.includes(itemKey))
+        return [];
+    row.noted = [...(row.noted ?? []), itemKey];
+    return [{ kind: 'accumulate', as: m, rowId: row.id, itemKey, payload, ...(metadata ? { metadata } : {}) }];
+}
+/** A started or progress event, noted on the row that holds the job. */
+function noteJobEvent(state, m, b, at) {
+    const unitId = b.unitId;
+    if (!state.lifecycle || typeof unitId !== 'string')
+        return [];
+    const row = holderRow(state, m, unitId, b.jobId);
+    if (!row)
+        return [];
+    const when = at ?? new Date().toISOString();
+    if (b.event === protocol_1.JOB_EVENT.STARTED) {
+        return note(row, m, contracts_1.itemKeys.started(unitId), { unitId, jobId: b.jobId ?? null, at: when });
+    }
+    const step = progressStepOf(b.percent, state.progressStep);
+    const stage = typeof b.stage === 'string' && b.stage ? b.stage : null;
+    if (step === null && stage === null)
+        return [];
+    const progress = {
+        ...(typeof b.percent === 'number' ? { percent: b.percent } : {}), remainingSec: b.remainingSec ?? null, ...(stage ? { stage } : {}), at: when,
+        ...(b.detail && typeof b.detail === 'object' ? { detail: b.detail } : {}),
+    };
+    const facet = { [contracts_1.LIFECYCLE_FACET.PROGRESS]: progress };
+    const job = { unitId, jobId: b.jobId ?? null };
+    return [
+        ...(step === null ? [] : note(row, m, contracts_1.itemKeys.progress(unitId, step), { ...job, step, ...progress }, facet)),
+        ...(stage === null ? [] : note(row, m, contracts_1.itemKeys.stage(unitId, stage), { ...job, ...progress }, facet)),
+    ];
+}
+/** Rows a stopping alarm pauses: those running or leaving with its job, or every row the machine holds. */
+function notePause(state, track, b) {
+    if (!state.lifecycle)
+        return [];
+    const m = track.machineId;
+    const jobId = typeof b.jobId === 'string' ? b.jobId : undefined;
+    const rows = state.open().filter((r) => runningAt(r, m, jobId) || leavingFrom(r, m, jobId));
+    track.noted = rows.map((r) => r.id);
+    const payload = { alarmId: track.alarmId, code: b.code ?? null, level: b.level ?? null, text: b.text ?? null, raisedAt: b.raisedAt ?? null };
+    return rows.flatMap((r) => note(r, m, contracts_1.itemKeys.paused(track.alarmId, String(b.raisedAt ?? '')), payload));
+}
+/** The machine cleared an alarm: the rows it paused, still open, note the resume. */
+function noteResume(state, track, b) {
+    const key = contracts_1.itemKeys.resumed(track.alarmId, String(track.raisedAt ?? ''));
+    const payload = { alarmId: track.alarmId, clearedAt: b.clearedAt ?? new Date().toISOString(), clearedBy: b.clearedBy ?? null };
+    return (track.noted ?? [])
+        .map((id) => state.get(id))
+        .filter((r) => !!r && !r.ended)
+        .flatMap((r) => note(r, track.machineId, key, payload));
+}

package/build/services/machines/gateway/log.d.ts

@@ -1,3 +1,4 @@
+import { publishMachineLogEvent } from '../../../lib/events/publish';
 export declare const GATEWAY_LOG: {
     readonly STARTED: "machine_gateway.started";
     readonly RECONCILED: "machine_gateway.reconciled";
@@ -39,3 +40,10 @@ export interface GatewayLog {
 }
 export declare const consoleLog: GatewayLog;
 export declare const silentLog: GatewayLog;
+/** Each entry published as `machine.log.<site>.<line>.<event>` with its context, for the line's dashboard view. */
+export declare function eventsLog(line: {
+    siteId: string;
+    lineId: string;
+}, publish?: typeof publishMachineLogEvent): GatewayLog;
+/** Every entry goes to each log in turn. */
+export declare function composeLogs(...logs: GatewayLog[]): GatewayLog;

package/build/services/machines/gateway/log.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.silentLog = exports.consoleLog = exports.GATEWAY_LOG = void 0;
+exports.eventsLog = eventsLog;
+exports.composeLogs = composeLogs;
+const publish_1 = require("../../../lib/events/publish");
 exports.GATEWAY_LOG = {
     STARTED: 'machine_gateway.started',
     RECONCILED: 'machine_gateway.reconciled',
@@ -40,3 +43,18 @@ exports.consoleLog = {
     error: (event, context) => console.error(JSON.stringify({ level: 'error', event, ...context })),
 };
 exports.silentLog = { info: () => undefined, warn: () => undefined, error: () => undefined };
+/** Each entry published as `machine.log.<site>.<line>.<event>` with its context, for the line's dashboard view. */
+function eventsLog(line, publish = publish_1.publishMachineLogEvent) {
+    const emit = (level) => (event, context) => {
+        publish({ siteId: line.siteId, lineId: line.lineId, level, event, context }).catch(() => undefined);
+    };
+    return { info: emit('info'), warn: emit('warn'), error: emit('error') };
+}
+/** Every entry goes to each log in turn. */
+function composeLogs(...logs) {
+    const each = (level) => (event, context) => {
+        for (const log of logs)
+            log[level](event, context);
+    };
+    return { info: each('info'), warn: each('warn'), error: each('error') };
+}

package/build/services/machines/gateway/pacing.d.ts

@@ -11,6 +11,20 @@ import { type Action } from './actions';
 import { type GatewayState, type HandoffRow } from './state';
 /** Dispatch arguments for a handoff row. */
 export declare function dispatchArgs(row: HandoffRow, expiresAt: number): Record<string, any>;
+/**
+ * The validity round a dispatch belongs to. Rounds of one validity period count
+ * from when the dispatch became due, and a round begins only when the one
+ * before it has expired, so a machine never holds two valid commands for one
+ * row. Within a round a dispatch sent again, by a retry, a restarted machine or
+ * a restarted gateway, is the same command with the same arguments; a new round
+ * is a new command, never an old one with changed arguments.
+ */
+export declare function dispatchWindow(dueAt: number, now: number, ttlMs: number): {
+    round: number;
+    expiresAt: number;
+};
+/** The later of a row's due time and an arrival's time; an unreadable time leaves it as it was. */
+export declare function latestArrival(dueAt: number | undefined, at: unknown): number | undefined;
 export declare function dispatchAction(row: HandoffRow, machineId: string, dispatchTtlMs: number): Action;
 /** A machine on a version the line does not allow is refused before any dispatch. */
 export declare function softwareRefusal(state: GatewayState, line: LineDef, machineId: string): string | null;

package/build/services/machines/gateway/pacing.js

@@ -9,6 +9,8 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dispatchArgs = dispatchArgs;
+exports.dispatchWindow = dispatchWindow;
+exports.latestArrival = latestArrival;
 exports.dispatchAction = dispatchAction;
 exports.softwareRefusal = softwareRefusal;
 exports.artifactRefusal = artifactRefusal;
@@ -37,11 +39,34 @@ function dispatchArgs(row, expiresAt) {
         expiresAt: (0, protocol_1.isoTs)(expiresAt),
         ...(row.artifactUrl ? { artifact: { url: row.artifactUrl, sha256: row.artifactSha256 } } : {}),
         ...(row.parameters ? { parameters: row.parameters } : {}),
+        ...(row.refs ? { refs: row.refs } : {}),
     };
 }
+/**
+ * The validity round a dispatch belongs to. Rounds of one validity period count
+ * from when the dispatch became due, and a round begins only when the one
+ * before it has expired, so a machine never holds two valid commands for one
+ * row. Within a round a dispatch sent again, by a retry, a restarted machine or
+ * a restarted gateway, is the same command with the same arguments; a new round
+ * is a new command, never an old one with changed arguments.
+ */
+function dispatchWindow(dueAt, now, ttlMs) {
+    const round = Math.max(1, Math.floor((now - dueAt) / ttlMs) + 1);
+    return { round, expiresAt: dueAt + round * ttlMs };
+}
+/** The later of a row's due time and an arrival's time; an unreadable time leaves it as it was. */
+function latestArrival(dueAt, at) {
+    const t = typeof at === 'string' ? Date.parse(at) : NaN;
+    if (!Number.isFinite(t))
+        return dueAt;
+    return dueAt === undefined ? t : Math.max(dueAt, t);
+}
 function dispatchAction(row, machineId, dispatchTtlMs) {
+    const now = Date.now();
+    row.createdAt ??= now;
+    const { round, expiresAt } = dispatchWindow(row.dueAt ?? row.createdAt, now, dispatchTtlMs);
     return {
-        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id), args: dispatchArgs(row, Date.now() + dispatchTtlMs),
+        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt),
         ...(row.dispatch === state_1.DISPATCH_STATE.ARRIVING ? { receivedOnAccept: true } : {}),
     };
 }
@@ -57,7 +82,7 @@ function softwareRefusal(state, line, machineId) {
 function artifactRefusal(line, row) {
     if (!(0, line_1.machineById)(line, row.toMachine).requireArtifact || (row.artifactUrl && row.artifactSha256))
         return null;
-    if (row.units.every((u) => u.artifact))
+    if (row.units.every((u) => u.artifact || u.artifacts?.length))
         return null;
     return `${row.toMachine} requires an artifact (url and sha256), for the job or for every unit; the row has none`;
 }

package/build/services/machines/gateway/plan-escalation.js

@@ -29,9 +29,17 @@ const VERB = { CREATED: 'created', CLAIMED: 'claimed', RESOLVED: 'resolved', CAN
 const STATUS_FOR_VERB = {
     [VERB.CREATED]: 'pending', [VERB.CLAIMED]: 'pending', [VERB.RESOLVED]: 'resolved', [VERB.CANCELLED]: 'cancelled', [VERB.EXPIRED]: 'expired',
 };
+/** The metadata key an accumulator row parked with an item limit carries. */
+const ACCUMULATE_MAX = 'accumulate_max';
 const LINE_ROW_TYPES = [contracts_1.ROW_TYPE.ALARM, contracts_1.ROW_TYPE.MACHINE_OFFLINE, contracts_1.ROW_TYPE.MACHINE_QUIET, contracts_1.ROW_TYPE.MACHINE_COMMAND, contracts_1.ROW_TYPE.MACHINE_REGISTRATION];
-function handoffRow(id, role, facets, metadata) {
-    return { ...facets, id, role, dispatch: state_1.DISPATCH_STATE.NONE, trace: (0, trace_1.readTrace)(metadata) };
+function handoffRow(id, role, facets, data) {
+    const metadata = data?.metadata;
+    const capped = metadata?.[ACCUMULATE_MAX] != null;
+    const createdAt = data?.created_at ? new Date(data.created_at).getTime() : NaN;
+    return {
+        ...facets, id, role, dispatch: state_1.DISPATCH_STATE.NONE, trace: (0, trace_1.readTrace)(metadata),
+        ...(capped ? { capped } : {}), ...(Number.isFinite(createdAt) ? { createdAt } : {}),
+    };
 }
 /** Rows for another line belong to that line's gateway. */
 function onThisLine(line, at) {
@@ -85,8 +93,9 @@ function commandRefusal(state, line, c, role) {
     const own = (0, line_1.machineRole)((0, line_1.machineById)(line, c.machineId));
     if (role !== own)
         return `commands for ${c.machineId} are parked on ${own}, not ${role}`;
-    if (!protocol_1.CONTROL_OPS.includes(c.op))
-        return `${c.op} is not a command a row may ask for; one of ${protocol_1.CONTROL_OPS.join(', ')}`;
+    const allowed = rowOps(state);
+    if (!allowed.includes(c.op))
+        return `${c.op} is not a command a row may ask for; one of ${allowed.join(', ')}`;
     if (c.op === protocol_1.OP.CANCEL && typeof c.args.jobId !== 'string')
         return `${protocol_1.OP.CANCEL} needs args.jobId, the job to withdraw or stop`;
     if (!(0, registration_1.reachable)(state, line, c.machineId))
@@ -102,13 +111,18 @@ function planCommandRow(state, line, event) {
     if (event.verb !== VERB.CREATED && event.verb !== VERB.CLAIMED)
         return [];
     const c = (0, commands_1.readCommandFacets)(event.data?.metadata);
-    if (!c || !state.trackCommand(id))
+    if (!c || !state.trackCommand(id, c.machineId))
         return [];
     const refusal = commandRefusal(state, line, c, event.role);
     if (refusal) {
         return [{ kind: 'resolve', as: actions_1.CONTROLLER, rowId: id, payload: { accepted: false, machineId: c.machineId, op: c.op, status: contracts_1.COMMAND_STATUS.UNSUPPORTED, reason: refusal } }];
     }
-    return [{ kind: 'control', rowId: id, machineId: c.machineId, op: c.op, cmdId: (0, contracts_1.controlCommandId)(id), args: c.args }];
+    const createdAt = event.data?.created_at ? new Date(event.data.created_at).getTime() : NaN;
+    return [{ kind: 'control', rowId: id, machineId: c.machineId, op: c.op, args: c.args, ...(Number.isFinite(createdAt) ? { createdAt } : {}) }];
+}
+/** The ops a machine-command row may ask for: the control ops, a snapshot, and the codec's own. */
+function rowOps(state) {
+    return [...protocol_1.CONTROL_OPS, protocol_1.OP.SNAPSHOT, ...state.rowOps];
 }
 function onLine(line, machineId) {
     return !!machineId && line.machines.some((m) => m.machineId === machineId);
@@ -122,7 +136,7 @@ function planHandoff(state, line, event, facets, dispatchTtlMs) {
     if (event.verb === VERB.CREATED || event.verb === VERB.CLAIMED) {
         if (state.isClosed(id) || !facets)
             return [];
-        state.index(handoffRow(id, event.role, facets, event.data?.metadata));
+        state.index(handoffRow(id, event.role, facets, event.data));
         const row = state.get(id);
         // The units now wait on the sender as this row's sender job, when the sender holds them under it.
         if (row.fromMachine)

package/build/services/machines/gateway/plan-machine.d.ts

@@ -2,10 +2,11 @@
  * Machine messages to gateway actions: the escalation verbs a person would
  * perform, performed for the machine. Each unit's arrival and the sender's
  * finished record accumulate on the handoff row, and the last expected item
- * resolves it. A failure on either side resolves it with an error. Alarms that
- * stop work and machines that go dark or quiet become rows for people. A
- * restarted machine gets back the dispatches it lost. Everything else stays
- * in the stream.
+ * resolves it. A failure on either side resolves it with an error. A job's
+ * start, its progress steps, and the alarms that pause it are noted on the row
+ * that holds it (lifecycle.ts). Alarms that stop work and machines that go
+ * dark or quiet become rows for people. A restarted machine gets back the
+ * dispatches it lost. Everything else stays in the stream.
  *
  * A message whose row does not exist yet is nak'd and arrives again; one whose
  * row already closed is acked, so a replay never resolves anything twice.

package/build/services/machines/gateway/plan-machine.js

@@ -3,10 +3,11 @@
  * Machine messages to gateway actions: the escalation verbs a person would
  * perform, performed for the machine. Each unit's arrival and the sender's
  * finished record accumulate on the handoff row, and the last expected item
- * resolves it. A failure on either side resolves it with an error. Alarms that
- * stop work and machines that go dark or quiet become rows for people. A
- * restarted machine gets back the dispatches it lost. Everything else stays
- * in the stream.
+ * resolves it. A failure on either side resolves it with an error. A job's
+ * start, its progress steps, and the alarms that pause it are noted on the row
+ * that holds it (lifecycle.ts). Alarms that stop work and machines that go
+ * dark or quiet become rows for people. A restarted machine gets back the
+ * dispatches it lost. Everything else stays in the stream.
  *
  * A message whose row does not exist yet is nak'd and arrives again; one whose
  * row already closed is acked, so a replay never resolves anything twice.
@@ -21,6 +22,7 @@ const actions_1 = require("./actions");
 const pacing_1 = require("./pacing");
 const plan_shared_1 = require("./plan-shared");
 const jobs_1 = require("./jobs");
+const lifecycle_1 = require("./lifecycle");
 const trace_1 = require("./trace");
 const state_1 = require("./state");
 const ack = (actions = []) => ({ actions, disposition: transport_1.DISPOSITION.ACK });
@@ -34,7 +36,7 @@ function planMachineMessage(state, parsed, env, dispatchTtlMs) {
 function planByKind(state, m, parsed, env, b, ttl) {
     switch (parsed.kind) {
         case protocol_1.MESSAGE_KIND.REPLY:
-            return ack([{ kind: 'settleReply', cmdId: b.cmdId, status: b.status, reason: b.reason }]);
+            return ack([{ kind: 'settleReply', cmdId: b.cmdId, status: b.status, reason: b.reason, ...(typeof b.code === 'string' ? { code: b.code } : {}) }]);
         case protocol_1.MESSAGE_KIND.JOB:
             return planJob(state, m, b, env?.at, ttl);
         case protocol_1.MESSAGE_KIND.ALARM:
@@ -68,12 +70,29 @@ function ownsJob(jobId, reported) {
     return (0, jobs_1.jobOwns)(jobId, reported);
 }
 /** Keep the receiver's arrival event for the unit, to go out with its dispatch. */
-function withArrival(row, unitId, b) {
+function withArrival(row, unitId, b, at) {
     row.arrivals = { ...row.arrivals, [unitId]: b };
+    row.dueAt = (0, pacing_1.latestArrival)(row.dueAt, at);
     return row;
 }
+/** The unit an open handoff into `m` carries on `carrierId`, for a scan that read only the carrier. */
+function unitOnCarrier(state, m, carrierId) {
+    if (typeof carrierId !== 'string')
+        return undefined;
+    for (const r of state.open()) {
+        if (r.toMachine !== m)
+            continue;
+        const unit = r.units.find((u) => u.carrierId === carrierId);
+        if (unit)
+            return unit.unitId;
+    }
+    return undefined;
+}
 function planJob(state, m, b, at, ttl) {
-    const unitId = b.unitId;
+    const unitId = typeof b.unitId === 'string' ? b.unitId : unitOnCarrier(state, m, b.carrierId);
+    // A carrier no open handoff here carries yet: its row may still be on the way.
+    if (unitId === undefined)
+        return nak;
     // An event from an earlier attempt's job never touches a later attempt's row.
     const toHere = (r) => r.toMachine === m && ownsJob(r.jobId, b.jobId);
     const fromHere = (r) => r.fromMachine === m && ownsJob(r.senderJobId, b.jobId);
@@ -95,13 +114,14 @@ function planJob(state, m, b, at, ttl) {
         if (row.fromMachine)
             state.letGo(row.fromMachine, unitId);
         const units = row.units.map((u) => u.unitId);
+        const arrivedAt = at ?? new Date().toISOString();
         const complete = row.dispatch === state_1.DISPATCH_STATE.ARRIVING
-            ? { keys: units.map(contracts_1.itemKeys.arrived), then: [(0, pacing_1.dispatchAction)(withArrival(row, unitId, b), m, ttl)] }
+            ? { keys: units.map(contracts_1.itemKeys.arrived), then: [(0, pacing_1.dispatchAction)(withArrival(row, unitId, b, arrivedAt), m, ttl)] }
             : { keys: units.map(contracts_1.itemKeys.arrived), payload: { received: true, receiver: m, units } };
         return ack([...early, {
                 kind: 'accumulate', as: m, rowId: row.id, itemKey: contracts_1.itemKeys.arrived(unitId),
                 payload: {
-                    unitId, carrierId: b.carrierId ?? null, from: b.peer ?? null, at: at ?? new Date().toISOString(),
+                    unitId, carrierId: b.carrierId ?? null, from: b.peer ?? null, at: arrivedAt,
                     ...(row.dispatch === state_1.DISPATCH_STATE.ARRIVING ? { arrival: b } : {}),
                 },
                 complete,
@@ -119,7 +139,8 @@ function planJob(state, m, b, at, ttl) {
         if (b.result === protocol_1.JOB_RESULT.OK)
             return ack([released]);
         return ack([released, ...(0, plan_shared_1.failHandoff)(state, row, m, {
-                received: false, unitId, errorCode: contracts_1.HANDOFF_ERROR.SENDER_REJECTED, reason: b.reason ?? null, detail: `${m} finished ${unitId}: ${b.result}${b.reason ? ` (${b.reason})` : ''}`,
+                received: false, unitId, errorCode: contracts_1.HANDOFF_ERROR.SENDER_REJECTED, result: b.result, reason: b.reason ?? null,
+                detail: `${m} finished ${unitId}: ${b.result}${b.reason ? ` (${b.reason})` : ''}`,
             })]);
     }
     if (b.event === protocol_1.JOB_EVENT.TRANSFER_FAILED) {
@@ -140,6 +161,8 @@ function planJob(state, m, b, at, ttl) {
             return ack();
         return ack((0, plan_shared_1.failHandoff)(state, row, m, { received: false, unitId, errorCode: contracts_1.HANDOFF_ERROR.WITHDRAWN, detail: `${m} gave up ${b.jobId}` }));
     }
+    if (b.event === protocol_1.JOB_EVENT.STARTED || b.event === protocol_1.JOB_EVENT.PROGRESS)
+        return ack((0, lifecycle_1.noteJobEvent)(state, m, b, at));
     return ack();
 }
 function releasedAction(row, m, b) {
@@ -160,9 +183,10 @@ function planAlarm(state, parsed, alarmId, env) {
     if (!env || b.active === false) {
         const track = state.alarms.get(key);
         state.alarms.delete(key);
+        const resumed = track ? (0, lifecycle_1.noteResume)(state, track, b) : [];
         if (!track?.rowId || !track.open)
-            return ack();
-        return ack([{
+            return ack(resumed);
+        return ack([...resumed, {
                 kind: 'resolve', as: actions_1.CONTROLLER, rowId: track.rowId,
                 payload: { decision: contracts_1.DECISION.CLEARED, clearedBy: b.clearedBy ?? null, note: `cleared at ${m}${b.clearedBy ? ` by ${b.clearedBy}` : ''}` },
             }]);
@@ -175,8 +199,9 @@ function planAlarm(state, parsed, alarmId, env) {
     const superseded = known?.rowId && known.open
         ? [{ kind: 'resolve', as: actions_1.CONTROLLER, rowId: known.rowId, payload: { decision: contracts_1.DECISION.CLEARED, note: `${m} raised ${alarmId} again; this one ended` } }]
         : [];
-    state.alarms.set(key, { machineId: m, alarmId, raisedAt: b.raisedAt, open: true });
-    return ack([...superseded, {
+    const track = { machineId: m, alarmId, raisedAt: b.raisedAt, level: b.level, open: true };
+    state.alarms.set(key, track);
+    return ack([...superseded, ...(0, lifecycle_1.notePause)(state, track, b), {
             kind: 'createAlarmRow', machineId: m, alarmId,
             row: {
                 type: contracts_1.ROW_TYPE.ALARM, subtype: b.level, role: contracts_1.LINE_ROLE.OPERATOR, priority: b.level === protocol_1.LEVEL.EMERGENCY ? 1 : 2,
@@ -198,10 +223,12 @@ function catalogEnvelope(catalog) {
 }
 function planPresence(state, m, b) {
     const link = state.link(m);
+    const session = typeof b.sessionId === 'string' ? b.sessionId : undefined;
     if (b.online !== true) {
         if (link.online)
             link.since = Date.now();
         link.online = false;
+        link.reason = typeof b.reason === 'string' ? b.reason : undefined;
         if (link.rowId || link.watching)
             return ack();
         link.watching = true;
@@ -210,12 +237,17 @@ function planPresence(state, m, b) {
     if (b.software)
         link.software = b.software;
     const wasOffline = !link.online;
+    // A new session while online: the machine reconnected before its offline presence was heard.
+    const reconnected = !!session && !!link.sessionId && session !== link.sessionId;
     link.online = true;
     link.watching = false;
+    link.reason = undefined;
+    if (session)
+        link.sessionId = session;
     const actions = [];
     // Back from a drop: ask for the full state rather than wait for the next heartbeat.
-    if (wasOffline)
-        actions.push({ kind: 'command', machineId: m, op: protocol_1.OP.SNAPSHOT, cmdId: `${m}:resync:${link.since}`, args: {} });
+    if (wasOffline || reconnected)
+        actions.push({ kind: 'command', machineId: m, op: protocol_1.OP.SNAPSHOT, cmdId: `${m}:resync:${session ?? link.since}`, args: {} });
     if (link.rowId) {
         actions.push({ kind: 'resolve', as: actions_1.CONTROLLER, rowId: link.rowId, payload: { decision: contracts_1.DECISION.RESTORED, note: `${m} back online` } });
         link.rowId = undefined;
@@ -228,15 +260,34 @@ function planSnapshot(state, m, b, ttl) {
     if (b.software)
         link.software = b.software;
     const jobIds = Array.isArray(b.jobs) ? b.jobs.map((j) => j?.jobId).filter((id) => typeof id === 'string') : [];
-    if (Array.isArray(b.jobs))
+    if (Array.isArray(b.jobs)) {
         state.keepHeld(m, new Set(jobIds));
+        link.jobs = b.jobs;
+    }
+    const fresh = !state.retainedAlarms && !state.readingRetained && Array.isArray(b.alarms);
+    const cleared = fresh ? alarmsNoLongerListed(state, m, b.alarms) : [];
     const answer = { kind: 'settleSnapshot', machineId: m, jobIds };
     const previous = link.bootId;
     link.bootId = b.bootId;
     if (!previous || previous === b.bootId)
-        return ack([answer]);
+        return ack([...cleared, answer]);
     const held = new Set(jobIds);
     const missing = (r) => (0, jobs_1.dispatchJobs)(r.jobId, r.units, state.unitsPerDispatch).some((j) => !held.has(j.jobId));
     const lost = state.open().filter((r) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && missing(r));
-    return ack([answer, ...lost.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl))]);
+    return ack([...cleared, answer, ...lost.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl))]);
+}
+/** Alarm rows for `m` whose alarm its snapshot no longer lists: they cleared at the machine. */
+function alarmsNoLongerListed(state, m, listed) {
+    const active = new Set(listed.filter((id) => typeof id === 'string'));
+    const actions = [];
+    for (const [key, track] of [...state.alarms.entries()]) {
+        if (track.machineId !== m || active.has(track.alarmId))
+            continue;
+        state.alarms.delete(key);
+        actions.push(...(0, lifecycle_1.noteResume)(state, track, {}));
+        if (track.open && track.rowId) {
+            actions.push({ kind: 'resolve', as: actions_1.CONTROLLER, rowId: track.rowId, payload: { decision: contracts_1.DECISION.CLEARED, note: `cleared at ${m}; its snapshot no longer lists it` } });
+        }
+    }
+    return actions;
 }

package/build/services/machines/gateway/project.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The line as its leader sees it, for the dashboard: each machine's link,
+ * held units, open rows and alarms from the gateway state, and the claims
+ * waiting for a person. Pure: reads the state and the line, changes nothing.
+ */
+import { type LineDef } from '../../../lib/machines/line';
+import type { LineView } from '../../../types/machines-view';
+import type { GatewayState } from './state';
+export declare function projectLine(state: GatewayState, line: LineDef): LineView;

package/build/services/machines/gateway/project.js

@@ -0,0 +1,55 @@
+"use strict";
+/**
+ * The line as its leader sees it, for the dashboard: each machine's link,
+ * held units, open rows and alarms from the gateway state, and the claims
+ * waiting for a person. Pure: reads the state and the line, changes nothing.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.projectLine = projectLine;
+const line_1 = require("../../../lib/machines/line");
+const protocol_1 = require("../../../lib/machines/protocol");
+const at = (ms) => (ms ? (0, protocol_1.isoTs)(ms) : null);
+function addressOf(state, line, def) {
+    if ((0, line_1.learnsAddress)(line, def.machineId))
+        return state.learnedAddress(def.machineId) ?? null;
+    return def.address ?? def.machineId;
+}
+function alarmsOf(state, machineId) {
+    return [...state.alarms.values()]
+        .filter((track) => track.machineId === machineId)
+        .map((track) => ({ alarmId: track.alarmId, level: track.level ?? '', raisedAt: track.raisedAt ?? '', rowId: track.rowId ?? null }));
+}
+/** The handoff that brought a unit into a machine: open first, then the recently closed one. */
+function rowCarrying(state, machineId, unitId) {
+    const into = (r) => r.toMachine === machineId;
+    return state.openFor(unitId, into) ?? state.closedFor(unitId, into);
+}
+function machineView(state, line, def) {
+    const m = def.machineId;
+    const link = state.links.get(m);
+    const open = state.open();
+    return {
+        machineId: m, process: def.process, role: (0, line_1.machineRole)(def), address: addressOf(state, line, def),
+        online: link ? link.online : null, since: at(link?.since), lastSeen: at(link?.lastSeen),
+        software: link?.software ?? null, bootId: link?.bootId ?? null, sessionId: link?.sessionId ?? null,
+        heldUnits: state.heldUnitsAt(m).map(({ unitId }) => {
+            const row = rowCarrying(state, m, unitId);
+            return { unitId, carrierId: row?.units.find((u) => u.unitId === unitId)?.carrierId ?? null, rowId: row?.id ?? '' };
+        }),
+        rows: {
+            handoff: open.filter((r) => r.toMachine === m).length,
+            command: state.openCommandsFor(m),
+            alarm: [...state.alarms.values()].filter((t) => t.machineId === m && t.open && !!t.rowId).length,
+            offlineRowId: link?.rowId ?? null, quietRowId: link?.quietRowId ?? null,
+        },
+        alarms: alarmsOf(state, m),
+        jobs: link?.jobs ?? [],
+    };
+}
+function projectLine(state, line) {
+    return {
+        siteId: line.siteId, lineId: line.lineId, leading: true, takenAt: at(state.takenAt),
+        machines: line.machines.map((def) => machineView(state, line, def)),
+        contested: state.contested(),
+    };
+}