@hotmeshio/long-tail 0.26.0 → 0.27.0

package/build/services/machines/gateway/executor.js

@@ -15,6 +15,7 @@ const escalations_1 = require("./escalations");
 const log_1 = require("./log");
 const dispatcher_1 = require("./dispatcher");
 const plan_shared_1 = require("./plan-shared");
+const state_1 = require("./state");
 const NOT_FOUND = 404;
 const CONFLICT = 409;
 class Executor {
@@ -47,10 +48,14 @@ class Executor {
         for (const t of this.timers)
             clearTimeout(t);
         this.timers.clear();
+        this.dispatcher.stop();
         this.d.replies.cancelAll();
     }
     track(work) {
-        const tracked = work.catch((err) => this.d.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { error: String(err?.message ?? err) }))
+        const tracked = work.catch((err) => {
+            if (!(err instanceof dispatcher_1.GatewayStopped))
+                this.d.log.error(log_1.GATEWAY_LOG.VERB_FAILED, { error: String(err?.message ?? err) });
+        })
             .finally(() => this.inflight.delete(tracked));
         this.inflight.add(tracked);
     }
@@ -90,19 +95,29 @@ class Executor {
                 this.d.log.warn(log_1.GATEWAY_LOG.ADDRESS_REFUSED, { machine: a.machineId, address: a.address, reason: a.reason });
         }
     }
-    /** A dispatch's second reply: ready, or a failure to prepare that ends the handoff. */
+    /**
+     * A dispatch's second reply: ready, or a failure to prepare that ends the
+     * handoff. When the handoff into the machine has already ended, as one
+     * dispatched on arrival does at acceptance, the failure ends the handoff
+     * carrying the units out of it.
+     */
     async lateReply(a) {
-        const row = this.d.state.rowForDispatch(a.cmdId);
-        if (!row?.toMachine)
+        const { state, log } = this.d;
+        const into = state.rowForDispatch(a.cmdId);
+        const m = into?.toMachine;
+        if (!into || !m)
             return;
-        const context = { row: row.id, machine: row.toMachine, cmdId: a.cmdId, reason: a.reason };
+        const context = { row: into.id, machine: m, cmdId: a.cmdId, reason: a.reason };
         if (a.status === protocol_1.REPLY_STATUS.COMPLETED)
-            return this.d.log.info(log_1.GATEWAY_LOG.DISPATCH_READY, context);
+            return log.info(log_1.GATEWAY_LOG.DISPATCH_READY, context);
         if (a.status !== protocol_1.REPLY_STATUS.FAILED)
             return;
-        this.d.log.warn(log_1.GATEWAY_LOG.DISPATCH_FAILED, context);
-        await this.apply((0, plan_shared_1.failHandoff)(this.d.state, row, row.toMachine, {
-            received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_FAILED, detail: `${row.toMachine} could not prepare the job: ${a.reason}`,
+        const row = into.ended ? onwardRow(state, into, m) : into;
+        log.warn(log_1.GATEWAY_LOG.DISPATCH_FAILED, { ...context, ...(into.ended ? { onward: row?.id ?? null } : {}) });
+        if (!row)
+            return;
+        await this.apply((0, plan_shared_1.failHandoff)(state, row, m, {
+            received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_FAILED, detail: `${m} could not prepare the job: ${a.reason}`,
         }));
     }
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
@@ -193,3 +208,14 @@ class Executor {
     }
 }
 exports.Executor = Executor;
+/** The open handoff out of `m` for units a closed handoff brought into it. */
+function onwardRow(state, into, m) {
+    if (into.ended !== state_1.ROW_END.RESOLVED || into.dispatch !== state_1.DISPATCH_STATE.ACCEPTED)
+        return undefined;
+    for (const u of into.units) {
+        const row = state.openFor(u.unitId, (r) => r.fromMachine === m);
+        if (row)
+            return row;
+    }
+    return undefined;
+}

package/build/services/machines/gateway/gateway.d.ts

@@ -23,6 +23,10 @@ export interface GatewaySettings {
     quietMs: number;
     /** How long confirmed message ids gather before a receipt goes out (codecs with receipts). */
     receiptDelayMs?: number;
+    /** Note each job's start, progress steps and pauses on the row that holds it; off unless set. */
+    lifecycle?: boolean;
+    /** With lifecycle notes, progress is noted each time it crosses this many percent; 0 notes none. */
+    progressStep?: number;
 }
 export interface GatewayDeps {
     bus: MachineBus;
@@ -52,6 +56,8 @@ export declare class MachineGateway {
     constructor(deps: GatewayDeps);
     start(): Promise<void>;
     stop(): Promise<void>;
+    /** Machines that report only changes are asked for their full state once the gateway listens. */
+    private askEveryMachine;
     /** Resolves when no dispatch is in flight (tests). */
     idle(): Promise<void>;
     /**

package/build/services/machines/gateway/gateway.js

@@ -32,6 +32,9 @@ class MachineGateway {
         this.codec = deps.codec ?? (0, protocol_1.genericCodec)({ root: deps.settings.topicRoot, line: deps.line });
         this.state.unitsPerDispatch = this.codec.unitsPerDispatch;
         this.state.dispatchOnArrival = this.codec.dispatchOn === protocol_1.DISPATCH_ON.ARRIVAL;
+        this.state.lifecycle = deps.settings.lifecycle === true;
+        this.state.retainedAlarms = this.codec.retainedAlarms === true;
+        this.state.progressStep = deps.settings.progressStep ?? this.state.progressStep;
         this.addresses = deps.addresses ?? new address_book_1.MemoryAddressBook();
         this.executor = new executor_1.Executor({
             addresses: this.addresses,
@@ -45,9 +48,12 @@ class MachineGateway {
     async start() {
         await this.deps.bus.connect();
         await this.deps.feed.connect();
+        this.state.takenAt = Date.now();
         await this.reconcile();
         await this.deps.feed.consume((event) => this.onEscalation(event));
         await this.deps.bus.consume((msg) => this.onMachine(msg));
+        if (this.codec.snapshotOnStart)
+            await this.askEveryMachine();
         const { quietMs } = this.deps.settings;
         if (quietMs > 0) {
             const machines = this.deps.line.machines.map((m) => m.machineId);
@@ -59,10 +65,18 @@ class MachineGateway {
         if (this.quietTimer)
             clearInterval(this.quietTimer);
         this.executor.stop();
+        await this.executor.idle();
         await this.receipts.flush();
         await this.deps.feed.close();
         await this.deps.bus.close();
     }
+    /** Machines that report only changes are asked for their full state once the gateway listens. */
+    async askEveryMachine() {
+        const at = Date.now();
+        await this.executor.apply(this.deps.line.machines.map((m) => ({
+            kind: 'command', machineId: m.machineId, op: protocol_1.OP.SNAPSHOT, cmdId: `${m.machineId}:takeover:${at}`, args: {},
+        })));
+    }
     /** Resolves when no dispatch is in flight (tests). */
     idle() {
         return this.executor.idle();
@@ -80,6 +94,7 @@ class MachineGateway {
             const addresses = await (0, reconcile_1.restoreAddresses)(this.state, this.addresses, this.deps.line);
             const handoffs = await this.reconcileHandoffs();
             await this.executor.apply(await (0, reconcile_1.arrivalsOnRows)(this.state, this.deps.lt, this.deps.settings.dispatchTtlMs));
+            await (0, reconcile_1.notesOnRows)(this.state, this.deps.lt);
             const commands = await this.reconcileCommands();
             const restored = await (0, reconcile_1.restoreLineRows)(this.state, this.deps.lt, this.deps.line);
             await this.resync();
@@ -121,18 +136,26 @@ class MachineGateway {
     async resync() {
         const { line } = this.deps;
         const active = new Set();
-        for (const msg of await this.deps.bus.retained((0, protocol_1.lineFilter)(this.codec.root, line.siteId, line.lineId))) {
-            for (const raw of this.decode(msg.topic, msg.payload, true)) {
-                if (!raw.parsed)
-                    continue;
-                const seen = await (0, admission_1.admitSender)(this.admission, msg.topic, raw);
-                if (!('parsed' in seen) || !seen.parsed)
-                    continue;
-                if (seen.parsed.kind === protocol_1.MESSAGE_KIND.ALARM && seen.body?.body?.active)
-                    active.add(this.state.alarmKey(seen.parsed.ident.machineId, seen.parsed.key));
-                await this.applyMachine(msg.topic, seen.parsed, seen.body);
+        this.state.readingRetained = true;
+        try {
+            for (const msg of await this.deps.bus.retained((0, protocol_1.lineFilter)(this.codec.root, line.siteId, line.lineId))) {
+                for (const raw of this.decode(msg.topic, msg.payload, true)) {
+                    if (!raw.parsed)
+                        continue;
+                    const seen = await (0, admission_1.admitSender)(this.admission, msg.topic, raw);
+                    if (!('parsed' in seen) || !seen.parsed)
+                        continue;
+                    if (seen.parsed.kind === protocol_1.MESSAGE_KIND.ALARM && seen.body?.body?.active)
+                        active.add(this.state.alarmKey(seen.parsed.ident.machineId, seen.parsed.key));
+                    await this.applyMachine(msg.topic, seen.parsed, seen.body);
+                }
             }
         }
+        finally {
+            this.state.readingRetained = false;
+        }
+        if (!this.state.retainedAlarms)
+            return;
         for (const gone of (0, reconcile_1.alarmsClearedMeanwhile)(this.state, active)) {
             this.state.alarms.delete(gone.key);
             await this.executor.apply([{ kind: 'resolve', as: actions_1.CONTROLLER, rowId: gone.rowId, payload: { decision: contracts_1.DECISION.CLEARED, note: `cleared at ${gone.machineId} while the gateway was away` } }]);

package/build/services/machines/gateway/lifecycle.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Job lifecycle on the row. A machine's started and progress events, and the
+ * alarms that stop it, are noted as items on the open row that holds the job:
+ * the accepted dispatch into the machine until its units arrive, then the
+ * handoff out of it. Progress is noted only when it crosses a step, or reaches
+ * a named stage it has not reached before, so a row carries a handful of items
+ * while the stream keeps every report.
+ *
+ * Off unless the line turns it on (MACHINE_LIFECYCLE=on). A row parked with a
+ * limit on its items gets no notes, so they never use a slot its units need.
+ */
+import type { Action } from './actions';
+import { type AlarmTrack, type GatewayState } from './state';
+/** The progress step `percent` has reached; null below the first step, at 100, or when steps are off. */
+export declare function progressStepOf(percent: unknown, step: number): number | null;
+/** A started or progress event, noted on the row that holds the job. */
+export declare function noteJobEvent(state: GatewayState, m: string, b: Record<string, any>, at: string | undefined): Action[];
+/** Rows a stopping alarm pauses: those running or leaving with its job, or every row the machine holds. */
+export declare function notePause(state: GatewayState, track: AlarmTrack, b: Record<string, any>): Action[];
+/** The machine cleared an alarm: the rows it paused, still open, note the resume. */
+export declare function noteResume(state: GatewayState, track: AlarmTrack, b: Record<string, any>): Action[];

package/build/services/machines/gateway/lifecycle.js

@@ -0,0 +1,89 @@
+"use strict";
+/**
+ * Job lifecycle on the row. A machine's started and progress events, and the
+ * alarms that stop it, are noted as items on the open row that holds the job:
+ * the accepted dispatch into the machine until its units arrive, then the
+ * handoff out of it. Progress is noted only when it crosses a step, or reaches
+ * a named stage it has not reached before, so a row carries a handful of items
+ * while the stream keeps every report.
+ *
+ * Off unless the line turns it on (MACHINE_LIFECYCLE=on). A row parked with a
+ * limit on its items gets no notes, so they never use a slot its units need.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.progressStepOf = progressStepOf;
+exports.noteJobEvent = noteJobEvent;
+exports.notePause = notePause;
+exports.noteResume = noteResume;
+const contracts_1 = require("../contracts");
+const protocol_1 = require("../../../lib/machines/protocol");
+const jobs_1 = require("./jobs");
+const state_1 = require("./state");
+const owns = (jobId, reported) => !jobId || typeof reported !== 'string' || (0, jobs_1.jobOwns)(jobId, reported);
+const runningAt = (r, m, jobId) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && owns(r.jobId, jobId);
+const leavingFrom = (r, m, jobId) => r.fromMachine === m && !!r.senderJobId && owns(r.senderJobId, jobId);
+/** The open row holding `unitId` at `m`: its accepted dispatch, else the handoff out of `m`. */
+function holderRow(state, m, unitId, jobId) {
+    return state.openFor(unitId, (r) => runningAt(r, m, jobId)) ?? state.openFor(unitId, (r) => leavingFrom(r, m, jobId));
+}
+/** The progress step `percent` has reached; null below the first step, at 100, or when steps are off. */
+function progressStepOf(percent, step) {
+    if (step <= 0 || typeof percent !== 'number' || !Number.isFinite(percent))
+        return null;
+    const reached = Math.floor(percent / step) * step;
+    return reached > 0 && reached < 100 ? reached : null;
+}
+/** An item the row does not hold yet, added as the machine. */
+function note(row, m, itemKey, payload, metadata) {
+    if (row.capped || row.noted?.includes(itemKey))
+        return [];
+    row.noted = [...(row.noted ?? []), itemKey];
+    return [{ kind: 'accumulate', as: m, rowId: row.id, itemKey, payload, ...(metadata ? { metadata } : {}) }];
+}
+/** A started or progress event, noted on the row that holds the job. */
+function noteJobEvent(state, m, b, at) {
+    const unitId = b.unitId;
+    if (!state.lifecycle || typeof unitId !== 'string')
+        return [];
+    const row = holderRow(state, m, unitId, b.jobId);
+    if (!row)
+        return [];
+    const when = at ?? new Date().toISOString();
+    if (b.event === protocol_1.JOB_EVENT.STARTED) {
+        return note(row, m, contracts_1.itemKeys.started(unitId), { unitId, jobId: b.jobId ?? null, at: when });
+    }
+    const step = progressStepOf(b.percent, state.progressStep);
+    const stage = typeof b.stage === 'string' && b.stage ? b.stage : null;
+    if (step === null && stage === null)
+        return [];
+    const progress = {
+        ...(typeof b.percent === 'number' ? { percent: b.percent } : {}), remainingSec: b.remainingSec ?? null, ...(stage ? { stage } : {}), at: when,
+        ...(b.detail && typeof b.detail === 'object' ? { detail: b.detail } : {}),
+    };
+    const facet = { [contracts_1.LIFECYCLE_FACET.PROGRESS]: progress };
+    const job = { unitId, jobId: b.jobId ?? null };
+    return [
+        ...(step === null ? [] : note(row, m, contracts_1.itemKeys.progress(unitId, step), { ...job, step, ...progress }, facet)),
+        ...(stage === null ? [] : note(row, m, contracts_1.itemKeys.stage(unitId, stage), { ...job, ...progress }, facet)),
+    ];
+}
+/** Rows a stopping alarm pauses: those running or leaving with its job, or every row the machine holds. */
+function notePause(state, track, b) {
+    if (!state.lifecycle)
+        return [];
+    const m = track.machineId;
+    const jobId = typeof b.jobId === 'string' ? b.jobId : undefined;
+    const rows = state.open().filter((r) => runningAt(r, m, jobId) || leavingFrom(r, m, jobId));
+    track.noted = rows.map((r) => r.id);
+    const payload = { alarmId: track.alarmId, code: b.code ?? null, level: b.level ?? null, text: b.text ?? null, raisedAt: b.raisedAt ?? null };
+    return rows.flatMap((r) => note(r, m, contracts_1.itemKeys.paused(track.alarmId, String(b.raisedAt ?? '')), payload));
+}
+/** The machine cleared an alarm: the rows it paused, still open, note the resume. */
+function noteResume(state, track, b) {
+    const key = contracts_1.itemKeys.resumed(track.alarmId, String(track.raisedAt ?? ''));
+    const payload = { alarmId: track.alarmId, clearedAt: b.clearedAt ?? new Date().toISOString(), clearedBy: b.clearedBy ?? null };
+    return (track.noted ?? [])
+        .map((id) => state.get(id))
+        .filter((r) => !!r && !r.ended)
+        .flatMap((r) => note(r, track.machineId, key, payload));
+}

package/build/services/machines/gateway/pacing.d.ts

@@ -11,6 +11,20 @@ import { type Action } from './actions';
 import { type GatewayState, type HandoffRow } from './state';
 /** Dispatch arguments for a handoff row. */
 export declare function dispatchArgs(row: HandoffRow, expiresAt: number): Record<string, any>;
+/**
+ * The validity round a dispatch belongs to. Rounds of one validity period count
+ * from when the dispatch became due, and a round begins only when the one
+ * before it has expired, so a machine never holds two valid commands for one
+ * row. Within a round a dispatch sent again, by a retry, a restarted machine or
+ * a restarted gateway, is the same command with the same arguments; a new round
+ * is a new command, never an old one with changed arguments.
+ */
+export declare function dispatchWindow(dueAt: number, now: number, ttlMs: number): {
+    round: number;
+    expiresAt: number;
+};
+/** The later of a row's due time and an arrival's time; an unreadable time leaves it as it was. */
+export declare function latestArrival(dueAt: number | undefined, at: unknown): number | undefined;
 export declare function dispatchAction(row: HandoffRow, machineId: string, dispatchTtlMs: number): Action;
 /** A machine on a version the line does not allow is refused before any dispatch. */
 export declare function softwareRefusal(state: GatewayState, line: LineDef, machineId: string): string | null;

package/build/services/machines/gateway/pacing.js

@@ -9,6 +9,8 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.dispatchArgs = dispatchArgs;
+exports.dispatchWindow = dispatchWindow;
+exports.latestArrival = latestArrival;
 exports.dispatchAction = dispatchAction;
 exports.softwareRefusal = softwareRefusal;
 exports.artifactRefusal = artifactRefusal;
@@ -39,9 +41,31 @@ function dispatchArgs(row, expiresAt) {
         ...(row.parameters ? { parameters: row.parameters } : {}),
     };
 }
+/**
+ * The validity round a dispatch belongs to. Rounds of one validity period count
+ * from when the dispatch became due, and a round begins only when the one
+ * before it has expired, so a machine never holds two valid commands for one
+ * row. Within a round a dispatch sent again, by a retry, a restarted machine or
+ * a restarted gateway, is the same command with the same arguments; a new round
+ * is a new command, never an old one with changed arguments.
+ */
+function dispatchWindow(dueAt, now, ttlMs) {
+    const round = Math.max(1, Math.floor((now - dueAt) / ttlMs) + 1);
+    return { round, expiresAt: dueAt + round * ttlMs };
+}
+/** The later of a row's due time and an arrival's time; an unreadable time leaves it as it was. */
+function latestArrival(dueAt, at) {
+    const t = typeof at === 'string' ? Date.parse(at) : NaN;
+    if (!Number.isFinite(t))
+        return dueAt;
+    return dueAt === undefined ? t : Math.max(dueAt, t);
+}
 function dispatchAction(row, machineId, dispatchTtlMs) {
+    const now = Date.now();
+    row.createdAt ??= now;
+    const { round, expiresAt } = dispatchWindow(row.dueAt ?? row.createdAt, now, dispatchTtlMs);
     return {
-        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id), args: dispatchArgs(row, Date.now() + dispatchTtlMs),
+        kind: 'dispatch', rowId: row.id, machineId, cmdId: (0, contracts_1.dispatchCommandId)(row.id, round), args: dispatchArgs(row, expiresAt),
         ...(row.dispatch === state_1.DISPATCH_STATE.ARRIVING ? { receivedOnAccept: true } : {}),
     };
 }
@@ -57,7 +81,7 @@ function softwareRefusal(state, line, machineId) {
 function artifactRefusal(line, row) {
     if (!(0, line_1.machineById)(line, row.toMachine).requireArtifact || (row.artifactUrl && row.artifactSha256))
         return null;
-    if (row.units.every((u) => u.artifact))
+    if (row.units.every((u) => u.artifact || u.artifacts?.length))
         return null;
     return `${row.toMachine} requires an artifact (url and sha256), for the job or for every unit; the row has none`;
 }

package/build/services/machines/gateway/plan-escalation.js

@@ -29,9 +29,17 @@ const VERB = { CREATED: 'created', CLAIMED: 'claimed', RESOLVED: 'resolved', CAN
 const STATUS_FOR_VERB = {
     [VERB.CREATED]: 'pending', [VERB.CLAIMED]: 'pending', [VERB.RESOLVED]: 'resolved', [VERB.CANCELLED]: 'cancelled', [VERB.EXPIRED]: 'expired',
 };
+/** The metadata key an accumulator row parked with an item limit carries. */
+const ACCUMULATE_MAX = 'accumulate_max';
 const LINE_ROW_TYPES = [contracts_1.ROW_TYPE.ALARM, contracts_1.ROW_TYPE.MACHINE_OFFLINE, contracts_1.ROW_TYPE.MACHINE_QUIET, contracts_1.ROW_TYPE.MACHINE_COMMAND, contracts_1.ROW_TYPE.MACHINE_REGISTRATION];
-function handoffRow(id, role, facets, metadata) {
-    return { ...facets, id, role, dispatch: state_1.DISPATCH_STATE.NONE, trace: (0, trace_1.readTrace)(metadata) };
+function handoffRow(id, role, facets, data) {
+    const metadata = data?.metadata;
+    const capped = metadata?.[ACCUMULATE_MAX] != null;
+    const createdAt = data?.created_at ? new Date(data.created_at).getTime() : NaN;
+    return {
+        ...facets, id, role, dispatch: state_1.DISPATCH_STATE.NONE, trace: (0, trace_1.readTrace)(metadata),
+        ...(capped ? { capped } : {}), ...(Number.isFinite(createdAt) ? { createdAt } : {}),
+    };
 }
 /** Rows for another line belong to that line's gateway. */
 function onThisLine(line, at) {
@@ -108,7 +116,8 @@ function planCommandRow(state, line, event) {
     if (refusal) {
         return [{ kind: 'resolve', as: actions_1.CONTROLLER, rowId: id, payload: { accepted: false, machineId: c.machineId, op: c.op, status: contracts_1.COMMAND_STATUS.UNSUPPORTED, reason: refusal } }];
     }
-    return [{ kind: 'control', rowId: id, machineId: c.machineId, op: c.op, cmdId: (0, contracts_1.controlCommandId)(id), args: c.args }];
+    const createdAt = event.data?.created_at ? new Date(event.data.created_at).getTime() : NaN;
+    return [{ kind: 'control', rowId: id, machineId: c.machineId, op: c.op, args: c.args, ...(Number.isFinite(createdAt) ? { createdAt } : {}) }];
 }
 function onLine(line, machineId) {
     return !!machineId && line.machines.some((m) => m.machineId === machineId);
@@ -122,7 +131,7 @@ function planHandoff(state, line, event, facets, dispatchTtlMs) {
     if (event.verb === VERB.CREATED || event.verb === VERB.CLAIMED) {
         if (state.isClosed(id) || !facets)
             return [];
-        state.index(handoffRow(id, event.role, facets, event.data?.metadata));
+        state.index(handoffRow(id, event.role, facets, event.data));
         const row = state.get(id);
         // The units now wait on the sender as this row's sender job, when the sender holds them under it.
         if (row.fromMachine)

package/build/services/machines/gateway/plan-machine.d.ts

@@ -2,10 +2,11 @@
  * Machine messages to gateway actions: the escalation verbs a person would
  * perform, performed for the machine. Each unit's arrival and the sender's
  * finished record accumulate on the handoff row, and the last expected item
- * resolves it. A failure on either side resolves it with an error. Alarms that
- * stop work and machines that go dark or quiet become rows for people. A
- * restarted machine gets back the dispatches it lost. Everything else stays
- * in the stream.
+ * resolves it. A failure on either side resolves it with an error. A job's
+ * start, its progress steps, and the alarms that pause it are noted on the row
+ * that holds it (lifecycle.ts). Alarms that stop work and machines that go
+ * dark or quiet become rows for people. A restarted machine gets back the
+ * dispatches it lost. Everything else stays in the stream.
  *
  * A message whose row does not exist yet is nak'd and arrives again; one whose
  * row already closed is acked, so a replay never resolves anything twice.

package/build/services/machines/gateway/plan-machine.js

@@ -3,10 +3,11 @@
  * Machine messages to gateway actions: the escalation verbs a person would
  * perform, performed for the machine. Each unit's arrival and the sender's
  * finished record accumulate on the handoff row, and the last expected item
- * resolves it. A failure on either side resolves it with an error. Alarms that
- * stop work and machines that go dark or quiet become rows for people. A
- * restarted machine gets back the dispatches it lost. Everything else stays
- * in the stream.
+ * resolves it. A failure on either side resolves it with an error. A job's
+ * start, its progress steps, and the alarms that pause it are noted on the row
+ * that holds it (lifecycle.ts). Alarms that stop work and machines that go
+ * dark or quiet become rows for people. A restarted machine gets back the
+ * dispatches it lost. Everything else stays in the stream.
  *
  * A message whose row does not exist yet is nak'd and arrives again; one whose
  * row already closed is acked, so a replay never resolves anything twice.
@@ -21,6 +22,7 @@ const actions_1 = require("./actions");
 const pacing_1 = require("./pacing");
 const plan_shared_1 = require("./plan-shared");
 const jobs_1 = require("./jobs");
+const lifecycle_1 = require("./lifecycle");
 const trace_1 = require("./trace");
 const state_1 = require("./state");
 const ack = (actions = []) => ({ actions, disposition: transport_1.DISPOSITION.ACK });
@@ -68,12 +70,29 @@ function ownsJob(jobId, reported) {
     return (0, jobs_1.jobOwns)(jobId, reported);
 }
 /** Keep the receiver's arrival event for the unit, to go out with its dispatch. */
-function withArrival(row, unitId, b) {
+function withArrival(row, unitId, b, at) {
     row.arrivals = { ...row.arrivals, [unitId]: b };
+    row.dueAt = (0, pacing_1.latestArrival)(row.dueAt, at);
     return row;
 }
+/** The unit an open handoff into `m` carries on `carrierId`, for a scan that read only the carrier. */
+function unitOnCarrier(state, m, carrierId) {
+    if (typeof carrierId !== 'string')
+        return undefined;
+    for (const r of state.open()) {
+        if (r.toMachine !== m)
+            continue;
+        const unit = r.units.find((u) => u.carrierId === carrierId);
+        if (unit)
+            return unit.unitId;
+    }
+    return undefined;
+}
 function planJob(state, m, b, at, ttl) {
-    const unitId = b.unitId;
+    const unitId = typeof b.unitId === 'string' ? b.unitId : unitOnCarrier(state, m, b.carrierId);
+    // A carrier no open handoff here carries yet: its row may still be on the way.
+    if (unitId === undefined)
+        return nak;
     // An event from an earlier attempt's job never touches a later attempt's row.
     const toHere = (r) => r.toMachine === m && ownsJob(r.jobId, b.jobId);
     const fromHere = (r) => r.fromMachine === m && ownsJob(r.senderJobId, b.jobId);
@@ -95,13 +114,14 @@ function planJob(state, m, b, at, ttl) {
         if (row.fromMachine)
             state.letGo(row.fromMachine, unitId);
         const units = row.units.map((u) => u.unitId);
+        const arrivedAt = at ?? new Date().toISOString();
         const complete = row.dispatch === state_1.DISPATCH_STATE.ARRIVING
-            ? { keys: units.map(contracts_1.itemKeys.arrived), then: [(0, pacing_1.dispatchAction)(withArrival(row, unitId, b), m, ttl)] }
+            ? { keys: units.map(contracts_1.itemKeys.arrived), then: [(0, pacing_1.dispatchAction)(withArrival(row, unitId, b, arrivedAt), m, ttl)] }
             : { keys: units.map(contracts_1.itemKeys.arrived), payload: { received: true, receiver: m, units } };
         return ack([...early, {
                 kind: 'accumulate', as: m, rowId: row.id, itemKey: contracts_1.itemKeys.arrived(unitId),
                 payload: {
-                    unitId, carrierId: b.carrierId ?? null, from: b.peer ?? null, at: at ?? new Date().toISOString(),
+                    unitId, carrierId: b.carrierId ?? null, from: b.peer ?? null, at: arrivedAt,
                     ...(row.dispatch === state_1.DISPATCH_STATE.ARRIVING ? { arrival: b } : {}),
                 },
                 complete,
@@ -140,6 +160,8 @@ function planJob(state, m, b, at, ttl) {
             return ack();
         return ack((0, plan_shared_1.failHandoff)(state, row, m, { received: false, unitId, errorCode: contracts_1.HANDOFF_ERROR.WITHDRAWN, detail: `${m} gave up ${b.jobId}` }));
     }
+    if (b.event === protocol_1.JOB_EVENT.STARTED || b.event === protocol_1.JOB_EVENT.PROGRESS)
+        return ack((0, lifecycle_1.noteJobEvent)(state, m, b, at));
     return ack();
 }
 function releasedAction(row, m, b) {
@@ -160,9 +182,10 @@ function planAlarm(state, parsed, alarmId, env) {
     if (!env || b.active === false) {
         const track = state.alarms.get(key);
         state.alarms.delete(key);
+        const resumed = track ? (0, lifecycle_1.noteResume)(state, track, b) : [];
         if (!track?.rowId || !track.open)
-            return ack();
-        return ack([{
+            return ack(resumed);
+        return ack([...resumed, {
                 kind: 'resolve', as: actions_1.CONTROLLER, rowId: track.rowId,
                 payload: { decision: contracts_1.DECISION.CLEARED, clearedBy: b.clearedBy ?? null, note: `cleared at ${m}${b.clearedBy ? ` by ${b.clearedBy}` : ''}` },
             }]);
@@ -175,8 +198,9 @@ function planAlarm(state, parsed, alarmId, env) {
     const superseded = known?.rowId && known.open
         ? [{ kind: 'resolve', as: actions_1.CONTROLLER, rowId: known.rowId, payload: { decision: contracts_1.DECISION.CLEARED, note: `${m} raised ${alarmId} again; this one ended` } }]
         : [];
-    state.alarms.set(key, { machineId: m, alarmId, raisedAt: b.raisedAt, open: true });
-    return ack([...superseded, {
+    const track = { machineId: m, alarmId, raisedAt: b.raisedAt, open: true };
+    state.alarms.set(key, track);
+    return ack([...superseded, ...(0, lifecycle_1.notePause)(state, track, b), {
             kind: 'createAlarmRow', machineId: m, alarmId,
             row: {
                 type: contracts_1.ROW_TYPE.ALARM, subtype: b.level, role: contracts_1.LINE_ROLE.OPERATOR, priority: b.level === protocol_1.LEVEL.EMERGENCY ? 1 : 2,
@@ -230,13 +254,30 @@ function planSnapshot(state, m, b, ttl) {
     const jobIds = Array.isArray(b.jobs) ? b.jobs.map((j) => j?.jobId).filter((id) => typeof id === 'string') : [];
     if (Array.isArray(b.jobs))
         state.keepHeld(m, new Set(jobIds));
+    const fresh = !state.retainedAlarms && !state.readingRetained && Array.isArray(b.alarms);
+    const cleared = fresh ? alarmsNoLongerListed(state, m, b.alarms) : [];
     const answer = { kind: 'settleSnapshot', machineId: m, jobIds };
     const previous = link.bootId;
     link.bootId = b.bootId;
     if (!previous || previous === b.bootId)
-        return ack([answer]);
+        return ack([...cleared, answer]);
     const held = new Set(jobIds);
     const missing = (r) => (0, jobs_1.dispatchJobs)(r.jobId, r.units, state.unitsPerDispatch).some((j) => !held.has(j.jobId));
     const lost = state.open().filter((r) => r.toMachine === m && r.dispatch === state_1.DISPATCH_STATE.ACCEPTED && missing(r));
-    return ack([answer, ...lost.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl))]);
+    return ack([...cleared, answer, ...lost.map((r) => (0, pacing_1.dispatchAction)(r, m, ttl))]);
+}
+/** Alarm rows for `m` whose alarm its snapshot no longer lists: they cleared at the machine. */
+function alarmsNoLongerListed(state, m, listed) {
+    const active = new Set(listed.filter((id) => typeof id === 'string'));
+    const actions = [];
+    for (const [key, track] of [...state.alarms.entries()]) {
+        if (track.machineId !== m || active.has(track.alarmId))
+            continue;
+        state.alarms.delete(key);
+        actions.push(...(0, lifecycle_1.noteResume)(state, track, {}));
+        if (track.open && track.rowId) {
+            actions.push({ kind: 'resolve', as: actions_1.CONTROLLER, rowId: track.rowId, payload: { decision: contracts_1.DECISION.CLEARED, note: `cleared at ${m}; its snapshot no longer lists it` } });
+        }
+    }
+    return actions;
 }

package/build/services/machines/gateway/reconcile.d.ts

@@ -31,3 +31,5 @@ export declare function alarmsClearedMeanwhile(state: GatewayState, active: Set<
  * arrivals it holds.
  */
 export declare function arrivalsOnRows(state: GatewayState, lt: LineEscalations, dispatchTtlMs: number): Promise<Action[]>;
+/** With lifecycle notes on, the items each handoff out of a machine already holds, so a restart notes nothing twice. */
+export declare function notesOnRows(state: GatewayState, lt: LineEscalations): Promise<void>;

package/build/services/machines/gateway/reconcile.js

@@ -10,6 +10,7 @@ exports.restoreLineRows = restoreLineRows;
 exports.restoreAddresses = restoreAddresses;
 exports.alarmsClearedMeanwhile = alarmsClearedMeanwhile;
 exports.arrivalsOnRows = arrivalsOnRows;
+exports.notesOnRows = notesOnRows;
 const contracts_1 = require("../contracts");
 const line_1 = require("../../../lib/machines/line");
 const jobs_1 = require("./jobs");
@@ -65,6 +66,7 @@ async function arrivalsOnRows(state, lt, dispatchTtlMs) {
         if (!to || (row.dispatch !== state_1.DISPATCH_STATE.ARRIVING && row.dispatch !== state_1.DISPATCH_STATE.ACCEPTED))
             continue;
         const items = await lt.items(row.id);
+        row.noted = items.map((i) => i.itemKey);
         const units = row.units.map((u) => u.unitId);
         const { arrived, missing } = (0, contracts_1.handoffProgress)(units, items);
         for (const job of (0, jobs_1.dispatchJobs)(row.jobId, row.units, state.unitsPerDispatch)) {
@@ -80,7 +82,18 @@ async function arrivalsOnRows(state, lt, dispatchTtlMs) {
         }
         const byUnit = new Map(items.filter((i) => (0, contracts_1.arrivedUnit)(i.itemKey)).map((i) => [i.payload.unitId, i.payload]));
         row.arrivals = Object.fromEntries(units.map((u) => [u, byUnit.get(u)?.arrival ?? { unitId: u, carrierId: byUnit.get(u)?.carrierId ?? null }]));
+        for (const arrival of byUnit.values())
+            row.dueAt = (0, pacing_1.latestArrival)(row.dueAt, arrival.at);
         actions.push((0, pacing_1.dispatchAction)(row, to, dispatchTtlMs));
     }
     return actions;
 }
+/** With lifecycle notes on, the items each handoff out of a machine already holds, so a restart notes nothing twice. */
+async function notesOnRows(state, lt) {
+    if (!state.lifecycle)
+        return;
+    for (const row of state.open()) {
+        if (row.fromMachine && !row.noted)
+            row.noted = (await lt.items(row.id)).map((i) => i.itemKey);
+    }
+}