@hotmeshio/long-tail 0.26.0 → 0.27.0

package/build/scripts/bambu-boat.js

@@ -0,0 +1,134 @@
+"use strict";
+/**
+ * Make the example's first print: a small 3DBenchy (by Creative Tools), sliced
+ * for the printer with OrcaSlicer, written to examples/workflows/bambu-printer/files/boat.gcode.3mf.
+ * Run it on the host, where OrcaSlicer is installed:
+ *
+ *   npm run bambu:boat
+ *
+ * 1. Download the model once into files/.
+ * 2. Scale it (BAMBU_BOAT_SCALE, 0.5 by default: about ten minutes on a P1S).
+ * 3. Slice it with the printer, process and filament presets. Each is an
+ *    OrcaSlicer system preset name or a path to a preset JSON, which may
+ *    inherit a system preset by name.
+ *
+ * Env: BAMBU_BOAT_PRINTER, BAMBU_BOAT_PROCESS, BAMBU_BOAT_FILAMENT, BAMBU_BOAT_SCALE,
+ * ORCA_SLICER (the OrcaSlicer executable) and ORCA_PROFILES (its BBL profiles directory).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+try {
+    require('dotenv/config');
+}
+catch { /* dotenv optional */ }
+const child_process_1 = require("child_process");
+const fs_1 = require("fs");
+const os_1 = require("os");
+const path_1 = require("path");
+const env = process.env;
+const FILES = (0, path_1.resolve)('examples/workflows/bambu-printer/files');
+const MODEL_URL = 'https://raw.githubusercontent.com/CreativeTools/3DBenchy/master/Single-part/3DBenchy.stl';
+const ORCA = env.ORCA_SLICER || '/Applications/OrcaSlicer.app/Contents/MacOS/OrcaSlicer';
+const PROFILES = env.ORCA_PROFILES || '/Applications/OrcaSlicer.app/Contents/Resources/profiles/BBL';
+const PRESET = {
+    PRINTER: env.BAMBU_BOAT_PRINTER || 'Bambu Lab P1S 0.4 nozzle',
+    PROCESS: env.BAMBU_BOAT_PROCESS || '0.20mm Standard @BBL X1C',
+    FILAMENT: env.BAMBU_BOAT_FILAMENT || 'Bambu PLA Basic @BBL X1C',
+};
+const PRESET_DIR = { PRINTER: 'machine', PROCESS: 'process', FILAMENT: 'filament' };
+const STL_HEADER = 84;
+const STL_TRIANGLE = 50;
+const say = (msg) => console.log(`[${new Date().toISOString().slice(11, 19)}] ${msg}`);
+async function fetchModel() {
+    const path = (0, path_1.join)(FILES, '3DBenchy.stl');
+    if ((0, fs_1.existsSync)(path))
+        return path;
+    const res = await fetch(MODEL_URL);
+    if (!res.ok)
+        throw new Error(`GET ${MODEL_URL} -> ${res.status}`);
+    (0, fs_1.writeFileSync)(path, Buffer.from(await res.arrayBuffer()));
+    return path;
+}
+/** A binary STL with every vertex scaled about the origin; the slicer places it on the bed. */
+function scaleStl(stl, factor) {
+    const count = stl.readUInt32LE(STL_HEADER - 4);
+    if (stl.length !== STL_HEADER + count * STL_TRIANGLE)
+        throw new Error('the model is not a binary STL');
+    const out = Buffer.from(stl);
+    for (let t = 0; t < count; t++) {
+        const at = STL_HEADER + t * STL_TRIANGLE;
+        for (let v = 3; v < 12; v++)
+            out.writeFloatLE(stl.readFloatLE(at + v * 4) * factor, at + v * 4);
+    }
+    return out;
+}
+/** The system presets of one kind in ORCA_PROFILES, by name. */
+function systemPresets(kind) {
+    const dir = (0, path_1.join)(PROFILES, PRESET_DIR[kind]);
+    const byName = new Map();
+    for (const file of (0, fs_1.readdirSync)(dir).filter((f) => f.endsWith('.json'))) {
+        const preset = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(dir, file), 'utf8'));
+        if (preset.name)
+            byName.set(preset.name, preset);
+    }
+    return byName;
+}
+/**
+ * A preset by path or system preset name, with every setting it inherits
+ * written out: OrcaSlicer's command line reads only the keys in the file, so
+ * an inherited start, layer-change or end G-code would otherwise be lost.
+ */
+function fullPreset(kind, value) {
+    const system = systemPresets(kind);
+    const own = (0, fs_1.existsSync)(value) ? JSON.parse((0, fs_1.readFileSync)(value, 'utf8')) : system.get(value);
+    if (!own)
+        throw new Error(`no ${kind.toLowerCase()} preset ${value} (a path, or a name in ${PROFILES}/${PRESET_DIR[kind]})`);
+    const chain = [own];
+    for (let p = own; p.inherits;) {
+        p = system.get(p.inherits);
+        if (!p)
+            throw new Error(`${kind.toLowerCase()} preset ${value} inherits ${chain[chain.length - 1].inherits}, which is not in ${PROFILES}`);
+        chain.unshift(p);
+    }
+    return Object.assign({}, ...chain, { inherits: '' });
+}
+/** Write a full preset for the slicer; a process or filament is marked compatible with the printer, as OrcaSlicer requires. */
+function writePreset(dir, kind, preset, printerName) {
+    const path = (0, path_1.join)(dir, `${PRESET_DIR[kind]}.json`);
+    const compatibility = kind === 'PRINTER' ? {} : { from: 'User', compatible_printers: [printerName], compatible_printers_condition: '' };
+    (0, fs_1.writeFileSync)(path, JSON.stringify({ ...preset, ...compatibility }));
+    return path;
+}
+async function main() {
+    if (!(0, fs_1.existsSync)(ORCA))
+        throw new Error(`OrcaSlicer is not at ${ORCA}; install it or set ORCA_SLICER`);
+    (0, fs_1.mkdirSync)(FILES, { recursive: true });
+    const work = (0, fs_1.mkdtempSync)((0, path_1.join)((0, os_1.tmpdir)(), 'bambu-boat-'));
+    // 1. The model
+    const model = await fetchModel();
+    // 2. Scaled
+    const scale = Number(env.BAMBU_BOAT_SCALE || 0.5);
+    if (!(scale > 0))
+        throw new Error(`BAMBU_BOAT_SCALE="${env.BAMBU_BOAT_SCALE}" must be a number above 0`);
+    const scaled = (0, path_1.join)(work, 'boat.stl');
+    (0, fs_1.writeFileSync)(scaled, scaleStl((0, fs_1.readFileSync)(model), scale));
+    // 3. Sliced
+    const printerPreset = fullPreset('PRINTER', PRESET.PRINTER);
+    const printerName = printerPreset.name;
+    const printer = writePreset(work, 'PRINTER', printerPreset, printerName);
+    const process_ = writePreset(work, 'PROCESS', fullPreset('PROCESS', PRESET.PROCESS), printerName);
+    const filament = writePreset(work, 'FILAMENT', fullPreset('FILAMENT', PRESET.FILAMENT), printerName);
+    say(`slicing at ${scale}x for ${printerName}`);
+    (0, child_process_1.execFileSync)(ORCA, [
+        '--load-settings', `${printer};${process_}`, '--load-filaments', filament,
+        '--arrange', '1', '--slice', '0', '--outputdir', work, '--export-3mf', 'boat.gcode.3mf', scaled,
+    ], { stdio: 'inherit' });
+    const target = (0, path_1.join)(FILES, 'boat.gcode.3mf');
+    (0, fs_1.renameSync)((0, path_1.join)(work, 'boat.gcode.3mf'), target);
+    const gcode = (0, fs_1.readdirSync)(work).find((f) => f.endsWith('.gcode'));
+    const time = gcode ? (0, fs_1.readFileSync)((0, path_1.join)(work, gcode), 'utf8').slice(0, 2000).match(/total estimated time: ([^\n]+)/)?.[1] : undefined;
+    say(`${target}${time ? ` (about ${time})` : ''}`);
+}
+main().catch((err) => {
+    console.error(`failed: ${err.message}`);
+    process.exit(1);
+});

package/build/scripts/bambu-print.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Print one file on a Bambu printer through Long Tail, and follow its rows.
+ * Run it where the printer is reachable, e.g. in the Long Tail container:
+ *
+ *   docker compose exec long-tail npm run bambu:print -- [examples/workflows/bambu-printer/files/part.gcode.3mf] [station]
+ *
+ * Without a file it prints the example's boat, a half-size 3DBenchy for a P1S with a 0.8 nozzle and TPU;
+ * `npm run bambu:boat` slices one for another printer or filament.
+ *
+ * 1. Put the file on the printer (it prints only files on its own storage) and hash it: the unit's artifact.
+ * 2. Start the unit route: one print step at the station, then a person unloads.
+ * 3. Print the route's rows and their lifecycle items as they change, until the
+ *    route ends. The unload row is resolved by a person on the dashboard.
+ *
+ * Env: LT_USERNAME and LT_PASSWORD (a Long Tail login that may invoke the route), BASE_URL
+ * (http://localhost:3000), BAMBU_SITE_DIR (deploy/bambu-site), and the printers in .env.
+ */
+export {};

package/build/scripts/bambu-print.js

@@ -0,0 +1,114 @@
+"use strict";
+/**
+ * Print one file on a Bambu printer through Long Tail, and follow its rows.
+ * Run it where the printer is reachable, e.g. in the Long Tail container:
+ *
+ *   docker compose exec long-tail npm run bambu:print -- [examples/workflows/bambu-printer/files/part.gcode.3mf] [station]
+ *
+ * Without a file it prints the example's boat, a half-size 3DBenchy for a P1S with a 0.8 nozzle and TPU;
+ * `npm run bambu:boat` slices one for another printer or filament.
+ *
+ * 1. Put the file on the printer (it prints only files on its own storage) and hash it: the unit's artifact.
+ * 2. Start the unit route: one print step at the station, then a person unloads.
+ * 3. Print the route's rows and their lifecycle items as they change, until the
+ *    route ends. The unload row is resolved by a person on the dashboard.
+ *
+ * Env: LT_USERNAME and LT_PASSWORD (a Long Tail login that may invoke the route), BASE_URL
+ * (http://localhost:3000), BAMBU_SITE_DIR (deploy/bambu-site), and the printers in .env.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+try {
+    require('dotenv/config');
+}
+catch { /* dotenv optional */ }
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const path_1 = require("path");
+const env_1 = require("../examples/workflows/bambu-printer/env");
+const files_1 = require("../examples/workflows/bambu-printer/files");
+const route_1 = require("../examples/workflows/machine-line/workflows/route");
+const env = process.env;
+const BASE_URL = env.BASE_URL || 'http://localhost:3000';
+const SITE_DIR = env.BAMBU_SITE_DIR || 'deploy/bambu-site';
+const WATCH_MS = 3_000;
+const BOAT = 'examples/workflows/bambu-printer/boat/p1s-0.8-tpu.gcode.3mf';
+const ts = () => new Date().toISOString().slice(11, 19);
+const say = (msg) => console.log(`[${ts()}] ${msg}`);
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+let token = '';
+async function api(method, path, body) {
+    const res = await fetch(`${BASE_URL}/api${path}`, {
+        method,
+        headers: { 'content-type': 'application/json', ...(token ? { authorization: `Bearer ${token}` } : {}) },
+        body: body === undefined ? undefined : JSON.stringify(body),
+    });
+    const data = await res.json().catch(() => ({}));
+    if (!res.ok)
+        throw new Error(`${method} ${path} -> ${res.status}: ${JSON.stringify(data)}`);
+    return data;
+}
+/** The route's rows, and the items each holds, printed as they change; the route's result once it ends. */
+async function follow(orderRef, workflowId) {
+    const shown = new Map();
+    for (;;) {
+        try {
+            const result = await look(orderRef, workflowId, shown);
+            if (result)
+                return result;
+        }
+        catch (err) {
+            // A restart or a database blip is not the route ending: keep following.
+            say(`(could not read the route: ${err.message}; trying again)`);
+        }
+        await sleep(WATCH_MS);
+    }
+}
+async function look(orderRef, workflowId, shown) {
+    const facets = encodeURIComponent(JSON.stringify({ orderRef }));
+    const { escalations } = await api('GET', `/escalations?facets=${facets}&limit=20&sort_by=created_at&order=asc`);
+    for (const row of escalations) {
+        const { items = [] } = await api('GET', `/escalations/${row.id}/items`);
+        const view = `${row.type}/${row.role} ${row.status}${row.assigned_to ? ' claimed' : ''} [${items.map((i) => i.itemKey).join(', ')}]`;
+        if (shown.get(row.id) !== view)
+            say(`${row.id.slice(0, 8)} ${view}`);
+        shown.set(row.id, view);
+    }
+    const resp = await api('GET', `/workflows/${workflowId}/result`).catch(() => null);
+    return resp?.result?.data?.status ? resp.result.data : null;
+}
+async function main() {
+    const [file = BOAT, stationArg] = process.argv.slice(2);
+    if (!(0, fs_1.existsSync)(file))
+        throw new Error(`${file} is missing; usage: npm run bambu:print -- [path/to/part.gcode.3mf] [station]`);
+    const line = JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(SITE_DIR, 'line.json'), 'utf8'));
+    const station = stationArg || line.machines[0].machineId;
+    const printer = (0, env_1.printersFromEnv)().find((p) => p.station === station);
+    if (!printer || !line.machines.some((m) => m.machineId === station))
+        throw new Error(`${station} is not an enabled printer on the line in ${SITE_DIR}/line.json`);
+    // 1. The artifact, on the printer
+    const routeKey = `P${Date.now().toString(36)}`;
+    const sha256 = (0, crypto_1.createHash)('sha256').update((0, fs_1.readFileSync)(file)).digest('hex');
+    say(`copying ${(0, path_1.basename)(file)} to ${station} (a minute or two for a few MB)`);
+    const url = await (0, files_1.stageFile)(printer, file, `${routeKey}.gcode.3mf`);
+    say(`${url} (sha256 ${sha256.slice(0, 12)}…)`);
+    // 2. The route
+    const { LT_USERNAME: username, LT_PASSWORD: password } = env;
+    if (!username || !password)
+        throw new Error('set LT_USERNAME and LT_PASSWORD in .env');
+    token = (await api('POST', '/auth/login', { username, password })).token;
+    const input = {
+        siteId: line.siteId, lineId: line.lineId, routeKey, orderRef: `ORD-${routeKey}`,
+        units: [{ unitId: `U-${routeKey}`, carrierId: `PLATE-${routeKey}` }],
+        steps: [{ machineId: station, process: env_1.PRINT_PROCESS, recipe: { ...env_1.PRINT_RECIPE } }],
+        artifact: { url, sha256 }, exitRole: env_1.BAMBU_ROLES.UNLOAD, handoffTimeout: env.BAMBU_HANDOFF_TIMEOUT || '6h',
+    };
+    const workflowId = `bambu-${routeKey}`;
+    await api('POST', `/workflows/${route_1.UNIT_ROUTE_WORKFLOW}/invoke`, { data: input, metadata: { source: 'bambu-print' }, workflowId });
+    say(`started ${workflowId} on ${station}; unload the part on the dashboard (${env_1.BAMBU_ROLES.UNLOAD}) when it finishes`);
+    // 3. Follow it
+    say(`route ended: ${JSON.stringify(await follow(input.orderRef, workflowId))}`);
+}
+main().catch((err) => {
+    console.error(`[${ts()}] failed: ${err.message}`);
+    process.exit(1);
+});

package/build/services/machines/config.d.ts

@@ -37,7 +37,13 @@ export interface MachineLineConfig {
     dispatchTtlMs: number;
     /** How long confirmed message ids gather before a receipt goes out. */
     receiptDelayMs: number;
+    /** Note each job's start, progress steps and pauses on the row that holds it (lifecycle.ts). */
+    lifecycle: boolean;
+    /** With lifecycle notes, progress is noted each time it crosses this many percent; 0 notes none. */
+    progressStep: number;
     ltBaseUrl: string;
 }
 type Env = Record<string, string | undefined>;
+/** The generic line protocol's topic root when MACHINE_TOPIC_ROOT is unset. */
+export declare const DEFAULT_TOPIC_ROOT = "line/v1";
 export declare function loadMachineLineConfig(env?: Env): MachineLineConfig;

package/build/services/machines/config.js

@@ -4,11 +4,14 @@
  * an unknown or incomplete selection throws; nothing falls back silently.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ESCALATION_FEED = exports.MACHINE_TRANSPORT = void 0;
+exports.DEFAULT_TOPIC_ROOT = exports.ESCALATION_FEED = exports.MACHINE_TRANSPORT = void 0;
 exports.loadMachineLineConfig = loadMachineLineConfig;
 const protocol_1 = require("../../lib/machines/protocol");
 exports.MACHINE_TRANSPORT = { MEMORY: 'memory', NATS: 'nats', MQTT: 'mqtt' };
 exports.ESCALATION_FEED = { MEMORY: 'memory', CORE: 'core', IN_PROCESS: 'in-process' };
+/** The generic line protocol's topic root when MACHINE_TOPIC_ROOT is unset. */
+exports.DEFAULT_TOPIC_ROOT = 'line/v1';
+const SWITCH = { ON: 'on', OFF: 'off' };
 function oneOf(name, value, allowed) {
     if (!allowed.includes(value)) {
         throw new Error(`${name}="${value}" is not one of ${allowed.join(', ')}`);
@@ -24,6 +27,12 @@ function int(env, name, fallback) {
         throw new Error(`${name}="${raw}" must be a non-negative integer`);
     return n;
 }
+function percentStep(env, name, fallback) {
+    const n = int(env, name, fallback);
+    if (n > 100)
+        throw new Error(`${name}="${n}" must be at most 100`);
+    return n;
+}
 function required(env, name, mode) {
     const value = env[name];
     if (!value)
@@ -55,7 +64,7 @@ function loadMachineLineConfig(env = process.env) {
     const config = {
         transport,
         feed,
-        topicRoot: env.MACHINE_TOPIC_ROOT ?? 'line/v1',
+        topicRoot: env.MACHINE_TOPIC_ROOT ?? exports.DEFAULT_TOPIC_ROOT,
         lineConfigPath: env.MACHINE_LINE_CONFIG || undefined,
         replyTimeoutMs: int(env, 'MACHINE_REPLY_TIMEOUT_MS', protocol_1.REPLY_DEADLINE_MS),
         dispatchAttempts: Math.max(1, int(env, 'MACHINE_DISPATCH_ATTEMPTS', 3)),
@@ -64,10 +73,13 @@ function loadMachineLineConfig(env = process.env) {
         quietMs: int(env, 'MACHINE_QUIET_MS', 90_000),
         dispatchTtlMs: int(env, 'MACHINE_DISPATCH_TTL_MS', 600_000),
         receiptDelayMs: int(env, 'MACHINE_RECEIPT_DELAY_MS', 250),
+        lifecycle: oneOf('MACHINE_LIFECYCLE', env.MACHINE_LIFECYCLE || SWITCH.OFF, Object.values(SWITCH)) === SWITCH.ON,
+        progressStep: percentStep(env, 'MACHINE_PROGRESS_STEP', 25),
         ltBaseUrl: env.LT_BASE_URL ?? 'http://localhost:3000',
     };
     if (transport === exports.MACHINE_TRANSPORT.NATS) {
-        config.siteNatsUrl = required(env, 'MACHINE_SITE_NATS_URL', transport);
+        // A site NATS of its own; without one, the streams are Long Tail's messaging.
+        config.siteNatsUrl = env.MACHINE_SITE_NATS_URL || undefined;
         config.siteNatsUser = env.MACHINE_SITE_NATS_USER;
         config.siteNatsPassword = env.MACHINE_SITE_NATS_PASSWORD;
         config.mqtt = mqttEndpoint(env, transport, 5);
@@ -75,10 +87,8 @@ function loadMachineLineConfig(env = process.env) {
     if (transport === exports.MACHINE_TRANSPORT.MQTT)
         config.mqtt = mqttEndpoint(env, transport, 5);
     if (feed === exports.ESCALATION_FEED.CORE) {
-        config.ltNatsUrl = env.NATS_URL;
+        config.ltNatsUrl = env.NATS_URL || undefined;
         config.ltNatsToken = env.NATS_TOKEN;
-        if (!config.ltNatsUrl)
-            throw new Error('MACHINE_ESCALATION_FEED=core needs NATS_URL (Long Tail NATS)');
     }
     return config;
 }

package/build/services/machines/contracts.d.ts

@@ -29,12 +29,18 @@ export declare const LINE_ROLE: {
 export interface ArtifactRef {
     url: string;
     sha256: string;
+    /** The host's id for the file. */
+    id?: string;
+    contentType?: string;
+    sizeBytes?: number;
 }
 export interface HandoffUnit {
     unitId: string;
     carrierId: string | null;
     /** This unit's own file, when each unit has one (a per-unit model or toolpath). */
     artifact?: ArtifactRef;
+    /** This unit's files, when it has several. */
+    artifacts?: ArtifactRef[];
     /** Parameters for this unit alone, sent to the machine as given. */
     parameters?: Record<string, unknown>;
 }
@@ -69,6 +75,11 @@ export interface HandoffFacets {
 export declare const itemKeys: {
     released: (unitId: string) => string;
     arrived: (unitId: string) => string;
+    started: (unitId: string) => string;
+    progress: (unitId: string, step: number) => string;
+    stage: (unitId: string, stage: string) => string;
+    paused: (alarmId: string, raisedAt: string) => string;
+    resumed: (alarmId: string, raisedAt: string) => string;
 };
 /** The unit an arrival item names; null for any other item. */
 export declare function arrivedUnit(itemKey: string): string | null;
@@ -125,6 +136,10 @@ export declare const TRACE_FACET: {
     readonly TRACE: "trace";
     readonly LOTS: "lots";
 };
+/** The facet a row holding a job carries while it runs: its latest noted progress. */
+export declare const LIFECYCLE_FACET: {
+    readonly PROGRESS: "progress";
+};
 /** What resolves a handoff row, from a machine or from a person's form. */
 export interface HandoffResolution {
     received: boolean;
@@ -188,6 +203,8 @@ export interface CommandResolution {
     status: CommandStatus;
     reason?: string;
 }
-export declare const dispatchCommandId: (escalationId: string) => string;
-export declare const controlCommandId: (escalationId: string) => string;
+/** A row's dispatch command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+export declare const dispatchCommandId: (escalationId: string, round?: number) => string;
+/** A command row's command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+export declare const controlCommandId: (escalationId: string, round?: number) => string;
 export declare const cancelCommandId: (escalationId: string) => string;

package/build/services/machines/contracts.js

@@ -6,7 +6,7 @@
  * protocol that resolves it differs.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cancelCommandId = exports.controlCommandId = exports.dispatchCommandId = exports.COMMAND_STATUS = exports.signalKeys = exports.DECISION = exports.TRACE_FACET = exports.HANDOFF_ERROR = exports.itemKeys = exports.LINE_ROLE = exports.ROW_TYPE = void 0;
+exports.cancelCommandId = exports.controlCommandId = exports.dispatchCommandId = exports.COMMAND_STATUS = exports.signalKeys = exports.DECISION = exports.LIFECYCLE_FACET = exports.TRACE_FACET = exports.HANDOFF_ERROR = exports.itemKeys = exports.LINE_ROLE = exports.ROW_TYPE = void 0;
 exports.arrivedUnit = arrivedUnit;
 exports.handoffProgress = handoffProgress;
 exports.handoffDecisions = handoffDecisions;
@@ -43,6 +43,11 @@ const ARRIVED_PREFIX = 'arrived:';
 exports.itemKeys = {
     released: (unitId) => `${RELEASED_PREFIX}${unitId}`,
     arrived: (unitId) => `${ARRIVED_PREFIX}${unitId}`,
+    started: (unitId) => `started:${unitId}`,
+    progress: (unitId, step) => `progress:${unitId}:${step}`,
+    stage: (unitId, stage) => `stage:${unitId}:${stage}`,
+    paused: (alarmId, raisedAt) => `paused:${alarmId}:${raisedAt}`,
+    resumed: (alarmId, raisedAt) => `resumed:${alarmId}:${raisedAt}`,
 };
 /** The unit an arrival item names; null for any other item. */
 function arrivedUnit(itemKey) {
@@ -78,6 +83,8 @@ exports.HANDOFF_ERROR = {
  * containment-searchable, e.g. `{ lots: ['LOT-7'] }`.
  */
 exports.TRACE_FACET = { TRACE: 'trace', LOTS: 'lots' };
+/** The facet a row holding a job carries while it runs: its latest noted progress. */
+exports.LIFECYCLE_FACET = { PROGRESS: 'progress' };
 /** Decisions on line-operator rows: failed handoffs, alarms, dark machines, contested registrations. */
 exports.DECISION = {
     RETRY: 'retry',
@@ -107,9 +114,11 @@ exports.signalKeys = {
     jobStop: (jobKey, step, attempt) => `job-stop-${jobKey}-s${step}-a${attempt}`,
 };
 exports.COMMAND_STATUS = { ...protocol_1.REPLY_STATUS, NO_REPLY: 'no-reply', UNSUPPORTED: 'unsupported' };
-const dispatchCommandId = (escalationId) => `${escalationId}:dispatch`;
+/** A row's dispatch command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+const dispatchCommandId = (escalationId, round = 1) => (round > 1 ? `${escalationId}:dispatch:r${round}` : `${escalationId}:dispatch`);
 exports.dispatchCommandId = dispatchCommandId;
-const controlCommandId = (escalationId) => `${escalationId}:command`;
+/** A command row's command in its validity round; a later round is a new command (gateway/pacing.ts dispatchWindow). */
+const controlCommandId = (escalationId, round = 1) => (round > 1 ? `${escalationId}:command:r${round}` : `${escalationId}:command`);
 exports.controlCommandId = controlCommandId;
 const cancelCommandId = (escalationId) => `${escalationId}:cancel`;
 exports.cancelCommandId = cancelCommandId;

package/build/services/machines/gateway/actions.d.ts

@@ -39,8 +39,8 @@ export type Action =
     rowId: string;
     machineId: string;
     op: string;
-    cmdId: string;
     args: Record<string, any>;
+    createdAt?: number;
 } | {
     kind: 'claim';
     as: Actor;

package/build/services/machines/gateway/dispatcher.d.ts

@@ -37,18 +37,31 @@ type CommandAction = Extract<Action, {
 type ControlAction = Extract<Action, {
     kind: 'control';
 }>;
+/** Ends a dispatch or command still running when the gateway stops; the next leader takes it up from the row. */
+export declare class GatewayStopped extends Error {
+    constructor();
+}
 export declare class Dispatcher {
     private readonly d;
+    private readonly sent;
+    private readonly stopping;
     constructor(d: DispatcherDeps);
-    /** False when the codec has no message for the command. */
-    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>): Promise<boolean>;
+    /** Work still running sends nothing more and changes no row. */
+    stop(): void;
+    private live;
+    /** This send's attempt number, and whether the machine may have the command already. */
+    private countSend;
+    /** False when the codec has no message for the command. `since` is when the command's row was created. */
+    publish(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<boolean>;
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>): Promise<CommandReply | null>;
+    ask(machineId: string, op: string, cmdId: string, args: Record<string, any>, since?: number): Promise<CommandReply | null>;
     command(a: CommandAction): Promise<void>;
     /**
      * A machine-command row: send it, then resolve the row with the reply, as the
      * machine that gave it. The command expires when the asking ends, so one that
      * reaches a machine after its row resolved no-reply is refused, never run.
+     * Rounds of the asking count from the row's creation, so the command sent
+     * again by a new gateway is the same command with the same arguments.
      */
     control(a: ControlAction): Promise<void>;
     dispatch(a: DispatchAction): Promise<void>;

package/build/services/machines/gateway/dispatcher.js

@@ -11,7 +11,7 @@
  * codec has no message for is unsupported: nothing is sent or awaited.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Dispatcher = void 0;
+exports.Dispatcher = exports.GatewayStopped = void 0;
 const promises_1 = require("timers/promises");
 const contracts_1 = require("../contracts");
 const registration_1 = require("./registration");
@@ -19,6 +19,7 @@ const protocol_1 = require("../../../lib/machines/protocol");
 const actions_1 = require("./actions");
 const escalations_1 = require("./escalations");
 const jobs_1 = require("./jobs");
+const pacing_1 = require("./pacing");
 const log_1 = require("./log");
 const state_1 = require("./state");
 /** A reply the machine sent, as opposed to none or an unsupported command. */
@@ -29,25 +30,56 @@ function fromMachine(reply) {
 function took(reply) {
     return reply?.status === protocol_1.REPLY_STATUS.ACCEPTED || reply?.status === protocol_1.REPLY_STATUS.COMPLETED;
 }
+/** Most command ids whose sends are counted; the oldest are forgotten first. */
+const SENT_REMEMBERED = 10_000;
+/** Ends a dispatch or command still running when the gateway stops; the next leader takes it up from the row. */
+class GatewayStopped extends Error {
+    constructor() {
+        super('the gateway stopped');
+    }
+}
+exports.GatewayStopped = GatewayStopped;
 class Dispatcher {
     constructor(d) {
         this.d = d;
+        this.sent = new Map();
+        this.stopping = new AbortController();
+    }
+    /** Work still running sends nothing more and changes no row. */
+    stop() {
+        this.stopping.abort();
+    }
+    live() {
+        if (this.stopping.signal.aborted)
+            throw new GatewayStopped();
+    }
+    /** This send's attempt number, and whether the machine may have the command already. */
+    countSend(cmdId, since) {
+        const attempt = (this.sent.get(cmdId) ?? 0) + 1;
+        this.sent.delete(cmdId);
+        this.sent.set(cmdId, attempt);
+        if (this.sent.size > SENT_REMEMBERED)
+            this.sent.delete(this.sent.keys().next().value);
+        return { attempt, retry: attempt > 1 || (since !== undefined && since < this.d.state.takenAt) };
     }
-    /** False when the codec has no message for the command. */
-    async publish(machineId, op, cmdId, args) {
-        const wires = this.d.codec.encode((0, registration_1.identFor)(this.d.state, this.d.line, machineId), { cmdId, op, args });
+    /** False when the codec has no message for the command. `since` is when the command's row was created. */
+    async publish(machineId, op, cmdId, args, since) {
+        this.live();
+        const wires = this.d.codec.encode((0, registration_1.identFor)(this.d.state, this.d.line, machineId), { cmdId, op, args, ...this.countSend(cmdId, since) });
         for (const wire of wires)
             await this.d.bus.publish(wire.topic, wire.payload);
         return wires.length > 0;
     }
     /** Publish and wait for the reply, asking again with the same id; null when none came. */
-    async ask(machineId, op, cmdId, args) {
+    async ask(machineId, op, cmdId, args, since) {
         for (let attempt = 1; attempt <= this.d.dispatchAttempts; attempt++) {
+            this.live();
             const waiting = this.d.replies.wait(cmdId, this.d.replyTimeoutMs);
-            if (!await this.publish(machineId, op, cmdId, args)) {
+            if (!await this.publish(machineId, op, cmdId, args, since)) {
                 this.d.replies.settle({ cmdId, status: contracts_1.COMMAND_STATUS.UNSUPPORTED, reason: `the line's codec has no ${op} command` });
             }
             const reply = await waiting;
+            this.live();
             if (reply)
                 return reply;
         }
@@ -67,13 +99,17 @@ class Dispatcher {
      * A machine-command row: send it, then resolve the row with the reply, as the
      * machine that gave it. The command expires when the asking ends, so one that
      * reaches a machine after its row resolved no-reply is refused, never run.
+     * Rounds of the asking count from the row's creation, so the command sent
+     * again by a new gateway is the same command with the same arguments.
      */
     async control(a) {
-        const expiresAt = Date.now() + this.d.replyTimeoutMs * this.d.dispatchAttempts;
-        const reply = await this.ask(a.machineId, a.op, a.cmdId, { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) });
+        const now = Date.now();
+        const { round, expiresAt } = (0, pacing_1.dispatchWindow)(a.createdAt ?? now, now, this.d.replyTimeoutMs * this.d.dispatchAttempts);
+        const reply = await this.ask(a.machineId, a.op, (0, contracts_1.controlCommandId)(a.rowId, round), { ...a.args, expiresAt: (0, protocol_1.isoTs)(expiresAt) }, a.createdAt);
         // Timers may fire a little early; the command must have expired before the row says no-reply.
         if (!reply && Date.now() <= expiresAt)
-            await (0, promises_1.setTimeout)(expiresAt - Date.now() + 1);
+            await (0, promises_1.setTimeout)(expiresAt - Date.now() + 1, undefined, { signal: this.stopping.signal }).catch(() => undefined);
+        this.live();
         const outcome = reply
             ? { accepted: reply.status === protocol_1.REPLY_STATUS.ACCEPTED, machineId: a.machineId, op: a.op, status: reply.status, reason: reply.reason }
             : { accepted: false, machineId: a.machineId, op: a.op, status: contracts_1.COMMAND_STATUS.NO_REPLY, reason: `no reply after ${this.d.dispatchAttempts} attempts` };
@@ -97,10 +133,12 @@ class Dispatcher {
         // Known before any reply, so a failure that follows an acceptance finds its row.
         for (const job of jobs)
             state.rememberDispatch(`${a.cmdId}${job.suffix}`, a.rowId);
-        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units })));
+        const since = state.get(a.rowId)?.createdAt;
+        const replies = await Promise.all(jobs.map((job) => this.ask(a.machineId, protocol_1.OP.DISPATCH, `${a.cmdId}${job.suffix}`, { ...a.args, jobId: job.jobId, units: job.units }, since)));
         const refused = replies.find((r) => r && !took(r));
         const silent = jobs.filter((_, i) => !replies[i]);
         const held = !refused && silent.length ? await this.heldJobs(a.machineId, a.cmdId) : null;
+        this.live();
         const reconciled = !!held && silent.every((j) => held.has(j.jobId));
         const row = state.get(a.rowId);
         if (!refused && (!silent.length || reconciled)) {
@@ -132,12 +170,14 @@ class Dispatcher {
     }
     /** The units were already at the machine: its acceptance completes the handoff. */
     async received(a) {
+        this.live();
         const units = a.args.units.map((u) => u.unitId);
         this.check(a.rowId, 'resolve', await this.d.lt.resolve(a.machineId, a.rowId, { received: true, receiver: a.machineId, units }));
         this.d.state.close(a.rowId);
     }
     /** Ask a silent machine what it holds; null when it cannot say or does not answer in time. */
     async heldJobs(machineId, cmdId) {
+        this.live();
         const waiting = this.d.replies.waitSnapshot(machineId, this.d.replyTimeoutMs);
         if (!await this.publish(machineId, protocol_1.OP.SNAPSHOT, `${cmdId}:status`, {}))
             this.d.replies.settleSnapshot(machineId, null);

package/build/services/machines/gateway/executor.d.ts

@@ -12,7 +12,7 @@ import type { AddressBook } from './address-book';
 import { type LineEscalations } from './escalations';
 import { type GatewayLog } from './log';
 import type { ReplyWaiter } from './reply-waiter';
-import type { GatewayState } from './state';
+import { type GatewayState } from './state';
 export interface ExecutorDeps {
     bus: MachineBus;
     lt: LineEscalations;
@@ -41,7 +41,12 @@ export declare class Executor {
     stop(): void;
     private track;
     private run;
-    /** A dispatch's second reply: ready, or a failure to prepare that ends the handoff. */
+    /**
+     * A dispatch's second reply: ready, or a failure to prepare that ends the
+     * handoff. When the handoff into the machine has already ended, as one
+     * dispatched on arrival does at acceptance, the failure ends the handoff
+     * carrying the units out of it.
+     */
     private lateReply;
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     private accumulate;