@hotmeshio/long-tail 0.26.0 → 0.27.0

package/build/services/machines/gateway/state.d.ts

@@ -33,6 +33,14 @@ export interface HandoffRow extends HandoffFacets {
     trace?: TraceEntry[];
     /** Each unit's arrival event, as its receiver reported it (dispatch on arrival). */
     arrivals?: Record<string, Record<string, any>>;
+    /** Lifecycle item keys already on the row (lifecycle.ts). */
+    noted?: string[];
+    /** Parked with a limit on its items, which lifecycle notes must never use up. */
+    capped?: boolean;
+    /** When the row was created (epoch ms). */
+    createdAt?: number;
+    /** When its last unit arrived (epoch ms, dispatch on arrival); its dispatch windows count from here, else from createdAt. */
+    dueAt?: number;
 }
 export interface AlarmTrack {
     machineId: string;
@@ -42,6 +50,8 @@ export interface AlarmTrack {
     rowId?: string;
     /** Created and not yet resolved by anyone. */
     open: boolean;
+    /** Rows the alarm paused, noted again when it clears. */
+    noted?: string[];
 }
 export interface LinkTrack {
     online: boolean;
@@ -58,6 +68,16 @@ export declare class GatewayState {
     unitsPerDispatch: number | null;
     /** Jobs go to a receiver once its units arrive there, from the codec. */
     dispatchOnArrival: boolean;
+    /** Note each job's start, progress steps and pauses on the row that holds it (lifecycle.ts). */
+    lifecycle: boolean;
+    /** Active alarms are retained on the broker; otherwise snapshots list them (codec.ts retainedAlarms). */
+    retainedAlarms: boolean;
+    /** Reading the broker's retained messages on taking the line; a snapshot read back may be older than its alarms. */
+    readingRetained: boolean;
+    /** When this gateway took the line (epoch ms); a command for an older row may have been sent by an earlier one. */
+    takenAt: number;
+    /** With lifecycle notes, progress is noted each time it crosses this many percent; 0 notes none. */
+    progressStep: number;
     /** While reconciling, new handoffs wait so pacing sees the whole line before anything is sent. */
     holdDispatch: boolean;
     /** Units at each machine that arrived and have not left: machine → unit → job. */
@@ -102,7 +122,7 @@ export declare class GatewayState {
     rememberFinish(machineId: string, unitId: string, body: Record<string, any>): void;
     takeFinish(machineId: string, unitId: string): Record<string, any> | undefined;
     rememberDispatch(cmdId: string, rowId: string): void;
-    /** The open row an accepted dispatch command belongs to. */
+    /** The row a dispatch command belongs to, open or recently closed. */
     rowForDispatch(cmdId: string): HandoffRow | undefined;
     /** Track a machine-command row; false when it is already tracked or has ended. */
     trackCommand(id: string): boolean;

package/build/services/machines/gateway/state.js

@@ -23,6 +23,16 @@ class GatewayState {
         this.unitsPerDispatch = null;
         /** Jobs go to a receiver once its units arrive there, from the codec. */
         this.dispatchOnArrival = false;
+        /** Note each job's start, progress steps and pauses on the row that holds it (lifecycle.ts). */
+        this.lifecycle = false;
+        /** Active alarms are retained on the broker; otherwise snapshots list them (codec.ts retainedAlarms). */
+        this.retainedAlarms = false;
+        /** Reading the broker's retained messages on taking the line; a snapshot read back may be older than its alarms. */
+        this.readingRetained = false;
+        /** When this gateway took the line (epoch ms); a command for an older row may have been sent by an earlier one. */
+        this.takenAt = 0;
+        /** With lifecycle notes, progress is noted each time it crosses this many percent; 0 notes none. */
+        this.progressStep = 25;
         /** While reconciling, new handoffs wait so pacing sees the whole line before anything is sent. */
         this.holdDispatch = false;
         /** Units at each machine that arrived and have not left: machine → unit → job. */
@@ -55,7 +65,10 @@ class GatewayState {
         const existing = this.openRows.get(row.id);
         // An event may carry metadata older than what the gateway already wrote: keep both traces.
         this.openRows.set(row.id, existing
-            ? { ...row, dispatch: existing.dispatch, trace: (0, trace_1.mergeTrace)(row.trace, existing.trace), arrivals: existing.arrivals }
+            ? {
+                ...row, dispatch: existing.dispatch, trace: (0, trace_1.mergeTrace)(row.trace, existing.trace), arrivals: existing.arrivals,
+                dueAt: existing.dueAt, noted: existing.noted ?? row.noted,
+            }
             : row);
     }
     get(id) {
@@ -135,10 +148,10 @@ class GatewayState {
                 this.dispatches.delete(oldest);
         }
     }
-    /** The open row an accepted dispatch command belongs to. */
+    /** The row a dispatch command belongs to, open or recently closed. */
     rowForDispatch(cmdId) {
         const rowId = this.dispatches.get(cmdId);
-        return rowId ? this.openRows.get(rowId) : undefined;
+        return rowId ? this.openRows.get(rowId) ?? this.closedRows.get(rowId) : undefined;
     }
     /** Track a machine-command row; false when it is already tracked or has ended. */
     trackCommand(id) {

package/build/services/machines/leadership.d.ts

@@ -4,19 +4,32 @@
  * waits on the server, so standby replicas hold no timers and the next one
  * takes over the moment the leader's session ends.
  */
+import type { PoolClient } from 'pg';
+export interface LeadershipOptions {
+    /** A dedicated connection for the lock; the shared pool's by default. */
+    connect?: () => Promise<PoolClient>;
+    /** The pause before contending again. */
+    retryMs?: number;
+}
 export declare class LineLeadership {
     readonly key: string;
     private client?;
     private stopped;
     private leading;
-    constructor(key: string);
+    private endTurn?;
+    private readonly connect;
+    private readonly retryMs;
+    constructor(key: string, opts?: LeadershipOptions);
     /**
-     * Wait for the lock, then call `lead`. When leading fails or the
-     * connection drops, the lock is released, `lost` runs, and the line
-     * contends again after a pause.
+     * Wait for the lock, then call `lead`. When the connection is refused,
+     * leading fails or the connection drops, `lost` runs before this replica lets
+     * the lock go, so no other replica leads alongside a gateway that is
+     * stopping; then the line contends again after a pause.
      */
     run(lead: () => Promise<void>, lost: (err: Error) => Promise<void>): Promise<void>;
     private leadOnce;
+    /** Lead while the lock is held; the error that ended it. */
+    private holdLock;
     /** A leader unlocks; a replica still waiting drops its connection, which ends the wait. */
     stop(): Promise<void>;
 }

package/build/services/machines/leadership.js

@@ -12,49 +12,74 @@ const LOCK_SQL = 'SELECT pg_advisory_lock(hashtext($1))';
 const RETRY_MS = 5_000;
 const UNLOCK_SQL = 'SELECT pg_advisory_unlock(hashtext($1))';
 class LineLeadership {
-    constructor(key) {
+    constructor(key, opts = {}) {
         this.key = key;
         this.stopped = false;
         this.leading = false;
+        this.connect = opts.connect ?? (() => (0, db_1.getPool)().connect());
+        this.retryMs = opts.retryMs ?? RETRY_MS;
     }
     /**
-     * Wait for the lock, then call `lead`. When leading fails or the
-     * connection drops, the lock is released, `lost` runs, and the line
-     * contends again after a pause.
+     * Wait for the lock, then call `lead`. When the connection is refused,
+     * leading fails or the connection drops, `lost` runs before this replica lets
+     * the lock go, so no other replica leads alongside a gateway that is
+     * stopping; then the line contends again after a pause.
      */
     async run(lead, lost) {
         while (!this.stopped) {
-            const err = await this.leadOnce(lead);
+            await this.leadOnce(lead, lost);
             if (this.stopped)
                 return;
-            await lost(err);
-            await new Promise((r) => setTimeout(r, RETRY_MS));
+            await new Promise((r) => setTimeout(r, this.retryMs));
         }
     }
-    async leadOnce(lead) {
-        const client = await (0, db_1.getPool)().connect();
+    async leadOnce(lead, lost) {
+        let client;
+        try {
+            client = await this.connect();
+        }
+        catch (err) {
+            if (!this.stopped)
+                await lost(err);
+            return;
+        }
+        if (this.stopped) {
+            client.release(true);
+            return;
+        }
         this.client = client;
-        const dropped = new Promise((resolve) => client.once('error', resolve));
+        const err = await this.holdLock(client, lead);
+        this.leading = false;
+        this.endTurn = undefined;
+        // stop() has let the connection go.
+        if (this.stopped)
+            return;
+        await lost(err);
+        this.client = undefined;
+        client.release(true);
+    }
+    /** Lead while the lock is held; the error that ended it. */
+    async holdLock(client, lead) {
+        const ended = new Promise((resolve) => {
+            client.once('error', resolve);
+            this.endTurn = () => resolve(new Error('leadership stopped'));
+        });
         try {
             await client.query(LOCK_SQL, [this.key]);
+            if (this.stopped)
+                return new Error('leadership stopped');
             this.leading = true;
             await lead();
-            return await dropped;
+            return await ended;
         }
         catch (err) {
             return err;
         }
-        finally {
-            this.leading = false;
-            if (!this.stopped) {
-                this.client = undefined;
-                client.release(true);
-            }
-        }
     }
     /** A leader unlocks; a replica still waiting drops its connection, which ends the wait. */
     async stop() {
         this.stopped = true;
+        this.endTurn?.();
         const client = this.client;
         this.client = undefined;
         if (!client)

package/build/services/machines/rows.js

@@ -16,15 +16,30 @@ function paramsOf(v) {
     return v && typeof v === 'object' && !Array.isArray(v) ? v : undefined;
 }
 function artifactOf(v) {
-    return v && typeof v.url === 'string' && v.url && typeof v.sha256 === 'string' && v.sha256 ? { url: v.url, sha256: v.sha256 } : undefined;
+    if (!v || typeof v.url !== 'string' || !v.url || typeof v.sha256 !== 'string' || !v.sha256)
+        return undefined;
+    return {
+        url: v.url, sha256: v.sha256,
+        ...(typeof v.id === 'string' && v.id ? { id: v.id } : {}),
+        ...(typeof v.contentType === 'string' && v.contentType ? { contentType: v.contentType } : {}),
+        ...(Number.isInteger(v.sizeBytes) && v.sizeBytes >= 0 ? { sizeBytes: v.sizeBytes } : {}),
+    };
+}
+function artifactsOf(v) {
+    const list = Array.isArray(v) ? v.map(artifactOf).filter((a) => !!a) : [];
+    return list.length ? list : undefined;
 }
 function unitsOf(v) {
     if (!Array.isArray(v))
         return [];
     return v.filter((u) => u && u.unitId).map((u) => {
         const artifact = artifactOf(u.artifact);
+        const artifacts = artifactsOf(u.artifacts);
         const parameters = paramsOf(u.parameters);
-        return { unitId: String(u.unitId), carrierId: str(u.carrierId), ...(artifact ? { artifact } : {}), ...(parameters ? { parameters } : {}) };
+        return {
+            unitId: String(u.unitId), carrierId: str(u.carrierId),
+            ...(artifact ? { artifact } : {}), ...(artifacts ? { artifacts } : {}), ...(parameters ? { parameters } : {}),
+        };
     });
 }
 /** Read standard handoff facets; null when the line, job or units are missing. */

package/build/services/machines/select.d.ts

@@ -6,12 +6,19 @@
 import { type MachineLineConfig } from './config';
 import { type LineDef } from '../../lib/machines/line';
 import { type LineCodec } from '../../lib/machines/protocol';
+import { messagingRegistry } from '../../lib/messaging';
 import type { CallbackEventAdapter } from '../../lib/events/callback';
 import type { EscalationFeed, MachineBus, MachineLink } from '../../lib/machines/transport/delivery';
 /** The codec for a line when the host supplies none: the line protocol under MACHINE_TOPIC_ROOT. */
 export declare function defaultCodec(config: MachineLineConfig, line?: LineDef): LineCodec;
-/** The gateway's view of every machine on the line. */
-export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec): MachineBus;
+/** What the stream bus needs from Long Tail's messaging; the registry by default. */
+export type LineMessaging = Pick<typeof messagingRegistry, 'streams' | 'pubsub'>;
+/**
+ * The gateway's view of every machine on the line. With `nats`, events are read
+ * from Long Tail's messaging streams; MACHINE_SITE_NATS_URL instead names a NATS
+ * server of the site's own, which the bus connects to itself.
+ */
+export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec, messaging?: LineMessaging): MachineBus;
 /** Long Tail's escalation events for the gateway; `in-process` needs the running instance's callback adapter. */
 export declare function createEscalationFeed(config: MachineLineConfig, line: LineDef, callbacks?: CallbackEventAdapter): EscalationFeed;
 /** One MQTT link per simulated machine; machines speak MQTT in every networked mode. */

package/build/services/machines/select.js

@@ -15,8 +15,11 @@ const protocol_1 = require("../../lib/machines/protocol");
 const mqtt_bus_1 = require("../../lib/machines/transport/mqtt-bus");
 const mqtt_link_1 = require("../../lib/machines/transport/mqtt-link");
 const nats_bus_1 = require("../../lib/machines/transport/nats-bus");
+const stream_bus_1 = require("../../lib/machines/transport/stream-bus");
 const callback_feed_1 = require("../../lib/machines/transport/callback-feed");
 const nats_feed_1 = require("../../lib/machines/transport/nats-feed");
+const pubsub_feed_1 = require("../../lib/machines/transport/pubsub-feed");
+const messaging_1 = require("../../lib/messaging");
 const IN_PROCESS_ONLY = 'lives inside one process; build MemoryMachineBus and MemoryEscalationFeed directly';
 function mqttOf(config) {
     if (!config.mqtt)
@@ -27,14 +30,20 @@ function mqttOf(config) {
 function defaultCodec(config, line) {
     return (0, protocol_1.genericCodec)({ root: config.topicRoot, line });
 }
-/** The gateway's view of every machine on the line. */
-function createMachineBus(config, line, codec = defaultCodec(config, line)) {
+/**
+ * The gateway's view of every machine on the line. With `nats`, events are read
+ * from Long Tail's messaging streams; MACHINE_SITE_NATS_URL instead names a NATS
+ * server of the site's own, which the bus connects to itself.
+ */
+function createMachineBus(config, line, codec = defaultCodec(config, line), messaging = messaging_1.messagingRegistry) {
     switch (config.transport) {
-        case config_1.MACHINE_TRANSPORT.NATS:
-            return new nats_bus_1.NatsMachineBus({
-                natsUrl: config.siteNatsUrl, user: config.siteNatsUser, password: config.siteNatsPassword,
-                mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec,
-            });
+        case config_1.MACHINE_TRANSPORT.NATS: {
+            const where = { mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec };
+            if (config.siteNatsUrl) {
+                return new nats_bus_1.NatsMachineBus({ natsUrl: config.siteNatsUrl, user: config.siteNatsUser, password: config.siteNatsPassword, ...where });
+            }
+            return new stream_bus_1.StreamMachineBus({ streams: messaging.streams, pubsub: messaging.pubsub, ...where });
+        }
         case config_1.MACHINE_TRANSPORT.MQTT:
             return new mqtt_bus_1.MqttMachineBus(mqttOf(config), (0, line_1.controllerId)(line), (0, protocol_1.lineFilter)(codec.root, line.siteId, line.lineId));
         default:
@@ -49,6 +58,10 @@ function createEscalationFeed(config, line, callbacks) {
                 throw new Error('MACHINE_ESCALATION_FEED=in-process runs inside start({ machines })');
             return new callback_feed_1.CallbackEscalationFeed(callbacks);
         case config_1.ESCALATION_FEED.CORE:
+            if (messaging_1.messagingRegistry.hasProvider)
+                return new pubsub_feed_1.PubSubEscalationFeed(messaging_1.messagingRegistry.pubsub);
+            if (!config.ltNatsUrl)
+                throw new Error('MACHINE_ESCALATION_FEED=core needs Long Tail messaging (LT_MESSAGING=nats) or NATS_URL');
             return new nats_feed_1.CoreEscalationFeed({ natsUrl: config.ltNatsUrl, token: config.ltNatsToken });
         default:
             throw new Error(`MACHINE_ESCALATION_FEED=${config.feed} ${IN_PROCESS_ONLY}`);

package/build/services/machines/surface.js

@@ -30,24 +30,30 @@ class LineRun {
         this.config = config;
         this.opts = opts;
         this.log = log;
+        this.stopped = false;
         this.leadership = new leadership_1.LineLeadership(`lt-machine-line:${line.siteId}/${line.lineId}`);
     }
     start() {
         const where = { site: this.line.siteId, line: this.line.lineId };
         this.log.info(log_1.GATEWAY_LOG.STANDBY, where);
-        this.leadership.run(() => this.lead(), async (err) => {
+        this.leadership.run(() => (this.leading = this.lead()), async (err) => {
             this.log.warn(log_1.GATEWAY_LOG.LEADERSHIP_LOST, { ...where, error: err.message });
             await this.gateway?.stop();
             this.gateway = undefined;
         }).catch((err) => this.log.error(log_1.GATEWAY_LOG.START_FAILED, { ...where, error: err.message }));
     }
+    /** A gateway still starting finishes first, so none is left running after stop. */
     async stop() {
+        this.stopped = true;
+        await this.leading?.catch(() => undefined);
         await this.gateway?.stop();
         await this.leadership.stop();
     }
     async lead() {
         // 1. Identities, 2. gateway on the line's bus and feed, 3. reconcile and consume (gateway.start)
         const ids = await (0, identities_1.ensureLineBots)(this.line);
+        if (this.stopped)
+            return;
         const userFor = (as) => {
             const id = ids.get(as);
             if (!id)
@@ -68,7 +74,7 @@ class LineRun {
             settings: {
                 topicRoot: config.topicRoot, replyTimeoutMs: config.replyTimeoutMs, dispatchAttempts: config.dispatchAttempts,
                 offlineDebounceMs: config.offlineDebounceMs, dispatchTtlMs: config.dispatchTtlMs, quietMs: config.quietMs,
-                receiptDelayMs: config.receiptDelayMs,
+                receiptDelayMs: config.receiptDelayMs, lifecycle: config.lifecycle, progressStep: config.progressStep,
             },
         });
         await this.gateway.start();

package/build/services/machines/testing/fake-long-tail.d.ts

@@ -26,6 +26,7 @@ export interface FakeRow {
     resolved_by?: string;
     envelope?: string;
     escalation_payload?: string;
+    created_at?: string;
 }
 export interface Call {
     verb: string;

package/build/services/machines/testing/fake-long-tail.js

@@ -43,7 +43,7 @@ class FakeLongTail {
         });
     }
     add(input) {
-        const row = { ...input, id: `e${++this.seq}`, status: 'pending', items: [] };
+        const row = { ...input, id: `e${++this.seq}`, status: 'pending', items: [], created_at: new Date().toISOString() };
         this.rows.set(row.id, row);
         this.emit(row, 'created');
         return row;
@@ -132,7 +132,7 @@ class FakeLongTail {
             return null;
         return {
             id: row.id, type: row.type, subtype: row.subtype, role: row.role, status: row.status,
-            assigned_to: row.assigned_to ?? null, resolved_by: row.resolved_by ?? null, metadata: { ...row.metadata },
+            assigned_to: row.assigned_to ?? null, resolved_by: row.resolved_by ?? null, metadata: { ...row.metadata }, created_at: row.created_at ?? null,
             // Stored serialized, as Long Tail returns it.
             resolver_payload: row.resolver_payload ? JSON.stringify(row.resolver_payload) : null,
         };
@@ -140,7 +140,10 @@ class FakeLongTail {
     async listOpen(query) {
         return [...this.rows.values()]
             .filter((r) => r.status === 'pending' && (!query.type || r.type === query.type) && (!query.role || r.role === query.role))
-            .map((r) => ({ id: r.id, type: r.type, subtype: r.subtype, role: r.role, status: r.status, assigned_to: r.assigned_to ?? null, metadata: r.metadata }));
+            .map((r) => ({
+            id: r.id, type: r.type, subtype: r.subtype, role: r.role, status: r.status, assigned_to: r.assigned_to ?? null, metadata: r.metadata,
+            created_at: r.created_at ?? null,
+        }));
     }
     ofType(type) {
         return [...this.rows.values()].filter((r) => r.type === type);

package/build/start/adapters.js

@@ -6,6 +6,8 @@ const pino_1 = require("../lib/logger/pino");
 const telemetry_1 = require("../lib/telemetry");
 const honeycomb_1 = require("../lib/telemetry/honeycomb");
 const events_1 = require("../lib/events");
+const messaging_1 = require("../lib/messaging");
+const config_1 = require("../lib/messaging/config");
 const nats_1 = require("../lib/events/nats");
 const socketio_1 = require("../lib/events/socketio");
 const maintenance_1 = require("../services/maintenance");
@@ -44,6 +46,10 @@ function registerAdapters(startConfig) {
     else if (startConfig.telemetry?.honeycomb) {
         telemetry_1.telemetryRegistry.register(new honeycomb_1.HoneycombTelemetryAdapter(startConfig.telemetry.honeycomb));
     }
+    // Messaging: one provider shared by events and the machine surface.
+    const messaging = (0, config_1.resolveMessaging)(startConfig);
+    if (messaging.provider)
+        messaging_1.messagingRegistry.register(messaging.provider);
     // Events — register the configured transport.
     // Socket.IO is the default for single-container ("nothing but Postgres").
     // When NATS or custom adapters are configured, Socket.IO is NOT registered —
@@ -54,8 +60,11 @@ function registerAdapters(startConfig) {
             events_1.eventRegistry.register(adapter);
         }
     }
-    else if (startConfig.events?.nats) {
-        events_1.eventRegistry.register(new nats_1.NatsEventAdapter(startConfig.events.nats));
+    else if (messaging.provider?.name === config_1.MESSAGING_KIND.NATS) {
+        events_1.eventRegistry.register(new nats_1.NatsEventAdapter({
+            messaging: messaging.provider, wsUrl: messaging.ws.url, wsProxy: messaging.ws.proxy, token: messaging.ws.token,
+            subjectPrefix: startConfig.events?.nats?.subjectPrefix,
+        }));
     }
     else {
         events_1.eventRegistry.register(new socketio_1.SocketIOEventAdapter({

package/build/start/index.js

@@ -5,6 +5,7 @@ const hotmesh_1 = require("@hotmeshio/hotmesh");
 const logger_1 = require("../lib/logger");
 const telemetry_1 = require("../lib/telemetry");
 const events_1 = require("../lib/events");
+const messaging_1 = require("../lib/messaging");
 const callback_1 = require("../lib/events/callback");
 const surface_1 = require("../services/machines/surface");
 const roles_1 = require("../services/machines/roles");
@@ -66,8 +67,12 @@ async function start(startConfig) {
     (0, config_1.applyDatabaseConfig)(startConfig.database);
     // 2. Apply server/auth config
     await (0, config_1.applyServerAuthConfig)(startConfig);
-    // 3. Register adapters
+    // 3. Register adapters; messaging connects first, since events and machines use it.
+    // An unreachable NATS is logged and Long Tail starts, as events always have; what needs streams fails then.
     (0, adapters_1.registerAdapters)(startConfig);
+    await messaging_1.messagingRegistry.connect().catch((err) => {
+        logger_1.loggerRegistry.error(`${messaging_1.MESSAGING_LOG.CONNECT_FAILED} ${err instanceof Error ? err.message : String(err)}`);
+    });
     logger_1.loggerRegistry.info('[long-tail] starting...');
     // 4-5. Collect workers, run migrations, start workers, seed data
     if (startConfig.machines && startConfig.machines.declareRoles !== false) {
@@ -110,6 +115,7 @@ async function start(startConfig) {
         if (events_1.eventRegistry.hasAdapters) {
             await events_1.eventRegistry.disconnect();
         }
+        await messaging_1.messagingRegistry.disconnect();
         if (telemetry_1.telemetryRegistry.hasAdapter) {
             await telemetry_1.telemetryRegistry.disconnect();
         }