@hotmeshio/long-tail 0.28.1 → 0.28.3

package/build/lib/events/publish.js

@@ -18,12 +18,13 @@ const index_1 = require("./index");
 const matching_1 = require("./matching");
 const logger_1 = require("../logger");
 /**
- * Fire-and-forget publish helper. Swallows errors (best-effort).
+ * Fire-and-forget publish helper. Swallows errors (best-effort). The publish
+ * is noted at debug: a line's wire can carry many events a second.
  */
 function fireAndForget(event) {
     if (!index_1.eventRegistry.hasAdapters)
         return Promise.resolve();
-    logger_1.loggerRegistry.info(`[lt-pub] ${event.type} ${event.workflowId || ''} ${event.escalationId || event.taskId || ''}`);
+    logger_1.loggerRegistry.debug(`[lt-pub] ${event.type} ${event.workflowId || ''} ${event.escalationId || event.taskId || ''}`);
     return index_1.eventRegistry.publish(event).catch(() => { });
 }
 /**

package/build/lib/machines/transport/nats-bus.d.ts

@@ -20,6 +20,8 @@ export interface NatsBusOptions {
     /** The codecs of every line on this site, so the streams keep all their subjects. */
     siteCodecs?: StreamCodec[];
     nakDelayMs?: number;
+    /** Told once when the consumer can deliver nothing more, so the line can give its lead up and contend again. */
+    onLost?: (reason: string) => void;
 }
 export declare class NatsMachineBus implements MachineBus {
     private readonly opts;

package/build/lib/machines/transport/stream-bus.d.ts

@@ -24,6 +24,8 @@ export interface StreamBusOptions {
     /** The codecs of every line on this site; the streams keep all their subjects, whichever line starts last. */
     siteCodecs?: StreamCodec[];
     nakDelayMs?: number;
+    /** Told once when the consumer can deliver nothing more, so the line can give its lead up and contend again. */
+    onLost?: (reason: string) => void;
 }
 export declare class StreamMachineBus implements MachineBus {
     private readonly opts;

package/build/lib/machines/transport/stream-bus.js

@@ -30,6 +30,12 @@ class StreamMachineBus {
             await streams.ensureStream(spec);
         const consumer = (0, streams_1.lineEventConsumerSpec)(codec, siteId, lineId, this.durableName);
         await streams.ensureConsumer(streams_1.LINE_STREAM.EVENTS, consumer);
+        const held = await streams.describeConsumer(streams_1.LINE_STREAM.EVENTS, consumer.name);
+        const reads = [...(held?.filterSubjects ?? [])].sort().join(', ');
+        const wants = [...consumer.filterSubjects].sort().join(', ');
+        if (reads !== wants) {
+            throw new Error(`${consumer.name} reads ${reads || 'nothing'} and not ${wants}; the consumer did not follow the line`);
+        }
         const events = await streams.describeStream(streams_1.LINE_STREAM.EVENTS);
         const missing = events ? (0, streams_1.uncoveredSubjects)(events, consumer.filterSubjects) : consumer.filterSubjects;
         if (missing.length) {
@@ -38,8 +44,8 @@ class StreamMachineBus {
         this.session = await mqtt_session_1.MqttSession.open(this.opts.mqtt, controllerId);
     }
     async consume(handler) {
-        const { streams, pubsub, codec, siteId, lineId, nakDelayMs = 1_000 } = this.opts;
-        this.subs.push(await streams.consume(streams_1.LINE_STREAM.EVENTS, this.durableName, (m) => this.queue.run(() => handler({ topic: (0, protocol_1.toTopic)(m.subject), payload: m.payload, deliveries: m.deliveries })), { nakDelayMs }));
+        const { streams, pubsub, codec, siteId, lineId, nakDelayMs = 1_000, onLost } = this.opts;
+        this.subs.push(await streams.consume(streams_1.LINE_STREAM.EVENTS, this.durableName, (m) => this.queue.run(() => handler({ topic: (0, protocol_1.toTopic)(m.subject), payload: m.payload, deliveries: m.deliveries })), { nakDelayMs, onLost }));
         for (const subject of (0, streams_1.telemetrySubjects)(codec, siteId, lineId)) {
             this.subs.push(pubsub.subscribe(subject, (m) => void this.queue.run(() => handler({ topic: (0, protocol_1.toTopic)(m.subject), payload: m.payload, deliveries: 1 }))));
         }

package/build/lib/machines/transport/streams.d.ts

@@ -16,7 +16,8 @@ export declare function filterToSubject(filter: string): string;
 /**
  * The site's streams: every line under each codec's root, since lines on
  * different protocols share one site NATS. A root alone means the generic
- * layout; several codecs give the union of their subjects, in order.
+ * layout; several codecs give the union of their subjects, sorted, so every
+ * line asks the site for the same stream whichever codec it lists first.
  */
 export declare function lineStreamSpecs(codecs: StreamCodec | string | (StreamCodec | string)[]): StreamSpec[];
 /** Whether a stream subject pattern keeps everything a consumer filter would read. */

package/build/lib/machines/transport/streams.js

@@ -31,11 +31,12 @@ function subjectsFor(codec, channels, siteId, lineId) {
 /**
  * The site's streams: every line under each codec's root, since lines on
  * different protocols share one site NATS. A root alone means the generic
- * layout; several codecs give the union of their subjects, in order.
+ * layout; several codecs give the union of their subjects, sorted, so every
+ * line asks the site for the same stream whichever codec it lists first.
  */
 function lineStreamSpecs(codecs) {
     const list = (Array.isArray(codecs) ? codecs : [codecs]).map(codecOf);
-    const union = (channel) => [...new Set(list.flatMap((c) => subjectsFor(c, c.channels[channel])))];
+    const union = (channel) => [...new Set(list.flatMap((c) => subjectsFor(c, c.channels[channel])))].sort();
     return [
         { name: exports.LINE_STREAM.EVENTS, subjects: union('events'), maxAgeMs: 30 * DAY_MS },
         { name: exports.LINE_STREAM.TELEMETRY, subjects: union('telemetry'), maxAgeMs: 3 * DAY_MS },

package/build/lib/messaging/memory.js

@@ -39,16 +39,20 @@ class MemoryStreams {
     async describeStream(name) {
         return this.streams.get(name)?.spec ?? null;
     }
+    /** An existing consumer keeps its place while its subjects match; one whose subjects changed starts again. */
     async ensureConsumer(stream, spec) {
         const s = this.stream(stream);
         const existing = s.consumers.get(spec.name);
-        if (existing) {
-            existing.spec = { ...existing.spec, filterSubjects: spec.filterSubjects };
+        const same = (a, b) => [...a].sort().join('|') === [...b].sort().join('|');
+        if (existing && same(existing.spec.filterSubjects, spec.filterSubjects))
             return;
-        }
         const floor = (spec.startAt ?? constants_1.START_AT.NEW) === constants_1.START_AT.ALL ? 0 : s.seq;
         s.consumers.set(spec.name, { spec, floor, settled: new Set(), deliveries: new Map(), retryAt: new Map() });
     }
+    async describeConsumer(stream, name) {
+        const spec = this.streams.get(stream)?.consumers.get(name)?.spec;
+        return spec ? { ...spec, filterSubjects: [...spec.filterSubjects] } : null;
+    }
     async append(subject, payload) {
         const now = Date.now();
         for (const s of this.streams.values()) {

package/build/lib/messaging/nats/permissions.js

@@ -15,6 +15,7 @@ function jsPermissions({ streams, durable }) {
         `$JS.API.CONSUMER.CREATE.${s}.${durable}`,
         `$JS.API.CONSUMER.CREATE.${s}.${durable}.>`,
         `$JS.API.CONSUMER.DURABLE.CREATE.${s}.${durable}`,
+        `$JS.API.CONSUMER.DELETE.${s}.${durable}`,
         `$JS.API.CONSUMER.MSG.NEXT.${s}.${durable}`,
         `$JS.ACK.${s}.${durable}.>`,
     ]);

package/build/lib/messaging/nats/streams.d.ts

@@ -1,18 +1,31 @@
 /**
  * Durable streams on JetStream. A spec becomes a file-backed stream or an
- * explicit-ack durable consumer; ensuring either is safe on every boot, and an
- * existing consumer keeps its position while its subjects follow the spec.
+ * explicit-ack durable consumer; ensuring either is safe on every boot and
+ * from several lines at once. An existing consumer keeps its position while
+ * its subjects match the spec; one whose subjects changed is made again, since
+ * a position in another protocol's messages means nothing to this one.
  */
 import { type ConsumerConfig, type JetStreamClient, type JetStreamManager, type StreamConfig } from 'nats';
 import type { ConsumerSpec, Delivery, LTConsumeOptions, LTStreamMessage, LTStreams, LTSubscription, StreamSpec, Subject } from '../../../types/messaging';
 export declare function streamConfig(spec: StreamSpec): Partial<StreamConfig>;
 export declare function consumerConfig(spec: ConsumerSpec): Partial<ConsumerConfig>;
-/** Create the stream, or update an existing one to the spec. */
+/**
+ * Create the stream, or update an existing one to the spec. Two lines may
+ * ensure the same stream at once: the create that loses the race finds the
+ * stream in use and updates it instead.
+ */
 export declare function ensureStream(jsm: JetStreamManager, spec: StreamSpec): Promise<void>;
 /** The stream as JetStream holds it, or null when there is none. */
 export declare function describeStream(jsm: JetStreamManager, name: string): Promise<StreamSpec | null>;
-/** Create the consumer, or move an existing one to the spec's subjects without losing its place. */
+/**
+ * Create the consumer, or keep an existing one whose subjects match the spec,
+ * position and all. One whose subjects differ is deleted and made again at
+ * the stream's newest message, since its old position was in another
+ * protocol's messages.
+ */
 export declare function ensureConsumer(jsm: JetStreamManager, stream: string, spec: ConsumerSpec): Promise<void>;
+/** The consumer as JetStream holds it, or null when there is none. */
+export declare function describeConsumer(jsm: JetStreamManager, stream: string, name: string): Promise<ConsumerSpec | null>;
 export declare function payloadOf(m: {
     data: Uint8Array;
 }): string;
@@ -23,6 +36,7 @@ export declare class NatsStreams implements LTStreams {
     ensureStream(spec: StreamSpec): Promise<void>;
     ensureConsumer(stream: string, spec: ConsumerSpec): Promise<void>;
     describeStream(name: string): Promise<StreamSpec | null>;
+    describeConsumer(stream: string, name: string): Promise<ConsumerSpec | null>;
     append(subject: Subject, payload: string): Promise<void>;
     /** One message at a time: the next is not taken until the handler settles this one. */
     consume(stream: string, consumer: string, handler: (m: LTStreamMessage) => Promise<Delivery>, opts?: LTConsumeOptions): Promise<LTSubscription>;

package/build/lib/messaging/nats/streams.js

@@ -1,8 +1,10 @@
 "use strict";
 /**
  * Durable streams on JetStream. A spec becomes a file-backed stream or an
- * explicit-ack durable consumer; ensuring either is safe on every boot, and an
- * existing consumer keeps its position while its subjects follow the spec.
+ * explicit-ack durable consumer; ensuring either is safe on every boot and
+ * from several lines at once. An existing consumer keeps its position while
+ * its subjects match the spec; one whose subjects changed is made again, since
+ * a position in another protocol's messages means nothing to this one.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.NatsStreams = void 0;
@@ -11,11 +13,15 @@ exports.consumerConfig = consumerConfig;
 exports.ensureStream = ensureStream;
 exports.describeStream = describeStream;
 exports.ensureConsumer = ensureConsumer;
+exports.describeConsumer = describeConsumer;
 exports.payloadOf = payloadOf;
 const nats_1 = require("nats");
 const constants_1 = require("../constants");
 const DEFAULT_ACK_WAIT_MS = 30_000;
 const DEFAULT_NAK_DELAY_MS = 1_000;
+/** Consumer events after which the consumer delivers nothing more: its stream or itself is gone, or the server fell silent. */
+const CONSUMER_LOST = new Set(['consumer_deleted', 'consumer_not_found', 'stream_not_found', 'heartbeats_missed']);
+const sameSubjects = (a = [], b = []) => [...a].sort().join('|') === [...b].sort().join('|');
 function streamConfig(spec) {
     return { name: spec.name, subjects: spec.subjects, max_age: (0, nats_1.nanos)(spec.maxAgeMs), storage: nats_1.StorageType.File };
 }
@@ -28,7 +34,11 @@ function consumerConfig(spec) {
         filter_subjects: spec.filterSubjects,
     };
 }
-/** Create the stream, or update an existing one to the spec. */
+/**
+ * Create the stream, or update an existing one to the spec. Two lines may
+ * ensure the same stream at once: the create that loses the race finds the
+ * stream in use and updates it instead.
+ */
 async function ensureStream(jsm, spec) {
     const config = streamConfig(spec);
     try {
@@ -37,8 +47,14 @@ async function ensureStream(jsm, spec) {
     catch (err) {
         if (!/stream not found/i.test(err?.message ?? ''))
             throw err;
-        await jsm.streams.add(config);
-        return;
+        try {
+            await jsm.streams.add(config);
+            return;
+        }
+        catch (added) {
+            if (!/already in use/i.test(added?.message ?? ''))
+                throw added;
+        }
     }
     await jsm.streams.update(spec.name, config);
 }
@@ -54,7 +70,12 @@ async function describeStream(jsm, name) {
         throw err;
     }
 }
-/** Create the consumer, or move an existing one to the spec's subjects without losing its place. */
+/**
+ * Create the consumer, or keep an existing one whose subjects match the spec,
+ * position and all. One whose subjects differ is deleted and made again at
+ * the stream's newest message, since its old position was in another
+ * protocol's messages.
+ */
 async function ensureConsumer(jsm, stream, spec) {
     let existing;
     try {
@@ -66,10 +87,22 @@ async function ensureConsumer(jsm, stream, spec) {
         await jsm.consumers.add(stream, consumerConfig(spec));
         return;
     }
-    const want = [...spec.filterSubjects].sort();
-    const have = [...(existing.config.filter_subjects ?? [])].sort();
-    if (want.join('|') !== have.join('|'))
-        await jsm.consumers.update(stream, spec.name, { filter_subjects: spec.filterSubjects });
+    if (sameSubjects(spec.filterSubjects, existing.config.filter_subjects))
+        return;
+    await jsm.consumers.delete(stream, spec.name);
+    await jsm.consumers.add(stream, consumerConfig(spec));
+}
+/** The consumer as JetStream holds it, or null when there is none. */
+async function describeConsumer(jsm, stream, name) {
+    try {
+        const info = await jsm.consumers.info(stream, name);
+        return { name, filterSubjects: info.config.filter_subjects ?? (info.config.filter_subject ? [info.config.filter_subject] : []) };
+    }
+    catch (err) {
+        if (/consumer not found|stream not found/i.test(err?.message ?? ''))
+            return null;
+        throw err;
+    }
 }
 function payloadOf(m) {
     return Buffer.from(m.data).toString('utf8');
@@ -88,6 +121,9 @@ class NatsStreams {
     describeStream(name) {
         return describeStream(this.jsm, name);
     }
+    describeConsumer(stream, name) {
+        return describeConsumer(this.jsm, stream, name);
+    }
     async append(subject, payload) {
         await this.js.publish(subject, payload);
     }
@@ -95,6 +131,15 @@ class NatsStreams {
     async consume(stream, consumer, handler, opts = {}) {
         const nakDelayMs = opts.nakDelayMs ?? DEFAULT_NAK_DELAY_MS;
         const messages = await (await this.js.consumers.get(stream, consumer)).consume();
+        // The consumer's own status: once its stream or itself is gone, or the server fell silent, the reader is told.
+        void (async () => {
+            for await (const status of await messages.status()) {
+                if (!CONSUMER_LOST.has(String(status.type)))
+                    continue;
+                opts.onLost?.(String(status.type));
+                return;
+            }
+        })().catch(() => { });
         void (async () => {
             for await (const m of messages) {
                 const delivery = await handler({ subject: m.subject, payload: payloadOf(m), deliveries: m.info.deliveryCount })

package/build/services/machines/gateway/dispatcher.js

@@ -164,20 +164,32 @@ class Dispatcher {
                 row.dispatch = state_1.DISPATCH_STATE.ACCEPTED;
             for (const job of jobs)
                 state.rememberSent(a.machineId, job.jobId, a.rowId);
+            // The machine may say a job is ready before the claim below returns: the set it is taken out of is there first, and a ready that came earlier still is already out of it.
+            const preparing = a.receivedOnAccept ? jobs.filter((job, i) => replies[i]?.status === protocol_1.REPLY_STATUS.ACCEPTED && replies[i]?.preparing).map((job) => `${a.cmdId}${job.suffix}`) : [];
+            if (row && preparing.length) {
+                row.awaitingReady = new Set(preparing.filter((cmdId) => !row.readyEarly?.has(cmdId)));
+                row.readyEarly = undefined;
+            }
             if (reconciled)
                 log.info(log_1.GATEWAY_LOG.DISPATCH_RECONCILED, { row: a.rowId, machine: a.machineId, jobs: silent.map((j) => j.jobId) });
             log.info(log_1.GATEWAY_LOG.DISPATCHED, { row: a.rowId, machine: a.machineId, reason: replies.map((r) => r?.reason ?? 'held').join('; ') });
-            this.check(a.rowId, 'claim', await lt.claim(a.machineId, a.rowId));
+            const claiming = lt.claim(a.machineId, a.rowId);
+            if (row)
+                row.claiming = claiming;
+            this.check(a.rowId, 'claim', await claiming);
+            if (row)
+                row.claiming = undefined;
             if (!a.receivedOnAccept)
                 return;
-            const preparing = jobs.filter((job, i) => replies[i]?.status === protocol_1.REPLY_STATUS.ACCEPTED && replies[i]?.preparing).map((job) => `${a.cmdId}${job.suffix}`);
-            if (!preparing.length) {
+            // A ready that came while the claim was out has received the row already.
+            const live = state.get(a.rowId);
+            if (!live || live.ended)
+                return;
+            if (!preparing.length || !live.awaitingReady?.size) {
                 await this.received(a);
                 return;
             }
-            if (row)
-                row.awaitingReady = new Set(preparing);
-            log.info(log_1.GATEWAY_LOG.DISPATCH_PREPARING, { row: a.rowId, machine: a.machineId, jobs: preparing });
+            log.info(log_1.GATEWAY_LOG.DISPATCH_PREPARING, { row: a.rowId, machine: a.machineId, jobs: [...live.awaitingReady] });
             return;
         }
         // The handoff may have ended meanwhile: a machine that ran the job anyway reports the arrival.

package/build/services/machines/gateway/executor.d.ts

@@ -57,7 +57,12 @@ export declare class Executor {
      * carrying the units out of it.
      */
     private lateReply;
-    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    /**
+     * On arrival dispatch, a row that waited for the machine to prepare is
+     * received once every job it sent is ready. A ready that comes before the
+     * dispatch has recorded what it waits for is kept for it; one that comes
+     * while the dispatch's claim is still out waits for that claim.
+     */
     private readyForArrival;
     /** Add the item; when the row then holds every key the action waits for, resolve it. */
     private accumulate;

package/build/services/machines/gateway/executor.js

@@ -155,14 +155,31 @@ class Executor {
             received: false, errorCode: contracts_1.HANDOFF_ERROR.DISPATCH_FAILED, detail: `${m} could not prepare the job: ${a.reason}`, ...(a.code ? { code: a.code } : {}),
         }));
     }
-    /** On arrival dispatch, a row that waited for the machine to prepare is received once every job it sent is ready. */
+    /**
+     * On arrival dispatch, a row that waited for the machine to prepare is
+     * received once every job it sent is ready. A ready that comes before the
+     * dispatch has recorded what it waits for is kept for it; one that comes
+     * while the dispatch's claim is still out waits for that claim.
+     */
     async readyForArrival(into, m, cmdId) {
         const { lt, state } = this.d;
-        if (into.ended || !into.awaitingReady?.has(cmdId))
+        if (into.ended)
+            return [];
+        if (!into.awaitingReady) {
+            if (into.dispatch !== state_1.DISPATCH_STATE.ACCEPTED && into.dispatch !== state_1.DISPATCH_STATE.PENDING)
+                return [];
+            (into.readyEarly ??= new Set()).add(cmdId);
+            return [{ verb: 'none', detail: `ready before its dispatch settled; kept` }];
+        }
+        if (!into.awaitingReady.has(cmdId))
             return [];
         into.awaitingReady.delete(cmdId);
         if (into.awaitingReady.size)
             return [];
+        if (into.claiming)
+            await into.claiming;
+        if (into.ended)
+            return [];
         const units = into.units.map((u) => u.unitId);
         const resolved = await lt.resolve(m, into.id, { received: true, receiver: m, units });
         state.close(into.id);

package/build/services/machines/gateway/log.d.ts

@@ -32,6 +32,7 @@ export declare const GATEWAY_LOG: {
     readonly LEADING: "machine_gateway.line.leading";
     readonly STANDBY: "machine_gateway.line.standby";
     readonly LEADERSHIP_LOST: "machine_gateway.line.leadership_lost";
+    readonly WIRE_LOST: "machine_gateway.line.wire_lost";
     readonly START_FAILED: "machine_gateway.line.start_failed";
 };
 export type GatewayLogEvent = typeof GATEWAY_LOG[keyof typeof GATEWAY_LOG];

package/build/services/machines/gateway/log.js

@@ -37,6 +37,7 @@ exports.GATEWAY_LOG = {
     LEADING: 'machine_gateway.line.leading',
     STANDBY: 'machine_gateway.line.standby',
     LEADERSHIP_LOST: 'machine_gateway.line.leadership_lost',
+    WIRE_LOST: 'machine_gateway.line.wire_lost',
     START_FAILED: 'machine_gateway.line.start_failed',
 };
 exports.consoleLog = {

package/build/services/machines/gateway/plan-escalation.js

@@ -131,6 +131,23 @@ function heldUnder(state, machineId, unitId, senderJobId) {
     const held = state.heldJob(machineId, unitId);
     return !!held && (!senderJobId || (0, jobs_1.jobOwns)(senderJobId, held));
 }
+/**
+ * Units a new row sends to a machine that already holds them from an earlier
+ * attempt (a job that failed to prepare, then a retry) need no second scan:
+ * the row is dispatched now, with the arrivals the gateway kept for them.
+ */
+function arrivedAlready(state, row, dispatchTtlMs) {
+    const m = row.toMachine;
+    if (!m || !row.units.length || !row.units.every((u) => state.heldJob(m, u.unitId)))
+        return [];
+    const arrivals = row.units.map((u) => {
+        const before = state.closedFor(u.unitId, (r) => r.toMachine === m && !!r.arrivals?.[u.unitId]);
+        return [u.unitId, before?.arrivals?.[u.unitId] ?? { unitId: u.unitId, carrierId: u.carrierId ?? null }];
+    });
+    row.arrivals = Object.fromEntries(arrivals);
+    row.dueAt = Date.now();
+    return [(0, pacing_1.dispatchAction)(row, m, dispatchTtlMs)];
+}
 function planHandoff(state, line, event, facets, dispatchTtlMs) {
     const id = event.escalationId;
     if (event.verb === VERB.CREATED || event.verb === VERB.CLAIMED) {
@@ -152,9 +169,10 @@ function planHandoff(state, line, event, facets, dispatchTtlMs) {
         }
         if (state.dispatchOnArrival) {
             const refused = (0, pacing_1.refuseUnfit)(state, line, row);
-            if (!refused.length)
-                row.dispatch = state_1.DISPATCH_STATE.ARRIVING;
-            return refused;
+            if (refused.length)
+                return refused;
+            row.dispatch = state_1.DISPATCH_STATE.ARRIVING;
+            return arrivedAlready(state, row, dispatchTtlMs);
         }
         const refused = !!(0, pacing_1.softwareRefusal)(state, line, row.toMachine) || !!(0, pacing_1.jobRefusal)(state, row);
         if (state.holdDispatch || (!refused && (!(0, pacing_1.hasRoom)(state, line, row) || !(0, registration_1.reachable)(state, line, row.toMachine)))) {

package/build/services/machines/gateway/state.d.ts

@@ -37,6 +37,10 @@ export interface HandoffRow extends HandoffFacets {
     arrivals?: Record<string, Record<string, any>>;
     /** Dispatch commands accepted as preparing; on arrival dispatch the row is received once each is ready. */
     awaitingReady?: Set<string>;
+    /** Jobs the machine said were ready before the dispatch recorded what it waits for. */
+    readyEarly?: Set<string>;
+    /** The claim the dispatch has out; a ready that completes the row waits for it. */
+    claiming?: Promise<unknown>;
     /** Lifecycle item keys already on the row (lifecycle.ts). */
     noted?: string[];
     /** Parked with a limit on its items, which lifecycle notes must never use up. */

package/build/services/machines/gateway/state.js

@@ -80,7 +80,7 @@ class GatewayState {
         this.openRows.set(row.id, existing
             ? {
                 ...row, dispatch: existing.dispatch, trace: (0, trace_1.mergeTrace)(row.trace, existing.trace), arrivals: existing.arrivals,
-                dueAt: existing.dueAt, noted: existing.noted ?? row.noted, awaitingReady: existing.awaitingReady,
+                dueAt: existing.dueAt, noted: existing.noted ?? row.noted, awaitingReady: existing.awaitingReady, readyEarly: existing.readyEarly, claiming: existing.claiming,
             }
             : row);
     }

package/build/services/machines/leadership.d.ts

@@ -30,6 +30,8 @@ export declare class LineLeadership {
     private leadOnce;
     /** Lead while the lock is held; the error that ended it. */
     private holdLock;
+    /** The leader gives its turn up without stopping: `lost` runs, the lock goes, and the line contends again after the pause. */
+    relinquish(reason: string): void;
     /** A leader unlocks; a replica still waiting drops its connection, which ends the wait. */
     stop(): Promise<void>;
 }

package/build/services/machines/leadership.js

@@ -62,7 +62,7 @@ class LineLeadership {
     async holdLock(client, lead) {
         const ended = new Promise((resolve) => {
             client.once('error', resolve);
-            this.endTurn = () => resolve(new Error('leadership stopped'));
+            this.endTurn = (err) => resolve(err ?? new Error('leadership stopped'));
         });
         try {
             await client.query(LOCK_SQL, [this.key]);
@@ -76,6 +76,12 @@ class LineLeadership {
             return err;
         }
     }
+    /** The leader gives its turn up without stopping: `lost` runs, the lock goes, and the line contends again after the pause. */
+    relinquish(reason) {
+        if (!this.leading)
+            return;
+        this.endTurn?.(new Error(reason));
+    }
     /** A leader unlocks; a replica still waiting drops its connection, which ends the wait. */
     async stop() {
         this.stopped = true;

package/build/services/machines/select.d.ts

@@ -19,7 +19,7 @@ export type LineMessaging = Pick<typeof messagingRegistry, 'streams' | 'pubsub'>
  * server of the site's own, which the bus connects to itself. `siteCodecs` are
  * the codecs of every line on the site, so the streams keep all their subjects.
  */
-export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec, messaging?: LineMessaging, siteCodecs?: LineCodec[]): MachineBus;
+export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec, messaging?: LineMessaging, siteCodecs?: LineCodec[], onLost?: (reason: string) => void): MachineBus;
 /** Long Tail's escalation events for the gateway; `in-process` needs the running instance's callback adapter. */
 export declare function createEscalationFeed(config: MachineLineConfig, line: LineDef, callbacks?: CallbackEventAdapter): EscalationFeed;
 /** One MQTT link per simulated machine; machines speak MQTT in every networked mode. */

package/build/services/machines/select.js

@@ -36,10 +36,10 @@ function defaultCodec(config, line) {
  * server of the site's own, which the bus connects to itself. `siteCodecs` are
  * the codecs of every line on the site, so the streams keep all their subjects.
  */
-function createMachineBus(config, line, codec = defaultCodec(config, line), messaging = messaging_1.messagingRegistry, siteCodecs = []) {
+function createMachineBus(config, line, codec = defaultCodec(config, line), messaging = messaging_1.messagingRegistry, siteCodecs = [], onLost) {
     switch (config.transport) {
         case config_1.MACHINE_TRANSPORT.NATS: {
-            const where = { mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec, siteCodecs };
+            const where = { mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec, siteCodecs, onLost };
             if (config.siteNatsUrl) {
                 return new nats_bus_1.NatsMachineBus({ natsUrl: config.siteNatsUrl, user: config.siteNatsUser, password: config.siteNatsPassword, ...where });
             }

package/build/services/machines/surface.js

@@ -103,8 +103,14 @@ class LineRun {
             return id;
         };
         const { config, line, codec } = this;
+        const where = { site: line.siteId, line: line.lineId };
+        // A wire that stops delivering ends this turn: the gateway stops, the line contends again and ensures its streams anew.
+        const onLost = (reason) => {
+            this.log.warn(log_1.GATEWAY_LOG.WIRE_LOST, { ...where, reason });
+            this.leadership.relinquish(`wire lost: ${reason}`);
+        };
         this.gateway = new gateway_1.MachineGateway({
-            bus: (0, select_1.createMachineBus)(config, line, codec, undefined, this.siteCodecs()),
+            bus: (0, select_1.createMachineBus)(config, line, codec, undefined, this.siteCodecs(), onLost),
             codec,
             rows: this.opts.rows,
             feed: (0, select_1.createEscalationFeed)(config, line, this.opts.callbacks),
@@ -121,7 +127,7 @@ class LineRun {
         });
         await this.gateway.start();
         this.ready = true;
-        this.log.info(log_1.GATEWAY_LOG.LEADING, { site: line.siteId, line: line.lineId });
+        this.log.info(log_1.GATEWAY_LOG.LEADING, where);
     }
 }
 class MachineSurface {

package/build/types/messaging.d.ts

@@ -47,15 +47,19 @@ export interface LTStreamMessage extends LTMessage {
 export interface LTConsumeOptions {
     /** Delay before a nak'd message is delivered again. */
     nakDelayMs?: number;
+    /** Told once when the consumer can deliver nothing more: its stream or itself is gone, or the server fell silent. */
+    onLost?: (reason: string) => void;
 }
 /** Durable streams: messages kept, delivered to named consumers in order, one at a time, until settled. */
 export interface LTStreams {
     /** Create the stream, or bring an existing one's subjects and age up to the spec. */
     ensureStream(spec: StreamSpec): Promise<void>;
-    /** Create the consumer, or bring an existing one's filter up to the spec; its position is kept. */
+    /** Create the consumer, or keep an existing one whose subjects match the spec; one whose subjects changed is made again. */
     ensureConsumer(stream: string, spec: ConsumerSpec): Promise<void>;
     /** The stream as it is now, or null when there is none. */
     describeStream(name: string): Promise<StreamSpec | null>;
+    /** The consumer as it is now, or null when there is none. */
+    describeConsumer(stream: string, name: string): Promise<ConsumerSpec | null>;
     /** Append a message; resolves once the stream holds it. */
     append(subject: Subject, payload: string): Promise<void>;
     /** Deliver the consumer's messages to `handler` one at a time; a thrown handler counts as a nak. */

package/dashboard/dist/assets/{AdminDashboard-Bm7VG65H.js → AdminDashboard-DRE0hSDS.js}

@@ -1,2 +1,2 @@
-import{j as o}from"./vendor-query-DvWWuNIB.js";import{c as n}from"./tasks-Dab4hExW.js";import{z as m,a9 as c}from"./index-CkMHS-hG.js";import{u}from"./mcp-CmcUXYce.js";import{P as d}from"./PageHeaderWithStats-BEQYYBJv.js";import"./vendor-react-_9U4mKFr.js";import"./vendor-icons-fxBvvHBi.js";function S(){var e;m();const{data:t}=n({limit:1}),{data:a}=c({status:"pending",limit:1}),{data:s}=u(),l=((e=s==null?void 0:s.servers)==null?void 0:e.filter(i=>i.status==="connected").length)??0,r=s?`${l}/${s.total}`:"—";return o.jsx("div",{children:o.jsx(d,{title:"Admin",stats:[{label:"Tasks",value:(t==null?void 0:t.total)??"—"},{label:"Pending Escalations",value:(a==null?void 0:a.total)??"—",dotClass:"bg-status-pending"},{label:"MCP Servers",value:r},{label:"System",value:"Healthy",dotClass:"bg-status-success"}]})})}export{S as AdminDashboard};
-//# sourceMappingURL=AdminDashboard-Bm7VG65H.js.map
+import{j as o}from"./vendor-query-DvWWuNIB.js";import{c as n}from"./tasks-ClJ-mbj9.js";import{z as m,a9 as c}from"./index-pzbGpihI.js";import{u}from"./mcp-_0kFRcyT.js";import{P as d}from"./PageHeaderWithStats-BEQYYBJv.js";import"./vendor-react-_9U4mKFr.js";import"./vendor-icons-fxBvvHBi.js";function S(){var e;m();const{data:t}=n({limit:1}),{data:a}=c({status:"pending",limit:1}),{data:s}=u(),l=((e=s==null?void 0:s.servers)==null?void 0:e.filter(i=>i.status==="connected").length)??0,r=s?`${l}/${s.total}`:"—";return o.jsx("div",{children:o.jsx(d,{title:"Admin",stats:[{label:"Tasks",value:(t==null?void 0:t.total)??"—"},{label:"Pending Escalations",value:(a==null?void 0:a.total)??"—",dotClass:"bg-status-pending"},{label:"MCP Servers",value:r},{label:"System",value:"Healthy",dotClass:"bg-status-success"}]})})}export{S as AdminDashboard};
+//# sourceMappingURL=AdminDashboard-DRE0hSDS.js.map

package/dashboard/dist/assets/{AdminDashboard-Bm7VG65H.js.map → AdminDashboard-DRE0hSDS.js.map}

@@ -1 +1 @@
-{"version":3,"file":"AdminDashboard-Bm7VG65H.js","sources":["../../src/pages/admin/AdminDashboard.tsx"],"sourcesContent":["import { useTasks } from '../../api/tasks';\nimport { useEscalations } from '../../api/escalations';\nimport { useEscalationListEvents } from '../../hooks/useEventHooks';\nimport { useMcpServers } from '../../api/mcp';\nimport { PageHeaderWithStats } from '../../components/common/layout/PageHeaderWithStats';\n\nexport function AdminDashboard() {\n  useEscalationListEvents();\n  const { data: taskData } = useTasks({ limit: 1 });\n  const { data: pendingEsc } = useEscalations({ status: 'pending', limit: 1 });\n  const { data: mcpData } = useMcpServers();\n\n  const connectedServers =\n    mcpData?.servers?.filter((s) => s.status === 'connected').length ?? 0;\n  const mcpLabel = mcpData ? `${connectedServers}/${mcpData.total}` : '—';\n\n  return (\n    <div>\n      <PageHeaderWithStats\n        title=\"Admin\"\n        stats={[\n          { label: 'Tasks', value: taskData?.total ?? '—' },\n          { label: 'Pending Escalations', value: pendingEsc?.total ?? '—', dotClass: 'bg-status-pending' },\n          { label: 'MCP Servers', value: mcpLabel },\n          { label: 'System', value: 'Healthy', dotClass: 'bg-status-success' },\n        ]}\n      />\n    </div>\n  );\n}\n"],"names":["AdminDashboard","useEscalationListEvents","taskData","useTasks","pendingEsc","useEscalations","mcpData","useMcpServers","connectedServers","_a","s","mcpLabel","jsx","PageHeaderWithStats"],"mappings":"oSAMO,SAASA,GAAiB,OAC/BC,EAAA,EACA,KAAM,CAAE,KAAMC,CAAA,EAAaC,EAAS,CAAE,MAAO,EAAG,EAC1C,CAAE,KAAMC,CAAA,EAAeC,EAAe,CAAE,OAAQ,UAAW,MAAO,EAAG,EACrE,CAAE,KAAMC,CAAA,EAAYC,EAAA,EAEpBC,IACJC,EAAAH,GAAA,YAAAA,EAAS,UAAT,YAAAG,EAAkB,OAAQC,GAAMA,EAAE,SAAW,aAAa,SAAU,EAChEC,EAAWL,EAAU,GAAGE,CAAgB,IAAIF,EAAQ,KAAK,GAAK,IAEpE,aACG,MAAA,CACC,SAAAM,EAAAA,IAACC,EAAA,CACC,MAAM,QACN,MAAO,CACL,CAAE,MAAO,QAAS,OAAOX,GAAA,YAAAA,EAAU,QAAS,GAAA,EAC5C,CAAE,MAAO,sBAAuB,OAAOE,GAAA,YAAAA,EAAY,QAAS,IAAK,SAAU,mBAAA,EAC3E,CAAE,MAAO,cAAe,MAAOO,CAAA,EAC/B,CAAE,MAAO,SAAU,MAAO,UAAW,SAAU,mBAAA,CAAoB,CACrE,CAAA,EAEJ,CAEJ"}
+{"version":3,"file":"AdminDashboard-DRE0hSDS.js","sources":["../../src/pages/admin/AdminDashboard.tsx"],"sourcesContent":["import { useTasks } from '../../api/tasks';\nimport { useEscalations } from '../../api/escalations';\nimport { useEscalationListEvents } from '../../hooks/useEventHooks';\nimport { useMcpServers } from '../../api/mcp';\nimport { PageHeaderWithStats } from '../../components/common/layout/PageHeaderWithStats';\n\nexport function AdminDashboard() {\n  useEscalationListEvents();\n  const { data: taskData } = useTasks({ limit: 1 });\n  const { data: pendingEsc } = useEscalations({ status: 'pending', limit: 1 });\n  const { data: mcpData } = useMcpServers();\n\n  const connectedServers =\n    mcpData?.servers?.filter((s) => s.status === 'connected').length ?? 0;\n  const mcpLabel = mcpData ? `${connectedServers}/${mcpData.total}` : '—';\n\n  return (\n    <div>\n      <PageHeaderWithStats\n        title=\"Admin\"\n        stats={[\n          { label: 'Tasks', value: taskData?.total ?? '—' },\n          { label: 'Pending Escalations', value: pendingEsc?.total ?? '—', dotClass: 'bg-status-pending' },\n          { label: 'MCP Servers', value: mcpLabel },\n          { label: 'System', value: 'Healthy', dotClass: 'bg-status-success' },\n        ]}\n      />\n    </div>\n  );\n}\n"],"names":["AdminDashboard","useEscalationListEvents","taskData","useTasks","pendingEsc","useEscalations","mcpData","useMcpServers","connectedServers","_a","s","mcpLabel","jsx","PageHeaderWithStats"],"mappings":"oSAMO,SAASA,GAAiB,OAC/BC,EAAA,EACA,KAAM,CAAE,KAAMC,CAAA,EAAaC,EAAS,CAAE,MAAO,EAAG,EAC1C,CAAE,KAAMC,CAAA,EAAeC,EAAe,CAAE,OAAQ,UAAW,MAAO,EAAG,EACrE,CAAE,KAAMC,CAAA,EAAYC,EAAA,EAEpBC,IACJC,EAAAH,GAAA,YAAAA,EAAS,UAAT,YAAAG,EAAkB,OAAQC,GAAMA,EAAE,SAAW,aAAa,SAAU,EAChEC,EAAWL,EAAU,GAAGE,CAAgB,IAAIF,EAAQ,KAAK,GAAK,IAEpE,aACG,MAAA,CACC,SAAAM,EAAAA,IAACC,EAAA,CACC,MAAM,QACN,MAAO,CACL,CAAE,MAAO,QAAS,OAAOX,GAAA,YAAAA,EAAU,QAAS,GAAA,EAC5C,CAAE,MAAO,sBAAuB,OAAOE,GAAA,YAAAA,EAAY,QAAS,IAAK,SAAU,mBAAA,EAC3E,CAAE,MAAO,cAAe,MAAOO,CAAA,EAC/B,CAAE,MAAO,SAAU,MAAO,UAAW,SAAU,mBAAA,CAAoB,CACrE,CAAA,EAEJ,CAEJ"}