@hotmeshio/long-tail 0.26.0 → 0.28.0

package/build/services/machines/registry.d.ts

@@ -0,0 +1,77 @@
+/**
+ * The line registry: every line this deployment controls, in Postgres, so
+ * every replica contends for the same lines and the Lines index reads one
+ * list. Lines come from code (start({ machines })) and from the API; the
+ * configSource rules decide who wins at boot, the version says what changed.
+ */
+import { type LineDef } from '../../lib/machines/line';
+import type { LineHealth, LineSource, SiteSummary } from '../../types/machines-view';
+import { LINE_ORDER, LINE_SOURCE } from './registry-sql';
+export { LINE_SOURCE };
+/** A registry row. */
+export interface LineRecord {
+    siteId: string;
+    lineId: string;
+    definition: LineDef;
+    source: LineSource;
+    version: number;
+    enabled: boolean;
+    createdAt: string;
+    updatedAt: string;
+}
+/** A registry row with its pending rows counted. */
+export interface LineRecordWithHealth extends LineRecord {
+    health: LineHealth;
+}
+export interface LineListQuery {
+    q?: string;
+    site?: string;
+    /** One line by id, with `site`. */
+    line?: string;
+    attention?: boolean;
+    sort?: keyof typeof LINE_ORDER;
+    limit?: number;
+    offset?: number;
+}
+export interface LineListPage {
+    lines: LineRecordWithHealth[];
+    total: number;
+    limit: number;
+    offset: number;
+}
+/** What boot did with each code line, and the code lines it no longer declares. */
+export interface LineApplyReport {
+    applied: string[];
+    unchanged: string[];
+    dbOwned: string[];
+    orphans: string[];
+}
+/** A write that named a version the row no longer has. */
+export declare class LineVersionConflict extends Error {
+    readonly siteId: string;
+    readonly lineId: string;
+    readonly expected: number;
+    constructor(siteId: string, lineId: string, expected: number);
+}
+export declare const DEFAULT_LINE_PAGE = 50;
+export declare const MAX_LINE_PAGE = 200;
+/** The index's search text: everything a person might type to find the line. */
+export declare function searchTextOf(line: LineDef): string;
+/** Declare a line through the API: insert, or replace and advance its version. */
+export declare function declareApiLine(def: LineDef, expectedVersion?: number): Promise<LineRecord>;
+/**
+ * Apply one code line. Db-owned: insert if absent, otherwise leave the row
+ * and log drift. Code-owned: write when the definition differs, once.
+ */
+export declare function applyCodeLine(def: LineDef, codeOwned: boolean): Promise<'applied' | 'unchanged' | 'db-owned'>;
+/** Boot: apply every code line and report, naming code lines the registry still holds that code no longer declares. */
+export declare function applyDeclaredLines(lines: LineDef[], codeOwned: boolean): Promise<LineApplyReport>;
+export declare function getLine(siteId: string, lineId: string): Promise<LineRecord | null>;
+export declare function deleteLine(siteId: string, lineId: string): Promise<LineRecord | null>;
+/** Every enabled line, code and API alike: what a replica contends for. */
+export declare function listEnabledLines(): Promise<LineRecord[]>;
+/** One page of the index, searched and filtered in SQL, with each line's pending rows counted. */
+export declare function listLines(query?: LineListQuery): Promise<LineListPage>;
+/** One line with its pending rows counted, or null. */
+export declare function getLineWithHealth(siteId: string, lineId: string): Promise<LineRecordWithHealth | null>;
+export declare function listSites(): Promise<SiteSummary[]>;

package/build/services/machines/registry.js

@@ -0,0 +1,158 @@
+"use strict";
+/**
+ * The line registry: every line this deployment controls, in Postgres, so
+ * every replica contends for the same lines and the Lines index reads one
+ * list. Lines come from code (start({ machines })) and from the API; the
+ * configSource rules decide who wins at boot, the version says what changed.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_LINE_PAGE = exports.DEFAULT_LINE_PAGE = exports.LineVersionConflict = exports.LINE_SOURCE = void 0;
+exports.searchTextOf = searchTextOf;
+exports.declareApiLine = declareApiLine;
+exports.applyCodeLine = applyCodeLine;
+exports.applyDeclaredLines = applyDeclaredLines;
+exports.getLine = getLine;
+exports.deleteLine = deleteLine;
+exports.listEnabledLines = listEnabledLines;
+exports.listLines = listLines;
+exports.getLineWithHealth = getLineWithHealth;
+exports.listSites = listSites;
+const node_util_1 = require("node:util");
+const db_1 = require("../../lib/db");
+const logger_1 = require("../../lib/logger");
+const line_1 = require("../../lib/machines/line");
+const registry_sql_1 = require("./registry-sql");
+Object.defineProperty(exports, "LINE_SOURCE", { enumerable: true, get: function () { return registry_sql_1.LINE_SOURCE; } });
+/** A write that named a version the row no longer has. */
+class LineVersionConflict extends Error {
+    constructor(siteId, lineId, expected) {
+        super(`line ${siteId}/${lineId} is not at version ${expected}`);
+        this.siteId = siteId;
+        this.lineId = lineId;
+        this.expected = expected;
+        this.name = 'LineVersionConflict';
+    }
+}
+exports.LineVersionConflict = LineVersionConflict;
+exports.DEFAULT_LINE_PAGE = 50;
+exports.MAX_LINE_PAGE = 200;
+const LOG = '[lt-machines]';
+/** The index's search text: everything a person might type to find the line. */
+function searchTextOf(line) {
+    const words = [line.siteId, line.lineId];
+    for (const m of line.machines)
+        words.push(m.machineId, m.process, (0, line_1.machineRole)(m), m.address ?? '');
+    return [...new Set(words.filter(Boolean).map((w) => w.toLowerCase()))].join(' ');
+}
+function toRecord(row) {
+    return {
+        siteId: row.site_id, lineId: row.line_id, definition: row.definition, source: row.source,
+        version: row.version, enabled: row.enabled,
+        createdAt: new Date(row.created_at).toISOString(), updatedAt: new Date(row.updated_at).toISOString(),
+    };
+}
+function toHealth(row) {
+    const machines = Array.isArray(row.definition?.machines) ? row.definition.machines.length : 0;
+    return {
+        machines, offline: row.offline, quiet: row.quiet, alarms: row.alarms, people: row.people,
+        handoff: row.handoff, command: row.command, attention: row.attention === true,
+    };
+}
+function keyOf(siteId, lineId) {
+    return `${siteId}/${lineId}`;
+}
+/** A LIKE pattern that matches the text anywhere, with the person's wildcards taken literally. */
+function likePattern(q) {
+    return `%${q.trim().toLowerCase().replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
+}
+/** Declare a line through the API: insert, or replace and advance its version. */
+async function declareApiLine(def, expectedVersion) {
+    const line = (0, line_1.validateLine)(def);
+    const { rows } = await (0, db_1.getPool)().query(registry_sql_1.UPSERT_API_LINE, [
+        line.siteId, line.lineId, JSON.stringify(line), searchTextOf(line), expectedVersion ?? null,
+    ]);
+    if (!rows.length)
+        throw new LineVersionConflict(line.siteId, line.lineId, expectedVersion);
+    return toRecord(rows[0]);
+}
+/**
+ * Apply one code line. Db-owned: insert if absent, otherwise leave the row
+ * and log drift. Code-owned: write when the definition differs, once.
+ */
+async function applyCodeLine(def, codeOwned) {
+    const line = (0, line_1.validateLine)(def);
+    const params = [line.siteId, line.lineId, JSON.stringify(line), searchTextOf(line)];
+    const { rows } = await (0, db_1.getPool)().query(codeOwned ? registry_sql_1.APPLY_CODE_LINE : registry_sql_1.SEED_CODE_LINE, params);
+    if (rows.length)
+        return 'applied';
+    if (codeOwned)
+        return 'unchanged';
+    const existing = await getLine(line.siteId, line.lineId);
+    if (existing && !(0, node_util_1.isDeepStrictEqual)(existing.definition, line)) {
+        logger_1.loggerRegistry.warn(`${LOG} line ${keyOf(line.siteId, line.lineId)} differs from its declaration; the registry row stands (db-owned)`);
+    }
+    return 'db-owned';
+}
+/** Boot: apply every code line and report, naming code lines the registry still holds that code no longer declares. */
+async function applyDeclaredLines(lines, codeOwned) {
+    const report = { applied: [], unchanged: [], dbOwned: [], orphans: [] };
+    const declared = new Set();
+    for (const line of lines) {
+        const key = keyOf(line.siteId, line.lineId);
+        declared.add(key);
+        const outcome = await applyCodeLine(line, codeOwned);
+        if (outcome === 'applied')
+            report.applied.push(key);
+        else if (outcome === 'unchanged')
+            report.unchanged.push(key);
+        else
+            report.dbOwned.push(key);
+    }
+    const { rows } = await (0, db_1.getPool)().query(registry_sql_1.LIST_CODE_LINES);
+    report.orphans = rows.map((r) => keyOf(r.site_id, r.line_id)).filter((k) => !declared.has(k));
+    const counts = `applied ${report.applied.length}, unchanged ${report.unchanged.length}, db-owned ${report.dbOwned.length}`;
+    if (report.orphans.length)
+        logger_1.loggerRegistry.warn(`${LOG} lines: ${counts}, declared in code before and not now: [${report.orphans.join(', ')}]`);
+    else
+        logger_1.loggerRegistry.info(`${LOG} lines: ${counts}`);
+    return report;
+}
+async function getLine(siteId, lineId) {
+    const { rows } = await (0, db_1.getPool)().query(registry_sql_1.GET_LINE, [siteId, lineId]);
+    return rows.length ? toRecord(rows[0]) : null;
+}
+async function deleteLine(siteId, lineId) {
+    const { rows } = await (0, db_1.getPool)().query(registry_sql_1.DELETE_LINE, [siteId, lineId]);
+    return rows.length ? toRecord(rows[0]) : null;
+}
+/** Every enabled line, code and API alike: what a replica contends for. */
+async function listEnabledLines() {
+    const { rows } = await (0, db_1.getPool)().query(registry_sql_1.LIST_ENABLED_LINES);
+    return rows.map(toRecord);
+}
+/** One page of the index, searched and filtered in SQL, with each line's pending rows counted. */
+async function listLines(query = {}) {
+    const limit = Math.min(Math.max(1, Math.floor(query.limit ?? exports.DEFAULT_LINE_PAGE)), exports.MAX_LINE_PAGE);
+    const offset = Math.max(0, Math.floor(query.offset ?? 0));
+    const order = registry_sql_1.LINE_ORDER[query.sort ?? 'attention'];
+    if (!order)
+        throw new Error(`sort must be one of ${Object.keys(registry_sql_1.LINE_ORDER).join(', ')}`);
+    const { rows } = await (0, db_1.getPool)().query((0, registry_sql_1.listLinesSql)(order), [
+        [...registry_sql_1.HEALTH_ROW_TYPES], query.q?.trim() ? likePattern(query.q) : null, query.site || null,
+        query.attention ?? null, limit, offset, query.line || null,
+    ]);
+    return {
+        lines: rows.map((r) => ({ ...toRecord(r), health: toHealth(r) })),
+        total: rows.length ? Number(rows[0].total) : 0,
+        limit, offset,
+    };
+}
+/** One line with its pending rows counted, or null. */
+async function getLineWithHealth(siteId, lineId) {
+    const page = await listLines({ site: siteId, line: lineId, limit: 1 });
+    return page.lines[0] ?? null;
+}
+async function listSites() {
+    const { rows } = await (0, db_1.getPool)().query(registry_sql_1.LIST_SITES, [[...registry_sql_1.HEALTH_ROW_TYPES]]);
+    return rows.map((r) => ({ siteId: r.site_id, lines: r.lines, attention: r.attention }));
+}

package/build/services/machines/rows.js

@@ -15,16 +15,39 @@ const str = (v) => (v === undefined || v === null || v === '' ? null : String(v)
 function paramsOf(v) {
     return v && typeof v === 'object' && !Array.isArray(v) ? v : undefined;
 }
+/** Named references with string values; other values are dropped, and an empty set is no refs. */
+function refsOf(v) {
+    if (!v || typeof v !== 'object' || Array.isArray(v))
+        return undefined;
+    const refs = Object.fromEntries(Object.entries(v).filter((e) => typeof e[1] === 'string'));
+    return Object.keys(refs).length ? refs : undefined;
+}
 function artifactOf(v) {
-    return v && typeof v.url === 'string' && v.url && typeof v.sha256 === 'string' && v.sha256 ? { url: v.url, sha256: v.sha256 } : undefined;
+    if (!v || typeof v.url !== 'string' || !v.url || typeof v.sha256 !== 'string' || !v.sha256)
+        return undefined;
+    return {
+        url: v.url, sha256: v.sha256,
+        ...(typeof v.id === 'string' && v.id ? { id: v.id } : {}),
+        ...(typeof v.contentType === 'string' && v.contentType ? { contentType: v.contentType } : {}),
+        ...(Number.isInteger(v.sizeBytes) && v.sizeBytes >= 0 ? { sizeBytes: v.sizeBytes } : {}),
+    };
+}
+function artifactsOf(v) {
+    const list = Array.isArray(v) ? v.map(artifactOf).filter((a) => !!a) : [];
+    return list.length ? list : undefined;
 }
 function unitsOf(v) {
     if (!Array.isArray(v))
         return [];
     return v.filter((u) => u && u.unitId).map((u) => {
         const artifact = artifactOf(u.artifact);
+        const artifacts = artifactsOf(u.artifacts);
         const parameters = paramsOf(u.parameters);
-        return { unitId: String(u.unitId), carrierId: str(u.carrierId), ...(artifact ? { artifact } : {}), ...(parameters ? { parameters } : {}) };
+        const refs = refsOf(u.refs);
+        return {
+            unitId: String(u.unitId), carrierId: str(u.carrierId),
+            ...(artifact ? { artifact } : {}), ...(artifacts ? { artifacts } : {}), ...(parameters ? { parameters } : {}), ...(refs ? { refs } : {}),
+        };
     });
 }
 /** Read standard handoff facets; null when the line, job or units are missing. */
@@ -48,6 +71,7 @@ function readHandoffFacets(md = {}) {
         artifactSha256: str(md.artifactSha256),
         attempt: Number(md.attempt ?? 1),
         ...(paramsOf(md.parameters) ? { parameters: paramsOf(md.parameters) } : {}),
+        ...(refsOf(md.refs) ? { refs: refsOf(md.refs) } : {}),
     };
 }
 exports.standardRows = {

package/build/services/machines/select.d.ts

@@ -6,12 +6,19 @@
 import { type MachineLineConfig } from './config';
 import { type LineDef } from '../../lib/machines/line';
 import { type LineCodec } from '../../lib/machines/protocol';
+import { messagingRegistry } from '../../lib/messaging';
 import type { CallbackEventAdapter } from '../../lib/events/callback';
 import type { EscalationFeed, MachineBus, MachineLink } from '../../lib/machines/transport/delivery';
 /** The codec for a line when the host supplies none: the line protocol under MACHINE_TOPIC_ROOT. */
 export declare function defaultCodec(config: MachineLineConfig, line?: LineDef): LineCodec;
-/** The gateway's view of every machine on the line. */
-export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec): MachineBus;
+/** What the stream bus needs from Long Tail's messaging; the registry by default. */
+export type LineMessaging = Pick<typeof messagingRegistry, 'streams' | 'pubsub'>;
+/**
+ * The gateway's view of every machine on the line. With `nats`, events are read
+ * from Long Tail's messaging streams; MACHINE_SITE_NATS_URL instead names a NATS
+ * server of the site's own, which the bus connects to itself.
+ */
+export declare function createMachineBus(config: MachineLineConfig, line: LineDef, codec?: LineCodec, messaging?: LineMessaging): MachineBus;
 /** Long Tail's escalation events for the gateway; `in-process` needs the running instance's callback adapter. */
 export declare function createEscalationFeed(config: MachineLineConfig, line: LineDef, callbacks?: CallbackEventAdapter): EscalationFeed;
 /** One MQTT link per simulated machine; machines speak MQTT in every networked mode. */

package/build/services/machines/select.js

@@ -15,8 +15,11 @@ const protocol_1 = require("../../lib/machines/protocol");
 const mqtt_bus_1 = require("../../lib/machines/transport/mqtt-bus");
 const mqtt_link_1 = require("../../lib/machines/transport/mqtt-link");
 const nats_bus_1 = require("../../lib/machines/transport/nats-bus");
+const stream_bus_1 = require("../../lib/machines/transport/stream-bus");
 const callback_feed_1 = require("../../lib/machines/transport/callback-feed");
 const nats_feed_1 = require("../../lib/machines/transport/nats-feed");
+const pubsub_feed_1 = require("../../lib/machines/transport/pubsub-feed");
+const messaging_1 = require("../../lib/messaging");
 const IN_PROCESS_ONLY = 'lives inside one process; build MemoryMachineBus and MemoryEscalationFeed directly';
 function mqttOf(config) {
     if (!config.mqtt)
@@ -27,14 +30,20 @@ function mqttOf(config) {
 function defaultCodec(config, line) {
     return (0, protocol_1.genericCodec)({ root: config.topicRoot, line });
 }
-/** The gateway's view of every machine on the line. */
-function createMachineBus(config, line, codec = defaultCodec(config, line)) {
+/**
+ * The gateway's view of every machine on the line. With `nats`, events are read
+ * from Long Tail's messaging streams; MACHINE_SITE_NATS_URL instead names a NATS
+ * server of the site's own, which the bus connects to itself.
+ */
+function createMachineBus(config, line, codec = defaultCodec(config, line), messaging = messaging_1.messagingRegistry) {
     switch (config.transport) {
-        case config_1.MACHINE_TRANSPORT.NATS:
-            return new nats_bus_1.NatsMachineBus({
-                natsUrl: config.siteNatsUrl, user: config.siteNatsUser, password: config.siteNatsPassword,
-                mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec,
-            });
+        case config_1.MACHINE_TRANSPORT.NATS: {
+            const where = { mqtt: mqttOf(config), controllerId: (0, line_1.controllerId)(line), siteId: line.siteId, lineId: line.lineId, codec };
+            if (config.siteNatsUrl) {
+                return new nats_bus_1.NatsMachineBus({ natsUrl: config.siteNatsUrl, user: config.siteNatsUser, password: config.siteNatsPassword, ...where });
+            }
+            return new stream_bus_1.StreamMachineBus({ streams: messaging.streams, pubsub: messaging.pubsub, ...where });
+        }
         case config_1.MACHINE_TRANSPORT.MQTT:
             return new mqtt_bus_1.MqttMachineBus(mqttOf(config), (0, line_1.controllerId)(line), (0, protocol_1.lineFilter)(codec.root, line.siteId, line.lineId));
         default:
@@ -49,6 +58,10 @@ function createEscalationFeed(config, line, callbacks) {
                 throw new Error('MACHINE_ESCALATION_FEED=in-process runs inside start({ machines })');
             return new callback_feed_1.CallbackEscalationFeed(callbacks);
         case config_1.ESCALATION_FEED.CORE:
+            if (messaging_1.messagingRegistry.hasProvider)
+                return new pubsub_feed_1.PubSubEscalationFeed(messaging_1.messagingRegistry.pubsub);
+            if (!config.ltNatsUrl)
+                throw new Error('MACHINE_ESCALATION_FEED=core needs Long Tail messaging (LT_MESSAGING=nats) or NATS_URL');
             return new nats_feed_1.CoreEscalationFeed({ natsUrl: config.ltNatsUrl, token: config.ltNatsToken });
         default:
             throw new Error(`MACHINE_ESCALATION_FEED=${config.feed} ${IN_PROCESS_ONLY}`);

package/build/services/machines/surface.d.ts

@@ -3,13 +3,17 @@
  * controls, one replica leads (an advisory lock), makes sure the line's bot
  * accounts exist, and runs the gateway: escalation events in, machine
  * messages in, verbs performed in process as each machine. Workflows never
- * see any of it; their rows are resolved the way a form resolves them.
+ * see any of it; their rows are resolved the way a form resolves them. The
+ * surface also describes its lines for the dashboard: every replica lists
+ * them, and the leader of a line projects it.
  */
 import type { CallbackEventAdapter } from '../../lib/events/callback';
-import type { LineDef } from '../../lib/machines/line';
+import { type LineDef } from '../../lib/machines/line';
+import { type LineCodec } from '../../lib/machines/protocol';
+import type { InboundMessage } from '../../lib/machines/transport';
+import type { InspectResult, LineSummary, LineView } from '../../types/machines-view';
 import { type MachineLineConfig } from './config';
 import { type GatewayLog } from './gateway/log';
-import type { LineCodec } from '../../lib/machines/protocol';
 import type { HandoffRowAdapter } from './rows';
 type Env = Record<string, string | undefined>;
 export interface MachineSurfaceOptions {
@@ -21,13 +25,49 @@ export interface MachineSurfaceOptions {
     codec?: LineCodec;
     rows?: HandoffRowAdapter;
 }
+/** A message to dry-run against a line. */
+export type WireProbe = Pick<InboundMessage, 'topic' | 'payload' | 'retained'>;
+/** What settings and the API read of the surface. */
+export interface LineSurface {
+    /** Every declared line, on every replica. */
+    describeLines(): LineSummary[];
+    /** The line as its leader sees it; null when this replica is not leading it, or the line is not declared. */
+    describeLine(siteId: string, lineId: string): LineView | null;
+    /** A dry run of one message; null when this replica is not leading the line. */
+    inspect(siteId: string, lineId: string, probe: WireProbe): InspectResult | null;
+    /** The declared line, or undefined. */
+    findLine(siteId: string, lineId: string): LineDef | undefined;
+    /** The ops a machine-command row on this line may ask for. */
+    commandOps(siteId: string, lineId: string): string[];
+    /** Run a line on this replica: started, replaced when its definition changed, or unchanged. */
+    declare(line: LineDef): Promise<DeclareOutcome>;
+    /** Stop running a line on this replica; false when it was not running. */
+    retire(siteId: string, lineId: string): Promise<boolean>;
+}
+export type DeclareOutcome = 'started' | 'replaced' | 'unchanged';
 /** Inside Long Tail the gateway reads escalation events in process. */
 export declare function surfaceConfig(env: Env): MachineLineConfig;
-export declare class MachineSurface {
+/** The line as every replica can describe it: its machines, roles, declared addresses and the codec's own ops. */
+export declare function summarizeLine(line: LineDef, codec: Pick<LineCodec, 'rowOps'>): LineSummary;
+/** The ops a machine-command row may ask for: the control ops, a snapshot, and the codec's own. */
+export declare function commandOpsOf(codec: Pick<LineCodec, 'rowOps'>): string[];
+export declare class MachineSurface implements LineSurface {
+    private readonly config;
+    private readonly opts;
+    private readonly log;
     private readonly runs;
     private constructor();
-    /** Validate the configuration, then contend for every line in the background. */
+    /** Validate the configuration, then contend for every line in the background. The registry may add lines later. */
     static start(opts: MachineSurfaceOptions): MachineSurface;
     stop(): Promise<void>;
+    declare(line: LineDef): Promise<DeclareOutcome>;
+    retire(siteId: string, lineId: string): Promise<boolean>;
+    describeLines(): LineSummary[];
+    describeLine(siteId: string, lineId: string): LineView | null;
+    inspect(siteId: string, lineId: string, probe: WireProbe): InspectResult | null;
+    findLine(siteId: string, lineId: string): LineDef | undefined;
+    commandOps(siteId: string, lineId: string): string[];
+    private begin;
+    private run;
 }
 export {};

package/build/services/machines/surface.js

@@ -4,11 +4,18 @@
  * controls, one replica leads (an advisory lock), makes sure the line's bot
  * accounts exist, and runs the gateway: escalation events in, machine
  * messages in, verbs performed in process as each machine. Workflows never
- * see any of it; their rows are resolved the way a form resolves them.
+ * see any of it; their rows are resolved the way a form resolves them. The
+ * surface also describes its lines for the dashboard: every replica lists
+ * them, and the leader of a line projects it.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MachineSurface = void 0;
 exports.surfaceConfig = surfaceConfig;
+exports.summarizeLine = summarizeLine;
+exports.commandOpsOf = commandOpsOf;
+const node_util_1 = require("node:util");
+const line_1 = require("../../lib/machines/line");
+const protocol_1 = require("../../lib/machines/protocol");
 const config_1 = require("./config");
 const api_escalations_1 = require("./gateway/api-escalations");
 const gateway_1 = require("./gateway/gateway");
@@ -24,38 +31,70 @@ function surfaceConfig(env) {
     }
     return (0, config_1.loadMachineLineConfig)({ ...env, MACHINE_ESCALATION_FEED: env.MACHINE_ESCALATION_FEED ?? config_1.ESCALATION_FEED.IN_PROCESS });
 }
+/** The line as every replica can describe it: its machines, roles, declared addresses and the codec's own ops. */
+function summarizeLine(line, codec) {
+    const rowOps = (0, protocol_1.declaredRowOps)(codec);
+    return {
+        siteId: line.siteId, lineId: line.lineId,
+        machines: line.machines.map((m) => {
+            const learned = (0, line_1.learnsAddress)(line, m.machineId);
+            return {
+                machineId: m.machineId, process: m.process, role: (0, line_1.machineRole)(m),
+                address: learned ? null : m.address ?? m.machineId, addressing: learned ? 'learned' : 'declared',
+                ...(rowOps.length ? { rowOps } : {}),
+            };
+        }),
+    };
+}
+/** The ops a machine-command row may ask for: the control ops, a snapshot, and the codec's own. */
+function commandOpsOf(codec) {
+    return [...new Set([...protocol_1.CONTROL_OPS, protocol_1.OP.SNAPSHOT, ...(0, protocol_1.declaredRowOps)(codec)])];
+}
 class LineRun {
     constructor(line, config, opts, log) {
         this.line = line;
         this.config = config;
         this.opts = opts;
         this.log = log;
+        this.ready = false;
+        this.stopped = false;
         this.leadership = new leadership_1.LineLeadership(`lt-machine-line:${line.siteId}/${line.lineId}`);
+        this.codec = opts.codec ?? (0, select_1.defaultCodec)(config, line);
     }
     start() {
         const where = { site: this.line.siteId, line: this.line.lineId };
         this.log.info(log_1.GATEWAY_LOG.STANDBY, where);
-        this.leadership.run(() => this.lead(), async (err) => {
+        this.leadership.run(() => (this.leading = this.lead()), async (err) => {
             this.log.warn(log_1.GATEWAY_LOG.LEADERSHIP_LOST, { ...where, error: err.message });
+            this.ready = false;
             await this.gateway?.stop();
             this.gateway = undefined;
         }).catch((err) => this.log.error(log_1.GATEWAY_LOG.START_FAILED, { ...where, error: err.message }));
     }
+    /** A gateway still starting finishes first, so none is left running after stop. */
     async stop() {
+        this.stopped = true;
+        this.ready = false;
+        await this.leading?.catch(() => undefined);
         await this.gateway?.stop();
         await this.leadership.stop();
     }
+    /** The gateway, once it leads the line. */
+    leader() {
+        return this.ready ? this.gateway : undefined;
+    }
     async lead() {
         // 1. Identities, 2. gateway on the line's bus and feed, 3. reconcile and consume (gateway.start)
         const ids = await (0, identities_1.ensureLineBots)(this.line);
+        if (this.stopped)
+            return;
         const userFor = (as) => {
             const id = ids.get(as);
             if (!id)
                 throw new Error(`no bot for "${as}" on ${this.line.siteId}/${this.line.lineId}`);
             return id;
         };
-        const { config, line } = this;
-        const codec = this.opts.codec ?? (0, select_1.defaultCodec)(config, line);
+        const { config, line, codec } = this;
         this.gateway = new gateway_1.MachineGateway({
             bus: (0, select_1.createMachineBus)(config, line, codec),
             codec,
@@ -68,31 +107,78 @@ class LineRun {
             settings: {
                 topicRoot: config.topicRoot, replyTimeoutMs: config.replyTimeoutMs, dispatchAttempts: config.dispatchAttempts,
                 offlineDebounceMs: config.offlineDebounceMs, dispatchTtlMs: config.dispatchTtlMs, quietMs: config.quietMs,
-                receiptDelayMs: config.receiptDelayMs,
+                receiptDelayMs: config.receiptDelayMs, lifecycle: config.lifecycle, progressStep: config.progressStep, wireTelemetry: config.wireTelemetry,
             },
         });
         await this.gateway.start();
+        this.ready = true;
         this.log.info(log_1.GATEWAY_LOG.LEADING, { site: line.siteId, line: line.lineId });
     }
 }
 class MachineSurface {
-    constructor(runs) {
-        this.runs = runs;
+    constructor(config, opts, log) {
+        this.config = config;
+        this.opts = opts;
+        this.log = log;
+        this.runs = new Map();
     }
-    /** Validate the configuration, then contend for every line in the background. */
+    /** Validate the configuration, then contend for every line in the background. The registry may add lines later. */
     static start(opts) {
-        if (!opts.lines.length)
-            throw new Error('start({ machines }) needs at least one line');
         const config = surfaceConfig(opts.env ?? process.env);
-        const log = opts.log ?? log_1.consoleLog;
-        const runs = opts.lines.map((line) => new LineRun(line, config, opts, log));
-        for (const run of runs)
-            run.start();
-        return new MachineSurface(runs);
+        const surface = new MachineSurface(config, opts, opts.log ?? log_1.consoleLog);
+        for (const line of opts.lines)
+            surface.begin(line);
+        return surface;
     }
     async stop() {
-        for (const run of this.runs)
+        for (const run of this.runs.values())
             await run.stop();
+        this.runs.clear();
+    }
+    async declare(line) {
+        const current = this.run(line.siteId, line.lineId);
+        if (current && (0, node_util_1.isDeepStrictEqual)(current.line, line))
+            return 'unchanged';
+        if (current)
+            await current.stop();
+        this.begin(line);
+        return current ? 'replaced' : 'started';
+    }
+    async retire(siteId, lineId) {
+        const current = this.run(siteId, lineId);
+        if (!current)
+            return false;
+        this.runs.delete(keyOf(siteId, lineId));
+        await current.stop();
+        return true;
+    }
+    describeLines() {
+        return [...this.runs.values()].map((run) => summarizeLine(run.line, run.codec));
+    }
+    describeLine(siteId, lineId) {
+        const gateway = this.run(siteId, lineId)?.leader();
+        return gateway ? gateway.state.project(this.findLine(siteId, lineId)) : null;
+    }
+    inspect(siteId, lineId, probe) {
+        return this.run(siteId, lineId)?.leader()?.inspect(probe) ?? null;
+    }
+    findLine(siteId, lineId) {
+        return this.run(siteId, lineId)?.line;
+    }
+    commandOps(siteId, lineId) {
+        const run = this.run(siteId, lineId);
+        return run ? commandOpsOf(run.codec) : [];
+    }
+    begin(line) {
+        const run = new LineRun(line, this.config, this.opts, (0, log_1.composeLogs)(this.log, (0, log_1.eventsLog)(line)));
+        this.runs.set(keyOf(line.siteId, line.lineId), run);
+        run.start();
+    }
+    run(siteId, lineId) {
+        return this.runs.get(keyOf(siteId, lineId));
     }
 }
 exports.MachineSurface = MachineSurface;
+function keyOf(siteId, lineId) {
+    return `${siteId}/${lineId}`;
+}

package/build/services/machines/testing/fake-long-tail.d.ts

@@ -26,6 +26,7 @@ export interface FakeRow {
     resolved_by?: string;
     envelope?: string;
     escalation_payload?: string;
+    created_at?: string;
 }
 export interface Call {
     verb: string;

package/build/services/machines/testing/fake-long-tail.js

@@ -43,7 +43,7 @@ class FakeLongTail {
         });
     }
     add(input) {
-        const row = { ...input, id: `e${++this.seq}`, status: 'pending', items: [] };
+        const row = { ...input, id: `e${++this.seq}`, status: 'pending', items: [], created_at: new Date().toISOString() };
         this.rows.set(row.id, row);
         this.emit(row, 'created');
         return row;
@@ -132,7 +132,7 @@ class FakeLongTail {
             return null;
         return {
             id: row.id, type: row.type, subtype: row.subtype, role: row.role, status: row.status,
-            assigned_to: row.assigned_to ?? null, resolved_by: row.resolved_by ?? null, metadata: { ...row.metadata },
+            assigned_to: row.assigned_to ?? null, resolved_by: row.resolved_by ?? null, metadata: { ...row.metadata }, created_at: row.created_at ?? null,
             // Stored serialized, as Long Tail returns it.
             resolver_payload: row.resolver_payload ? JSON.stringify(row.resolver_payload) : null,
         };
@@ -140,7 +140,10 @@ class FakeLongTail {
     async listOpen(query) {
         return [...this.rows.values()]
             .filter((r) => r.status === 'pending' && (!query.type || r.type === query.type) && (!query.role || r.role === query.role))
-            .map((r) => ({ id: r.id, type: r.type, subtype: r.subtype, role: r.role, status: r.status, assigned_to: r.assigned_to ?? null, metadata: r.metadata }));
+            .map((r) => ({
+            id: r.id, type: r.type, subtype: r.subtype, role: r.role, status: r.status, assigned_to: r.assigned_to ?? null, metadata: r.metadata,
+            created_at: r.created_at ?? null,
+        }));
     }
     ofType(type) {
         return [...this.rows.values()].filter((r) => r.type === type);