@yarkivaev/scada 2.3.116 → 2.4.1

package/README.md

@@ -395,3 +395,93 @@ before applying the replacement. This trades a collection gap for no overlap.
 The caller must bind each selector to its old Deployment; no namespace-wide prune
 is performed. Render/check/diff do not delete anything. An optional process `image`
 overrides the release image, so unrelated collectors can retain their pinned runtime.
+
+## Universal collector runtime
+
+`scada-collector manifest.json` (or `scada-collector -` for stdin) runs all
+registered adapters inside one Node.js process. Every instance has the same
+registry; assignments select its work. `COLLECTOR_MANIFEST` accepts inline JSON;
+`COLLECTOR_MANIFEST_PATH` selects a file. SIGHUP reconciles the file, SIGTERM/SIGINT
+perform bounded shutdown. No cluster scheduler, lease service or network control
+API is included. HTTP request handlers and their process-local read models remain
+owned by the HTTP application; use `COLLECTOR_MODE=external` to disable its
+transferable background collection before assigning that work elsewhere.
+
+```js
+import { createRegistry, createRuntime } from '@yarkivaev/scada/runtime';
+const runtime = createRuntime({ registry: createRegistry() });
+await runtime.apply({ taskId: 'line-reader', generation: 1, specVersion: 1,
+  spec: { adapter: 'modbus', url: 'mqtt://broker', config: {
+    addresses: ['tcp://127.0.0.1:502'], streams: [], maps: [], interval: 5
+  } } });
+await runtime.remove({ taskId: 'line-reader', generation: 2 });
+await runtime.stop();
+```
+
+Adapters include `stream` (registered source/transform/sinks), `modbus`,
+`activity`, `amqp-relay`, `amqp-metrics`, `mqtt-metrics`, `operation-sync`,
+`amqp-pg-metrics`, `silent-streams` and `stomp-ingest`. Plant packages extend the
+registry with domain-specific adapters; OPC UA belongs here, not in
+source-to-sink. Constructors do not open connections. A task owns independent
+connections and must close them in `stop()`. Factories are trusted code, not
+arbitrary scripts supplied in manifests.
+
+Assignments have stable task IDs, externally supplied integer generations and
+`specVersion: 1`. An optional `specHash` is checked against canonical JSON.
+Identical repeats are idempotent; conflicting/stale generations are rejected.
+Replacement waits for the previous owner to stop. Removed IDs retain bounded
+in-memory tombstones. Cleanup timeout quarantines resources and marks the
+runtime fatal; it never authorizes a second owner. Generation and PostgreSQL CAS
+are local/conflict safeguards, **not distributed fencing**. The external agent
+must establish cross-node ownership and provide current assignments after restart.
+
+Secrets can be references such as `{ "$secret": "AMQP_URL" }`, resolved by the
+host immediately before adapter construction. Status excludes configuration and
+exception messages. Optional `COLLECTOR_HEALTH_PORT` and
+`COLLECTOR_HEALTH_HOST` (default `127.0.0.1`) expose read-only `/live`, `/ready`
+and `/status`. Readiness represents admission, not the health of every remote
+server. Fatal cleanup makes liveness fail. A blocked event loop cannot answer
+an external probe; synchronous infinite loops/native crashes still affect the
+entire process. No hard CPU or native-memory isolation is claimed.
+
+The process defaults to 16 MiB / 16,384 buffered records, each task to 1 MiB /
+1,024, with bounded control admission and start/stop deadlines. Adapter-owned
+buffers and actual sink promises reserve capacity; driver/internal allocations
+and transient JSON decoding are not a hard RSS limit. Configure capacities for
+all simultaneously retained pipeline stages. MQTT and Loki collectors flush
+individual accepted records because their source delivery is sequential;
+AMQP pipelines retain bounded timed batching. Timeout is not proof that a remote
+side effect did not happen.
+
+`stream` requires explicit `delivery: "live" | "durable"`. Durable streams need
+`stateVersion`, `compatibilityKey`, stable `eventId` and a shared state store.
+Durable payloads and checkpoints use JSON; encode binary explicitly as strings.
+They persist an outbox and each recipient acknowledgement before advancing the
+checkpoint. MQTT requires QoS 1/2; AMQP sets persistent/mandatory publication and
+checks returned messages. Broker confirmation does not prove downstream business
+processing. An uncertain send can repeat with the same event ID; receivers must
+deduplicate. Sources with a non-null checkpoint must implement `restore()`;
+unsupported restoration fails closed. Live telemetry cannot replay downtime.
+Loki fails a saturated page rather than silently advancing across truncated data.
+
+The runner accepts `COLLECTOR_STATE_URL` for PostgreSQL. Apply
+`db/migrations/V0009__collector_task_state.sql` explicitly and grant the service
+role access before starting stateful assignments; the runner never runs schema
+migrations or falls back to local disk. The storage must be reachable from every
+eligible node. The built-in store rejects unsupported ownership tokens rather
+than pretending to enforce an external fencing protocol.
+
+The coordinated review branches require new source-to-sink/scada package releases
+before dependency installation can reproduce all runtime tests. Stable release/pin
+updates and deployment remain separate steps. Unit, transport,
+state recovery and mixed-runtime tests do not constitute a 24-hour production
+soak or prove cluster-wide exactly-once effects.
+
+## Streaming package boundary
+
+`@yarkivaev/source-to-sink` 2.x contains only protocol-independent composition
+primitives with no runtime dependencies. Protocol/storage implementations and
+their integration tests live in `src/infrastructure/streams` and are exported as
+`@yarkivaev/scada/streams`. This includes AMQP, MQTT, STOMP, serial, Modbus, Loki,
+PostgreSQL, ClickHouse and SQLite; OPC UA stays in `@yarkivaev/scada/opcua`.
+Plant-specific transformations stay in the plant package.

package/charts/scada/templates/processes.yaml

@@ -58,6 +58,28 @@ spec:
           image: {{ required "process image is required" (default $.Values.image .image) | quote }}
           imagePullPolicy: IfNotPresent
           command: {{ required "process command is required" .command | toJson }}
+          {{- with .health }}
+          ports:
+            - name: health
+              containerPort: {{ .port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: {{ default "/live" .livenessPath | quote }}
+              port: health
+            initialDelaySeconds: {{ if hasKey . "initialDelaySeconds" }}{{ .initialDelaySeconds }}{{ else }}2{{ end }}
+            periodSeconds: {{ default 10 .periodSeconds }}
+            timeoutSeconds: {{ default 2 .timeoutSeconds }}
+            failureThreshold: {{ default 3 .failureThreshold }}
+          readinessProbe:
+            httpGet:
+              path: {{ default "/ready" .readinessPath | quote }}
+              port: health
+            initialDelaySeconds: {{ if hasKey . "initialDelaySeconds" }}{{ .initialDelaySeconds }}{{ else }}2{{ end }}
+            periodSeconds: {{ default 10 .periodSeconds }}
+            timeoutSeconds: {{ default 2 .timeoutSeconds }}
+            failureThreshold: {{ default 3 .failureThreshold }}
+          {{- end }}
           {{- if .hostDevices }}
           securityContext:
             privileged: true

package/charts/scada/values.schema.json

@@ -11,6 +11,18 @@
           "name": { "type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" },
           "command": { "type": "array", "minItems": 1, "items": { "type": "string" } },
           "env": { "type": "object", "additionalProperties": { "type": ["string", "number", "boolean"] } },
+          "health": {
+            "type": "object", "required": ["port"], "additionalProperties": false,
+            "properties": {
+              "port": { "type": "integer", "minimum": 1, "maximum": 65535 },
+              "livenessPath": { "type": "string", "pattern": "^/" },
+              "readinessPath": { "type": "string", "pattern": "^/" },
+              "initialDelaySeconds": { "type": "integer", "minimum": 0 },
+              "periodSeconds": { "type": "integer", "minimum": 1 },
+              "timeoutSeconds": { "type": "integer", "minimum": 1 },
+              "failureThreshold": { "type": "integer", "minimum": 1 }
+            }
+          },
           "hostDevices": {
             "type": "array", "uniqueItems": true,
             "items": { "type": "string", "pattern": "^/dev/([a-zA-Z0-9_-][a-zA-Z0-9_.-]*/)*[a-zA-Z0-9_-][a-zA-Z0-9_.-]*$" }

package/db/migrations/V0009__collector_task_state.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS collector_task_state (
+    task_id text PRIMARY KEY,
+    version bigint NOT NULL CHECK (version > 0),
+    document jsonb NOT NULL,
+    updated_at timestamptz NOT NULL DEFAULT now()
+);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yarkivaev/scada",
-  "version": "2.3.116",
+  "version": "2.4.1",
   "description": "SCADA domain objects, state persistence, and plant monitoring",
   "repository": {
     "type": "git",
@@ -17,11 +17,14 @@
     "./rabbitmq/definitions": "./src/infrastructure/messaging/rabbitmq/definitions.js",
     "./rabbitmq/reconcile": "./src/infrastructure/messaging/rabbitmq/reconcile.js",
     "./opcua": "./src/infrastructure/opcua/index.js",
-    "./opcua/config": "./src/infrastructure/opcua/config.js"
+    "./opcua/config": "./src/infrastructure/opcua/config.js",
+    "./runtime": "./src/application/runtime/index.js",
+    "./streams": "./src/infrastructure/streams/index.js"
   },
   "bin": {
     "modbus-to-mqtt": "./src/bin/modbus-to-mqtt.js",
-    "supervisor-sink": "./src/bin/supervisor-sink.js"
+    "supervisor-sink": "./src/bin/supervisor-sink.js",
+    "scada-collector": "./src/bin/collector.js"
   },
   "scripts": {
     "test": "mocha --exit 'test/**/test*.js'",
@@ -35,16 +38,19 @@
     "@clickhouse/client": "^1.0.0",
     "@modelcontextprotocol/sdk": "^1.24.3",
     "@yarkivaev/simple-server": "1.0.2",
-    "@yarkivaev/source-to-sink": "1.3.11",
+    "@yarkivaev/source-to-sink": "2.0.0",
     "amqplib": "^0.10.9",
+    "better-sqlite3": "^11.0.0",
+    "modbus-serial": "^8.0.0",
+    "mqtt": "^5.0.0",
     "node-opcua": "2.179.0",
     "pg": "^8.0.0",
+    "serialport": "^12.0.0",
     "stompit": "^1.0.0",
     "zod": "^4.6.7"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.2",
-    "better-sqlite3": "^11.0.0",
     "c8": "^10.1.3",
     "cassandra-driver": "^4.7.2",
     "eslint": "^9.39.2",

package/src/application/deploy/readDocument.js

@@ -4,10 +4,13 @@ import applyIncludes from './applyIncludes.js';
  * Parses one scalar token.
  *
  * @param {string} text - raw scalar
- * @returns {string|number|boolean|null} scalar
+ * @returns {string|number|boolean|null|Array|object} scalar
  */
 function scalar(text) {
     const trimmed = text.trim();
+    if (trimmed === '[]' || trimmed === '{}') {
+        return trimmed === '[]' ? [] : {};
+    }
     if (trimmed === 'true') {
         return true;
     }

package/src/application/plantServer.js

@@ -1,5 +1,5 @@
 import http from 'http';
-import { stompSource } from '@yarkivaev/source-to-sink';
+import { stompSource } from '../infrastructure/streams/index.js';
 import plantApi from './plantApi.js';
 import pgAlerts from '../infrastructure/persistence/pg/alerts.js';
 import postgresPool from '../infrastructure/persistence/postgresPool.js';

package/src/application/runtime/additionalAdapters.js

@@ -0,0 +1,139 @@
+import { clickhouseSink } from '../../infrastructure/streams/index.js';
+import amqpMetricsIngest from '../../infrastructure/ingest/telemetry/amqpMetricsIngest.js';
+import operationSyncIngest from '../../infrastructure/sync/operationSyncIngest.js';
+import operationsFromPg from '../../infrastructure/persistence/pg/operations.js';
+import { metricsSinkFromPool } from '../../infrastructure/persistence/pg/metrics.js';
+import postgresPool from '../../infrastructure/persistence/postgresPool.js';
+import foldedMetricsSink from '../foldedMetricsSink.js';
+import silentStreams from '../../infrastructure/ingest/modbus/silentStreams.js';
+import mx210Tcp from '../../infrastructure/ingest/modbus/mx210Tcp.js';
+import segmentPipeline from '../../infrastructure/ingest/pipelines/segmentPipeline.js';
+import decisionPipeline from '../../infrastructure/ingest/pipelines/decisionPipeline.js';
+import alertPipeline from '../../infrastructure/ingest/pipelines/alertPipeline.js';
+import resolveSecrets from './resolveSecrets.js';
+
+function urlOptions(spec) {
+    const protocols = { amqp: ['amqp:', 'amqps:'], stomp: ['stomp:', 'stomps:'],
+        postgres: ['postgres:', 'postgresql:'], clickhouse: ['http:', 'https:'] };
+    Object.entries(protocols).forEach(([key, allowed]) => {
+        const value = spec[key];
+        if (value === undefined || value?.$secret) { return; }
+        if (typeof value !== 'string' || !allowed.includes(new URL(value).protocol)) { throw new TypeError(`Invalid ${key} URL`); }
+    });
+}
+function boundedOptions(spec) {
+    const maxRecords = Math.min(spec.options?.maxRecords ?? 1000, spec.limits?.maxRecords ?? 1024);
+    const maxBytes = Math.min(spec.options?.maxBytes ?? 1048576, spec.limits?.maxBytes ?? 1048576);
+    if (!Number.isSafeInteger(maxRecords) || maxRecords < 1 || !Number.isSafeInteger(maxBytes) || maxBytes < 1) {
+        throw new TypeError('Ingest limits must be positive safe integers');
+    }
+    return { ...spec.options, maxRecords, maxBytes, maxMessageBytes: maxBytes, maxPendingBytes: maxBytes, size: Math.min(spec.options?.size ?? 100, maxRecords),
+        prefetch: Math.min(spec.options?.prefetch ?? 32, maxRecords) };
+}
+function ownedSink(pipeline, sink) {
+    return { start() { return pipeline.start(); }, async stop() {
+        try { await pipeline.stop(); } finally { await sink.stop?.(); }
+    } };
+}
+function together(items) {
+    return {
+        start() { return Promise.all(items.map((item) => { return item.start(); })); },
+        async stop() {
+            const results = await Promise.allSettled(items.map((item) => { return item.stop(); }));
+            const failure = results.find((result) => { return result.status === 'rejected'; });
+            if (failure) { throw failure.reason; }
+        }
+    };
+}
+function borrowed(pool) {
+    return { query(...args) { return pool.query(...args); }, connect() { return pool.connect(); },
+        end() { return Promise.resolve(); } };
+}
+function telemetry(spec, pool) {
+    const inner = spec.clickhouse ? clickhouseSink(spec.clickhouse, spec.table ?? 'scada.metrics') : metricsSinkFromPool(pool);
+    return foldedMetricsSink(inner, pool, spec.devices);
+}
+function metrics(spec, pool, context) {
+    const sink = telemetry(spec, pool);
+    return ownedSink(amqpMetricsIngest(spec.amqp, spec.queue, sink, { ...boundedOptions(spec), budget: context.budget }), sink);
+}
+function silent(spec, pool, context) {
+    const sink = telemetry(spec, pool);
+    const streams = silentStreams({ budget: spec.budget, interval: spec.interval,
+        sources: spec.sources.map((source) => { return mx210Tcp(source.name, source.host, source.port); }),
+        onError() { context.report?.({ state: 'degraded' }); } });
+    streams.bind(sink);
+    const ingest = amqpMetricsIngest(spec.amqp, spec.queue, sink, {
+        ...boundedOptions(spec), budget: context.budget, onSeen(name) { streams.seen(name); }
+    });
+    return ownedSink(together([ingest, streams]), sink);
+}
+function stomp(spec, pool, context) {
+    const shared = borrowed(pool);
+    const config = { ...spec.config, ...boundedOptions({ ...spec, options: spec.config }), budget: context.budget };
+    return together([
+        segmentPipeline(spec.stomp, spec.postgres, shared, config),
+        decisionPipeline(spec.stomp, shared, config),
+        alertPipeline(spec.stomp, shared, config)
+    ]);
+}
+function close(state) {
+    state.closing ??= Promise.resolve().then(async () => {
+        try { await state.pipeline?.stop(); } finally { await state.pool?.end(); }
+    });
+    return state.closing;
+}
+function task(factory, spec, context) {
+    const state = { pipeline: null, pool: null, stopped: false };
+    return {
+        async start() {
+            if (state.stopped) { throw new Error('Ingest task stopped'); }
+            context.signal.throwIfAborted();
+            const resolved = resolveSecrets(spec, context.secrets);
+            urlOptions(resolved);
+            state.pool = resolved.postgres ? postgresPool({ connectionString: resolved.postgres }) : null;
+            try {
+                state.pipeline = factory(resolved, state.pool, context);
+                await state.pipeline.start();
+                context.signal.throwIfAborted();
+            } catch (error) {
+                await close(state);
+                throw error;
+            }
+        },
+        async stop() {
+            state.stopped = true;
+            await close(state);
+        }
+    };
+}
+function required(spec, keys) {
+    keys.forEach((key) => { if (!spec[key]) { throw new TypeError(`Missing ingest ${key}`); } });
+}
+function silentConfig(spec) {
+    if (!spec.clickhouse && !spec.postgres) { throw new TypeError('Missing silent metrics storage'); }
+    if (spec.devices && Object.keys(spec.devices).length && !spec.postgres) { throw new TypeError('Missing folding postgres'); }
+    if (![spec.budget, spec.interval].every((value) => { return Number.isFinite(value) && value > 0; })) {
+        throw new TypeError('Silent budget and interval must be positive');
+    }
+    if (!Array.isArray(spec.sources) || !spec.sources.length || spec.sources.some((source) => {
+        return !source.name || !source.host || !Number.isInteger(source.port) || source.port < 1 || source.port > 65535;
+    })) { throw new TypeError('Invalid silent source'); }
+}
+function descriptor(factory, keys, validate = () => { return undefined; }, resources = () => { return []; }) {
+    return { validate(spec) { required(spec, keys); urlOptions(spec); boundedOptions(spec); validate(spec); }, resources,
+        create(spec, context) { return task(factory, spec, context); } };
+}
+/** Portable consumers previously embedded in site HTTP processes; migrations remain external. */
+export default function additionalAdapters() {
+    return new Map([
+        ['operation-sync', descriptor((spec, pool, context) => {
+            return operationSyncIngest(spec.amqp, spec.queue, operationsFromPg(pool), { ...boundedOptions(spec), budget: context.budget });
+        }, ['amqp', 'queue', 'postgres'])],
+        ['amqp-pg-metrics', descriptor(metrics, ['amqp', 'queue', 'postgres'])],
+        ['silent-streams', descriptor(silent, ['amqp', 'queue', 'sources'], silentConfig, (spec) => {
+            return [...new Set(spec.sources.map((source) => { return `modbus-tcp:${source.host}:${source.port}`; }))];
+        })],
+        ['stomp-ingest', descriptor(stomp, ['stomp', 'postgres', 'config'])]
+    ]);
+}

package/src/application/runtime/assignment.js

@@ -0,0 +1,30 @@
+import { createHash } from 'node:crypto';
+
+function canonical(value) {
+    if (Array.isArray(value)) { return value.map(canonical); }
+    if (value && typeof value === 'object') {
+        return Object.fromEntries(Object.keys(value).sort().map((key) => { return [key, canonical(value[key])]; }));
+    }
+    return value;
+}
+export function identity(input) {
+    if (!input || typeof input.taskId !== 'string' || !/^[\w./:-]{1,200}$/u.test(input.taskId)) {
+        throw new TypeError('Invalid taskId');
+    }
+    if (!Number.isSafeInteger(input.generation) || input.generation < 0) { throw new TypeError('Invalid generation'); }
+}
+export default function assignment(input, registry) {
+    identity(input);
+    if (input.specVersion !== 1) { throw new TypeError('Unsupported specVersion'); }
+    const json = JSON.stringify(input.spec);
+    if (!json || Buffer.byteLength(json) > 1048576) { throw new TypeError('Invalid or oversized task spec'); }
+    const spec = JSON.parse(json);
+    Object.entries(spec.limits ?? {}).forEach(([key, value]) => {
+        if (!Number.isSafeInteger(value) || value < 1) { throw new TypeError(`Invalid ${key}`); }
+    });
+    const adapter = registry.get(spec.adapter);
+    if (!adapter) { throw new TypeError('Unknown task adapter'); }
+    const hash = createHash('sha256').update(JSON.stringify(canonical(spec))).digest('hex');
+    if (input.specHash && input.specHash !== hash) { throw new TypeError('Invalid specHash'); }
+    return { ...input, spec, specHash: hash, adapter };
+}

package/src/application/runtime/budgetedSink.js

@@ -0,0 +1,17 @@
+/** Reservation follows the driver's actual promise, not an outer timeout race. */
+export default function budgetedSink(sink, budget, countRecords = true) {
+    return {
+        start() { return sink.start?.(); },
+        stop() { return sink.stop?.(); },
+        write(records, options) {
+            try {
+                options?.signal?.throwIfAborted();
+                const release = budget.reserve({ bytes: Buffer.byteLength(JSON.stringify(records)), records: countRecords ? records.length : 0 });
+                return Promise.resolve().then(() => {
+                    options?.signal?.throwIfAborted();
+                    return sink.write(records, options);
+                }).finally(release);
+            } catch (error) { return Promise.reject(error); }
+        }
+    };
+}

package/src/application/runtime/collectorSignals.js

@@ -0,0 +1,14 @@
+/** Only the executable runner owns process-global signal handlers. */
+export default function collectorSignals(collector) {
+    function report(error) { process.stderr.write(`collector: ${error.name}\n`); }
+    function stop() { void collector.stop().then(() => { process.exit(0); }, () => { process.exit(1); }); }
+    function reload() { void collector.reload().catch(report); }
+    process.once('SIGTERM', stop);
+    process.once('SIGINT', stop);
+    process.on('SIGHUP', reload);
+    return () => {
+        process.off('SIGTERM', stop);
+        process.off('SIGINT', stop);
+        process.off('SIGHUP', reload);
+    };
+}

package/src/application/runtime/compositeLifecycle.js

@@ -0,0 +1,49 @@
+async function close(state) {
+    const failures = [];
+    for (const part of [...state.opened].reverse()) {
+        try {
+            // Reverse startup order; every owned resource receives a stop attempt.
+            // eslint-disable-next-line no-await-in-loop
+            await part.stop();
+        } catch (error) { failures.push(error); }
+    }
+    if (failures.length) { throw new AggregateError(failures, 'Composite cleanup failed'); }
+    state.opened = [];
+}
+function cleanup(state) {
+    state.closing ??= close(state);
+    return state.closing;
+}
+async function open(state, parts) {
+    try {
+        for (const part of parts) {
+            if (state.stopped) { throw new Error('Composite startup aborted'); }
+            state.opened.push(part);
+            // Register ownership before awaiting potentially cancellable startup.
+            // eslint-disable-next-line no-await-in-loop
+            await part.start();
+            if (state.stopped) { throw new Error('Composite startup aborted'); }
+        }
+    } catch (error) {
+        try { await cleanup(state); } catch (failure) { throw new AggregateError([error, failure], 'Start and cleanup failed'); }
+        throw error;
+    }
+}
+/** Stop attempts cancellation immediately, then waits for startup to settle. */
+export default function compositeLifecycle({ inputs, outputs }) {
+    const state = { opened: [], starting: null, closing: null, stopping: null, stopped: false };
+    return {
+        start() {
+            if (state.stopped) { return Promise.reject(new Error('Composite stopped')); }
+            state.starting ??= open(state, [...outputs, ...inputs]);
+            return state.starting;
+        },
+        stop() {
+            state.stopped = true;
+            state.stopping ??= Promise.all([cleanup(state), Promise.resolve(state.starting).catch(() => {
+                // Startup failure remains visible to its own caller.
+            })]);
+            return state.stopping;
+        }
+    };
+}

package/src/application/runtime/createRegistry.js

@@ -0,0 +1,20 @@
+import { resources, sinks, sources } from './streamPorts.js';
+import pipelineAdapters from './pipelineAdapters.js';
+import additionalAdapters from './additionalAdapters.js';
+import streamTask from './streamTask.js';
+
+function extend(base, additions = new Map()) {
+    additions.forEach((value, key) => {
+        if (base.has(key)) { throw new Error(`duplicate runtime adapter: ${key}`); }
+        base.set(key, value);
+    });
+    return base;
+}
+/** Same protocol capabilities on every executor; only registered extensions may run. */
+export default function createRegistry(options = {}) {
+    const ports = { sources: extend(sources(), options.sources), sinks: extend(sinks(), options.sinks), transforms: options.transforms ?? new Map() };
+    const stream = streamTask(ports);
+    const registry = extend(pipelineAdapters(), additionalAdapters());
+    registry.set('stream', { ...stream, resources(spec) { return [...new Set([...resources(spec), ...(spec.resources ?? [])])]; } });
+    return extend(registry, options.adapters);
+}

package/src/application/runtime/deadline.js

@@ -0,0 +1,8 @@
+/** Bounds waiting; callers must also abort/close resources and fence late results. */
+export default function deadline(action, milliseconds) {
+    let timer;
+    const timeout = new Promise((resolve, reject) => {
+        timer = setTimeout(() => { reject(new Error('Task operation timed out')); }, milliseconds);
+    });
+    return Promise.race([Promise.resolve().then(action), timeout]).finally(() => { clearTimeout(timer); });
+}

package/src/application/runtime/index.js

@@ -0,0 +1,10 @@
+export { default as createRuntime } from './runtime.js';
+export { default as taskBudget } from './taskBudget.js';
+export { default as taskState } from './taskState.js';
+export { default as streamTask } from './streamTask.js';
+export { default as compositeLifecycle } from './compositeLifecycle.js';
+export { default as runCollector } from './runCollector.js';
+export { default as taskQueue } from './taskQueue.js';
+export { default as createRegistry } from './createRegistry.js';
+export { default as resolveSecrets } from './resolveSecrets.js';
+export { default as taskStatePg } from '../../infrastructure/persistence/runtime/taskStatePg.js';

package/src/application/runtime/manifest.js

@@ -0,0 +1,38 @@
+import { createReadStream } from 'node:fs';
+import assignment, { identity } from './assignment.js';
+
+async function read(stream) {
+    const chunks = [];
+    let bytes = 0;
+    for await (const chunk of stream) {
+        bytes += Buffer.byteLength(chunk);
+        if (bytes > 1048576) { throw new Error('Manifest capacity exceeded'); }
+        chunks.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(chunks).toString('utf8');
+}
+export async function readManifest(env, argv) {
+    const text = env.COLLECTOR_MANIFEST ?? await read(argv[0] === '-' ? process.stdin : createReadStream(argv[0] ?? env.COLLECTOR_MANIFEST_PATH));
+    if (Buffer.byteLength(text) > 1048576) { throw new Error('Manifest capacity exceeded'); }
+    return JSON.parse(text);
+}
+export async function validateManifest(document, registry) {
+    if (!document || !Array.isArray(document.assignments) || !Array.isArray(document.removals ?? [])) {
+        throw new TypeError('Manifest requires assignments and optional removals arrays');
+    }
+    const ids = new Set();
+    const resources = new Set();
+    for (const input of [...document.assignments, ...(document.removals ?? [])]) {
+        identity(input);
+        if (ids.has(input.taskId)) { throw new TypeError('Duplicate manifest taskId'); }
+        ids.add(input.taskId);
+    }
+    await Promise.all(document.assignments.map(async (input) => {
+        const request = assignment(input, registry);
+        await request.adapter.validate(request.spec);
+        request.adapter.resources(request.spec).forEach((key) => {
+            if (resources.has(key)) { throw new Error('Manifest resource already claimed'); }
+            resources.add(key);
+        });
+    }));
+}

package/src/application/runtime/pipelineAdapters.js

@@ -0,0 +1,60 @@
+import { clickhouseSink, mqttSink } from '../../infrastructure/streams/index.js';
+import activityTracking from '../../infrastructure/ingest/activity/activityTracking.js';
+import amqpMetricsIngest from '../../infrastructure/ingest/telemetry/amqpMetricsIngest.js';
+import amqpMqttRelay from '../../infrastructure/ingest/telemetry/amqpMqttRelay.js';
+import modbusPublish from '../../infrastructure/ingest/mqtt/modbusPublish.js';
+import mqttMetrics from '../../infrastructure/ingest/mqtt/mqttMetrics.js';
+import { parseModbusDeviceSpec } from '../../infrastructure/ingest/mqtt/modbusDeviceSpec.js';
+import compositeLifecycle from './compositeLifecycle.js';
+import resolveSecrets from './resolveSecrets.js';
+
+function configured(spec, context) {
+    const options = { maxRecords: spec.limits?.maxRecords, maxBytes: spec.limits?.maxBytes, budget: context.budget };
+    const defined = Object.fromEntries(Object.entries(options).filter(([, value]) => { return value !== undefined; }));
+    return { ...resolveSecrets(spec, context.secrets),
+        config: { ...resolveSecrets(spec.config, context.secrets), ...defined },
+        options: { ...resolveSecrets(spec.options, context.secrets), ...defined } };
+}
+function adapter(factory, required, resourceKeys = () => { return []; }) {
+    return {
+        validate(spec) {
+            required.forEach((key) => { if (!spec[key]) { throw new TypeError(`Missing pipeline ${key}`); } });
+        },
+        resources(spec) { return resourceKeys(spec); },
+        create(spec, context) {
+            let pipeline;
+            let stopped = false;
+            return {
+                start() {
+                    context.signal.throwIfAborted();
+                    if (stopped) { throw new Error('Pipeline stopped'); }
+                    pipeline ??= factory(configured(spec, context));
+                    return pipeline.start();
+                },
+                stop() { stopped = true; return pipeline?.stop(); }
+            };
+        }
+    };
+}
+function modbusResources(spec) {
+    const devices = spec.config.devices?.split(',').map(parseModbusDeviceSpec) ?? [];
+    const paths = devices.filter((device) => { return device.kind === 'rtu'; }).map((device) => { return `serial:${device.path}`; });
+    const addresses = spec.config.addresses ?? [];
+    addresses.filter((address) => { return address.startsWith('rtu://'); }).forEach((address) => {
+        paths.push(`serial:${address.slice(6).split(':')[0]}`);
+    });
+    return [...new Set(paths)];
+}
+function metrics(spec, factory) {
+    const sink = clickhouseSink(spec.clickhouseUrl, spec.table);
+    return compositeLifecycle({ outputs: [sink], inputs: [factory(sink)] });
+}
+export default function pipelineAdapters() {
+    return new Map([
+        ['modbus', adapter((spec) => { return modbusPublish(spec.url, spec.config); }, ['url', 'config'], modbusResources)],
+        ['activity', adapter((spec) => { return activityTracking(spec.lokiUrl, spec.clickhouseUrl, spec.query, spec.config); }, ['lokiUrl', 'clickhouseUrl', 'query', 'config'])],
+        ['amqp-relay', adapter((spec) => { return amqpMqttRelay(spec.url, spec.queue, mqttSink(spec.mqttUrl, spec.options), spec.options); }, ['url', 'queue', 'mqttUrl'])],
+        ['amqp-metrics', adapter((spec) => { return metrics(spec, (sink) => { return amqpMetricsIngest(spec.url, spec.queue, sink, spec.options); }); }, ['url', 'queue', 'clickhouseUrl', 'table'])],
+        ['mqtt-metrics', adapter((spec) => { return metrics(spec, (sink) => { return mqttMetrics(spec.url, sink, spec.topic, spec.config); }); }, ['url', 'topic', 'clickhouseUrl', 'table', 'config'])]
+    ]);
+}

package/src/application/runtime/resolveSecrets.js

@@ -0,0 +1,12 @@
+/** Resolve references only at resource construction; hashes and manifests retain refs. */
+export default function resolveSecrets(value, secrets) {
+    if (Array.isArray(value)) { return value.map((item) => { return resolveSecrets(item, secrets); }); }
+    if (!value || typeof value !== 'object') { return value; }
+    if (Object.hasOwn(value, '$secret')) {
+        if (Object.keys(value).length !== 1 || typeof value.$secret !== 'string') { throw new TypeError('Invalid secret reference'); }
+        const resolved = secrets?.get(value.$secret);
+        if (resolved === undefined || resolved === null) { throw new Error('Required collector secret is unavailable'); }
+        return resolved;
+    }
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => { return [key, resolveSecrets(item, secrets)]; }));
+}