@winkjs/composer 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/emitter.js +44 -14
- package/src/core/emitter-manager/mqtt/index.js +2 -1
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +4 -4
- package/src/core/env-vars.js +44 -3
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +95 -36
- package/src/core/source-manager/mqtt/dedup.js +3 -3
- package/src/core/source-manager/mqtt/index.js +1 -1
- package/src/core/source-manager/mqtt/status.js +67 -14
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/index.js +55 -17
- package/src/core/storage-manager/questdb/persist-plan.js +32 -8
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/callback-guard/index.js +155 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +4 -3
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +4 -3
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +41 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +176 -38
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/emitter-manager/mqtt/mqtt-store.js +0 -538
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@winkjs/composer",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Composable Streaming Intelligence — turn streaming data into actionable insights, from edge to cloud",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"streaming",
|
|
@@ -27,11 +27,10 @@
|
|
|
27
27
|
},
|
|
28
28
|
"scripts": {
|
|
29
29
|
"pretest": "npm run lint",
|
|
30
|
-
"test": "mocha --recursive
|
|
31
|
-
"test:hardening": "node --expose-gc node_modules
|
|
32
|
-
"test:coverage": "c8 --reporter=html mocha --recursive
|
|
33
|
-
"lint": "eslint
|
|
34
|
-
"docs-serve": "docsify serve ./docs"
|
|
30
|
+
"test": "c8 --reporter=text-summary --check-coverage --statements 100 --branches 100 --functions 100 --lines 100 mocha --recursive \"src/**/*.specs.js\" --ignore \"src/**/slow-*.specs.js\"",
|
|
31
|
+
"test:hardening": "node --expose-gc node_modules/mocha/bin/mocha.js --recursive \"src/**/slow-*.specs.js\" --timeout 300000 --exit",
|
|
32
|
+
"test:coverage": "c8 --reporter=html mocha --recursive \"src/**/*.specs.js\" --ignore \"src/**/slow-*.specs.js\"",
|
|
33
|
+
"lint": "eslint \"src/**/*.js\""
|
|
35
34
|
},
|
|
36
35
|
"repository": {
|
|
37
36
|
"type": "git",
|
|
@@ -51,7 +50,8 @@
|
|
|
51
50
|
"src/tools/**/*.js",
|
|
52
51
|
"src/windowing/**/*.js",
|
|
53
52
|
"!src/**/*.specs.js",
|
|
54
|
-
"!src/**/test/"
|
|
53
|
+
"!src/**/test/",
|
|
54
|
+
"!src/core/emitter-manager/mqtt/mqtt-store.js"
|
|
55
55
|
],
|
|
56
56
|
"bugs": {
|
|
57
57
|
"url": "https://github.com/winkjs/composer/issues"
|
|
@@ -63,6 +63,7 @@
|
|
|
63
63
|
"devDependencies": {
|
|
64
64
|
"c8": "^10.1.3",
|
|
65
65
|
"chai": "^5.2.0",
|
|
66
|
+
"classic-level": "^3.0.0",
|
|
66
67
|
"eslint": "^9.21.0",
|
|
67
68
|
"eslint-plugin-security": "^4.0.0",
|
|
68
69
|
"mocha": "^11.1.0",
|
|
@@ -70,7 +71,6 @@
|
|
|
70
71
|
},
|
|
71
72
|
"dependencies": {
|
|
72
73
|
"@questdb/nodejs-client": "^4.2.0",
|
|
73
|
-
"classic-level": "^3.0.0",
|
|
74
74
|
"glob": "^13.0.6",
|
|
75
75
|
"mqtt": "^5.15.1",
|
|
76
76
|
"msgpackr": "^1.11.10",
|
|
@@ -43,13 +43,13 @@
|
|
|
43
43
|
export const createAooEncoder = function ( { maxRows, fieldOrder, codec } ) {
|
|
44
44
|
// Input validation - fail fast on misconfiguration
|
|
45
45
|
if ( !Array.isArray( fieldOrder ) || fieldOrder.length === 0 ) {
|
|
46
|
-
throw new Error( 'fieldOrder must be non-empty array of field names' );
|
|
46
|
+
throw new Error( 'winkComposer/batch: fieldOrder must be non-empty array of field names' );
|
|
47
47
|
}
|
|
48
48
|
if ( !maxRows || maxRows <= 0 ) {
|
|
49
|
-
throw new Error( 'maxRows must be positive integer' );
|
|
49
|
+
throw new Error( 'winkComposer/batch: maxRows must be positive integer' );
|
|
50
50
|
}
|
|
51
51
|
if ( !codec || typeof codec.pack !== 'function' ) {
|
|
52
|
-
throw new Error( 'codec must have pack() method');
|
|
52
|
+
throw new Error( 'winkComposer/batch: codec must have pack() method' );
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
// Pre-allocate row objects with stable shape.
|
|
@@ -98,7 +98,7 @@ export const createAooEncoder = function ( { maxRows, fieldOrder, codec } ) {
|
|
|
98
98
|
const encode = function ( producerId, seq, srcRows, count ) {
|
|
99
99
|
// Validate count doesn't exceed pre-allocated capacity
|
|
100
100
|
if ( count > maxRows ) {
|
|
101
|
-
throw new Error( `Cannot encode ${count} rows, max capacity is ${maxRows}` );
|
|
101
|
+
throw new Error( `winkComposer/batch: Cannot encode ${count} rows, max capacity is ${maxRows}` );
|
|
102
102
|
}
|
|
103
103
|
|
|
104
104
|
// Copy source data into pre-allocated row objects
|
|
@@ -7,10 +7,10 @@
|
|
|
7
7
|
export const createWindow = function ( capacity, fieldOrder ) {
|
|
8
8
|
// Input validation
|
|
9
9
|
if ( !capacity || capacity <= 0 ) {
|
|
10
|
-
throw new Error( 'capacity must be positive integer' );
|
|
10
|
+
throw new Error( 'winkComposer/batch: capacity must be positive integer' );
|
|
11
11
|
}
|
|
12
12
|
if ( !Array.isArray( fieldOrder ) || fieldOrder.length === 0 ) {
|
|
13
|
-
throw new Error( 'fieldOrder must be non-empty array' );
|
|
13
|
+
throw new Error( 'winkComposer/batch: fieldOrder must be non-empty array' );
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
// Pre-allocate all row objects with stable shape
|
package/src/core/codec/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// src/core/codec/index.js
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @fileoverview Universal codec for
|
|
4
|
+
* @fileoverview Universal codec for winkComposer and Dashboard
|
|
5
5
|
*
|
|
6
6
|
* Provides efficient message serialization for MQTT transport with support
|
|
7
7
|
* for both MessagePack (binary, efficient) and JSON (text, debuggable).
|
|
8
|
-
* Works identically in Node.js (
|
|
8
|
+
* Works identically in Node.js (winkComposer) and browsers (Dashboard).
|
|
9
9
|
*
|
|
10
10
|
* Design principles:
|
|
11
11
|
* - Zero dependencies on Node.js-specific APIs
|
|
@@ -53,7 +53,7 @@ const toUint8 = function ( bytes ) {
|
|
|
53
53
|
return new Uint8Array( bytes.buffer, bytes.byteOffset, bytes.byteLength );
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
throw new TypeError( 'Expected BufferSource' );
|
|
56
|
+
throw new TypeError( 'winkComposer/codec: Expected BufferSource' );
|
|
57
57
|
};
|
|
58
58
|
|
|
59
59
|
/**
|
|
@@ -126,7 +126,7 @@ export const createCodec = function ( format = 'msgpack' ) {
|
|
|
126
126
|
} );
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
-
throw new TypeError( `Unknown format: ${format}` );
|
|
129
|
+
throw new TypeError( `winkComposer/codec: Unknown format: ${format}` );
|
|
130
130
|
};
|
|
131
131
|
|
|
132
132
|
// Pre-created codecs for convenience
|
|
@@ -107,6 +107,11 @@
|
|
|
107
107
|
* so a disconnected shutdown with pending messages is a real loss and
|
|
108
108
|
* is reported as one (this differs from the wal-backed design, which
|
|
109
109
|
* held them for the next session).
|
|
110
|
+
* - `CALLBACK_FAILED` — a user callback (`onDeliveryFailure`,
|
|
111
|
+
* `onCritical`, `onBackpressure`) itself threw or rejected. The
|
|
112
|
+
* shared callback guard contains the fault (ADR-018): the emitter
|
|
113
|
+
* keeps publishing and each fault becomes one classified console
|
|
114
|
+
* line. Fix the callback; the line names it and carries the detail.
|
|
110
115
|
*
|
|
111
116
|
* @module mqtt-emitter
|
|
112
117
|
*/
|
|
@@ -123,6 +128,8 @@ import {
|
|
|
123
128
|
DEFAULT_MAX_QUEUE_SIZE,
|
|
124
129
|
MQTT_INFLIGHT_ID_LIMIT
|
|
125
130
|
} from './constants.js';
|
|
131
|
+
import { wrapCallback } from '../../utils/callback-guard/index.js';
|
|
132
|
+
import { logger } from '../../logger/index.js';
|
|
126
133
|
|
|
127
134
|
/**
|
|
128
135
|
* Pre-flight reject threshold on store pressure.
|
|
@@ -185,7 +192,7 @@ const ERR_SHUTTING_DOWN = {
|
|
|
185
192
|
* @returns {TypeError}
|
|
186
193
|
*/
|
|
187
194
|
const invalidConfig = function ( message ) {
|
|
188
|
-
const err = new TypeError( message );
|
|
195
|
+
const err = new TypeError( 'winkComposer/mqttEmitter: ' + message );
|
|
189
196
|
err.code = 'INVALID_CONFIG';
|
|
190
197
|
return err;
|
|
191
198
|
};
|
|
@@ -306,6 +313,29 @@ export const createEmitter = function ( config ) {
|
|
|
306
313
|
assertOptionalCallback( config.onCritical, 'onCritical' );
|
|
307
314
|
assertOptionalCallback( config.onBackpressure, 'onBackpressure' );
|
|
308
315
|
|
|
316
|
+
// The three notification callbacks are armed by the shared
|
|
317
|
+
// callback guard. They were validated raw above and are wrapped
|
|
318
|
+
// once here. The wrap matters because all three run inside
|
|
319
|
+
// mqtt.js's publish ack chain, where a user throw would land in
|
|
320
|
+
// the client library (ADR-018). Each wrap is null when the
|
|
321
|
+
// callback is absent, so every no-handler path below keeps its
|
|
322
|
+
// exact meaning. That includes the deliberate unhandled-rejection
|
|
323
|
+
// escape hatch for an unhandled delivery failure.
|
|
324
|
+
const reportCallbackFault = function ( severity, name, detail ) {
|
|
325
|
+
logger.error(
|
|
326
|
+
`winkComposer/mqttEmitter: user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
|
|
327
|
+
);
|
|
328
|
+
};
|
|
329
|
+
const onDeliveryFailure = wrapCallback( config.onDeliveryFailure, {
|
|
330
|
+
name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
|
|
331
|
+
} );
|
|
332
|
+
const onCritical = wrapCallback( config.onCritical, {
|
|
333
|
+
name: 'onCritical', severity: 'red', report: reportCallbackFault
|
|
334
|
+
} );
|
|
335
|
+
const onBackpressure = wrapCallback( config.onBackpressure, {
|
|
336
|
+
name: 'onBackpressure', severity: 'yellow', report: reportCallbackFault
|
|
337
|
+
} );
|
|
338
|
+
|
|
309
339
|
// First-connack grace: explicit config → MQTT_CONNECT_GRACE_MS env
|
|
310
340
|
// fallback → 500 ms default (ADR-018 precedence; the fallback lives
|
|
311
341
|
// here in the factory body, never as a schema sigil). `??` keeps an
|
|
@@ -349,8 +379,8 @@ export const createEmitter = function ( config ) {
|
|
|
349
379
|
// (same clamp the LevelDB store applied, kept with the same warning).
|
|
350
380
|
const requestedQueueSize = config.maxQueueSize || DEFAULT_MAX_QUEUE_SIZE;
|
|
351
381
|
if ( requestedQueueSize > MQTT_INFLIGHT_ID_LIMIT ) {
|
|
352
|
-
|
|
353
|
-
`
|
|
382
|
+
logger.warn(
|
|
383
|
+
`winkComposer/mqttEmitter: maxQueueSize ${requestedQueueSize} exceeds the MQTT ` +
|
|
354
384
|
`packet-id ceiling — clamped to ${MQTT_INFLIGHT_ID_LIMIT}`
|
|
355
385
|
);
|
|
356
386
|
}
|
|
@@ -413,12 +443,12 @@ export const createEmitter = function ( config ) {
|
|
|
413
443
|
const checkBackpressure = function () {
|
|
414
444
|
const pressure = getPressure();
|
|
415
445
|
|
|
416
|
-
if ( ( pressure > QUEUE_CRITICAL_THRESHOLD ) &&
|
|
417
|
-
|
|
446
|
+
if ( ( pressure > QUEUE_CRITICAL_THRESHOLD ) && onCritical ) {
|
|
447
|
+
onCritical( 'QUEUE_CRITICAL', pressure );
|
|
418
448
|
}
|
|
419
449
|
|
|
420
|
-
if (
|
|
421
|
-
|
|
450
|
+
if ( onBackpressure ) {
|
|
451
|
+
onBackpressure( pressure );
|
|
422
452
|
}
|
|
423
453
|
};
|
|
424
454
|
|
|
@@ -522,12 +552,12 @@ export const createEmitter = function ( config ) {
|
|
|
522
552
|
if ( err ) {
|
|
523
553
|
state.stats.publishErrors += 1;
|
|
524
554
|
const deliveryErr = new Error(
|
|
525
|
-
`
|
|
555
|
+
`winkComposer/mqttEmitter: publish failed (topic=${topic}): ${err.message || err.code || 'unknown'}`
|
|
526
556
|
);
|
|
527
557
|
deliveryErr.code = 'DELIVERY_FAILED';
|
|
528
558
|
deliveryErr.cause = err;
|
|
529
|
-
if (
|
|
530
|
-
|
|
559
|
+
if ( onDeliveryFailure ) {
|
|
560
|
+
onDeliveryFailure( deliveryErr, { topic } );
|
|
531
561
|
} else {
|
|
532
562
|
// No handler — surface as unhandledRejection so the
|
|
533
563
|
// process logs loudly and (Node 15+) exits. Mirrors
|
|
@@ -652,7 +682,7 @@ export const createEmitter = function ( config ) {
|
|
|
652
682
|
// an incomplete drain must not read as a clean one (ADR-018).
|
|
653
683
|
if ( undeliveredCount > 0 ) {
|
|
654
684
|
const err = new Error(
|
|
655
|
-
`
|
|
685
|
+
`winkComposer/mqttEmitter: shutdown closed with ${undeliveredCount} message(s) unacknowledged`
|
|
656
686
|
);
|
|
657
687
|
err.code = 'SHUTDOWN_TIMEOUT';
|
|
658
688
|
err.dropped = { count: undeliveredCount };
|
|
@@ -698,21 +728,21 @@ export const createEmitter = function ( config ) {
|
|
|
698
728
|
}
|
|
699
729
|
state.hasConnectedOnce = true;
|
|
700
730
|
if ( config.debug ) {
|
|
701
|
-
|
|
731
|
+
logger.info( `winkComposer/mqttEmitter: Connected to ${redactedBrokerUrl}` );
|
|
702
732
|
}
|
|
703
733
|
} );
|
|
704
734
|
|
|
705
735
|
client.on( 'offline', () => {
|
|
706
736
|
state.connected = false;
|
|
707
737
|
if ( config.debug ) {
|
|
708
|
-
|
|
738
|
+
logger.info( `winkComposer/mqttEmitter: Offline - ${state.unacked} messages in flight` );
|
|
709
739
|
}
|
|
710
740
|
} );
|
|
711
741
|
|
|
712
742
|
client.on( 'error', ( err ) => {
|
|
713
743
|
state.stats.errors += 1;
|
|
714
744
|
if ( config.debug ) {
|
|
715
|
-
|
|
745
|
+
logger.error( `winkComposer/mqttEmitter: client error: ${err.message}` );
|
|
716
746
|
}
|
|
717
747
|
} );
|
|
718
748
|
|
|
@@ -41,7 +41,8 @@
|
|
|
41
41
|
* ├── index.js # Module surface: id, schema, factory
|
|
42
42
|
* ├── emitter.js # The emitter implementation
|
|
43
43
|
* ├── mqtt-store.js # DORMANT LevelDB store (ADR-021; kept
|
|
44
|
-
* │ # for the WAL successor, not wired
|
|
44
|
+
* │ # for the WAL successor, not wired,
|
|
45
|
+
* │ # not shipped in the npm package)
|
|
45
46
|
* ├── constants.js # Shared configuration
|
|
46
47
|
* └── test/ # Spec files
|
|
47
48
|
*
|
|
@@ -84,7 +84,7 @@ const assertColumnFacts = function ( columns ) {
|
|
|
84
84
|
!Number.isFinite( colSpec.resolution ) ||
|
|
85
85
|
colSpec.resolution <= 0 ) {
|
|
86
86
|
const err = new Error(
|
|
87
|
-
`
|
|
87
|
+
`winkComposer/terminal: column '${colName}' has type 'float64' ` +
|
|
88
88
|
`but resolution is '${colSpec.resolution}'; ` +
|
|
89
89
|
'expected positive finite number (or omit the field to use the global precision)'
|
|
90
90
|
);
|
|
@@ -281,15 +281,15 @@ export const createEmitter = function ( config = {} ) {
|
|
|
281
281
|
const ts = new Date().toLocaleTimeString();
|
|
282
282
|
const header = `${linePrefix}── ${ts} ── ${topic} ──`;
|
|
283
283
|
|
|
284
|
-
console.log( header );
|
|
284
|
+
console.log( header ); // eslint-disable-line no-console -- product output, not diagnostics
|
|
285
285
|
|
|
286
286
|
if ( verbose ) {
|
|
287
|
-
console.log( formatVerbose( msg, formatValue ) );
|
|
287
|
+
console.log( formatVerbose( msg, formatValue ) ); // eslint-disable-line no-console -- product output, not diagnostics
|
|
288
288
|
} else {
|
|
289
|
-
console.log( formatCompact( msg, formatValue ) );
|
|
289
|
+
console.log( formatCompact( msg, formatValue ) ); // eslint-disable-line no-console -- product output, not diagnostics
|
|
290
290
|
}
|
|
291
291
|
|
|
292
|
-
console.log(); //
|
|
292
|
+
console.log(); // eslint-disable-line no-console -- product output: blank line for readability
|
|
293
293
|
|
|
294
294
|
return RESULT_OK;
|
|
295
295
|
},
|
package/src/core/env-vars.js
CHANGED
|
@@ -16,14 +16,36 @@ import os from 'os';
|
|
|
16
16
|
// to nothing intended.
|
|
17
17
|
const rawYieldMs = ( process.env.YIELD_TIME_THRESHOLD_MS ?? '500' ).trim();
|
|
18
18
|
|
|
19
|
+
// The log-level default depends on NODE_ENV, and the ENV_VARS object
|
|
20
|
+
// literal cannot read its own nodeEnv entry while it is being built.
|
|
21
|
+
// So NODE_ENV is resolved once here and used in both places (the same
|
|
22
|
+
// hoisting precedent as rawYieldMs above). Production defaults to
|
|
23
|
+
// 'info' so an edge box does not fill its journal with debug lines;
|
|
24
|
+
// everything else defaults to 'debug' for development visibility.
|
|
25
|
+
// ADR-028 records the logger architecture.
|
|
26
|
+
const rawNodeEnv = ( process.env.NODE_ENV ?? 'development' ).trim();
|
|
27
|
+
|
|
19
28
|
const ENV_VARS = {
|
|
20
29
|
// Core Configuration
|
|
21
30
|
edgeDeviceId: ( process.env.EDGE_DEVICE_ID ?? os.hostname() ).trim(),
|
|
22
|
-
nodeEnv:
|
|
31
|
+
nodeEnv: rawNodeEnv,
|
|
32
|
+
|
|
33
|
+
// Logging (ADR-028). 'logger' picks the transport the facade
|
|
34
|
+
// writes through; 'logLevel' picks the lowest level that prints.
|
|
35
|
+
logger: ( process.env.COMPOSER_LOGGER ?? 'console' ).trim(),
|
|
36
|
+
logLevel: ( process.env.COMPOSER_LOG_LEVEL ?? ( ( rawNodeEnv === 'production' ) ? 'info' : 'debug' ) ).trim(),
|
|
23
37
|
|
|
24
38
|
// Partition Management (see ADR-016)
|
|
25
39
|
maxPartitionsAllowed: parseInt( process.env.COMPOSER_MAX_PARTITIONS_ALLOWED ?? '10000', 10 ),
|
|
26
40
|
|
|
41
|
+
// Fault containment (ADR-018 — the flow runtime owns per-message
|
|
42
|
+
// dispatch failure). This many CONSECUTIVE message failures stop
|
|
43
|
+
// the flow in the terminal 'errored' phase; one success resets
|
|
44
|
+
// the count. The same value caps a partition's consecutive
|
|
45
|
+
// creation failures before it is quarantined. A run of failures
|
|
46
|
+
// means something systemic; one bad message costs only itself.
|
|
47
|
+
messageFailureThreshold: parseInt( process.env.COMPOSER_MESSAGE_FAILURE_THRESHOLD ?? '5', 10 ),
|
|
48
|
+
|
|
27
49
|
// STORAGE_DIR retired 2026-07-09 — the emitter's LevelDB store
|
|
28
50
|
// (removed by ADR-021) was the only thing that ever wrote there.
|
|
29
51
|
// A future WAL adapter defines its own disk-path configuration.
|
|
@@ -109,6 +131,22 @@ const validators = {
|
|
|
109
131
|
return null;
|
|
110
132
|
},
|
|
111
133
|
|
|
134
|
+
logger: function ( value ) {
|
|
135
|
+
const validTransports = [ 'console', 'json', 'silent' ];
|
|
136
|
+
if ( !validTransports.includes( value ) ) {
|
|
137
|
+
return `Must be one of ${validTransports.join( ', ' )}, got: "${value}"`;
|
|
138
|
+
}
|
|
139
|
+
return null;
|
|
140
|
+
},
|
|
141
|
+
|
|
142
|
+
logLevel: function ( value ) {
|
|
143
|
+
const validLevels = [ 'debug', 'info', 'warn', 'error' ];
|
|
144
|
+
if ( !validLevels.includes( value ) ) {
|
|
145
|
+
return `Must be one of ${validLevels.join( ', ' )}, got: "${value}"`;
|
|
146
|
+
}
|
|
147
|
+
return null;
|
|
148
|
+
},
|
|
149
|
+
|
|
112
150
|
mqttMsgExpiry: function ( value, originalEnv ) {
|
|
113
151
|
if ( isNaN( value ) || value <= 0 ) {
|
|
114
152
|
return `Must be positive integer, got: "${originalEnv}"`;
|
|
@@ -185,7 +223,10 @@ const validators = {
|
|
|
185
223
|
const validationConfig = [
|
|
186
224
|
{ field: 'edgeDeviceId', validator: validators.edgeDeviceId },
|
|
187
225
|
{ field: 'nodeEnv', validator: validators.nodeEnv },
|
|
226
|
+
{ field: 'logger', validator: validators.logger, label: 'COMPOSER_LOGGER' },
|
|
227
|
+
{ field: 'logLevel', validator: validators.logLevel, label: 'COMPOSER_LOG_LEVEL' },
|
|
188
228
|
{ field: 'maxPartitionsAllowed', validator: validators.positiveInt, originalEnv: 'COMPOSER_MAX_PARTITIONS_ALLOWED' },
|
|
229
|
+
{ field: 'messageFailureThreshold', validator: validators.positiveInt, originalEnv: 'COMPOSER_MESSAGE_FAILURE_THRESHOLD' },
|
|
189
230
|
{ field: 'yieldTimeThresholdMs', validator: validators.nonNegativeNumberOrInfinity, originalEnv: 'YIELD_TIME_THRESHOLD_MS' },
|
|
190
231
|
{ field: 'mqttBrokerUrl', validator: validators.mqttUrl, label: 'MQTT_BROKER_URL' },
|
|
191
232
|
{ field: 'mqttMsgExpiry', validator: validators.mqttMsgExpiry, originalEnv: 'MQTT_MSG_EXPIRY' },
|
|
@@ -233,8 +274,8 @@ const validate = function () {
|
|
|
233
274
|
|
|
234
275
|
// Fail fast with clear error message
|
|
235
276
|
if ( errors.length > 0 ) {
|
|
236
|
-
console.error( '
|
|
237
|
-
errors.forEach( ( err ) => console.error( ` - ${err}` ) );
|
|
277
|
+
console.error( 'winkComposer/envVars: Environment variable validation failed:' ); // eslint-disable-line no-console -- bootstrap failure runs before any logger can exist
|
|
278
|
+
errors.forEach( ( err ) => console.error( ` - ${err}` ) ); // eslint-disable-line no-console -- bootstrap failure runs before any logger can exist
|
|
238
279
|
process.exit( 1 ); // eslint-disable-line no-process-exit
|
|
239
280
|
}
|
|
240
281
|
}; // validate()
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// core/logger/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The framework's logging facade. Every production
|
|
5
|
+
* log line goes through one small object with four methods: debug,
|
|
6
|
+
* info, warn, and error. Each method takes a message string and an
|
|
7
|
+
* optional flat fields object. The transport behind the facade —
|
|
8
|
+
* where a record actually goes — is picked once at startup from
|
|
9
|
+
* COMPOSER_LOGGER, and the lowest level that prints is picked from
|
|
10
|
+
* COMPOSER_LOG_LEVEL. ADR-028 records the design.
|
|
11
|
+
*
|
|
12
|
+
* Three guarantees shape the code:
|
|
13
|
+
*
|
|
14
|
+
* - **Zero cost when suppressed.** Levels below the threshold are
|
|
15
|
+
* resolved at build time: every suppressed method IS one shared
|
|
16
|
+
* empty function. No comparison, no allocation, no argument
|
|
17
|
+
* inspection happens on a suppressed call. JavaScript still
|
|
18
|
+
* evaluates the arguments themselves, so a hot-path call site
|
|
19
|
+
* guards a fields literal with the precomputed `debugOn` or
|
|
20
|
+
* `infoOn` boolean instead of relying on the no-op.
|
|
21
|
+
* - **A log call never throws.** Logging exists to report faults,
|
|
22
|
+
* so it must not create them. A broken transport, a bad level
|
|
23
|
+
* name, or unserializable fields each degrade to a working
|
|
24
|
+
* console line. The last-resort line interpolates nothing.
|
|
25
|
+
* - **Stream identity.** The console transport mirrors each level
|
|
26
|
+
* to the same console method a raw call would have used, and the
|
|
27
|
+
* json transport keeps the same stderr/stdout split. A line
|
|
28
|
+
* never moves between streams because the facade arrived.
|
|
29
|
+
*
|
|
30
|
+
* There is deliberately no file transport: journald, kubelet, and
|
|
31
|
+
* logrotate own files in every deployment shape (ADR-028 records
|
|
32
|
+
* the rejection and the escape hatches).
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { ENV_VARS } from '../env-vars.js';
|
|
36
|
+
|
|
37
|
+
// Level order: a method emits when its rank is at or above the
|
|
38
|
+
// configured threshold. Internal constant keys — a plain literal.
|
|
39
|
+
const LEVELS = { debug: 10, info: 20, warn: 30, error: 40 };
|
|
40
|
+
|
|
41
|
+
// Which console method carries each level. error and warn go to
|
|
42
|
+
// stderr, info and debug to stdout — the same split raw calls had.
|
|
43
|
+
const CONSOLE_METHODS = { debug: 'log', info: 'log', warn: 'warn', error: 'error' };
|
|
44
|
+
|
|
45
|
+
// The one shared no-op every suppressed method resolves to.
|
|
46
|
+
const NO_OP = function () { /* the shared suppressed-level sink */ };
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Stringify a fields object without ever throwing. Circular or
|
|
50
|
+
* otherwise unserializable fields become a quoted placeholder, so
|
|
51
|
+
* the surrounding line stays valid.
|
|
52
|
+
*
|
|
53
|
+
* @param {*} value - The fields value to render.
|
|
54
|
+
* @returns {string} JSON text, or the placeholder literal.
|
|
55
|
+
*/
|
|
56
|
+
const safeStringify = function ( value ) {
|
|
57
|
+
try {
|
|
58
|
+
return JSON.stringify( value );
|
|
59
|
+
} catch {
|
|
60
|
+
return '"[unserializable fields]"';
|
|
61
|
+
}
|
|
62
|
+
}; // safeStringify()
|
|
63
|
+
|
|
64
|
+
const consoleTransport = {
|
|
65
|
+
emit: function ( level, msg, fields ) {
|
|
66
|
+
const line = ( fields === undefined ) ? msg : ( msg + ' ' + safeStringify( fields ) );
|
|
67
|
+
console[ CONSOLE_METHODS[ level ] ]( line ); // eslint-disable-line no-console -- the console transport IS the console boundary
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const jsonTransport = {
|
|
72
|
+
emit: function ( level, msg, fields ) {
|
|
73
|
+
let line;
|
|
74
|
+
try {
|
|
75
|
+
// JSON.stringify drops a fields key holding undefined,
|
|
76
|
+
// so the no-fields record needs no separate branch.
|
|
77
|
+
line = JSON.stringify( { level, msg, fields } );
|
|
78
|
+
} catch {
|
|
79
|
+
line = JSON.stringify( { level, msg, fields: '[unserializable fields]' } );
|
|
80
|
+
}
|
|
81
|
+
console[ CONSOLE_METHODS[ level ] ]( line ); // eslint-disable-line no-console -- the json transport IS the console boundary
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const silentTransport = {
|
|
86
|
+
emit: NO_OP
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// The env-selectable transports. The mqtt transport joins this map
|
|
90
|
+
// when ADR-029 lands.
|
|
91
|
+
const transports = {
|
|
92
|
+
console: consoleTransport,
|
|
93
|
+
json: jsonTransport,
|
|
94
|
+
silent: silentTransport
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Create an in-memory transport for specs. Each instance records
|
|
99
|
+
* `{ level, msg, fields }` per emission and can be reset. Never
|
|
100
|
+
* selectable through COMPOSER_LOGGER.
|
|
101
|
+
*
|
|
102
|
+
* @returns {object} `{ emit, records, reset }`.
|
|
103
|
+
*/
|
|
104
|
+
const createMemoryTransport = function () {
|
|
105
|
+
const records = [];
|
|
106
|
+
return {
|
|
107
|
+
records,
|
|
108
|
+
reset: function () {
|
|
109
|
+
records.length = 0;
|
|
110
|
+
},
|
|
111
|
+
emit: function ( level, msg, fields ) {
|
|
112
|
+
records.push( { level, msg, fields } );
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
}; // createMemoryTransport()
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Build a logger over a transport at a level. Methods are
|
|
119
|
+
* precompiled here: suppressed levels become the shared no-op, and
|
|
120
|
+
* active levels close over the transport. Invalid inputs fall back
|
|
121
|
+
* to a working console logger with one warning line — building a
|
|
122
|
+
* logger never throws, because logging must not crash its host.
|
|
123
|
+
*
|
|
124
|
+
* @param {object} transport - An object with `emit( level, msg, fields )`.
|
|
125
|
+
* @param {string} levelName - One of debug, info, warn, error.
|
|
126
|
+
* @returns {object} `{ debug, info, warn, error, debugOn, infoOn, level }`.
|
|
127
|
+
*/
|
|
128
|
+
const buildLogger = function ( transport, levelName ) {
|
|
129
|
+
let safeTransport = transport;
|
|
130
|
+
if ( !transport || typeof transport.emit !== 'function' ) {
|
|
131
|
+
console.warn( 'winkComposer/logger: transport has no emit function — falling back to console' ); // eslint-disable-line no-console -- init fallback runs before a working transport exists
|
|
132
|
+
safeTransport = consoleTransport;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
let safeLevel = levelName;
|
|
136
|
+
if ( LEVELS[ safeLevel ] === undefined ) {
|
|
137
|
+
console.warn( 'winkComposer/logger: unknown log level — falling back to info' ); // eslint-disable-line no-console -- init fallback runs before a working transport exists
|
|
138
|
+
safeLevel = 'info';
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const threshold = LEVELS[ safeLevel ];
|
|
142
|
+
|
|
143
|
+
const makeMethod = function ( name ) {
|
|
144
|
+
if ( LEVELS[ name ] < threshold ) {
|
|
145
|
+
return NO_OP;
|
|
146
|
+
}
|
|
147
|
+
return function ( msg, fields ) {
|
|
148
|
+
try {
|
|
149
|
+
safeTransport.emit( name, msg, fields );
|
|
150
|
+
} catch {
|
|
151
|
+
// Last resort, interpolation-free: a broken
|
|
152
|
+
// transport must not crash the host process.
|
|
153
|
+
console.error( 'winkComposer/logger: transport emit failed — record dropped' ); // eslint-disable-line no-console -- last resort when the transport itself threw
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
debug: makeMethod( 'debug' ),
|
|
160
|
+
info: makeMethod( 'info' ),
|
|
161
|
+
warn: makeMethod( 'warn' ),
|
|
162
|
+
error: makeMethod( 'error' ),
|
|
163
|
+
debugOn: ( threshold <= LEVELS.debug ),
|
|
164
|
+
infoOn: ( threshold <= LEVELS.info ),
|
|
165
|
+
level: safeLevel
|
|
166
|
+
};
|
|
167
|
+
}; // buildLogger()
|
|
168
|
+
|
|
169
|
+
// The default instance every production module imports. ENV_VARS
|
|
170
|
+
// validated both values at startup, so the fallbacks above are a
|
|
171
|
+
// second line of defense, not the expected path.
|
|
172
|
+
const logger = buildLogger( transports[ ENV_VARS.logger ], ENV_VARS.logLevel );
|
|
173
|
+
|
|
174
|
+
export { logger, buildLogger, createMemoryTransport, transports };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// core/mqtt-protocol.js
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @fileoverview Shared MQTT protocol invariants for
|
|
4
|
+
* @fileoverview Shared MQTT protocol invariants for winkComposer.
|
|
5
5
|
*
|
|
6
6
|
* These are design decisions, NOT deployment-configurable values.
|
|
7
7
|
* Both source and emitter depend on identical values for interoperability:
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
// ============================================================================
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
* MQTT v5 user property keys for
|
|
21
|
+
* MQTT v5 user property keys for winkComposer metadata.
|
|
22
22
|
* Publisher (emitter) writes these; subscriber (source) reads them.
|
|
23
23
|
*
|
|
24
24
|
* @type {Object}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// core/partition-manager/get-stats.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Read-only statistics snapshot for the partition
|
|
5
|
+
* manager. Lets an operator count dropped messages and partitions
|
|
6
|
+
* instead of scraping log lines (logger-facade epic, Story 6). The
|
|
7
|
+
* counter itself is initialized in init.js and maintained at the
|
|
8
|
+
* drop site in update.js; this file only reads.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Build a snapshot of the partition manager's counters. Cold path:
|
|
13
|
+
* allocates one small object per call and is never called per
|
|
14
|
+
* message.
|
|
15
|
+
*
|
|
16
|
+
* @param {Object} composerState - State object created by init().
|
|
17
|
+
* @returns {{droppedUnknownSpecialization: number,
|
|
18
|
+
* totalPartitionsCreated: number, activePartitions: number}}
|
|
19
|
+
* Messages dropped for an unknown specialization; partition
|
|
20
|
+
* creations attempted, accepted or cap-rejected (ADR-016); and
|
|
21
|
+
* partitions currently held.
|
|
22
|
+
*/
|
|
23
|
+
const getStats = function ( composerState ) {
|
|
24
|
+
return {
|
|
25
|
+
droppedUnknownSpecialization: composerState.droppedUnknownSpecialization,
|
|
26
|
+
totalPartitionsCreated: composerState.totalPartitionsCreated,
|
|
27
|
+
activePartitions: composerState.partitionSpecializations.size
|
|
28
|
+
};
|
|
29
|
+
}; // getStats()
|
|
30
|
+
|
|
31
|
+
export default getStats;
|
|
@@ -3,5 +3,6 @@ export { default as update } from './update.js';
|
|
|
3
3
|
export { default as reset } from './reset.js';
|
|
4
4
|
export { default as recompute } from './recompute.js';
|
|
5
5
|
export { default as publishTo } from './publish-to.js';
|
|
6
|
+
export { default as getStats } from './get-stats.js';
|
|
6
7
|
// Re-export everything from introspection.js
|
|
7
8
|
export * from './introspect.js';
|
|
@@ -18,6 +18,21 @@ const init = function ( flow ) {
|
|
|
18
18
|
// rejection count. See ADR-016.
|
|
19
19
|
composerState.totalPartitionsCreated = 0;
|
|
20
20
|
|
|
21
|
+
// Messages dropped because their specialization field named no
|
|
22
|
+
// known specialization (the update.js drop site). A lifetime
|
|
23
|
+
// counter, read through getStats() — operators count drops
|
|
24
|
+
// instead of scraping log lines.
|
|
25
|
+
composerState.droppedUnknownSpecialization = 0;
|
|
26
|
+
|
|
27
|
+
// Per-partition creation-failure ledgers (ADR-018 fault
|
|
28
|
+
// containment): keyed by each partition's specializedGraphs
|
|
29
|
+
// object, holding { failures, quarantined }. A WeakMap on object
|
|
30
|
+
// identity — specialization names are user-controlled strings, so
|
|
31
|
+
// no in-object key is collision-safe, and the ledger dies with its
|
|
32
|
+
// partition entry automatically. Entries allocate only when a
|
|
33
|
+
// creation fails; the healthy hot path never touches this.
|
|
34
|
+
composerState.creationFailures = new WeakMap();
|
|
35
|
+
|
|
21
36
|
// Yield state (ADR-024): update() sets `yieldPending` when the time
|
|
22
37
|
// threshold is crossed; the flow runtime clears it and takes the
|
|
23
38
|
// event-loop breath after running the message's pipeline.
|