@winkjs/composer 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -3
- package/src/composer.js +1 -1
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +108 -633
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +20 -5
- package/src/core/env-vars.js +143 -16
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +276 -83
- package/src/core/source-manager/mqtt/dedup.js +19 -14
- package/src/core/source-manager/mqtt/index.js +33 -6
- package/src/core/source-manager/mqtt/status.js +211 -46
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +592 -827
- package/src/core/storage-manager/questdb/persist-plan.js +147 -162
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +176 -0
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +7 -4
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +14 -5
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +62 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +200 -40
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// core/utils/callback-guard/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The shared guard for user-supplied callbacks.
|
|
5
|
+
*
|
|
6
|
+
* Adapters and the flow runtime hand the user's functions to this
|
|
7
|
+
* module once, at setup. The armed version they get back contains
|
|
8
|
+
* every fault the user's code can produce. That covers a plain
|
|
9
|
+
* throw, a rejected promise, and even a `throw null` with no
|
|
10
|
+
* message to read. Each fault becomes one classified
|
|
11
|
+
* CALLBACK_FAILED report. The operation that invoked the callback
|
|
12
|
+
* continues unharmed. This is ADR-018's callback-isolation rule:
|
|
13
|
+
* a misbehaving user callback never reaches transport code and
|
|
14
|
+
* never fails silently. ADR-027 scopes which callbacks are
|
|
15
|
+
* wrapped: notification callbacks only. A callback whose throw
|
|
16
|
+
* carries meaning (QuestDB's strict-mode onWarning) and the
|
|
17
|
+
* flow-guarded onMessage stay unwrapped.
|
|
18
|
+
*
|
|
19
|
+
* Design constraints, in force at every call site:
|
|
20
|
+
* - Wrap once at setup; never per message. The armed closure's
|
|
21
|
+
* success-path cost is one try frame and one typeof check. An
|
|
22
|
+
* async callback additionally costs one derived promise per
|
|
23
|
+
* call — that cost follows the user's own choice of an async
|
|
24
|
+
* handler.
|
|
25
|
+
* - Fixed two-argument signature: every wrapped callback takes at
|
|
26
|
+
* most two arguments, so no rest-array is allocated per call.
|
|
27
|
+
* A one-argument callback observes a trailing undefined; that is
|
|
28
|
+
* deliberate and documented here.
|
|
29
|
+
* - The fault reporter is throw-proof. Every containment guarantee
|
|
30
|
+
* reduces to "reporting a fault cannot itself fault". The report
|
|
31
|
+
* closure receives an already-safe detail string, never the raw
|
|
32
|
+
* thrown value. When the report closure itself fails, the guard
|
|
33
|
+
* falls back to one bare log line with no user-value
|
|
34
|
+
* interpolation.
|
|
35
|
+
* - The fault report is bounded per wrapped callback (ADR-029, the
|
|
36
|
+
* bounded loss line). A callback that fails on every call would
|
|
37
|
+
* print one line per call, which for a status callback on a busy
|
|
38
|
+
* source is a line per message. So the first two faults of an
|
|
39
|
+
* episode report in full, later faults are counted, and one summary
|
|
40
|
+
* reports per minute with the count. A quiet minute ends the
|
|
41
|
+
* episode. The summary goes through the site's own channel with the
|
|
42
|
+
* count appended to the detail, so no site changes.
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
import { logger } from '../../logger/index.js';
|
|
46
|
+
import { createLineBound } from '../line-rate/index.js';
|
|
47
|
+
|
|
48
|
+
/** The bound on the fault report (see the file header). */
|
|
49
|
+
const FULL_FAULT_LINES_PER_EPISODE = 2;
|
|
50
|
+
const FAULT_SUMMARY_INTERVAL_MS = 60000;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Frozen sentinel `wrapTransform` returns when the transform threw.
|
|
54
|
+
* Distinguishes "threw" from the documented null/undefined drop.
|
|
55
|
+
* Never exported from composer's public surface, so no user value
|
|
56
|
+
* can ever be identical to it.
|
|
57
|
+
*/
|
|
58
|
+
const TRANSFORM_THREW = Object.freeze( {} );
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Renders a thrown value as a safe string. `throw null`, a throwing
|
|
62
|
+
* `message` getter, and an object with no string form all yield a
|
|
63
|
+
* printable result instead of a second throw.
|
|
64
|
+
*
|
|
65
|
+
* @param {*} err - Whatever the user's code threw or rejected with
|
|
66
|
+
* @returns {string} A printable detail, never a throw
|
|
67
|
+
*/
|
|
68
|
+
const describeFault = function ( err ) {
|
|
69
|
+
try {
|
|
70
|
+
if ( err && err.message ) {
|
|
71
|
+
return String( err.message );
|
|
72
|
+
}
|
|
73
|
+
return String( err );
|
|
74
|
+
} catch {
|
|
75
|
+
return 'unprintable error';
|
|
76
|
+
}
|
|
77
|
+
}; // describeFault()
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Last-resort line when the site's own report closure fails. Bare on
|
|
81
|
+
* purpose: interpolating anything user-controlled here could throw
|
|
82
|
+
* again, and this is the floor the whole guard stands on.
|
|
83
|
+
*/
|
|
84
|
+
const reportFallback = function () {
|
|
85
|
+
logger.error( 'winkComposer/callbackGuard: a callback fault report failed [CALLBACK_FAILED]' );
|
|
86
|
+
}; // reportFallback()
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Arms one user callback. Returns null when no function was given,
|
|
90
|
+
* so `if ( onX )` presence checks at the sites keep their meaning
|
|
91
|
+
* (absent stays absent — no-handler fallbacks are untouched).
|
|
92
|
+
*
|
|
93
|
+
* @param {Function} fn - The user's callback, already validated by
|
|
94
|
+
* the site's own config validation (validate raw, then wrap)
|
|
95
|
+
* @param {Object} spec - `{ name, severity, report }`
|
|
96
|
+
* @param {string} spec.name - Callback name for the report
|
|
97
|
+
* @param {string} spec.severity - 'red' or 'yellow' per ADR-018:
|
|
98
|
+
* red when the callback signals something happening now, yellow
|
|
99
|
+
* for post-event reporting
|
|
100
|
+
* @param {Function} spec.report - Site channel: called as
|
|
101
|
+
* `report( severity, name, detail )` with detail already safe
|
|
102
|
+
* @returns {Function|null} The armed callback, or null
|
|
103
|
+
*/
|
|
104
|
+
const wrapCallback = function ( fn, { name, severity, report } ) {
|
|
105
|
+
if ( typeof fn !== 'function' ) {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
const bounded = createLineBound( {
|
|
109
|
+
fullLines: FULL_FAULT_LINES_PER_EPISODE,
|
|
110
|
+
intervalMs: FAULT_SUMMARY_INTERVAL_MS,
|
|
111
|
+
printFull: report,
|
|
112
|
+
printSummary: function ( count, seconds, sev, callbackName, detail ) {
|
|
113
|
+
report( sev, callbackName, `${detail}; ${count} more fault(s) in the last ${seconds} s` );
|
|
114
|
+
}
|
|
115
|
+
} );
|
|
116
|
+
const faultFn = function ( err ) {
|
|
117
|
+
try {
|
|
118
|
+
bounded( severity, name, describeFault( err ) );
|
|
119
|
+
} catch {
|
|
120
|
+
reportFallback();
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
return function ( a, b ) {
|
|
124
|
+
try {
|
|
125
|
+
const result = fn( a, b );
|
|
126
|
+
if ( result && typeof result.then === 'function' ) {
|
|
127
|
+
// An async callback costs one derived promise per call.
|
|
128
|
+
// That allocation is proportional to the user's own
|
|
129
|
+
// choice of an async handler; a sync callback allocates
|
|
130
|
+
// nothing here.
|
|
131
|
+
const settled = result.then( undefined, faultFn );
|
|
132
|
+
// A native promise cannot reject past this point, because
|
|
133
|
+
// faultFn cannot throw. A non-conforming thenable can
|
|
134
|
+
// ignore its handlers and RETURN a rejected promise
|
|
135
|
+
// instead; sink that one level so it cannot leak as an
|
|
136
|
+
// unhandled rejection. The guard contains bugs, not
|
|
137
|
+
// in-process adversaries — a deeper chain of hostile
|
|
138
|
+
// thenables stays the caller's own fault.
|
|
139
|
+
if ( !( result instanceof Promise ) && settled && typeof settled.then === 'function' ) {
|
|
140
|
+
settled.then( undefined, faultFn );
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
} catch ( err ) {
|
|
144
|
+
faultFn( err );
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
}; // wrapCallback()
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Arms a value-returning callback (the sources' `transform`). The
|
|
151
|
+
* caller keeps its control flow — a transform's return decides the
|
|
152
|
+
* row's fate — so this face only converts a throw into the sentinel
|
|
153
|
+
* and a safe fault report. Return-shape checks and the null drop
|
|
154
|
+
* stay at the site.
|
|
155
|
+
*
|
|
156
|
+
* @param {Function} fn - The transform, called as `fn( value )`
|
|
157
|
+
* @param {Function} onFault - Hoisted site closure, called as
|
|
158
|
+
* `onFault( detail, context )` on the failure path only
|
|
159
|
+
* @returns {Function} `( value, context ) => result | TRANSFORM_THREW`
|
|
160
|
+
*/
|
|
161
|
+
const wrapTransform = function ( fn, onFault ) {
|
|
162
|
+
return function ( value, context ) {
|
|
163
|
+
try {
|
|
164
|
+
return fn( value );
|
|
165
|
+
} catch ( err ) {
|
|
166
|
+
try {
|
|
167
|
+
onFault( describeFault( err ), context );
|
|
168
|
+
} catch {
|
|
169
|
+
reportFallback();
|
|
170
|
+
}
|
|
171
|
+
return TRANSFORM_THREW;
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
}; // wrapTransform()
|
|
175
|
+
|
|
176
|
+
export { wrapCallback, wrapTransform, TRANSFORM_THREW, FULL_FAULT_LINES_PER_EPISODE, FAULT_SUMMARY_INTERVAL_MS };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// core/utils/clock/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The stopwatch clock for time rules that must not
|
|
5
|
+
* follow the wall clock.
|
|
6
|
+
*
|
|
7
|
+
* `Date.now()` reads the wall clock, and the wall clock can step. NTP
|
|
8
|
+
* corrects a board after boot or after a long time without network,
|
|
9
|
+
* and an operator can set the time by hand. A rule such as "red after
|
|
10
|
+
* 30 s disconnected" or "expire a dedup entry after 120 s" would then
|
|
11
|
+
* fire early or late by the size of the step. `performance.now()`
|
|
12
|
+
* reads a clock that only counts up, from process start, in
|
|
13
|
+
* milliseconds with a fraction. A step in the wall clock does not
|
|
14
|
+
* move it (ADR-018, long-running stability).
|
|
15
|
+
*
|
|
16
|
+
* The function reads `globalThis.performance` at every call, for two
|
|
17
|
+
* reasons. The method must be called on its own object, so the bare
|
|
18
|
+
* method cannot be passed around as a clock. And a test that fakes the
|
|
19
|
+
* stopwatch replaces the global object rather than patching the
|
|
20
|
+
* method, so a captured reference would keep reading the real clock.
|
|
21
|
+
*
|
|
22
|
+
* Use this clock for durations and expiry. Use `Date.now()` for
|
|
23
|
+
* timestamps that leave the process, such as a record's time field.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Reads the stopwatch clock.
|
|
28
|
+
*
|
|
29
|
+
* @returns {number} Milliseconds since process start, with a fraction;
|
|
30
|
+
* never decreases.
|
|
31
|
+
*/
|
|
32
|
+
export const monotonicNow = function () {
|
|
33
|
+
return globalThis.performance.now();
|
|
34
|
+
}; // monotonicNow()
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
export const halfLifeToAlpha = function ( halfLife ) {
|
|
16
16
|
if ( typeof halfLife !== 'number' || !Number.isFinite( halfLife ) || ( halfLife <= 0 ) ) {
|
|
17
|
-
throw new Error( 'Half-life must be a finite number > 0' );
|
|
17
|
+
throw new Error( 'winkComposer/halfLife: Half-life must be a finite number > 0' );
|
|
18
18
|
}
|
|
19
19
|
// alpha = 1 - exp( -ln(2) / halfLife ); use expm1 to avoid cancellation
|
|
20
20
|
let alpha = -Math.expm1( -( Math.LN2 / halfLife ) );
|
|
@@ -37,7 +37,7 @@ export const halfLifeToAlpha = function ( halfLife ) {
|
|
|
37
37
|
*/
|
|
38
38
|
export const alphaToHalfLife = function ( alpha ) {
|
|
39
39
|
if ( typeof alpha !== 'number' || !Number.isFinite( alpha ) || ( alpha <= 0 ) || ( alpha >= 1 ) ) {
|
|
40
|
-
throw new Error( 'Alpha must be a finite number in (0,1)' );
|
|
40
|
+
throw new Error( 'winkComposer/halfLife: Alpha must be a finite number in (0,1)' );
|
|
41
41
|
}
|
|
42
42
|
// halfLife = ln(2) / -ln(1 - alpha) ; use log1p(-alpha) for stability
|
|
43
43
|
return Math.LN2 / ( -Math.log1p( -alpha ) );
|
|
@@ -53,10 +53,10 @@ export const alphaToHalfLife = function ( alpha ) {
|
|
|
53
53
|
*/
|
|
54
54
|
export const halfLifeToWarmupSamples = function ( halfLifeSamples, settledFraction = 0.95 ) {
|
|
55
55
|
if ( typeof halfLifeSamples !== 'number' || !Number.isFinite( halfLifeSamples ) || ( halfLifeSamples <= 0 ) ) {
|
|
56
|
-
throw new Error( 'halfLifeSamples must be a finite number > 0' );
|
|
56
|
+
throw new Error( 'winkComposer/halfLife: halfLifeSamples must be a finite number > 0' );
|
|
57
57
|
}
|
|
58
58
|
if ( typeof settledFraction !== 'number' || !Number.isFinite( settledFraction ) || ( settledFraction <= 0 ) || ( settledFraction >= 1 ) ) {
|
|
59
|
-
throw new Error( 'settledFraction must be a finite number in (0,1)' );
|
|
59
|
+
throw new Error( 'winkComposer/halfLife: settledFraction must be a finite number in (0,1)' );
|
|
60
60
|
}
|
|
61
61
|
// nHL = -log2(1 - settledFraction) = -ln(1 - s)/ln2
|
|
62
62
|
const nHalfLives = ( -Math.log1p( -settledFraction ) ) / Math.LN2;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// core/utils/jitter/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Adds a random share to a reconnect period, once per
|
|
5
|
+
* client, so a fleet does not retry in step.
|
|
6
|
+
*
|
|
7
|
+
* mqtt.js retries at a fixed period. When one broker goes away, every
|
|
8
|
+
* client that used it fails at the same moment and retries on the same
|
|
9
|
+
* beat. Attempt `k` from every client lands on the recovering broker
|
|
10
|
+
* at `k` times the period, all at once. This module gives each client
|
|
11
|
+
* its own period: the configured one plus a random share of up to a
|
|
12
|
+
* fifth of it. The share is drawn once at setup, and because attempt
|
|
13
|
+
* `k` fires at `k` times the period, a fixed offset spreads the fleet
|
|
14
|
+
* further with each attempt.
|
|
15
|
+
*
|
|
16
|
+
* The configured period is the operator's floor. The helper never
|
|
17
|
+
* shortens it, and a period of zero stays zero, so a reconnect that
|
|
18
|
+
* was switched off stays off. Both MQTT adapters use this module; the
|
|
19
|
+
* reconnect loop itself stays the library's (ADR-018).
|
|
20
|
+
*
|
|
21
|
+
* The random source is a parameter, so a spec can name its result.
|
|
22
|
+
* Nothing here runs on a message path.
|
|
23
|
+
*
|
|
24
|
+
* @see ADR-018
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/** The largest share of the period the jitter may add. */
|
|
28
|
+
export const RECONNECT_JITTER_FRACTION = 0.2;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Returns the period plus the floor of a random share of it.
|
|
32
|
+
*
|
|
33
|
+
* @param {number} periodMs - The configured period, a finite number of milliseconds, zero or more
|
|
34
|
+
* @param {Object} [options]
|
|
35
|
+
* @param {number} [options.fraction=RECONNECT_JITTER_FRACTION] - The largest share to add, in `[0, 1]`
|
|
36
|
+
* @param {Function} [options.random=Math.random] - Returns a number in `[0, 1)`
|
|
37
|
+
* @returns {number} `periodMs + Math.floor( random() * periodMs * fraction )`
|
|
38
|
+
* @throws {TypeError} when an argument is out of range
|
|
39
|
+
*/
|
|
40
|
+
export const jitteredPeriod = function ( periodMs, { fraction = RECONNECT_JITTER_FRACTION, random = Math.random } = {} ) {
|
|
41
|
+
if ( ( typeof periodMs !== 'number' ) || !Number.isFinite( periodMs ) || ( periodMs < 0 ) ) {
|
|
42
|
+
throw new TypeError( `winkComposer/jitter: periodMs must be a finite number of milliseconds, zero or more, got ${String( periodMs )}` );
|
|
43
|
+
}
|
|
44
|
+
if ( ( typeof fraction !== 'number' ) || !( ( fraction >= 0 ) && ( fraction <= 1 ) ) ) {
|
|
45
|
+
throw new TypeError( `winkComposer/jitter: fraction must be a number in [0, 1], got ${String( fraction )}` );
|
|
46
|
+
}
|
|
47
|
+
if ( typeof random !== 'function' ) {
|
|
48
|
+
throw new TypeError( 'winkComposer/jitter: random must be a function' );
|
|
49
|
+
}
|
|
50
|
+
return periodMs + Math.floor( random() * periodMs * fraction );
|
|
51
|
+
}; // jitteredPeriod()
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// core/utils/line-rate/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Bounds how often a repeating event reaches the log.
|
|
5
|
+
*
|
|
6
|
+
* A dead sensor, a stuck callback, or an unreachable database produces
|
|
7
|
+
* the same event once per row. Printing every one fills a log file with
|
|
8
|
+
* one line per second for a whole night, and the line that matters is
|
|
9
|
+
* lost in the repeats. This module keeps the first lines of such an
|
|
10
|
+
* episode in full and then prints one summary per interval. An episode
|
|
11
|
+
* is a run of events with no quiet interval between them. A quiet
|
|
12
|
+
* interval ends the episode, so the next event prints in full again.
|
|
13
|
+
*
|
|
14
|
+
* The bound is a pure closure over a stopwatch reading, `monotonicNow`,
|
|
15
|
+
* so a step in the wall clock can neither end an episode nor move a
|
|
16
|
+
* summary (ADR-018, long-running stability). It holds no timer, so it
|
|
17
|
+
* needs no shutdown and cannot keep a process alive. The
|
|
18
|
+
* summary prints at the next event after the interval, never on a
|
|
19
|
+
* timer. So a burst that stops shows its count only when the next
|
|
20
|
+
* episode starts, or never. That is the accepted trade for zero timers.
|
|
21
|
+
*
|
|
22
|
+
* Two sites share this shape. The QuestDB adapter's flush-failure loss
|
|
23
|
+
* line uses the same algorithm inline (ADR-029, the bounded loss line).
|
|
24
|
+
* The adapter's default row-skip warning and the callback guard's
|
|
25
|
+
* `CALLBACK_FAILED` line both use this module.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { monotonicNow } from '../clock/index.js';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Creates a line bound.
|
|
32
|
+
*
|
|
33
|
+
* The returned function takes the event's arguments, up to three, and
|
|
34
|
+
* forwards them by identity to `printFull` for the first `fullLines`
|
|
35
|
+
* events of an episode. What the arguments mean is the caller's
|
|
36
|
+
* business: the callback guard passes a severity, a callback name and
|
|
37
|
+
* a fault description; the row-skip warning passes a raw value and a
|
|
38
|
+
* partition id. Later events inside the interval are counted. The
|
|
39
|
+
* first event a full interval after the last printed line calls
|
|
40
|
+
* `printSummary` with the count since that line, the seconds elapsed,
|
|
41
|
+
* and the latest arguments. When an episode ends with counted events,
|
|
42
|
+
* their summary prints before the next episode's first full line.
|
|
43
|
+
*
|
|
44
|
+
* Three named parameters stand in for a rest parameter on purpose. A
|
|
45
|
+
* rest parameter builds an array on every call, and the row-skip
|
|
46
|
+
* warning runs once per skipped row.
|
|
47
|
+
*
|
|
48
|
+
* @param {Object} options
|
|
49
|
+
* @param {number} options.fullLines - How many events per episode print in full.
|
|
50
|
+
* @param {number} options.intervalMs - The summary interval, also the quiet gap that ends an episode.
|
|
51
|
+
* @param {Function} options.printFull - Called as `printFull( arg1, arg2, arg3 )` with the event's arguments.
|
|
52
|
+
* @param {Function} options.printSummary - Called as `printSummary( count, seconds, arg1, arg2, arg3 )`.
|
|
53
|
+
* @returns {Function} `( arg1, arg2, arg3 ) => void`.
|
|
54
|
+
*/
|
|
55
|
+
export const createLineBound = function ( { fullLines, intervalMs, printFull, printSummary } ) {
|
|
56
|
+
let lastEventAt = 0;
|
|
57
|
+
let lastLineAt = 0;
|
|
58
|
+
let printed = 0;
|
|
59
|
+
let counted = 0;
|
|
60
|
+
|
|
61
|
+
// The state clears before the printer runs, as on the full-line
|
|
62
|
+
// path. A printer that throws then costs that one line only. With
|
|
63
|
+
// the old order a throw left the count and the stamp behind, and
|
|
64
|
+
// every later event ran straight into the printer again.
|
|
65
|
+
const summarize = function ( now, arg1, arg2, arg3 ) {
|
|
66
|
+
const seconds = Math.round( ( now - lastLineAt ) / 1000 );
|
|
67
|
+
const count = counted;
|
|
68
|
+
lastLineAt = now;
|
|
69
|
+
counted = 0;
|
|
70
|
+
printSummary( count, seconds, arg1, arg2, arg3 );
|
|
71
|
+
}; // summarize()
|
|
72
|
+
|
|
73
|
+
// The latest arguments are kept for the summary that closes an
|
|
74
|
+
// episode. They belong to the last counted event, not the new one.
|
|
75
|
+
let lastArg1;
|
|
76
|
+
let lastArg2;
|
|
77
|
+
let lastArg3;
|
|
78
|
+
|
|
79
|
+
return function ( arg1, arg2, arg3 ) {
|
|
80
|
+
const now = monotonicNow();
|
|
81
|
+
if ( ( now - lastEventAt ) >= intervalMs ) {
|
|
82
|
+
// A quiet interval ended the previous episode.
|
|
83
|
+
if ( counted > 0 ) {
|
|
84
|
+
summarize( now, lastArg1, lastArg2, lastArg3 );
|
|
85
|
+
}
|
|
86
|
+
printed = 0;
|
|
87
|
+
}
|
|
88
|
+
lastEventAt = now;
|
|
89
|
+
lastArg1 = arg1;
|
|
90
|
+
lastArg2 = arg2;
|
|
91
|
+
lastArg3 = arg3;
|
|
92
|
+
if ( printed < fullLines ) {
|
|
93
|
+
printed += 1;
|
|
94
|
+
lastLineAt = now;
|
|
95
|
+
printFull( arg1, arg2, arg3 );
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
counted += 1;
|
|
99
|
+
if ( ( now - lastLineAt ) < intervalMs ) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
summarize( now, arg1, arg2, arg3 );
|
|
103
|
+
};
|
|
104
|
+
}; // createLineBound()
|
|
@@ -11,24 +11,30 @@
|
|
|
11
11
|
* for human reading. Both used to carry their own copy of the same
|
|
12
12
|
* formula; this module is the single source.
|
|
13
13
|
*
|
|
14
|
-
* The quantizer
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* recomputation.
|
|
14
|
+
* The quantizer chooses its arithmetic once, at build time, so the
|
|
15
|
+
* per-message hot path is a few float operations and no allocation.
|
|
16
|
+
* Three shapes of resolution get three paths:
|
|
18
17
|
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* -
|
|
25
|
-
*
|
|
26
|
-
*
|
|
18
|
+
* - The inverse is a whole number (0.1, 0.01, 0.001, 0.25, 0.5, 0.2,
|
|
19
|
+
* 0.125). The value is `Math.round( v * inverse ) / inverse`. A
|
|
20
|
+
* division by an exact whole number is correctly rounded, so it
|
|
21
|
+
* returns the nearest double to the grid value: 101 / 10 is 10.1
|
|
22
|
+
* and 95 / 4 is 23.75. No string is built.
|
|
23
|
+
* - The resolution is a whole number (5, 10, 7). The value is
|
|
24
|
+
* `Math.round( v / resolution ) * resolution`, exact for whole
|
|
25
|
+
* numbers. No string is built.
|
|
26
|
+
* - Anything else (0.3, 0.15, 2.5). The old path stays: multiply,
|
|
27
|
+
* round, multiply, then `toFixed` to trim the noise and `Number` to
|
|
28
|
+
* parse it back. The trim keeps as many decimals as the resolution
|
|
29
|
+
* itself has. This path allocates one short string per value. It is
|
|
30
|
+
* rare: a declared resolution is nearly always a power of ten.
|
|
27
31
|
*
|
|
28
|
-
*
|
|
29
|
-
* `
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
+
* Why not `toFixed` for everything. `Math.round( 10.1 * 10 ) * 0.1` is
|
|
33
|
+
* `10.100000000000001`, because 0.1 is not exact in binary, so a
|
|
34
|
+
* multiply by the resolution needs a trim. The trim built a string per
|
|
35
|
+
* value on the storage hot path. It also cut grid values short: with a
|
|
36
|
+
* resolution of 0.25 the trim kept one decimal, so 23.75 became 23.8,
|
|
37
|
+
* off the grid. Division by the exact inverse has neither problem.
|
|
32
38
|
*
|
|
33
39
|
* Returning `null` for `resolution === 1` (or undefined) is a small but
|
|
34
40
|
* load-bearing optimisation: the common case is "no resolution declared"
|
|
@@ -62,9 +68,27 @@ const buildResolutionQuantizer = function ( resolution ) {
|
|
|
62
68
|
if ( resolution === undefined || resolution === 1 ) return null;
|
|
63
69
|
|
|
64
70
|
const invResolution = 1 / resolution;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
71
|
+
|
|
72
|
+
if ( Number.isInteger( invResolution ) ) {
|
|
73
|
+
return function ( v ) {
|
|
74
|
+
return Math.round( v * invResolution ) / invResolution;
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if ( Number.isInteger( resolution ) ) {
|
|
79
|
+
return function ( v ) {
|
|
80
|
+
return Math.round( v / resolution ) * resolution;
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// The string path trims the noise to the decimals the grid needs.
|
|
85
|
+
// The count comes from the resolution's own digits, so 0.15 gets
|
|
86
|
+
// two and 2.5 gets one. A resolution below one millionth prints in
|
|
87
|
+
// exponent form, so it takes the count from its magnitude instead.
|
|
88
|
+
const text = String( resolution );
|
|
89
|
+
const decimalPlaces = ( text.indexOf( 'e' ) === -1 ) ?
|
|
90
|
+
( text.length - text.indexOf( '.' ) - 1 ) :
|
|
91
|
+
Math.ceil( -Math.log10( resolution ) );
|
|
68
92
|
|
|
69
93
|
return function ( v ) {
|
|
70
94
|
const quantized = Math.round( v * invResolution ) * resolution;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* winkComposer-specific validators
|
|
3
3
|
* Domain-specific validation functions for streaming analytics nodes
|
|
4
4
|
*/
|
|
5
5
|
|
|
@@ -121,7 +121,7 @@ const noSelfTriggers = {
|
|
|
121
121
|
!trgr.targets.includes( spec.name )
|
|
122
122
|
);
|
|
123
123
|
},
|
|
124
|
-
error: '
|
|
124
|
+
error: 'winkComposer/validation: Node cannot trigger control methods on itself'
|
|
125
125
|
}; // noSelfTriggers
|
|
126
126
|
|
|
127
127
|
// Export all composer validators
|
|
@@ -95,7 +95,7 @@ const validateWithSchema = function ( schema, object, pathPrefix = 'object' ) {
|
|
|
95
95
|
// Throw if invalid - used in init functions
|
|
96
96
|
throwIfInvalid: ( nodeType ) => {
|
|
97
97
|
if ( !valid ) {
|
|
98
|
-
const errorMessage = `
|
|
98
|
+
const errorMessage = `winkComposer/${nodeType}: validation failed:\n ${errors.join( '\n ' )}`;
|
|
99
99
|
throw new TypeError( errorMessage );
|
|
100
100
|
}
|
|
101
101
|
}
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* - Startup runs once per flow; the check costs nothing at runtime.
|
|
18
18
|
*
|
|
19
19
|
* Error message format (same across sinks):
|
|
20
|
-
* `
|
|
20
|
+
* `winkComposer/adapter: '<adapterId>' missing required method '<name>'`
|
|
21
21
|
*
|
|
22
22
|
* @see ADR-018 (sink method surface)
|
|
23
23
|
*/
|
|
@@ -32,23 +32,23 @@
|
|
|
32
32
|
* @param {*} handle - the value returned from the factory
|
|
33
33
|
* @param {string[]} requiredMethods - method names that must be functions
|
|
34
34
|
* on the handle (e.g., [ 'publishNow', 'shutdown', 'getHealth' ])
|
|
35
|
-
* @throws {Error} `
|
|
35
|
+
* @throws {Error} `winkComposer/adapter: '<adapterId>' factory returned non-object handle`
|
|
36
36
|
* if `handle` is null/undefined/non-object.
|
|
37
|
-
* @throws {Error} `
|
|
37
|
+
* @throws {Error} `winkComposer/adapter: '<adapterId>' missing required method '<name>'`
|
|
38
38
|
* on the first missing or non-function method (fails fast — does not
|
|
39
39
|
* enumerate every gap, since one is enough to abort wiring).
|
|
40
40
|
*/
|
|
41
41
|
const assertHandle = function ( adapterId, handle, requiredMethods ) {
|
|
42
42
|
if ( !handle || typeof handle !== 'object' ) {
|
|
43
43
|
throw new Error(
|
|
44
|
-
`
|
|
44
|
+
`winkComposer/adapter: '${adapterId}' factory returned non-object handle`
|
|
45
45
|
);
|
|
46
46
|
}
|
|
47
47
|
for ( let i = 0; i < requiredMethods.length; i += 1 ) {
|
|
48
48
|
const name = requiredMethods[ i ];
|
|
49
49
|
if ( typeof handle[ name ] !== 'function' ) {
|
|
50
50
|
throw new Error(
|
|
51
|
-
`
|
|
51
|
+
`winkComposer/adapter: '${adapterId}' missing required method '${name}'`
|
|
52
52
|
);
|
|
53
53
|
}
|
|
54
54
|
}
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* `src/core/test/adapter-module-surface.specs.js`.
|
|
20
20
|
*
|
|
21
21
|
* Error message format (same shape as assert-handle.js):
|
|
22
|
-
* `
|
|
22
|
+
* `winkComposer/adapter: '<adapterId>' module missing valid
|
|
23
23
|
* 'durabilityClass' export — got <value>; expected one of ...`
|
|
24
24
|
*
|
|
25
25
|
* @see ADR-018 (module surface, durability)
|
|
@@ -52,7 +52,7 @@ const assertModuleDurability = function ( adapterId, module ) {
|
|
|
52
52
|
}
|
|
53
53
|
const got = ( typeof value === 'string' ) ? `'${value}'` : String( value );
|
|
54
54
|
const err = new Error(
|
|
55
|
-
`
|
|
55
|
+
`winkComposer/adapter: '${adapterId}' module missing valid 'durabilityClass' export ` +
|
|
56
56
|
`(ADR-018) — got ${got}; expected one of ` +
|
|
57
57
|
`${DURABILITY_CLASSES.map( ( c ) => `'${c}'` ).join( ' | ' )}`
|
|
58
58
|
);
|
|
@@ -45,6 +45,7 @@
|
|
|
45
45
|
import { assertHandle } from './assert-handle.js';
|
|
46
46
|
import { assertModuleDurability } from './assert-module.js';
|
|
47
47
|
import { applySemanticsRequirement } from './wire-semantics.js';
|
|
48
|
+
import { logger } from '../logger/index.js';
|
|
48
49
|
|
|
49
50
|
/**
|
|
50
51
|
* Methods the framework calls on an emitter handle later, so they have to
|
|
@@ -53,8 +54,10 @@ import { applySemanticsRequirement } from './wire-semantics.js';
|
|
|
53
54
|
* contract's method surface.
|
|
54
55
|
* `getPressure` is optional — its presence decides whether the handle is
|
|
55
56
|
* collected into `backpressureAwareSinks`, not whether the wire succeeds.
|
|
57
|
+
* `flush` is required, as it is for storages: ADR-018 §6 lists it in
|
|
58
|
+
* the sink floor for both roles.
|
|
56
59
|
*/
|
|
57
|
-
const REQUIRED_EMITTER_METHODS = [ 'publishNow', 'shutdown', 'getHealth' ];
|
|
60
|
+
const REQUIRED_EMITTER_METHODS = [ 'publishNow', 'flush', 'shutdown', 'getHealth' ];
|
|
58
61
|
|
|
59
62
|
/**
|
|
60
63
|
* Serializes a rejection's `dropped` payload for the shutdown log. A
|
|
@@ -121,7 +124,7 @@ const emitters = ( function () {
|
|
|
121
124
|
const module = emitterModules[ target ];
|
|
122
125
|
const factory = module && module.createEmitter;
|
|
123
126
|
if ( typeof factory !== 'function' ) {
|
|
124
|
-
throw new Error( `
|
|
127
|
+
throw new Error( `winkComposer/wiring: Invalid emitter module: ${target}` );
|
|
125
128
|
}
|
|
126
129
|
|
|
127
130
|
// ADR-018 module-surface check: the module must say what
|
|
@@ -234,8 +237,8 @@ const emitters = ( function () {
|
|
|
234
237
|
if ( result.status === 'rejected' ) {
|
|
235
238
|
const err = result.reason || {};
|
|
236
239
|
const dropped = err.dropped ? ` dropped=${describeDropped( err.dropped )}` : '';
|
|
237
|
-
|
|
238
|
-
`
|
|
240
|
+
logger.error(
|
|
241
|
+
`winkComposer/wiring: emitter '${names[ i ]}' shutdown failed [${err.code || 'UNKNOWN'}]: ${err.message || String( err )}${dropped}`
|
|
239
242
|
);
|
|
240
243
|
}
|
|
241
244
|
} );
|
|
@@ -11,7 +11,7 @@ const wireLinearGraph = function ( specs, nodeModules, options = {} ) {
|
|
|
11
11
|
const nodeModule = nodeModules[ moduleName ];
|
|
12
12
|
|
|
13
13
|
if (!nodeModule) {
|
|
14
|
-
throw new Error( `
|
|
14
|
+
throw new Error( `winkComposer/wiring: Node module '${moduleName}' not found for nodeType '${spec.nodeType}'` );
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
const nextHops = nextHop ? [ nextHop ] : [];
|