@winkjs/composer 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +2 -2
- package/src/composer.js +1 -1
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +103 -658
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/index.js +16 -1
- package/src/core/env-vars.js +92 -24
- package/src/core/logger/index.js +1 -1
- package/src/core/source-manager/mqtt/client.js +197 -63
- package/src/core/source-manager/mqtt/dedup.js +16 -11
- package/src/core/source-manager/mqtt/index.js +32 -5
- package/src/core/source-manager/mqtt/status.js +168 -56
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +542 -862
- package/src/core/storage-manager/questdb/persist-plan.js +128 -167
- package/src/core/storage-manager/questdb/resolve-options.js +266 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +23 -2
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/wiring/wire-emitters.js +3 -1
- package/src/core/wiring/wire-storages.js +10 -2
- package/src/flow/flow.js +24 -3
- package/src/flow/run.js +24 -2
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
// core/utils/address/probe.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The setup probe: does the configured address answer on
|
|
5
|
+
* every address it resolves to?
|
|
6
|
+
*
|
|
7
|
+
* Why it exists (ADR-030 item 4): a name such as `db.plant.local` can
|
|
8
|
+
* resolve to two addresses, `::1` and `127.0.0.1`, and a service may
|
|
9
|
+
* listen on only one. Node connects to the first address the resolver
|
|
10
|
+
* returns. On the run-5 rig the first address refused while the
|
|
11
|
+
* second answered, the flow ran on the second for 32 hours, and then
|
|
12
|
+
* the name stopped working with no change composer could see. So the
|
|
13
|
+
* QuestDB adapter probes both of its endpoints at setup, before any
|
|
14
|
+
* client is built, and fails setup unless every resolved address
|
|
15
|
+
* answers. The message names each address and its result and, when
|
|
16
|
+
* one answered, the literal to set.
|
|
17
|
+
*
|
|
18
|
+
* How it probes:
|
|
19
|
+
* - An IP literal is probed once. A name is resolved with Node's
|
|
20
|
+
* default lookup, all addresses, no family or order forced. That is
|
|
21
|
+
* the same resolve path the QuestDB client takes, so the probe sees
|
|
22
|
+
* what the client will see. A shell tool cannot stand in for it:
|
|
23
|
+
* `curl` and `nc` resolve and fall back differently from Node.
|
|
24
|
+
* - The lookup has the same time limit as a connect. A resolver pool
|
|
25
|
+
* that stalls is a real failure mode: on the run-5 rig one held
|
|
26
|
+
* every flush open for minutes. A lookup that does not answer in
|
|
27
|
+
* time fails the probe as `timeout`, and no address is dialled.
|
|
28
|
+
* - Every address is probed in resolver order, one at a time, with a
|
|
29
|
+
* time limit each. A probe is one TCP connect; the socket is
|
|
30
|
+
* destroyed the moment it answers, fails, or times out. The socket
|
|
31
|
+
* is not `unref`'d: an unref'd socket would let the process exit
|
|
32
|
+
* with the probe still pending when nothing else holds the event
|
|
33
|
+
* loop open, which is the case for a headless flow at setup. The
|
|
34
|
+
* time limit bounds the wait instead.
|
|
35
|
+
* - A late event after the first (an error after a connect) changes
|
|
36
|
+
* nothing; each attempt settles once.
|
|
37
|
+
* - Results are `answers`, `refused` (ECONNREFUSED), `timeout`
|
|
38
|
+
* (ETIMEDOUT or the time limit), or the Node error code as is.
|
|
39
|
+
*
|
|
40
|
+
* The lookup and the connect are injectable so the adapter specs run
|
|
41
|
+
* against scripted sockets. ADR-029's adapter calls the same probe
|
|
42
|
+
* once when a flush fails to connect, off the hot path, to attach a
|
|
43
|
+
* diagnosis to the report. Setup path only; nothing here runs per
|
|
44
|
+
* message.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import dns from 'node:dns';
|
|
48
|
+
import net from 'node:net';
|
|
49
|
+
|
|
50
|
+
import { formatHostPort, suggestLiteral } from './index.js';
|
|
51
|
+
|
|
52
|
+
const DEFAULT_PROBE_TIMEOUT_MS = 2000;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Names the outcome of a failed connect.
|
|
56
|
+
*
|
|
57
|
+
* @param {Error} err - The socket error
|
|
58
|
+
* @returns {string} `refused`, `timeout`, the Node code, or `error`
|
|
59
|
+
*/
|
|
60
|
+
const resultForError = function ( err ) {
|
|
61
|
+
if ( err.code === 'ECONNREFUSED' ) {
|
|
62
|
+
return 'refused';
|
|
63
|
+
}
|
|
64
|
+
if ( err.code === 'ETIMEDOUT' ) {
|
|
65
|
+
return 'timeout';
|
|
66
|
+
}
|
|
67
|
+
return err.code || 'error';
|
|
68
|
+
}; // resultForError()
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Builds the one-address dialler over the injected connect.
|
|
72
|
+
*
|
|
73
|
+
* @param {function} connectFn - `net.connect`-shaped: `( { host, port } ) => socket`
|
|
74
|
+
* @param {number} timeoutMs - The time limit per attempt
|
|
75
|
+
* @returns {function} `( address, port ) => Promise<string>` resolving to the result name
|
|
76
|
+
*/
|
|
77
|
+
const makeDial = function ( connectFn, timeoutMs ) {
|
|
78
|
+
return function ( address, port ) {
|
|
79
|
+
return new Promise( ( resolve ) => {
|
|
80
|
+
const socket = connectFn( { host: address, port } );
|
|
81
|
+
let settled = false;
|
|
82
|
+
const finish = function ( result ) {
|
|
83
|
+
if ( settled ) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
settled = true;
|
|
87
|
+
socket.destroy();
|
|
88
|
+
resolve( result );
|
|
89
|
+
};
|
|
90
|
+
socket.once( 'connect', () => finish( 'answers' ) );
|
|
91
|
+
socket.once( 'error', ( err ) => finish( resultForError( err ) ) );
|
|
92
|
+
socket.setTimeout( timeoutMs, () => finish( 'timeout' ) );
|
|
93
|
+
} );
|
|
94
|
+
};
|
|
95
|
+
}; // makeDial()
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Resolves a name within the time limit. The timer is cleared the
|
|
99
|
+
* moment the lookup settles, so a normal lookup leaves nothing behind.
|
|
100
|
+
* The timer is not `unref`'d, for the same reason as the socket above.
|
|
101
|
+
*
|
|
102
|
+
* @param {function} lookupFn - The resolver, `( host, { all: true } ) => Promise<targets>`
|
|
103
|
+
* @param {string} host - The name to resolve
|
|
104
|
+
* @param {number} timeoutMs - The time limit
|
|
105
|
+
* @returns {Promise<Array>} The resolved targets; rejects with code `timeout` at the limit
|
|
106
|
+
*/
|
|
107
|
+
const lookupWithin = function ( lookupFn, host, timeoutMs ) {
|
|
108
|
+
return new Promise( ( resolve, reject ) => {
|
|
109
|
+
const pending = lookupFn( host, { all: true } );
|
|
110
|
+
const timer = setTimeout( () => {
|
|
111
|
+
const err = new Error( `lookup of '${host}' did not answer within ${timeoutMs} ms` );
|
|
112
|
+
err.code = 'timeout';
|
|
113
|
+
reject( err );
|
|
114
|
+
}, timeoutMs );
|
|
115
|
+
pending.then(
|
|
116
|
+
( targets ) => {
|
|
117
|
+
clearTimeout( timer );
|
|
118
|
+
resolve( targets );
|
|
119
|
+
},
|
|
120
|
+
( err ) => {
|
|
121
|
+
clearTimeout( timer );
|
|
122
|
+
reject( err );
|
|
123
|
+
}
|
|
124
|
+
);
|
|
125
|
+
} );
|
|
126
|
+
}; // lookupWithin()
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Probes the targets one at a time, in order. Written as a recursion
|
|
130
|
+
* rather than a loop so no `await` sits inside a loop body; the depth
|
|
131
|
+
* is the number of resolved addresses, two or three in practice.
|
|
132
|
+
*
|
|
133
|
+
* @param {Array<{address: string, family: number}>} targets - Resolved addresses
|
|
134
|
+
* @param {number} port - The port to probe
|
|
135
|
+
* @param {function} dial - The dialler from `makeDial`
|
|
136
|
+
* @param {Array} attempts - The results so far
|
|
137
|
+
* @returns {Promise<Array>} One `{ address, family, result }` per target
|
|
138
|
+
*/
|
|
139
|
+
const probeEach = async function ( targets, port, dial, attempts ) {
|
|
140
|
+
if ( attempts.length === targets.length ) {
|
|
141
|
+
return attempts;
|
|
142
|
+
}
|
|
143
|
+
const target = targets[ attempts.length ];
|
|
144
|
+
const result = await dial( target.address, port );
|
|
145
|
+
attempts.push( { address: target.address, family: target.family, result } );
|
|
146
|
+
return probeEach( targets, port, dial, attempts );
|
|
147
|
+
}; // probeEach()
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Probes one classified address. Passes only when every resolved
|
|
151
|
+
* address answers.
|
|
152
|
+
*
|
|
153
|
+
* @param {Object} address - A `classifyAddress` result of kind `ip` or `name`, with a port
|
|
154
|
+
* @param {Object} [options] - Injection points
|
|
155
|
+
* @param {function} [options.lookupFn=dns.promises.lookup] - The resolver
|
|
156
|
+
* @param {function} [options.connectFn=net.connect] - The connect
|
|
157
|
+
* @param {number} [options.timeoutMs=2000] - The time limit per address, and for the lookup
|
|
158
|
+
* @returns {Promise<Object>} `{ ok, host, port, attempts }`, or
|
|
159
|
+
* `{ ok: false, host, port, lookupError, attempts: [] }` when the name did not resolve in time
|
|
160
|
+
*/
|
|
161
|
+
const probeAddress = async function ( address, options = {} ) {
|
|
162
|
+
const {
|
|
163
|
+
lookupFn = dns.promises.lookup,
|
|
164
|
+
connectFn = net.connect,
|
|
165
|
+
timeoutMs = DEFAULT_PROBE_TIMEOUT_MS
|
|
166
|
+
} = options;
|
|
167
|
+
const dial = makeDial( connectFn, timeoutMs );
|
|
168
|
+
|
|
169
|
+
let targets;
|
|
170
|
+
if ( address.kind === 'ip' ) {
|
|
171
|
+
targets = [ { address: address.host, family: address.family } ];
|
|
172
|
+
} else {
|
|
173
|
+
try {
|
|
174
|
+
targets = await lookupWithin( lookupFn, address.host, timeoutMs );
|
|
175
|
+
} catch ( err ) {
|
|
176
|
+
return { ok: false, host: address.host, port: address.port, lookupError: err.code || err.message, attempts: [] };
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const attempts = await probeEach( targets, address.port, dial, [] );
|
|
181
|
+
const ok = attempts.every( ( attempt ) => attempt.result === 'answers' );
|
|
182
|
+
return { ok, host: address.host, port: address.port, attempts };
|
|
183
|
+
}; // probeAddress()
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Renders a probe outcome for an operator: each address with its
|
|
187
|
+
* result and, when one answered but another did not, the literal to
|
|
188
|
+
* set. The rig's own case reads
|
|
189
|
+
* `[::1]:9000 refused, 127.0.0.1:9000 answers; set ilpUrl to 127.0.0.1:9000`.
|
|
190
|
+
*
|
|
191
|
+
* @param {Object} outcome - A `probeAddress` result
|
|
192
|
+
* @param {string} field - The config key, for the suggestion
|
|
193
|
+
* @param {Object} address - The classified address that was probed
|
|
194
|
+
* @returns {string} The detail sentence
|
|
195
|
+
*/
|
|
196
|
+
const describeProbe = function ( outcome, field, address ) {
|
|
197
|
+
if ( outcome.lookupError !== undefined ) {
|
|
198
|
+
return `'${outcome.host}' did not resolve (${outcome.lookupError})`;
|
|
199
|
+
}
|
|
200
|
+
const listed = outcome.attempts
|
|
201
|
+
.map( ( attempt ) => `${formatHostPort( attempt.address, outcome.port )} ${attempt.result}` )
|
|
202
|
+
.join( ', ' );
|
|
203
|
+
const answering = outcome.attempts.find( ( attempt ) => attempt.result === 'answers' );
|
|
204
|
+
if ( outcome.ok || ( answering === undefined ) ) {
|
|
205
|
+
return listed;
|
|
206
|
+
}
|
|
207
|
+
return `${listed}; set ${field} to ${suggestLiteral( address, answering.address )}`;
|
|
208
|
+
}; // describeProbe()
|
|
209
|
+
|
|
210
|
+
export { probeAddress, describeProbe, DEFAULT_PROBE_TIMEOUT_MS };
|
|
@@ -32,9 +32,22 @@
|
|
|
32
32
|
* thrown value. When the report closure itself fails, the guard
|
|
33
33
|
* falls back to one bare log line with no user-value
|
|
34
34
|
* interpolation.
|
|
35
|
+
* - The fault report is bounded per wrapped callback (ADR-029, the
|
|
36
|
+
* bounded loss line). A callback that fails on every call would
|
|
37
|
+
* print one line per call, which for a status callback on a busy
|
|
38
|
+
* source is a line per message. So the first two faults of an
|
|
39
|
+
* episode report in full, later faults are counted, and one summary
|
|
40
|
+
* reports per minute with the count. A quiet minute ends the
|
|
41
|
+
* episode. The summary goes through the site's own channel with the
|
|
42
|
+
* count appended to the detail, so no site changes.
|
|
35
43
|
*/
|
|
36
44
|
|
|
37
45
|
import { logger } from '../../logger/index.js';
|
|
46
|
+
import { createLineBound } from '../line-rate/index.js';
|
|
47
|
+
|
|
48
|
+
/** The bound on the fault report (see the file header). */
|
|
49
|
+
const FULL_FAULT_LINES_PER_EPISODE = 2;
|
|
50
|
+
const FAULT_SUMMARY_INTERVAL_MS = 60000;
|
|
38
51
|
|
|
39
52
|
/**
|
|
40
53
|
* Frozen sentinel `wrapTransform` returns when the transform threw.
|
|
@@ -92,9 +105,17 @@ const wrapCallback = function ( fn, { name, severity, report } ) {
|
|
|
92
105
|
if ( typeof fn !== 'function' ) {
|
|
93
106
|
return null;
|
|
94
107
|
}
|
|
108
|
+
const bounded = createLineBound( {
|
|
109
|
+
fullLines: FULL_FAULT_LINES_PER_EPISODE,
|
|
110
|
+
intervalMs: FAULT_SUMMARY_INTERVAL_MS,
|
|
111
|
+
printFull: report,
|
|
112
|
+
printSummary: function ( count, seconds, sev, callbackName, detail ) {
|
|
113
|
+
report( sev, callbackName, `${detail}; ${count} more fault(s) in the last ${seconds} s` );
|
|
114
|
+
}
|
|
115
|
+
} );
|
|
95
116
|
const faultFn = function ( err ) {
|
|
96
117
|
try {
|
|
97
|
-
|
|
118
|
+
bounded( severity, name, describeFault( err ) );
|
|
98
119
|
} catch {
|
|
99
120
|
reportFallback();
|
|
100
121
|
}
|
|
@@ -152,4 +173,4 @@ const wrapTransform = function ( fn, onFault ) {
|
|
|
152
173
|
};
|
|
153
174
|
}; // wrapTransform()
|
|
154
175
|
|
|
155
|
-
export { wrapCallback, wrapTransform, TRANSFORM_THREW };
|
|
176
|
+
export { wrapCallback, wrapTransform, TRANSFORM_THREW, FULL_FAULT_LINES_PER_EPISODE, FAULT_SUMMARY_INTERVAL_MS };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// core/utils/clock/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The stopwatch clock for time rules that must not
|
|
5
|
+
* follow the wall clock.
|
|
6
|
+
*
|
|
7
|
+
* `Date.now()` reads the wall clock, and the wall clock can step. NTP
|
|
8
|
+
* corrects a board after boot or after a long time without network,
|
|
9
|
+
* and an operator can set the time by hand. A rule such as "red after
|
|
10
|
+
* 30 s disconnected" or "expire a dedup entry after 120 s" would then
|
|
11
|
+
* fire early or late by the size of the step. `performance.now()`
|
|
12
|
+
* reads a clock that only counts up, from process start, in
|
|
13
|
+
* milliseconds with a fraction. A step in the wall clock does not
|
|
14
|
+
* move it (ADR-018, long-running stability).
|
|
15
|
+
*
|
|
16
|
+
* The function reads `globalThis.performance` at every call, for two
|
|
17
|
+
* reasons. The method must be called on its own object, so the bare
|
|
18
|
+
* method cannot be passed around as a clock. And a test that fakes the
|
|
19
|
+
* stopwatch replaces the global object rather than patching the
|
|
20
|
+
* method, so a captured reference would keep reading the real clock.
|
|
21
|
+
*
|
|
22
|
+
* Use this clock for durations and expiry. Use `Date.now()` for
|
|
23
|
+
* timestamps that leave the process, such as a record's time field.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Reads the stopwatch clock.
|
|
28
|
+
*
|
|
29
|
+
* @returns {number} Milliseconds since process start, with a fraction;
|
|
30
|
+
* never decreases.
|
|
31
|
+
*/
|
|
32
|
+
export const monotonicNow = function () {
|
|
33
|
+
return globalThis.performance.now();
|
|
34
|
+
}; // monotonicNow()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// core/utils/jitter/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Adds a random share to a reconnect period, once per
|
|
5
|
+
* client, so a fleet does not retry in step.
|
|
6
|
+
*
|
|
7
|
+
* mqtt.js retries at a fixed period. When one broker goes away, every
|
|
8
|
+
* client that used it fails at the same moment and retries on the same
|
|
9
|
+
* beat. Attempt `k` from every client lands on the recovering broker
|
|
10
|
+
* at `k` times the period, all at once. This module gives each client
|
|
11
|
+
* its own period: the configured one plus a random share of up to a
|
|
12
|
+
* fifth of it. The share is drawn once at setup, and because attempt
|
|
13
|
+
* `k` fires at `k` times the period, a fixed offset spreads the fleet
|
|
14
|
+
* further with each attempt.
|
|
15
|
+
*
|
|
16
|
+
* The configured period is the operator's floor. The helper never
|
|
17
|
+
* shortens it, and a period of zero stays zero, so a reconnect that
|
|
18
|
+
* was switched off stays off. Both MQTT adapters use this module; the
|
|
19
|
+
* reconnect loop itself stays the library's (ADR-018).
|
|
20
|
+
*
|
|
21
|
+
* The random source is a parameter, so a spec can name its result.
|
|
22
|
+
* Nothing here runs on a message path.
|
|
23
|
+
*
|
|
24
|
+
* @see ADR-018
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/** The largest share of the period the jitter may add. */
|
|
28
|
+
export const RECONNECT_JITTER_FRACTION = 0.2;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Returns the period plus the floor of a random share of it.
|
|
32
|
+
*
|
|
33
|
+
* @param {number} periodMs - The configured period, a finite number of milliseconds, zero or more
|
|
34
|
+
* @param {Object} [options]
|
|
35
|
+
* @param {number} [options.fraction=RECONNECT_JITTER_FRACTION] - The largest share to add, in `[0, 1]`
|
|
36
|
+
* @param {Function} [options.random=Math.random] - Returns a number in `[0, 1)`
|
|
37
|
+
* @returns {number} `periodMs + Math.floor( random() * periodMs * fraction )`
|
|
38
|
+
* @throws {TypeError} when an argument is out of range
|
|
39
|
+
*/
|
|
40
|
+
export const jitteredPeriod = function ( periodMs, { fraction = RECONNECT_JITTER_FRACTION, random = Math.random } = {} ) {
|
|
41
|
+
if ( ( typeof periodMs !== 'number' ) || !Number.isFinite( periodMs ) || ( periodMs < 0 ) ) {
|
|
42
|
+
throw new TypeError( `winkComposer/jitter: periodMs must be a finite number of milliseconds, zero or more, got ${String( periodMs )}` );
|
|
43
|
+
}
|
|
44
|
+
if ( ( typeof fraction !== 'number' ) || !( ( fraction >= 0 ) && ( fraction <= 1 ) ) ) {
|
|
45
|
+
throw new TypeError( `winkComposer/jitter: fraction must be a number in [0, 1], got ${String( fraction )}` );
|
|
46
|
+
}
|
|
47
|
+
if ( typeof random !== 'function' ) {
|
|
48
|
+
throw new TypeError( 'winkComposer/jitter: random must be a function' );
|
|
49
|
+
}
|
|
50
|
+
return periodMs + Math.floor( random() * periodMs * fraction );
|
|
51
|
+
}; // jitteredPeriod()
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// core/utils/line-rate/index.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Bounds how often a repeating event reaches the log.
|
|
5
|
+
*
|
|
6
|
+
* A dead sensor, a stuck callback, or an unreachable database produces
|
|
7
|
+
* the same event once per row. Printing every one fills a log file with
|
|
8
|
+
* one line per second for a whole night, and the line that matters is
|
|
9
|
+
* lost in the repeats. This module keeps the first lines of such an
|
|
10
|
+
* episode in full and then prints one summary per interval. An episode
|
|
11
|
+
* is a run of events with no quiet interval between them. A quiet
|
|
12
|
+
* interval ends the episode, so the next event prints in full again.
|
|
13
|
+
*
|
|
14
|
+
* The bound is a pure closure over a stopwatch reading, `monotonicNow`,
|
|
15
|
+
* so a step in the wall clock can neither end an episode nor move a
|
|
16
|
+
* summary (ADR-018, long-running stability). It holds no timer, so it
|
|
17
|
+
* needs no shutdown and cannot keep a process alive. The
|
|
18
|
+
* summary prints at the next event after the interval, never on a
|
|
19
|
+
* timer. So a burst that stops shows its count only when the next
|
|
20
|
+
* episode starts, or never. That is the accepted trade for zero timers.
|
|
21
|
+
*
|
|
22
|
+
* Two sites share this shape. The QuestDB adapter's flush-failure loss
|
|
23
|
+
* line uses the same algorithm inline (ADR-029, the bounded loss line).
|
|
24
|
+
* The adapter's default row-skip warning and the callback guard's
|
|
25
|
+
* `CALLBACK_FAILED` line both use this module.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { monotonicNow } from '../clock/index.js';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Creates a line bound.
|
|
32
|
+
*
|
|
33
|
+
* The returned function takes the event's arguments, up to three, and
|
|
34
|
+
* forwards them by identity to `printFull` for the first `fullLines`
|
|
35
|
+
* events of an episode. What the arguments mean is the caller's
|
|
36
|
+
* business: the callback guard passes a severity, a callback name and
|
|
37
|
+
* a fault description; the row-skip warning passes a raw value and a
|
|
38
|
+
* partition id. Later events inside the interval are counted. The
|
|
39
|
+
* first event a full interval after the last printed line calls
|
|
40
|
+
* `printSummary` with the count since that line, the seconds elapsed,
|
|
41
|
+
* and the latest arguments. When an episode ends with counted events,
|
|
42
|
+
* their summary prints before the next episode's first full line.
|
|
43
|
+
*
|
|
44
|
+
* Three named parameters stand in for a rest parameter on purpose. A
|
|
45
|
+
* rest parameter builds an array on every call, and the row-skip
|
|
46
|
+
* warning runs once per skipped row.
|
|
47
|
+
*
|
|
48
|
+
* @param {Object} options
|
|
49
|
+
* @param {number} options.fullLines - How many events per episode print in full.
|
|
50
|
+
* @param {number} options.intervalMs - The summary interval, also the quiet gap that ends an episode.
|
|
51
|
+
* @param {Function} options.printFull - Called as `printFull( arg1, arg2, arg3 )` with the event's arguments.
|
|
52
|
+
* @param {Function} options.printSummary - Called as `printSummary( count, seconds, arg1, arg2, arg3 )`.
|
|
53
|
+
* @returns {Function} `( arg1, arg2, arg3 ) => void`.
|
|
54
|
+
*/
|
|
55
|
+
export const createLineBound = function ( { fullLines, intervalMs, printFull, printSummary } ) {
|
|
56
|
+
let lastEventAt = 0;
|
|
57
|
+
let lastLineAt = 0;
|
|
58
|
+
let printed = 0;
|
|
59
|
+
let counted = 0;
|
|
60
|
+
|
|
61
|
+
// The state clears before the printer runs, as on the full-line
|
|
62
|
+
// path. A printer that throws then costs that one line only. With
|
|
63
|
+
// the old order a throw left the count and the stamp behind, and
|
|
64
|
+
// every later event ran straight into the printer again.
|
|
65
|
+
const summarize = function ( now, arg1, arg2, arg3 ) {
|
|
66
|
+
const seconds = Math.round( ( now - lastLineAt ) / 1000 );
|
|
67
|
+
const count = counted;
|
|
68
|
+
lastLineAt = now;
|
|
69
|
+
counted = 0;
|
|
70
|
+
printSummary( count, seconds, arg1, arg2, arg3 );
|
|
71
|
+
}; // summarize()
|
|
72
|
+
|
|
73
|
+
// The latest arguments are kept for the summary that closes an
|
|
74
|
+
// episode. They belong to the last counted event, not the new one.
|
|
75
|
+
let lastArg1;
|
|
76
|
+
let lastArg2;
|
|
77
|
+
let lastArg3;
|
|
78
|
+
|
|
79
|
+
return function ( arg1, arg2, arg3 ) {
|
|
80
|
+
const now = monotonicNow();
|
|
81
|
+
if ( ( now - lastEventAt ) >= intervalMs ) {
|
|
82
|
+
// A quiet interval ended the previous episode.
|
|
83
|
+
if ( counted > 0 ) {
|
|
84
|
+
summarize( now, lastArg1, lastArg2, lastArg3 );
|
|
85
|
+
}
|
|
86
|
+
printed = 0;
|
|
87
|
+
}
|
|
88
|
+
lastEventAt = now;
|
|
89
|
+
lastArg1 = arg1;
|
|
90
|
+
lastArg2 = arg2;
|
|
91
|
+
lastArg3 = arg3;
|
|
92
|
+
if ( printed < fullLines ) {
|
|
93
|
+
printed += 1;
|
|
94
|
+
lastLineAt = now;
|
|
95
|
+
printFull( arg1, arg2, arg3 );
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
counted += 1;
|
|
99
|
+
if ( ( now - lastLineAt ) < intervalMs ) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
summarize( now, arg1, arg2, arg3 );
|
|
103
|
+
};
|
|
104
|
+
}; // createLineBound()
|
|
@@ -11,24 +11,30 @@
|
|
|
11
11
|
* for human reading. Both used to carry their own copy of the same
|
|
12
12
|
* formula; this module is the single source.
|
|
13
13
|
*
|
|
14
|
-
* The quantizer
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* recomputation.
|
|
14
|
+
* The quantizer chooses its arithmetic once, at build time, so the
|
|
15
|
+
* per-message hot path is a few float operations and no allocation.
|
|
16
|
+
* Three shapes of resolution get three paths:
|
|
18
17
|
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* -
|
|
25
|
-
*
|
|
26
|
-
*
|
|
18
|
+
* - The inverse is a whole number (0.1, 0.01, 0.001, 0.25, 0.5, 0.2,
|
|
19
|
+
* 0.125). The value is `Math.round( v * inverse ) / inverse`. A
|
|
20
|
+
* division by an exact whole number is correctly rounded, so it
|
|
21
|
+
* returns the nearest double to the grid value: 101 / 10 is 10.1
|
|
22
|
+
* and 95 / 4 is 23.75. No string is built.
|
|
23
|
+
* - The resolution is a whole number (5, 10, 7). The value is
|
|
24
|
+
* `Math.round( v / resolution ) * resolution`, exact for whole
|
|
25
|
+
* numbers. No string is built.
|
|
26
|
+
* - Anything else (0.3, 0.15, 2.5). The old path stays: multiply,
|
|
27
|
+
* round, multiply, then `toFixed` to trim the noise and `Number` to
|
|
28
|
+
* parse it back. The trim keeps as many decimals as the resolution
|
|
29
|
+
* itself has. This path allocates one short string per value. It is
|
|
30
|
+
* rare: a declared resolution is nearly always a power of ten.
|
|
27
31
|
*
|
|
28
|
-
*
|
|
29
|
-
* `
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
+
* Why not `toFixed` for everything. `Math.round( 10.1 * 10 ) * 0.1` is
|
|
33
|
+
* `10.100000000000001`, because 0.1 is not exact in binary, so a
|
|
34
|
+
* multiply by the resolution needs a trim. The trim built a string per
|
|
35
|
+
* value on the storage hot path. It also cut grid values short: with a
|
|
36
|
+
* resolution of 0.25 the trim kept one decimal, so 23.75 became 23.8,
|
|
37
|
+
* off the grid. Division by the exact inverse has neither problem.
|
|
32
38
|
*
|
|
33
39
|
* Returning `null` for `resolution === 1` (or undefined) is a small but
|
|
34
40
|
* load-bearing optimisation: the common case is "no resolution declared"
|
|
@@ -62,9 +68,27 @@ const buildResolutionQuantizer = function ( resolution ) {
|
|
|
62
68
|
if ( resolution === undefined || resolution === 1 ) return null;
|
|
63
69
|
|
|
64
70
|
const invResolution = 1 / resolution;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
71
|
+
|
|
72
|
+
if ( Number.isInteger( invResolution ) ) {
|
|
73
|
+
return function ( v ) {
|
|
74
|
+
return Math.round( v * invResolution ) / invResolution;
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if ( Number.isInteger( resolution ) ) {
|
|
79
|
+
return function ( v ) {
|
|
80
|
+
return Math.round( v / resolution ) * resolution;
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// The string path trims the noise to the decimals the grid needs.
|
|
85
|
+
// The count comes from the resolution's own digits, so 0.15 gets
|
|
86
|
+
// two and 2.5 gets one. A resolution below one millionth prints in
|
|
87
|
+
// exponent form, so it takes the count from its magnitude instead.
|
|
88
|
+
const text = String( resolution );
|
|
89
|
+
const decimalPlaces = ( text.indexOf( 'e' ) === -1 ) ?
|
|
90
|
+
( text.length - text.indexOf( '.' ) - 1 ) :
|
|
91
|
+
Math.ceil( -Math.log10( resolution ) );
|
|
68
92
|
|
|
69
93
|
return function ( v ) {
|
|
70
94
|
const quantized = Math.round( v * invResolution ) * resolution;
|
|
@@ -54,8 +54,10 @@ import { logger } from '../logger/index.js';
|
|
|
54
54
|
* contract's method surface.
|
|
55
55
|
* `getPressure` is optional — its presence decides whether the handle is
|
|
56
56
|
* collected into `backpressureAwareSinks`, not whether the wire succeeds.
|
|
57
|
+
* `flush` is required, as it is for storages: ADR-018 §6 lists it in
|
|
58
|
+
* the sink floor for both roles.
|
|
57
59
|
*/
|
|
58
|
-
const REQUIRED_EMITTER_METHODS = [ 'publishNow', 'shutdown', 'getHealth' ];
|
|
60
|
+
const REQUIRED_EMITTER_METHODS = [ 'publishNow', 'flush', 'shutdown', 'getHealth' ];
|
|
59
61
|
|
|
60
62
|
/**
|
|
61
63
|
* Serializes a rejection's `dropped` payload for the shutdown log. A
|
|
@@ -146,6 +146,13 @@ const storages = ( function () {
|
|
|
146
146
|
// to assetClass.name if not explicitly specified.
|
|
147
147
|
const wire = async function ( specs, storageConfigs, storageModules, assetClass = null ) {
|
|
148
148
|
const pendingStorages = [];
|
|
149
|
+
// Names whose factory has been called in THIS wire() call. The
|
|
150
|
+
// singleton slot is filled only after every pending factory has
|
|
151
|
+
// settled. Without this set, a flow with several persist-if nodes
|
|
152
|
+
// on one storage would call the factory once per node and keep
|
|
153
|
+
// only the last handle. Each orphan would hold a flush timer, an
|
|
154
|
+
// agent, and a sender for the life of the process.
|
|
155
|
+
const pendingNames = new Set();
|
|
149
156
|
|
|
150
157
|
for ( const spec of specs ) {
|
|
151
158
|
// Skip non-persistIf nodes
|
|
@@ -155,8 +162,8 @@ const storages = ( function () {
|
|
|
155
162
|
|
|
156
163
|
const storageName = spec.storageName;
|
|
157
164
|
|
|
158
|
-
// Skip if already in singleton registry
|
|
159
|
-
if ( singletons[ storageName ] ) {
|
|
165
|
+
// Skip if already in singleton registry, or already being built
|
|
166
|
+
if ( singletons[ storageName ] || pendingNames.has( storageName ) ) {
|
|
160
167
|
continue; // eslint-disable-line no-continue
|
|
161
168
|
}
|
|
162
169
|
|
|
@@ -196,6 +203,7 @@ const storages = ( function () {
|
|
|
196
203
|
|
|
197
204
|
// createStorage may return Promise (async) or storage directly (sync)
|
|
198
205
|
const storageOrPromise = module.createStorage( effectiveConfig );
|
|
206
|
+
pendingNames.add( storageName );
|
|
199
207
|
pendingStorages.push( { storageName, storageOrPromise } );
|
|
200
208
|
}
|
|
201
209
|
|
package/src/flow/flow.js
CHANGED
|
@@ -47,6 +47,27 @@ import { validateWithSchema } from '../core/utils/validate/index.js';
|
|
|
47
47
|
import { assetClassSchema } from '../core/semantics/schemas/index.js';
|
|
48
48
|
import { expandTemplateSpec } from './expand-template-spec.js';
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Throws the flow's setup error for an adapter config that the
|
|
52
|
+
* adapter's own schema rejected. The error carries `INVALID_CONFIG`,
|
|
53
|
+
* the ADR-018 setup code, and the ADR-028 bracket. A program matches
|
|
54
|
+
* the code; an operator reads the remedy the schema names.
|
|
55
|
+
*
|
|
56
|
+
* @param {{valid: boolean, errors: string[]}} validation - The schema result
|
|
57
|
+
* @param {string} label - `flow.<method>.<adapterId>`, for the message
|
|
58
|
+
* @throws {Error} INVALID_CONFIG when the config is invalid
|
|
59
|
+
*/
|
|
60
|
+
const throwIfAdapterConfigInvalid = function ( validation, label ) {
|
|
61
|
+
if ( validation.valid ) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
const err = new Error(
|
|
65
|
+
`winkComposer/${label}: validation failed [INVALID_CONFIG]:\n - ${validation.errors.join( '\n - ' )}`
|
|
66
|
+
);
|
|
67
|
+
err.code = 'INVALID_CONFIG';
|
|
68
|
+
throw err;
|
|
69
|
+
}; // throwIfAdapterConfigInvalid()
|
|
70
|
+
|
|
50
71
|
const flow = function ( flowName = 'flow1' ) {
|
|
51
72
|
const api = Object.create( null );
|
|
52
73
|
|
|
@@ -171,7 +192,7 @@ const flow = function ( flowName = 'flow1' ) {
|
|
|
171
192
|
// not later inside adapter.start().
|
|
172
193
|
if ( adapter.configSchema ) {
|
|
173
194
|
const validation = validateWithSchema( adapter.configSchema, sourceConfig, 'source' );
|
|
174
|
-
|
|
195
|
+
throwIfAdapterConfigInvalid( validation, `flow.source.${adapter.id || 'unknown'}` );
|
|
175
196
|
}
|
|
176
197
|
|
|
177
198
|
runtime.source = { adapter, config: sourceConfig };
|
|
@@ -195,7 +216,7 @@ const flow = function ( flowName = 'flow1' ) {
|
|
|
195
216
|
// not later inside adapter.createEmitter().
|
|
196
217
|
if ( adapter.configSchema ) {
|
|
197
218
|
const validation = validateWithSchema( adapter.configSchema, emitterConfig, 'emitter' );
|
|
198
|
-
|
|
219
|
+
throwIfAdapterConfigInvalid( validation, `flow.emitter.${adapter.id}` );
|
|
199
220
|
}
|
|
200
221
|
|
|
201
222
|
runtime.emitters[ adapter.id ] = { adapter, config: emitterConfig };
|
|
@@ -219,7 +240,7 @@ const flow = function ( flowName = 'flow1' ) {
|
|
|
219
240
|
// not later inside adapter.createStorage().
|
|
220
241
|
if ( adapter.configSchema ) {
|
|
221
242
|
const validation = validateWithSchema( adapter.configSchema, storageConfig, 'storage' );
|
|
222
|
-
|
|
243
|
+
throwIfAdapterConfigInvalid( validation, `flow.storage.${adapter.id}` );
|
|
223
244
|
}
|
|
224
245
|
|
|
225
246
|
runtime.storages[ adapter.id ] = { adapter, config: storageConfig };
|