@winkjs/composer 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +2 -2
- package/src/composer.js +1 -1
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +103 -658
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/index.js +16 -1
- package/src/core/env-vars.js +99 -13
- package/src/core/logger/index.js +1 -1
- package/src/core/source-manager/mqtt/client.js +197 -63
- package/src/core/source-manager/mqtt/dedup.js +16 -11
- package/src/core/source-manager/mqtt/index.js +32 -5
- package/src/core/source-manager/mqtt/status.js +168 -56
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +574 -847
- package/src/core/storage-manager/questdb/persist-plan.js +128 -167
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +23 -2
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/wiring/wire-emitters.js +3 -1
- package/src/core/wiring/wire-storages.js +10 -2
- package/src/flow/flow.js +24 -3
- package/src/flow/run.js +24 -2
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
// core/storage-manager/questdb/resolve-options.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview Turns the QuestDB storage options into the settings the
|
|
5
|
+
* adapter runs on.
|
|
6
|
+
*
|
|
7
|
+
* The adapter accepts options from two places: the `.storage()` config
|
|
8
|
+
* in the flow, and the `QUESTDB_*` environment variables read by
|
|
9
|
+
* `env-vars.js`. This module merges the two, fills in the defaults, and
|
|
10
|
+
* derives the buffer ceiling from the flush threshold. It also computes
|
|
11
|
+
* the deadline of one flush from the rows that flush carries. It is a
|
|
12
|
+
* pure function of its inputs, so it can be tested without a flow, a
|
|
13
|
+
* client, or a live environment. The adapter calls `resolveOptions`
|
|
14
|
+
* once, at setup, and `flushDeadlineFor` once per flush. Nothing here
|
|
15
|
+
* runs on the per-row path.
|
|
16
|
+
*
|
|
17
|
+
* Why the derived values live here and not in `env-vars.js`. The buffer
|
|
18
|
+
* ceiling defaults to ten times the flush threshold. A fixed ceiling in
|
|
19
|
+
* the environment would fail setup whenever a user raised the threshold
|
|
20
|
+
* in config without also raising the ceiling. So the environment carries
|
|
21
|
+
* a value only when the operator set one, and this module derives the
|
|
22
|
+
* rest from what actually won.
|
|
23
|
+
*
|
|
24
|
+
* Why the deadline depends on the row count. The client gives a bigger
|
|
25
|
+
* request more time: its request timeout is an inactivity timeout that
|
|
26
|
+
* grows with the bytes sent, at a planning speed of 100 KiB per second
|
|
27
|
+
* (`request_min_throughput`, a 4.2.0 fact). A fixed deadline would
|
|
28
|
+
* abandon a large but healthy send over a slow link and report rows
|
|
29
|
+
* lost that then land. So the deadline is the client's worst case for
|
|
30
|
+
* that batch, plus a margin. One attempt is the request timeout plus
|
|
31
|
+
* the transfer time at a planning size of 512 bytes a row. The client
|
|
32
|
+
* checks its retry window only when an attempt ends, so its last
|
|
33
|
+
* attempt can run in full past the window. The deadline is therefore
|
|
34
|
+
* the retry window, two attempts, the longest backoff, and 5 seconds.
|
|
35
|
+
* One row gets about 36 seconds; 5000 rows get 86 seconds; a full
|
|
36
|
+
* 50000-row catch-up send gets about 9 minutes. When the operator
|
|
37
|
+
* sets `flushDeadlineMs`, that fixed value is used for every flush.
|
|
38
|
+
*
|
|
39
|
+
* Precedence, highest first (ADR-018 §10, extended for the legacy
|
|
40
|
+
* aliases of ADR-029 item 10):
|
|
41
|
+
*
|
|
42
|
+
* 1. the new key in the config (`flushRows`)
|
|
43
|
+
* 2. the legacy key in the config (`autoFlushRows`)
|
|
44
|
+
* 3. the new environment variable (`QUESTDB_FLUSH_ROWS`)
|
|
45
|
+
* 4. the legacy environment variable (`QUESTDB_AUTO_FLUSH_ROWS`)
|
|
46
|
+
* 5. the default
|
|
47
|
+
*
|
|
48
|
+
* Two legacy keys map to a new one: `autoFlushRows` to `flushRows`, and
|
|
49
|
+
* `idleFlushCheckMs` to `flushIntervalMs`. The other three,
|
|
50
|
+
* `flushMode`, `idleFlushAfterMs` and `autoFlushIntervalMs`, have no
|
|
51
|
+
* meaning once composer owns every flush (ADR-029). They are accepted
|
|
52
|
+
* and ignored. Every supplied legacy key is named in the deprecation
|
|
53
|
+
* report, with what happened to it, so the adapter can print one
|
|
54
|
+
* `DEPRECATED_OPTION` line at setup. All five keys are removed in
|
|
55
|
+
* 0.8.0.
|
|
56
|
+
*
|
|
57
|
+
* The one relation enforced here: the ceiling is never below twice
|
|
58
|
+
* the threshold. Rows in flight count against the ceiling, so a
|
|
59
|
+
* ceiling equal to the threshold would refuse every row that arrives
|
|
60
|
+
* during a row-triggered flush. Twice the threshold holds one batch
|
|
61
|
+
* in flight and one batch buffering. A smaller ceiling fails setup
|
|
62
|
+
* with `INVALID_CONFIG`.
|
|
63
|
+
*
|
|
64
|
+
* The client constants used in the deadline are `@questdb/nodejs-client`
|
|
65
|
+
* 4.2.0 facts, cited by line in the client's `dist/es/index.mjs`. The
|
|
66
|
+
* defaults sit at lines 230 to 234: `request_min_throughput` 102400
|
|
67
|
+
* bytes a second, `request_timeout` 10 s, `retry_timeout` 10 s. The
|
|
68
|
+
* throughput option is documented at line 404 and read at line 275.
|
|
69
|
+
* The request timeout grows with the bytes sent at line 1142.
|
|
70
|
+
*
|
|
71
|
+
* The retry clock starts when the first attempt ends, at line 1183.
|
|
72
|
+
* The window is checked when a later attempt ends, at lines 1186 to
|
|
73
|
+
* 1188. The backoff between attempts doubles and stops at 1000 ms,
|
|
74
|
+
* with a jitter of up to 5 ms, at lines 1192 to 1194. Re-verify these
|
|
75
|
+
* lines on a client upgrade. A configured `requestTimeout` or
|
|
76
|
+
* `retryTimeout` replaces its default in the sum, so an operator who
|
|
77
|
+
* lengthens a timeout lengthens the deadline with it.
|
|
78
|
+
*
|
|
79
|
+
* The transport setting has two vocabularies. In config, `stdlibHttp`
|
|
80
|
+
* is a boolean. In the environment, `QUESTDB_STDLIB_HTTP` takes the
|
|
81
|
+
* client's own words `on` and `off`. This module maps the words to the
|
|
82
|
+
* boolean, and the default is the standard-library transport
|
|
83
|
+
* (`DEFAULT_STDLIB_HTTP` explains why).
|
|
84
|
+
*
|
|
85
|
+
* @see ADR-029
|
|
86
|
+
*/
|
|
87
|
+
|
|
88
|
+
// ============================================================================
|
|
89
|
+
// DEFAULTS
|
|
90
|
+
// ============================================================================
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Rows that trigger a flush from inside `write()`. At the measured 130
|
|
94
|
+
* to 290 bytes a row this is 0.65 to 1.5 MB per request, sized for an
|
|
95
|
+
* edge device and a slow plant link. Larger deployments raise it by
|
|
96
|
+
* config.
|
|
97
|
+
* @type {number}
|
|
98
|
+
*/
|
|
99
|
+
const DEFAULT_FLUSH_ROWS = 5000;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Period of the flush timer. Rows land within about this long after
|
|
103
|
+
* they were written, busy stream or idle stream alike, and a power cut
|
|
104
|
+
* costs at most about this much data.
|
|
105
|
+
* @type {number}
|
|
106
|
+
*/
|
|
107
|
+
const DEFAULT_FLUSH_INTERVAL_MS = 1000;
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The ceiling is this many flush thresholds. Past it, `write()` sheds
|
|
111
|
+
* new rows with `STORAGE_FULL`. With flushes held while the endpoint is
|
|
112
|
+
* unreachable (ADR-029), the ceiling is the outage the adapter rides
|
|
113
|
+
* through without loss: ten batches hold 50 seconds at 1000 rows a
|
|
114
|
+
* second and hours at plant rate, in 6.5 to 15 MB of memory.
|
|
115
|
+
* @type {number}
|
|
116
|
+
*/
|
|
117
|
+
const DEFAULT_CEILING_MULTIPLIER = 10;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The client's standard-library HTTP transport is selected unless the
|
|
121
|
+
* operator opts out. It is the transport whose requests end: a refused
|
|
122
|
+
* connection rejects at once, and a stalled request ends within the
|
|
123
|
+
* retry window plus one request timeout. The client's own default,
|
|
124
|
+
* undici, retries a refused connection without end, and its abort
|
|
125
|
+
* cannot end that retry (ADR-029).
|
|
126
|
+
* @type {boolean}
|
|
127
|
+
*/
|
|
128
|
+
const DEFAULT_STDLIB_HTTP = true;
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The client's default `request_timeout` (4.2.0 fact).
|
|
132
|
+
* @type {number}
|
|
133
|
+
*/
|
|
134
|
+
const CLIENT_DEFAULT_REQUEST_TIMEOUT_MS = 10000;
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The client's default `retry_timeout` (4.2.0 fact).
|
|
138
|
+
* @type {number}
|
|
139
|
+
*/
|
|
140
|
+
const CLIENT_DEFAULT_RETRY_TIMEOUT_MS = 10000;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The client's default `request_min_throughput`, in bytes per second
|
|
144
|
+
* (4.2.0 fact). The client extends its request timeout by the time the
|
|
145
|
+
* request would take at this speed.
|
|
146
|
+
* @type {number}
|
|
147
|
+
*/
|
|
148
|
+
const CLIENT_DEFAULT_MIN_THROUGHPUT_BPS = 102400;
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Planning size of one row for the deadline, above the widest row
|
|
152
|
+
* measured (291 bytes for 16 float columns). A row wider than this over
|
|
153
|
+
* a link at the planning speed needs an explicit `flushDeadlineMs`.
|
|
154
|
+
* @type {number}
|
|
155
|
+
*/
|
|
156
|
+
const ROW_BYTES_PLANNING = 512;
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* The longest pause the client takes between two attempts (4.2.0 fact:
|
|
160
|
+
* the backoff doubles from 10 ms and stops at 1000 ms).
|
|
161
|
+
* @type {number}
|
|
162
|
+
*/
|
|
163
|
+
const CLIENT_MAX_RETRY_BACKOFF_MS = 1000;
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Added on top of the client's own bound, so the client gets to reject
|
|
167
|
+
* first with a real error and the deadline stays a backstop for a hang
|
|
168
|
+
* the client cannot see.
|
|
169
|
+
* @type {number}
|
|
170
|
+
*/
|
|
171
|
+
const FLUSH_DEADLINE_MARGIN_MS = 5000;
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* The release that removes the five legacy keys.
|
|
175
|
+
* @type {string}
|
|
176
|
+
*/
|
|
177
|
+
const REMOVAL_RELEASE = '0.8.0';
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* The five legacy keys in report order. `envField` is the property on
|
|
181
|
+
* the `ENV_VARS` object, `envVar` the variable name an operator sees.
|
|
182
|
+
* `target` names the new key a legacy value maps to; a key without one
|
|
183
|
+
* is accepted and ignored.
|
|
184
|
+
* @type {Array<{key: string, envField: string, envVar: string, target?: string}>}
|
|
185
|
+
*/
|
|
186
|
+
const LEGACY_KEYS = [
|
|
187
|
+
{ key: 'flushMode', envField: 'questdbFlushMode', envVar: 'QUESTDB_FLUSH_MODE' },
|
|
188
|
+
{ key: 'idleFlushAfterMs', envField: 'questdbIdleFlushAfterMs', envVar: 'QUESTDB_IDLE_FLUSH_AFTER_MS' },
|
|
189
|
+
{
|
|
190
|
+
key: 'idleFlushCheckMs',
|
|
191
|
+
envField: 'questdbIdleFlushCheckMs',
|
|
192
|
+
envVar: 'QUESTDB_IDLE_FLUSH_CHECK_MS',
|
|
193
|
+
target: 'flushIntervalMs'
|
|
194
|
+
},
|
|
195
|
+
{ key: 'autoFlushRows', envField: 'questdbAutoFlushRows', envVar: 'QUESTDB_AUTO_FLUSH_ROWS', target: 'flushRows' },
|
|
196
|
+
{ key: 'autoFlushIntervalMs', envField: 'questdbAutoFlushIntervalMs', envVar: 'QUESTDB_AUTO_FLUSH_INTERVAL_MS' }
|
|
197
|
+
];
|
|
198
|
+
|
|
199
|
+
// ============================================================================
|
|
200
|
+
// RESOLUTION
|
|
201
|
+
// ============================================================================
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Resolves one setting that has a legacy alias, walking the five layers
|
|
205
|
+
* in precedence order. Returns the value and which layer supplied it,
|
|
206
|
+
* so the deprecation report can say whether a legacy value took effect.
|
|
207
|
+
*
|
|
208
|
+
* @param {Object} options - The raw storage options
|
|
209
|
+
* @param {Object} envVars - The `ENV_VARS` object
|
|
210
|
+
* @param {Object} spec - `{ newKey, legacyKey, newEnvField, legacyEnvField, fallback }`
|
|
211
|
+
* @returns {{value: *, winner: string}} The value and its layer
|
|
212
|
+
*/
|
|
213
|
+
const resolveWithAlias = function ( options, envVars, spec ) {
|
|
214
|
+
if ( options[ spec.newKey ] !== undefined ) {
|
|
215
|
+
return { value: options[ spec.newKey ], winner: 'config' };
|
|
216
|
+
}
|
|
217
|
+
if ( options[ spec.legacyKey ] !== undefined ) {
|
|
218
|
+
return { value: options[ spec.legacyKey ], winner: 'legacyConfig' };
|
|
219
|
+
}
|
|
220
|
+
if ( envVars[ spec.newEnvField ] !== undefined ) {
|
|
221
|
+
return { value: envVars[ spec.newEnvField ], winner: 'env' };
|
|
222
|
+
}
|
|
223
|
+
if ( envVars[ spec.legacyEnvField ] !== undefined ) {
|
|
224
|
+
return { value: envVars[ spec.legacyEnvField ], winner: 'legacyEnv' };
|
|
225
|
+
}
|
|
226
|
+
return { value: spec.fallback, winner: 'default' };
|
|
227
|
+
}; // resolveWithAlias()
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Describes one supplied legacy key for the report. A key with a target
|
|
231
|
+
* is `mapped` when its layer is the one that won that target, and
|
|
232
|
+
* `ignored` otherwise. A key without a target is always `ignored`.
|
|
233
|
+
*
|
|
234
|
+
* @param {string} name - The config key or the environment variable name
|
|
235
|
+
* @param {Object} legacy - The `LEGACY_KEYS` entry
|
|
236
|
+
* @param {Object} winners - Winning layer per target setting
|
|
237
|
+
* @param {string} layer - `'legacyConfig'` or `'legacyEnv'`
|
|
238
|
+
* @returns {{name: string, effect: string, target?: string}} One report entry
|
|
239
|
+
*/
|
|
240
|
+
const describeLegacy = function ( name, legacy, winners, layer ) {
|
|
241
|
+
if ( ( legacy.target !== undefined ) && ( winners[ legacy.target ] === layer ) ) {
|
|
242
|
+
return { name, effect: 'mapped', target: legacy.target };
|
|
243
|
+
}
|
|
244
|
+
return { name, effect: 'ignored' };
|
|
245
|
+
}; // describeLegacy()
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Builds the deprecation report: every supplied legacy config key in
|
|
249
|
+
* order, then every supplied legacy environment variable in the same
|
|
250
|
+
* order.
|
|
251
|
+
*
|
|
252
|
+
* @param {Object} options - The raw storage options
|
|
253
|
+
* @param {Object} envVars - The `ENV_VARS` object
|
|
254
|
+
* @param {Object} winners - Winning layer per target setting
|
|
255
|
+
* @returns {Array<Object>} The report entries, possibly empty
|
|
256
|
+
*/
|
|
257
|
+
const collectDeprecations = function ( options, envVars, winners ) {
|
|
258
|
+
const report = [];
|
|
259
|
+
for ( const legacy of LEGACY_KEYS ) {
|
|
260
|
+
if ( options[ legacy.key ] !== undefined ) {
|
|
261
|
+
report.push( describeLegacy( legacy.key, legacy, winners, 'legacyConfig' ) );
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
for ( const legacy of LEGACY_KEYS ) {
|
|
265
|
+
if ( envVars[ legacy.envField ] !== undefined ) {
|
|
266
|
+
report.push( describeLegacy( legacy.envVar, legacy, winners, 'legacyEnv' ) );
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return report;
|
|
270
|
+
}; // collectDeprecations()
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Maps the words of `QUESTDB_STDLIB_HTTP` to a boolean. The words are
|
|
274
|
+
* the client's own for `stdlib_http`, `on` and `off`, so an operator
|
|
275
|
+
* reads one vocabulary in the variable and in the config string.
|
|
276
|
+
* `env-vars.js` has already refused any other word.
|
|
277
|
+
*
|
|
278
|
+
* @param {string|undefined} word - `'on'`, `'off'`, or undefined when unset
|
|
279
|
+
* @returns {boolean|undefined} The boolean, or undefined when unset
|
|
280
|
+
*/
|
|
281
|
+
const onOffToBoolean = function ( word ) {
|
|
282
|
+
if ( word === undefined ) {
|
|
283
|
+
return undefined;
|
|
284
|
+
}
|
|
285
|
+
return word === 'on';
|
|
286
|
+
}; // onOffToBoolean()
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Resolves the storage options into the adapter's settings and the
|
|
290
|
+
* deprecation report. `settings.flushDeadlineMs` is undefined unless the
|
|
291
|
+
* operator set it; the engine then derives each flush's deadline with
|
|
292
|
+
* `flushDeadlineFor`.
|
|
293
|
+
*
|
|
294
|
+
* @param {Object} options - The raw storage options from `.storage()` config
|
|
295
|
+
* @param {Object} envVars - The `ENV_VARS` object (injected, for purity)
|
|
296
|
+
* @returns {{settings: Object, deprecations: Array<Object>}} Settings and report
|
|
297
|
+
* @throws {Error} INVALID_CONFIG when the ceiling is below twice the threshold
|
|
298
|
+
*/
|
|
299
|
+
const resolveOptions = function ( options, envVars ) {
|
|
300
|
+
const rows = resolveWithAlias( options, envVars, {
|
|
301
|
+
newKey: 'flushRows',
|
|
302
|
+
legacyKey: 'autoFlushRows',
|
|
303
|
+
newEnvField: 'questdbFlushRows',
|
|
304
|
+
legacyEnvField: 'questdbAutoFlushRows',
|
|
305
|
+
fallback: DEFAULT_FLUSH_ROWS
|
|
306
|
+
} );
|
|
307
|
+
const interval = resolveWithAlias( options, envVars, {
|
|
308
|
+
newKey: 'flushIntervalMs',
|
|
309
|
+
legacyKey: 'idleFlushCheckMs',
|
|
310
|
+
newEnvField: 'questdbFlushIntervalMs',
|
|
311
|
+
legacyEnvField: 'questdbIdleFlushCheckMs',
|
|
312
|
+
fallback: DEFAULT_FLUSH_INTERVAL_MS
|
|
313
|
+
} );
|
|
314
|
+
|
|
315
|
+
const bufferCeilingRows = options.bufferCeilingRows ??
|
|
316
|
+
envVars.questdbBufferCeilingRows ??
|
|
317
|
+
( rows.value * DEFAULT_CEILING_MULTIPLIER );
|
|
318
|
+
|
|
319
|
+
// Rows in flight count against the ceiling. A ceiling equal to the
|
|
320
|
+
// threshold would refuse every row that arrives during a
|
|
321
|
+
// row-triggered flush, because the batch in flight fills it. Twice
|
|
322
|
+
// the threshold is the least that holds one batch in flight and
|
|
323
|
+
// one batch buffering.
|
|
324
|
+
if ( bufferCeilingRows < ( 2 * rows.value ) ) {
|
|
325
|
+
const err = new Error(
|
|
326
|
+
`winkComposer/questdb: bufferCeilingRows ${bufferCeilingRows} is below twice flushRows ${rows.value} [INVALID_CONFIG]: ` +
|
|
327
|
+
'the ceiling must hold one batch in flight and one batch buffering; raise bufferCeilingRows or lower flushRows'
|
|
328
|
+
);
|
|
329
|
+
err.code = 'INVALID_CONFIG';
|
|
330
|
+
throw err;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
const settings = {
|
|
334
|
+
ilpUrl: options.ilpUrl ?? envVars.questdbIlpUrl,
|
|
335
|
+
pgUrl: options.pgUrl ?? envVars.questdbPgUrl,
|
|
336
|
+
flushRows: rows.value,
|
|
337
|
+
flushIntervalMs: interval.value,
|
|
338
|
+
bufferCeilingRows,
|
|
339
|
+
flushDeadlineMs: options.flushDeadlineMs ?? envVars.questdbFlushDeadlineMs,
|
|
340
|
+
stdlibHttp: options.stdlibHttp ?? onOffToBoolean( envVars.questdbStdlibHttp ) ?? DEFAULT_STDLIB_HTTP,
|
|
341
|
+
requestTimeout: options.requestTimeout ?? envVars.questdbRequestTimeout,
|
|
342
|
+
retryTimeout: options.retryTimeout ?? envVars.questdbRetryTimeout,
|
|
343
|
+
initBufSize: options.initBufSize ?? envVars.questdbInitBufSize,
|
|
344
|
+
maxBufSize: options.maxBufSize ?? envVars.questdbMaxBufSize,
|
|
345
|
+
partitionBy: options.partitionBy ?? 'DAY',
|
|
346
|
+
onWarning: options.onWarning,
|
|
347
|
+
onDeliveryFailure: options.onDeliveryFailure
|
|
348
|
+
};
|
|
349
|
+
|
|
350
|
+
const winners = { flushRows: rows.winner, flushIntervalMs: interval.winner };
|
|
351
|
+
return { settings, deprecations: collectDeprecations( options, envVars, winners ) };
|
|
352
|
+
}; // resolveOptions()
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* The deadline of one flush, in milliseconds. A fixed `flushDeadlineMs`
|
|
356
|
+
* wins. Otherwise it is the client's worst case for a batch of this
|
|
357
|
+
* many rows plus a margin. One attempt lasts the request timeout plus
|
|
358
|
+
* the transfer time at the planning speed and row size (see the
|
|
359
|
+
* header). The client checks its retry window only when an attempt
|
|
360
|
+
* ends, and it starts that clock when the first attempt ends. So the
|
|
361
|
+
* last attempt can begin just inside the window and run in full. The
|
|
362
|
+
* worst case is two attempts around the window, plus the longest
|
|
363
|
+
* backoff between attempts. A configured `retryTimeout` or
|
|
364
|
+
* `requestTimeout` replaces the client default in that sum. One
|
|
365
|
+
* multiply per flush, nothing per row.
|
|
366
|
+
*
|
|
367
|
+
* @param {number} rows - Rows the flush carries
|
|
368
|
+
* @param {Object} settings - The resolved settings
|
|
369
|
+
* @returns {number} The deadline in milliseconds
|
|
370
|
+
*/
|
|
371
|
+
const flushDeadlineFor = function ( rows, settings ) {
|
|
372
|
+
if ( settings.flushDeadlineMs !== undefined ) {
|
|
373
|
+
return settings.flushDeadlineMs;
|
|
374
|
+
}
|
|
375
|
+
const retryTimeout = settings.retryTimeout ?? CLIENT_DEFAULT_RETRY_TIMEOUT_MS;
|
|
376
|
+
const requestTimeout = settings.requestTimeout ?? CLIENT_DEFAULT_REQUEST_TIMEOUT_MS;
|
|
377
|
+
const transferMs = Math.ceil( ( rows * ROW_BYTES_PLANNING * 1000 ) / CLIENT_DEFAULT_MIN_THROUGHPUT_BPS );
|
|
378
|
+
const attemptMs = requestTimeout + transferMs;
|
|
379
|
+
return retryTimeout + ( 2 * attemptMs ) + CLIENT_MAX_RETRY_BACKOFF_MS + FLUSH_DEADLINE_MARGIN_MS;
|
|
380
|
+
}; // flushDeadlineFor()
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Formats the deprecation report as the one console line the adapter
|
|
384
|
+
* prints at setup, in the ADR-028 message grammar.
|
|
385
|
+
*
|
|
386
|
+
* @param {Array<Object>} deprecations - A non-empty report from `resolveOptions`
|
|
387
|
+
* @returns {string} The console line
|
|
388
|
+
*/
|
|
389
|
+
const deprecationMessage = function ( deprecations ) {
|
|
390
|
+
const parts = deprecations.map( function ( entry ) {
|
|
391
|
+
return ( entry.effect === 'mapped' ) ?
|
|
392
|
+
`${entry.name} maps to ${entry.target}` :
|
|
393
|
+
`${entry.name} is ignored`;
|
|
394
|
+
} );
|
|
395
|
+
return 'winkComposer/questdb: deprecated storage options in use [DEPRECATED_OPTION]: ' +
|
|
396
|
+
`${parts.join( '; ' )}; all five deprecated keys are removed in ${REMOVAL_RELEASE}`;
|
|
397
|
+
}; // deprecationMessage()
|
|
398
|
+
|
|
399
|
+
// ============================================================================
|
|
400
|
+
// EXPORTS
|
|
401
|
+
// ============================================================================
|
|
402
|
+
|
|
403
|
+
export { resolveOptions, flushDeadlineFor, deprecationMessage, CLIENT_MAX_RETRY_BACKOFF_MS };
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// core/storage-manager/questdb/shutdown-drain.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview The drain-then-close shutdown of the QuestDB adapter
|
|
5
|
+
* (ADR-018 §7, ADR-029).
|
|
6
|
+
*
|
|
7
|
+
* A clean shutdown resolve is a delivery statement: everything
|
|
8
|
+
* buffered or in flight was delivered. The drain therefore settles
|
|
9
|
+
* every flush still in the ledger (engine, recovery, explicit) plus one
|
|
10
|
+
* final flush for whatever is still buffered. It never fires a blind
|
|
11
|
+
* flush at a buffer an earlier copy-out emptied. That blind flush is
|
|
12
|
+
* what once let shutdown report clean over a hung flush.
|
|
13
|
+
*
|
|
14
|
+
* The caller's `{ timeout }` bounds the whole drain. Without one, the
|
|
15
|
+
* deadline on each flush (`flush-tracker.js`) bounds it instead, so a
|
|
16
|
+
* shutdown with no budget still ends. On loss the drain rejects with a
|
|
17
|
+
* classified error carrying `dropped: { count }`, exact:
|
|
18
|
+
* - `DELIVERY_FAILED` when any awaited flush failed or was abandoned.
|
|
19
|
+
* The first error is on `cause`; the count is the rows on the
|
|
20
|
+
* flushes that failed.
|
|
21
|
+
* - `SHUTDOWN_TIMEOUT` when the drain did not settle within the budget.
|
|
22
|
+
* The count is the rows not confirmed delivered.
|
|
23
|
+
*
|
|
24
|
+
* `dropped` is a statement about this session: those rows were not
|
|
25
|
+
* confirmed delivered before close. An abandoned flush may still land
|
|
26
|
+
* its rows if the server recovers, so the count is a floor on
|
|
27
|
+
* uncertainty, not a proof of loss.
|
|
28
|
+
*
|
|
29
|
+
* How the tally works. Each entry in the ledger has an `onDone`
|
|
30
|
+
* handler owned by whoever started the flush. The drain wraps that
|
|
31
|
+
* handler: the owner's handler still runs, and the drain adds the
|
|
32
|
+
* entry's rows to delivered or failed. The wrapped promises never
|
|
33
|
+
* reject, so the only rejection the race can surface is the timeout.
|
|
34
|
+
*
|
|
35
|
+
* The transport close runs on every path, clean or lossy. The client's
|
|
36
|
+
* own `close()` is an empty function on its HTTP transports (a 4.2.0
|
|
37
|
+
* fact). What ends a request still on the wire is the agent the
|
|
38
|
+
* factory owns: `closeTransport` closes the sender and then destroys
|
|
39
|
+
* that agent (ADR-029). In a flow, the shutdown manager's forced exit
|
|
40
|
+
* is the final backstop. A close failure on a loss path is logged,
|
|
41
|
+
* because the loss report matters more than the close.
|
|
42
|
+
*
|
|
43
|
+
* @see ADR-018
|
|
44
|
+
* @see ADR-029
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
import { logger } from '../../logger/index.js';
|
|
48
|
+
|
|
49
|
+
/** The final flush has no owner of its own: the drain tallies it. */
|
|
50
|
+
const NOOP = function () {
|
|
51
|
+
// Intentionally empty.
|
|
52
|
+
}; // NOOP()
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Races the drain against the caller's time budget. No budget (0 or
|
|
56
|
+
* undefined) means no race. The timer is `unref`'d and cleared however
|
|
57
|
+
* the race ends, so a finished drain leaves nothing behind. The losing
|
|
58
|
+
* side is left as it is: the drain never rejects, and a cleared timer
|
|
59
|
+
* never fires.
|
|
60
|
+
*
|
|
61
|
+
* @param {Promise} drain - The drain; it settles but never rejects
|
|
62
|
+
* @param {number} timeoutMs - Budget in ms; 0 or absent disables the race
|
|
63
|
+
* @returns {Promise} Settles with the drain, or rejects SHUTDOWN_TIMEOUT
|
|
64
|
+
*/
|
|
65
|
+
const raceTimeout = function ( drain, timeoutMs ) {
|
|
66
|
+
if ( !( timeoutMs > 0 ) ) {
|
|
67
|
+
return drain;
|
|
68
|
+
}
|
|
69
|
+
let timer = null;
|
|
70
|
+
const timeout = new Promise( function ( resolve, reject ) {
|
|
71
|
+
timer = setTimeout( function () {
|
|
72
|
+
const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
|
|
73
|
+
err.code = 'SHUTDOWN_TIMEOUT';
|
|
74
|
+
reject( err );
|
|
75
|
+
}, timeoutMs );
|
|
76
|
+
timer.unref();
|
|
77
|
+
} );
|
|
78
|
+
return Promise.race( [ drain, timeout ] ).finally( function () {
|
|
79
|
+
clearTimeout( timer );
|
|
80
|
+
} );
|
|
81
|
+
}; // raceTimeout()
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Builds the drain over the engine's sender and ledger.
|
|
85
|
+
*
|
|
86
|
+
* @param {Object} parts - What the drain runs on
|
|
87
|
+
* @param {Object} parts.sender - The connected ILP sender
|
|
88
|
+
* @param {{entries: Set}} parts.ledger - The tracker's ledger of flushes in flight
|
|
89
|
+
* @param {function} parts.track - The tracker's `track`, for the final flush
|
|
90
|
+
* @param {function} parts.takeBufferedRows - `() => number`: returns the buffered count and zeroes it
|
|
91
|
+
* @param {function} parts.closeTransport - `() => Promise<void>`: closes the sender, then destroys the factory's agent
|
|
92
|
+
* @returns {function} `drain( timeoutMs ) => Promise<void>`
|
|
93
|
+
*/
|
|
94
|
+
const createShutdownDrain = function ( { sender, ledger, track, takeBufferedRows, closeTransport } ) {
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Best-effort transport close on the lossy path.
|
|
98
|
+
*
|
|
99
|
+
* @returns {Promise<void>} Resolves whether or not the close worked
|
|
100
|
+
*/
|
|
101
|
+
const closeQuietly = function () {
|
|
102
|
+
return closeTransport().catch( function ( closeErr ) {
|
|
103
|
+
logger.error( `winkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
|
|
104
|
+
} );
|
|
105
|
+
}; // closeQuietly()
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Runs the drain once. The engine latches the returned promise so
|
|
109
|
+
* every caller receives this one outcome.
|
|
110
|
+
*
|
|
111
|
+
* @param {number} timeoutMs - The caller's budget; 0 means none
|
|
112
|
+
* @returns {Promise<void>} Resolves clean, or rejects classified
|
|
113
|
+
*/
|
|
114
|
+
const drain = async function ( timeoutMs ) {
|
|
115
|
+
let totalRows = 0;
|
|
116
|
+
let deliveredRows = 0;
|
|
117
|
+
let failedRows = 0;
|
|
118
|
+
let firstFailure = null;
|
|
119
|
+
const waits = [];
|
|
120
|
+
|
|
121
|
+
const awaitDelivery = function ( entry ) {
|
|
122
|
+
totalRows += entry.rows;
|
|
123
|
+
const ownerDone = entry.onDone;
|
|
124
|
+
waits.push( new Promise( function ( resolve ) {
|
|
125
|
+
entry.onDone = function ( err, abandoned ) {
|
|
126
|
+
ownerDone( err, abandoned );
|
|
127
|
+
if ( err ) {
|
|
128
|
+
failedRows += entry.rows;
|
|
129
|
+
if ( !firstFailure ) {
|
|
130
|
+
firstFailure = err;
|
|
131
|
+
}
|
|
132
|
+
} else {
|
|
133
|
+
deliveredRows += entry.rows;
|
|
134
|
+
}
|
|
135
|
+
resolve();
|
|
136
|
+
};
|
|
137
|
+
} ) );
|
|
138
|
+
}; // awaitDelivery()
|
|
139
|
+
|
|
140
|
+
// Everything delivery still owes: flushes already in flight
|
|
141
|
+
// (their rows left the buffer at their call) plus one final
|
|
142
|
+
// flush for whatever is still buffered.
|
|
143
|
+
ledger.entries.forEach( awaitDelivery );
|
|
144
|
+
const buffered = takeBufferedRows();
|
|
145
|
+
if ( buffered > 0 ) {
|
|
146
|
+
awaitDelivery( track( sender.flush(), buffered, NOOP ) );
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if ( waits.length > 0 ) {
|
|
150
|
+
try {
|
|
151
|
+
await raceTimeout( Promise.all( waits ), timeoutMs );
|
|
152
|
+
} catch ( err ) {
|
|
153
|
+
await closeQuietly();
|
|
154
|
+
const dropped = totalRows - deliveredRows;
|
|
155
|
+
const timedOut = new Error(
|
|
156
|
+
`winkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
|
|
157
|
+
);
|
|
158
|
+
timedOut.code = 'SHUTDOWN_TIMEOUT';
|
|
159
|
+
timedOut.dropped = { count: dropped };
|
|
160
|
+
throw timedOut;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
if ( failedRows > 0 ) {
|
|
164
|
+
await closeQuietly();
|
|
165
|
+
const failure = new Error(
|
|
166
|
+
`winkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
|
|
167
|
+
);
|
|
168
|
+
failure.code = 'DELIVERY_FAILED';
|
|
169
|
+
failure.dropped = { count: failedRows };
|
|
170
|
+
failure.cause = firstFailure;
|
|
171
|
+
throw failure;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
await closeTransport();
|
|
176
|
+
}; // drain()
|
|
177
|
+
|
|
178
|
+
return drain;
|
|
179
|
+
}; // createShutdownDrain()
|
|
180
|
+
|
|
181
|
+
export { createShutdownDrain };
|