@winkjs/composer 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/README.md +1 -1
  2. package/package.json +3 -3
  3. package/src/composer.js +1 -1
  4. package/src/core/batch/aoo-encoder.js +4 -4
  5. package/src/core/batch/sliding-window.js +2 -2
  6. package/src/core/codec/index.js +4 -4
  7. package/src/core/emitter-manager/mqtt/connection.js +209 -0
  8. package/src/core/emitter-manager/mqtt/constants.js +43 -29
  9. package/src/core/emitter-manager/mqtt/emitter.js +108 -633
  10. package/src/core/emitter-manager/mqtt/health.js +151 -0
  11. package/src/core/emitter-manager/mqtt/index.js +68 -4
  12. package/src/core/emitter-manager/mqtt/publish.js +292 -0
  13. package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
  14. package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
  15. package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
  16. package/src/core/emitter-manager/terminal/index.js +20 -5
  17. package/src/core/env-vars.js +143 -16
  18. package/src/core/logger/index.js +174 -0
  19. package/src/core/mqtt-protocol.js +2 -2
  20. package/src/core/partition-manager/get-stats.js +31 -0
  21. package/src/core/partition-manager/index.js +1 -0
  22. package/src/core/partition-manager/init.js +15 -0
  23. package/src/core/partition-manager/update.js +143 -87
  24. package/src/core/partition-manager/validate-flow.js +1 -1
  25. package/src/core/semantics/loader.js +1 -1
  26. package/src/core/semantics/warnings.js +5 -3
  27. package/src/core/shutdown-manager/index.js +71 -21
  28. package/src/core/source-manager/csv/start.js +109 -35
  29. package/src/core/source-manager/mqtt/client.js +276 -83
  30. package/src/core/source-manager/mqtt/dedup.js +19 -14
  31. package/src/core/source-manager/mqtt/index.js +33 -6
  32. package/src/core/source-manager/mqtt/status.js +211 -46
  33. package/src/core/source-manager/record-shape.js +55 -0
  34. package/src/core/source-manager/test-harness/field-generator.js +1 -1
  35. package/src/core/source-manager/test-harness/start.js +52 -10
  36. package/src/core/source-manager/test-harness/validate.js +1 -1
  37. package/src/core/storage-manager/questdb/address-policy.js +264 -0
  38. package/src/core/storage-manager/questdb/assert-columns.js +2 -2
  39. package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
  40. package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
  41. package/src/core/storage-manager/questdb/flush-engine.js +852 -0
  42. package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
  43. package/src/core/storage-manager/questdb/index.js +592 -827
  44. package/src/core/storage-manager/questdb/persist-plan.js +147 -162
  45. package/src/core/storage-manager/questdb/resolve-options.js +403 -0
  46. package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
  47. package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
  48. package/src/core/test-utils/tcp-proxy.js +194 -1
  49. package/src/core/tunable/helpers.js +1 -1
  50. package/src/core/utils/address/index.js +362 -0
  51. package/src/core/utils/address/probe.js +210 -0
  52. package/src/core/utils/callback-guard/index.js +176 -0
  53. package/src/core/utils/clock/index.js +34 -0
  54. package/src/core/utils/half-life/index.js +4 -4
  55. package/src/core/utils/jitter/index.js +51 -0
  56. package/src/core/utils/line-rate/index.js +104 -0
  57. package/src/core/utils/quantize/index.js +43 -19
  58. package/src/core/utils/validate/composer-validators.js +2 -2
  59. package/src/core/utils/validate/index.js +1 -1
  60. package/src/core/utils/validate/validate.js +1 -1
  61. package/src/core/wiring/assert-handle.js +5 -5
  62. package/src/core/wiring/assert-module.js +2 -2
  63. package/src/core/wiring/wire-emitters.js +7 -4
  64. package/src/core/wiring/wire-linear-graph.js +1 -1
  65. package/src/core/wiring/wire-node.js +6 -6
  66. package/src/core/wiring/wire-semantics.js +1 -1
  67. package/src/core/wiring/wire-storages.js +14 -5
  68. package/src/flow/driver.js +31 -6
  69. package/src/flow/expand-single-input-array.js +4 -4
  70. package/src/flow/flow-api-schemas.js +4 -4
  71. package/src/flow/flow.js +62 -41
  72. package/src/flow/for-each.js +12 -11
  73. package/src/flow/get-target-array.js +2 -2
  74. package/src/flow/load-node-module.js +1 -1
  75. package/src/flow/make-node-method.js +4 -4
  76. package/src/flow/resolve-pick-by-field.js +1 -1
  77. package/src/flow/run.js +200 -40
  78. package/src/flow/serialize.js +1 -1
  79. package/src/flow/spec-builder.js +1 -1
  80. package/src/flow/validate.js +1 -1
  81. package/src/nodes/butterworth-filter/init.js +5 -4
  82. package/src/nodes/categorize/update.js +3 -1
  83. package/src/nodes/controller/update.js +2 -1
  84. package/src/nodes/dwell-time-tracker/update.js +2 -1
  85. package/src/nodes/emit-if/helpers.js +57 -11
  86. package/src/nodes/emit-if/init.js +1 -1
  87. package/src/nodes/emit-if/update.js +8 -11
  88. package/src/nodes/page-hinkley/update.js +2 -1
  89. package/src/nodes/pass-if/update.js +3 -1
  90. package/src/nodes/persist-if/helpers.js +60 -4
  91. package/src/nodes/persist-if/init.js +1 -1
  92. package/src/nodes/persist-if/update.js +8 -16
  93. package/src/nodes/persistence-check/update.js +2 -1
  94. package/src/nodes/sanitize/update.js +4 -2
  95. package/src/nodes/swing-watch/update.js +3 -2
  96. package/src/nodes/threshold/update.js +2 -1
  97. package/src/nodes/trend/compute-confidence.js +4 -4
  98. package/src/nodes/trend/update.js +2 -1
  99. package/src/nodes/winnow/update.js +3 -1
  100. package/src/tools/training/features.js +13 -13
  101. package/src/tools/training/logistic.js +1 -1
  102. package/src/tools/training/metrics.js +2 -2
  103. package/src/tools/training/scale.js +2 -2
  104. package/src/tools/training/validate.js +7 -7
  105. package/src/core/utils/console-formatter/index.js +0 -207
  106. package/src/nodes/archive/csv/index.js +0 -6
  107. package/src/nodes/archive/csv/init.js +0 -36
  108. package/src/nodes/archive/csv/load.js +0 -70
  109. package/src/nodes/archive/csv/reset.js +0 -14
  110. package/src/nodes/archive/csv/simple-parser.js +0 -216
  111. package/src/nodes/archive/csv/test.js +0 -173
  112. package/src/nodes/archive/csv/update.js +0 -63
  113. package/src/nodes/sse-emitter/index.js +0 -7
  114. package/src/nodes/sse-emitter/init.js +0 -105
  115. package/src/nodes/sse-emitter/introspect.js +0 -134
  116. package/src/nodes/sse-emitter/publish-to.js +0 -6
  117. package/src/nodes/sse-emitter/recompute.js +0 -16
  118. package/src/nodes/sse-emitter/reset.js +0 -19
  119. package/src/nodes/sse-emitter/update.js +0 -69
  120. package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
  121. package/src/nodes/sse-emitter/utils/build-message.js +0 -31
  122. package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
  123. package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
  124. package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
  125. package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
@@ -0,0 +1,403 @@
1
+ // core/storage-manager/questdb/resolve-options.js
2
+
3
+ /**
4
+ * @fileoverview Turns the QuestDB storage options into the settings the
5
+ * adapter runs on.
6
+ *
7
+ * The adapter accepts options from two places: the `.storage()` config
8
+ * in the flow, and the `QUESTDB_*` environment variables read by
9
+ * `env-vars.js`. This module merges the two, fills in the defaults, and
10
+ * derives the buffer ceiling from the flush threshold. It also computes
11
+ * the deadline of one flush from the rows that flush carries. It is a
12
+ * pure function of its inputs, so it can be tested without a flow, a
13
+ * client, or a live environment. The adapter calls `resolveOptions`
14
+ * once, at setup, and `flushDeadlineFor` once per flush. Nothing here
15
+ * runs on the per-row path.
16
+ *
17
+ * Why the derived values live here and not in `env-vars.js`. The buffer
18
+ * ceiling defaults to ten times the flush threshold. A fixed ceiling in
19
+ * the environment would fail setup whenever a user raised the threshold
20
+ * in config without also raising the ceiling. So the environment carries
21
+ * a value only when the operator set one, and this module derives the
22
+ * rest from what actually won.
23
+ *
24
+ * Why the deadline depends on the row count. The client gives a bigger
25
+ * request more time: its request timeout is an inactivity timeout that
26
+ * grows with the bytes sent, at a planning speed of 100 KiB per second
27
+ * (`request_min_throughput`, a 4.2.0 fact). A fixed deadline would
28
+ * abandon a large but healthy send over a slow link and report rows
29
+ * lost that then land. So the deadline is the client's worst case for
30
+ * that batch, plus a margin. One attempt is the request timeout plus
31
+ * the transfer time at a planning size of 512 bytes a row. The client
32
+ * checks its retry window only when an attempt ends, so its last
33
+ * attempt can run in full past the window. The deadline is therefore
34
+ * the retry window, two attempts, the longest backoff, and 5 seconds.
35
+ * One row gets about 36 seconds; 5000 rows get 86 seconds; a full
36
+ * 50000-row catch-up send gets about 9 minutes. When the operator
37
+ * sets `flushDeadlineMs`, that fixed value is used for every flush.
38
+ *
39
+ * Precedence, highest first (ADR-018 §10, extended for the legacy
40
+ * aliases of ADR-029 item 10):
41
+ *
42
+ * 1. the new key in the config (`flushRows`)
43
+ * 2. the legacy key in the config (`autoFlushRows`)
44
+ * 3. the new environment variable (`QUESTDB_FLUSH_ROWS`)
45
+ * 4. the legacy environment variable (`QUESTDB_AUTO_FLUSH_ROWS`)
46
+ * 5. the default
47
+ *
48
+ * Two legacy keys map to a new one: `autoFlushRows` to `flushRows`, and
49
+ * `idleFlushCheckMs` to `flushIntervalMs`. The other three,
50
+ * `flushMode`, `idleFlushAfterMs` and `autoFlushIntervalMs`, have no
51
+ * meaning once composer owns every flush (ADR-029). They are accepted
52
+ * and ignored. Every supplied legacy key is named in the deprecation
53
+ * report, with what happened to it, so the adapter can print one
54
+ * `DEPRECATED_OPTION` line at setup. All five keys are removed in
55
+ * 0.8.0.
56
+ *
57
+ * The one relation enforced here: the ceiling is never below twice
58
+ * the threshold. Rows in flight count against the ceiling, so a
59
+ * ceiling equal to the threshold would refuse every row that arrives
60
+ * during a row-triggered flush. Twice the threshold holds one batch
61
+ * in flight and one batch buffering. A smaller ceiling fails setup
62
+ * with `INVALID_CONFIG`.
63
+ *
64
+ * The client constants used in the deadline are `@questdb/nodejs-client`
65
+ * 4.2.0 facts, cited by line in the client's `dist/es/index.mjs`. The
66
+ * defaults sit at lines 230 to 234: `request_min_throughput` 102400
67
+ * bytes a second, `request_timeout` 10 s, `retry_timeout` 10 s. The
68
+ * throughput option is documented at line 404 and read at line 275.
69
+ * The request timeout grows with the bytes sent at line 1142.
70
+ *
71
+ * The retry clock starts when the first attempt ends, at line 1183.
72
+ * The window is checked when a later attempt ends, at lines 1186 to
73
+ * 1188. The backoff between attempts doubles and stops at 1000 ms,
74
+ * with a jitter of up to 5 ms, at lines 1192 to 1194. Re-verify these
75
+ * lines on a client upgrade. A configured `requestTimeout` or
76
+ * `retryTimeout` replaces its default in the sum, so an operator who
77
+ * lengthens a timeout lengthens the deadline with it.
78
+ *
79
+ * The transport setting has two vocabularies. In config, `stdlibHttp`
80
+ * is a boolean. In the environment, `QUESTDB_STDLIB_HTTP` takes the
81
+ * client's own words `on` and `off`. This module maps the words to the
82
+ * boolean, and the default is the standard-library transport
83
+ * (`DEFAULT_STDLIB_HTTP` explains why).
84
+ *
85
+ * @see ADR-029
86
+ */
87
+
88
+ // ============================================================================
89
+ // DEFAULTS
90
+ // ============================================================================
91
+
92
+ /**
93
+ * Rows that trigger a flush from inside `write()`. At the measured 130
94
+ * to 290 bytes a row this is 0.65 to 1.5 MB per request, sized for an
95
+ * edge device and a slow plant link. Larger deployments raise it by
96
+ * config.
97
+ * @type {number}
98
+ */
99
+ const DEFAULT_FLUSH_ROWS = 5000;
100
+
101
+ /**
102
+ * Period of the flush timer. Rows land within about this long after
103
+ * they were written, busy stream or idle stream alike, and a power cut
104
+ * costs at most about this much data.
105
+ * @type {number}
106
+ */
107
+ const DEFAULT_FLUSH_INTERVAL_MS = 1000;
108
+
109
+ /**
110
+ * The ceiling is this many flush thresholds. Past it, `write()` sheds
111
+ * new rows with `STORAGE_FULL`. With flushes held while the endpoint is
112
+ * unreachable (ADR-029), the ceiling is the outage the adapter rides
113
+ * through without loss: ten batches hold 50 seconds at 1000 rows a
114
+ * second and hours at plant rate, in 6.5 to 15 MB of memory.
115
+ * @type {number}
116
+ */
117
+ const DEFAULT_CEILING_MULTIPLIER = 10;
118
+
119
+ /**
120
+ * The client's standard-library HTTP transport is selected unless the
121
+ * operator opts out. It is the transport whose requests end: a refused
122
+ * connection rejects at once, and a stalled request ends within the
123
+ * retry window plus one request timeout. The client's own default,
124
+ * undici, retries a refused connection without end, and its abort
125
+ * cannot end that retry (ADR-029).
126
+ * @type {boolean}
127
+ */
128
+ const DEFAULT_STDLIB_HTTP = true;
129
+
130
+ /**
131
+ * The client's default `request_timeout` (4.2.0 fact).
132
+ * @type {number}
133
+ */
134
+ const CLIENT_DEFAULT_REQUEST_TIMEOUT_MS = 10000;
135
+
136
+ /**
137
+ * The client's default `retry_timeout` (4.2.0 fact).
138
+ * @type {number}
139
+ */
140
+ const CLIENT_DEFAULT_RETRY_TIMEOUT_MS = 10000;
141
+
142
+ /**
143
+ * The client's default `request_min_throughput`, in bytes per second
144
+ * (4.2.0 fact). The client extends its request timeout by the time the
145
+ * request would take at this speed.
146
+ * @type {number}
147
+ */
148
+ const CLIENT_DEFAULT_MIN_THROUGHPUT_BPS = 102400;
149
+
150
+ /**
151
+ * Planning size of one row for the deadline, above the widest row
152
+ * measured (291 bytes for 16 float columns). A row wider than this over
153
+ * a link at the planning speed needs an explicit `flushDeadlineMs`.
154
+ * @type {number}
155
+ */
156
+ const ROW_BYTES_PLANNING = 512;
157
+
158
+ /**
159
+ * The longest pause the client takes between two attempts (4.2.0 fact:
160
+ * the backoff doubles from 10 ms and stops at 1000 ms).
161
+ * @type {number}
162
+ */
163
+ const CLIENT_MAX_RETRY_BACKOFF_MS = 1000;
164
+
165
+ /**
166
+ * Added on top of the client's own bound, so the client gets to reject
167
+ * first with a real error and the deadline stays a backstop for a hang
168
+ * the client cannot see.
169
+ * @type {number}
170
+ */
171
+ const FLUSH_DEADLINE_MARGIN_MS = 5000;
172
+
173
+ /**
174
+ * The release that removes the five legacy keys.
175
+ * @type {string}
176
+ */
177
+ const REMOVAL_RELEASE = '0.8.0';
178
+
179
+ /**
180
+ * The five legacy keys in report order. `envField` is the property on
181
+ * the `ENV_VARS` object, `envVar` the variable name an operator sees.
182
+ * `target` names the new key a legacy value maps to; a key without one
183
+ * is accepted and ignored.
184
+ * @type {Array<{key: string, envField: string, envVar: string, target?: string}>}
185
+ */
186
+ const LEGACY_KEYS = [
187
+ { key: 'flushMode', envField: 'questdbFlushMode', envVar: 'QUESTDB_FLUSH_MODE' },
188
+ { key: 'idleFlushAfterMs', envField: 'questdbIdleFlushAfterMs', envVar: 'QUESTDB_IDLE_FLUSH_AFTER_MS' },
189
+ {
190
+ key: 'idleFlushCheckMs',
191
+ envField: 'questdbIdleFlushCheckMs',
192
+ envVar: 'QUESTDB_IDLE_FLUSH_CHECK_MS',
193
+ target: 'flushIntervalMs'
194
+ },
195
+ { key: 'autoFlushRows', envField: 'questdbAutoFlushRows', envVar: 'QUESTDB_AUTO_FLUSH_ROWS', target: 'flushRows' },
196
+ { key: 'autoFlushIntervalMs', envField: 'questdbAutoFlushIntervalMs', envVar: 'QUESTDB_AUTO_FLUSH_INTERVAL_MS' }
197
+ ];
198
+
199
+ // ============================================================================
200
+ // RESOLUTION
201
+ // ============================================================================
202
+
203
+ /**
204
+ * Resolves one setting that has a legacy alias, walking the five layers
205
+ * in precedence order. Returns the value and which layer supplied it,
206
+ * so the deprecation report can say whether a legacy value took effect.
207
+ *
208
+ * @param {Object} options - The raw storage options
209
+ * @param {Object} envVars - The `ENV_VARS` object
210
+ * @param {Object} spec - `{ newKey, legacyKey, newEnvField, legacyEnvField, fallback }`
211
+ * @returns {{value: *, winner: string}} The value and its layer
212
+ */
213
+ const resolveWithAlias = function ( options, envVars, spec ) {
214
+ if ( options[ spec.newKey ] !== undefined ) {
215
+ return { value: options[ spec.newKey ], winner: 'config' };
216
+ }
217
+ if ( options[ spec.legacyKey ] !== undefined ) {
218
+ return { value: options[ spec.legacyKey ], winner: 'legacyConfig' };
219
+ }
220
+ if ( envVars[ spec.newEnvField ] !== undefined ) {
221
+ return { value: envVars[ spec.newEnvField ], winner: 'env' };
222
+ }
223
+ if ( envVars[ spec.legacyEnvField ] !== undefined ) {
224
+ return { value: envVars[ spec.legacyEnvField ], winner: 'legacyEnv' };
225
+ }
226
+ return { value: spec.fallback, winner: 'default' };
227
+ }; // resolveWithAlias()
228
+
229
+ /**
230
+ * Describes one supplied legacy key for the report. A key with a target
231
+ * is `mapped` when its layer is the one that won that target, and
232
+ * `ignored` otherwise. A key without a target is always `ignored`.
233
+ *
234
+ * @param {string} name - The config key or the environment variable name
235
+ * @param {Object} legacy - The `LEGACY_KEYS` entry
236
+ * @param {Object} winners - Winning layer per target setting
237
+ * @param {string} layer - `'legacyConfig'` or `'legacyEnv'`
238
+ * @returns {{name: string, effect: string, target?: string}} One report entry
239
+ */
240
+ const describeLegacy = function ( name, legacy, winners, layer ) {
241
+ if ( ( legacy.target !== undefined ) && ( winners[ legacy.target ] === layer ) ) {
242
+ return { name, effect: 'mapped', target: legacy.target };
243
+ }
244
+ return { name, effect: 'ignored' };
245
+ }; // describeLegacy()
246
+
247
+ /**
248
+ * Builds the deprecation report: every supplied legacy config key in
249
+ * order, then every supplied legacy environment variable in the same
250
+ * order.
251
+ *
252
+ * @param {Object} options - The raw storage options
253
+ * @param {Object} envVars - The `ENV_VARS` object
254
+ * @param {Object} winners - Winning layer per target setting
255
+ * @returns {Array<Object>} The report entries, possibly empty
256
+ */
257
+ const collectDeprecations = function ( options, envVars, winners ) {
258
+ const report = [];
259
+ for ( const legacy of LEGACY_KEYS ) {
260
+ if ( options[ legacy.key ] !== undefined ) {
261
+ report.push( describeLegacy( legacy.key, legacy, winners, 'legacyConfig' ) );
262
+ }
263
+ }
264
+ for ( const legacy of LEGACY_KEYS ) {
265
+ if ( envVars[ legacy.envField ] !== undefined ) {
266
+ report.push( describeLegacy( legacy.envVar, legacy, winners, 'legacyEnv' ) );
267
+ }
268
+ }
269
+ return report;
270
+ }; // collectDeprecations()
271
+
272
+ /**
273
+ * Maps the words of `QUESTDB_STDLIB_HTTP` to a boolean. The words are
274
+ * the client's own for `stdlib_http`, `on` and `off`, so an operator
275
+ * reads one vocabulary in the variable and in the config string.
276
+ * `env-vars.js` has already refused any other word.
277
+ *
278
+ * @param {string|undefined} word - `'on'`, `'off'`, or undefined when unset
279
+ * @returns {boolean|undefined} The boolean, or undefined when unset
280
+ */
281
+ const onOffToBoolean = function ( word ) {
282
+ if ( word === undefined ) {
283
+ return undefined;
284
+ }
285
+ return word === 'on';
286
+ }; // onOffToBoolean()
287
+
288
+ /**
289
+ * Resolves the storage options into the adapter's settings and the
290
+ * deprecation report. `settings.flushDeadlineMs` is undefined unless the
291
+ * operator set it; the engine then derives each flush's deadline with
292
+ * `flushDeadlineFor`.
293
+ *
294
+ * @param {Object} options - The raw storage options from `.storage()` config
295
+ * @param {Object} envVars - The `ENV_VARS` object (injected, for purity)
296
+ * @returns {{settings: Object, deprecations: Array<Object>}} Settings and report
297
+ * @throws {Error} INVALID_CONFIG when the ceiling is below twice the threshold
298
+ */
299
+ const resolveOptions = function ( options, envVars ) {
300
+ const rows = resolveWithAlias( options, envVars, {
301
+ newKey: 'flushRows',
302
+ legacyKey: 'autoFlushRows',
303
+ newEnvField: 'questdbFlushRows',
304
+ legacyEnvField: 'questdbAutoFlushRows',
305
+ fallback: DEFAULT_FLUSH_ROWS
306
+ } );
307
+ const interval = resolveWithAlias( options, envVars, {
308
+ newKey: 'flushIntervalMs',
309
+ legacyKey: 'idleFlushCheckMs',
310
+ newEnvField: 'questdbFlushIntervalMs',
311
+ legacyEnvField: 'questdbIdleFlushCheckMs',
312
+ fallback: DEFAULT_FLUSH_INTERVAL_MS
313
+ } );
314
+
315
+ const bufferCeilingRows = options.bufferCeilingRows ??
316
+ envVars.questdbBufferCeilingRows ??
317
+ ( rows.value * DEFAULT_CEILING_MULTIPLIER );
318
+
319
+ // Rows in flight count against the ceiling. A ceiling equal to the
320
+ // threshold would refuse every row that arrives during a
321
+ // row-triggered flush, because the batch in flight fills it. Twice
322
+ // the threshold is the least that holds one batch in flight and
323
+ // one batch buffering.
324
+ if ( bufferCeilingRows < ( 2 * rows.value ) ) {
325
+ const err = new Error(
326
+ `winkComposer/questdb: bufferCeilingRows ${bufferCeilingRows} is below twice flushRows ${rows.value} [INVALID_CONFIG]: ` +
327
+ 'the ceiling must hold one batch in flight and one batch buffering; raise bufferCeilingRows or lower flushRows'
328
+ );
329
+ err.code = 'INVALID_CONFIG';
330
+ throw err;
331
+ }
332
+
333
+ const settings = {
334
+ ilpUrl: options.ilpUrl ?? envVars.questdbIlpUrl,
335
+ pgUrl: options.pgUrl ?? envVars.questdbPgUrl,
336
+ flushRows: rows.value,
337
+ flushIntervalMs: interval.value,
338
+ bufferCeilingRows,
339
+ flushDeadlineMs: options.flushDeadlineMs ?? envVars.questdbFlushDeadlineMs,
340
+ stdlibHttp: options.stdlibHttp ?? onOffToBoolean( envVars.questdbStdlibHttp ) ?? DEFAULT_STDLIB_HTTP,
341
+ requestTimeout: options.requestTimeout ?? envVars.questdbRequestTimeout,
342
+ retryTimeout: options.retryTimeout ?? envVars.questdbRetryTimeout,
343
+ initBufSize: options.initBufSize ?? envVars.questdbInitBufSize,
344
+ maxBufSize: options.maxBufSize ?? envVars.questdbMaxBufSize,
345
+ partitionBy: options.partitionBy ?? 'DAY',
346
+ onWarning: options.onWarning,
347
+ onDeliveryFailure: options.onDeliveryFailure
348
+ };
349
+
350
+ const winners = { flushRows: rows.winner, flushIntervalMs: interval.winner };
351
+ return { settings, deprecations: collectDeprecations( options, envVars, winners ) };
352
+ }; // resolveOptions()
353
+
354
+ /**
355
+ * The deadline of one flush, in milliseconds. A fixed `flushDeadlineMs`
356
+ * wins. Otherwise it is the client's worst case for a batch of this
357
+ * many rows plus a margin. One attempt lasts the request timeout plus
358
+ * the transfer time at the planning speed and row size (see the
359
+ * header). The client checks its retry window only when an attempt
360
+ * ends, and it starts that clock when the first attempt ends. So the
361
+ * last attempt can begin just inside the window and run in full. The
362
+ * worst case is two attempts around the window, plus the longest
363
+ * backoff between attempts. A configured `retryTimeout` or
364
+ * `requestTimeout` replaces the client default in that sum. One
365
+ * multiply per flush, nothing per row.
366
+ *
367
+ * @param {number} rows - Rows the flush carries
368
+ * @param {Object} settings - The resolved settings
369
+ * @returns {number} The deadline in milliseconds
370
+ */
371
+ const flushDeadlineFor = function ( rows, settings ) {
372
+ if ( settings.flushDeadlineMs !== undefined ) {
373
+ return settings.flushDeadlineMs;
374
+ }
375
+ const retryTimeout = settings.retryTimeout ?? CLIENT_DEFAULT_RETRY_TIMEOUT_MS;
376
+ const requestTimeout = settings.requestTimeout ?? CLIENT_DEFAULT_REQUEST_TIMEOUT_MS;
377
+ const transferMs = Math.ceil( ( rows * ROW_BYTES_PLANNING * 1000 ) / CLIENT_DEFAULT_MIN_THROUGHPUT_BPS );
378
+ const attemptMs = requestTimeout + transferMs;
379
+ return retryTimeout + ( 2 * attemptMs ) + CLIENT_MAX_RETRY_BACKOFF_MS + FLUSH_DEADLINE_MARGIN_MS;
380
+ }; // flushDeadlineFor()
381
+
382
+ /**
383
+ * Formats the deprecation report as the one console line the adapter
384
+ * prints at setup, in the ADR-028 message grammar.
385
+ *
386
+ * @param {Array<Object>} deprecations - A non-empty report from `resolveOptions`
387
+ * @returns {string} The console line
388
+ */
389
+ const deprecationMessage = function ( deprecations ) {
390
+ const parts = deprecations.map( function ( entry ) {
391
+ return ( entry.effect === 'mapped' ) ?
392
+ `${entry.name} maps to ${entry.target}` :
393
+ `${entry.name} is ignored`;
394
+ } );
395
+ return 'winkComposer/questdb: deprecated storage options in use [DEPRECATED_OPTION]: ' +
396
+ `${parts.join( '; ' )}; all five deprecated keys are removed in ${REMOVAL_RELEASE}`;
397
+ }; // deprecationMessage()
398
+
399
+ // ============================================================================
400
+ // EXPORTS
401
+ // ============================================================================
402
+
403
+ export { resolveOptions, flushDeadlineFor, deprecationMessage, CLIENT_MAX_RETRY_BACKOFF_MS };
@@ -0,0 +1,181 @@
1
+ // core/storage-manager/questdb/shutdown-drain.js
2
+
3
+ /**
4
+ * @fileoverview The drain-then-close shutdown of the QuestDB adapter
5
+ * (ADR-018 §7, ADR-029).
6
+ *
7
+ * A clean shutdown resolve is a delivery statement: everything
8
+ * buffered or in flight was delivered. The drain therefore settles
9
+ * every flush still in the ledger (engine, recovery, explicit) plus one
10
+ * final flush for whatever is still buffered. It never fires a blind
11
+ * flush at a buffer an earlier copy-out emptied. That blind flush is
12
+ * what once let shutdown report clean over a hung flush.
13
+ *
14
+ * The caller's `{ timeout }` bounds the whole drain. Without one, the
15
+ * deadline on each flush (`flush-tracker.js`) bounds it instead, so a
16
+ * shutdown with no budget still ends. On loss the drain rejects with a
17
+ * classified error carrying `dropped: { count }`, exact:
18
+ * - `DELIVERY_FAILED` when any awaited flush failed or was abandoned.
19
+ * The first error is on `cause`; the count is the rows on the
20
+ * flushes that failed.
21
+ * - `SHUTDOWN_TIMEOUT` when the drain did not settle within the budget.
22
+ * The count is the rows not confirmed delivered.
23
+ *
24
+ * `dropped` is a statement about this session: those rows were not
25
+ * confirmed delivered before close. An abandoned flush may still land
26
+ * its rows if the server recovers, so the count is a floor on
27
+ * uncertainty, not a proof of loss.
28
+ *
29
+ * How the tally works. Each entry in the ledger has an `onDone`
30
+ * handler owned by whoever started the flush. The drain wraps that
31
+ * handler: the owner's handler still runs, and the drain adds the
32
+ * entry's rows to delivered or failed. The wrapped promises never
33
+ * reject, so the only rejection the race can surface is the timeout.
34
+ *
35
+ * The transport close runs on every path, clean or lossy. The client's
36
+ * own `close()` is an empty function on its HTTP transports (a 4.2.0
37
+ * fact). What ends a request still on the wire is the agent the
38
+ * factory owns: `closeTransport` closes the sender and then destroys
39
+ * that agent (ADR-029). In a flow, the shutdown manager's forced exit
40
+ * is the final backstop. A close failure on a loss path is logged,
41
+ * because the loss report matters more than the close.
42
+ *
43
+ * @see ADR-018
44
+ * @see ADR-029
45
+ */
46
+
47
+ import { logger } from '../../logger/index.js';
48
+
49
+ /** The final flush has no owner of its own: the drain tallies it. */
50
+ const NOOP = function () {
51
+ // Intentionally empty.
52
+ }; // NOOP()
53
+
54
+ /**
55
+ * Races the drain against the caller's time budget. No budget (0 or
56
+ * undefined) means no race. The timer is `unref`'d and cleared however
57
+ * the race ends, so a finished drain leaves nothing behind. The losing
58
+ * side is left as it is: the drain never rejects, and a cleared timer
59
+ * never fires.
60
+ *
61
+ * @param {Promise} drain - The drain; it settles but never rejects
62
+ * @param {number} timeoutMs - Budget in ms; 0 or absent disables the race
63
+ * @returns {Promise} Settles with the drain, or rejects SHUTDOWN_TIMEOUT
64
+ */
65
+ const raceTimeout = function ( drain, timeoutMs ) {
66
+ if ( !( timeoutMs > 0 ) ) {
67
+ return drain;
68
+ }
69
+ let timer = null;
70
+ const timeout = new Promise( function ( resolve, reject ) {
71
+ timer = setTimeout( function () {
72
+ const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
73
+ err.code = 'SHUTDOWN_TIMEOUT';
74
+ reject( err );
75
+ }, timeoutMs );
76
+ timer.unref();
77
+ } );
78
+ return Promise.race( [ drain, timeout ] ).finally( function () {
79
+ clearTimeout( timer );
80
+ } );
81
+ }; // raceTimeout()
82
+
83
+ /**
84
+ * Builds the drain over the engine's sender and ledger.
85
+ *
86
+ * @param {Object} parts - What the drain runs on
87
+ * @param {Object} parts.sender - The connected ILP sender
88
+ * @param {{entries: Set}} parts.ledger - The tracker's ledger of flushes in flight
89
+ * @param {function} parts.track - The tracker's `track`, for the final flush
90
+ * @param {function} parts.takeBufferedRows - `() => number`: returns the buffered count and zeroes it
91
+ * @param {function} parts.closeTransport - `() => Promise<void>`: closes the sender, then destroys the factory's agent
92
+ * @returns {function} `drain( timeoutMs ) => Promise<void>`
93
+ */
94
+ const createShutdownDrain = function ( { sender, ledger, track, takeBufferedRows, closeTransport } ) {
95
+
96
+ /**
97
+ * Best-effort transport close on the lossy path.
98
+ *
99
+ * @returns {Promise<void>} Resolves whether or not the close worked
100
+ */
101
+ const closeQuietly = function () {
102
+ return closeTransport().catch( function ( closeErr ) {
103
+ logger.error( `winkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
104
+ } );
105
+ }; // closeQuietly()
106
+
107
+ /**
108
+ * Runs the drain once. The engine latches the returned promise so
109
+ * every caller receives this one outcome.
110
+ *
111
+ * @param {number} timeoutMs - The caller's budget; 0 means none
112
+ * @returns {Promise<void>} Resolves clean, or rejects classified
113
+ */
114
+ const drain = async function ( timeoutMs ) {
115
+ let totalRows = 0;
116
+ let deliveredRows = 0;
117
+ let failedRows = 0;
118
+ let firstFailure = null;
119
+ const waits = [];
120
+
121
+ const awaitDelivery = function ( entry ) {
122
+ totalRows += entry.rows;
123
+ const ownerDone = entry.onDone;
124
+ waits.push( new Promise( function ( resolve ) {
125
+ entry.onDone = function ( err, abandoned ) {
126
+ ownerDone( err, abandoned );
127
+ if ( err ) {
128
+ failedRows += entry.rows;
129
+ if ( !firstFailure ) {
130
+ firstFailure = err;
131
+ }
132
+ } else {
133
+ deliveredRows += entry.rows;
134
+ }
135
+ resolve();
136
+ };
137
+ } ) );
138
+ }; // awaitDelivery()
139
+
140
+ // Everything delivery still owes: flushes already in flight
141
+ // (their rows left the buffer at their call) plus one final
142
+ // flush for whatever is still buffered.
143
+ ledger.entries.forEach( awaitDelivery );
144
+ const buffered = takeBufferedRows();
145
+ if ( buffered > 0 ) {
146
+ awaitDelivery( track( sender.flush(), buffered, NOOP ) );
147
+ }
148
+
149
+ if ( waits.length > 0 ) {
150
+ try {
151
+ await raceTimeout( Promise.all( waits ), timeoutMs );
152
+ } catch ( err ) {
153
+ await closeQuietly();
154
+ const dropped = totalRows - deliveredRows;
155
+ const timedOut = new Error(
156
+ `winkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
157
+ );
158
+ timedOut.code = 'SHUTDOWN_TIMEOUT';
159
+ timedOut.dropped = { count: dropped };
160
+ throw timedOut;
161
+ }
162
+
163
+ if ( failedRows > 0 ) {
164
+ await closeQuietly();
165
+ const failure = new Error(
166
+ `winkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
167
+ );
168
+ failure.code = 'DELIVERY_FAILED';
169
+ failure.dropped = { count: failedRows };
170
+ failure.cause = firstFailure;
171
+ throw failure;
172
+ }
173
+ }
174
+
175
+ await closeTransport();
176
+ }; // drain()
177
+
178
+ return drain;
179
+ }; // createShutdownDrain()
180
+
181
+ export { createShutdownDrain };