@winkjs/composer 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +1 -1
  2. package/package.json +2 -2
  3. package/src/composer.js +1 -1
  4. package/src/core/emitter-manager/mqtt/connection.js +209 -0
  5. package/src/core/emitter-manager/mqtt/constants.js +43 -29
  6. package/src/core/emitter-manager/mqtt/emitter.js +103 -658
  7. package/src/core/emitter-manager/mqtt/health.js +151 -0
  8. package/src/core/emitter-manager/mqtt/index.js +68 -4
  9. package/src/core/emitter-manager/mqtt/publish.js +292 -0
  10. package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
  11. package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
  12. package/src/core/emitter-manager/terminal/index.js +16 -1
  13. package/src/core/env-vars.js +99 -13
  14. package/src/core/logger/index.js +1 -1
  15. package/src/core/source-manager/mqtt/client.js +197 -63
  16. package/src/core/source-manager/mqtt/dedup.js +16 -11
  17. package/src/core/source-manager/mqtt/index.js +32 -5
  18. package/src/core/source-manager/mqtt/status.js +168 -56
  19. package/src/core/storage-manager/questdb/address-policy.js +264 -0
  20. package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
  21. package/src/core/storage-manager/questdb/flush-engine.js +852 -0
  22. package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
  23. package/src/core/storage-manager/questdb/index.js +574 -847
  24. package/src/core/storage-manager/questdb/persist-plan.js +128 -167
  25. package/src/core/storage-manager/questdb/resolve-options.js +403 -0
  26. package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
  27. package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
  28. package/src/core/test-utils/tcp-proxy.js +194 -1
  29. package/src/core/utils/address/index.js +362 -0
  30. package/src/core/utils/address/probe.js +210 -0
  31. package/src/core/utils/callback-guard/index.js +23 -2
  32. package/src/core/utils/clock/index.js +34 -0
  33. package/src/core/utils/jitter/index.js +51 -0
  34. package/src/core/utils/line-rate/index.js +104 -0
  35. package/src/core/utils/quantize/index.js +43 -19
  36. package/src/core/wiring/wire-emitters.js +3 -1
  37. package/src/core/wiring/wire-storages.js +10 -2
  38. package/src/flow/flow.js +24 -3
  39. package/src/flow/run.js +24 -2
@@ -1,64 +1,134 @@
1
1
  // core/storage-manager/questdb/index.js
2
2
 
3
3
  /**
4
- * @fileoverview QuestDB storage adapter for time-series data persistence.
5
- *
6
- * Creates storage interface that integrates with partition-manager:
7
- * - ensureTables: Creates tables via PostgreSQL wire protocol at startup
8
- * - write(insightType, msg, partitionId): Writes row via ILP, sync return
9
- * per the ADR-018 sink contract — `{ ok: true }` on success,
10
- * `{ ok: false, error: { code, message } }` on failure.
11
- * - flush(): Manual flush (in manual mode) or no-op (in auto mode)
12
- * - shutdown(): Graceful shutdown
13
- * - getPressure(): Buffer fill ratio in [0, 1] per ADR-018 (sync, O(1),
14
- * allocation-free).
15
- *
16
- * Two flush modes:
17
- * - auto: QuestDB client handles flushing based on rows/interval
18
- * - manual: Application controls flush timing via idle timer
19
- *
20
- * Adapter contract (ADR-018, structured-sink role):
21
- * - Module-level exports per the ADR-018 module surface: `id`, `configSchema`,
22
- * `createStorage`, `durabilityClass`, `semanticsRequirement`, and the
23
- * default aggregate (`questdbAdapter`) referencing the same constants.
24
- * - Durability class (ADR-018): `'in-memory'` — rows accepted by `write()`
25
- * live in the ILP client's in-process buffer until a flush reaches the
26
- * server.
27
- * A process crash loses every un-flushed row; the shutdown drain and
28
- * its classified loss reporting exist precisely to shrink that window.
29
- * - Hot-path `write` returns sync per ADR-013; the partition-manager never
30
- * awaits a Promise on the per-message path.
31
- * - `getPressure()` returns `(bufferedRows + inFlightRows) / autoFlushRows`,
32
- * clamped to 1.0. `bufferedRows` counts rows awaiting the next flush and
33
- * increments synchronously when the persist plan opens a row; a flush call
34
- * moves its rows to `inFlightRows` (the client copies them out of the
35
- * buffer synchronously — copy-out), where they stay until the flush
36
- * settles. In auto mode the counter mirrors QuestDB's internal flush
37
- * trigger via a heuristic reset at the `autoFlushRows` boundary; the
38
- * `auto_flush_interval` trigger is not directly observable but self-heals
39
- * via `checkIdleFlush` after `idleFlushAfterMs` of write-idle.
40
- *
41
- * Health-status semantics (uniform across sinks):
42
- * - `getHealth()` returns `{status, connected, pressure, ...}` — the ADR-018
43
- * health floor.
44
- * - `connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD`.
45
- * QuestDB has no persistent connection state to inspect (ILP sender is
46
- * fire-and-forget); `connected` is *derived* from recent write success.
47
- * - Status derivation:
48
- * `red` if `!connected` (transport unhealthy or shutting down)
49
- * `yellow` if `pressure >= 0.66` (threshold shared with ADR-020's
50
- * Draft yield proposal) OR
51
- * `consecutiveWriteErrors >= 1` (any error elevates concern)
52
- * `green` otherwise
53
- * - The counter resets to 0 on every successful write, so a single recovery
54
- * write returns `status` to `green`.
4
+ * @fileoverview QuestDB storage adapter. It writes rows to QuestDB over
5
+ * ILP (InfluxDB Line Protocol) on HTTP, and creates the tables over the
6
+ * PostgreSQL wire protocol at startup.
7
+ *
8
+ * Contract role (ADR-018): storage, buffered. Startup posture: eager.
9
+ * Setup throws when either endpoint does not answer, so a flow never
10
+ * starts against a dead database.
11
+ *
12
+ * Module layout. This file holds the factory, the config schema, and
13
+ * the adapter export. The work lives in four modules next to it:
14
+ * - `resolve-options.js` merges the config and the environment into
15
+ * the settings, and derives the ceiling and the deadline.
16
+ * - `address-policy.js` holds the ADR-030 address checks, the setup
17
+ * probe wrapper, and the classification of a connect failure.
18
+ * - `flush-engine.js` owns the hot path, every flush, the counters
19
+ * behind pressure and health, and the shutdown drain (ADR-029).
20
+ * - `persist-plan.js` compiles one row writer per insight type.
21
+ *
22
+ * The handle the factory returns:
23
+ * - `write(insightType, msg, partitionId)`: the hot path. Appends one
24
+ * row to the client's buffer and returns at once. `{ ok: true }`
25
+ * means accepted into the buffer, not delivered. `{ ok: false,
26
+ * error: { code, message } }` names the refusal. Never throws.
27
+ * - `flush()`: sends everything buffered now. Resolves when the server
28
+ * confirmed the rows, rejects when it did not.
29
+ * - `shutdown({ timeout })`: drain, then close. A clean resolve means
30
+ * every row was delivered. A loss is a classified rejection.
31
+ * - `getPressure()`: buffer fill in [0, 1]. Sync, O(1), no allocation.
32
+ * - `getHealth()`: `{ status, connected, pressure, ... }`.
33
+ * `flush-engine.js` documents how each one behaves.
34
+ *
35
+ * Composer owns every flush (ADR-029). The client's own trigger is
36
+ * off (`auto_flush=off`), so a batch leaves the process only when the
37
+ * engine says so. A write that brings the buffer to `flushRows` starts
38
+ * a flush at once. A timer starts a flush every `flushIntervalMs` when
39
+ * anything is buffered. Only one engine flush runs at a time. A failed
40
+ * flush is reported once, with the exact rows lost, and the process
41
+ * keeps running. Every flush has a deadline, derived from the rows it
42
+ * carries, and a flush past it is abandoned and reported. A failed or
43
+ * abandoned flush runs the ADR-030 probe. While the probe fails,
44
+ * delivery pauses: rows are held up to the ceiling and each tick
45
+ * probes again. `flush-engine.js` carries the detail.
46
+ *
47
+ * Row order. Rows reach QuestDB in write order. The engine runs one
48
+ * flush at a time, each flush carries its rows in write order, and
49
+ * the standard-library transport keeps one socket (`maxSockets: 1`),
50
+ * so a later request waits for the earlier one. Two sends can be
51
+ * outstanding only at shutdown, when the drain's final flush starts
52
+ * while an earlier flush is still on the wire. On the undici
53
+ * transport those two batches can land out of order. QuestDB orders
54
+ * each table by its designated timestamp, so a query never sees the
55
+ * difference.
56
+ *
57
+ * Durability in plain words (ADR-018). `durabilityClass` is
58
+ * `'in-memory'`. Rows live in the client's buffer until a flush reaches
59
+ * the server. Three things lose rows: a process crash loses everything
60
+ * buffered or in flight, a failed flush loses its batch, and the
61
+ * ceiling sheds new rows. The shed policy: once the rows buffered plus
62
+ * the rows in flight reach `bufferCeilingRows`, `write()` refuses new
63
+ * rows with `STORAGE_FULL` and the rows already accepted stay safe.
64
+ * Outage budget with the defaults: the ceiling of 50,000 rows holds
65
+ * 50 seconds at 1,000 rows a second and about 14 hours at one row a
66
+ * second, in 6.5 to 15 MB of memory at the measured 130 to 290 bytes
67
+ * a row.
68
+ *
69
+ * Edge-baseline defaults (every one runs within a 2 GB device):
70
+ *
71
+ * | Setting | Default | Environment variable |
72
+ * |---------------------|------------------|------------------------------|
73
+ * | `flushRows` | 5000 | `QUESTDB_FLUSH_ROWS` |
74
+ * | `flushIntervalMs` | 1000 | `QUESTDB_FLUSH_INTERVAL_MS` |
75
+ * | `bufferCeilingRows` | 10 × `flushRows` | `QUESTDB_BUFFER_CEILING_ROWS`|
76
+ * | `flushDeadlineMs` | derived per flush| `QUESTDB_FLUSH_DEADLINE_MS` |
77
+ * | `stdlibHttp` | true | `QUESTDB_STDLIB_HTTP` |
78
+ * | `requestTimeout` | client default | `QUESTDB_REQUEST_TIMEOUT` |
79
+ * | `retryTimeout` | client default | `QUESTDB_RETRY_TIMEOUT` |
80
+ * | `initBufSize` | client default | `QUESTDB_INIT_BUF_SIZE` |
81
+ * | `maxBufSize` | client default | `QUESTDB_MAX_BUF_SIZE` |
82
+ *
83
+ * `resolve-options.js` merges the config and the environment and
84
+ * explains each default. `ilpUrl` and `pgUrl` fall back to
85
+ * `QUESTDB_ILP_URL` and `QUESTDB_PG_URL`; the credentials come from
86
+ * `QUESTDB_DATABASE`, `QUESTDB_USER` and `QUESTDB_PASSWORD`.
87
+ *
88
+ * Long-running commitments (ADR-018 §12). One timer, created at setup
89
+ * and cleared at shutdown; it does not hold the process open. Every
90
+ * counter is bounded by the ceiling. The per-row path allocates
91
+ * nothing beyond the one derived promise `persist-plan.js` documents.
92
+ * Reconnection and request retries belong to the client. No listeners
93
+ * are attached. `flush-engine.js` carries the detail.
94
+ *
95
+ * The transport (ADR-029). The client's default HTTP library, undici,
96
+ * retries a refused connection without end, and its abort cannot end
97
+ * that retry. So the adapter selects the client's standard-library
98
+ * transport (`stdlib_http=on`) unless `stdlibHttp` is false. With it, a refused connection rejects at once and every
99
+ * request ends within `retryTimeout` plus one request timeout. The
100
+ * adapter owns one `http.Agent` for that transport, keep-alive on one
101
+ * socket, and destroys it after the sender closes on every shutdown
102
+ * path. Destroying the agent ends a request still on its socket, so
103
+ * the process can exit. The agent also closes its socket after 4
104
+ * seconds idle, before QuestDB's own idle close at 5 minutes, so no
105
+ * flush meets a socket the server has just closed. An operator who
106
+ * selects undici accepts that an abandoned flush can keep the process
107
+ * alive.
108
+ *
109
+ * Health (ADR-018 §8). `status` is `red` when not connected or at
110
+ * capacity. It is `yellow` at `pressure >= 0.66`, on an outstanding
111
+ * write error, or after one failed flush. Otherwise it is `green`.
112
+ * `connected` is derived, because the ILP client exposes no socket
113
+ * state. It is false while delivery is paused, after five write errors
114
+ * in a row, after two failed flushes in a row, or after one abandoned
115
+ * flush. The health object carries `pausedSince`, `abandonedFlushes`,
116
+ * `consecutiveFlushFailures`, `lastFlushAt`, and `lastFlushError`.
117
+ * `flush-engine.js` documents the derivation.
118
+ *
119
+ * Deprecated options (ADR-029, removed in 0.8.0). `autoFlushRows` maps
120
+ * to `flushRows` and `idleFlushCheckMs` maps to `flushIntervalMs`.
121
+ * `flushMode`, `idleFlushAfterMs` and `autoFlushIntervalMs` are
122
+ * accepted and ignored. The same holds for their `QUESTDB_*`
123
+ * variables. Setup prints one `DEPRECATED_OPTION` line naming every
124
+ * legacy key in use and what happened to it.
55
125
  *
56
126
  * `err.code` vocabulary (per-adapter; ADR-018 has each adapter document
57
127
  * its own codes in this header):
58
128
  *
59
129
  * Setup-time throws (ADR-018 fail-fast setup):
60
130
  * - `INVALID_CONFIG` — the supplied configuration does not
61
- * work. Five current sub-cases:
131
+ * work. Ten current sub-cases:
62
132
  * (a) required transport URL missing (`ilpUrl`, `pgUrl`);
63
133
  * (b) PostgreSQL endpoint answered but rejected the connection
64
134
  * — wrong credentials or a protocol-level refusal;
@@ -72,18 +142,57 @@
72
142
  * of wedging the sender mid-row at runtime;
73
143
  * (e) an insightType uses the reserved column name `assetId`.
74
144
  * Composer writes that column itself, from the partition id.
75
- * Checked at plan build in `persist-plan.js`.
145
+ * Checked at plan build in `persist-plan.js`;
146
+ * (f) `ilpUrl` or `pgUrl` names `localhost` (ADR-030). The name
147
+ * can resolve to two addresses, `::1` and `127.0.0.1`, and
148
+ * QuestDB may listen on only one. Refused before any socket
149
+ * opens, at the schema (flow definition) and here (the env
150
+ * fallback and direct callers). The message names the
151
+ * literal to set;
152
+ * (g) `ilpUrl` is an IPv6 literal. The client (4.2.0) splits the
153
+ * address on its first colon and cannot read one. `pgUrl`
154
+ * accepts `[::1]:8812`;
155
+ * (h) the ILP client rejected the sender configuration for a
156
+ * reason that is not a network error (`fromConfig`);
157
+ * (i) `bufferCeilingRows` is below twice `flushRows` (ADR-029).
158
+ * Rows in flight count against the ceiling, so it must hold
159
+ * one batch in flight and one batch buffering. A smaller
160
+ * ceiling would refuse every row that arrives during a
161
+ * row-triggered flush. Checked in `resolve-options.js`;
162
+ * (j) `tablePrefix` is not an identifier: letters, digits, `_`
163
+ * and `$`, not starting with a digit. The prefix opens every
164
+ * table name, and `ensure-tables.js` writes that name unquoted
165
+ * in CREATE TABLE. QuestDB reads an unquoted name as one
166
+ * token. So a hyphen, a space, a dot, a semicolon or a
167
+ * backtick ends the statement early with "unexpected token"
168
+ * (verified live on 9.4.3). QuestDB's own rule for a quoted
169
+ * name is wider. Its CREATE TABLE reference says a name with
170
+ * a space or a dot must be double-quoted. The server's
171
+ * `TableUtils.isValidTableName` forbids `? , ' " \ / : ( ) +
172
+ * * % ~`, control characters, and a leading or trailing dot
173
+ * or space. Sources: https://questdb.com/docs/query/sql/create-table/
174
+ * and https://questdb.com/docs/connect/compatibility/ilp/advanced-settings/.
175
+ * The identifier rule is the one the semantics schema applies
176
+ * to asset class, insight type and column names, so every
177
+ * part of a table name follows one rule. Checked at the
178
+ * schema and here, which covers the default prefix (the asset
179
+ * class name) and a direct caller.
76
180
  * Operator remediation: fix the supplied config or the relevant
77
- * `QUESTDB_*` env var. The underlying transport error (sub-case b)
78
- * is preserved on `err.cause` for diagnostics.
79
- * - `TRANSPORT_UNREACHABLE` — the PostgreSQL endpoint did not answer
80
- * at setup: nothing listening on the port, host unresolvable, no
81
- * route, or the attempt timed out (a Node syscall code —
82
- * `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, ...). Distinct from
83
- * `INVALID_CONFIG` per the one split the ADR-018 error vocabulary
84
- * mandates: the
85
- * connection string may be fine — check the network, the firewall,
86
- * whether QuestDB is running. Underlying error on `err.cause`.
181
+ * `QUESTDB_*` env var. The underlying error (sub-cases b and h) is
182
+ * preserved on `err.cause` for diagnostics.
183
+ * - `TRANSPORT_UNREACHABLE` — an endpoint did not answer at setup.
184
+ * Three sources. (1) The setup probe (ADR-030 item 4): before any
185
+ * client is built, `pgUrl` and then `ilpUrl` are probed with one
186
+ * TCP connect per address. A name is resolved with Node's default
187
+ * lookup and every address it resolves to must answer; the message
188
+ * lists each address with its result and names the literal to set.
189
+ * (2) The PostgreSQL connect failed with a Node syscall code
190
+ * (`ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, ...). (3) The ILP
191
+ * client could not be built for the same class of reason.
192
+ * Distinct from `INVALID_CONFIG` per the one split the ADR-018
193
+ * error vocabulary mandates: the connection string may be fine —
194
+ * check the network, the firewall, whether QuestDB is running.
195
+ * Underlying error on `err.cause` for (2) and (3).
87
196
  * - `MISSING_ASSET_CLASS` — `assetClass` not provided to `createStorage`.
88
197
  * Distinct from `INVALID_CONFIG` because operator remediation differs:
89
198
  * they need to add `.assetClass(assetClassDef)` to the flow, not edit env
@@ -91,26 +200,72 @@
91
200
  * - `SCHEMA_ERROR` — DDL `CREATE TABLE` failed for a reason other
92
201
  * than already-exists (see `ensure-tables.js`).
93
202
  *
94
- * Runtime returns:
95
- * - `INVALID_INSIGHT_TYPE` — sync; the insightType has no persist plan
203
+ * Runtime returns (sync, from `write()`):
204
+ * - `INVALID_INSIGHT_TYPE` — the insightType has no persist plan
96
205
  * (typically a config typo or asset-class drift).
97
- * - `SEND_FAILED` — sync; the persist plan threw while invoking
98
- * sender methods (ILP buffer overflow, type coercion failure, etc.).
206
+ * - `STORAGE_FULL` — the buffer is at `bufferCeilingRows`. The
207
+ * row was shed. One shared result object; shedding allocates
208
+ * nothing. Remediation: the endpoint is not taking rows; check it.
209
+ * Raise the ceiling only when memory allows.
210
+ * - `SHUTTING_DOWN` — `shutdown()` has been called; no flusher
211
+ * is left for a new row. One shared result object.
212
+ * - `SEND_FAILED` — the persist plan threw while invoking
213
+ * sender methods (type coercion failure, a client throw). The
214
+ * half-written row is cancelled (mid-row recovery, in
215
+ * `flush-engine.js`).
99
216
  * - `INVALID_TIMESTAMP` — reserved; today persist-plan handles invalid
100
217
  * timestamps via `onWarning + skip-row`. It may later be surfaced as a
101
218
  * hard return error.
102
219
  *
103
220
  * Shutdown-time throws (ADR-018 — a clean shutdown resolve means
104
221
  * everything buffered or in flight was delivered; both carry
105
- * `dropped: { count }`, exact). Shutdown settles every unsettled flush
106
- * (idle, recovery) plus one final flush, raced against `{ timeout }`;
107
- * its outcome is latched so repeated calls cannot contradict it:
222
+ * `dropped: { count }`, exact):
108
223
  * - `DELIVERY_FAILED` — a flush failed; first flush error on `cause`.
109
224
  * - `SHUTDOWN_TIMEOUT` — delivery did not settle within the caller's
110
- * `{ timeout }` (a send against an unreachable server never settles —
111
- * see the mid-row recovery section below).
112
- *
113
- * Runtime console classification:
225
+ * `{ timeout }`.
226
+ *
227
+ * Runtime console classification (tokens, not `err.code` values):
228
+ * - `DELIVERY_FAILED` — an engine or recovery flush failed or was
229
+ * abandoned at its deadline, or the client refused a row append, and
230
+ * no `onDeliveryFailure` was given. One `logger.error` line per
231
+ * event, carrying the exact rows lost and the client's message. A
232
+ * flush line ends with the probe finding when a probe ran. The
233
+ * process keeps running: an unattended deployment must report a lost
234
+ * batch, not stop on it. The health surface carries the same fact.
235
+ * - `CIRCUIT_OPEN` — delivery paused or resumed (ADR-029 hold and
236
+ * probe). One `logger.warn` line when a failing probe pauses
237
+ * delivery, naming the held rows and the finding. One `logger.warn`
238
+ * line when a tick probe passes and delivery resumes, at warn so a
239
+ * log transport with a warn floor carries the episode end. Nothing
240
+ * per tick. Remediation: the endpoint refused a TCP connect; check
241
+ * that QuestDB is running and reachable. Rows are held up to the
242
+ * ceiling meanwhile, and health reads red with `pausedSince`.
243
+ * - `DELIVERY_HEALTH` — the delivery ladder changed state. One
244
+ * `logger.warn` line when it enters yellow (the first failed flush),
245
+ * one `logger.error` line when it enters red (two failed in a row,
246
+ * one abandoned, or a failing probe pausing delivery, printed just
247
+ * before the `CIRCUIT_OPEN` line), and one `logger.warn` line when
248
+ * it returns to green, naming the episode length and the rows
249
+ * reported lost in it.
250
+ * Nothing while a state persists, whatever the outage length, and
251
+ * the lines print with or without `onDeliveryFailure`. Remediation:
252
+ * the detail is the client's message or the deadline; a red with no
253
+ * `pausedSince` in health is a server that answers and refuses.
254
+ * - `STORAGE_FULL` (lines) — one `logger.warn` line when the first row
255
+ * of an episode is refused at the ceiling, and one when the buffer
256
+ * has room again, with the count refused. The end is found at the
257
+ * interval tick, so its line can lag the first free slot by one
258
+ * interval. Remediation: the endpoint is not taking rows; read the
259
+ * `CIRCUIT_OPEN` and `DELIVERY_HEALTH` lines beside it, and raise
260
+ * `bufferCeilingRows` when a longer outage must be ridden through.
261
+ * - `DEPRECATED_OPTION` — one `logger.warn` line at setup naming the
262
+ * legacy keys in use (see Deprecated options above).
263
+ * - `ADDRESS_IS_NAME` — `ilpUrl` or `pgUrl` is a name other than
264
+ * `localhost`. One `logger.warn` line per field at setup, before any
265
+ * socket opens (ADR-030). A name is allowed and the adapter
266
+ * proceeds. The line tells the operator that only a literal address
267
+ * is immune to a resolver that changes its answer under a running
268
+ * process.
114
269
  * - `CALLBACK_FAILED` — the user's `onDeliveryFailure` itself threw or
115
270
  * rejected. The shared callback guard contains the fault (ADR-018).
116
271
  * The adapter keeps writing and flushing, and each fault becomes one
@@ -142,31 +297,22 @@
142
297
  * extend this section and add corresponding assertions in
143
298
  * `assert-columns.js`.
144
299
  *
145
- * Async-rejection containment:
146
- * - QuestDB's `sender.at(...)` is `async` — the buffer mutation is sync but
147
- * the trailing `await this.tryFlush()` can fire a network flush. To prevent
148
- * silent loss when that flush fails, `persist-plan.js` attaches a `.catch()`
149
- * to the `at()` Promise and routes the failure to `onDeliveryFailure` when
150
- * provided — otherwise it throws a classified `DELIVERY_FAILED` so the loss
151
- * is loud. See `persist-plan.js` for the wrapper.
152
- *
153
- * Mid-row recovery (ADR-018 — a rejected message costs that message,
154
- * nothing else):
155
- * - A persist plan that throws between sender.table() and sender.at() leaves
156
- * the client holding a half-written row. Without recovery, every later
157
- * write fails with "Table name has already been set" — the 2026-06-10
158
- * silent-write-failure incident lost 98.6% of a replay's rows this way.
159
- * - write()'s catch therefore calls recoverSender(), which composes two
160
- * documented client calls: flush() ships every COMPLETED row out of the
161
- * buffer (the copy-out happens synchronously, before any network I/O; the
162
- * client documents that an unfinished row stays behind), then reset()
163
- * clears the buffer — at that point holding only the broken stub — and
164
- * lowers the client's row-in-progress flags.
165
- * - The client (4.2.0) has no row-cancel API, while its sibling clients do
166
- * (.NET CancelRow, Rust/C rewind_to_marker, Java recovers automatically).
167
- * Upstream issue #60 tracks the gap:
168
- * https://github.com/questdb/nodejs-questdb-client/issues/60
169
- * When a release ships cancelRow(), recoverSender() becomes that one call.
300
+ * Row append rejections. The client declares `sender.at()` as `async`,
301
+ * so its promise can reject. With the client trigger off, the only
302
+ * rejection left is the client's own byte ceiling (`max_buf_size`,
303
+ * 100 MiB by default): the append throws inside the async function
304
+ * and arrives as a rejection, and the row never completed.
305
+ * `persist-plan.js` attaches one handler per insight type to that
306
+ * promise and routes the failure to `onDeliveryFailure` as
307
+ * `( err, { tableName } )`, or prints one `DELIVERY_FAILED` line. The
308
+ * next write finds the sender holding the unfinished row, throws, and
309
+ * the mid-row recovery in `flush-engine.js` clears it.
310
+ *
311
+ * Client dependency note. `package.json` pins `@questdb/nodejs-client`
312
+ * to `~4.2.0`. `flush-engine.js` lists the five facts of that release
313
+ * the engine relies on; re-verify each on an upgrade. `persist-plan.js`
314
+ * also imports `SenderBufferV1`, an exported but undocumented member,
315
+ * for name validation at setup only.
170
316
  *
171
317
  * Integration with persist-if node:
172
318
  * - persist-if calls: state.storage.write(insightType, msg, partitionId)
@@ -201,225 +347,205 @@
201
347
  * @see https://questdb.com/docs/clients/ingest-node/
202
348
  * @see docs/architecture/storage-layer.md
203
349
  * @see ADR-018
350
+ * @see ADR-029
204
351
  */
205
352
 
353
+ import http from 'node:http';
354
+
206
355
  import { Sender } from '@questdb/nodejs-client';
207
356
  import pg from 'pg';
208
357
 
209
358
  import { ENV_VARS } from '../../env-vars.js';
210
359
  import { logger } from '../../logger/index.js';
211
360
  import { validators } from '../../utils/validate/index.js';
212
- import { wrapCallback } from '../../utils/callback-guard/index.js';
361
+ import { probeAddress, describeProbe } from '../../utils/address/probe.js';
213
362
  import { buildPersistPlans } from './persist-plan.js';
214
363
  import { ensureTables } from './ensure-tables.js';
215
364
  import { assertColumnFacts } from './assert-columns.js';
216
-
217
- // ============================================================================
218
- // HOT-PATH SINGLETONS
219
- // ============================================================================
220
-
221
- /**
222
- * Singleton success result reused on every successful write. Hot-path zero
223
- * allocation per ADR-013 / ADR-004. Plain literal — not frozen (V8 hot paths
224
- * handle plain objects more predictably; no caller mutates this).
225
- * @type {{ok: true}}
226
- */
227
- const RESULT_OK = { ok: true };
228
-
229
- /**
230
- * Console channel for the callback guard: one classified line in this
231
- * adapter's family. Receives an already-safe detail string, never the
232
- * raw thrown value.
233
- */
234
- const reportCallbackFault = function ( severity, name, detail ) {
235
- logger.error(
236
- `winkComposer/questdb: user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
237
- );
238
- }; // reportCallbackFault()
239
-
240
- // Error results (INVALID_INSIGHT_TYPE, SEND_FAILED) are constructed per-call
241
- // because each carries dynamic content (the offending insightType name and the
242
- // underlying err.message respectively). These paths are rare; per-occurrence
243
- // allocation on errors is acceptable. The RESULT_OK singleton above covers the
244
- // per-message hot path.
245
-
246
- /**
247
- * Health status thresholds. consecutiveWriteErrors crossing the YELLOW
248
- * threshold elevates status to 'yellow'; crossing the RED threshold flips
249
- * `connected` to false (and therefore `status` to 'red'). Tuned for "any
250
- * error is worth flagging" + "sustained errors mean the transport is gone."
251
- */
252
- const HEALTH_ERROR_YELLOW_THRESHOLD = 1;
253
- const HEALTH_ERROR_RED_THRESHOLD = 5;
254
-
255
- /**
256
- * Pressure threshold above which `status` elevates to at least 'yellow'.
257
- * The value matches the pressure-aware-yield design (ADR-020, still a
258
- * Draft). That design proposes 0.66 as the pressure level where the
259
- * flow would start yielding to let sinks drain; the number still needs
260
- * benchmark confirmation. Today the yield trigger is time-only (ADR-024), so
261
- * this alignment is forward-looking, not a description of current yield
262
- * behaviour. Numeric — the constant exists once here rather than scattered
263
- * as a magic number.
264
- */
265
- const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
365
+ import { resolveOptions, deprecationMessage } from './resolve-options.js';
366
+ import {
367
+ isAllowedIlpUrl,
368
+ isAllowedPgUrl,
369
+ assertNotLocalhost,
370
+ assertIlpNotIPv6,
371
+ warnIfName,
372
+ assertReachable,
373
+ pgConnectionTarget,
374
+ NETWORK_ERROR_CODES,
375
+ buildSender
376
+ } from './address-policy.js';
377
+ import { createFlushEngine } from './flush-engine.js';
266
378
 
267
379
  // ============================================================================
268
380
  // CONFIGURATION BUILDER
269
381
  // ============================================================================
270
382
 
271
383
  /**
272
- * Build QuestDB Sender configuration from options.
384
+ * Build the QuestDB Sender configuration string. The client's own flush
385
+ * trigger is always off (ADR-029): composer starts every flush itself,
386
+ * so the client never sends a batch this module did not ask for. The
387
+ * transport is stated either way, `stdlib_http=on` unless `stdlibHttp`
388
+ * is false, so a printed config reads the same however it was set.
273
389
  *
274
- * @param {Object} options - Storage options
390
+ * @param {Object} options - Resolved settings
275
391
  * @param {string} options.ilpUrl - ILP endpoint (host:port)
276
- * @param {string} options.flushMode - 'auto' or 'manual'
277
- * @param {number} [options.autoFlushRows] - Rows before auto-flush
278
- * @param {number} [options.autoFlushIntervalMs] - Time interval for auto-flush
279
- * @param {number} [options.maxBufSize] - Maximum buffer size
280
- * @param {number} [options.retryTimeout] - Retry timeout in ms
392
+ * @param {boolean} [options.stdlibHttp] - False selects undici; anything else the standard library
393
+ * @param {number} [options.initBufSize] - Initial client buffer size in bytes
394
+ * @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
395
+ * @param {number} [options.requestTimeout] - Client request timeout in ms
396
+ * @param {number} [options.retryTimeout] - Client retry window in ms
281
397
  * @returns {string} Configuration string for Sender.fromConfig()
282
398
  */
283
399
  const buildSenderConfig = function ( options ) {
284
- const { ilpUrl, flushMode, autoFlushRows, autoFlushIntervalMs, maxBufSize, retryTimeout } = options;
400
+ const { ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout } = options;
285
401
 
286
- // Base configuration: HTTP transport
287
- let config = `http::addr=${ilpUrl};`;
402
+ let config = `http::addr=${ilpUrl};auto_flush=off;stdlib_http=${( stdlibHttp === false ) ? 'off' : 'on'};`;
288
403
 
289
- // Auto-flush configuration
290
- if ( flushMode === 'auto' ) {
291
- // Use QuestDB's built-in auto-flush
292
- if ( autoFlushRows !== undefined ) {
293
- config += `auto_flush_rows=${autoFlushRows};`;
294
- }
295
- if ( autoFlushIntervalMs !== undefined ) {
296
- config += `auto_flush_interval=${autoFlushIntervalMs};`;
297
- }
298
- } else {
299
- // Manual mode: disable auto-flush, we control timing
300
- config += 'auto_flush=off;';
404
+ if ( initBufSize !== undefined ) {
405
+ config += `init_buf_size=${initBufSize};`;
301
406
  }
302
-
303
- // Optional buffer/retry settings
304
407
  if ( maxBufSize !== undefined ) {
305
- config += `init_buf_size=${maxBufSize};`;
408
+ config += `max_buf_size=${maxBufSize};`;
409
+ }
410
+ if ( requestTimeout !== undefined ) {
411
+ config += `request_timeout=${requestTimeout};`;
306
412
  }
307
413
  if ( retryTimeout !== undefined ) {
308
414
  config += `retry_timeout=${retryTimeout};`;
309
415
  }
310
416
 
311
417
  return config;
312
- };
418
+ }; // buildSenderConfig()
313
419
 
314
420
  // ============================================================================
315
- // STORAGE FACTORY
421
+ // THE TRANSPORT
316
422
  // ============================================================================
317
423
 
318
424
  /**
319
- * Race a final flush against the shutdown time budget. No budget
320
- * (0/undefined) means no enforcement — the await is unbounded, preserving
321
- * direct-caller behavior. On timeout the flush promise is deliberately
322
- * left pending: the client's send may never settle (see the file header),
323
- * and the process is shutting down anyway. Its eventual rejection, if
324
- * any, is absorbed by the race's own handlers — never an
325
- * unhandledRejection.
326
- *
327
- * @param {Promise} flushPromise - the in-flight sender.flush()
328
- * @param {number} timeoutMs - budget in ms; 0/absent disables the race
329
- * @returns {Promise} settles with the flush, or rejects SHUTDOWN_TIMEOUT
425
+ * How long an idle socket stays open before the adapter closes it.
426
+ * QuestDB closes an idle connection after 300 seconds and announces no
427
+ * keep-alive timeout, so Node gets no hint. A flush that starts in the
428
+ * moment between the server's close and Node's notice of it writes
429
+ * into a dead socket, and this transport does not retry a reset. That
430
+ * would report one batch lost for an outage that never happened.
431
+ * Closing first removes that moment (ADR-029). The value sits below
432
+ * QuestDB's advertised keep-alive of 5 seconds and below the 5-second
433
+ * timeout of Node's own default agent. Node applies it to free sockets
434
+ * only; a request in flight keeps the client's own request timeout.
330
435
  */
331
- const raceFlushTimeout = function ( flushPromise, timeoutMs ) {
332
- if ( !( timeoutMs > 0 ) ) {
333
- return flushPromise;
334
- }
335
- return new Promise( function ( resolve, reject ) {
336
- const timer = setTimeout( function () {
337
- const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
338
- err.code = 'SHUTDOWN_TIMEOUT';
339
- reject( err );
340
- }, timeoutMs );
341
- timer.unref();
342
- flushPromise.then(
343
- function ( value ) {
344
- clearTimeout( timer );
345
- resolve( value );
346
- },
436
+ const IDLE_SOCKET_TIMEOUT_MS = 4000;
347
437
 
348
- /* c8 ignore start -- unreachable from the sole caller:
349
- doShutdown races a Promise.all over waits that absorb
350
- their own rejections (that is what keeps the dropped
351
- count exact), so the raced promise cannot reject today.
352
- The handler stays because this utility's contract is
353
- generic — without it, a rejecting promise from a future
354
- caller would become an unhandled rejection and a race
355
- that never settles. */
356
- function ( err ) {
357
- clearTimeout( timer );
358
- reject( err );
359
- }
360
-
361
- /* c8 ignore stop */
362
- );
363
- } );
364
- }; // raceFlushTimeout()
438
+ /**
439
+ * Builds the one HTTP agent the standard-library transport uses. One
440
+ * socket is enough, because composer starts one flush at a time
441
+ * (ADR-029). Keep-alive reuses that socket from flush to flush, and
442
+ * the adapter closes it after `IDLE_SOCKET_TIMEOUT_MS` without a flush.
443
+ * The adapter destroys the agent at shutdown, which ends any request
444
+ * still on the socket.
445
+ *
446
+ * @returns {http.Agent} The agent
447
+ */
448
+ const createKeepAliveAgent = function () {
449
+ return new http.Agent( { keepAlive: true, maxSockets: 1, timeout: IDLE_SOCKET_TIMEOUT_MS } );
450
+ }; // createKeepAliveAgent()
365
451
 
366
452
  /**
367
- * Node syscall codes that mean the PostgreSQL endpoint did not answer.
368
- * Used at setup to classify a connect failure as TRANSPORT_UNREACHABLE
369
- * (the one split the ADR-018 error vocabulary mandates — see the
370
- * connect wrap below).
371
- * Module-level Set: allocated once at load, membership check at setup.
372
- *
373
- * @type {Set<string>}
453
+ * Opens the ILP transport: the agent when the standard-library
454
+ * transport is selected, then the sender. Returns the sender and the
455
+ * function that closes both at the end of the drain. A failed sender
456
+ * build destroys the agent before the classified error goes on, so no
457
+ * agent outlives a failed setup.
458
+ *
459
+ * `closeTransport` closes the sender first, then destroys the agent
460
+ * whatever the sender's close did. Destroying the agent ends any
461
+ * request still on its socket, so the process can exit (ADR-029). With
462
+ * undici there is no agent, and `closeTransport` is the sender's close.
463
+ *
464
+ * @param {Object} parts - The transport inputs
465
+ * @param {Object} parts.SenderClass - The client's Sender class
466
+ * @param {string} parts.senderConfig - The sender configuration string
467
+ * @param {string} parts.ilpUrl - The configured `ilpUrl`, for messages
468
+ * @param {boolean} parts.stdlibHttp - Whether the standard-library transport is selected
469
+ * @param {function} parts.createAgent - Builds the agent
470
+ * @returns {Promise<{sender: Object, closeTransport: function}>} The sender and its close
471
+ * @throws {Error} TRANSPORT_UNREACHABLE or INVALID_CONFIG from `buildSender`
374
472
  */
375
- const NETWORK_ERROR_CODES = new Set( [
376
- 'ECONNREFUSED',
377
- 'ENOTFOUND',
378
- 'ETIMEDOUT',
379
- 'EHOSTUNREACH',
380
- 'ENETUNREACH',
381
- 'ECONNRESET',
382
- 'EAI_AGAIN',
383
- 'EPIPE'
384
- ] );
473
+ const openTransport = async function ( { SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent } ) {
474
+ if ( stdlibHttp === false ) {
475
+ const sender = await buildSender( SenderClass, senderConfig, ilpUrl, null );
476
+ const closeTransport = function () {
477
+ return sender.close();
478
+ }; // closeTransport()
479
+ return { sender, closeTransport };
480
+ }
481
+
482
+ const agent = createAgent();
483
+ let sender;
484
+ try {
485
+ sender = await buildSender( SenderClass, senderConfig, ilpUrl, agent );
486
+ } catch ( err ) {
487
+ agent.destroy();
488
+ throw err;
489
+ }
490
+ const closeTransport = async function () {
491
+ try {
492
+ await sender.close();
493
+ } finally {
494
+ agent.destroy();
495
+ }
496
+ }; // closeTransport()
497
+ return { sender, closeTransport };
498
+ }; // openTransport()
499
+
500
+ // ============================================================================
501
+ // STORAGE FACTORY
502
+ // ============================================================================
385
503
 
386
504
  /**
387
505
  * Create QuestDB storage adapter.
388
506
  *
507
+ * The options are resolved by `resolve-options.js` (config over
508
+ * environment over default; legacy aliases mapped or ignored). Only the
509
+ * keys that reach the engine are listed here; the resolver documents
510
+ * the rest.
511
+ *
389
512
  * @param {Object} assetClass - Asset class definition with columns and insightTypes
390
513
  * @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
391
514
  * @param {Object} options - Configuration options
392
- * @param {string} options.ilpUrl - ILP endpoint (host:port)
393
- * @param {string} options.pgUrl - PostgreSQL endpoint (host:port)
394
- * @param {string} [options.flushMode='auto'] - 'auto' or 'manual'
395
- * @param {number} [options.idleFlushAfterMs=5000] - Idle time before flush (manual mode)
396
- * @param {number} [options.idleFlushCheckMs=1000] - Idle check interval (manual mode)
397
- * @param {number} [options.autoFlushRows] - Rows before auto-flush (auto mode)
398
- * @param {number} [options.autoFlushIntervalMs] - Time interval for auto-flush (auto mode)
399
- * @param {number} [options.maxBufSize] - Maximum buffer size
400
- * @param {number} [options.retryTimeout] - Retry timeout in ms
515
+ * @param {string} [options.ilpUrl] - ILP endpoint (host:port)
516
+ * @param {string} [options.pgUrl] - PostgreSQL endpoint (host:port)
517
+ * @param {number} [options.flushRows=5000] - Rows that start a flush from inside write()
518
+ * @param {number} [options.flushIntervalMs=1000] - Period of the flush timer
519
+ * @param {number} [options.bufferCeilingRows] - Most rows held; default ten times flushRows
520
+ * @param {number} [options.flushDeadlineMs] - Fixed deadline per flush; derived from the rows when unset
521
+ * @param {boolean} [options.stdlibHttp=true] - The standard-library transport; false selects undici
522
+ * @param {number} [options.requestTimeout] - Client request timeout in ms
523
+ * @param {number} [options.retryTimeout] - Client retry window in ms
524
+ * @param {number} [options.initBufSize] - Initial client buffer size in bytes
525
+ * @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
401
526
  * @param {string} [options.partitionBy='DAY'] - Table partition interval
402
- * @param {function} [options.onWarning] - Warning callback for null column values
527
+ * @param {function} [options.onWarning] - Warning callback for skipped values
528
+ * @param {function} [options.onDeliveryFailure] - Called once per lost flush
403
529
  * @param {Object} [deps={}] - Injectable dependencies (for testing)
404
530
  * @param {Object} [deps.SenderClass] - QuestDB Sender class (default: @questdb/nodejs-client Sender)
405
531
  * @param {Object} [deps.PgClientClass] - PostgreSQL Client class (default: pg.Client)
406
- * @returns {Promise<Object>} Storage adapter with write, flush, close methods
532
+ * @param {function} [deps.probeFn] - Setup probe (default: `probeAddress`, ADR-030)
533
+ * @param {function} [deps.createAgent] - Builds the transport agent (default: `createKeepAliveAgent`)
534
+ * @returns {Promise<Object>} Storage adapter with write, flush, shutdown, getPressure, getHealth
407
535
  */
408
536
  const createQuestDBStorage = async function ( assetClass, tablePrefix, options, deps = {} ) {
537
+ // Options become settings here, once (ADR-029). A ceiling below the
538
+ // threshold fails setup inside the resolver with INVALID_CONFIG.
539
+ const { settings, deprecations } = resolveOptions( options, ENV_VARS );
409
540
  const {
410
- ilpUrl = ENV_VARS.questdbIlpUrl,
411
- pgUrl = ENV_VARS.questdbPgUrl,
412
- flushMode = ENV_VARS.questdbFlushMode,
413
- idleFlushAfterMs = ENV_VARS.questdbIdleFlushAfterMs,
414
- idleFlushCheckMs = ENV_VARS.questdbIdleFlushCheckMs,
415
- autoFlushRows = ENV_VARS.questdbAutoFlushRows,
416
- autoFlushIntervalMs = ENV_VARS.questdbAutoFlushIntervalMs,
417
- maxBufSize = ENV_VARS.questdbMaxBufSize,
418
- retryTimeout = ENV_VARS.questdbRetryTimeout,
419
- partitionBy = 'DAY',
420
- onWarning,
421
- onDeliveryFailure
422
- } = options;
541
+ ilpUrl, pgUrl, stdlibHttp, requestTimeout, retryTimeout, initBufSize, maxBufSize,
542
+ partitionBy, onWarning, onDeliveryFailure
543
+ } = settings;
544
+
545
+ // One line names every legacy key in use and what happened to it.
546
+ if ( deprecations.length > 0 ) {
547
+ logger.warn( deprecationMessage( deprecations ) );
548
+ }
423
549
 
424
550
  // Runtime validation — required from either DSL config or ENV_VARS.
425
551
  // Per ADR-018, setup-time throws carry classified err.code.
@@ -434,35 +560,56 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
434
560
  throw err;
435
561
  }
436
562
 
563
+ // The prefix opens every table name, and CREATE TABLE writes that
564
+ // name unquoted (header sub-case (j)). The schema checks a flow's
565
+ // config. This check covers a direct caller and the default, which
566
+ // is the asset class name.
567
+ if ( !validators.identifier( tablePrefix ) ) {
568
+ const err = new Error(
569
+ `winkComposer/questdb: tablePrefix '${tablePrefix}' is not a valid table name prefix [INVALID_CONFIG]: ` +
570
+ 'use letters, digits, _ and $ only, and do not start with a digit'
571
+ );
572
+ err.code = 'INVALID_CONFIG';
573
+ throw err;
574
+ }
575
+
576
+ // Address policy (ADR-030), before any socket opens. `localhost` is
577
+ // refused. An IPv6 literal is refused for the ILP path. A name gets
578
+ // one warning per field. See header sub-cases (f), (g) and the
579
+ // ADDRESS_IS_NAME console token.
580
+ const ilpAddress = assertNotLocalhost( 'ilpUrl', ilpUrl, 'QUESTDB_ILP_URL' );
581
+ const pgAddress = assertNotLocalhost( 'pgUrl', pgUrl, 'QUESTDB_PG_URL' );
582
+ assertIlpNotIPv6( ilpAddress );
583
+ warnIfName( 'ilpUrl', ilpAddress );
584
+ warnIfName( 'pgUrl', pgAddress );
585
+
437
586
  // Injectable dependencies with defaults
438
587
  const {
439
588
  SenderClass = Sender,
440
- PgClientClass = pg.Client
589
+ PgClientClass = pg.Client,
590
+ probeFn = probeAddress,
591
+ createAgent = createKeepAliveAgent
441
592
  } = deps;
442
593
 
443
594
  // Build persist plans (pre-compiled closures). The callbacks go in
444
595
  // RAW: buildPersistPlans validates them itself and arms its own
445
- // guarded copy for the at-flush site. Handing it a pre-wrapped
596
+ // guarded copy for the row-append site. Handing it a pre-wrapped
446
597
  // function would defeat that validation (the guard turns a
447
598
  // non-function into null instead of the fail-fast INVALID_CONFIG).
599
+ // The engine wraps its own copy after this validation (ADR-027:
600
+ // wrap once, after the site's own validation).
448
601
  const persistPlans = buildPersistPlans( assetClass, tablePrefix, { onWarning, onDeliveryFailure } );
449
602
 
450
- // Arm the delivery-failure callback for this module's own report
451
- // sites: the idle-flush timer and the mid-row recovery flush. Both
452
- // fire inside promise chains, where a broken handler used to become
453
- // an unhandled rejection. The guard classifies the fault instead
454
- // (ADR-018) and its cost stays inside the handler. Absent stays
455
- // null, so the no-handler console/throw fallbacks keep their
456
- // meaning.
457
- const safeOnDeliveryFailure = wrapCallback( onDeliveryFailure, {
458
- name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
459
- } );
603
+ // Setup probe, PostgreSQL side (ADR-030 item 4): every address the
604
+ // endpoint resolves to must answer before the client opens. The
605
+ // ILP side is probed below, before the sender is built.
606
+ await assertReachable( 'pgUrl', pgAddress, probeFn );
460
607
 
461
608
  // Ensure tables exist via PostgreSQL wire protocol
462
- const [ pgHost, pgPort ] = pgUrl.split( ':' );
609
+ const { host: pgHost, port: pgPort } = pgConnectionTarget( pgUrl, pgAddress );
463
610
  const pgClient = new PgClientClass( {
464
611
  host: pgHost,
465
- port: parseInt( pgPort, 10 ),
612
+ port: pgPort,
466
613
  database: ENV_VARS.questdbDatabase,
467
614
  user: ENV_VARS.questdbUser,
468
615
  password: ENV_VARS.questdbPassword
@@ -470,15 +617,16 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
470
617
 
471
618
  // Wrap the PostgreSQL connect so the failure carries a classified
472
619
  // `err.code` an operator can route on. The ADR-018 error vocabulary
473
- // mandates one split here: an endpoint that does not answer (a Node syscall
474
- // code like ECONNREFUSED — see NETWORK_ERROR_CODES above) is
475
- // TRANSPORT_UNREACHABLE, because the remediation is "check the
476
- // network, the firewall, whether the service is running" — the
477
- // connection string may be perfectly fine. Everything else — an
478
- // auth failure (the host answered), a protocol error, an
479
- // unclassified throw — stays INVALID_CONFIG: fix the supplied
480
- // config. The underlying error is preserved via `err.cause` for
481
- // diagnostics either way.
620
+ // mandates one split here. An endpoint that does not answer is
621
+ // TRANSPORT_UNREACHABLE: a Node syscall code like ECONNREFUSED (see
622
+ // NETWORK_ERROR_CODES in address-policy.js). The remediation there
623
+ // is "check the network, the firewall, whether the service is
624
+ // running", and the connection string may be fine.
625
+ //
626
+ // Everything else stays INVALID_CONFIG: an auth failure (the host
627
+ // answered), a protocol error, an unclassified throw. The fix there
628
+ // is the supplied config. The underlying error is preserved on
629
+ // `err.cause` for diagnostics either way.
482
630
  try {
483
631
  await pgClient.connect();
484
632
  } catch ( connErr ) {
@@ -498,539 +646,52 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
498
646
  await pgClient.end();
499
647
  }
500
648
 
501
- // Create ILP sender
649
+ // The sender config. The client trigger is off and the transport is
650
+ // stated: see buildSenderConfig.
502
651
  const senderConfig = buildSenderConfig( {
503
- ilpUrl,
504
- flushMode,
505
- autoFlushRows,
506
- autoFlushIntervalMs,
507
- maxBufSize,
508
- retryTimeout
652
+ ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout
509
653
  } );
510
654
 
511
- // fromConfig returns a Promise that resolves to a connected sender
512
- const sender = await SenderClass.fromConfig( senderConfig );
513
-
514
- // Buffer state — `bufferedRows` counts rows awaiting the NEXT flush.
515
- // The client's flush() copies its rows out of the buffer synchronously
516
- // (copy-out) before any network I/O, so the moment a flush is called
517
- // its rows are no longer buffered — they move to the in-flight tally
518
- // below. Conflating the two quantities in one counter let shutdown
519
- // report clean over a hung idle flush.
520
- let lastWriteTime = 0;
521
- let idleFlushTimer = null;
522
- let bufferedRows = 0;
523
-
524
- // Rows inside flush copies that have not settled yet, plus the flush
525
- // promises carrying them. getPressure() adds this to bufferedRows (a
526
- // hung flush is undelivered data and must read as pressure), and
527
- // shutdown() settles these entries instead of firing a blind flush
528
- // at a buffer the copy-out already emptied.
529
- let inFlightRows = 0;
530
- const inFlightFlushes = new Set();
531
-
532
- // Shutdown outcome, latched on the first call. A lossy shutdown's
533
- // failed flush already emptied the buffer via copy-out, so a re-run
534
- // would find nothing to flush and resolve clean — contradicting the
535
- // recorded loss. Every caller gets the first call's promise instead.
536
- let shutdownPromise = null;
537
-
538
- // Health state — drives getHealth()'s status/connected derivation.
539
- // shuttingDown flips true at the start of shutdown(); never resets.
540
- // consecutiveWriteErrors increments on every write() catch and resets to 0
541
- // on every successful persist plan call (recovery). Pure derived signal —
542
- // no extra hot-path allocation; just two integer/boolean operations.
543
- let shuttingDown = false;
544
- let consecutiveWriteErrors = 0;
545
-
546
- // One idle flush in flight at a time. Against an unreachable server a
547
- // flush can never settle (the client's retry loop — see the mid-row
548
- // recovery section in the file header); without this guard every check
549
- // interval would start ANOTHER stuck flush. A hung flush leaves the
550
- // flag set, the buffer keeps filling, and the condition surfaces as
551
- // rising pressure in getHealth() — visible, not multiplied.
552
- let idleFlushInFlight = false;
553
-
554
- /**
555
- * Registers a flush the moment it is called. Copy-out means the rows
556
- * leave the buffer NOW, delivered or not, so the caller hands them
557
- * over in the same breath. Settlement — either way — removes them
558
- * from the in-flight tally: resolved means delivered; rejected means
559
- * lost, and loss REPORTING stays at the call site (the idle flush
560
- * logs or routes to onDeliveryFailure, recovery routes to
561
- * onDeliveryFailure, shutdown throws classified).
562
- *
563
- * @param {Promise} flushPromise - the just-fired sender.flush()
564
- * @param {number} rows - row count the flush copy carries
565
- * @returns {{promise: Promise, rows: number}} the tracked entry
566
- */
567
- const trackFlush = function ( flushPromise, rows ) {
568
- const entry = { promise: flushPromise, rows };
569
- inFlightRows += rows;
570
- inFlightFlushes.add( entry );
571
- const settle = function () {
572
- inFlightFlushes.delete( entry );
573
- inFlightRows -= rows;
574
- };
575
- flushPromise.then( settle, settle );
576
- return entry;
577
- }; // trackFlush()
578
-
579
- /**
580
- * Check and flush if idle for idleFlushAfterMs.
581
- *
582
- * Runs in BOTH modes (see longer comment block at the setInterval call
583
- * below). The rows move to the in-flight tally the moment the flush
584
- * fires; a successful settle clears them, which also bounds the
585
- * worst-case pressure-counter lag in auto mode (when QuestDB's silent
586
- * `auto_flush_interval` trigger fires without us observing it, this
587
- * is the self-healing point).
588
- */
589
- const checkIdleFlush = async function () {
590
- if ( idleFlushInFlight || bufferedRows === 0 ) {
591
- return;
592
- }
593
-
594
- const now = Date.now();
595
- const idleTime = now - lastWriteTime;
596
-
597
- if ( idleTime >= idleFlushAfterMs ) {
598
- idleFlushInFlight = true;
599
- const rows = bufferedRows;
600
- bufferedRows = 0;
601
- const entry = trackFlush( sender.flush(), rows );
602
- try {
603
- await entry.promise;
604
- } catch ( err ) {
605
- // Copy-out: the failed flush carried the rows with it —
606
- // the buffer no longer holds them and a retry cannot
607
- // resend them. They are lost, and this is the report
608
- // (the settle handler already cleared them from
609
- // pressure). Same routing policy as the recovery flush:
610
- // the caller owns the response when it asked to.
611
- if ( safeOnDeliveryFailure ) {
612
- safeOnDeliveryFailure( err, { idleFlush: true, rowsLost: rows } );
613
- } else {
614
- logger.error(
615
- `winkComposer/questdb: idle flush failed; ${rows} buffered row(s) lost: ${err.message}`
616
- );
617
- }
618
- } finally {
619
- idleFlushInFlight = false;
620
- }
621
- }
622
- };
623
-
624
- // Start idle flush timer as safety net for "data stopped flowing" case.
625
- // QuestDB's auto_flush_interval only checks elapsed time when NEW data is added -
626
- // it is NOT a real background timer. If no more data arrives, the buffer sits
627
- // indefinitely. QuestDB docs recommend: "implement your own timer-based logic."
628
- // See: https://py-questdb-client.readthedocs.io/en/stable/conf.html
629
- //
630
- // This timer runs in BOTH modes:
631
- // - Auto mode: Safety net for when data stops flowing (QuestDB won't flush on its own)
632
- // - Manual mode: Primary flush mechanism (QuestDB auto-flush is disabled)
633
- idleFlushTimer = setInterval( checkIdleFlush, idleFlushCheckMs );
634
- // Don't block process exit
635
- idleFlushTimer.unref();
636
-
637
- // ========================================================================
638
- // STORAGE INTERFACE
639
- // ========================================================================
640
-
641
- /**
642
- * Cancels a half-written ILP row after a mid-row throw, so the NEXT write
643
- * starts clean (see "Mid-row recovery" in the file header for the full
644
- * account and the upstream issue link).
645
- *
646
- * flush() copies every completed row out of the buffer synchronously and
647
- * sends them in the background; the unfinished row stays behind. reset()
648
- * then clears the buffer — only the broken stub remains at that point —
649
- * and lowers the client's row-in-progress flags. Net effect: the broken
650
- * row vanishes, every good row is on its way, the sender accepts the
651
- * next write.
652
- *
653
- * The early flush carries real data. If its send ultimately fails (after
654
- * the client's own retries), that loss is reported like any other
655
- * delivery failure: routed to `onDeliveryFailure` when provided,
656
- * otherwise thrown as a classified unhandled rejection — the same policy
657
- * as persist-plan's sender.at() failure path. Never silent.
658
- */
659
- const recoverSender = function () {
660
- try {
661
- // flush() is an async function: it can never throw synchronously,
662
- // and the rows it sends live in its own copy of the buffer.
663
- // Tracked like every flush, so shutdown settles it and its
664
- // rows stay visible as pressure until it settles.
665
- const entry = trackFlush( sender.flush(), bufferedRows );
666
- sender.reset();
667
- bufferedRows = 0;
668
-
669
- entry.promise.catch( ( flushErr ) => {
670
- if ( safeOnDeliveryFailure ) {
671
- safeOnDeliveryFailure( flushErr, { recovery: true } );
672
- return;
673
- }
674
- const failure = new Error(
675
- `winkComposer/questdb: recovery flush failed after a mid-row write error: ${flushErr.message}. ` +
676
- 'Completed rows in that batch were dropped. Provide an `onDeliveryFailure` callback ' +
677
- 'in the storage config to handle these explicitly.'
678
- );
679
- failure.code = 'DELIVERY_FAILED';
680
- failure.cause = flushErr;
681
- throw failure;
682
- } );
683
- } catch ( recoveryErr ) {
684
- // Defensive: with the 4.2.0 client neither call can throw here
685
- // (flush is async, reset is trivial buffer bookkeeping). If a
686
- // future client changes that, the sender may stay wedged — but
687
- // write() must still return its classified result rather than
688
- // throw (ADR-018: the hot path never throws), and the
689
- // failure must be visible.
690
- logger.error( `winkComposer/questdb: sender recovery failed: ${recoveryErr.message}` );
691
- }
692
- }; // recoverSender()
693
-
694
- /**
695
- * Write a message to QuestDB for a given insightType.
696
- *
697
- * Sync hot-path return per ADR-018 / ADR-013:
698
- * - `RESULT_OK` (singleton) on successful enqueue.
699
- * - `{ ok: false, error: { code: 'INVALID_INSIGHT_TYPE', message } }`
700
- * when the insightType has no persist plan (config error).
701
- * - `{ ok: false, error: { code: 'SEND_FAILED', message } }` when the
702
- * persist plan throws (sender method failure: ILP buffer overflow,
703
- * type coercion failure, etc.).
704
- *
705
- * Never throws. Async tryFlush rejections inside `sender.at()` are
706
- * contained at the call site in `persist-plan.js` (see file header).
707
- *
708
- * @param {string} insightType - SignalType name (must exist in assetClass)
709
- * @param {Object} message - Message with column values
710
- * @param {string} partitionId - Partition identifier (stored as SYMBOL)
711
- * @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
712
- */
713
- const write = function ( insightType, message, partitionId ) {
714
- if ( shuttingDown ) {
715
- // The idle timer is already stopped and the final flush may
716
- // have run: a row accepted now would have no flusher left.
717
- // Refuse loudly instead of stranding it — uniform with the
718
- // MQTT emitter's SHUTTING_DOWN refusal.
719
- return {
720
- ok: false,
721
- error: {
722
- code: 'SHUTTING_DOWN',
723
- message: 'winkComposer/questdb: write rejected — storage is shutting down'
724
- }
725
- };
726
- }
727
-
728
- const persistPlan = persistPlans[ insightType ];
729
-
730
- if ( !persistPlan ) {
731
- return {
732
- ok: false,
733
- error: {
734
- code: 'INVALID_INSIGHT_TYPE',
735
- message: `No persist plan for insightType '${insightType}'`
736
- }
737
- };
738
- }
739
-
740
- try {
741
- // The plan reports whether it actually opened a row. A row
742
- // skipped in phase 1 (bad designated timestamp) never touched
743
- // the sender and must not count as buffered — it would inflate
744
- // pressure and shutdown's dropped count.
745
- const written = persistPlan( sender, message, partitionId );
746
- consecutiveWriteErrors = 0; // recovery — health flips back to green
747
- if ( written ) {
748
- lastWriteTime = Date.now();
749
- bufferedRows += 1;
750
- // Auto mode: QuestDB triggers an internal flush on the sender.at()
751
- // call that brings pendingRowCount to autoFlushRows (verified
752
- // against @questdb/nodejs-client v4.2.0 dist/cjs/index.js
753
- // tryFlush: pendingRowCount >= autoFlushRows). Mirror that reset
754
- // so getPressure() reflects QuestDB's actual buffer state.
755
- //
756
- // Guarded on `autoFlushRows > 0` because the value is optional in
757
- // ENV_VARS / config; when unset, QuestDB falls back to its own
758
- // internal default which we cannot observe. In that case the
759
- // heuristic does not fire and QuestDB contributes no pressure
760
- // signal — `getPressure()` (below) returns 0.
761
- if ( flushMode === 'auto' && autoFlushRows > 0 && bufferedRows >= autoFlushRows ) {
762
- bufferedRows = 0;
763
- }
764
- }
765
- return RESULT_OK;
766
- } catch ( err ) {
767
- consecutiveWriteErrors += 1; // health degradation signal
768
- // The throw may have left a half-written row in the sender;
769
- // cancel it so this failure costs one row, not the rest of the
770
- // run (ADR-018). Safe to call even when no row was open: flushing
771
- // early is harmless and reset() on a consistent buffer is a no-op.
772
- recoverSender();
773
- return {
774
- ok: false,
775
- error: {
776
- code: 'SEND_FAILED',
777
- message: err.message
778
- }
779
- };
780
- }
781
- };
782
-
783
- /**
784
- * Flush pending rows to QuestDB.
785
- * In auto mode, this is a no-op (QuestDB handles it).
786
- * In manual mode, this forces an immediate flush. The rows move to
787
- * the in-flight tally at the call (copy-out); a failure rejects to
788
- * the caller exactly as before, and the settle handler keeps the
789
- * pressure accounting straight either way.
790
- *
791
- * @returns {Promise<void>}
792
- */
793
- const flush = async function () {
794
- if ( flushMode === 'manual' && bufferedRows > 0 ) {
795
- const entry = trackFlush( sender.flush(), bufferedRows );
796
- bufferedRows = 0;
797
- await entry.promise;
798
- }
799
- // Auto mode: no-op, QuestDB handles flushing
800
- };
655
+ // Setup probe, ILP side (ADR-030 item 4). Before this change the
656
+ // write path was never opened at setup; the first sign of a dead
657
+ // endpoint was the first flush.
658
+ await assertReachable( 'ilpUrl', ilpAddress, probeFn );
801
659
 
802
- /**
803
- * Backpressure metric for the partition manager. Returns the buffer fill
804
- * ratio in [0, 1] per ADR-018 (sync, O(1), allocation-free). Clamped
805
- * to 1.0 — the codebase invariant for getPressure across adapters.
806
- *
807
- * Accuracy by mode:
808
- * - Manual mode: exact (we own every flush).
809
- * - Auto mode, heavy flow: exact (autoFlushRows heuristic reset matches).
810
- * - Auto mode, slow flow + auto_flush_interval triggers: over-states by
811
- * up to `idleFlushAfterMs`, self-heals via `checkIdleFlush`.
812
- *
813
- * Rows inside unsettled flush copies count as pressure: a hung flush
814
- * is undelivered data, and hiding it is what let shutdown report
815
- * clean over it. They leave the tally when their flush settles —
816
- * delivered or reported lost.
817
- *
818
- * When `autoFlushRows` is not configured (left to QuestDB's internal
819
- * default), we have no observable capacity reference and return
820
- * 0 — QuestDB then contributes nothing to the flow's pressure signal.
821
- * The time-based yield (ADR-024) and message-count safety nets remain
822
- * effective.
823
- *
824
- * @returns {number} Pressure value in [0, 1]
825
- */
826
- const getPressure = function () {
827
- if ( !( autoFlushRows > 0 ) ) {
828
- return 0;
829
- }
830
- return Math.min( 1.0, ( bufferedRows + inFlightRows ) / autoFlushRows );
831
- };
832
-
833
- /**
834
- * Health snapshot for operator monitoring (uniform across sinks).
835
- * Returns the ADR-018 health floor `{status, connected, ...}`, plus a
836
- * small set of QuestDB-specific diagnostic fields.
837
- *
838
- * Status derivation (kept in code, not config — operator mental model is
839
- * load-bearing institutional knowledge):
840
- * - `red` if `!connected` (shutting down or sustained write failure)
841
- * - `yellow` if `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD` OR
842
- * `consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD`
843
- * - `green` otherwise
844
- *
845
- * `connected` here is *derived* — QuestDB's ILP sender is fire-and-forget
846
- * with no observable socket state, so we infer transport health from
847
- * recent write success.
848
- *
849
- * @returns {{status: 'green'|'yellow'|'red', connected: boolean, pressure: number, consecutiveWriteErrors: number, bufferedRows: number, flushMode: string}}
850
- */
851
- const getHealth = function () {
852
- const pressure = getPressure();
853
- const connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD;
660
+ // The agent, when the standard-library transport is selected, then
661
+ // the sender. A failure is classified, cause attached (see
662
+ // buildSender), and leaves no agent behind (see openTransport).
663
+ const { sender, closeTransport } = await openTransport( {
664
+ SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent
665
+ } );
854
666
 
855
- let status;
856
- if ( !connected ) {
857
- status = 'red';
858
- } else if ( pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD || consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD ) {
859
- status = 'yellow';
860
- } else {
861
- status = 'green';
667
+ // The probe the engine runs after a failed flush (ADR-029 hold and
668
+ // probe), bound to the ILP address so the engine knows nothing
669
+ // about addresses. Setup ran the same probe on both endpoints.
670
+ const probe = {
671
+ run: function () {
672
+ return probeFn( ilpAddress );
673
+ },
674
+ describe: function ( outcome ) {
675
+ return describeProbe( outcome, 'ilpUrl', ilpAddress );
862
676
  }
863
-
864
- return {
865
- // Required health floor (ADR-018)
866
- status,
867
- connected,
868
- pressure,
869
- // Adapter-specific diagnostics
870
- consecutiveWriteErrors,
871
- bufferedRows,
872
- flushMode
873
- };
874
677
  };
875
678
 
876
- /**
877
- * Best-effort transport close on the lossy path: the loss report
878
- * (the classified throw that follows) matters more than a close
879
- * failure, which is only logged.
880
- */
881
- const closeQuietly = function () {
882
- return sender.close().catch( function ( closeErr ) {
883
- logger.error( `winkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
884
- } );
885
- }; // closeQuietly()
886
-
887
- /**
888
- * The real shutdown body. `shutdown` below latches its promise so
889
- * every caller — including re-entrant and post-failure callers —
890
- * receives this one outcome.
891
- *
892
- * A clean resolve is a delivery statement (ADR-018): everything
893
- * buffered OR in flight was delivered. Shutdown therefore settles
894
- * every unsettled flush (idle, recovery) plus one final flush for
895
- * whatever is still buffered, all raced against the caller's
896
- * `{ timeout }` (ADR-018 drain-then-close). It never fires a blind flush at a
897
- * buffer an earlier copy-out emptied — that is what let it report
898
- * clean over a hung idle flush.
899
- *
900
- * On loss it rejects classified, `dropped: { count }` exact:
901
- * - any awaited flush fails → `DELIVERY_FAILED`, first flush error
902
- * on `cause`, count = rows on the flushes that failed;
903
- * - the combined wait does not settle in time → `SHUTDOWN_TIMEOUT`,
904
- * count = rows not confirmed delivered. A send against an
905
- * unreachable server never settles (the client's retry loop — see
906
- * the file header), so the bound is what keeps shutdown finite.
907
- * `dropped` is a statement about THIS session: those rows were not
908
- * confirmed delivered before close. An abandoned flush keeps
909
- * retrying and may still land its rows later if the server
910
- * recovers — the count is a floor on uncertainty, not a proof of
911
- * loss.
912
- *
913
- * The transport close is attempted in both loss paths, but on the
914
- * HTTP transport the client's `close()` is an empty function
915
- * (verified against @questdb/nodejs-client 4.2.0), so nothing can
916
- * abort an abandoned flush's retry timers from here; they keep the
917
- * event loop alive. Process exit is the final backstop — in a flow,
918
- * the shutdown manager's `SHUTDOWN_FORCE_TIMEOUT_MS` exit covers
919
- * this. No timeout supplied = no enforcement (unbounded await),
920
- * preserving direct-caller behavior.
921
- */
922
- const doShutdown = async function ( timeout ) {
923
- // Flip the health flag first so any concurrent getHealth() call
924
- // immediately sees the shutdown and returns red/disconnected —
925
- // and write() starts refusing new rows (SHUTTING_DOWN).
926
- shuttingDown = true;
927
-
928
- // Stop idle flush timer
929
- if ( idleFlushTimer ) {
930
- clearInterval( idleFlushTimer );
931
- idleFlushTimer = null;
932
- }
933
-
934
- // Everything delivery still owes: flushes already in flight
935
- // (their rows left the buffer at their call) plus one final
936
- // flush for whatever is still buffered. Each wait records its
937
- // outcome into the tallies below; the mapped promises never
938
- // reject, so the only rejection the race can surface is the
939
- // timeout itself.
940
- let totalRows = 0;
941
- let deliveredRows = 0;
942
- let failedRows = 0;
943
- let firstFailure = null;
944
- const waits = [];
945
-
946
- const awaitDelivery = function ( entry ) {
947
- totalRows += entry.rows;
948
- waits.push( entry.promise.then(
949
- function () {
950
- deliveredRows += entry.rows;
951
- },
952
- function ( err ) {
953
- failedRows += entry.rows;
954
- if ( !firstFailure ) {
955
- firstFailure = err;
956
- }
957
- }
958
- ) );
959
- }; // awaitDelivery()
960
-
961
- inFlightFlushes.forEach( awaitDelivery );
962
- if ( bufferedRows > 0 ) {
963
- const entry = trackFlush( sender.flush(), bufferedRows );
964
- bufferedRows = 0;
965
- awaitDelivery( entry );
966
- }
967
-
968
- if ( waits.length > 0 ) {
969
- try {
970
- await raceFlushTimeout( Promise.all( waits ), timeout );
971
- } catch ( err ) {
972
- await closeQuietly();
973
- const dropped = totalRows - deliveredRows;
974
- const timedOut = new Error(
975
- `winkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
976
- );
977
- timedOut.code = 'SHUTDOWN_TIMEOUT';
978
- timedOut.dropped = { count: dropped };
979
- throw timedOut;
980
- }
981
-
982
- if ( failedRows > 0 ) {
983
- await closeQuietly();
984
- const failure = new Error(
985
- `winkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
986
- );
987
- failure.code = 'DELIVERY_FAILED';
988
- failure.dropped = { count: failedRows };
989
- failure.cause = firstFailure;
990
- throw failure;
991
- }
992
- }
993
-
994
- // Close sender
995
- await sender.close();
996
- }; // doShutdown()
997
-
998
- /**
999
- * Shutdown the storage adapter gracefully.
1000
- * Flushes pending data and closes connections.
1001
- *
1002
- * Called by wire-storages.shutdown() which is invoked during:
1003
- * - Pipeline shutdown (flowHandle.shutdown())
1004
- * - Process signal handlers (SIGINT, SIGTERM)
1005
- *
1006
- * The outcome is latched: the first call runs the shutdown, every
1007
- * later call returns the same promise. A lossy shutdown's failed
1008
- * flush already emptied the buffer (copy-out), so a re-run would
1009
- * find nothing to flush and resolve clean, contradicting the
1010
- * recorded loss. One consequence: the first caller's `{ timeout }`
1011
- * governs; a later caller's is ignored.
1012
- *
1013
- * @param {{timeout?: number}} [options]
1014
- * @returns {Promise<void>}
1015
- */
1016
- const shutdown = function ( { timeout = 0 } = {} ) {
1017
- if ( !shutdownPromise ) {
1018
- shutdownPromise = doShutdown( timeout );
1019
- }
1020
- return shutdownPromise;
1021
- }; // shutdown()
679
+ // Everything after setup is the engine's: the hot path, the
680
+ // flushes, the counters, the drain. The transport close stays the
681
+ // factory's, because the factory owns the agent.
682
+ const engine = createFlushEngine( { sender, persistPlans, settings, onDeliveryFailure, probe, closeTransport } );
1022
683
 
1023
684
  return {
1024
- write,
1025
- flush,
1026
- shutdown,
1027
- getPressure,
1028
- getHealth,
685
+ write: engine.write,
686
+ flush: engine.flush,
687
+ shutdown: engine.shutdown,
688
+ getPressure: engine.getPressure,
689
+ getHealth: engine.getHealth,
1029
690
  // Expose for testing/debugging
1030
691
  _sender: sender,
1031
692
  _persistPlans: persistPlans
1032
693
  };
1033
- };
694
+ }; // createQuestDBStorage()
1034
695
 
1035
696
  // ============================================================================
1036
697
  // CONFIG SCHEMA
@@ -1060,65 +721,116 @@ const configSchema = {
1060
721
  'idleFlushCheckMs',
1061
722
  'autoFlushRows',
1062
723
  'autoFlushIntervalMs',
1063
- 'maxBufSize',
724
+ 'flushRows',
725
+ 'flushIntervalMs',
726
+ 'bufferCeilingRows',
727
+ 'flushDeadlineMs',
728
+ 'stdlibHttp',
729
+ 'requestTimeout',
1064
730
  'retryTimeout',
731
+ 'initBufSize',
732
+ 'maxBufSize',
1065
733
  'partitionBy',
1066
734
  'onWarning',
1067
735
  'onDeliveryFailure'
1068
736
  ],
737
+ // Both addresses refuse `localhost` at flow definition (ADR-030).
738
+ // The non-empty rule lives inside the validator, so one static
739
+ // error string covers every refusal.
1069
740
  ilpUrl: {
1070
741
  type: 'string',
1071
742
  required: false,
1072
- minLength: 1,
1073
- error: 'ilpUrl must be a non-empty string (e.g., localhost:9000)'
743
+ validator: isAllowedIlpUrl,
744
+ error: 'ilpUrl must be host:port with a literal address or a name, never localhost and never ' +
745
+ 'an IPv6 literal (the QuestDB client cannot read one); e.g., 127.0.0.1:9000'
1074
746
  },
1075
747
  pgUrl: {
1076
748
  type: 'string',
1077
749
  required: false,
1078
- minLength: 1,
1079
- error: 'pgUrl must be a non-empty string (e.g., localhost:8812)'
750
+ validator: isAllowedPgUrl,
751
+ error: 'pgUrl must be host:port with a literal address or a name, never localhost; e.g., 127.0.0.1:8812'
1080
752
  },
1081
753
  tablePrefix: {
1082
754
  type: 'string',
1083
755
  required: false,
1084
756
  minLength: 1,
1085
- error: 'tablePrefix must be a non-empty string (defaults to assetClass.name when omitted)'
757
+ // The prefix opens every table name, written unquoted in CREATE
758
+ // TABLE. See header sub-case (j) for the rule and its sources.
759
+ validator: validators.identifier,
760
+ error: 'tablePrefix must use letters, digits, _ and $ only, and not start with a digit ' +
761
+ '(defaults to assetClass.name when omitted)'
1086
762
  },
763
+ // The five legacy keys (deprecated, ADR-029; removed in 0.8.0). They
764
+ // keep their old validation so an existing flow still passes the
765
+ // schema. The option resolver maps or ignores them and setup prints
766
+ // one DEPRECATED_OPTION line.
1087
767
  flushMode: {
1088
768
  type: 'string',
1089
769
  required: false,
1090
770
  validator: validators.oneOf( [ 'auto', 'manual' ] ),
1091
- error: 'flushMode must be "auto" or "manual"'
771
+ error: 'flushMode must be "auto" or "manual" (deprecated: composer owns every flush)'
1092
772
  },
1093
773
  idleFlushAfterMs: {
1094
774
  type: 'number',
1095
775
  required: false,
1096
776
  validator: validators.positiveInteger,
1097
- error: 'idleFlushAfterMs must be a positive integer'
777
+ error: 'idleFlushAfterMs must be a positive integer (deprecated: ignored)'
1098
778
  },
1099
779
  idleFlushCheckMs: {
1100
780
  type: 'number',
1101
781
  required: false,
1102
782
  validator: validators.positiveInteger,
1103
- error: 'idleFlushCheckMs must be a positive integer'
783
+ error: 'idleFlushCheckMs must be a positive integer (deprecated: use flushIntervalMs)'
1104
784
  },
1105
785
  autoFlushRows: {
1106
786
  type: 'number',
1107
787
  required: false,
1108
788
  validator: validators.positiveInteger,
1109
- error: 'autoFlushRows must be a positive integer'
789
+ error: 'autoFlushRows must be a positive integer (deprecated: use flushRows)'
1110
790
  },
1111
791
  autoFlushIntervalMs: {
1112
792
  type: 'number',
1113
793
  required: false,
1114
794
  validator: validators.positiveInteger,
1115
- error: 'autoFlushIntervalMs must be a positive integer'
795
+ error: 'autoFlushIntervalMs must be a positive integer (deprecated: ignored)'
1116
796
  },
1117
- maxBufSize: {
797
+ // The flush settings composer owns (ADR-029). The relation between
798
+ // the ceiling and the threshold is checked by the option resolver
799
+ // at setup, where both values are known.
800
+ flushRows: {
1118
801
  type: 'number',
1119
802
  required: false,
1120
803
  validator: validators.positiveInteger,
1121
- error: 'maxBufSize must be a positive integer'
804
+ error: 'flushRows must be a positive integer'
805
+ },
806
+ flushIntervalMs: {
807
+ type: 'number',
808
+ required: false,
809
+ validator: validators.positiveInteger,
810
+ error: 'flushIntervalMs must be a positive integer'
811
+ },
812
+ bufferCeilingRows: {
813
+ type: 'number',
814
+ required: false,
815
+ validator: validators.positiveInteger,
816
+ error: 'bufferCeilingRows must be a positive integer'
817
+ },
818
+ flushDeadlineMs: {
819
+ type: 'number',
820
+ required: false,
821
+ validator: validators.positiveInteger,
822
+ error: 'flushDeadlineMs must be a positive integer'
823
+ },
824
+ stdlibHttp: {
825
+ type: 'boolean',
826
+ required: false,
827
+ error: 'stdlibHttp must be a boolean'
828
+ },
829
+ requestTimeout: {
830
+ type: 'number',
831
+ required: false,
832
+ validator: validators.positiveInteger,
833
+ error: 'requestTimeout must be a positive integer'
1122
834
  },
1123
835
  retryTimeout: {
1124
836
  type: 'number',
@@ -1126,6 +838,18 @@ const configSchema = {
1126
838
  validator: validators.positiveInteger,
1127
839
  error: 'retryTimeout must be a positive integer'
1128
840
  },
841
+ initBufSize: {
842
+ type: 'number',
843
+ required: false,
844
+ validator: validators.positiveInteger,
845
+ error: 'initBufSize must be a positive integer'
846
+ },
847
+ maxBufSize: {
848
+ type: 'number',
849
+ required: false,
850
+ validator: validators.positiveInteger,
851
+ error: 'maxBufSize must be a positive integer'
852
+ },
1129
853
  partitionBy: {
1130
854
  type: 'string',
1131
855
  required: false,
@@ -1166,23 +890,24 @@ const durabilityClass = 'in-memory';
1166
890
 
1167
891
  // Capability declaration per ADR-018.
1168
892
  //
1169
- // QuestDB needs the asset class because (a) ensure-tables.js reads
1170
- // `columns` and `insightTypes` to issue CREATE TABLE statements, and
1171
- // (b) persist-plan.js reads the same to dispatch column writers and
893
+ // QuestDB needs the asset class for two reasons. `ensure-tables.js`
894
+ // reads `columns` and `insightTypes` to issue CREATE TABLE statements.
895
+ // `persist-plan.js` reads the same to dispatch column writers and
1172
896
  // pre-compile resolution-aware quantization. `name` drives the default
1173
897
  // `tablePrefix` when the caller did not pass one.
1174
898
  //
1175
899
  // Declared `required: true` because there is no useful behaviour for
1176
- // QuestDB without an assetClass — the wiring layer fails fast with
1177
- // MISSING_ASSET_CLASS rather than calling `createStorage` and letting
900
+ // QuestDB without an assetClass. The wiring layer fails fast with
901
+ // MISSING_ASSET_CLASS instead of calling `createStorage` and letting
1178
902
  // the failure surface deeper.
1179
903
  //
1180
904
  // `fields` lists exactly the top-level fields read. The slicing is
1181
- // top-level only; column-internal fields (`type`, `resolution`) are
905
+ // top-level only. Column-internal fields (`type`, `resolution`) are
1182
906
  // read directly from the `columns` slice and validated by Layer 2
1183
- // assertions inside `createStorage` below. A declarative shape for
1184
- // column-internal capability is deferred until a second adapter
1185
- // needs it.
907
+ // assertions inside `createStorage` below.
908
+ //
909
+ // A declarative shape for column-internal capability is deferred
910
+ // until a second adapter needs it.
1186
911
  const semanticsRequirement = {
1187
912
  assetClass: {
1188
913
  required: true,
@@ -1196,17 +921,20 @@ const semanticsRequirement = {
1196
921
  * @param {Object} config - Configuration object
1197
922
  * @param {Object} config.assetClass - Asset class definition with columns and insightTypes
1198
923
  * @param {string} config.tablePrefix - Prefix for table names (defaults to assetClass.name)
1199
- * @param {string} config.ilpUrl - ILP endpoint (host:port)
1200
- * @param {string} config.pgUrl - PostgreSQL endpoint (host:port)
1201
- * @param {string} [config.flushMode='auto'] - 'auto' or 'manual'
1202
- * @param {number} [config.idleFlushAfterMs=5000] - Idle time before flush (manual mode)
1203
- * @param {number} [config.idleFlushCheckMs=1000] - Idle check interval (manual mode)
1204
- * @param {number} [config.autoFlushRows] - Rows before auto-flush (auto mode)
1205
- * @param {number} [config.autoFlushIntervalMs] - Time interval for auto-flush (auto mode)
1206
- * @param {number} [config.maxBufSize] - Maximum buffer size
1207
- * @param {number} [config.retryTimeout] - Retry timeout in ms
924
+ * @param {string} [config.ilpUrl] - ILP endpoint (host:port)
925
+ * @param {string} [config.pgUrl] - PostgreSQL endpoint (host:port)
926
+ * @param {number} [config.flushRows=5000] - Rows that start a flush from inside write()
927
+ * @param {number} [config.flushIntervalMs=1000] - Period of the flush timer
928
+ * @param {number} [config.bufferCeilingRows] - Most rows held; default ten times flushRows
929
+ * @param {number} [config.flushDeadlineMs] - Fixed deadline per flush
930
+ * @param {boolean} [config.stdlibHttp=true] - The standard-library transport; false selects undici
931
+ * @param {number} [config.requestTimeout] - Client request timeout in ms
932
+ * @param {number} [config.retryTimeout] - Client retry window in ms
933
+ * @param {number} [config.initBufSize] - Initial client buffer size in bytes
934
+ * @param {number} [config.maxBufSize] - Byte ceiling of the client buffer
1208
935
  * @param {string} [config.partitionBy='DAY'] - Table partition interval
1209
- * @param {function} [config.onWarning] - Warning callback for null column values
936
+ * @param {function} [config.onWarning] - Warning callback for skipped values
937
+ * @param {function} [config.onDeliveryFailure] - Called once per lost flush
1210
938
  * @param {Object} [config._deps] - Injectable dependencies (for testing)
1211
939
  * @returns {Promise<Object>} Storage instance with write, flush, shutdown methods
1212
940
  */
@@ -1228,15 +956,14 @@ const createStorage = function ( config ) {
1228
956
  }
1229
957
 
1230
958
  // Adapter-side defensive validation of column-internal facts
1231
- // QuestDB consumes. Per the ADR-018 column-internal facts
1232
- // pattern: the universal semantics
1233
- // schema validates top-level
1234
- // structure (type required, type in COLUMN_TYPES) at load and
1235
- // at .assetClass() time; this layer catches bypasses (e.g.,
1236
- // direct createStorage call from test code) and adapter-specific
1237
- // requirements like "float64 columns with declared resolution
1238
- // must have a positive value." See `assert-columns.js` for the
1239
- // full rationale.
959
+ // QuestDB consumes (the ADR-018 column-internal facts pattern). The
960
+ // universal semantics schema validates the top-level structure at
961
+ // load and at .assetClass() time: type required, type in
962
+ // COLUMN_TYPES. This layer catches bypasses, such as a direct
963
+ // createStorage call from test code. It also checks adapter-specific
964
+ // requirements, such as "a float64 column with a declared resolution
965
+ // must have a positive value". See `assert-columns.js` for the full
966
+ // rationale.
1240
967
  assertColumnFacts( assetClass );
1241
968
 
1242
969
  // Default `tablePrefix` to the asset class name when the caller
@@ -1264,8 +991,8 @@ const createStorage = function ( config ) {
1264
991
  * .assetClass( assetClassDef ) // wire-storages injects the slice
1265
992
  * .storage( questdbAdapter, {
1266
993
  * tablePrefix: 'myPrefix',
1267
- * ilpUrl: 'localhost:9000',
1268
- * pgUrl: 'localhost:8812'
994
+ * ilpUrl: '127.0.0.1:9000',
995
+ * pgUrl: '127.0.0.1:8812'
1269
996
  * } )
1270
997
  *
1271
998
  * @type {Object}