@winkjs/composer 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +2 -2
- package/src/composer.js +1 -1
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +103 -658
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/index.js +16 -1
- package/src/core/env-vars.js +92 -24
- package/src/core/logger/index.js +1 -1
- package/src/core/source-manager/mqtt/client.js +197 -63
- package/src/core/source-manager/mqtt/dedup.js +16 -11
- package/src/core/source-manager/mqtt/index.js +32 -5
- package/src/core/source-manager/mqtt/status.js +168 -56
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +542 -862
- package/src/core/storage-manager/questdb/persist-plan.js +128 -167
- package/src/core/storage-manager/questdb/resolve-options.js +266 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +23 -2
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/wiring/wire-emitters.js +3 -1
- package/src/core/wiring/wire-storages.js +10 -2
- package/src/flow/flow.js +24 -3
- package/src/flow/run.js +24 -2
|
@@ -1,64 +1,135 @@
|
|
|
1
1
|
// core/storage-manager/questdb/index.js
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @fileoverview QuestDB storage adapter
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* -
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* -
|
|
30
|
-
*
|
|
31
|
-
* - `getPressure()
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
4
|
+
* @fileoverview QuestDB storage adapter. It writes rows to QuestDB over
|
|
5
|
+
* ILP (InfluxDB Line Protocol) on HTTP, and creates the tables over the
|
|
6
|
+
* PostgreSQL wire protocol at startup.
|
|
7
|
+
*
|
|
8
|
+
* Contract role (ADR-018): storage, buffered. Startup posture: eager.
|
|
9
|
+
* Setup throws when either endpoint does not answer, so a flow never
|
|
10
|
+
* starts against a dead database.
|
|
11
|
+
*
|
|
12
|
+
* Module layout. This file holds the factory, the config schema, and
|
|
13
|
+
* the adapter export. The work lives in four modules next to it:
|
|
14
|
+
* - `resolve-options.js` merges the config and the environment into
|
|
15
|
+
* the settings, and derives the ceiling and the deadline.
|
|
16
|
+
* - `address-policy.js` holds the ADR-030 address checks, the setup
|
|
17
|
+
* probe wrapper, and the classification of a connect failure.
|
|
18
|
+
* - `flush-engine.js` owns the hot path, every flush, the counters
|
|
19
|
+
* behind pressure and health, and the shutdown drain (ADR-029).
|
|
20
|
+
* - `persist-plan.js` compiles one row writer per insight type.
|
|
21
|
+
*
|
|
22
|
+
* The handle the factory returns:
|
|
23
|
+
* - `write(insightType, msg, partitionId)`: the hot path. Appends one
|
|
24
|
+
* row to the client's buffer and returns at once. `{ ok: true }`
|
|
25
|
+
* means accepted into the buffer, not delivered. `{ ok: false,
|
|
26
|
+
* error: { code, message } }` names the refusal. Never throws.
|
|
27
|
+
* - `flush()`: sends everything buffered now. Resolves when the server
|
|
28
|
+
* confirmed the rows, rejects when it did not.
|
|
29
|
+
* - `shutdown({ timeout })`: drain, then close. A clean resolve means
|
|
30
|
+
* every row was delivered. A loss is a classified rejection.
|
|
31
|
+
* - `getPressure()`: buffer fill in [0, 1]. Sync, O(1), no allocation.
|
|
32
|
+
* - `getHealth()`: `{ status, connected, pressure, ... }`.
|
|
33
|
+
* `flush-engine.js` documents how each one behaves.
|
|
34
|
+
*
|
|
35
|
+
* Composer owns every flush (ADR-029). The client's own trigger is
|
|
36
|
+
* off (`auto_flush=off`), so a batch leaves the process only when the
|
|
37
|
+
* engine says so. A write that brings the buffer to `flushRows` starts
|
|
38
|
+
* a flush at once. A timer starts a flush every `flushIntervalMs` when
|
|
39
|
+
* anything is buffered. Only one engine flush runs at a time. A failed
|
|
40
|
+
* flush is reported once, with the exact rows lost, and the process
|
|
41
|
+
* keeps running. Every flush has a deadline, derived from the rows it
|
|
42
|
+
* carries, and a flush past it is abandoned and reported. A failed or
|
|
43
|
+
* abandoned flush runs the ADR-030 probe. While the probe fails,
|
|
44
|
+
* delivery pauses: rows are held up to the ceiling and each tick
|
|
45
|
+
* probes again. `flush-engine.js` carries the detail.
|
|
46
|
+
*
|
|
47
|
+
* Row order. Rows reach QuestDB in write order. The engine runs one
|
|
48
|
+
* flush at a time, each flush carries its rows in write order, and
|
|
49
|
+
* the standard-library transport keeps one socket (`maxSockets: 1`),
|
|
50
|
+
* so a later request waits for the earlier one. Two sends can be
|
|
51
|
+
* outstanding only at shutdown, when the drain's final flush starts
|
|
52
|
+
* while an earlier flush is still on the wire. On the undici
|
|
53
|
+
* transport those two batches can land out of order. QuestDB orders
|
|
54
|
+
* each table by its designated timestamp, so a query never sees the
|
|
55
|
+
* difference.
|
|
56
|
+
*
|
|
57
|
+
* Durability in plain words (ADR-018). `durabilityClass` is
|
|
58
|
+
* `'in-memory'`. Rows live in the client's buffer until a flush reaches
|
|
59
|
+
* the server. Three things lose rows: a process crash loses everything
|
|
60
|
+
* buffered or in flight, a failed flush loses its batch, and the
|
|
61
|
+
* ceiling sheds new rows. The shed policy: once the rows buffered plus
|
|
62
|
+
* the rows in flight reach `bufferCeilingRows`, `write()` refuses new
|
|
63
|
+
* rows with `STORAGE_FULL` and the rows already accepted stay safe.
|
|
64
|
+
* Outage budget with the defaults: the ceiling of 50,000 rows holds
|
|
65
|
+
* 50 seconds at 1,000 rows a second and about 14 hours at one row a
|
|
66
|
+
* second, in 6.5 to 15 MB of memory at the measured 130 to 290 bytes
|
|
67
|
+
* a row.
|
|
68
|
+
*
|
|
69
|
+
* Edge-baseline defaults (every one runs within a 2 GB device):
|
|
70
|
+
*
|
|
71
|
+
* | Setting | Default | Environment variable |
|
|
72
|
+
* |---------------------|------------------|------------------------------|
|
|
73
|
+
* | `flushRows` | 5000 | `QUESTDB_FLUSH_ROWS` |
|
|
74
|
+
* | `flushIntervalMs` | 1000 | `QUESTDB_FLUSH_INTERVAL_MS` |
|
|
75
|
+
* | `bufferCeilingRows` | 10 × `flushRows` | `QUESTDB_BUFFER_CEILING_ROWS`|
|
|
76
|
+
* | `flushDeadlineMs` | derived per flush| `QUESTDB_FLUSH_DEADLINE_MS` |
|
|
77
|
+
* | `stdlibHttp` | true | `QUESTDB_STDLIB_HTTP` |
|
|
78
|
+
* | `requestTimeout` | client default | `QUESTDB_REQUEST_TIMEOUT` |
|
|
79
|
+
* | `retryTimeout` | client default | `QUESTDB_RETRY_TIMEOUT` |
|
|
80
|
+
* | `initBufSize` | client default | `QUESTDB_INIT_BUF_SIZE` |
|
|
81
|
+
* | `maxBufSize` | client default | `QUESTDB_MAX_BUF_SIZE` |
|
|
82
|
+
*
|
|
83
|
+
* `resolve-options.js` merges the config and the environment and
|
|
84
|
+
* explains each default. `ilpUrl` and `pgUrl` fall back to
|
|
85
|
+
* `QUESTDB_ILP_URL` and `QUESTDB_PG_URL`; the credentials come from
|
|
86
|
+
* `QUESTDB_DATABASE`, `QUESTDB_USER` and `QUESTDB_PASSWORD`.
|
|
87
|
+
*
|
|
88
|
+
* Long-running commitments (ADR-018 §12). One timer, created at setup
|
|
89
|
+
* and cleared at shutdown; it does not hold the process open. Every
|
|
90
|
+
* counter is bounded by the ceiling. The per-row path allocates
|
|
91
|
+
* nothing beyond the one derived promise `persist-plan.js` documents.
|
|
92
|
+
* Reconnection and request retries belong to the client. No listeners
|
|
93
|
+
* are attached. `flush-engine.js` carries the detail.
|
|
94
|
+
*
|
|
95
|
+
* The transport (ADR-029). The client's default HTTP library, undici,
|
|
96
|
+
* retries a refused connection without end, and its abort cannot end
|
|
97
|
+
* that retry. So the adapter selects the client's standard-library
|
|
98
|
+
* transport (`stdlib_http=on`) unless `stdlibHttp` is false. With it, a refused connection rejects at once and every
|
|
99
|
+
* request ends within `retryTimeout` plus one request timeout. The
|
|
100
|
+
* adapter owns one `http.Agent` for that transport, keep-alive on one
|
|
101
|
+
* socket, and destroys it after the sender closes on every shutdown
|
|
102
|
+
* path. Destroying the agent ends a request still on its socket, so
|
|
103
|
+
* the process can exit. The agent also closes its socket after 4
|
|
104
|
+
* seconds idle, before QuestDB's own idle close at 5 minutes, so no
|
|
105
|
+
* flush meets a socket the server has just closed. An operator who
|
|
106
|
+
* selects undici accepts that an abandoned flush can keep the process
|
|
107
|
+
* alive.
|
|
108
|
+
*
|
|
109
|
+
* Health (ADR-018 §8). `status` is `red` when not connected or at
|
|
110
|
+
* capacity. It is `yellow` at `pressure >= 0.66`, on an outstanding
|
|
111
|
+
* write error, or after one failed flush. Otherwise it is `green`.
|
|
112
|
+
* `connected` is derived, because the ILP client exposes no socket
|
|
113
|
+
* state. It is false while delivery is paused, after five write errors
|
|
114
|
+
* in a row, after two failed flushes in a row, or after one abandoned
|
|
115
|
+
* flush. The health object carries `pausedSince`, `abandonedFlushes`,
|
|
116
|
+
* `consecutiveFlushFailures`, `lastFlushAt`, and `lastFlushError`.
|
|
117
|
+
* `flush-engine.js` documents the derivation.
|
|
118
|
+
*
|
|
119
|
+
* The five legacy flush keys of 0.7.0 (`flushMode`, `idleFlushAfterMs`,
|
|
120
|
+
* `idleFlushCheckMs`, `autoFlushRows`, `autoFlushIntervalMs`) are gone
|
|
121
|
+
* since 0.8.0 (ADR-029 item 10). The schema refuses each one as an
|
|
122
|
+
* unknown key at flow definition, with `INVALID_CONFIG`. `env-vars.js`
|
|
123
|
+
* refuses their `QUESTDB_*` variables at import. `autoFlushRows`
|
|
124
|
+
* became `flushRows` and `idleFlushCheckMs` became `flushIntervalMs`.
|
|
125
|
+
* The other three have no replacement: composer owns every flush.
|
|
55
126
|
*
|
|
56
127
|
* `err.code` vocabulary (per-adapter; ADR-018 has each adapter document
|
|
57
128
|
* its own codes in this header):
|
|
58
129
|
*
|
|
59
130
|
* Setup-time throws (ADR-018 fail-fast setup):
|
|
60
131
|
* - `INVALID_CONFIG` — the supplied configuration does not
|
|
61
|
-
* work.
|
|
132
|
+
* work. Ten current sub-cases:
|
|
62
133
|
* (a) required transport URL missing (`ilpUrl`, `pgUrl`);
|
|
63
134
|
* (b) PostgreSQL endpoint answered but rejected the connection
|
|
64
135
|
* — wrong credentials or a protocol-level refusal;
|
|
@@ -72,18 +143,57 @@
|
|
|
72
143
|
* of wedging the sender mid-row at runtime;
|
|
73
144
|
* (e) an insightType uses the reserved column name `assetId`.
|
|
74
145
|
* Composer writes that column itself, from the partition id.
|
|
75
|
-
* Checked at plan build in `persist-plan.js
|
|
146
|
+
* Checked at plan build in `persist-plan.js`;
|
|
147
|
+
* (f) `ilpUrl` or `pgUrl` names `localhost` (ADR-030). The name
|
|
148
|
+
* can resolve to two addresses, `::1` and `127.0.0.1`, and
|
|
149
|
+
* QuestDB may listen on only one. Refused before any socket
|
|
150
|
+
* opens, at the schema (flow definition) and here (the env
|
|
151
|
+
* fallback and direct callers). The message names the
|
|
152
|
+
* literal to set;
|
|
153
|
+
* (g) `ilpUrl` is an IPv6 literal. The client (4.2.0) splits the
|
|
154
|
+
* address on its first colon and cannot read one. `pgUrl`
|
|
155
|
+
* accepts `[::1]:8812`;
|
|
156
|
+
* (h) the ILP client rejected the sender configuration for a
|
|
157
|
+
* reason that is not a network error (`fromConfig`);
|
|
158
|
+
* (i) `bufferCeilingRows` is below twice `flushRows` (ADR-029).
|
|
159
|
+
* Rows in flight count against the ceiling, so it must hold
|
|
160
|
+
* one batch in flight and one batch buffering. A smaller
|
|
161
|
+
* ceiling would refuse every row that arrives during a
|
|
162
|
+
* row-triggered flush. Checked in `resolve-options.js`;
|
|
163
|
+
* (j) `tablePrefix` is not an identifier: letters, digits, `_`
|
|
164
|
+
* and `$`, not starting with a digit. The prefix opens every
|
|
165
|
+
* table name, and `ensure-tables.js` writes that name unquoted
|
|
166
|
+
* in CREATE TABLE. QuestDB reads an unquoted name as one
|
|
167
|
+
* token. So a hyphen, a space, a dot, a semicolon or a
|
|
168
|
+
* backtick ends the statement early with "unexpected token"
|
|
169
|
+
* (verified live on 9.4.3). QuestDB's own rule for a quoted
|
|
170
|
+
* name is wider. Its CREATE TABLE reference says a name with
|
|
171
|
+
* a space or a dot must be double-quoted. The server's
|
|
172
|
+
* `TableUtils.isValidTableName` forbids `? , ' " \ / : ( ) +
|
|
173
|
+
* * % ~`, control characters, and a leading or trailing dot
|
|
174
|
+
* or space. Sources: https://questdb.com/docs/query/sql/create-table/
|
|
175
|
+
* and https://questdb.com/docs/connect/compatibility/ilp/advanced-settings/.
|
|
176
|
+
* The identifier rule is the one the semantics schema applies
|
|
177
|
+
* to asset class, insight type and column names, so every
|
|
178
|
+
* part of a table name follows one rule. Checked at the
|
|
179
|
+
* schema and here, which covers the default prefix (the asset
|
|
180
|
+
* class name) and a direct caller.
|
|
76
181
|
* Operator remediation: fix the supplied config or the relevant
|
|
77
|
-
* `QUESTDB_*` env var. The underlying
|
|
78
|
-
*
|
|
79
|
-
* - `TRANSPORT_UNREACHABLE` —
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
182
|
+
* `QUESTDB_*` env var. The underlying error (sub-cases b and h) is
|
|
183
|
+
* preserved on `err.cause` for diagnostics.
|
|
184
|
+
* - `TRANSPORT_UNREACHABLE` — an endpoint did not answer at setup.
|
|
185
|
+
* Three sources. (1) The setup probe (ADR-030 item 4): before any
|
|
186
|
+
* client is built, `pgUrl` and then `ilpUrl` are probed with one
|
|
187
|
+
* TCP connect per address. A name is resolved with Node's default
|
|
188
|
+
* lookup and every address it resolves to must answer; the message
|
|
189
|
+
* lists each address with its result and names the literal to set.
|
|
190
|
+
* (2) The PostgreSQL connect failed with a Node syscall code
|
|
191
|
+
* (`ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, ...). (3) The ILP
|
|
192
|
+
* client could not be built for the same class of reason.
|
|
193
|
+
* Distinct from `INVALID_CONFIG` per the one split the ADR-018
|
|
194
|
+
* error vocabulary mandates: the connection string may be fine —
|
|
195
|
+
* check the network, the firewall, whether QuestDB is running.
|
|
196
|
+
* Underlying error on `err.cause` for (2) and (3).
|
|
87
197
|
* - `MISSING_ASSET_CLASS` — `assetClass` not provided to `createStorage`.
|
|
88
198
|
* Distinct from `INVALID_CONFIG` because operator remediation differs:
|
|
89
199
|
* they need to add `.assetClass(assetClassDef)` to the flow, not edit env
|
|
@@ -91,26 +201,70 @@
|
|
|
91
201
|
* - `SCHEMA_ERROR` — DDL `CREATE TABLE` failed for a reason other
|
|
92
202
|
* than already-exists (see `ensure-tables.js`).
|
|
93
203
|
*
|
|
94
|
-
* Runtime returns:
|
|
95
|
-
* - `INVALID_INSIGHT_TYPE` —
|
|
204
|
+
* Runtime returns (sync, from `write()`):
|
|
205
|
+
* - `INVALID_INSIGHT_TYPE` — the insightType has no persist plan
|
|
96
206
|
* (typically a config typo or asset-class drift).
|
|
97
|
-
* - `
|
|
98
|
-
*
|
|
207
|
+
* - `STORAGE_FULL` — the buffer is at `bufferCeilingRows`. The
|
|
208
|
+
* row was shed. One shared result object; shedding allocates
|
|
209
|
+
* nothing. Remediation: the endpoint is not taking rows; check it.
|
|
210
|
+
* Raise the ceiling only when memory allows.
|
|
211
|
+
* - `SHUTTING_DOWN` — `shutdown()` has been called; no flusher
|
|
212
|
+
* is left for a new row. One shared result object.
|
|
213
|
+
* - `SEND_FAILED` — the persist plan threw while invoking
|
|
214
|
+
* sender methods (type coercion failure, a client throw). The
|
|
215
|
+
* half-written row is cancelled (mid-row recovery, in
|
|
216
|
+
* `flush-engine.js`).
|
|
99
217
|
* - `INVALID_TIMESTAMP` — reserved; today persist-plan handles invalid
|
|
100
218
|
* timestamps via `onWarning + skip-row`. It may later be surfaced as a
|
|
101
219
|
* hard return error.
|
|
102
220
|
*
|
|
103
221
|
* Shutdown-time throws (ADR-018 — a clean shutdown resolve means
|
|
104
222
|
* everything buffered or in flight was delivered; both carry
|
|
105
|
-
* `dropped: { count }`, exact)
|
|
106
|
-
* (idle, recovery) plus one final flush, raced against `{ timeout }`;
|
|
107
|
-
* its outcome is latched so repeated calls cannot contradict it:
|
|
223
|
+
* `dropped: { count }`, exact):
|
|
108
224
|
* - `DELIVERY_FAILED` — a flush failed; first flush error on `cause`.
|
|
109
225
|
* - `SHUTDOWN_TIMEOUT` — delivery did not settle within the caller's
|
|
110
|
-
* `{ timeout }
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
226
|
+
* `{ timeout }`.
|
|
227
|
+
*
|
|
228
|
+
* Runtime console classification (tokens, not `err.code` values):
|
|
229
|
+
* - `DELIVERY_FAILED` — an engine or recovery flush failed or was
|
|
230
|
+
* abandoned at its deadline, or the client refused a row append, and
|
|
231
|
+
* no `onDeliveryFailure` was given. One `logger.error` line per
|
|
232
|
+
* event, carrying the exact rows lost and the client's message. A
|
|
233
|
+
* flush line ends with the probe finding when a probe ran. The
|
|
234
|
+
* process keeps running: an unattended deployment must report a lost
|
|
235
|
+
* batch, not stop on it. The health surface carries the same fact.
|
|
236
|
+
* - `CIRCUIT_OPEN` — delivery paused or resumed (ADR-029 hold and
|
|
237
|
+
* probe). One `logger.warn` line when a failing probe pauses
|
|
238
|
+
* delivery, naming the held rows and the finding. One `logger.warn`
|
|
239
|
+
* line when a tick probe passes and delivery resumes, at warn so a
|
|
240
|
+
* log transport with a warn floor carries the episode end. Nothing
|
|
241
|
+
* per tick. Remediation: the endpoint refused a TCP connect; check
|
|
242
|
+
* that QuestDB is running and reachable. Rows are held up to the
|
|
243
|
+
* ceiling meanwhile, and health reads red with `pausedSince`.
|
|
244
|
+
* - `DELIVERY_HEALTH` — the delivery ladder changed state. One
|
|
245
|
+
* `logger.warn` line when it enters yellow (the first failed flush),
|
|
246
|
+
* one `logger.error` line when it enters red (two failed in a row,
|
|
247
|
+
* one abandoned, or a failing probe pausing delivery, printed just
|
|
248
|
+
* before the `CIRCUIT_OPEN` line), and one `logger.warn` line when
|
|
249
|
+
* it returns to green, naming the episode length and the rows
|
|
250
|
+
* reported lost in it.
|
|
251
|
+
* Nothing while a state persists, whatever the outage length, and
|
|
252
|
+
* the lines print with or without `onDeliveryFailure`. Remediation:
|
|
253
|
+
* the detail is the client's message or the deadline; a red with no
|
|
254
|
+
* `pausedSince` in health is a server that answers and refuses.
|
|
255
|
+
* - `STORAGE_FULL` (lines) — one `logger.warn` line when the first row
|
|
256
|
+
* of an episode is refused at the ceiling, and one when the buffer
|
|
257
|
+
* has room again, with the count refused. The end is found at the
|
|
258
|
+
* interval tick, so its line can lag the first free slot by one
|
|
259
|
+
* interval. Remediation: the endpoint is not taking rows; read the
|
|
260
|
+
* `CIRCUIT_OPEN` and `DELIVERY_HEALTH` lines beside it, and raise
|
|
261
|
+
* `bufferCeilingRows` when a longer outage must be ridden through.
|
|
262
|
+
* - `ADDRESS_IS_NAME` — `ilpUrl` or `pgUrl` is a name other than
|
|
263
|
+
* `localhost`. One `logger.warn` line per field at setup, before any
|
|
264
|
+
* socket opens (ADR-030). A name is allowed and the adapter
|
|
265
|
+
* proceeds. The line tells the operator that only a literal address
|
|
266
|
+
* is immune to a resolver that changes its answer under a running
|
|
267
|
+
* process.
|
|
114
268
|
* - `CALLBACK_FAILED` — the user's `onDeliveryFailure` itself threw or
|
|
115
269
|
* rejected. The shared callback guard contains the fault (ADR-018).
|
|
116
270
|
* The adapter keeps writing and flushing, and each fault becomes one
|
|
@@ -142,31 +296,22 @@
|
|
|
142
296
|
* extend this section and add corresponding assertions in
|
|
143
297
|
* `assert-columns.js`.
|
|
144
298
|
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
* buffer (the copy-out happens synchronously, before any network I/O; the
|
|
162
|
-
* client documents that an unfinished row stays behind), then reset()
|
|
163
|
-
* clears the buffer — at that point holding only the broken stub — and
|
|
164
|
-
* lowers the client's row-in-progress flags.
|
|
165
|
-
* - The client (4.2.0) has no row-cancel API, while its sibling clients do
|
|
166
|
-
* (.NET CancelRow, Rust/C rewind_to_marker, Java recovers automatically).
|
|
167
|
-
* Upstream issue #60 tracks the gap:
|
|
168
|
-
* https://github.com/questdb/nodejs-questdb-client/issues/60
|
|
169
|
-
* When a release ships cancelRow(), recoverSender() becomes that one call.
|
|
299
|
+
* Row append rejections. The client declares `sender.at()` as `async`,
|
|
300
|
+
* so its promise can reject. With the client trigger off, the only
|
|
301
|
+
* rejection left is the client's own byte ceiling (`max_buf_size`,
|
|
302
|
+
* 100 MiB by default): the append throws inside the async function
|
|
303
|
+
* and arrives as a rejection, and the row never completed.
|
|
304
|
+
* `persist-plan.js` attaches one handler per insight type to that
|
|
305
|
+
* promise and routes the failure to `onDeliveryFailure` as
|
|
306
|
+
* `( err, { tableName } )`, or prints one `DELIVERY_FAILED` line. The
|
|
307
|
+
* next write finds the sender holding the unfinished row, throws, and
|
|
308
|
+
* the mid-row recovery in `flush-engine.js` clears it.
|
|
309
|
+
*
|
|
310
|
+
* Client dependency note. `package.json` pins `@questdb/nodejs-client`
|
|
311
|
+
* to `~4.2.0`. `flush-engine.js` lists the five facts of that release
|
|
312
|
+
* the engine relies on; re-verify each on an upgrade. `persist-plan.js`
|
|
313
|
+
* also imports `SenderBufferV1`, an exported but undocumented member,
|
|
314
|
+
* for name validation at setup only.
|
|
170
315
|
*
|
|
171
316
|
* Integration with persist-if node:
|
|
172
317
|
* - persist-if calls: state.storage.write(insightType, msg, partitionId)
|
|
@@ -201,225 +346,198 @@
|
|
|
201
346
|
* @see https://questdb.com/docs/clients/ingest-node/
|
|
202
347
|
* @see docs/architecture/storage-layer.md
|
|
203
348
|
* @see ADR-018
|
|
349
|
+
* @see ADR-029
|
|
204
350
|
*/
|
|
205
351
|
|
|
352
|
+
import http from 'node:http';
|
|
353
|
+
|
|
206
354
|
import { Sender } from '@questdb/nodejs-client';
|
|
207
355
|
import pg from 'pg';
|
|
208
356
|
|
|
209
357
|
import { ENV_VARS } from '../../env-vars.js';
|
|
210
|
-
import { logger } from '../../logger/index.js';
|
|
211
358
|
import { validators } from '../../utils/validate/index.js';
|
|
212
|
-
import {
|
|
359
|
+
import { probeAddress, describeProbe } from '../../utils/address/probe.js';
|
|
213
360
|
import { buildPersistPlans } from './persist-plan.js';
|
|
214
361
|
import { ensureTables } from './ensure-tables.js';
|
|
215
362
|
import { assertColumnFacts } from './assert-columns.js';
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
/**
|
|
230
|
-
* Console channel for the callback guard: one classified line in this
|
|
231
|
-
* adapter's family. Receives an already-safe detail string, never the
|
|
232
|
-
* raw thrown value.
|
|
233
|
-
*/
|
|
234
|
-
const reportCallbackFault = function ( severity, name, detail ) {
|
|
235
|
-
logger.error(
|
|
236
|
-
`winkComposer/questdb: user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
|
|
237
|
-
);
|
|
238
|
-
}; // reportCallbackFault()
|
|
239
|
-
|
|
240
|
-
// Error results (INVALID_INSIGHT_TYPE, SEND_FAILED) are constructed per-call
|
|
241
|
-
// because each carries dynamic content (the offending insightType name and the
|
|
242
|
-
// underlying err.message respectively). These paths are rare; per-occurrence
|
|
243
|
-
// allocation on errors is acceptable. The RESULT_OK singleton above covers the
|
|
244
|
-
// per-message hot path.
|
|
245
|
-
|
|
246
|
-
/**
|
|
247
|
-
* Health status thresholds. consecutiveWriteErrors crossing the YELLOW
|
|
248
|
-
* threshold elevates status to 'yellow'; crossing the RED threshold flips
|
|
249
|
-
* `connected` to false (and therefore `status` to 'red'). Tuned for "any
|
|
250
|
-
* error is worth flagging" + "sustained errors mean the transport is gone."
|
|
251
|
-
*/
|
|
252
|
-
const HEALTH_ERROR_YELLOW_THRESHOLD = 1;
|
|
253
|
-
const HEALTH_ERROR_RED_THRESHOLD = 5;
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* Pressure threshold above which `status` elevates to at least 'yellow'.
|
|
257
|
-
* The value matches the pressure-aware-yield design (ADR-020, still a
|
|
258
|
-
* Draft). That design proposes 0.66 as the pressure level where the
|
|
259
|
-
* flow would start yielding to let sinks drain; the number still needs
|
|
260
|
-
* benchmark confirmation. Today the yield trigger is time-only (ADR-024), so
|
|
261
|
-
* this alignment is forward-looking, not a description of current yield
|
|
262
|
-
* behaviour. Numeric — the constant exists once here rather than scattered
|
|
263
|
-
* as a magic number.
|
|
264
|
-
*/
|
|
265
|
-
const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
|
|
363
|
+
import { resolveOptions } from './resolve-options.js';
|
|
364
|
+
import {
|
|
365
|
+
isAllowedIlpUrl,
|
|
366
|
+
isAllowedPgUrl,
|
|
367
|
+
assertNotLocalhost,
|
|
368
|
+
assertIlpNotIPv6,
|
|
369
|
+
warnIfName,
|
|
370
|
+
assertReachable,
|
|
371
|
+
pgConnectionTarget,
|
|
372
|
+
NETWORK_ERROR_CODES,
|
|
373
|
+
buildSender
|
|
374
|
+
} from './address-policy.js';
|
|
375
|
+
import { createFlushEngine } from './flush-engine.js';
|
|
266
376
|
|
|
267
377
|
// ============================================================================
|
|
268
378
|
// CONFIGURATION BUILDER
|
|
269
379
|
// ============================================================================
|
|
270
380
|
|
|
271
381
|
/**
|
|
272
|
-
* Build QuestDB Sender configuration
|
|
382
|
+
* Build the QuestDB Sender configuration string. The client's own flush
|
|
383
|
+
* trigger is always off (ADR-029): composer starts every flush itself,
|
|
384
|
+
* so the client never sends a batch this module did not ask for. The
|
|
385
|
+
* transport is stated either way, `stdlib_http=on` unless `stdlibHttp`
|
|
386
|
+
* is false, so a printed config reads the same however it was set.
|
|
273
387
|
*
|
|
274
|
-
* @param {Object} options -
|
|
388
|
+
* @param {Object} options - Resolved settings
|
|
275
389
|
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
276
|
-
* @param {
|
|
277
|
-
* @param {number} [options.
|
|
278
|
-
* @param {number} [options.
|
|
279
|
-
* @param {number} [options.
|
|
280
|
-
* @param {number} [options.retryTimeout] -
|
|
390
|
+
* @param {boolean} [options.stdlibHttp] - False selects undici; anything else the standard library
|
|
391
|
+
* @param {number} [options.initBufSize] - Initial client buffer size in bytes
|
|
392
|
+
* @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
|
|
393
|
+
* @param {number} [options.requestTimeout] - Client request timeout in ms
|
|
394
|
+
* @param {number} [options.retryTimeout] - Client retry window in ms
|
|
281
395
|
* @returns {string} Configuration string for Sender.fromConfig()
|
|
282
396
|
*/
|
|
283
397
|
const buildSenderConfig = function ( options ) {
|
|
284
|
-
const { ilpUrl,
|
|
398
|
+
const { ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout } = options;
|
|
285
399
|
|
|
286
|
-
|
|
287
|
-
let config = `http::addr=${ilpUrl};`;
|
|
400
|
+
let config = `http::addr=${ilpUrl};auto_flush=off;stdlib_http=${( stdlibHttp === false ) ? 'off' : 'on'};`;
|
|
288
401
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
// Use QuestDB's built-in auto-flush
|
|
292
|
-
if ( autoFlushRows !== undefined ) {
|
|
293
|
-
config += `auto_flush_rows=${autoFlushRows};`;
|
|
294
|
-
}
|
|
295
|
-
if ( autoFlushIntervalMs !== undefined ) {
|
|
296
|
-
config += `auto_flush_interval=${autoFlushIntervalMs};`;
|
|
297
|
-
}
|
|
298
|
-
} else {
|
|
299
|
-
// Manual mode: disable auto-flush, we control timing
|
|
300
|
-
config += 'auto_flush=off;';
|
|
402
|
+
if ( initBufSize !== undefined ) {
|
|
403
|
+
config += `init_buf_size=${initBufSize};`;
|
|
301
404
|
}
|
|
302
|
-
|
|
303
|
-
// Optional buffer/retry settings
|
|
304
405
|
if ( maxBufSize !== undefined ) {
|
|
305
|
-
config += `
|
|
406
|
+
config += `max_buf_size=${maxBufSize};`;
|
|
407
|
+
}
|
|
408
|
+
if ( requestTimeout !== undefined ) {
|
|
409
|
+
config += `request_timeout=${requestTimeout};`;
|
|
306
410
|
}
|
|
307
411
|
if ( retryTimeout !== undefined ) {
|
|
308
412
|
config += `retry_timeout=${retryTimeout};`;
|
|
309
413
|
}
|
|
310
414
|
|
|
311
415
|
return config;
|
|
312
|
-
};
|
|
416
|
+
}; // buildSenderConfig()
|
|
313
417
|
|
|
314
418
|
// ============================================================================
|
|
315
|
-
//
|
|
419
|
+
// THE TRANSPORT
|
|
316
420
|
// ============================================================================
|
|
317
421
|
|
|
318
422
|
/**
|
|
319
|
-
*
|
|
320
|
-
*
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
328
|
-
*
|
|
329
|
-
* @returns {Promise} settles with the flush, or rejects SHUTDOWN_TIMEOUT
|
|
423
|
+
* How long an idle socket stays open before the adapter closes it.
|
|
424
|
+
* QuestDB closes an idle connection after 300 seconds and announces no
|
|
425
|
+
* keep-alive timeout, so Node gets no hint. A flush that starts in the
|
|
426
|
+
* moment between the server's close and Node's notice of it writes
|
|
427
|
+
* into a dead socket, and this transport does not retry a reset. That
|
|
428
|
+
* would report one batch lost for an outage that never happened.
|
|
429
|
+
* Closing first removes that moment (ADR-029). The value sits below
|
|
430
|
+
* QuestDB's advertised keep-alive of 5 seconds and below the 5-second
|
|
431
|
+
* timeout of Node's own default agent. Node applies it to free sockets
|
|
432
|
+
* only; a request in flight keeps the client's own request timeout.
|
|
330
433
|
*/
|
|
331
|
-
const
|
|
332
|
-
if ( !( timeoutMs > 0 ) ) {
|
|
333
|
-
return flushPromise;
|
|
334
|
-
}
|
|
335
|
-
return new Promise( function ( resolve, reject ) {
|
|
336
|
-
const timer = setTimeout( function () {
|
|
337
|
-
const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
|
|
338
|
-
err.code = 'SHUTDOWN_TIMEOUT';
|
|
339
|
-
reject( err );
|
|
340
|
-
}, timeoutMs );
|
|
341
|
-
timer.unref();
|
|
342
|
-
flushPromise.then(
|
|
343
|
-
function ( value ) {
|
|
344
|
-
clearTimeout( timer );
|
|
345
|
-
resolve( value );
|
|
346
|
-
},
|
|
434
|
+
const IDLE_SOCKET_TIMEOUT_MS = 4000;
|
|
347
435
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
/* c8 ignore stop */
|
|
362
|
-
);
|
|
363
|
-
} );
|
|
364
|
-
}; // raceFlushTimeout()
|
|
436
|
+
/**
|
|
437
|
+
* Builds the one HTTP agent the standard-library transport uses. One
|
|
438
|
+
* socket is enough, because composer starts one flush at a time
|
|
439
|
+
* (ADR-029). Keep-alive reuses that socket from flush to flush, and
|
|
440
|
+
* the adapter closes it after `IDLE_SOCKET_TIMEOUT_MS` without a flush.
|
|
441
|
+
* The adapter destroys the agent at shutdown, which ends any request
|
|
442
|
+
* still on the socket.
|
|
443
|
+
*
|
|
444
|
+
* @returns {http.Agent} The agent
|
|
445
|
+
*/
|
|
446
|
+
const createKeepAliveAgent = function () {
|
|
447
|
+
return new http.Agent( { keepAlive: true, maxSockets: 1, timeout: IDLE_SOCKET_TIMEOUT_MS } );
|
|
448
|
+
}; // createKeepAliveAgent()
|
|
365
449
|
|
|
366
450
|
/**
|
|
367
|
-
*
|
|
368
|
-
*
|
|
369
|
-
*
|
|
370
|
-
*
|
|
371
|
-
*
|
|
372
|
-
*
|
|
373
|
-
*
|
|
451
|
+
* Opens the ILP transport: the agent when the standard-library
|
|
452
|
+
* transport is selected, then the sender. Returns the sender and the
|
|
453
|
+
* function that closes both at the end of the drain. A failed sender
|
|
454
|
+
* build destroys the agent before the classified error goes on, so no
|
|
455
|
+
* agent outlives a failed setup.
|
|
456
|
+
*
|
|
457
|
+
* `closeTransport` closes the sender first, then destroys the agent
|
|
458
|
+
* whatever the sender's close did. Destroying the agent ends any
|
|
459
|
+
* request still on its socket, so the process can exit (ADR-029). With
|
|
460
|
+
* undici there is no agent, and `closeTransport` is the sender's close.
|
|
461
|
+
*
|
|
462
|
+
* @param {Object} parts - The transport inputs
|
|
463
|
+
* @param {Object} parts.SenderClass - The client's Sender class
|
|
464
|
+
* @param {string} parts.senderConfig - The sender configuration string
|
|
465
|
+
* @param {string} parts.ilpUrl - The configured `ilpUrl`, for messages
|
|
466
|
+
* @param {boolean} parts.stdlibHttp - Whether the standard-library transport is selected
|
|
467
|
+
* @param {function} parts.createAgent - Builds the agent
|
|
468
|
+
* @returns {Promise<{sender: Object, closeTransport: function}>} The sender and its close
|
|
469
|
+
* @throws {Error} TRANSPORT_UNREACHABLE or INVALID_CONFIG from `buildSender`
|
|
374
470
|
*/
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
471
|
+
const openTransport = async function ( { SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent } ) {
|
|
472
|
+
if ( stdlibHttp === false ) {
|
|
473
|
+
const sender = await buildSender( SenderClass, senderConfig, ilpUrl, null );
|
|
474
|
+
const closeTransport = function () {
|
|
475
|
+
return sender.close();
|
|
476
|
+
}; // closeTransport()
|
|
477
|
+
return { sender, closeTransport };
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
const agent = createAgent();
|
|
481
|
+
let sender;
|
|
482
|
+
try {
|
|
483
|
+
sender = await buildSender( SenderClass, senderConfig, ilpUrl, agent );
|
|
484
|
+
} catch ( err ) {
|
|
485
|
+
agent.destroy();
|
|
486
|
+
throw err;
|
|
487
|
+
}
|
|
488
|
+
const closeTransport = async function () {
|
|
489
|
+
try {
|
|
490
|
+
await sender.close();
|
|
491
|
+
} finally {
|
|
492
|
+
agent.destroy();
|
|
493
|
+
}
|
|
494
|
+
}; // closeTransport()
|
|
495
|
+
return { sender, closeTransport };
|
|
496
|
+
}; // openTransport()
|
|
497
|
+
|
|
498
|
+
// ============================================================================
|
|
499
|
+
// STORAGE FACTORY
|
|
500
|
+
// ============================================================================
|
|
385
501
|
|
|
386
502
|
/**
|
|
387
503
|
* Create QuestDB storage adapter.
|
|
388
504
|
*
|
|
505
|
+
* The options are resolved by `resolve-options.js` (config over
|
|
506
|
+
* environment over default). Only the keys that reach the engine are
|
|
507
|
+
* listed here; the resolver documents the rest.
|
|
508
|
+
*
|
|
389
509
|
* @param {Object} assetClass - Asset class definition with columns and insightTypes
|
|
390
510
|
* @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
|
|
391
511
|
* @param {Object} options - Configuration options
|
|
392
|
-
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
393
|
-
* @param {string} options.pgUrl - PostgreSQL endpoint (host:port)
|
|
394
|
-
* @param {
|
|
395
|
-
* @param {number} [options.
|
|
396
|
-
* @param {number} [options.
|
|
397
|
-
* @param {number} [options.
|
|
398
|
-
* @param {
|
|
399
|
-
* @param {number} [options.
|
|
400
|
-
* @param {number} [options.retryTimeout] -
|
|
512
|
+
* @param {string} [options.ilpUrl] - ILP endpoint (host:port)
|
|
513
|
+
* @param {string} [options.pgUrl] - PostgreSQL endpoint (host:port)
|
|
514
|
+
* @param {number} [options.flushRows=5000] - Rows that start a flush from inside write()
|
|
515
|
+
* @param {number} [options.flushIntervalMs=1000] - Period of the flush timer
|
|
516
|
+
* @param {number} [options.bufferCeilingRows] - Most rows held; default ten times flushRows
|
|
517
|
+
* @param {number} [options.flushDeadlineMs] - Fixed deadline per flush; derived from the rows when unset
|
|
518
|
+
* @param {boolean} [options.stdlibHttp=true] - The standard-library transport; false selects undici
|
|
519
|
+
* @param {number} [options.requestTimeout] - Client request timeout in ms
|
|
520
|
+
* @param {number} [options.retryTimeout] - Client retry window in ms
|
|
521
|
+
* @param {number} [options.initBufSize] - Initial client buffer size in bytes
|
|
522
|
+
* @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
|
|
401
523
|
* @param {string} [options.partitionBy='DAY'] - Table partition interval
|
|
402
|
-
* @param {function} [options.onWarning] - Warning callback for
|
|
524
|
+
* @param {function} [options.onWarning] - Warning callback for skipped values
|
|
525
|
+
* @param {function} [options.onDeliveryFailure] - Called once per lost flush
|
|
403
526
|
* @param {Object} [deps={}] - Injectable dependencies (for testing)
|
|
404
527
|
* @param {Object} [deps.SenderClass] - QuestDB Sender class (default: @questdb/nodejs-client Sender)
|
|
405
528
|
* @param {Object} [deps.PgClientClass] - PostgreSQL Client class (default: pg.Client)
|
|
406
|
-
* @
|
|
529
|
+
* @param {function} [deps.probeFn] - Setup probe (default: `probeAddress`, ADR-030)
|
|
530
|
+
* @param {function} [deps.createAgent] - Builds the transport agent (default: `createKeepAliveAgent`)
|
|
531
|
+
* @returns {Promise<Object>} Storage adapter with write, flush, shutdown, getPressure, getHealth
|
|
407
532
|
*/
|
|
408
533
|
const createQuestDBStorage = async function ( assetClass, tablePrefix, options, deps = {} ) {
|
|
534
|
+
// Options become settings here, once (ADR-029). A ceiling below the
|
|
535
|
+
// threshold fails setup inside the resolver with INVALID_CONFIG.
|
|
536
|
+
const settings = resolveOptions( options, ENV_VARS );
|
|
409
537
|
const {
|
|
410
|
-
ilpUrl
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
idleFlushAfterMs = ENV_VARS.questdbIdleFlushAfterMs,
|
|
414
|
-
idleFlushCheckMs = ENV_VARS.questdbIdleFlushCheckMs,
|
|
415
|
-
autoFlushRows = ENV_VARS.questdbAutoFlushRows,
|
|
416
|
-
autoFlushIntervalMs = ENV_VARS.questdbAutoFlushIntervalMs,
|
|
417
|
-
maxBufSize = ENV_VARS.questdbMaxBufSize,
|
|
418
|
-
retryTimeout = ENV_VARS.questdbRetryTimeout,
|
|
419
|
-
partitionBy = 'DAY',
|
|
420
|
-
onWarning,
|
|
421
|
-
onDeliveryFailure
|
|
422
|
-
} = options;
|
|
538
|
+
ilpUrl, pgUrl, stdlibHttp, requestTimeout, retryTimeout, initBufSize, maxBufSize,
|
|
539
|
+
partitionBy, onWarning, onDeliveryFailure
|
|
540
|
+
} = settings;
|
|
423
541
|
|
|
424
542
|
// Runtime validation — required from either DSL config or ENV_VARS.
|
|
425
543
|
// Per ADR-018, setup-time throws carry classified err.code.
|
|
@@ -434,35 +552,56 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
434
552
|
throw err;
|
|
435
553
|
}
|
|
436
554
|
|
|
555
|
+
// The prefix opens every table name, and CREATE TABLE writes that
|
|
556
|
+
// name unquoted (header sub-case (j)). The schema checks a flow's
|
|
557
|
+
// config. This check covers a direct caller and the default, which
|
|
558
|
+
// is the asset class name.
|
|
559
|
+
if ( !validators.identifier( tablePrefix ) ) {
|
|
560
|
+
const err = new Error(
|
|
561
|
+
`winkComposer/questdb: tablePrefix '${tablePrefix}' is not a valid table name prefix [INVALID_CONFIG]: ` +
|
|
562
|
+
'use letters, digits, _ and $ only, and do not start with a digit'
|
|
563
|
+
);
|
|
564
|
+
err.code = 'INVALID_CONFIG';
|
|
565
|
+
throw err;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// Address policy (ADR-030), before any socket opens. `localhost` is
|
|
569
|
+
// refused. An IPv6 literal is refused for the ILP path. A name gets
|
|
570
|
+
// one warning per field. See header sub-cases (f), (g) and the
|
|
571
|
+
// ADDRESS_IS_NAME console token.
|
|
572
|
+
const ilpAddress = assertNotLocalhost( 'ilpUrl', ilpUrl, 'QUESTDB_ILP_URL' );
|
|
573
|
+
const pgAddress = assertNotLocalhost( 'pgUrl', pgUrl, 'QUESTDB_PG_URL' );
|
|
574
|
+
assertIlpNotIPv6( ilpAddress );
|
|
575
|
+
warnIfName( 'ilpUrl', ilpAddress );
|
|
576
|
+
warnIfName( 'pgUrl', pgAddress );
|
|
577
|
+
|
|
437
578
|
// Injectable dependencies with defaults
|
|
438
579
|
const {
|
|
439
580
|
SenderClass = Sender,
|
|
440
|
-
PgClientClass = pg.Client
|
|
581
|
+
PgClientClass = pg.Client,
|
|
582
|
+
probeFn = probeAddress,
|
|
583
|
+
createAgent = createKeepAliveAgent
|
|
441
584
|
} = deps;
|
|
442
585
|
|
|
443
586
|
// Build persist plans (pre-compiled closures). The callbacks go in
|
|
444
587
|
// RAW: buildPersistPlans validates them itself and arms its own
|
|
445
|
-
// guarded copy for the
|
|
588
|
+
// guarded copy for the row-append site. Handing it a pre-wrapped
|
|
446
589
|
// function would defeat that validation (the guard turns a
|
|
447
590
|
// non-function into null instead of the fail-fast INVALID_CONFIG).
|
|
591
|
+
// The engine wraps its own copy after this validation (ADR-027:
|
|
592
|
+
// wrap once, after the site's own validation).
|
|
448
593
|
const persistPlans = buildPersistPlans( assetClass, tablePrefix, { onWarning, onDeliveryFailure } );
|
|
449
594
|
|
|
450
|
-
//
|
|
451
|
-
//
|
|
452
|
-
//
|
|
453
|
-
|
|
454
|
-
// (ADR-018) and its cost stays inside the handler. Absent stays
|
|
455
|
-
// null, so the no-handler console/throw fallbacks keep their
|
|
456
|
-
// meaning.
|
|
457
|
-
const safeOnDeliveryFailure = wrapCallback( onDeliveryFailure, {
|
|
458
|
-
name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
|
|
459
|
-
} );
|
|
595
|
+
// Setup probe, PostgreSQL side (ADR-030 item 4): every address the
|
|
596
|
+
// endpoint resolves to must answer before the client opens. The
|
|
597
|
+
// ILP side is probed below, before the sender is built.
|
|
598
|
+
await assertReachable( 'pgUrl', pgAddress, probeFn );
|
|
460
599
|
|
|
461
600
|
// Ensure tables exist via PostgreSQL wire protocol
|
|
462
|
-
const
|
|
601
|
+
const { host: pgHost, port: pgPort } = pgConnectionTarget( pgUrl, pgAddress );
|
|
463
602
|
const pgClient = new PgClientClass( {
|
|
464
603
|
host: pgHost,
|
|
465
|
-
port:
|
|
604
|
+
port: pgPort,
|
|
466
605
|
database: ENV_VARS.questdbDatabase,
|
|
467
606
|
user: ENV_VARS.questdbUser,
|
|
468
607
|
password: ENV_VARS.questdbPassword
|
|
@@ -470,15 +609,16 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
470
609
|
|
|
471
610
|
// Wrap the PostgreSQL connect so the failure carries a classified
|
|
472
611
|
// `err.code` an operator can route on. The ADR-018 error vocabulary
|
|
473
|
-
// mandates one split here
|
|
474
|
-
// code like ECONNREFUSED
|
|
475
|
-
//
|
|
476
|
-
// network, the firewall, whether the service is
|
|
477
|
-
// connection string may be
|
|
478
|
-
//
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
//
|
|
612
|
+
// mandates one split here. An endpoint that does not answer is
|
|
613
|
+
// TRANSPORT_UNREACHABLE: a Node syscall code like ECONNREFUSED (see
|
|
614
|
+
// NETWORK_ERROR_CODES in address-policy.js). The remediation there
|
|
615
|
+
// is "check the network, the firewall, whether the service is
|
|
616
|
+
// running", and the connection string may be fine.
|
|
617
|
+
//
|
|
618
|
+
// Everything else stays INVALID_CONFIG: an auth failure (the host
|
|
619
|
+
// answered), a protocol error, an unclassified throw. The fix there
|
|
620
|
+
// is the supplied config. The underlying error is preserved on
|
|
621
|
+
// `err.cause` for diagnostics either way.
|
|
482
622
|
try {
|
|
483
623
|
await pgClient.connect();
|
|
484
624
|
} catch ( connErr ) {
|
|
@@ -498,539 +638,52 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
498
638
|
await pgClient.end();
|
|
499
639
|
}
|
|
500
640
|
|
|
501
|
-
//
|
|
641
|
+
// The sender config. The client trigger is off and the transport is
|
|
642
|
+
// stated: see buildSenderConfig.
|
|
502
643
|
const senderConfig = buildSenderConfig( {
|
|
503
|
-
ilpUrl,
|
|
504
|
-
flushMode,
|
|
505
|
-
autoFlushRows,
|
|
506
|
-
autoFlushIntervalMs,
|
|
507
|
-
maxBufSize,
|
|
508
|
-
retryTimeout
|
|
644
|
+
ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout
|
|
509
645
|
} );
|
|
510
646
|
|
|
511
|
-
//
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
// The client's flush() copies its rows out of the buffer synchronously
|
|
516
|
-
// (copy-out) before any network I/O, so the moment a flush is called
|
|
517
|
-
// its rows are no longer buffered — they move to the in-flight tally
|
|
518
|
-
// below. Conflating the two quantities in one counter let shutdown
|
|
519
|
-
// report clean over a hung idle flush.
|
|
520
|
-
let lastWriteTime = 0;
|
|
521
|
-
let idleFlushTimer = null;
|
|
522
|
-
let bufferedRows = 0;
|
|
523
|
-
|
|
524
|
-
// Rows inside flush copies that have not settled yet, plus the flush
|
|
525
|
-
// promises carrying them. getPressure() adds this to bufferedRows (a
|
|
526
|
-
// hung flush is undelivered data and must read as pressure), and
|
|
527
|
-
// shutdown() settles these entries instead of firing a blind flush
|
|
528
|
-
// at a buffer the copy-out already emptied.
|
|
529
|
-
let inFlightRows = 0;
|
|
530
|
-
const inFlightFlushes = new Set();
|
|
531
|
-
|
|
532
|
-
// Shutdown outcome, latched on the first call. A lossy shutdown's
|
|
533
|
-
// failed flush already emptied the buffer via copy-out, so a re-run
|
|
534
|
-
// would find nothing to flush and resolve clean — contradicting the
|
|
535
|
-
// recorded loss. Every caller gets the first call's promise instead.
|
|
536
|
-
let shutdownPromise = null;
|
|
647
|
+
// Setup probe, ILP side (ADR-030 item 4). Before this change the
|
|
648
|
+
// write path was never opened at setup; the first sign of a dead
|
|
649
|
+
// endpoint was the first flush.
|
|
650
|
+
await assertReachable( 'ilpUrl', ilpAddress, probeFn );
|
|
537
651
|
|
|
538
|
-
//
|
|
539
|
-
//
|
|
540
|
-
//
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
let consecutiveWriteErrors = 0;
|
|
545
|
-
|
|
546
|
-
// One idle flush in flight at a time. Against an unreachable server a
|
|
547
|
-
// flush can never settle (the client's retry loop — see the mid-row
|
|
548
|
-
// recovery section in the file header); without this guard every check
|
|
549
|
-
// interval would start ANOTHER stuck flush. A hung flush leaves the
|
|
550
|
-
// flag set, the buffer keeps filling, and the condition surfaces as
|
|
551
|
-
// rising pressure in getHealth() — visible, not multiplied.
|
|
552
|
-
let idleFlushInFlight = false;
|
|
553
|
-
|
|
554
|
-
/**
|
|
555
|
-
* Registers a flush the moment it is called. Copy-out means the rows
|
|
556
|
-
* leave the buffer NOW, delivered or not, so the caller hands them
|
|
557
|
-
* over in the same breath. Settlement — either way — removes them
|
|
558
|
-
* from the in-flight tally: resolved means delivered; rejected means
|
|
559
|
-
* lost, and loss REPORTING stays at the call site (the idle flush
|
|
560
|
-
* logs or routes to onDeliveryFailure, recovery routes to
|
|
561
|
-
* onDeliveryFailure, shutdown throws classified).
|
|
562
|
-
*
|
|
563
|
-
* @param {Promise} flushPromise - the just-fired sender.flush()
|
|
564
|
-
* @param {number} rows - row count the flush copy carries
|
|
565
|
-
* @returns {{promise: Promise, rows: number}} the tracked entry
|
|
566
|
-
*/
|
|
567
|
-
const trackFlush = function ( flushPromise, rows ) {
|
|
568
|
-
const entry = { promise: flushPromise, rows };
|
|
569
|
-
inFlightRows += rows;
|
|
570
|
-
inFlightFlushes.add( entry );
|
|
571
|
-
const settle = function () {
|
|
572
|
-
inFlightFlushes.delete( entry );
|
|
573
|
-
inFlightRows -= rows;
|
|
574
|
-
};
|
|
575
|
-
flushPromise.then( settle, settle );
|
|
576
|
-
return entry;
|
|
577
|
-
}; // trackFlush()
|
|
578
|
-
|
|
579
|
-
/**
|
|
580
|
-
* Check and flush if idle for idleFlushAfterMs.
|
|
581
|
-
*
|
|
582
|
-
* Runs in BOTH modes (see longer comment block at the setInterval call
|
|
583
|
-
* below). The rows move to the in-flight tally the moment the flush
|
|
584
|
-
* fires; a successful settle clears them, which also bounds the
|
|
585
|
-
* worst-case pressure-counter lag in auto mode (when QuestDB's silent
|
|
586
|
-
* `auto_flush_interval` trigger fires without us observing it, this
|
|
587
|
-
* is the self-healing point).
|
|
588
|
-
*/
|
|
589
|
-
const checkIdleFlush = async function () {
|
|
590
|
-
if ( idleFlushInFlight || bufferedRows === 0 ) {
|
|
591
|
-
return;
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
const now = Date.now();
|
|
595
|
-
const idleTime = now - lastWriteTime;
|
|
596
|
-
|
|
597
|
-
if ( idleTime >= idleFlushAfterMs ) {
|
|
598
|
-
idleFlushInFlight = true;
|
|
599
|
-
const rows = bufferedRows;
|
|
600
|
-
bufferedRows = 0;
|
|
601
|
-
const entry = trackFlush( sender.flush(), rows );
|
|
602
|
-
try {
|
|
603
|
-
await entry.promise;
|
|
604
|
-
} catch ( err ) {
|
|
605
|
-
// Copy-out: the failed flush carried the rows with it —
|
|
606
|
-
// the buffer no longer holds them and a retry cannot
|
|
607
|
-
// resend them. They are lost, and this is the report
|
|
608
|
-
// (the settle handler already cleared them from
|
|
609
|
-
// pressure). Same routing policy as the recovery flush:
|
|
610
|
-
// the caller owns the response when it asked to.
|
|
611
|
-
if ( safeOnDeliveryFailure ) {
|
|
612
|
-
safeOnDeliveryFailure( err, { idleFlush: true, rowsLost: rows } );
|
|
613
|
-
} else {
|
|
614
|
-
logger.error(
|
|
615
|
-
`winkComposer/questdb: idle flush failed; ${rows} buffered row(s) lost: ${err.message}`
|
|
616
|
-
);
|
|
617
|
-
}
|
|
618
|
-
} finally {
|
|
619
|
-
idleFlushInFlight = false;
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
};
|
|
623
|
-
|
|
624
|
-
// Start idle flush timer as safety net for "data stopped flowing" case.
|
|
625
|
-
// QuestDB's auto_flush_interval only checks elapsed time when NEW data is added -
|
|
626
|
-
// it is NOT a real background timer. If no more data arrives, the buffer sits
|
|
627
|
-
// indefinitely. QuestDB docs recommend: "implement your own timer-based logic."
|
|
628
|
-
// See: https://py-questdb-client.readthedocs.io/en/stable/conf.html
|
|
629
|
-
//
|
|
630
|
-
// This timer runs in BOTH modes:
|
|
631
|
-
// - Auto mode: Safety net for when data stops flowing (QuestDB won't flush on its own)
|
|
632
|
-
// - Manual mode: Primary flush mechanism (QuestDB auto-flush is disabled)
|
|
633
|
-
idleFlushTimer = setInterval( checkIdleFlush, idleFlushCheckMs );
|
|
634
|
-
// Don't block process exit
|
|
635
|
-
idleFlushTimer.unref();
|
|
636
|
-
|
|
637
|
-
// ========================================================================
|
|
638
|
-
// STORAGE INTERFACE
|
|
639
|
-
// ========================================================================
|
|
640
|
-
|
|
641
|
-
/**
|
|
642
|
-
* Cancels a half-written ILP row after a mid-row throw, so the NEXT write
|
|
643
|
-
* starts clean (see "Mid-row recovery" in the file header for the full
|
|
644
|
-
* account and the upstream issue link).
|
|
645
|
-
*
|
|
646
|
-
* flush() copies every completed row out of the buffer synchronously and
|
|
647
|
-
* sends them in the background; the unfinished row stays behind. reset()
|
|
648
|
-
* then clears the buffer — only the broken stub remains at that point —
|
|
649
|
-
* and lowers the client's row-in-progress flags. Net effect: the broken
|
|
650
|
-
* row vanishes, every good row is on its way, the sender accepts the
|
|
651
|
-
* next write.
|
|
652
|
-
*
|
|
653
|
-
* The early flush carries real data. If its send ultimately fails (after
|
|
654
|
-
* the client's own retries), that loss is reported like any other
|
|
655
|
-
* delivery failure: routed to `onDeliveryFailure` when provided,
|
|
656
|
-
* otherwise thrown as a classified unhandled rejection — the same policy
|
|
657
|
-
* as persist-plan's sender.at() failure path. Never silent.
|
|
658
|
-
*/
|
|
659
|
-
const recoverSender = function () {
|
|
660
|
-
try {
|
|
661
|
-
// flush() is an async function: it can never throw synchronously,
|
|
662
|
-
// and the rows it sends live in its own copy of the buffer.
|
|
663
|
-
// Tracked like every flush, so shutdown settles it and its
|
|
664
|
-
// rows stay visible as pressure until it settles.
|
|
665
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
666
|
-
sender.reset();
|
|
667
|
-
bufferedRows = 0;
|
|
668
|
-
|
|
669
|
-
entry.promise.catch( ( flushErr ) => {
|
|
670
|
-
if ( safeOnDeliveryFailure ) {
|
|
671
|
-
safeOnDeliveryFailure( flushErr, { recovery: true } );
|
|
672
|
-
return;
|
|
673
|
-
}
|
|
674
|
-
const failure = new Error(
|
|
675
|
-
`winkComposer/questdb: recovery flush failed after a mid-row write error: ${flushErr.message}. ` +
|
|
676
|
-
'Completed rows in that batch were dropped. Provide an `onDeliveryFailure` callback ' +
|
|
677
|
-
'in the storage config to handle these explicitly.'
|
|
678
|
-
);
|
|
679
|
-
failure.code = 'DELIVERY_FAILED';
|
|
680
|
-
failure.cause = flushErr;
|
|
681
|
-
throw failure;
|
|
682
|
-
} );
|
|
683
|
-
} catch ( recoveryErr ) {
|
|
684
|
-
// Defensive: with the 4.2.0 client neither call can throw here
|
|
685
|
-
// (flush is async, reset is trivial buffer bookkeeping). If a
|
|
686
|
-
// future client changes that, the sender may stay wedged — but
|
|
687
|
-
// write() must still return its classified result rather than
|
|
688
|
-
// throw (ADR-018: the hot path never throws), and the
|
|
689
|
-
// failure must be visible.
|
|
690
|
-
logger.error( `winkComposer/questdb: sender recovery failed: ${recoveryErr.message}` );
|
|
691
|
-
}
|
|
692
|
-
}; // recoverSender()
|
|
693
|
-
|
|
694
|
-
/**
|
|
695
|
-
* Write a message to QuestDB for a given insightType.
|
|
696
|
-
*
|
|
697
|
-
* Sync hot-path return per ADR-018 / ADR-013:
|
|
698
|
-
* - `RESULT_OK` (singleton) on successful enqueue.
|
|
699
|
-
* - `{ ok: false, error: { code: 'INVALID_INSIGHT_TYPE', message } }`
|
|
700
|
-
* when the insightType has no persist plan (config error).
|
|
701
|
-
* - `{ ok: false, error: { code: 'SEND_FAILED', message } }` when the
|
|
702
|
-
* persist plan throws (sender method failure: ILP buffer overflow,
|
|
703
|
-
* type coercion failure, etc.).
|
|
704
|
-
*
|
|
705
|
-
* Never throws. Async tryFlush rejections inside `sender.at()` are
|
|
706
|
-
* contained at the call site in `persist-plan.js` (see file header).
|
|
707
|
-
*
|
|
708
|
-
* @param {string} insightType - SignalType name (must exist in assetClass)
|
|
709
|
-
* @param {Object} message - Message with column values
|
|
710
|
-
* @param {string} partitionId - Partition identifier (stored as SYMBOL)
|
|
711
|
-
* @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
|
|
712
|
-
*/
|
|
713
|
-
const write = function ( insightType, message, partitionId ) {
|
|
714
|
-
if ( shuttingDown ) {
|
|
715
|
-
// The idle timer is already stopped and the final flush may
|
|
716
|
-
// have run: a row accepted now would have no flusher left.
|
|
717
|
-
// Refuse loudly instead of stranding it — uniform with the
|
|
718
|
-
// MQTT emitter's SHUTTING_DOWN refusal.
|
|
719
|
-
return {
|
|
720
|
-
ok: false,
|
|
721
|
-
error: {
|
|
722
|
-
code: 'SHUTTING_DOWN',
|
|
723
|
-
message: 'winkComposer/questdb: write rejected — storage is shutting down'
|
|
724
|
-
}
|
|
725
|
-
};
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
const persistPlan = persistPlans[ insightType ];
|
|
729
|
-
|
|
730
|
-
if ( !persistPlan ) {
|
|
731
|
-
return {
|
|
732
|
-
ok: false,
|
|
733
|
-
error: {
|
|
734
|
-
code: 'INVALID_INSIGHT_TYPE',
|
|
735
|
-
message: `No persist plan for insightType '${insightType}'`
|
|
736
|
-
}
|
|
737
|
-
};
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
try {
|
|
741
|
-
// The plan reports whether it actually opened a row. A row
|
|
742
|
-
// skipped in phase 1 (bad designated timestamp) never touched
|
|
743
|
-
// the sender and must not count as buffered — it would inflate
|
|
744
|
-
// pressure and shutdown's dropped count.
|
|
745
|
-
const written = persistPlan( sender, message, partitionId );
|
|
746
|
-
consecutiveWriteErrors = 0; // recovery — health flips back to green
|
|
747
|
-
if ( written ) {
|
|
748
|
-
lastWriteTime = Date.now();
|
|
749
|
-
bufferedRows += 1;
|
|
750
|
-
// Auto mode: QuestDB triggers an internal flush on the sender.at()
|
|
751
|
-
// call that brings pendingRowCount to autoFlushRows (verified
|
|
752
|
-
// against @questdb/nodejs-client v4.2.0 dist/cjs/index.js
|
|
753
|
-
// tryFlush: pendingRowCount >= autoFlushRows). Mirror that reset
|
|
754
|
-
// so getPressure() reflects QuestDB's actual buffer state.
|
|
755
|
-
//
|
|
756
|
-
// Guarded on `autoFlushRows > 0` because the value is optional in
|
|
757
|
-
// ENV_VARS / config; when unset, QuestDB falls back to its own
|
|
758
|
-
// internal default which we cannot observe. In that case the
|
|
759
|
-
// heuristic does not fire and QuestDB contributes no pressure
|
|
760
|
-
// signal — `getPressure()` (below) returns 0.
|
|
761
|
-
if ( flushMode === 'auto' && autoFlushRows > 0 && bufferedRows >= autoFlushRows ) {
|
|
762
|
-
bufferedRows = 0;
|
|
763
|
-
}
|
|
764
|
-
}
|
|
765
|
-
return RESULT_OK;
|
|
766
|
-
} catch ( err ) {
|
|
767
|
-
consecutiveWriteErrors += 1; // health degradation signal
|
|
768
|
-
// The throw may have left a half-written row in the sender;
|
|
769
|
-
// cancel it so this failure costs one row, not the rest of the
|
|
770
|
-
// run (ADR-018). Safe to call even when no row was open: flushing
|
|
771
|
-
// early is harmless and reset() on a consistent buffer is a no-op.
|
|
772
|
-
recoverSender();
|
|
773
|
-
return {
|
|
774
|
-
ok: false,
|
|
775
|
-
error: {
|
|
776
|
-
code: 'SEND_FAILED',
|
|
777
|
-
message: err.message
|
|
778
|
-
}
|
|
779
|
-
};
|
|
780
|
-
}
|
|
781
|
-
};
|
|
782
|
-
|
|
783
|
-
/**
|
|
784
|
-
* Flush pending rows to QuestDB.
|
|
785
|
-
* In auto mode, this is a no-op (QuestDB handles it).
|
|
786
|
-
* In manual mode, this forces an immediate flush. The rows move to
|
|
787
|
-
* the in-flight tally at the call (copy-out); a failure rejects to
|
|
788
|
-
* the caller exactly as before, and the settle handler keeps the
|
|
789
|
-
* pressure accounting straight either way.
|
|
790
|
-
*
|
|
791
|
-
* @returns {Promise<void>}
|
|
792
|
-
*/
|
|
793
|
-
const flush = async function () {
|
|
794
|
-
if ( flushMode === 'manual' && bufferedRows > 0 ) {
|
|
795
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
796
|
-
bufferedRows = 0;
|
|
797
|
-
await entry.promise;
|
|
798
|
-
}
|
|
799
|
-
// Auto mode: no-op, QuestDB handles flushing
|
|
800
|
-
};
|
|
801
|
-
|
|
802
|
-
/**
|
|
803
|
-
* Backpressure metric for the partition manager. Returns the buffer fill
|
|
804
|
-
* ratio in [0, 1] per ADR-018 (sync, O(1), allocation-free). Clamped
|
|
805
|
-
* to 1.0 — the codebase invariant for getPressure across adapters.
|
|
806
|
-
*
|
|
807
|
-
* Accuracy by mode:
|
|
808
|
-
* - Manual mode: exact (we own every flush).
|
|
809
|
-
* - Auto mode, heavy flow: exact (autoFlushRows heuristic reset matches).
|
|
810
|
-
* - Auto mode, slow flow + auto_flush_interval triggers: over-states by
|
|
811
|
-
* up to `idleFlushAfterMs`, self-heals via `checkIdleFlush`.
|
|
812
|
-
*
|
|
813
|
-
* Rows inside unsettled flush copies count as pressure: a hung flush
|
|
814
|
-
* is undelivered data, and hiding it is what let shutdown report
|
|
815
|
-
* clean over it. They leave the tally when their flush settles —
|
|
816
|
-
* delivered or reported lost.
|
|
817
|
-
*
|
|
818
|
-
* When `autoFlushRows` is not configured (left to QuestDB's internal
|
|
819
|
-
* default), we have no observable capacity reference and return
|
|
820
|
-
* 0 — QuestDB then contributes nothing to the flow's pressure signal.
|
|
821
|
-
* The time-based yield (ADR-024) and message-count safety nets remain
|
|
822
|
-
* effective.
|
|
823
|
-
*
|
|
824
|
-
* @returns {number} Pressure value in [0, 1]
|
|
825
|
-
*/
|
|
826
|
-
const getPressure = function () {
|
|
827
|
-
if ( !( autoFlushRows > 0 ) ) {
|
|
828
|
-
return 0;
|
|
829
|
-
}
|
|
830
|
-
return Math.min( 1.0, ( bufferedRows + inFlightRows ) / autoFlushRows );
|
|
831
|
-
};
|
|
832
|
-
|
|
833
|
-
/**
|
|
834
|
-
* Health snapshot for operator monitoring (uniform across sinks).
|
|
835
|
-
* Returns the ADR-018 health floor `{status, connected, ...}`, plus a
|
|
836
|
-
* small set of QuestDB-specific diagnostic fields.
|
|
837
|
-
*
|
|
838
|
-
* Status derivation (kept in code, not config — operator mental model is
|
|
839
|
-
* load-bearing institutional knowledge):
|
|
840
|
-
* - `red` if `!connected` (shutting down or sustained write failure)
|
|
841
|
-
* - `yellow` if `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD` OR
|
|
842
|
-
* `consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD`
|
|
843
|
-
* - `green` otherwise
|
|
844
|
-
*
|
|
845
|
-
* `connected` here is *derived* — QuestDB's ILP sender is fire-and-forget
|
|
846
|
-
* with no observable socket state, so we infer transport health from
|
|
847
|
-
* recent write success.
|
|
848
|
-
*
|
|
849
|
-
* @returns {{status: 'green'|'yellow'|'red', connected: boolean, pressure: number, consecutiveWriteErrors: number, bufferedRows: number, flushMode: string}}
|
|
850
|
-
*/
|
|
851
|
-
const getHealth = function () {
|
|
852
|
-
const pressure = getPressure();
|
|
853
|
-
const connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD;
|
|
652
|
+
// The agent, when the standard-library transport is selected, then
|
|
653
|
+
// the sender. A failure is classified, cause attached (see
|
|
654
|
+
// buildSender), and leaves no agent behind (see openTransport).
|
|
655
|
+
const { sender, closeTransport } = await openTransport( {
|
|
656
|
+
SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent
|
|
657
|
+
} );
|
|
854
658
|
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
659
|
+
// The probe the engine runs after a failed flush (ADR-029 hold and
|
|
660
|
+
// probe), bound to the ILP address so the engine knows nothing
|
|
661
|
+
// about addresses. Setup ran the same probe on both endpoints.
|
|
662
|
+
const probe = {
|
|
663
|
+
run: function () {
|
|
664
|
+
return probeFn( ilpAddress );
|
|
665
|
+
},
|
|
666
|
+
describe: function ( outcome ) {
|
|
667
|
+
return describeProbe( outcome, 'ilpUrl', ilpAddress );
|
|
862
668
|
}
|
|
863
|
-
|
|
864
|
-
return {
|
|
865
|
-
// Required health floor (ADR-018)
|
|
866
|
-
status,
|
|
867
|
-
connected,
|
|
868
|
-
pressure,
|
|
869
|
-
// Adapter-specific diagnostics
|
|
870
|
-
consecutiveWriteErrors,
|
|
871
|
-
bufferedRows,
|
|
872
|
-
flushMode
|
|
873
|
-
};
|
|
874
669
|
};
|
|
875
670
|
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
*/
|
|
881
|
-
const closeQuietly = function () {
|
|
882
|
-
return sender.close().catch( function ( closeErr ) {
|
|
883
|
-
logger.error( `winkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
|
|
884
|
-
} );
|
|
885
|
-
}; // closeQuietly()
|
|
886
|
-
|
|
887
|
-
/**
|
|
888
|
-
* The real shutdown body. `shutdown` below latches its promise so
|
|
889
|
-
* every caller — including re-entrant and post-failure callers —
|
|
890
|
-
* receives this one outcome.
|
|
891
|
-
*
|
|
892
|
-
* A clean resolve is a delivery statement (ADR-018): everything
|
|
893
|
-
* buffered OR in flight was delivered. Shutdown therefore settles
|
|
894
|
-
* every unsettled flush (idle, recovery) plus one final flush for
|
|
895
|
-
* whatever is still buffered, all raced against the caller's
|
|
896
|
-
* `{ timeout }` (ADR-018 drain-then-close). It never fires a blind flush at a
|
|
897
|
-
* buffer an earlier copy-out emptied — that is what let it report
|
|
898
|
-
* clean over a hung idle flush.
|
|
899
|
-
*
|
|
900
|
-
* On loss it rejects classified, `dropped: { count }` exact:
|
|
901
|
-
* - any awaited flush fails → `DELIVERY_FAILED`, first flush error
|
|
902
|
-
* on `cause`, count = rows on the flushes that failed;
|
|
903
|
-
* - the combined wait does not settle in time → `SHUTDOWN_TIMEOUT`,
|
|
904
|
-
* count = rows not confirmed delivered. A send against an
|
|
905
|
-
* unreachable server never settles (the client's retry loop — see
|
|
906
|
-
* the file header), so the bound is what keeps shutdown finite.
|
|
907
|
-
* `dropped` is a statement about THIS session: those rows were not
|
|
908
|
-
* confirmed delivered before close. An abandoned flush keeps
|
|
909
|
-
* retrying and may still land its rows later if the server
|
|
910
|
-
* recovers — the count is a floor on uncertainty, not a proof of
|
|
911
|
-
* loss.
|
|
912
|
-
*
|
|
913
|
-
* The transport close is attempted in both loss paths, but on the
|
|
914
|
-
* HTTP transport the client's `close()` is an empty function
|
|
915
|
-
* (verified against @questdb/nodejs-client 4.2.0), so nothing can
|
|
916
|
-
* abort an abandoned flush's retry timers from here; they keep the
|
|
917
|
-
* event loop alive. Process exit is the final backstop — in a flow,
|
|
918
|
-
* the shutdown manager's `SHUTDOWN_FORCE_TIMEOUT_MS` exit covers
|
|
919
|
-
* this. No timeout supplied = no enforcement (unbounded await),
|
|
920
|
-
* preserving direct-caller behavior.
|
|
921
|
-
*/
|
|
922
|
-
const doShutdown = async function ( timeout ) {
|
|
923
|
-
// Flip the health flag first so any concurrent getHealth() call
|
|
924
|
-
// immediately sees the shutdown and returns red/disconnected —
|
|
925
|
-
// and write() starts refusing new rows (SHUTTING_DOWN).
|
|
926
|
-
shuttingDown = true;
|
|
927
|
-
|
|
928
|
-
// Stop idle flush timer
|
|
929
|
-
if ( idleFlushTimer ) {
|
|
930
|
-
clearInterval( idleFlushTimer );
|
|
931
|
-
idleFlushTimer = null;
|
|
932
|
-
}
|
|
933
|
-
|
|
934
|
-
// Everything delivery still owes: flushes already in flight
|
|
935
|
-
// (their rows left the buffer at their call) plus one final
|
|
936
|
-
// flush for whatever is still buffered. Each wait records its
|
|
937
|
-
// outcome into the tallies below; the mapped promises never
|
|
938
|
-
// reject, so the only rejection the race can surface is the
|
|
939
|
-
// timeout itself.
|
|
940
|
-
let totalRows = 0;
|
|
941
|
-
let deliveredRows = 0;
|
|
942
|
-
let failedRows = 0;
|
|
943
|
-
let firstFailure = null;
|
|
944
|
-
const waits = [];
|
|
945
|
-
|
|
946
|
-
const awaitDelivery = function ( entry ) {
|
|
947
|
-
totalRows += entry.rows;
|
|
948
|
-
waits.push( entry.promise.then(
|
|
949
|
-
function () {
|
|
950
|
-
deliveredRows += entry.rows;
|
|
951
|
-
},
|
|
952
|
-
function ( err ) {
|
|
953
|
-
failedRows += entry.rows;
|
|
954
|
-
if ( !firstFailure ) {
|
|
955
|
-
firstFailure = err;
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
) );
|
|
959
|
-
}; // awaitDelivery()
|
|
960
|
-
|
|
961
|
-
inFlightFlushes.forEach( awaitDelivery );
|
|
962
|
-
if ( bufferedRows > 0 ) {
|
|
963
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
964
|
-
bufferedRows = 0;
|
|
965
|
-
awaitDelivery( entry );
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
if ( waits.length > 0 ) {
|
|
969
|
-
try {
|
|
970
|
-
await raceFlushTimeout( Promise.all( waits ), timeout );
|
|
971
|
-
} catch ( err ) {
|
|
972
|
-
await closeQuietly();
|
|
973
|
-
const dropped = totalRows - deliveredRows;
|
|
974
|
-
const timedOut = new Error(
|
|
975
|
-
`winkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
|
|
976
|
-
);
|
|
977
|
-
timedOut.code = 'SHUTDOWN_TIMEOUT';
|
|
978
|
-
timedOut.dropped = { count: dropped };
|
|
979
|
-
throw timedOut;
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
if ( failedRows > 0 ) {
|
|
983
|
-
await closeQuietly();
|
|
984
|
-
const failure = new Error(
|
|
985
|
-
`winkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
|
|
986
|
-
);
|
|
987
|
-
failure.code = 'DELIVERY_FAILED';
|
|
988
|
-
failure.dropped = { count: failedRows };
|
|
989
|
-
failure.cause = firstFailure;
|
|
990
|
-
throw failure;
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
// Close sender
|
|
995
|
-
await sender.close();
|
|
996
|
-
}; // doShutdown()
|
|
997
|
-
|
|
998
|
-
/**
|
|
999
|
-
* Shutdown the storage adapter gracefully.
|
|
1000
|
-
* Flushes pending data and closes connections.
|
|
1001
|
-
*
|
|
1002
|
-
* Called by wire-storages.shutdown() which is invoked during:
|
|
1003
|
-
* - Pipeline shutdown (flowHandle.shutdown())
|
|
1004
|
-
* - Process signal handlers (SIGINT, SIGTERM)
|
|
1005
|
-
*
|
|
1006
|
-
* The outcome is latched: the first call runs the shutdown, every
|
|
1007
|
-
* later call returns the same promise. A lossy shutdown's failed
|
|
1008
|
-
* flush already emptied the buffer (copy-out), so a re-run would
|
|
1009
|
-
* find nothing to flush and resolve clean, contradicting the
|
|
1010
|
-
* recorded loss. One consequence: the first caller's `{ timeout }`
|
|
1011
|
-
* governs; a later caller's is ignored.
|
|
1012
|
-
*
|
|
1013
|
-
* @param {{timeout?: number}} [options]
|
|
1014
|
-
* @returns {Promise<void>}
|
|
1015
|
-
*/
|
|
1016
|
-
const shutdown = function ( { timeout = 0 } = {} ) {
|
|
1017
|
-
if ( !shutdownPromise ) {
|
|
1018
|
-
shutdownPromise = doShutdown( timeout );
|
|
1019
|
-
}
|
|
1020
|
-
return shutdownPromise;
|
|
1021
|
-
}; // shutdown()
|
|
671
|
+
// Everything after setup is the engine's: the hot path, the
|
|
672
|
+
// flushes, the counters, the drain. The transport close stays the
|
|
673
|
+
// factory's, because the factory owns the agent.
|
|
674
|
+
const engine = createFlushEngine( { sender, persistPlans, settings, onDeliveryFailure, probe, closeTransport } );
|
|
1022
675
|
|
|
1023
676
|
return {
|
|
1024
|
-
write,
|
|
1025
|
-
flush,
|
|
1026
|
-
shutdown,
|
|
1027
|
-
getPressure,
|
|
1028
|
-
getHealth,
|
|
677
|
+
write: engine.write,
|
|
678
|
+
flush: engine.flush,
|
|
679
|
+
shutdown: engine.shutdown,
|
|
680
|
+
getPressure: engine.getPressure,
|
|
681
|
+
getHealth: engine.getHealth,
|
|
1029
682
|
// Expose for testing/debugging
|
|
1030
683
|
_sender: sender,
|
|
1031
684
|
_persistPlans: persistPlans
|
|
1032
685
|
};
|
|
1033
|
-
};
|
|
686
|
+
}; // createQuestDBStorage()
|
|
1034
687
|
|
|
1035
688
|
// ============================================================================
|
|
1036
689
|
// CONFIG SCHEMA
|
|
@@ -1055,70 +708,82 @@ const configSchema = {
|
|
|
1055
708
|
'ilpUrl',
|
|
1056
709
|
'pgUrl',
|
|
1057
710
|
'tablePrefix',
|
|
1058
|
-
'
|
|
1059
|
-
'
|
|
1060
|
-
'
|
|
1061
|
-
'
|
|
1062
|
-
'
|
|
1063
|
-
'
|
|
711
|
+
'flushRows',
|
|
712
|
+
'flushIntervalMs',
|
|
713
|
+
'bufferCeilingRows',
|
|
714
|
+
'flushDeadlineMs',
|
|
715
|
+
'stdlibHttp',
|
|
716
|
+
'requestTimeout',
|
|
1064
717
|
'retryTimeout',
|
|
718
|
+
'initBufSize',
|
|
719
|
+
'maxBufSize',
|
|
1065
720
|
'partitionBy',
|
|
1066
721
|
'onWarning',
|
|
1067
722
|
'onDeliveryFailure'
|
|
1068
723
|
],
|
|
724
|
+
// Both addresses refuse `localhost` at flow definition (ADR-030).
|
|
725
|
+
// The non-empty rule lives inside the validator, so one static
|
|
726
|
+
// error string covers every refusal.
|
|
1069
727
|
ilpUrl: {
|
|
1070
728
|
type: 'string',
|
|
1071
729
|
required: false,
|
|
1072
|
-
|
|
1073
|
-
error: 'ilpUrl must be a
|
|
730
|
+
validator: isAllowedIlpUrl,
|
|
731
|
+
error: 'ilpUrl must be host:port with a literal address or a name, never localhost and never ' +
|
|
732
|
+
'an IPv6 literal (the QuestDB client cannot read one); e.g., 127.0.0.1:9000'
|
|
1074
733
|
},
|
|
1075
734
|
pgUrl: {
|
|
1076
735
|
type: 'string',
|
|
1077
736
|
required: false,
|
|
1078
|
-
|
|
1079
|
-
error: 'pgUrl must be a
|
|
737
|
+
validator: isAllowedPgUrl,
|
|
738
|
+
error: 'pgUrl must be host:port with a literal address or a name, never localhost; e.g., 127.0.0.1:8812'
|
|
1080
739
|
},
|
|
1081
740
|
tablePrefix: {
|
|
1082
741
|
type: 'string',
|
|
1083
742
|
required: false,
|
|
1084
743
|
minLength: 1,
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
validator: validators.oneOf( [ 'auto', 'manual' ] ),
|
|
1091
|
-
error: 'flushMode must be "auto" or "manual"'
|
|
744
|
+
// The prefix opens every table name, written unquoted in CREATE
|
|
745
|
+
// TABLE. See header sub-case (j) for the rule and its sources.
|
|
746
|
+
validator: validators.identifier,
|
|
747
|
+
error: 'tablePrefix must use letters, digits, _ and $ only, and not start with a digit ' +
|
|
748
|
+
'(defaults to assetClass.name when omitted)'
|
|
1092
749
|
},
|
|
1093
|
-
|
|
750
|
+
// The flush settings composer owns (ADR-029). The relation between
|
|
751
|
+
// the ceiling and the threshold is checked by the option resolver
|
|
752
|
+
// at setup, where both values are known.
|
|
753
|
+
flushRows: {
|
|
1094
754
|
type: 'number',
|
|
1095
755
|
required: false,
|
|
1096
756
|
validator: validators.positiveInteger,
|
|
1097
|
-
error: '
|
|
757
|
+
error: 'flushRows must be a positive integer'
|
|
1098
758
|
},
|
|
1099
|
-
|
|
759
|
+
flushIntervalMs: {
|
|
1100
760
|
type: 'number',
|
|
1101
761
|
required: false,
|
|
1102
762
|
validator: validators.positiveInteger,
|
|
1103
|
-
error: '
|
|
763
|
+
error: 'flushIntervalMs must be a positive integer'
|
|
1104
764
|
},
|
|
1105
|
-
|
|
765
|
+
bufferCeilingRows: {
|
|
1106
766
|
type: 'number',
|
|
1107
767
|
required: false,
|
|
1108
768
|
validator: validators.positiveInteger,
|
|
1109
|
-
error: '
|
|
769
|
+
error: 'bufferCeilingRows must be a positive integer'
|
|
1110
770
|
},
|
|
1111
|
-
|
|
771
|
+
flushDeadlineMs: {
|
|
1112
772
|
type: 'number',
|
|
1113
773
|
required: false,
|
|
1114
774
|
validator: validators.positiveInteger,
|
|
1115
|
-
error: '
|
|
775
|
+
error: 'flushDeadlineMs must be a positive integer'
|
|
1116
776
|
},
|
|
1117
|
-
|
|
777
|
+
stdlibHttp: {
|
|
778
|
+
type: 'boolean',
|
|
779
|
+
required: false,
|
|
780
|
+
error: 'stdlibHttp must be a boolean'
|
|
781
|
+
},
|
|
782
|
+
requestTimeout: {
|
|
1118
783
|
type: 'number',
|
|
1119
784
|
required: false,
|
|
1120
785
|
validator: validators.positiveInteger,
|
|
1121
|
-
error: '
|
|
786
|
+
error: 'requestTimeout must be a positive integer'
|
|
1122
787
|
},
|
|
1123
788
|
retryTimeout: {
|
|
1124
789
|
type: 'number',
|
|
@@ -1126,6 +791,18 @@ const configSchema = {
|
|
|
1126
791
|
validator: validators.positiveInteger,
|
|
1127
792
|
error: 'retryTimeout must be a positive integer'
|
|
1128
793
|
},
|
|
794
|
+
initBufSize: {
|
|
795
|
+
type: 'number',
|
|
796
|
+
required: false,
|
|
797
|
+
validator: validators.positiveInteger,
|
|
798
|
+
error: 'initBufSize must be a positive integer'
|
|
799
|
+
},
|
|
800
|
+
maxBufSize: {
|
|
801
|
+
type: 'number',
|
|
802
|
+
required: false,
|
|
803
|
+
validator: validators.positiveInteger,
|
|
804
|
+
error: 'maxBufSize must be a positive integer'
|
|
805
|
+
},
|
|
1129
806
|
partitionBy: {
|
|
1130
807
|
type: 'string',
|
|
1131
808
|
required: false,
|
|
@@ -1166,23 +843,24 @@ const durabilityClass = 'in-memory';
|
|
|
1166
843
|
|
|
1167
844
|
// Capability declaration per ADR-018.
|
|
1168
845
|
//
|
|
1169
|
-
// QuestDB needs the asset class
|
|
1170
|
-
// `columns` and `insightTypes` to issue CREATE TABLE statements
|
|
1171
|
-
//
|
|
846
|
+
// QuestDB needs the asset class for two reasons. `ensure-tables.js`
|
|
847
|
+
// reads `columns` and `insightTypes` to issue CREATE TABLE statements.
|
|
848
|
+
// `persist-plan.js` reads the same to dispatch column writers and
|
|
1172
849
|
// pre-compile resolution-aware quantization. `name` drives the default
|
|
1173
850
|
// `tablePrefix` when the caller did not pass one.
|
|
1174
851
|
//
|
|
1175
852
|
// Declared `required: true` because there is no useful behaviour for
|
|
1176
|
-
// QuestDB without an assetClass
|
|
1177
|
-
// MISSING_ASSET_CLASS
|
|
853
|
+
// QuestDB without an assetClass. The wiring layer fails fast with
|
|
854
|
+
// MISSING_ASSET_CLASS instead of calling `createStorage` and letting
|
|
1178
855
|
// the failure surface deeper.
|
|
1179
856
|
//
|
|
1180
857
|
// `fields` lists exactly the top-level fields read. The slicing is
|
|
1181
|
-
// top-level only
|
|
858
|
+
// top-level only. Column-internal fields (`type`, `resolution`) are
|
|
1182
859
|
// read directly from the `columns` slice and validated by Layer 2
|
|
1183
|
-
// assertions inside `createStorage` below.
|
|
1184
|
-
//
|
|
1185
|
-
//
|
|
860
|
+
// assertions inside `createStorage` below.
|
|
861
|
+
//
|
|
862
|
+
// A declarative shape for column-internal capability is deferred
|
|
863
|
+
// until a second adapter needs it.
|
|
1186
864
|
const semanticsRequirement = {
|
|
1187
865
|
assetClass: {
|
|
1188
866
|
required: true,
|
|
@@ -1196,17 +874,20 @@ const semanticsRequirement = {
|
|
|
1196
874
|
* @param {Object} config - Configuration object
|
|
1197
875
|
* @param {Object} config.assetClass - Asset class definition with columns and insightTypes
|
|
1198
876
|
* @param {string} config.tablePrefix - Prefix for table names (defaults to assetClass.name)
|
|
1199
|
-
* @param {string} config.ilpUrl - ILP endpoint (host:port)
|
|
1200
|
-
* @param {string} config.pgUrl - PostgreSQL endpoint (host:port)
|
|
1201
|
-
* @param {
|
|
1202
|
-
* @param {number} [config.
|
|
1203
|
-
* @param {number} [config.
|
|
1204
|
-
* @param {number} [config.
|
|
1205
|
-
* @param {
|
|
1206
|
-
* @param {number} [config.
|
|
1207
|
-
* @param {number} [config.retryTimeout] -
|
|
877
|
+
* @param {string} [config.ilpUrl] - ILP endpoint (host:port)
|
|
878
|
+
* @param {string} [config.pgUrl] - PostgreSQL endpoint (host:port)
|
|
879
|
+
* @param {number} [config.flushRows=5000] - Rows that start a flush from inside write()
|
|
880
|
+
* @param {number} [config.flushIntervalMs=1000] - Period of the flush timer
|
|
881
|
+
* @param {number} [config.bufferCeilingRows] - Most rows held; default ten times flushRows
|
|
882
|
+
* @param {number} [config.flushDeadlineMs] - Fixed deadline per flush
|
|
883
|
+
* @param {boolean} [config.stdlibHttp=true] - The standard-library transport; false selects undici
|
|
884
|
+
* @param {number} [config.requestTimeout] - Client request timeout in ms
|
|
885
|
+
* @param {number} [config.retryTimeout] - Client retry window in ms
|
|
886
|
+
* @param {number} [config.initBufSize] - Initial client buffer size in bytes
|
|
887
|
+
* @param {number} [config.maxBufSize] - Byte ceiling of the client buffer
|
|
1208
888
|
* @param {string} [config.partitionBy='DAY'] - Table partition interval
|
|
1209
|
-
* @param {function} [config.onWarning] - Warning callback for
|
|
889
|
+
* @param {function} [config.onWarning] - Warning callback for skipped values
|
|
890
|
+
* @param {function} [config.onDeliveryFailure] - Called once per lost flush
|
|
1210
891
|
* @param {Object} [config._deps] - Injectable dependencies (for testing)
|
|
1211
892
|
* @returns {Promise<Object>} Storage instance with write, flush, shutdown methods
|
|
1212
893
|
*/
|
|
@@ -1228,15 +909,14 @@ const createStorage = function ( config ) {
|
|
|
1228
909
|
}
|
|
1229
910
|
|
|
1230
911
|
// Adapter-side defensive validation of column-internal facts
|
|
1231
|
-
// QuestDB consumes
|
|
1232
|
-
//
|
|
1233
|
-
//
|
|
1234
|
-
//
|
|
1235
|
-
//
|
|
1236
|
-
//
|
|
1237
|
-
//
|
|
1238
|
-
//
|
|
1239
|
-
// full rationale.
|
|
912
|
+
// QuestDB consumes (the ADR-018 column-internal facts pattern). The
|
|
913
|
+
// universal semantics schema validates the top-level structure at
|
|
914
|
+
// load and at .assetClass() time: type required, type in
|
|
915
|
+
// COLUMN_TYPES. This layer catches bypasses, such as a direct
|
|
916
|
+
// createStorage call from test code. It also checks adapter-specific
|
|
917
|
+
// requirements, such as "a float64 column with a declared resolution
|
|
918
|
+
// must have a positive value". See `assert-columns.js` for the full
|
|
919
|
+
// rationale.
|
|
1240
920
|
assertColumnFacts( assetClass );
|
|
1241
921
|
|
|
1242
922
|
// Default `tablePrefix` to the asset class name when the caller
|
|
@@ -1264,8 +944,8 @@ const createStorage = function ( config ) {
|
|
|
1264
944
|
* .assetClass( assetClassDef ) // wire-storages injects the slice
|
|
1265
945
|
* .storage( questdbAdapter, {
|
|
1266
946
|
* tablePrefix: 'myPrefix',
|
|
1267
|
-
* ilpUrl: '
|
|
1268
|
-
* pgUrl: '
|
|
947
|
+
* ilpUrl: '127.0.0.1:9000',
|
|
948
|
+
* pgUrl: '127.0.0.1:8812'
|
|
1269
949
|
* } )
|
|
1270
950
|
*
|
|
1271
951
|
* @type {Object}
|