@winkjs/composer 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +2 -2
- package/src/composer.js +1 -1
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +103 -658
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/index.js +16 -1
- package/src/core/env-vars.js +99 -13
- package/src/core/logger/index.js +1 -1
- package/src/core/source-manager/mqtt/client.js +197 -63
- package/src/core/source-manager/mqtt/dedup.js +16 -11
- package/src/core/source-manager/mqtt/index.js +32 -5
- package/src/core/source-manager/mqtt/status.js +168 -56
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +574 -847
- package/src/core/storage-manager/questdb/persist-plan.js +128 -167
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +23 -2
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/wiring/wire-emitters.js +3 -1
- package/src/core/wiring/wire-storages.js +10 -2
- package/src/flow/flow.js +24 -3
- package/src/flow/run.js +24 -2
|
@@ -1,64 +1,134 @@
|
|
|
1
1
|
// core/storage-manager/questdb/index.js
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @fileoverview QuestDB storage adapter
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* -
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* -
|
|
30
|
-
*
|
|
31
|
-
* - `getPressure()
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
4
|
+
* @fileoverview QuestDB storage adapter. It writes rows to QuestDB over
|
|
5
|
+
* ILP (InfluxDB Line Protocol) on HTTP, and creates the tables over the
|
|
6
|
+
* PostgreSQL wire protocol at startup.
|
|
7
|
+
*
|
|
8
|
+
* Contract role (ADR-018): storage, buffered. Startup posture: eager.
|
|
9
|
+
* Setup throws when either endpoint does not answer, so a flow never
|
|
10
|
+
* starts against a dead database.
|
|
11
|
+
*
|
|
12
|
+
* Module layout. This file holds the factory, the config schema, and
|
|
13
|
+
* the adapter export. The work lives in four modules next to it:
|
|
14
|
+
* - `resolve-options.js` merges the config and the environment into
|
|
15
|
+
* the settings, and derives the ceiling and the deadline.
|
|
16
|
+
* - `address-policy.js` holds the ADR-030 address checks, the setup
|
|
17
|
+
* probe wrapper, and the classification of a connect failure.
|
|
18
|
+
* - `flush-engine.js` owns the hot path, every flush, the counters
|
|
19
|
+
* behind pressure and health, and the shutdown drain (ADR-029).
|
|
20
|
+
* - `persist-plan.js` compiles one row writer per insight type.
|
|
21
|
+
*
|
|
22
|
+
* The handle the factory returns:
|
|
23
|
+
* - `write(insightType, msg, partitionId)`: the hot path. Appends one
|
|
24
|
+
* row to the client's buffer and returns at once. `{ ok: true }`
|
|
25
|
+
* means accepted into the buffer, not delivered. `{ ok: false,
|
|
26
|
+
* error: { code, message } }` names the refusal. Never throws.
|
|
27
|
+
* - `flush()`: sends everything buffered now. Resolves when the server
|
|
28
|
+
* confirmed the rows, rejects when it did not.
|
|
29
|
+
* - `shutdown({ timeout })`: drain, then close. A clean resolve means
|
|
30
|
+
* every row was delivered. A loss is a classified rejection.
|
|
31
|
+
* - `getPressure()`: buffer fill in [0, 1]. Sync, O(1), no allocation.
|
|
32
|
+
* - `getHealth()`: `{ status, connected, pressure, ... }`.
|
|
33
|
+
* `flush-engine.js` documents how each one behaves.
|
|
34
|
+
*
|
|
35
|
+
* Composer owns every flush (ADR-029). The client's own trigger is
|
|
36
|
+
* off (`auto_flush=off`), so a batch leaves the process only when the
|
|
37
|
+
* engine says so. A write that brings the buffer to `flushRows` starts
|
|
38
|
+
* a flush at once. A timer starts a flush every `flushIntervalMs` when
|
|
39
|
+
* anything is buffered. Only one engine flush runs at a time. A failed
|
|
40
|
+
* flush is reported once, with the exact rows lost, and the process
|
|
41
|
+
* keeps running. Every flush has a deadline, derived from the rows it
|
|
42
|
+
* carries, and a flush past it is abandoned and reported. A failed or
|
|
43
|
+
* abandoned flush runs the ADR-030 probe. While the probe fails,
|
|
44
|
+
* delivery pauses: rows are held up to the ceiling and each tick
|
|
45
|
+
* probes again. `flush-engine.js` carries the detail.
|
|
46
|
+
*
|
|
47
|
+
* Row order. Rows reach QuestDB in write order. The engine runs one
|
|
48
|
+
* flush at a time, each flush carries its rows in write order, and
|
|
49
|
+
* the standard-library transport keeps one socket (`maxSockets: 1`),
|
|
50
|
+
* so a later request waits for the earlier one. Two sends can be
|
|
51
|
+
* outstanding only at shutdown, when the drain's final flush starts
|
|
52
|
+
* while an earlier flush is still on the wire. On the undici
|
|
53
|
+
* transport those two batches can land out of order. QuestDB orders
|
|
54
|
+
* each table by its designated timestamp, so a query never sees the
|
|
55
|
+
* difference.
|
|
56
|
+
*
|
|
57
|
+
* Durability in plain words (ADR-018). `durabilityClass` is
|
|
58
|
+
* `'in-memory'`. Rows live in the client's buffer until a flush reaches
|
|
59
|
+
* the server. Three things lose rows: a process crash loses everything
|
|
60
|
+
* buffered or in flight, a failed flush loses its batch, and the
|
|
61
|
+
* ceiling sheds new rows. The shed policy: once the rows buffered plus
|
|
62
|
+
* the rows in flight reach `bufferCeilingRows`, `write()` refuses new
|
|
63
|
+
* rows with `STORAGE_FULL` and the rows already accepted stay safe.
|
|
64
|
+
* Outage budget with the defaults: the ceiling of 50,000 rows holds
|
|
65
|
+
* 50 seconds at 1,000 rows a second and about 14 hours at one row a
|
|
66
|
+
* second, in 6.5 to 15 MB of memory at the measured 130 to 290 bytes
|
|
67
|
+
* a row.
|
|
68
|
+
*
|
|
69
|
+
* Edge-baseline defaults (every one runs within a 2 GB device):
|
|
70
|
+
*
|
|
71
|
+
* | Setting | Default | Environment variable |
|
|
72
|
+
* |---------------------|------------------|------------------------------|
|
|
73
|
+
* | `flushRows` | 5000 | `QUESTDB_FLUSH_ROWS` |
|
|
74
|
+
* | `flushIntervalMs` | 1000 | `QUESTDB_FLUSH_INTERVAL_MS` |
|
|
75
|
+
* | `bufferCeilingRows` | 10 × `flushRows` | `QUESTDB_BUFFER_CEILING_ROWS`|
|
|
76
|
+
* | `flushDeadlineMs` | derived per flush| `QUESTDB_FLUSH_DEADLINE_MS` |
|
|
77
|
+
* | `stdlibHttp` | true | `QUESTDB_STDLIB_HTTP` |
|
|
78
|
+
* | `requestTimeout` | client default | `QUESTDB_REQUEST_TIMEOUT` |
|
|
79
|
+
* | `retryTimeout` | client default | `QUESTDB_RETRY_TIMEOUT` |
|
|
80
|
+
* | `initBufSize` | client default | `QUESTDB_INIT_BUF_SIZE` |
|
|
81
|
+
* | `maxBufSize` | client default | `QUESTDB_MAX_BUF_SIZE` |
|
|
82
|
+
*
|
|
83
|
+
* `resolve-options.js` merges the config and the environment and
|
|
84
|
+
* explains each default. `ilpUrl` and `pgUrl` fall back to
|
|
85
|
+
* `QUESTDB_ILP_URL` and `QUESTDB_PG_URL`; the credentials come from
|
|
86
|
+
* `QUESTDB_DATABASE`, `QUESTDB_USER` and `QUESTDB_PASSWORD`.
|
|
87
|
+
*
|
|
88
|
+
* Long-running commitments (ADR-018 §12). One timer, created at setup
|
|
89
|
+
* and cleared at shutdown; it does not hold the process open. Every
|
|
90
|
+
* counter is bounded by the ceiling. The per-row path allocates
|
|
91
|
+
* nothing beyond the one derived promise `persist-plan.js` documents.
|
|
92
|
+
* Reconnection and request retries belong to the client. No listeners
|
|
93
|
+
* are attached. `flush-engine.js` carries the detail.
|
|
94
|
+
*
|
|
95
|
+
* The transport (ADR-029). The client's default HTTP library, undici,
|
|
96
|
+
* retries a refused connection without end, and its abort cannot end
|
|
97
|
+
* that retry. So the adapter selects the client's standard-library
|
|
98
|
+
* transport (`stdlib_http=on`) unless `stdlibHttp` is false. With it, a refused connection rejects at once and every
|
|
99
|
+
* request ends within `retryTimeout` plus one request timeout. The
|
|
100
|
+
* adapter owns one `http.Agent` for that transport, keep-alive on one
|
|
101
|
+
* socket, and destroys it after the sender closes on every shutdown
|
|
102
|
+
* path. Destroying the agent ends a request still on its socket, so
|
|
103
|
+
* the process can exit. The agent also closes its socket after 4
|
|
104
|
+
* seconds idle, before QuestDB's own idle close at 5 minutes, so no
|
|
105
|
+
* flush meets a socket the server has just closed. An operator who
|
|
106
|
+
* selects undici accepts that an abandoned flush can keep the process
|
|
107
|
+
* alive.
|
|
108
|
+
*
|
|
109
|
+
* Health (ADR-018 §8). `status` is `red` when not connected or at
|
|
110
|
+
* capacity. It is `yellow` at `pressure >= 0.66`, on an outstanding
|
|
111
|
+
* write error, or after one failed flush. Otherwise it is `green`.
|
|
112
|
+
* `connected` is derived, because the ILP client exposes no socket
|
|
113
|
+
* state. It is false while delivery is paused, after five write errors
|
|
114
|
+
* in a row, after two failed flushes in a row, or after one abandoned
|
|
115
|
+
* flush. The health object carries `pausedSince`, `abandonedFlushes`,
|
|
116
|
+
* `consecutiveFlushFailures`, `lastFlushAt`, and `lastFlushError`.
|
|
117
|
+
* `flush-engine.js` documents the derivation.
|
|
118
|
+
*
|
|
119
|
+
* Deprecated options (ADR-029, removed in 0.8.0). `autoFlushRows` maps
|
|
120
|
+
* to `flushRows` and `idleFlushCheckMs` maps to `flushIntervalMs`.
|
|
121
|
+
* `flushMode`, `idleFlushAfterMs` and `autoFlushIntervalMs` are
|
|
122
|
+
* accepted and ignored. The same holds for their `QUESTDB_*`
|
|
123
|
+
* variables. Setup prints one `DEPRECATED_OPTION` line naming every
|
|
124
|
+
* legacy key in use and what happened to it.
|
|
55
125
|
*
|
|
56
126
|
* `err.code` vocabulary (per-adapter; ADR-018 has each adapter document
|
|
57
127
|
* its own codes in this header):
|
|
58
128
|
*
|
|
59
129
|
* Setup-time throws (ADR-018 fail-fast setup):
|
|
60
130
|
* - `INVALID_CONFIG` — the supplied configuration does not
|
|
61
|
-
* work.
|
|
131
|
+
* work. Ten current sub-cases:
|
|
62
132
|
* (a) required transport URL missing (`ilpUrl`, `pgUrl`);
|
|
63
133
|
* (b) PostgreSQL endpoint answered but rejected the connection
|
|
64
134
|
* — wrong credentials or a protocol-level refusal;
|
|
@@ -72,18 +142,57 @@
|
|
|
72
142
|
* of wedging the sender mid-row at runtime;
|
|
73
143
|
* (e) an insightType uses the reserved column name `assetId`.
|
|
74
144
|
* Composer writes that column itself, from the partition id.
|
|
75
|
-
* Checked at plan build in `persist-plan.js
|
|
145
|
+
* Checked at plan build in `persist-plan.js`;
|
|
146
|
+
* (f) `ilpUrl` or `pgUrl` names `localhost` (ADR-030). The name
|
|
147
|
+
* can resolve to two addresses, `::1` and `127.0.0.1`, and
|
|
148
|
+
* QuestDB may listen on only one. Refused before any socket
|
|
149
|
+
* opens, at the schema (flow definition) and here (the env
|
|
150
|
+
* fallback and direct callers). The message names the
|
|
151
|
+
* literal to set;
|
|
152
|
+
* (g) `ilpUrl` is an IPv6 literal. The client (4.2.0) splits the
|
|
153
|
+
* address on its first colon and cannot read one. `pgUrl`
|
|
154
|
+
* accepts `[::1]:8812`;
|
|
155
|
+
* (h) the ILP client rejected the sender configuration for a
|
|
156
|
+
* reason that is not a network error (`fromConfig`);
|
|
157
|
+
* (i) `bufferCeilingRows` is below twice `flushRows` (ADR-029).
|
|
158
|
+
* Rows in flight count against the ceiling, so it must hold
|
|
159
|
+
* one batch in flight and one batch buffering. A smaller
|
|
160
|
+
* ceiling would refuse every row that arrives during a
|
|
161
|
+
* row-triggered flush. Checked in `resolve-options.js`;
|
|
162
|
+
* (j) `tablePrefix` is not an identifier: letters, digits, `_`
|
|
163
|
+
* and `$`, not starting with a digit. The prefix opens every
|
|
164
|
+
* table name, and `ensure-tables.js` writes that name unquoted
|
|
165
|
+
* in CREATE TABLE. QuestDB reads an unquoted name as one
|
|
166
|
+
* token. So a hyphen, a space, a dot, a semicolon or a
|
|
167
|
+
* backtick ends the statement early with "unexpected token"
|
|
168
|
+
* (verified live on 9.4.3). QuestDB's own rule for a quoted
|
|
169
|
+
* name is wider. Its CREATE TABLE reference says a name with
|
|
170
|
+
* a space or a dot must be double-quoted. The server's
|
|
171
|
+
* `TableUtils.isValidTableName` forbids `? , ' " \ / : ( ) +
|
|
172
|
+
* * % ~`, control characters, and a leading or trailing dot
|
|
173
|
+
* or space. Sources: https://questdb.com/docs/query/sql/create-table/
|
|
174
|
+
* and https://questdb.com/docs/connect/compatibility/ilp/advanced-settings/.
|
|
175
|
+
* The identifier rule is the one the semantics schema applies
|
|
176
|
+
* to asset class, insight type and column names, so every
|
|
177
|
+
* part of a table name follows one rule. Checked at the
|
|
178
|
+
* schema and here, which covers the default prefix (the asset
|
|
179
|
+
* class name) and a direct caller.
|
|
76
180
|
* Operator remediation: fix the supplied config or the relevant
|
|
77
|
-
* `QUESTDB_*` env var. The underlying
|
|
78
|
-
*
|
|
79
|
-
* - `TRANSPORT_UNREACHABLE` —
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
181
|
+
* `QUESTDB_*` env var. The underlying error (sub-cases b and h) is
|
|
182
|
+
* preserved on `err.cause` for diagnostics.
|
|
183
|
+
* - `TRANSPORT_UNREACHABLE` — an endpoint did not answer at setup.
|
|
184
|
+
* Three sources. (1) The setup probe (ADR-030 item 4): before any
|
|
185
|
+
* client is built, `pgUrl` and then `ilpUrl` are probed with one
|
|
186
|
+
* TCP connect per address. A name is resolved with Node's default
|
|
187
|
+
* lookup and every address it resolves to must answer; the message
|
|
188
|
+
* lists each address with its result and names the literal to set.
|
|
189
|
+
* (2) The PostgreSQL connect failed with a Node syscall code
|
|
190
|
+
* (`ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, ...). (3) The ILP
|
|
191
|
+
* client could not be built for the same class of reason.
|
|
192
|
+
* Distinct from `INVALID_CONFIG` per the one split the ADR-018
|
|
193
|
+
* error vocabulary mandates: the connection string may be fine —
|
|
194
|
+
* check the network, the firewall, whether QuestDB is running.
|
|
195
|
+
* Underlying error on `err.cause` for (2) and (3).
|
|
87
196
|
* - `MISSING_ASSET_CLASS` — `assetClass` not provided to `createStorage`.
|
|
88
197
|
* Distinct from `INVALID_CONFIG` because operator remediation differs:
|
|
89
198
|
* they need to add `.assetClass(assetClassDef)` to the flow, not edit env
|
|
@@ -91,26 +200,72 @@
|
|
|
91
200
|
* - `SCHEMA_ERROR` — DDL `CREATE TABLE` failed for a reason other
|
|
92
201
|
* than already-exists (see `ensure-tables.js`).
|
|
93
202
|
*
|
|
94
|
-
* Runtime returns:
|
|
95
|
-
* - `INVALID_INSIGHT_TYPE` —
|
|
203
|
+
* Runtime returns (sync, from `write()`):
|
|
204
|
+
* - `INVALID_INSIGHT_TYPE` — the insightType has no persist plan
|
|
96
205
|
* (typically a config typo or asset-class drift).
|
|
97
|
-
* - `
|
|
98
|
-
*
|
|
206
|
+
* - `STORAGE_FULL` — the buffer is at `bufferCeilingRows`. The
|
|
207
|
+
* row was shed. One shared result object; shedding allocates
|
|
208
|
+
* nothing. Remediation: the endpoint is not taking rows; check it.
|
|
209
|
+
* Raise the ceiling only when memory allows.
|
|
210
|
+
* - `SHUTTING_DOWN` — `shutdown()` has been called; no flusher
|
|
211
|
+
* is left for a new row. One shared result object.
|
|
212
|
+
* - `SEND_FAILED` — the persist plan threw while invoking
|
|
213
|
+
* sender methods (type coercion failure, a client throw). The
|
|
214
|
+
* half-written row is cancelled (mid-row recovery, in
|
|
215
|
+
* `flush-engine.js`).
|
|
99
216
|
* - `INVALID_TIMESTAMP` — reserved; today persist-plan handles invalid
|
|
100
217
|
* timestamps via `onWarning + skip-row`. It may later be surfaced as a
|
|
101
218
|
* hard return error.
|
|
102
219
|
*
|
|
103
220
|
* Shutdown-time throws (ADR-018 — a clean shutdown resolve means
|
|
104
221
|
* everything buffered or in flight was delivered; both carry
|
|
105
|
-
* `dropped: { count }`, exact)
|
|
106
|
-
* (idle, recovery) plus one final flush, raced against `{ timeout }`;
|
|
107
|
-
* its outcome is latched so repeated calls cannot contradict it:
|
|
222
|
+
* `dropped: { count }`, exact):
|
|
108
223
|
* - `DELIVERY_FAILED` — a flush failed; first flush error on `cause`.
|
|
109
224
|
* - `SHUTDOWN_TIMEOUT` — delivery did not settle within the caller's
|
|
110
|
-
* `{ timeout }
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
225
|
+
* `{ timeout }`.
|
|
226
|
+
*
|
|
227
|
+
* Runtime console classification (tokens, not `err.code` values):
|
|
228
|
+
* - `DELIVERY_FAILED` — an engine or recovery flush failed or was
|
|
229
|
+
* abandoned at its deadline, or the client refused a row append, and
|
|
230
|
+
* no `onDeliveryFailure` was given. One `logger.error` line per
|
|
231
|
+
* event, carrying the exact rows lost and the client's message. A
|
|
232
|
+
* flush line ends with the probe finding when a probe ran. The
|
|
233
|
+
* process keeps running: an unattended deployment must report a lost
|
|
234
|
+
* batch, not stop on it. The health surface carries the same fact.
|
|
235
|
+
* - `CIRCUIT_OPEN` — delivery paused or resumed (ADR-029 hold and
|
|
236
|
+
* probe). One `logger.warn` line when a failing probe pauses
|
|
237
|
+
* delivery, naming the held rows and the finding. One `logger.warn`
|
|
238
|
+
* line when a tick probe passes and delivery resumes, at warn so a
|
|
239
|
+
* log transport with a warn floor carries the episode end. Nothing
|
|
240
|
+
* per tick. Remediation: the endpoint refused a TCP connect; check
|
|
241
|
+
* that QuestDB is running and reachable. Rows are held up to the
|
|
242
|
+
* ceiling meanwhile, and health reads red with `pausedSince`.
|
|
243
|
+
* - `DELIVERY_HEALTH` — the delivery ladder changed state. One
|
|
244
|
+
* `logger.warn` line when it enters yellow (the first failed flush),
|
|
245
|
+
* one `logger.error` line when it enters red (two failed in a row,
|
|
246
|
+
* one abandoned, or a failing probe pausing delivery, printed just
|
|
247
|
+
* before the `CIRCUIT_OPEN` line), and one `logger.warn` line when
|
|
248
|
+
* it returns to green, naming the episode length and the rows
|
|
249
|
+
* reported lost in it.
|
|
250
|
+
* Nothing while a state persists, whatever the outage length, and
|
|
251
|
+
* the lines print with or without `onDeliveryFailure`. Remediation:
|
|
252
|
+
* the detail is the client's message or the deadline; a red with no
|
|
253
|
+
* `pausedSince` in health is a server that answers and refuses.
|
|
254
|
+
* - `STORAGE_FULL` (lines) — one `logger.warn` line when the first row
|
|
255
|
+
* of an episode is refused at the ceiling, and one when the buffer
|
|
256
|
+
* has room again, with the count refused. The end is found at the
|
|
257
|
+
* interval tick, so its line can lag the first free slot by one
|
|
258
|
+
* interval. Remediation: the endpoint is not taking rows; read the
|
|
259
|
+
* `CIRCUIT_OPEN` and `DELIVERY_HEALTH` lines beside it, and raise
|
|
260
|
+
* `bufferCeilingRows` when a longer outage must be ridden through.
|
|
261
|
+
* - `DEPRECATED_OPTION` — one `logger.warn` line at setup naming the
|
|
262
|
+
* legacy keys in use (see Deprecated options above).
|
|
263
|
+
* - `ADDRESS_IS_NAME` — `ilpUrl` or `pgUrl` is a name other than
|
|
264
|
+
* `localhost`. One `logger.warn` line per field at setup, before any
|
|
265
|
+
* socket opens (ADR-030). A name is allowed and the adapter
|
|
266
|
+
* proceeds. The line tells the operator that only a literal address
|
|
267
|
+
* is immune to a resolver that changes its answer under a running
|
|
268
|
+
* process.
|
|
114
269
|
* - `CALLBACK_FAILED` — the user's `onDeliveryFailure` itself threw or
|
|
115
270
|
* rejected. The shared callback guard contains the fault (ADR-018).
|
|
116
271
|
* The adapter keeps writing and flushing, and each fault becomes one
|
|
@@ -142,31 +297,22 @@
|
|
|
142
297
|
* extend this section and add corresponding assertions in
|
|
143
298
|
* `assert-columns.js`.
|
|
144
299
|
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
* buffer (the copy-out happens synchronously, before any network I/O; the
|
|
162
|
-
* client documents that an unfinished row stays behind), then reset()
|
|
163
|
-
* clears the buffer — at that point holding only the broken stub — and
|
|
164
|
-
* lowers the client's row-in-progress flags.
|
|
165
|
-
* - The client (4.2.0) has no row-cancel API, while its sibling clients do
|
|
166
|
-
* (.NET CancelRow, Rust/C rewind_to_marker, Java recovers automatically).
|
|
167
|
-
* Upstream issue #60 tracks the gap:
|
|
168
|
-
* https://github.com/questdb/nodejs-questdb-client/issues/60
|
|
169
|
-
* When a release ships cancelRow(), recoverSender() becomes that one call.
|
|
300
|
+
* Row append rejections. The client declares `sender.at()` as `async`,
|
|
301
|
+
* so its promise can reject. With the client trigger off, the only
|
|
302
|
+
* rejection left is the client's own byte ceiling (`max_buf_size`,
|
|
303
|
+
* 100 MiB by default): the append throws inside the async function
|
|
304
|
+
* and arrives as a rejection, and the row never completed.
|
|
305
|
+
* `persist-plan.js` attaches one handler per insight type to that
|
|
306
|
+
* promise and routes the failure to `onDeliveryFailure` as
|
|
307
|
+
* `( err, { tableName } )`, or prints one `DELIVERY_FAILED` line. The
|
|
308
|
+
* next write finds the sender holding the unfinished row, throws, and
|
|
309
|
+
* the mid-row recovery in `flush-engine.js` clears it.
|
|
310
|
+
*
|
|
311
|
+
* Client dependency note. `package.json` pins `@questdb/nodejs-client`
|
|
312
|
+
* to `~4.2.0`. `flush-engine.js` lists the five facts of that release
|
|
313
|
+
* the engine relies on; re-verify each on an upgrade. `persist-plan.js`
|
|
314
|
+
* also imports `SenderBufferV1`, an exported but undocumented member,
|
|
315
|
+
* for name validation at setup only.
|
|
170
316
|
*
|
|
171
317
|
* Integration with persist-if node:
|
|
172
318
|
* - persist-if calls: state.storage.write(insightType, msg, partitionId)
|
|
@@ -201,225 +347,205 @@
|
|
|
201
347
|
* @see https://questdb.com/docs/clients/ingest-node/
|
|
202
348
|
* @see docs/architecture/storage-layer.md
|
|
203
349
|
* @see ADR-018
|
|
350
|
+
* @see ADR-029
|
|
204
351
|
*/
|
|
205
352
|
|
|
353
|
+
import http from 'node:http';
|
|
354
|
+
|
|
206
355
|
import { Sender } from '@questdb/nodejs-client';
|
|
207
356
|
import pg from 'pg';
|
|
208
357
|
|
|
209
358
|
import { ENV_VARS } from '../../env-vars.js';
|
|
210
359
|
import { logger } from '../../logger/index.js';
|
|
211
360
|
import { validators } from '../../utils/validate/index.js';
|
|
212
|
-
import {
|
|
361
|
+
import { probeAddress, describeProbe } from '../../utils/address/probe.js';
|
|
213
362
|
import { buildPersistPlans } from './persist-plan.js';
|
|
214
363
|
import { ensureTables } from './ensure-tables.js';
|
|
215
364
|
import { assertColumnFacts } from './assert-columns.js';
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
/**
|
|
230
|
-
* Console channel for the callback guard: one classified line in this
|
|
231
|
-
* adapter's family. Receives an already-safe detail string, never the
|
|
232
|
-
* raw thrown value.
|
|
233
|
-
*/
|
|
234
|
-
const reportCallbackFault = function ( severity, name, detail ) {
|
|
235
|
-
logger.error(
|
|
236
|
-
`winkComposer/questdb: user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
|
|
237
|
-
);
|
|
238
|
-
}; // reportCallbackFault()
|
|
239
|
-
|
|
240
|
-
// Error results (INVALID_INSIGHT_TYPE, SEND_FAILED) are constructed per-call
|
|
241
|
-
// because each carries dynamic content (the offending insightType name and the
|
|
242
|
-
// underlying err.message respectively). These paths are rare; per-occurrence
|
|
243
|
-
// allocation on errors is acceptable. The RESULT_OK singleton above covers the
|
|
244
|
-
// per-message hot path.
|
|
245
|
-
|
|
246
|
-
/**
|
|
247
|
-
* Health status thresholds. consecutiveWriteErrors crossing the YELLOW
|
|
248
|
-
* threshold elevates status to 'yellow'; crossing the RED threshold flips
|
|
249
|
-
* `connected` to false (and therefore `status` to 'red'). Tuned for "any
|
|
250
|
-
* error is worth flagging" + "sustained errors mean the transport is gone."
|
|
251
|
-
*/
|
|
252
|
-
const HEALTH_ERROR_YELLOW_THRESHOLD = 1;
|
|
253
|
-
const HEALTH_ERROR_RED_THRESHOLD = 5;
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* Pressure threshold above which `status` elevates to at least 'yellow'.
|
|
257
|
-
* The value matches the pressure-aware-yield design (ADR-020, still a
|
|
258
|
-
* Draft). That design proposes 0.66 as the pressure level where the
|
|
259
|
-
* flow would start yielding to let sinks drain; the number still needs
|
|
260
|
-
* benchmark confirmation. Today the yield trigger is time-only (ADR-024), so
|
|
261
|
-
* this alignment is forward-looking, not a description of current yield
|
|
262
|
-
* behaviour. Numeric — the constant exists once here rather than scattered
|
|
263
|
-
* as a magic number.
|
|
264
|
-
*/
|
|
265
|
-
const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
|
|
365
|
+
import { resolveOptions, deprecationMessage } from './resolve-options.js';
|
|
366
|
+
import {
|
|
367
|
+
isAllowedIlpUrl,
|
|
368
|
+
isAllowedPgUrl,
|
|
369
|
+
assertNotLocalhost,
|
|
370
|
+
assertIlpNotIPv6,
|
|
371
|
+
warnIfName,
|
|
372
|
+
assertReachable,
|
|
373
|
+
pgConnectionTarget,
|
|
374
|
+
NETWORK_ERROR_CODES,
|
|
375
|
+
buildSender
|
|
376
|
+
} from './address-policy.js';
|
|
377
|
+
import { createFlushEngine } from './flush-engine.js';
|
|
266
378
|
|
|
267
379
|
// ============================================================================
|
|
268
380
|
// CONFIGURATION BUILDER
|
|
269
381
|
// ============================================================================
|
|
270
382
|
|
|
271
383
|
/**
|
|
272
|
-
* Build QuestDB Sender configuration
|
|
384
|
+
* Build the QuestDB Sender configuration string. The client's own flush
|
|
385
|
+
* trigger is always off (ADR-029): composer starts every flush itself,
|
|
386
|
+
* so the client never sends a batch this module did not ask for. The
|
|
387
|
+
* transport is stated either way, `stdlib_http=on` unless `stdlibHttp`
|
|
388
|
+
* is false, so a printed config reads the same however it was set.
|
|
273
389
|
*
|
|
274
|
-
* @param {Object} options -
|
|
390
|
+
* @param {Object} options - Resolved settings
|
|
275
391
|
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
276
|
-
* @param {
|
|
277
|
-
* @param {number} [options.
|
|
278
|
-
* @param {number} [options.
|
|
279
|
-
* @param {number} [options.
|
|
280
|
-
* @param {number} [options.retryTimeout] -
|
|
392
|
+
* @param {boolean} [options.stdlibHttp] - False selects undici; anything else the standard library
|
|
393
|
+
* @param {number} [options.initBufSize] - Initial client buffer size in bytes
|
|
394
|
+
* @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
|
|
395
|
+
* @param {number} [options.requestTimeout] - Client request timeout in ms
|
|
396
|
+
* @param {number} [options.retryTimeout] - Client retry window in ms
|
|
281
397
|
* @returns {string} Configuration string for Sender.fromConfig()
|
|
282
398
|
*/
|
|
283
399
|
const buildSenderConfig = function ( options ) {
|
|
284
|
-
const { ilpUrl,
|
|
400
|
+
const { ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout } = options;
|
|
285
401
|
|
|
286
|
-
|
|
287
|
-
let config = `http::addr=${ilpUrl};`;
|
|
402
|
+
let config = `http::addr=${ilpUrl};auto_flush=off;stdlib_http=${( stdlibHttp === false ) ? 'off' : 'on'};`;
|
|
288
403
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
// Use QuestDB's built-in auto-flush
|
|
292
|
-
if ( autoFlushRows !== undefined ) {
|
|
293
|
-
config += `auto_flush_rows=${autoFlushRows};`;
|
|
294
|
-
}
|
|
295
|
-
if ( autoFlushIntervalMs !== undefined ) {
|
|
296
|
-
config += `auto_flush_interval=${autoFlushIntervalMs};`;
|
|
297
|
-
}
|
|
298
|
-
} else {
|
|
299
|
-
// Manual mode: disable auto-flush, we control timing
|
|
300
|
-
config += 'auto_flush=off;';
|
|
404
|
+
if ( initBufSize !== undefined ) {
|
|
405
|
+
config += `init_buf_size=${initBufSize};`;
|
|
301
406
|
}
|
|
302
|
-
|
|
303
|
-
// Optional buffer/retry settings
|
|
304
407
|
if ( maxBufSize !== undefined ) {
|
|
305
|
-
config += `
|
|
408
|
+
config += `max_buf_size=${maxBufSize};`;
|
|
409
|
+
}
|
|
410
|
+
if ( requestTimeout !== undefined ) {
|
|
411
|
+
config += `request_timeout=${requestTimeout};`;
|
|
306
412
|
}
|
|
307
413
|
if ( retryTimeout !== undefined ) {
|
|
308
414
|
config += `retry_timeout=${retryTimeout};`;
|
|
309
415
|
}
|
|
310
416
|
|
|
311
417
|
return config;
|
|
312
|
-
};
|
|
418
|
+
}; // buildSenderConfig()
|
|
313
419
|
|
|
314
420
|
// ============================================================================
|
|
315
|
-
//
|
|
421
|
+
// THE TRANSPORT
|
|
316
422
|
// ============================================================================
|
|
317
423
|
|
|
318
424
|
/**
|
|
319
|
-
*
|
|
320
|
-
*
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
328
|
-
*
|
|
329
|
-
* @returns {Promise} settles with the flush, or rejects SHUTDOWN_TIMEOUT
|
|
425
|
+
* How long an idle socket stays open before the adapter closes it.
|
|
426
|
+
* QuestDB closes an idle connection after 300 seconds and announces no
|
|
427
|
+
* keep-alive timeout, so Node gets no hint. A flush that starts in the
|
|
428
|
+
* moment between the server's close and Node's notice of it writes
|
|
429
|
+
* into a dead socket, and this transport does not retry a reset. That
|
|
430
|
+
* would report one batch lost for an outage that never happened.
|
|
431
|
+
* Closing first removes that moment (ADR-029). The value sits below
|
|
432
|
+
* QuestDB's advertised keep-alive of 5 seconds and below the 5-second
|
|
433
|
+
* timeout of Node's own default agent. Node applies it to free sockets
|
|
434
|
+
* only; a request in flight keeps the client's own request timeout.
|
|
330
435
|
*/
|
|
331
|
-
const
|
|
332
|
-
if ( !( timeoutMs > 0 ) ) {
|
|
333
|
-
return flushPromise;
|
|
334
|
-
}
|
|
335
|
-
return new Promise( function ( resolve, reject ) {
|
|
336
|
-
const timer = setTimeout( function () {
|
|
337
|
-
const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
|
|
338
|
-
err.code = 'SHUTDOWN_TIMEOUT';
|
|
339
|
-
reject( err );
|
|
340
|
-
}, timeoutMs );
|
|
341
|
-
timer.unref();
|
|
342
|
-
flushPromise.then(
|
|
343
|
-
function ( value ) {
|
|
344
|
-
clearTimeout( timer );
|
|
345
|
-
resolve( value );
|
|
346
|
-
},
|
|
436
|
+
const IDLE_SOCKET_TIMEOUT_MS = 4000;
|
|
347
437
|
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
/* c8 ignore stop */
|
|
362
|
-
);
|
|
363
|
-
} );
|
|
364
|
-
}; // raceFlushTimeout()
|
|
438
|
+
/**
|
|
439
|
+
* Builds the one HTTP agent the standard-library transport uses. One
|
|
440
|
+
* socket is enough, because composer starts one flush at a time
|
|
441
|
+
* (ADR-029). Keep-alive reuses that socket from flush to flush, and
|
|
442
|
+
* the adapter closes it after `IDLE_SOCKET_TIMEOUT_MS` without a flush.
|
|
443
|
+
* The adapter destroys the agent at shutdown, which ends any request
|
|
444
|
+
* still on the socket.
|
|
445
|
+
*
|
|
446
|
+
* @returns {http.Agent} The agent
|
|
447
|
+
*/
|
|
448
|
+
const createKeepAliveAgent = function () {
|
|
449
|
+
return new http.Agent( { keepAlive: true, maxSockets: 1, timeout: IDLE_SOCKET_TIMEOUT_MS } );
|
|
450
|
+
}; // createKeepAliveAgent()
|
|
365
451
|
|
|
366
452
|
/**
|
|
367
|
-
*
|
|
368
|
-
*
|
|
369
|
-
*
|
|
370
|
-
*
|
|
371
|
-
*
|
|
372
|
-
*
|
|
373
|
-
*
|
|
453
|
+
* Opens the ILP transport: the agent when the standard-library
|
|
454
|
+
* transport is selected, then the sender. Returns the sender and the
|
|
455
|
+
* function that closes both at the end of the drain. A failed sender
|
|
456
|
+
* build destroys the agent before the classified error goes on, so no
|
|
457
|
+
* agent outlives a failed setup.
|
|
458
|
+
*
|
|
459
|
+
* `closeTransport` closes the sender first, then destroys the agent
|
|
460
|
+
* whatever the sender's close did. Destroying the agent ends any
|
|
461
|
+
* request still on its socket, so the process can exit (ADR-029). With
|
|
462
|
+
* undici there is no agent, and `closeTransport` is the sender's close.
|
|
463
|
+
*
|
|
464
|
+
* @param {Object} parts - The transport inputs
|
|
465
|
+
* @param {Object} parts.SenderClass - The client's Sender class
|
|
466
|
+
* @param {string} parts.senderConfig - The sender configuration string
|
|
467
|
+
* @param {string} parts.ilpUrl - The configured `ilpUrl`, for messages
|
|
468
|
+
* @param {boolean} parts.stdlibHttp - Whether the standard-library transport is selected
|
|
469
|
+
* @param {function} parts.createAgent - Builds the agent
|
|
470
|
+
* @returns {Promise<{sender: Object, closeTransport: function}>} The sender and its close
|
|
471
|
+
* @throws {Error} TRANSPORT_UNREACHABLE or INVALID_CONFIG from `buildSender`
|
|
374
472
|
*/
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
473
|
+
const openTransport = async function ( { SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent } ) {
|
|
474
|
+
if ( stdlibHttp === false ) {
|
|
475
|
+
const sender = await buildSender( SenderClass, senderConfig, ilpUrl, null );
|
|
476
|
+
const closeTransport = function () {
|
|
477
|
+
return sender.close();
|
|
478
|
+
}; // closeTransport()
|
|
479
|
+
return { sender, closeTransport };
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
const agent = createAgent();
|
|
483
|
+
let sender;
|
|
484
|
+
try {
|
|
485
|
+
sender = await buildSender( SenderClass, senderConfig, ilpUrl, agent );
|
|
486
|
+
} catch ( err ) {
|
|
487
|
+
agent.destroy();
|
|
488
|
+
throw err;
|
|
489
|
+
}
|
|
490
|
+
const closeTransport = async function () {
|
|
491
|
+
try {
|
|
492
|
+
await sender.close();
|
|
493
|
+
} finally {
|
|
494
|
+
agent.destroy();
|
|
495
|
+
}
|
|
496
|
+
}; // closeTransport()
|
|
497
|
+
return { sender, closeTransport };
|
|
498
|
+
}; // openTransport()
|
|
499
|
+
|
|
500
|
+
// ============================================================================
|
|
501
|
+
// STORAGE FACTORY
|
|
502
|
+
// ============================================================================
|
|
385
503
|
|
|
386
504
|
/**
|
|
387
505
|
* Create QuestDB storage adapter.
|
|
388
506
|
*
|
|
507
|
+
* The options are resolved by `resolve-options.js` (config over
|
|
508
|
+
* environment over default; legacy aliases mapped or ignored). Only the
|
|
509
|
+
* keys that reach the engine are listed here; the resolver documents
|
|
510
|
+
* the rest.
|
|
511
|
+
*
|
|
389
512
|
* @param {Object} assetClass - Asset class definition with columns and insightTypes
|
|
390
513
|
* @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
|
|
391
514
|
* @param {Object} options - Configuration options
|
|
392
|
-
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
393
|
-
* @param {string} options.pgUrl - PostgreSQL endpoint (host:port)
|
|
394
|
-
* @param {
|
|
395
|
-
* @param {number} [options.
|
|
396
|
-
* @param {number} [options.
|
|
397
|
-
* @param {number} [options.
|
|
398
|
-
* @param {
|
|
399
|
-
* @param {number} [options.
|
|
400
|
-
* @param {number} [options.retryTimeout] -
|
|
515
|
+
* @param {string} [options.ilpUrl] - ILP endpoint (host:port)
|
|
516
|
+
* @param {string} [options.pgUrl] - PostgreSQL endpoint (host:port)
|
|
517
|
+
* @param {number} [options.flushRows=5000] - Rows that start a flush from inside write()
|
|
518
|
+
* @param {number} [options.flushIntervalMs=1000] - Period of the flush timer
|
|
519
|
+
* @param {number} [options.bufferCeilingRows] - Most rows held; default ten times flushRows
|
|
520
|
+
* @param {number} [options.flushDeadlineMs] - Fixed deadline per flush; derived from the rows when unset
|
|
521
|
+
* @param {boolean} [options.stdlibHttp=true] - The standard-library transport; false selects undici
|
|
522
|
+
* @param {number} [options.requestTimeout] - Client request timeout in ms
|
|
523
|
+
* @param {number} [options.retryTimeout] - Client retry window in ms
|
|
524
|
+
* @param {number} [options.initBufSize] - Initial client buffer size in bytes
|
|
525
|
+
* @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
|
|
401
526
|
* @param {string} [options.partitionBy='DAY'] - Table partition interval
|
|
402
|
-
* @param {function} [options.onWarning] - Warning callback for
|
|
527
|
+
* @param {function} [options.onWarning] - Warning callback for skipped values
|
|
528
|
+
* @param {function} [options.onDeliveryFailure] - Called once per lost flush
|
|
403
529
|
* @param {Object} [deps={}] - Injectable dependencies (for testing)
|
|
404
530
|
* @param {Object} [deps.SenderClass] - QuestDB Sender class (default: @questdb/nodejs-client Sender)
|
|
405
531
|
* @param {Object} [deps.PgClientClass] - PostgreSQL Client class (default: pg.Client)
|
|
406
|
-
* @
|
|
532
|
+
* @param {function} [deps.probeFn] - Setup probe (default: `probeAddress`, ADR-030)
|
|
533
|
+
* @param {function} [deps.createAgent] - Builds the transport agent (default: `createKeepAliveAgent`)
|
|
534
|
+
* @returns {Promise<Object>} Storage adapter with write, flush, shutdown, getPressure, getHealth
|
|
407
535
|
*/
|
|
408
536
|
const createQuestDBStorage = async function ( assetClass, tablePrefix, options, deps = {} ) {
|
|
537
|
+
// Options become settings here, once (ADR-029). A ceiling below the
|
|
538
|
+
// threshold fails setup inside the resolver with INVALID_CONFIG.
|
|
539
|
+
const { settings, deprecations } = resolveOptions( options, ENV_VARS );
|
|
409
540
|
const {
|
|
410
|
-
ilpUrl
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
retryTimeout = ENV_VARS.questdbRetryTimeout,
|
|
419
|
-
partitionBy = 'DAY',
|
|
420
|
-
onWarning,
|
|
421
|
-
onDeliveryFailure
|
|
422
|
-
} = options;
|
|
541
|
+
ilpUrl, pgUrl, stdlibHttp, requestTimeout, retryTimeout, initBufSize, maxBufSize,
|
|
542
|
+
partitionBy, onWarning, onDeliveryFailure
|
|
543
|
+
} = settings;
|
|
544
|
+
|
|
545
|
+
// One line names every legacy key in use and what happened to it.
|
|
546
|
+
if ( deprecations.length > 0 ) {
|
|
547
|
+
logger.warn( deprecationMessage( deprecations ) );
|
|
548
|
+
}
|
|
423
549
|
|
|
424
550
|
// Runtime validation — required from either DSL config or ENV_VARS.
|
|
425
551
|
// Per ADR-018, setup-time throws carry classified err.code.
|
|
@@ -434,35 +560,56 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
434
560
|
throw err;
|
|
435
561
|
}
|
|
436
562
|
|
|
563
|
+
// The prefix opens every table name, and CREATE TABLE writes that
|
|
564
|
+
// name unquoted (header sub-case (j)). The schema checks a flow's
|
|
565
|
+
// config. This check covers a direct caller and the default, which
|
|
566
|
+
// is the asset class name.
|
|
567
|
+
if ( !validators.identifier( tablePrefix ) ) {
|
|
568
|
+
const err = new Error(
|
|
569
|
+
`winkComposer/questdb: tablePrefix '${tablePrefix}' is not a valid table name prefix [INVALID_CONFIG]: ` +
|
|
570
|
+
'use letters, digits, _ and $ only, and do not start with a digit'
|
|
571
|
+
);
|
|
572
|
+
err.code = 'INVALID_CONFIG';
|
|
573
|
+
throw err;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
// Address policy (ADR-030), before any socket opens. `localhost` is
|
|
577
|
+
// refused. An IPv6 literal is refused for the ILP path. A name gets
|
|
578
|
+
// one warning per field. See header sub-cases (f), (g) and the
|
|
579
|
+
// ADDRESS_IS_NAME console token.
|
|
580
|
+
const ilpAddress = assertNotLocalhost( 'ilpUrl', ilpUrl, 'QUESTDB_ILP_URL' );
|
|
581
|
+
const pgAddress = assertNotLocalhost( 'pgUrl', pgUrl, 'QUESTDB_PG_URL' );
|
|
582
|
+
assertIlpNotIPv6( ilpAddress );
|
|
583
|
+
warnIfName( 'ilpUrl', ilpAddress );
|
|
584
|
+
warnIfName( 'pgUrl', pgAddress );
|
|
585
|
+
|
|
437
586
|
// Injectable dependencies with defaults
|
|
438
587
|
const {
|
|
439
588
|
SenderClass = Sender,
|
|
440
|
-
PgClientClass = pg.Client
|
|
589
|
+
PgClientClass = pg.Client,
|
|
590
|
+
probeFn = probeAddress,
|
|
591
|
+
createAgent = createKeepAliveAgent
|
|
441
592
|
} = deps;
|
|
442
593
|
|
|
443
594
|
// Build persist plans (pre-compiled closures). The callbacks go in
|
|
444
595
|
// RAW: buildPersistPlans validates them itself and arms its own
|
|
445
|
-
// guarded copy for the
|
|
596
|
+
// guarded copy for the row-append site. Handing it a pre-wrapped
|
|
446
597
|
// function would defeat that validation (the guard turns a
|
|
447
598
|
// non-function into null instead of the fail-fast INVALID_CONFIG).
|
|
599
|
+
// The engine wraps its own copy after this validation (ADR-027:
|
|
600
|
+
// wrap once, after the site's own validation).
|
|
448
601
|
const persistPlans = buildPersistPlans( assetClass, tablePrefix, { onWarning, onDeliveryFailure } );
|
|
449
602
|
|
|
450
|
-
//
|
|
451
|
-
//
|
|
452
|
-
//
|
|
453
|
-
|
|
454
|
-
// (ADR-018) and its cost stays inside the handler. Absent stays
|
|
455
|
-
// null, so the no-handler console/throw fallbacks keep their
|
|
456
|
-
// meaning.
|
|
457
|
-
const safeOnDeliveryFailure = wrapCallback( onDeliveryFailure, {
|
|
458
|
-
name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
|
|
459
|
-
} );
|
|
603
|
+
// Setup probe, PostgreSQL side (ADR-030 item 4): every address the
|
|
604
|
+
// endpoint resolves to must answer before the client opens. The
|
|
605
|
+
// ILP side is probed below, before the sender is built.
|
|
606
|
+
await assertReachable( 'pgUrl', pgAddress, probeFn );
|
|
460
607
|
|
|
461
608
|
// Ensure tables exist via PostgreSQL wire protocol
|
|
462
|
-
const
|
|
609
|
+
const { host: pgHost, port: pgPort } = pgConnectionTarget( pgUrl, pgAddress );
|
|
463
610
|
const pgClient = new PgClientClass( {
|
|
464
611
|
host: pgHost,
|
|
465
|
-
port:
|
|
612
|
+
port: pgPort,
|
|
466
613
|
database: ENV_VARS.questdbDatabase,
|
|
467
614
|
user: ENV_VARS.questdbUser,
|
|
468
615
|
password: ENV_VARS.questdbPassword
|
|
@@ -470,15 +617,16 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
470
617
|
|
|
471
618
|
// Wrap the PostgreSQL connect so the failure carries a classified
|
|
472
619
|
// `err.code` an operator can route on. The ADR-018 error vocabulary
|
|
473
|
-
// mandates one split here
|
|
474
|
-
// code like ECONNREFUSED
|
|
475
|
-
//
|
|
476
|
-
// network, the firewall, whether the service is
|
|
477
|
-
// connection string may be
|
|
478
|
-
//
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
//
|
|
620
|
+
// mandates one split here. An endpoint that does not answer is
|
|
621
|
+
// TRANSPORT_UNREACHABLE: a Node syscall code like ECONNREFUSED (see
|
|
622
|
+
// NETWORK_ERROR_CODES in address-policy.js). The remediation there
|
|
623
|
+
// is "check the network, the firewall, whether the service is
|
|
624
|
+
// running", and the connection string may be fine.
|
|
625
|
+
//
|
|
626
|
+
// Everything else stays INVALID_CONFIG: an auth failure (the host
|
|
627
|
+
// answered), a protocol error, an unclassified throw. The fix there
|
|
628
|
+
// is the supplied config. The underlying error is preserved on
|
|
629
|
+
// `err.cause` for diagnostics either way.
|
|
482
630
|
try {
|
|
483
631
|
await pgClient.connect();
|
|
484
632
|
} catch ( connErr ) {
|
|
@@ -498,539 +646,52 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
498
646
|
await pgClient.end();
|
|
499
647
|
}
|
|
500
648
|
|
|
501
|
-
//
|
|
649
|
+
// The sender config. The client trigger is off and the transport is
|
|
650
|
+
// stated: see buildSenderConfig.
|
|
502
651
|
const senderConfig = buildSenderConfig( {
|
|
503
|
-
ilpUrl,
|
|
504
|
-
flushMode,
|
|
505
|
-
autoFlushRows,
|
|
506
|
-
autoFlushIntervalMs,
|
|
507
|
-
maxBufSize,
|
|
508
|
-
retryTimeout
|
|
652
|
+
ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout
|
|
509
653
|
} );
|
|
510
654
|
|
|
511
|
-
//
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
// The client's flush() copies its rows out of the buffer synchronously
|
|
516
|
-
// (copy-out) before any network I/O, so the moment a flush is called
|
|
517
|
-
// its rows are no longer buffered — they move to the in-flight tally
|
|
518
|
-
// below. Conflating the two quantities in one counter let shutdown
|
|
519
|
-
// report clean over a hung idle flush.
|
|
520
|
-
let lastWriteTime = 0;
|
|
521
|
-
let idleFlushTimer = null;
|
|
522
|
-
let bufferedRows = 0;
|
|
523
|
-
|
|
524
|
-
// Rows inside flush copies that have not settled yet, plus the flush
|
|
525
|
-
// promises carrying them. getPressure() adds this to bufferedRows (a
|
|
526
|
-
// hung flush is undelivered data and must read as pressure), and
|
|
527
|
-
// shutdown() settles these entries instead of firing a blind flush
|
|
528
|
-
// at a buffer the copy-out already emptied.
|
|
529
|
-
let inFlightRows = 0;
|
|
530
|
-
const inFlightFlushes = new Set();
|
|
531
|
-
|
|
532
|
-
// Shutdown outcome, latched on the first call. A lossy shutdown's
|
|
533
|
-
// failed flush already emptied the buffer via copy-out, so a re-run
|
|
534
|
-
// would find nothing to flush and resolve clean — contradicting the
|
|
535
|
-
// recorded loss. Every caller gets the first call's promise instead.
|
|
536
|
-
let shutdownPromise = null;
|
|
537
|
-
|
|
538
|
-
// Health state — drives getHealth()'s status/connected derivation.
|
|
539
|
-
// shuttingDown flips true at the start of shutdown(); never resets.
|
|
540
|
-
// consecutiveWriteErrors increments on every write() catch and resets to 0
|
|
541
|
-
// on every successful persist plan call (recovery). Pure derived signal —
|
|
542
|
-
// no extra hot-path allocation; just two integer/boolean operations.
|
|
543
|
-
let shuttingDown = false;
|
|
544
|
-
let consecutiveWriteErrors = 0;
|
|
545
|
-
|
|
546
|
-
// One idle flush in flight at a time. Against an unreachable server a
|
|
547
|
-
// flush can never settle (the client's retry loop — see the mid-row
|
|
548
|
-
// recovery section in the file header); without this guard every check
|
|
549
|
-
// interval would start ANOTHER stuck flush. A hung flush leaves the
|
|
550
|
-
// flag set, the buffer keeps filling, and the condition surfaces as
|
|
551
|
-
// rising pressure in getHealth() — visible, not multiplied.
|
|
552
|
-
let idleFlushInFlight = false;
|
|
553
|
-
|
|
554
|
-
/**
|
|
555
|
-
* Registers a flush the moment it is called. Copy-out means the rows
|
|
556
|
-
* leave the buffer NOW, delivered or not, so the caller hands them
|
|
557
|
-
* over in the same breath. Settlement — either way — removes them
|
|
558
|
-
* from the in-flight tally: resolved means delivered; rejected means
|
|
559
|
-
* lost, and loss REPORTING stays at the call site (the idle flush
|
|
560
|
-
* logs or routes to onDeliveryFailure, recovery routes to
|
|
561
|
-
* onDeliveryFailure, shutdown throws classified).
|
|
562
|
-
*
|
|
563
|
-
* @param {Promise} flushPromise - the just-fired sender.flush()
|
|
564
|
-
* @param {number} rows - row count the flush copy carries
|
|
565
|
-
* @returns {{promise: Promise, rows: number}} the tracked entry
|
|
566
|
-
*/
|
|
567
|
-
const trackFlush = function ( flushPromise, rows ) {
|
|
568
|
-
const entry = { promise: flushPromise, rows };
|
|
569
|
-
inFlightRows += rows;
|
|
570
|
-
inFlightFlushes.add( entry );
|
|
571
|
-
const settle = function () {
|
|
572
|
-
inFlightFlushes.delete( entry );
|
|
573
|
-
inFlightRows -= rows;
|
|
574
|
-
};
|
|
575
|
-
flushPromise.then( settle, settle );
|
|
576
|
-
return entry;
|
|
577
|
-
}; // trackFlush()
|
|
578
|
-
|
|
579
|
-
/**
|
|
580
|
-
* Check and flush if idle for idleFlushAfterMs.
|
|
581
|
-
*
|
|
582
|
-
* Runs in BOTH modes (see longer comment block at the setInterval call
|
|
583
|
-
* below). The rows move to the in-flight tally the moment the flush
|
|
584
|
-
* fires; a successful settle clears them, which also bounds the
|
|
585
|
-
* worst-case pressure-counter lag in auto mode (when QuestDB's silent
|
|
586
|
-
* `auto_flush_interval` trigger fires without us observing it, this
|
|
587
|
-
* is the self-healing point).
|
|
588
|
-
*/
|
|
589
|
-
const checkIdleFlush = async function () {
|
|
590
|
-
if ( idleFlushInFlight || bufferedRows === 0 ) {
|
|
591
|
-
return;
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
const now = Date.now();
|
|
595
|
-
const idleTime = now - lastWriteTime;
|
|
596
|
-
|
|
597
|
-
if ( idleTime >= idleFlushAfterMs ) {
|
|
598
|
-
idleFlushInFlight = true;
|
|
599
|
-
const rows = bufferedRows;
|
|
600
|
-
bufferedRows = 0;
|
|
601
|
-
const entry = trackFlush( sender.flush(), rows );
|
|
602
|
-
try {
|
|
603
|
-
await entry.promise;
|
|
604
|
-
} catch ( err ) {
|
|
605
|
-
// Copy-out: the failed flush carried the rows with it —
|
|
606
|
-
// the buffer no longer holds them and a retry cannot
|
|
607
|
-
// resend them. They are lost, and this is the report
|
|
608
|
-
// (the settle handler already cleared them from
|
|
609
|
-
// pressure). Same routing policy as the recovery flush:
|
|
610
|
-
// the caller owns the response when it asked to.
|
|
611
|
-
if ( safeOnDeliveryFailure ) {
|
|
612
|
-
safeOnDeliveryFailure( err, { idleFlush: true, rowsLost: rows } );
|
|
613
|
-
} else {
|
|
614
|
-
logger.error(
|
|
615
|
-
`winkComposer/questdb: idle flush failed; ${rows} buffered row(s) lost: ${err.message}`
|
|
616
|
-
);
|
|
617
|
-
}
|
|
618
|
-
} finally {
|
|
619
|
-
idleFlushInFlight = false;
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
};
|
|
623
|
-
|
|
624
|
-
// Start idle flush timer as safety net for "data stopped flowing" case.
|
|
625
|
-
// QuestDB's auto_flush_interval only checks elapsed time when NEW data is added -
|
|
626
|
-
// it is NOT a real background timer. If no more data arrives, the buffer sits
|
|
627
|
-
// indefinitely. QuestDB docs recommend: "implement your own timer-based logic."
|
|
628
|
-
// See: https://py-questdb-client.readthedocs.io/en/stable/conf.html
|
|
629
|
-
//
|
|
630
|
-
// This timer runs in BOTH modes:
|
|
631
|
-
// - Auto mode: Safety net for when data stops flowing (QuestDB won't flush on its own)
|
|
632
|
-
// - Manual mode: Primary flush mechanism (QuestDB auto-flush is disabled)
|
|
633
|
-
idleFlushTimer = setInterval( checkIdleFlush, idleFlushCheckMs );
|
|
634
|
-
// Don't block process exit
|
|
635
|
-
idleFlushTimer.unref();
|
|
636
|
-
|
|
637
|
-
// ========================================================================
|
|
638
|
-
// STORAGE INTERFACE
|
|
639
|
-
// ========================================================================
|
|
640
|
-
|
|
641
|
-
/**
|
|
642
|
-
* Cancels a half-written ILP row after a mid-row throw, so the NEXT write
|
|
643
|
-
* starts clean (see "Mid-row recovery" in the file header for the full
|
|
644
|
-
* account and the upstream issue link).
|
|
645
|
-
*
|
|
646
|
-
* flush() copies every completed row out of the buffer synchronously and
|
|
647
|
-
* sends them in the background; the unfinished row stays behind. reset()
|
|
648
|
-
* then clears the buffer — only the broken stub remains at that point —
|
|
649
|
-
* and lowers the client's row-in-progress flags. Net effect: the broken
|
|
650
|
-
* row vanishes, every good row is on its way, the sender accepts the
|
|
651
|
-
* next write.
|
|
652
|
-
*
|
|
653
|
-
* The early flush carries real data. If its send ultimately fails (after
|
|
654
|
-
* the client's own retries), that loss is reported like any other
|
|
655
|
-
* delivery failure: routed to `onDeliveryFailure` when provided,
|
|
656
|
-
* otherwise thrown as a classified unhandled rejection — the same policy
|
|
657
|
-
* as persist-plan's sender.at() failure path. Never silent.
|
|
658
|
-
*/
|
|
659
|
-
const recoverSender = function () {
|
|
660
|
-
try {
|
|
661
|
-
// flush() is an async function: it can never throw synchronously,
|
|
662
|
-
// and the rows it sends live in its own copy of the buffer.
|
|
663
|
-
// Tracked like every flush, so shutdown settles it and its
|
|
664
|
-
// rows stay visible as pressure until it settles.
|
|
665
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
666
|
-
sender.reset();
|
|
667
|
-
bufferedRows = 0;
|
|
668
|
-
|
|
669
|
-
entry.promise.catch( ( flushErr ) => {
|
|
670
|
-
if ( safeOnDeliveryFailure ) {
|
|
671
|
-
safeOnDeliveryFailure( flushErr, { recovery: true } );
|
|
672
|
-
return;
|
|
673
|
-
}
|
|
674
|
-
const failure = new Error(
|
|
675
|
-
`winkComposer/questdb: recovery flush failed after a mid-row write error: ${flushErr.message}. ` +
|
|
676
|
-
'Completed rows in that batch were dropped. Provide an `onDeliveryFailure` callback ' +
|
|
677
|
-
'in the storage config to handle these explicitly.'
|
|
678
|
-
);
|
|
679
|
-
failure.code = 'DELIVERY_FAILED';
|
|
680
|
-
failure.cause = flushErr;
|
|
681
|
-
throw failure;
|
|
682
|
-
} );
|
|
683
|
-
} catch ( recoveryErr ) {
|
|
684
|
-
// Defensive: with the 4.2.0 client neither call can throw here
|
|
685
|
-
// (flush is async, reset is trivial buffer bookkeeping). If a
|
|
686
|
-
// future client changes that, the sender may stay wedged — but
|
|
687
|
-
// write() must still return its classified result rather than
|
|
688
|
-
// throw (ADR-018: the hot path never throws), and the
|
|
689
|
-
// failure must be visible.
|
|
690
|
-
logger.error( `winkComposer/questdb: sender recovery failed: ${recoveryErr.message}` );
|
|
691
|
-
}
|
|
692
|
-
}; // recoverSender()
|
|
693
|
-
|
|
694
|
-
/**
|
|
695
|
-
* Write a message to QuestDB for a given insightType.
|
|
696
|
-
*
|
|
697
|
-
* Sync hot-path return per ADR-018 / ADR-013:
|
|
698
|
-
* - `RESULT_OK` (singleton) on successful enqueue.
|
|
699
|
-
* - `{ ok: false, error: { code: 'INVALID_INSIGHT_TYPE', message } }`
|
|
700
|
-
* when the insightType has no persist plan (config error).
|
|
701
|
-
* - `{ ok: false, error: { code: 'SEND_FAILED', message } }` when the
|
|
702
|
-
* persist plan throws (sender method failure: ILP buffer overflow,
|
|
703
|
-
* type coercion failure, etc.).
|
|
704
|
-
*
|
|
705
|
-
* Never throws. Async tryFlush rejections inside `sender.at()` are
|
|
706
|
-
* contained at the call site in `persist-plan.js` (see file header).
|
|
707
|
-
*
|
|
708
|
-
* @param {string} insightType - SignalType name (must exist in assetClass)
|
|
709
|
-
* @param {Object} message - Message with column values
|
|
710
|
-
* @param {string} partitionId - Partition identifier (stored as SYMBOL)
|
|
711
|
-
* @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
|
|
712
|
-
*/
|
|
713
|
-
const write = function ( insightType, message, partitionId ) {
|
|
714
|
-
if ( shuttingDown ) {
|
|
715
|
-
// The idle timer is already stopped and the final flush may
|
|
716
|
-
// have run: a row accepted now would have no flusher left.
|
|
717
|
-
// Refuse loudly instead of stranding it — uniform with the
|
|
718
|
-
// MQTT emitter's SHUTTING_DOWN refusal.
|
|
719
|
-
return {
|
|
720
|
-
ok: false,
|
|
721
|
-
error: {
|
|
722
|
-
code: 'SHUTTING_DOWN',
|
|
723
|
-
message: 'winkComposer/questdb: write rejected — storage is shutting down'
|
|
724
|
-
}
|
|
725
|
-
};
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
const persistPlan = persistPlans[ insightType ];
|
|
729
|
-
|
|
730
|
-
if ( !persistPlan ) {
|
|
731
|
-
return {
|
|
732
|
-
ok: false,
|
|
733
|
-
error: {
|
|
734
|
-
code: 'INVALID_INSIGHT_TYPE',
|
|
735
|
-
message: `No persist plan for insightType '${insightType}'`
|
|
736
|
-
}
|
|
737
|
-
};
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
try {
|
|
741
|
-
// The plan reports whether it actually opened a row. A row
|
|
742
|
-
// skipped in phase 1 (bad designated timestamp) never touched
|
|
743
|
-
// the sender and must not count as buffered — it would inflate
|
|
744
|
-
// pressure and shutdown's dropped count.
|
|
745
|
-
const written = persistPlan( sender, message, partitionId );
|
|
746
|
-
consecutiveWriteErrors = 0; // recovery — health flips back to green
|
|
747
|
-
if ( written ) {
|
|
748
|
-
lastWriteTime = Date.now();
|
|
749
|
-
bufferedRows += 1;
|
|
750
|
-
// Auto mode: QuestDB triggers an internal flush on the sender.at()
|
|
751
|
-
// call that brings pendingRowCount to autoFlushRows (verified
|
|
752
|
-
// against @questdb/nodejs-client v4.2.0 dist/cjs/index.js
|
|
753
|
-
// tryFlush: pendingRowCount >= autoFlushRows). Mirror that reset
|
|
754
|
-
// so getPressure() reflects QuestDB's actual buffer state.
|
|
755
|
-
//
|
|
756
|
-
// Guarded on `autoFlushRows > 0` because the value is optional in
|
|
757
|
-
// ENV_VARS / config; when unset, QuestDB falls back to its own
|
|
758
|
-
// internal default which we cannot observe. In that case the
|
|
759
|
-
// heuristic does not fire and QuestDB contributes no pressure
|
|
760
|
-
// signal — `getPressure()` (below) returns 0.
|
|
761
|
-
if ( flushMode === 'auto' && autoFlushRows > 0 && bufferedRows >= autoFlushRows ) {
|
|
762
|
-
bufferedRows = 0;
|
|
763
|
-
}
|
|
764
|
-
}
|
|
765
|
-
return RESULT_OK;
|
|
766
|
-
} catch ( err ) {
|
|
767
|
-
consecutiveWriteErrors += 1; // health degradation signal
|
|
768
|
-
// The throw may have left a half-written row in the sender;
|
|
769
|
-
// cancel it so this failure costs one row, not the rest of the
|
|
770
|
-
// run (ADR-018). Safe to call even when no row was open: flushing
|
|
771
|
-
// early is harmless and reset() on a consistent buffer is a no-op.
|
|
772
|
-
recoverSender();
|
|
773
|
-
return {
|
|
774
|
-
ok: false,
|
|
775
|
-
error: {
|
|
776
|
-
code: 'SEND_FAILED',
|
|
777
|
-
message: err.message
|
|
778
|
-
}
|
|
779
|
-
};
|
|
780
|
-
}
|
|
781
|
-
};
|
|
782
|
-
|
|
783
|
-
/**
|
|
784
|
-
* Flush pending rows to QuestDB.
|
|
785
|
-
* In auto mode, this is a no-op (QuestDB handles it).
|
|
786
|
-
* In manual mode, this forces an immediate flush. The rows move to
|
|
787
|
-
* the in-flight tally at the call (copy-out); a failure rejects to
|
|
788
|
-
* the caller exactly as before, and the settle handler keeps the
|
|
789
|
-
* pressure accounting straight either way.
|
|
790
|
-
*
|
|
791
|
-
* @returns {Promise<void>}
|
|
792
|
-
*/
|
|
793
|
-
const flush = async function () {
|
|
794
|
-
if ( flushMode === 'manual' && bufferedRows > 0 ) {
|
|
795
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
796
|
-
bufferedRows = 0;
|
|
797
|
-
await entry.promise;
|
|
798
|
-
}
|
|
799
|
-
// Auto mode: no-op, QuestDB handles flushing
|
|
800
|
-
};
|
|
655
|
+
// Setup probe, ILP side (ADR-030 item 4). Before this change the
|
|
656
|
+
// write path was never opened at setup; the first sign of a dead
|
|
657
|
+
// endpoint was the first flush.
|
|
658
|
+
await assertReachable( 'ilpUrl', ilpAddress, probeFn );
|
|
801
659
|
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
* - Manual mode: exact (we own every flush).
|
|
809
|
-
* - Auto mode, heavy flow: exact (autoFlushRows heuristic reset matches).
|
|
810
|
-
* - Auto mode, slow flow + auto_flush_interval triggers: over-states by
|
|
811
|
-
* up to `idleFlushAfterMs`, self-heals via `checkIdleFlush`.
|
|
812
|
-
*
|
|
813
|
-
* Rows inside unsettled flush copies count as pressure: a hung flush
|
|
814
|
-
* is undelivered data, and hiding it is what let shutdown report
|
|
815
|
-
* clean over it. They leave the tally when their flush settles —
|
|
816
|
-
* delivered or reported lost.
|
|
817
|
-
*
|
|
818
|
-
* When `autoFlushRows` is not configured (left to QuestDB's internal
|
|
819
|
-
* default), we have no observable capacity reference and return
|
|
820
|
-
* 0 — QuestDB then contributes nothing to the flow's pressure signal.
|
|
821
|
-
* The time-based yield (ADR-024) and message-count safety nets remain
|
|
822
|
-
* effective.
|
|
823
|
-
*
|
|
824
|
-
* @returns {number} Pressure value in [0, 1]
|
|
825
|
-
*/
|
|
826
|
-
const getPressure = function () {
|
|
827
|
-
if ( !( autoFlushRows > 0 ) ) {
|
|
828
|
-
return 0;
|
|
829
|
-
}
|
|
830
|
-
return Math.min( 1.0, ( bufferedRows + inFlightRows ) / autoFlushRows );
|
|
831
|
-
};
|
|
832
|
-
|
|
833
|
-
/**
|
|
834
|
-
* Health snapshot for operator monitoring (uniform across sinks).
|
|
835
|
-
* Returns the ADR-018 health floor `{status, connected, ...}`, plus a
|
|
836
|
-
* small set of QuestDB-specific diagnostic fields.
|
|
837
|
-
*
|
|
838
|
-
* Status derivation (kept in code, not config — operator mental model is
|
|
839
|
-
* load-bearing institutional knowledge):
|
|
840
|
-
* - `red` if `!connected` (shutting down or sustained write failure)
|
|
841
|
-
* - `yellow` if `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD` OR
|
|
842
|
-
* `consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD`
|
|
843
|
-
* - `green` otherwise
|
|
844
|
-
*
|
|
845
|
-
* `connected` here is *derived* — QuestDB's ILP sender is fire-and-forget
|
|
846
|
-
* with no observable socket state, so we infer transport health from
|
|
847
|
-
* recent write success.
|
|
848
|
-
*
|
|
849
|
-
* @returns {{status: 'green'|'yellow'|'red', connected: boolean, pressure: number, consecutiveWriteErrors: number, bufferedRows: number, flushMode: string}}
|
|
850
|
-
*/
|
|
851
|
-
const getHealth = function () {
|
|
852
|
-
const pressure = getPressure();
|
|
853
|
-
const connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD;
|
|
660
|
+
// The agent, when the standard-library transport is selected, then
|
|
661
|
+
// the sender. A failure is classified, cause attached (see
|
|
662
|
+
// buildSender), and leaves no agent behind (see openTransport).
|
|
663
|
+
const { sender, closeTransport } = await openTransport( {
|
|
664
|
+
SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent
|
|
665
|
+
} );
|
|
854
666
|
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
667
|
+
// The probe the engine runs after a failed flush (ADR-029 hold and
|
|
668
|
+
// probe), bound to the ILP address so the engine knows nothing
|
|
669
|
+
// about addresses. Setup ran the same probe on both endpoints.
|
|
670
|
+
const probe = {
|
|
671
|
+
run: function () {
|
|
672
|
+
return probeFn( ilpAddress );
|
|
673
|
+
},
|
|
674
|
+
describe: function ( outcome ) {
|
|
675
|
+
return describeProbe( outcome, 'ilpUrl', ilpAddress );
|
|
862
676
|
}
|
|
863
|
-
|
|
864
|
-
return {
|
|
865
|
-
// Required health floor (ADR-018)
|
|
866
|
-
status,
|
|
867
|
-
connected,
|
|
868
|
-
pressure,
|
|
869
|
-
// Adapter-specific diagnostics
|
|
870
|
-
consecutiveWriteErrors,
|
|
871
|
-
bufferedRows,
|
|
872
|
-
flushMode
|
|
873
|
-
};
|
|
874
677
|
};
|
|
875
678
|
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
*/
|
|
881
|
-
const closeQuietly = function () {
|
|
882
|
-
return sender.close().catch( function ( closeErr ) {
|
|
883
|
-
logger.error( `winkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
|
|
884
|
-
} );
|
|
885
|
-
}; // closeQuietly()
|
|
886
|
-
|
|
887
|
-
/**
|
|
888
|
-
* The real shutdown body. `shutdown` below latches its promise so
|
|
889
|
-
* every caller — including re-entrant and post-failure callers —
|
|
890
|
-
* receives this one outcome.
|
|
891
|
-
*
|
|
892
|
-
* A clean resolve is a delivery statement (ADR-018): everything
|
|
893
|
-
* buffered OR in flight was delivered. Shutdown therefore settles
|
|
894
|
-
* every unsettled flush (idle, recovery) plus one final flush for
|
|
895
|
-
* whatever is still buffered, all raced against the caller's
|
|
896
|
-
* `{ timeout }` (ADR-018 drain-then-close). It never fires a blind flush at a
|
|
897
|
-
* buffer an earlier copy-out emptied — that is what let it report
|
|
898
|
-
* clean over a hung idle flush.
|
|
899
|
-
*
|
|
900
|
-
* On loss it rejects classified, `dropped: { count }` exact:
|
|
901
|
-
* - any awaited flush fails → `DELIVERY_FAILED`, first flush error
|
|
902
|
-
* on `cause`, count = rows on the flushes that failed;
|
|
903
|
-
* - the combined wait does not settle in time → `SHUTDOWN_TIMEOUT`,
|
|
904
|
-
* count = rows not confirmed delivered. A send against an
|
|
905
|
-
* unreachable server never settles (the client's retry loop — see
|
|
906
|
-
* the file header), so the bound is what keeps shutdown finite.
|
|
907
|
-
* `dropped` is a statement about THIS session: those rows were not
|
|
908
|
-
* confirmed delivered before close. An abandoned flush keeps
|
|
909
|
-
* retrying and may still land its rows later if the server
|
|
910
|
-
* recovers — the count is a floor on uncertainty, not a proof of
|
|
911
|
-
* loss.
|
|
912
|
-
*
|
|
913
|
-
* The transport close is attempted in both loss paths, but on the
|
|
914
|
-
* HTTP transport the client's `close()` is an empty function
|
|
915
|
-
* (verified against @questdb/nodejs-client 4.2.0), so nothing can
|
|
916
|
-
* abort an abandoned flush's retry timers from here; they keep the
|
|
917
|
-
* event loop alive. Process exit is the final backstop — in a flow,
|
|
918
|
-
* the shutdown manager's `SHUTDOWN_FORCE_TIMEOUT_MS` exit covers
|
|
919
|
-
* this. No timeout supplied = no enforcement (unbounded await),
|
|
920
|
-
* preserving direct-caller behavior.
|
|
921
|
-
*/
|
|
922
|
-
const doShutdown = async function ( timeout ) {
|
|
923
|
-
// Flip the health flag first so any concurrent getHealth() call
|
|
924
|
-
// immediately sees the shutdown and returns red/disconnected —
|
|
925
|
-
// and write() starts refusing new rows (SHUTTING_DOWN).
|
|
926
|
-
shuttingDown = true;
|
|
927
|
-
|
|
928
|
-
// Stop idle flush timer
|
|
929
|
-
if ( idleFlushTimer ) {
|
|
930
|
-
clearInterval( idleFlushTimer );
|
|
931
|
-
idleFlushTimer = null;
|
|
932
|
-
}
|
|
933
|
-
|
|
934
|
-
// Everything delivery still owes: flushes already in flight
|
|
935
|
-
// (their rows left the buffer at their call) plus one final
|
|
936
|
-
// flush for whatever is still buffered. Each wait records its
|
|
937
|
-
// outcome into the tallies below; the mapped promises never
|
|
938
|
-
// reject, so the only rejection the race can surface is the
|
|
939
|
-
// timeout itself.
|
|
940
|
-
let totalRows = 0;
|
|
941
|
-
let deliveredRows = 0;
|
|
942
|
-
let failedRows = 0;
|
|
943
|
-
let firstFailure = null;
|
|
944
|
-
const waits = [];
|
|
945
|
-
|
|
946
|
-
const awaitDelivery = function ( entry ) {
|
|
947
|
-
totalRows += entry.rows;
|
|
948
|
-
waits.push( entry.promise.then(
|
|
949
|
-
function () {
|
|
950
|
-
deliveredRows += entry.rows;
|
|
951
|
-
},
|
|
952
|
-
function ( err ) {
|
|
953
|
-
failedRows += entry.rows;
|
|
954
|
-
if ( !firstFailure ) {
|
|
955
|
-
firstFailure = err;
|
|
956
|
-
}
|
|
957
|
-
}
|
|
958
|
-
) );
|
|
959
|
-
}; // awaitDelivery()
|
|
960
|
-
|
|
961
|
-
inFlightFlushes.forEach( awaitDelivery );
|
|
962
|
-
if ( bufferedRows > 0 ) {
|
|
963
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
964
|
-
bufferedRows = 0;
|
|
965
|
-
awaitDelivery( entry );
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
if ( waits.length > 0 ) {
|
|
969
|
-
try {
|
|
970
|
-
await raceFlushTimeout( Promise.all( waits ), timeout );
|
|
971
|
-
} catch ( err ) {
|
|
972
|
-
await closeQuietly();
|
|
973
|
-
const dropped = totalRows - deliveredRows;
|
|
974
|
-
const timedOut = new Error(
|
|
975
|
-
`winkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
|
|
976
|
-
);
|
|
977
|
-
timedOut.code = 'SHUTDOWN_TIMEOUT';
|
|
978
|
-
timedOut.dropped = { count: dropped };
|
|
979
|
-
throw timedOut;
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
if ( failedRows > 0 ) {
|
|
983
|
-
await closeQuietly();
|
|
984
|
-
const failure = new Error(
|
|
985
|
-
`winkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
|
|
986
|
-
);
|
|
987
|
-
failure.code = 'DELIVERY_FAILED';
|
|
988
|
-
failure.dropped = { count: failedRows };
|
|
989
|
-
failure.cause = firstFailure;
|
|
990
|
-
throw failure;
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
// Close sender
|
|
995
|
-
await sender.close();
|
|
996
|
-
}; // doShutdown()
|
|
997
|
-
|
|
998
|
-
/**
|
|
999
|
-
* Shutdown the storage adapter gracefully.
|
|
1000
|
-
* Flushes pending data and closes connections.
|
|
1001
|
-
*
|
|
1002
|
-
* Called by wire-storages.shutdown() which is invoked during:
|
|
1003
|
-
* - Pipeline shutdown (flowHandle.shutdown())
|
|
1004
|
-
* - Process signal handlers (SIGINT, SIGTERM)
|
|
1005
|
-
*
|
|
1006
|
-
* The outcome is latched: the first call runs the shutdown, every
|
|
1007
|
-
* later call returns the same promise. A lossy shutdown's failed
|
|
1008
|
-
* flush already emptied the buffer (copy-out), so a re-run would
|
|
1009
|
-
* find nothing to flush and resolve clean, contradicting the
|
|
1010
|
-
* recorded loss. One consequence: the first caller's `{ timeout }`
|
|
1011
|
-
* governs; a later caller's is ignored.
|
|
1012
|
-
*
|
|
1013
|
-
* @param {{timeout?: number}} [options]
|
|
1014
|
-
* @returns {Promise<void>}
|
|
1015
|
-
*/
|
|
1016
|
-
const shutdown = function ( { timeout = 0 } = {} ) {
|
|
1017
|
-
if ( !shutdownPromise ) {
|
|
1018
|
-
shutdownPromise = doShutdown( timeout );
|
|
1019
|
-
}
|
|
1020
|
-
return shutdownPromise;
|
|
1021
|
-
}; // shutdown()
|
|
679
|
+
// Everything after setup is the engine's: the hot path, the
|
|
680
|
+
// flushes, the counters, the drain. The transport close stays the
|
|
681
|
+
// factory's, because the factory owns the agent.
|
|
682
|
+
const engine = createFlushEngine( { sender, persistPlans, settings, onDeliveryFailure, probe, closeTransport } );
|
|
1022
683
|
|
|
1023
684
|
return {
|
|
1024
|
-
write,
|
|
1025
|
-
flush,
|
|
1026
|
-
shutdown,
|
|
1027
|
-
getPressure,
|
|
1028
|
-
getHealth,
|
|
685
|
+
write: engine.write,
|
|
686
|
+
flush: engine.flush,
|
|
687
|
+
shutdown: engine.shutdown,
|
|
688
|
+
getPressure: engine.getPressure,
|
|
689
|
+
getHealth: engine.getHealth,
|
|
1029
690
|
// Expose for testing/debugging
|
|
1030
691
|
_sender: sender,
|
|
1031
692
|
_persistPlans: persistPlans
|
|
1032
693
|
};
|
|
1033
|
-
};
|
|
694
|
+
}; // createQuestDBStorage()
|
|
1034
695
|
|
|
1035
696
|
// ============================================================================
|
|
1036
697
|
// CONFIG SCHEMA
|
|
@@ -1060,65 +721,116 @@ const configSchema = {
|
|
|
1060
721
|
'idleFlushCheckMs',
|
|
1061
722
|
'autoFlushRows',
|
|
1062
723
|
'autoFlushIntervalMs',
|
|
1063
|
-
'
|
|
724
|
+
'flushRows',
|
|
725
|
+
'flushIntervalMs',
|
|
726
|
+
'bufferCeilingRows',
|
|
727
|
+
'flushDeadlineMs',
|
|
728
|
+
'stdlibHttp',
|
|
729
|
+
'requestTimeout',
|
|
1064
730
|
'retryTimeout',
|
|
731
|
+
'initBufSize',
|
|
732
|
+
'maxBufSize',
|
|
1065
733
|
'partitionBy',
|
|
1066
734
|
'onWarning',
|
|
1067
735
|
'onDeliveryFailure'
|
|
1068
736
|
],
|
|
737
|
+
// Both addresses refuse `localhost` at flow definition (ADR-030).
|
|
738
|
+
// The non-empty rule lives inside the validator, so one static
|
|
739
|
+
// error string covers every refusal.
|
|
1069
740
|
ilpUrl: {
|
|
1070
741
|
type: 'string',
|
|
1071
742
|
required: false,
|
|
1072
|
-
|
|
1073
|
-
error: 'ilpUrl must be a
|
|
743
|
+
validator: isAllowedIlpUrl,
|
|
744
|
+
error: 'ilpUrl must be host:port with a literal address or a name, never localhost and never ' +
|
|
745
|
+
'an IPv6 literal (the QuestDB client cannot read one); e.g., 127.0.0.1:9000'
|
|
1074
746
|
},
|
|
1075
747
|
pgUrl: {
|
|
1076
748
|
type: 'string',
|
|
1077
749
|
required: false,
|
|
1078
|
-
|
|
1079
|
-
error: 'pgUrl must be a
|
|
750
|
+
validator: isAllowedPgUrl,
|
|
751
|
+
error: 'pgUrl must be host:port with a literal address or a name, never localhost; e.g., 127.0.0.1:8812'
|
|
1080
752
|
},
|
|
1081
753
|
tablePrefix: {
|
|
1082
754
|
type: 'string',
|
|
1083
755
|
required: false,
|
|
1084
756
|
minLength: 1,
|
|
1085
|
-
|
|
757
|
+
// The prefix opens every table name, written unquoted in CREATE
|
|
758
|
+
// TABLE. See header sub-case (j) for the rule and its sources.
|
|
759
|
+
validator: validators.identifier,
|
|
760
|
+
error: 'tablePrefix must use letters, digits, _ and $ only, and not start with a digit ' +
|
|
761
|
+
'(defaults to assetClass.name when omitted)'
|
|
1086
762
|
},
|
|
763
|
+
// The five legacy keys (deprecated, ADR-029; removed in 0.8.0). They
|
|
764
|
+
// keep their old validation so an existing flow still passes the
|
|
765
|
+
// schema. The option resolver maps or ignores them and setup prints
|
|
766
|
+
// one DEPRECATED_OPTION line.
|
|
1087
767
|
flushMode: {
|
|
1088
768
|
type: 'string',
|
|
1089
769
|
required: false,
|
|
1090
770
|
validator: validators.oneOf( [ 'auto', 'manual' ] ),
|
|
1091
|
-
error: 'flushMode must be "auto" or "manual"'
|
|
771
|
+
error: 'flushMode must be "auto" or "manual" (deprecated: composer owns every flush)'
|
|
1092
772
|
},
|
|
1093
773
|
idleFlushAfterMs: {
|
|
1094
774
|
type: 'number',
|
|
1095
775
|
required: false,
|
|
1096
776
|
validator: validators.positiveInteger,
|
|
1097
|
-
error: 'idleFlushAfterMs must be a positive integer'
|
|
777
|
+
error: 'idleFlushAfterMs must be a positive integer (deprecated: ignored)'
|
|
1098
778
|
},
|
|
1099
779
|
idleFlushCheckMs: {
|
|
1100
780
|
type: 'number',
|
|
1101
781
|
required: false,
|
|
1102
782
|
validator: validators.positiveInteger,
|
|
1103
|
-
error: 'idleFlushCheckMs must be a positive integer'
|
|
783
|
+
error: 'idleFlushCheckMs must be a positive integer (deprecated: use flushIntervalMs)'
|
|
1104
784
|
},
|
|
1105
785
|
autoFlushRows: {
|
|
1106
786
|
type: 'number',
|
|
1107
787
|
required: false,
|
|
1108
788
|
validator: validators.positiveInteger,
|
|
1109
|
-
error: 'autoFlushRows must be a positive integer'
|
|
789
|
+
error: 'autoFlushRows must be a positive integer (deprecated: use flushRows)'
|
|
1110
790
|
},
|
|
1111
791
|
autoFlushIntervalMs: {
|
|
1112
792
|
type: 'number',
|
|
1113
793
|
required: false,
|
|
1114
794
|
validator: validators.positiveInteger,
|
|
1115
|
-
error: 'autoFlushIntervalMs must be a positive integer'
|
|
795
|
+
error: 'autoFlushIntervalMs must be a positive integer (deprecated: ignored)'
|
|
1116
796
|
},
|
|
1117
|
-
|
|
797
|
+
// The flush settings composer owns (ADR-029). The relation between
|
|
798
|
+
// the ceiling and the threshold is checked by the option resolver
|
|
799
|
+
// at setup, where both values are known.
|
|
800
|
+
flushRows: {
|
|
1118
801
|
type: 'number',
|
|
1119
802
|
required: false,
|
|
1120
803
|
validator: validators.positiveInteger,
|
|
1121
|
-
error: '
|
|
804
|
+
error: 'flushRows must be a positive integer'
|
|
805
|
+
},
|
|
806
|
+
flushIntervalMs: {
|
|
807
|
+
type: 'number',
|
|
808
|
+
required: false,
|
|
809
|
+
validator: validators.positiveInteger,
|
|
810
|
+
error: 'flushIntervalMs must be a positive integer'
|
|
811
|
+
},
|
|
812
|
+
bufferCeilingRows: {
|
|
813
|
+
type: 'number',
|
|
814
|
+
required: false,
|
|
815
|
+
validator: validators.positiveInteger,
|
|
816
|
+
error: 'bufferCeilingRows must be a positive integer'
|
|
817
|
+
},
|
|
818
|
+
flushDeadlineMs: {
|
|
819
|
+
type: 'number',
|
|
820
|
+
required: false,
|
|
821
|
+
validator: validators.positiveInteger,
|
|
822
|
+
error: 'flushDeadlineMs must be a positive integer'
|
|
823
|
+
},
|
|
824
|
+
stdlibHttp: {
|
|
825
|
+
type: 'boolean',
|
|
826
|
+
required: false,
|
|
827
|
+
error: 'stdlibHttp must be a boolean'
|
|
828
|
+
},
|
|
829
|
+
requestTimeout: {
|
|
830
|
+
type: 'number',
|
|
831
|
+
required: false,
|
|
832
|
+
validator: validators.positiveInteger,
|
|
833
|
+
error: 'requestTimeout must be a positive integer'
|
|
1122
834
|
},
|
|
1123
835
|
retryTimeout: {
|
|
1124
836
|
type: 'number',
|
|
@@ -1126,6 +838,18 @@ const configSchema = {
|
|
|
1126
838
|
validator: validators.positiveInteger,
|
|
1127
839
|
error: 'retryTimeout must be a positive integer'
|
|
1128
840
|
},
|
|
841
|
+
initBufSize: {
|
|
842
|
+
type: 'number',
|
|
843
|
+
required: false,
|
|
844
|
+
validator: validators.positiveInteger,
|
|
845
|
+
error: 'initBufSize must be a positive integer'
|
|
846
|
+
},
|
|
847
|
+
maxBufSize: {
|
|
848
|
+
type: 'number',
|
|
849
|
+
required: false,
|
|
850
|
+
validator: validators.positiveInteger,
|
|
851
|
+
error: 'maxBufSize must be a positive integer'
|
|
852
|
+
},
|
|
1129
853
|
partitionBy: {
|
|
1130
854
|
type: 'string',
|
|
1131
855
|
required: false,
|
|
@@ -1166,23 +890,24 @@ const durabilityClass = 'in-memory';
|
|
|
1166
890
|
|
|
1167
891
|
// Capability declaration per ADR-018.
|
|
1168
892
|
//
|
|
1169
|
-
// QuestDB needs the asset class
|
|
1170
|
-
// `columns` and `insightTypes` to issue CREATE TABLE statements
|
|
1171
|
-
//
|
|
893
|
+
// QuestDB needs the asset class for two reasons. `ensure-tables.js`
|
|
894
|
+
// reads `columns` and `insightTypes` to issue CREATE TABLE statements.
|
|
895
|
+
// `persist-plan.js` reads the same to dispatch column writers and
|
|
1172
896
|
// pre-compile resolution-aware quantization. `name` drives the default
|
|
1173
897
|
// `tablePrefix` when the caller did not pass one.
|
|
1174
898
|
//
|
|
1175
899
|
// Declared `required: true` because there is no useful behaviour for
|
|
1176
|
-
// QuestDB without an assetClass
|
|
1177
|
-
// MISSING_ASSET_CLASS
|
|
900
|
+
// QuestDB without an assetClass. The wiring layer fails fast with
|
|
901
|
+
// MISSING_ASSET_CLASS instead of calling `createStorage` and letting
|
|
1178
902
|
// the failure surface deeper.
|
|
1179
903
|
//
|
|
1180
904
|
// `fields` lists exactly the top-level fields read. The slicing is
|
|
1181
|
-
// top-level only
|
|
905
|
+
// top-level only. Column-internal fields (`type`, `resolution`) are
|
|
1182
906
|
// read directly from the `columns` slice and validated by Layer 2
|
|
1183
|
-
// assertions inside `createStorage` below.
|
|
1184
|
-
//
|
|
1185
|
-
//
|
|
907
|
+
// assertions inside `createStorage` below.
|
|
908
|
+
//
|
|
909
|
+
// A declarative shape for column-internal capability is deferred
|
|
910
|
+
// until a second adapter needs it.
|
|
1186
911
|
const semanticsRequirement = {
|
|
1187
912
|
assetClass: {
|
|
1188
913
|
required: true,
|
|
@@ -1196,17 +921,20 @@ const semanticsRequirement = {
|
|
|
1196
921
|
* @param {Object} config - Configuration object
|
|
1197
922
|
* @param {Object} config.assetClass - Asset class definition with columns and insightTypes
|
|
1198
923
|
* @param {string} config.tablePrefix - Prefix for table names (defaults to assetClass.name)
|
|
1199
|
-
* @param {string} config.ilpUrl - ILP endpoint (host:port)
|
|
1200
|
-
* @param {string} config.pgUrl - PostgreSQL endpoint (host:port)
|
|
1201
|
-
* @param {
|
|
1202
|
-
* @param {number} [config.
|
|
1203
|
-
* @param {number} [config.
|
|
1204
|
-
* @param {number} [config.
|
|
1205
|
-
* @param {
|
|
1206
|
-
* @param {number} [config.
|
|
1207
|
-
* @param {number} [config.retryTimeout] -
|
|
924
|
+
* @param {string} [config.ilpUrl] - ILP endpoint (host:port)
|
|
925
|
+
* @param {string} [config.pgUrl] - PostgreSQL endpoint (host:port)
|
|
926
|
+
* @param {number} [config.flushRows=5000] - Rows that start a flush from inside write()
|
|
927
|
+
* @param {number} [config.flushIntervalMs=1000] - Period of the flush timer
|
|
928
|
+
* @param {number} [config.bufferCeilingRows] - Most rows held; default ten times flushRows
|
|
929
|
+
* @param {number} [config.flushDeadlineMs] - Fixed deadline per flush
|
|
930
|
+
* @param {boolean} [config.stdlibHttp=true] - The standard-library transport; false selects undici
|
|
931
|
+
* @param {number} [config.requestTimeout] - Client request timeout in ms
|
|
932
|
+
* @param {number} [config.retryTimeout] - Client retry window in ms
|
|
933
|
+
* @param {number} [config.initBufSize] - Initial client buffer size in bytes
|
|
934
|
+
* @param {number} [config.maxBufSize] - Byte ceiling of the client buffer
|
|
1208
935
|
* @param {string} [config.partitionBy='DAY'] - Table partition interval
|
|
1209
|
-
* @param {function} [config.onWarning] - Warning callback for
|
|
936
|
+
* @param {function} [config.onWarning] - Warning callback for skipped values
|
|
937
|
+
* @param {function} [config.onDeliveryFailure] - Called once per lost flush
|
|
1210
938
|
* @param {Object} [config._deps] - Injectable dependencies (for testing)
|
|
1211
939
|
* @returns {Promise<Object>} Storage instance with write, flush, shutdown methods
|
|
1212
940
|
*/
|
|
@@ -1228,15 +956,14 @@ const createStorage = function ( config ) {
|
|
|
1228
956
|
}
|
|
1229
957
|
|
|
1230
958
|
// Adapter-side defensive validation of column-internal facts
|
|
1231
|
-
// QuestDB consumes
|
|
1232
|
-
//
|
|
1233
|
-
//
|
|
1234
|
-
//
|
|
1235
|
-
//
|
|
1236
|
-
//
|
|
1237
|
-
//
|
|
1238
|
-
//
|
|
1239
|
-
// full rationale.
|
|
959
|
+
// QuestDB consumes (the ADR-018 column-internal facts pattern). The
|
|
960
|
+
// universal semantics schema validates the top-level structure at
|
|
961
|
+
// load and at .assetClass() time: type required, type in
|
|
962
|
+
// COLUMN_TYPES. This layer catches bypasses, such as a direct
|
|
963
|
+
// createStorage call from test code. It also checks adapter-specific
|
|
964
|
+
// requirements, such as "a float64 column with a declared resolution
|
|
965
|
+
// must have a positive value". See `assert-columns.js` for the full
|
|
966
|
+
// rationale.
|
|
1240
967
|
assertColumnFacts( assetClass );
|
|
1241
968
|
|
|
1242
969
|
// Default `tablePrefix` to the asset class name when the caller
|
|
@@ -1264,8 +991,8 @@ const createStorage = function ( config ) {
|
|
|
1264
991
|
* .assetClass( assetClassDef ) // wire-storages injects the slice
|
|
1265
992
|
* .storage( questdbAdapter, {
|
|
1266
993
|
* tablePrefix: 'myPrefix',
|
|
1267
|
-
* ilpUrl: '
|
|
1268
|
-
* pgUrl: '
|
|
994
|
+
* ilpUrl: '127.0.0.1:9000',
|
|
995
|
+
* pgUrl: '127.0.0.1:8812'
|
|
1269
996
|
* } )
|
|
1270
997
|
*
|
|
1271
998
|
* @type {Object}
|