@winkjs/composer 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -3
- package/src/composer.js +1 -1
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +108 -633
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +20 -5
- package/src/core/env-vars.js +143 -16
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +276 -83
- package/src/core/source-manager/mqtt/dedup.js +19 -14
- package/src/core/source-manager/mqtt/index.js +33 -6
- package/src/core/source-manager/mqtt/status.js +211 -46
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +592 -827
- package/src/core/storage-manager/questdb/persist-plan.js +147 -162
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +176 -0
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +7 -4
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +14 -5
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +62 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +200 -40
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
|
@@ -1,64 +1,134 @@
|
|
|
1
1
|
// core/storage-manager/questdb/index.js
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @fileoverview QuestDB storage adapter
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* -
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* -
|
|
30
|
-
*
|
|
31
|
-
* - `getPressure()
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
4
|
+
* @fileoverview QuestDB storage adapter. It writes rows to QuestDB over
|
|
5
|
+
* ILP (InfluxDB Line Protocol) on HTTP, and creates the tables over the
|
|
6
|
+
* PostgreSQL wire protocol at startup.
|
|
7
|
+
*
|
|
8
|
+
* Contract role (ADR-018): storage, buffered. Startup posture: eager.
|
|
9
|
+
* Setup throws when either endpoint does not answer, so a flow never
|
|
10
|
+
* starts against a dead database.
|
|
11
|
+
*
|
|
12
|
+
* Module layout. This file holds the factory, the config schema, and
|
|
13
|
+
* the adapter export. The work lives in four modules next to it:
|
|
14
|
+
* - `resolve-options.js` merges the config and the environment into
|
|
15
|
+
* the settings, and derives the ceiling and the deadline.
|
|
16
|
+
* - `address-policy.js` holds the ADR-030 address checks, the setup
|
|
17
|
+
* probe wrapper, and the classification of a connect failure.
|
|
18
|
+
* - `flush-engine.js` owns the hot path, every flush, the counters
|
|
19
|
+
* behind pressure and health, and the shutdown drain (ADR-029).
|
|
20
|
+
* - `persist-plan.js` compiles one row writer per insight type.
|
|
21
|
+
*
|
|
22
|
+
* The handle the factory returns:
|
|
23
|
+
* - `write(insightType, msg, partitionId)`: the hot path. Appends one
|
|
24
|
+
* row to the client's buffer and returns at once. `{ ok: true }`
|
|
25
|
+
* means accepted into the buffer, not delivered. `{ ok: false,
|
|
26
|
+
* error: { code, message } }` names the refusal. Never throws.
|
|
27
|
+
* - `flush()`: sends everything buffered now. Resolves when the server
|
|
28
|
+
* confirmed the rows, rejects when it did not.
|
|
29
|
+
* - `shutdown({ timeout })`: drain, then close. A clean resolve means
|
|
30
|
+
* every row was delivered. A loss is a classified rejection.
|
|
31
|
+
* - `getPressure()`: buffer fill in [0, 1]. Sync, O(1), no allocation.
|
|
32
|
+
* - `getHealth()`: `{ status, connected, pressure, ... }`.
|
|
33
|
+
* `flush-engine.js` documents how each one behaves.
|
|
34
|
+
*
|
|
35
|
+
* Composer owns every flush (ADR-029). The client's own trigger is
|
|
36
|
+
* off (`auto_flush=off`), so a batch leaves the process only when the
|
|
37
|
+
* engine says so. A write that brings the buffer to `flushRows` starts
|
|
38
|
+
* a flush at once. A timer starts a flush every `flushIntervalMs` when
|
|
39
|
+
* anything is buffered. Only one engine flush runs at a time. A failed
|
|
40
|
+
* flush is reported once, with the exact rows lost, and the process
|
|
41
|
+
* keeps running. Every flush has a deadline, derived from the rows it
|
|
42
|
+
* carries, and a flush past it is abandoned and reported. A failed or
|
|
43
|
+
* abandoned flush runs the ADR-030 probe. While the probe fails,
|
|
44
|
+
* delivery pauses: rows are held up to the ceiling and each tick
|
|
45
|
+
* probes again. `flush-engine.js` carries the detail.
|
|
46
|
+
*
|
|
47
|
+
* Row order. Rows reach QuestDB in write order. The engine runs one
|
|
48
|
+
* flush at a time, each flush carries its rows in write order, and
|
|
49
|
+
* the standard-library transport keeps one socket (`maxSockets: 1`),
|
|
50
|
+
* so a later request waits for the earlier one. Two sends can be
|
|
51
|
+
* outstanding only at shutdown, when the drain's final flush starts
|
|
52
|
+
* while an earlier flush is still on the wire. On the undici
|
|
53
|
+
* transport those two batches can land out of order. QuestDB orders
|
|
54
|
+
* each table by its designated timestamp, so a query never sees the
|
|
55
|
+
* difference.
|
|
56
|
+
*
|
|
57
|
+
* Durability in plain words (ADR-018). `durabilityClass` is
|
|
58
|
+
* `'in-memory'`. Rows live in the client's buffer until a flush reaches
|
|
59
|
+
* the server. Three things lose rows: a process crash loses everything
|
|
60
|
+
* buffered or in flight, a failed flush loses its batch, and the
|
|
61
|
+
* ceiling sheds new rows. The shed policy: once the rows buffered plus
|
|
62
|
+
* the rows in flight reach `bufferCeilingRows`, `write()` refuses new
|
|
63
|
+
* rows with `STORAGE_FULL` and the rows already accepted stay safe.
|
|
64
|
+
* Outage budget with the defaults: the ceiling of 50,000 rows holds
|
|
65
|
+
* 50 seconds at 1,000 rows a second and about 14 hours at one row a
|
|
66
|
+
* second, in 6.5 to 15 MB of memory at the measured 130 to 290 bytes
|
|
67
|
+
* a row.
|
|
68
|
+
*
|
|
69
|
+
* Edge-baseline defaults (every one runs within a 2 GB device):
|
|
70
|
+
*
|
|
71
|
+
* | Setting | Default | Environment variable |
|
|
72
|
+
* |---------------------|------------------|------------------------------|
|
|
73
|
+
* | `flushRows` | 5000 | `QUESTDB_FLUSH_ROWS` |
|
|
74
|
+
* | `flushIntervalMs` | 1000 | `QUESTDB_FLUSH_INTERVAL_MS` |
|
|
75
|
+
* | `bufferCeilingRows` | 10 × `flushRows` | `QUESTDB_BUFFER_CEILING_ROWS`|
|
|
76
|
+
* | `flushDeadlineMs` | derived per flush| `QUESTDB_FLUSH_DEADLINE_MS` |
|
|
77
|
+
* | `stdlibHttp` | true | `QUESTDB_STDLIB_HTTP` |
|
|
78
|
+
* | `requestTimeout` | client default | `QUESTDB_REQUEST_TIMEOUT` |
|
|
79
|
+
* | `retryTimeout` | client default | `QUESTDB_RETRY_TIMEOUT` |
|
|
80
|
+
* | `initBufSize` | client default | `QUESTDB_INIT_BUF_SIZE` |
|
|
81
|
+
* | `maxBufSize` | client default | `QUESTDB_MAX_BUF_SIZE` |
|
|
82
|
+
*
|
|
83
|
+
* `resolve-options.js` merges the config and the environment and
|
|
84
|
+
* explains each default. `ilpUrl` and `pgUrl` fall back to
|
|
85
|
+
* `QUESTDB_ILP_URL` and `QUESTDB_PG_URL`; the credentials come from
|
|
86
|
+
* `QUESTDB_DATABASE`, `QUESTDB_USER` and `QUESTDB_PASSWORD`.
|
|
87
|
+
*
|
|
88
|
+
* Long-running commitments (ADR-018 §12). One timer, created at setup
|
|
89
|
+
* and cleared at shutdown; it does not hold the process open. Every
|
|
90
|
+
* counter is bounded by the ceiling. The per-row path allocates
|
|
91
|
+
* nothing beyond the one derived promise `persist-plan.js` documents.
|
|
92
|
+
* Reconnection and request retries belong to the client. No listeners
|
|
93
|
+
* are attached. `flush-engine.js` carries the detail.
|
|
94
|
+
*
|
|
95
|
+
* The transport (ADR-029). The client's default HTTP library, undici,
|
|
96
|
+
* retries a refused connection without end, and its abort cannot end
|
|
97
|
+
* that retry. So the adapter selects the client's standard-library
|
|
98
|
+
* transport (`stdlib_http=on`) unless `stdlibHttp` is false. With it, a refused connection rejects at once and every
|
|
99
|
+
* request ends within `retryTimeout` plus one request timeout. The
|
|
100
|
+
* adapter owns one `http.Agent` for that transport, keep-alive on one
|
|
101
|
+
* socket, and destroys it after the sender closes on every shutdown
|
|
102
|
+
* path. Destroying the agent ends a request still on its socket, so
|
|
103
|
+
* the process can exit. The agent also closes its socket after 4
|
|
104
|
+
* seconds idle, before QuestDB's own idle close at 5 minutes, so no
|
|
105
|
+
* flush meets a socket the server has just closed. An operator who
|
|
106
|
+
* selects undici accepts that an abandoned flush can keep the process
|
|
107
|
+
* alive.
|
|
108
|
+
*
|
|
109
|
+
* Health (ADR-018 §8). `status` is `red` when not connected or at
|
|
110
|
+
* capacity. It is `yellow` at `pressure >= 0.66`, on an outstanding
|
|
111
|
+
* write error, or after one failed flush. Otherwise it is `green`.
|
|
112
|
+
* `connected` is derived, because the ILP client exposes no socket
|
|
113
|
+
* state. It is false while delivery is paused, after five write errors
|
|
114
|
+
* in a row, after two failed flushes in a row, or after one abandoned
|
|
115
|
+
* flush. The health object carries `pausedSince`, `abandonedFlushes`,
|
|
116
|
+
* `consecutiveFlushFailures`, `lastFlushAt`, and `lastFlushError`.
|
|
117
|
+
* `flush-engine.js` documents the derivation.
|
|
118
|
+
*
|
|
119
|
+
* Deprecated options (ADR-029, removed in 0.8.0). `autoFlushRows` maps
|
|
120
|
+
* to `flushRows` and `idleFlushCheckMs` maps to `flushIntervalMs`.
|
|
121
|
+
* `flushMode`, `idleFlushAfterMs` and `autoFlushIntervalMs` are
|
|
122
|
+
* accepted and ignored. The same holds for their `QUESTDB_*`
|
|
123
|
+
* variables. Setup prints one `DEPRECATED_OPTION` line naming every
|
|
124
|
+
* legacy key in use and what happened to it.
|
|
55
125
|
*
|
|
56
126
|
* `err.code` vocabulary (per-adapter; ADR-018 has each adapter document
|
|
57
127
|
* its own codes in this header):
|
|
58
128
|
*
|
|
59
129
|
* Setup-time throws (ADR-018 fail-fast setup):
|
|
60
130
|
* - `INVALID_CONFIG` — the supplied configuration does not
|
|
61
|
-
* work.
|
|
131
|
+
* work. Ten current sub-cases:
|
|
62
132
|
* (a) required transport URL missing (`ilpUrl`, `pgUrl`);
|
|
63
133
|
* (b) PostgreSQL endpoint answered but rejected the connection
|
|
64
134
|
* — wrong credentials or a protocol-level refusal;
|
|
@@ -72,18 +142,57 @@
|
|
|
72
142
|
* of wedging the sender mid-row at runtime;
|
|
73
143
|
* (e) an insightType uses the reserved column name `assetId`.
|
|
74
144
|
* Composer writes that column itself, from the partition id.
|
|
75
|
-
* Checked at plan build in `persist-plan.js
|
|
145
|
+
* Checked at plan build in `persist-plan.js`;
|
|
146
|
+
* (f) `ilpUrl` or `pgUrl` names `localhost` (ADR-030). The name
|
|
147
|
+
* can resolve to two addresses, `::1` and `127.0.0.1`, and
|
|
148
|
+
* QuestDB may listen on only one. Refused before any socket
|
|
149
|
+
* opens, at the schema (flow definition) and here (the env
|
|
150
|
+
* fallback and direct callers). The message names the
|
|
151
|
+
* literal to set;
|
|
152
|
+
* (g) `ilpUrl` is an IPv6 literal. The client (4.2.0) splits the
|
|
153
|
+
* address on its first colon and cannot read one. `pgUrl`
|
|
154
|
+
* accepts `[::1]:8812`;
|
|
155
|
+
* (h) the ILP client rejected the sender configuration for a
|
|
156
|
+
* reason that is not a network error (`fromConfig`);
|
|
157
|
+
* (i) `bufferCeilingRows` is below twice `flushRows` (ADR-029).
|
|
158
|
+
* Rows in flight count against the ceiling, so it must hold
|
|
159
|
+
* one batch in flight and one batch buffering. A smaller
|
|
160
|
+
* ceiling would refuse every row that arrives during a
|
|
161
|
+
* row-triggered flush. Checked in `resolve-options.js`;
|
|
162
|
+
* (j) `tablePrefix` is not an identifier: letters, digits, `_`
|
|
163
|
+
* and `$`, not starting with a digit. The prefix opens every
|
|
164
|
+
* table name, and `ensure-tables.js` writes that name unquoted
|
|
165
|
+
* in CREATE TABLE. QuestDB reads an unquoted name as one
|
|
166
|
+
* token. So a hyphen, a space, a dot, a semicolon or a
|
|
167
|
+
* backtick ends the statement early with "unexpected token"
|
|
168
|
+
* (verified live on 9.4.3). QuestDB's own rule for a quoted
|
|
169
|
+
* name is wider. Its CREATE TABLE reference says a name with
|
|
170
|
+
* a space or a dot must be double-quoted. The server's
|
|
171
|
+
* `TableUtils.isValidTableName` forbids `? , ' " \ / : ( ) +
|
|
172
|
+
* * % ~`, control characters, and a leading or trailing dot
|
|
173
|
+
* or space. Sources: https://questdb.com/docs/query/sql/create-table/
|
|
174
|
+
* and https://questdb.com/docs/connect/compatibility/ilp/advanced-settings/.
|
|
175
|
+
* The identifier rule is the one the semantics schema applies
|
|
176
|
+
* to asset class, insight type and column names, so every
|
|
177
|
+
* part of a table name follows one rule. Checked at the
|
|
178
|
+
* schema and here, which covers the default prefix (the asset
|
|
179
|
+
* class name) and a direct caller.
|
|
76
180
|
* Operator remediation: fix the supplied config or the relevant
|
|
77
|
-
* `QUESTDB_*` env var. The underlying
|
|
78
|
-
*
|
|
79
|
-
* - `TRANSPORT_UNREACHABLE` —
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
181
|
+
* `QUESTDB_*` env var. The underlying error (sub-cases b and h) is
|
|
182
|
+
* preserved on `err.cause` for diagnostics.
|
|
183
|
+
* - `TRANSPORT_UNREACHABLE` — an endpoint did not answer at setup.
|
|
184
|
+
* Three sources. (1) The setup probe (ADR-030 item 4): before any
|
|
185
|
+
* client is built, `pgUrl` and then `ilpUrl` are probed with one
|
|
186
|
+
* TCP connect per address. A name is resolved with Node's default
|
|
187
|
+
* lookup and every address it resolves to must answer; the message
|
|
188
|
+
* lists each address with its result and names the literal to set.
|
|
189
|
+
* (2) The PostgreSQL connect failed with a Node syscall code
|
|
190
|
+
* (`ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, ...). (3) The ILP
|
|
191
|
+
* client could not be built for the same class of reason.
|
|
192
|
+
* Distinct from `INVALID_CONFIG` per the one split the ADR-018
|
|
193
|
+
* error vocabulary mandates: the connection string may be fine —
|
|
194
|
+
* check the network, the firewall, whether QuestDB is running.
|
|
195
|
+
* Underlying error on `err.cause` for (2) and (3).
|
|
87
196
|
* - `MISSING_ASSET_CLASS` — `assetClass` not provided to `createStorage`.
|
|
88
197
|
* Distinct from `INVALID_CONFIG` because operator remediation differs:
|
|
89
198
|
* they need to add `.assetClass(assetClassDef)` to the flow, not edit env
|
|
@@ -91,24 +200,80 @@
|
|
|
91
200
|
* - `SCHEMA_ERROR` — DDL `CREATE TABLE` failed for a reason other
|
|
92
201
|
* than already-exists (see `ensure-tables.js`).
|
|
93
202
|
*
|
|
94
|
-
* Runtime returns:
|
|
95
|
-
* - `INVALID_INSIGHT_TYPE` —
|
|
203
|
+
* Runtime returns (sync, from `write()`):
|
|
204
|
+
* - `INVALID_INSIGHT_TYPE` — the insightType has no persist plan
|
|
96
205
|
* (typically a config typo or asset-class drift).
|
|
97
|
-
* - `
|
|
98
|
-
*
|
|
206
|
+
* - `STORAGE_FULL` — the buffer is at `bufferCeilingRows`. The
|
|
207
|
+
* row was shed. One shared result object; shedding allocates
|
|
208
|
+
* nothing. Remediation: the endpoint is not taking rows; check it.
|
|
209
|
+
* Raise the ceiling only when memory allows.
|
|
210
|
+
* - `SHUTTING_DOWN` — `shutdown()` has been called; no flusher
|
|
211
|
+
* is left for a new row. One shared result object.
|
|
212
|
+
* - `SEND_FAILED` — the persist plan threw while invoking
|
|
213
|
+
* sender methods (type coercion failure, a client throw). The
|
|
214
|
+
* half-written row is cancelled (mid-row recovery, in
|
|
215
|
+
* `flush-engine.js`).
|
|
99
216
|
* - `INVALID_TIMESTAMP` — reserved; today persist-plan handles invalid
|
|
100
217
|
* timestamps via `onWarning + skip-row`. It may later be surfaced as a
|
|
101
218
|
* hard return error.
|
|
102
219
|
*
|
|
103
220
|
* Shutdown-time throws (ADR-018 — a clean shutdown resolve means
|
|
104
221
|
* everything buffered or in flight was delivered; both carry
|
|
105
|
-
* `dropped: { count }`, exact)
|
|
106
|
-
* (idle, recovery) plus one final flush, raced against `{ timeout }`;
|
|
107
|
-
* its outcome is latched so repeated calls cannot contradict it:
|
|
222
|
+
* `dropped: { count }`, exact):
|
|
108
223
|
* - `DELIVERY_FAILED` — a flush failed; first flush error on `cause`.
|
|
109
224
|
* - `SHUTDOWN_TIMEOUT` — delivery did not settle within the caller's
|
|
110
|
-
* `{ timeout }
|
|
111
|
-
*
|
|
225
|
+
* `{ timeout }`.
|
|
226
|
+
*
|
|
227
|
+
* Runtime console classification (tokens, not `err.code` values):
|
|
228
|
+
* - `DELIVERY_FAILED` — an engine or recovery flush failed or was
|
|
229
|
+
* abandoned at its deadline, or the client refused a row append, and
|
|
230
|
+
* no `onDeliveryFailure` was given. One `logger.error` line per
|
|
231
|
+
* event, carrying the exact rows lost and the client's message. A
|
|
232
|
+
* flush line ends with the probe finding when a probe ran. The
|
|
233
|
+
* process keeps running: an unattended deployment must report a lost
|
|
234
|
+
* batch, not stop on it. The health surface carries the same fact.
|
|
235
|
+
* - `CIRCUIT_OPEN` — delivery paused or resumed (ADR-029 hold and
|
|
236
|
+
* probe). One `logger.warn` line when a failing probe pauses
|
|
237
|
+
* delivery, naming the held rows and the finding. One `logger.warn`
|
|
238
|
+
* line when a tick probe passes and delivery resumes, at warn so a
|
|
239
|
+
* log transport with a warn floor carries the episode end. Nothing
|
|
240
|
+
* per tick. Remediation: the endpoint refused a TCP connect; check
|
|
241
|
+
* that QuestDB is running and reachable. Rows are held up to the
|
|
242
|
+
* ceiling meanwhile, and health reads red with `pausedSince`.
|
|
243
|
+
* - `DELIVERY_HEALTH` — the delivery ladder changed state. One
|
|
244
|
+
* `logger.warn` line when it enters yellow (the first failed flush),
|
|
245
|
+
* one `logger.error` line when it enters red (two failed in a row,
|
|
246
|
+
* one abandoned, or a failing probe pausing delivery, printed just
|
|
247
|
+
* before the `CIRCUIT_OPEN` line), and one `logger.warn` line when
|
|
248
|
+
* it returns to green, naming the episode length and the rows
|
|
249
|
+
* reported lost in it.
|
|
250
|
+
* Nothing while a state persists, whatever the outage length, and
|
|
251
|
+
* the lines print with or without `onDeliveryFailure`. Remediation:
|
|
252
|
+
* the detail is the client's message or the deadline; a red with no
|
|
253
|
+
* `pausedSince` in health is a server that answers and refuses.
|
|
254
|
+
* - `STORAGE_FULL` (lines) — one `logger.warn` line when the first row
|
|
255
|
+
* of an episode is refused at the ceiling, and one when the buffer
|
|
256
|
+
* has room again, with the count refused. The end is found at the
|
|
257
|
+
* interval tick, so its line can lag the first free slot by one
|
|
258
|
+
* interval. Remediation: the endpoint is not taking rows; read the
|
|
259
|
+
* `CIRCUIT_OPEN` and `DELIVERY_HEALTH` lines beside it, and raise
|
|
260
|
+
* `bufferCeilingRows` when a longer outage must be ridden through.
|
|
261
|
+
* - `DEPRECATED_OPTION` — one `logger.warn` line at setup naming the
|
|
262
|
+
* legacy keys in use (see Deprecated options above).
|
|
263
|
+
* - `ADDRESS_IS_NAME` — `ilpUrl` or `pgUrl` is a name other than
|
|
264
|
+
* `localhost`. One `logger.warn` line per field at setup, before any
|
|
265
|
+
* socket opens (ADR-030). A name is allowed and the adapter
|
|
266
|
+
* proceeds. The line tells the operator that only a literal address
|
|
267
|
+
* is immune to a resolver that changes its answer under a running
|
|
268
|
+
* process.
|
|
269
|
+
* - `CALLBACK_FAILED` — the user's `onDeliveryFailure` itself threw or
|
|
270
|
+
* rejected. The shared callback guard contains the fault (ADR-018).
|
|
271
|
+
* The adapter keeps writing and flushing, and each fault becomes one
|
|
272
|
+
* classified console line carrying the detail. `onWarning` is
|
|
273
|
+
* deliberately NOT guarded. A throwing `onWarning` is strict mode:
|
|
274
|
+
* the throw is the instruction that rejects the row. Wrapping it
|
|
275
|
+
* would erase that contract (the ADR-027 exclusion, pinned by the
|
|
276
|
+
* strict-mode specs).
|
|
112
277
|
*
|
|
113
278
|
* Column-internal facts consumed (the ADR-018 column-internal facts
|
|
114
279
|
* pattern):
|
|
@@ -132,31 +297,22 @@
|
|
|
132
297
|
* extend this section and add corresponding assertions in
|
|
133
298
|
* `assert-columns.js`.
|
|
134
299
|
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
* buffer (the copy-out happens synchronously, before any network I/O; the
|
|
152
|
-
* client documents that an unfinished row stays behind), then reset()
|
|
153
|
-
* clears the buffer — at that point holding only the broken stub — and
|
|
154
|
-
* lowers the client's row-in-progress flags.
|
|
155
|
-
* - The client (4.2.0) has no row-cancel API, while its sibling clients do
|
|
156
|
-
* (.NET CancelRow, Rust/C rewind_to_marker, Java recovers automatically).
|
|
157
|
-
* Upstream issue #60 tracks the gap:
|
|
158
|
-
* https://github.com/questdb/nodejs-questdb-client/issues/60
|
|
159
|
-
* When a release ships cancelRow(), recoverSender() becomes that one call.
|
|
300
|
+
* Row append rejections. The client declares `sender.at()` as `async`,
|
|
301
|
+
* so its promise can reject. With the client trigger off, the only
|
|
302
|
+
* rejection left is the client's own byte ceiling (`max_buf_size`,
|
|
303
|
+
* 100 MiB by default): the append throws inside the async function
|
|
304
|
+
* and arrives as a rejection, and the row never completed.
|
|
305
|
+
* `persist-plan.js` attaches one handler per insight type to that
|
|
306
|
+
* promise and routes the failure to `onDeliveryFailure` as
|
|
307
|
+
* `( err, { tableName } )`, or prints one `DELIVERY_FAILED` line. The
|
|
308
|
+
* next write finds the sender holding the unfinished row, throws, and
|
|
309
|
+
* the mid-row recovery in `flush-engine.js` clears it.
|
|
310
|
+
*
|
|
311
|
+
* Client dependency note. `package.json` pins `@questdb/nodejs-client`
|
|
312
|
+
* to `~4.2.0`. `flush-engine.js` lists the five facts of that release
|
|
313
|
+
* the engine relies on; re-verify each on an upgrade. `persist-plan.js`
|
|
314
|
+
* also imports `SenderBufferV1`, an exported but undocumented member,
|
|
315
|
+
* for name validation at setup only.
|
|
160
316
|
*
|
|
161
317
|
* Integration with persist-if node:
|
|
162
318
|
* - persist-if calls: state.storage.write(insightType, msg, partitionId)
|
|
@@ -191,240 +347,269 @@
|
|
|
191
347
|
* @see https://questdb.com/docs/clients/ingest-node/
|
|
192
348
|
* @see docs/architecture/storage-layer.md
|
|
193
349
|
* @see ADR-018
|
|
350
|
+
* @see ADR-029
|
|
194
351
|
*/
|
|
195
352
|
|
|
353
|
+
import http from 'node:http';
|
|
354
|
+
|
|
196
355
|
import { Sender } from '@questdb/nodejs-client';
|
|
197
356
|
import pg from 'pg';
|
|
198
357
|
|
|
199
358
|
import { ENV_VARS } from '../../env-vars.js';
|
|
359
|
+
import { logger } from '../../logger/index.js';
|
|
200
360
|
import { validators } from '../../utils/validate/index.js';
|
|
361
|
+
import { probeAddress, describeProbe } from '../../utils/address/probe.js';
|
|
201
362
|
import { buildPersistPlans } from './persist-plan.js';
|
|
202
363
|
import { ensureTables } from './ensure-tables.js';
|
|
203
364
|
import { assertColumnFacts } from './assert-columns.js';
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
// Error results (INVALID_INSIGHT_TYPE, SEND_FAILED) are constructed per-call
|
|
218
|
-
// because each carries dynamic content (the offending insightType name and the
|
|
219
|
-
// underlying err.message respectively). These paths are rare; per-occurrence
|
|
220
|
-
// allocation on errors is acceptable. The RESULT_OK singleton above covers the
|
|
221
|
-
// per-message hot path.
|
|
222
|
-
|
|
223
|
-
/**
|
|
224
|
-
* Health status thresholds. consecutiveWriteErrors crossing the YELLOW
|
|
225
|
-
* threshold elevates status to 'yellow'; crossing the RED threshold flips
|
|
226
|
-
* `connected` to false (and therefore `status` to 'red'). Tuned for "any
|
|
227
|
-
* error is worth flagging" + "sustained errors mean the transport is gone."
|
|
228
|
-
*/
|
|
229
|
-
const HEALTH_ERROR_YELLOW_THRESHOLD = 1;
|
|
230
|
-
const HEALTH_ERROR_RED_THRESHOLD = 5;
|
|
231
|
-
|
|
232
|
-
/**
|
|
233
|
-
* Pressure threshold above which `status` elevates to at least 'yellow'.
|
|
234
|
-
* The value matches the pressure-aware-yield design (ADR-020, still a
|
|
235
|
-
* Draft). That design proposes 0.66 as the pressure level where the
|
|
236
|
-
* flow would start yielding to let sinks drain; the number still needs
|
|
237
|
-
* benchmark confirmation. Today the yield trigger is time-only (ADR-024), so
|
|
238
|
-
* this alignment is forward-looking, not a description of current yield
|
|
239
|
-
* behaviour. Numeric — the constant exists once here rather than scattered
|
|
240
|
-
* as a magic number.
|
|
241
|
-
*/
|
|
242
|
-
const HEALTH_PRESSURE_YELLOW_THRESHOLD = 0.66;
|
|
365
|
+
import { resolveOptions, deprecationMessage } from './resolve-options.js';
|
|
366
|
+
import {
|
|
367
|
+
isAllowedIlpUrl,
|
|
368
|
+
isAllowedPgUrl,
|
|
369
|
+
assertNotLocalhost,
|
|
370
|
+
assertIlpNotIPv6,
|
|
371
|
+
warnIfName,
|
|
372
|
+
assertReachable,
|
|
373
|
+
pgConnectionTarget,
|
|
374
|
+
NETWORK_ERROR_CODES,
|
|
375
|
+
buildSender
|
|
376
|
+
} from './address-policy.js';
|
|
377
|
+
import { createFlushEngine } from './flush-engine.js';
|
|
243
378
|
|
|
244
379
|
// ============================================================================
|
|
245
380
|
// CONFIGURATION BUILDER
|
|
246
381
|
// ============================================================================
|
|
247
382
|
|
|
248
383
|
/**
|
|
249
|
-
* Build QuestDB Sender configuration
|
|
384
|
+
* Build the QuestDB Sender configuration string. The client's own flush
|
|
385
|
+
* trigger is always off (ADR-029): composer starts every flush itself,
|
|
386
|
+
* so the client never sends a batch this module did not ask for. The
|
|
387
|
+
* transport is stated either way, `stdlib_http=on` unless `stdlibHttp`
|
|
388
|
+
* is false, so a printed config reads the same however it was set.
|
|
250
389
|
*
|
|
251
|
-
* @param {Object} options -
|
|
390
|
+
* @param {Object} options - Resolved settings
|
|
252
391
|
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
253
|
-
* @param {
|
|
254
|
-
* @param {number} [options.
|
|
255
|
-
* @param {number} [options.
|
|
256
|
-
* @param {number} [options.
|
|
257
|
-
* @param {number} [options.retryTimeout] -
|
|
392
|
+
* @param {boolean} [options.stdlibHttp] - False selects undici; anything else the standard library
|
|
393
|
+
* @param {number} [options.initBufSize] - Initial client buffer size in bytes
|
|
394
|
+
* @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
|
|
395
|
+
* @param {number} [options.requestTimeout] - Client request timeout in ms
|
|
396
|
+
* @param {number} [options.retryTimeout] - Client retry window in ms
|
|
258
397
|
* @returns {string} Configuration string for Sender.fromConfig()
|
|
259
398
|
*/
|
|
260
399
|
const buildSenderConfig = function ( options ) {
|
|
261
|
-
const { ilpUrl,
|
|
400
|
+
const { ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout } = options;
|
|
262
401
|
|
|
263
|
-
|
|
264
|
-
let config = `http::addr=${ilpUrl};`;
|
|
402
|
+
let config = `http::addr=${ilpUrl};auto_flush=off;stdlib_http=${( stdlibHttp === false ) ? 'off' : 'on'};`;
|
|
265
403
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
// Use QuestDB's built-in auto-flush
|
|
269
|
-
if ( autoFlushRows !== undefined ) {
|
|
270
|
-
config += `auto_flush_rows=${autoFlushRows};`;
|
|
271
|
-
}
|
|
272
|
-
if ( autoFlushIntervalMs !== undefined ) {
|
|
273
|
-
config += `auto_flush_interval=${autoFlushIntervalMs};`;
|
|
274
|
-
}
|
|
275
|
-
} else {
|
|
276
|
-
// Manual mode: disable auto-flush, we control timing
|
|
277
|
-
config += 'auto_flush=off;';
|
|
404
|
+
if ( initBufSize !== undefined ) {
|
|
405
|
+
config += `init_buf_size=${initBufSize};`;
|
|
278
406
|
}
|
|
279
|
-
|
|
280
|
-
// Optional buffer/retry settings
|
|
281
407
|
if ( maxBufSize !== undefined ) {
|
|
282
|
-
config += `
|
|
408
|
+
config += `max_buf_size=${maxBufSize};`;
|
|
409
|
+
}
|
|
410
|
+
if ( requestTimeout !== undefined ) {
|
|
411
|
+
config += `request_timeout=${requestTimeout};`;
|
|
283
412
|
}
|
|
284
413
|
if ( retryTimeout !== undefined ) {
|
|
285
414
|
config += `retry_timeout=${retryTimeout};`;
|
|
286
415
|
}
|
|
287
416
|
|
|
288
417
|
return config;
|
|
289
|
-
};
|
|
418
|
+
}; // buildSenderConfig()
|
|
290
419
|
|
|
291
420
|
// ============================================================================
|
|
292
|
-
//
|
|
421
|
+
// THE TRANSPORT
|
|
293
422
|
// ============================================================================
|
|
294
423
|
|
|
295
424
|
/**
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
*
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
*
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
* @returns {Promise} settles with the flush, or rejects SHUTDOWN_TIMEOUT
|
|
425
|
+
* How long an idle socket stays open before the adapter closes it.
|
|
426
|
+
* QuestDB closes an idle connection after 300 seconds and announces no
|
|
427
|
+
* keep-alive timeout, so Node gets no hint. A flush that starts in the
|
|
428
|
+
* moment between the server's close and Node's notice of it writes
|
|
429
|
+
* into a dead socket, and this transport does not retry a reset. That
|
|
430
|
+
* would report one batch lost for an outage that never happened.
|
|
431
|
+
* Closing first removes that moment (ADR-029). The value sits below
|
|
432
|
+
* QuestDB's advertised keep-alive of 5 seconds and below the 5-second
|
|
433
|
+
* timeout of Node's own default agent. Node applies it to free sockets
|
|
434
|
+
* only; a request in flight keeps the client's own request timeout.
|
|
307
435
|
*/
|
|
308
|
-
const
|
|
309
|
-
if ( !( timeoutMs > 0 ) ) {
|
|
310
|
-
return flushPromise;
|
|
311
|
-
}
|
|
312
|
-
return new Promise( function ( resolve, reject ) {
|
|
313
|
-
const timer = setTimeout( function () {
|
|
314
|
-
const err = new Error( `final flush did not settle within ${timeoutMs} ms` );
|
|
315
|
-
err.code = 'SHUTDOWN_TIMEOUT';
|
|
316
|
-
reject( err );
|
|
317
|
-
}, timeoutMs );
|
|
318
|
-
timer.unref();
|
|
319
|
-
flushPromise.then(
|
|
320
|
-
function ( value ) {
|
|
321
|
-
clearTimeout( timer );
|
|
322
|
-
resolve( value );
|
|
323
|
-
},
|
|
436
|
+
const IDLE_SOCKET_TIMEOUT_MS = 4000;
|
|
324
437
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
/* c8 ignore stop */
|
|
339
|
-
);
|
|
340
|
-
} );
|
|
341
|
-
}; // raceFlushTimeout()
|
|
438
|
+
/**
|
|
439
|
+
* Builds the one HTTP agent the standard-library transport uses. One
|
|
440
|
+
* socket is enough, because composer starts one flush at a time
|
|
441
|
+
* (ADR-029). Keep-alive reuses that socket from flush to flush, and
|
|
442
|
+
* the adapter closes it after `IDLE_SOCKET_TIMEOUT_MS` without a flush.
|
|
443
|
+
* The adapter destroys the agent at shutdown, which ends any request
|
|
444
|
+
* still on the socket.
|
|
445
|
+
*
|
|
446
|
+
* @returns {http.Agent} The agent
|
|
447
|
+
*/
|
|
448
|
+
const createKeepAliveAgent = function () {
|
|
449
|
+
return new http.Agent( { keepAlive: true, maxSockets: 1, timeout: IDLE_SOCKET_TIMEOUT_MS } );
|
|
450
|
+
}; // createKeepAliveAgent()
|
|
342
451
|
|
|
343
452
|
/**
|
|
344
|
-
*
|
|
345
|
-
*
|
|
346
|
-
*
|
|
347
|
-
*
|
|
348
|
-
*
|
|
349
|
-
*
|
|
350
|
-
*
|
|
453
|
+
* Opens the ILP transport: the agent when the standard-library
|
|
454
|
+
* transport is selected, then the sender. Returns the sender and the
|
|
455
|
+
* function that closes both at the end of the drain. A failed sender
|
|
456
|
+
* build destroys the agent before the classified error goes on, so no
|
|
457
|
+
* agent outlives a failed setup.
|
|
458
|
+
*
|
|
459
|
+
* `closeTransport` closes the sender first, then destroys the agent
|
|
460
|
+
* whatever the sender's close did. Destroying the agent ends any
|
|
461
|
+
* request still on its socket, so the process can exit (ADR-029). With
|
|
462
|
+
* undici there is no agent, and `closeTransport` is the sender's close.
|
|
463
|
+
*
|
|
464
|
+
* @param {Object} parts - The transport inputs
|
|
465
|
+
* @param {Object} parts.SenderClass - The client's Sender class
|
|
466
|
+
* @param {string} parts.senderConfig - The sender configuration string
|
|
467
|
+
* @param {string} parts.ilpUrl - The configured `ilpUrl`, for messages
|
|
468
|
+
* @param {boolean} parts.stdlibHttp - Whether the standard-library transport is selected
|
|
469
|
+
* @param {function} parts.createAgent - Builds the agent
|
|
470
|
+
* @returns {Promise<{sender: Object, closeTransport: function}>} The sender and its close
|
|
471
|
+
* @throws {Error} TRANSPORT_UNREACHABLE or INVALID_CONFIG from `buildSender`
|
|
351
472
|
*/
|
|
352
|
-
const
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
473
|
+
const openTransport = async function ( { SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent } ) {
|
|
474
|
+
if ( stdlibHttp === false ) {
|
|
475
|
+
const sender = await buildSender( SenderClass, senderConfig, ilpUrl, null );
|
|
476
|
+
const closeTransport = function () {
|
|
477
|
+
return sender.close();
|
|
478
|
+
}; // closeTransport()
|
|
479
|
+
return { sender, closeTransport };
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
const agent = createAgent();
|
|
483
|
+
let sender;
|
|
484
|
+
try {
|
|
485
|
+
sender = await buildSender( SenderClass, senderConfig, ilpUrl, agent );
|
|
486
|
+
} catch ( err ) {
|
|
487
|
+
agent.destroy();
|
|
488
|
+
throw err;
|
|
489
|
+
}
|
|
490
|
+
const closeTransport = async function () {
|
|
491
|
+
try {
|
|
492
|
+
await sender.close();
|
|
493
|
+
} finally {
|
|
494
|
+
agent.destroy();
|
|
495
|
+
}
|
|
496
|
+
}; // closeTransport()
|
|
497
|
+
return { sender, closeTransport };
|
|
498
|
+
}; // openTransport()
|
|
499
|
+
|
|
500
|
+
// ============================================================================
|
|
501
|
+
// STORAGE FACTORY
|
|
502
|
+
// ============================================================================
|
|
362
503
|
|
|
363
504
|
/**
|
|
364
505
|
* Create QuestDB storage adapter.
|
|
365
506
|
*
|
|
507
|
+
* The options are resolved by `resolve-options.js` (config over
|
|
508
|
+
* environment over default; legacy aliases mapped or ignored). Only the
|
|
509
|
+
* keys that reach the engine are listed here; the resolver documents
|
|
510
|
+
* the rest.
|
|
511
|
+
*
|
|
366
512
|
* @param {Object} assetClass - Asset class definition with columns and insightTypes
|
|
367
513
|
* @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
|
|
368
514
|
* @param {Object} options - Configuration options
|
|
369
|
-
* @param {string} options.ilpUrl - ILP endpoint (host:port)
|
|
370
|
-
* @param {string} options.pgUrl - PostgreSQL endpoint (host:port)
|
|
371
|
-
* @param {
|
|
372
|
-
* @param {number} [options.
|
|
373
|
-
* @param {number} [options.
|
|
374
|
-
* @param {number} [options.
|
|
375
|
-
* @param {
|
|
376
|
-
* @param {number} [options.
|
|
377
|
-
* @param {number} [options.retryTimeout] -
|
|
515
|
+
* @param {string} [options.ilpUrl] - ILP endpoint (host:port)
|
|
516
|
+
* @param {string} [options.pgUrl] - PostgreSQL endpoint (host:port)
|
|
517
|
+
* @param {number} [options.flushRows=5000] - Rows that start a flush from inside write()
|
|
518
|
+
* @param {number} [options.flushIntervalMs=1000] - Period of the flush timer
|
|
519
|
+
* @param {number} [options.bufferCeilingRows] - Most rows held; default ten times flushRows
|
|
520
|
+
* @param {number} [options.flushDeadlineMs] - Fixed deadline per flush; derived from the rows when unset
|
|
521
|
+
* @param {boolean} [options.stdlibHttp=true] - The standard-library transport; false selects undici
|
|
522
|
+
* @param {number} [options.requestTimeout] - Client request timeout in ms
|
|
523
|
+
* @param {number} [options.retryTimeout] - Client retry window in ms
|
|
524
|
+
* @param {number} [options.initBufSize] - Initial client buffer size in bytes
|
|
525
|
+
* @param {number} [options.maxBufSize] - Byte ceiling of the client buffer
|
|
378
526
|
* @param {string} [options.partitionBy='DAY'] - Table partition interval
|
|
379
|
-
* @param {function} [options.onWarning] - Warning callback for
|
|
527
|
+
* @param {function} [options.onWarning] - Warning callback for skipped values
|
|
528
|
+
* @param {function} [options.onDeliveryFailure] - Called once per lost flush
|
|
380
529
|
* @param {Object} [deps={}] - Injectable dependencies (for testing)
|
|
381
530
|
* @param {Object} [deps.SenderClass] - QuestDB Sender class (default: @questdb/nodejs-client Sender)
|
|
382
531
|
* @param {Object} [deps.PgClientClass] - PostgreSQL Client class (default: pg.Client)
|
|
383
|
-
* @
|
|
532
|
+
* @param {function} [deps.probeFn] - Setup probe (default: `probeAddress`, ADR-030)
|
|
533
|
+
* @param {function} [deps.createAgent] - Builds the transport agent (default: `createKeepAliveAgent`)
|
|
534
|
+
* @returns {Promise<Object>} Storage adapter with write, flush, shutdown, getPressure, getHealth
|
|
384
535
|
*/
|
|
385
536
|
const createQuestDBStorage = async function ( assetClass, tablePrefix, options, deps = {} ) {
|
|
537
|
+
// Options become settings here, once (ADR-029). A ceiling below the
|
|
538
|
+
// threshold fails setup inside the resolver with INVALID_CONFIG.
|
|
539
|
+
const { settings, deprecations } = resolveOptions( options, ENV_VARS );
|
|
386
540
|
const {
|
|
387
|
-
ilpUrl
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
retryTimeout = ENV_VARS.questdbRetryTimeout,
|
|
396
|
-
partitionBy = 'DAY',
|
|
397
|
-
onWarning,
|
|
398
|
-
onDeliveryFailure
|
|
399
|
-
} = options;
|
|
541
|
+
ilpUrl, pgUrl, stdlibHttp, requestTimeout, retryTimeout, initBufSize, maxBufSize,
|
|
542
|
+
partitionBy, onWarning, onDeliveryFailure
|
|
543
|
+
} = settings;
|
|
544
|
+
|
|
545
|
+
// One line names every legacy key in use and what happened to it.
|
|
546
|
+
if ( deprecations.length > 0 ) {
|
|
547
|
+
logger.warn( deprecationMessage( deprecations ) );
|
|
548
|
+
}
|
|
400
549
|
|
|
401
550
|
// Runtime validation — required from either DSL config or ENV_VARS.
|
|
402
551
|
// Per ADR-018, setup-time throws carry classified err.code.
|
|
403
552
|
if ( !ilpUrl ) {
|
|
404
|
-
const err = new Error( '
|
|
553
|
+
const err = new Error( 'winkComposer/questdb: ilpUrl required — set in .storage() config or QUESTDB_ILP_URL env var' );
|
|
405
554
|
err.code = 'INVALID_CONFIG';
|
|
406
555
|
throw err;
|
|
407
556
|
}
|
|
408
557
|
if ( !pgUrl ) {
|
|
409
|
-
const err = new Error( '
|
|
558
|
+
const err = new Error( 'winkComposer/questdb: pgUrl required — set in .storage() config or QUESTDB_PG_URL env var' );
|
|
559
|
+
err.code = 'INVALID_CONFIG';
|
|
560
|
+
throw err;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// The prefix opens every table name, and CREATE TABLE writes that
|
|
564
|
+
// name unquoted (header sub-case (j)). The schema checks a flow's
|
|
565
|
+
// config. This check covers a direct caller and the default, which
|
|
566
|
+
// is the asset class name.
|
|
567
|
+
if ( !validators.identifier( tablePrefix ) ) {
|
|
568
|
+
const err = new Error(
|
|
569
|
+
`winkComposer/questdb: tablePrefix '${tablePrefix}' is not a valid table name prefix [INVALID_CONFIG]: ` +
|
|
570
|
+
'use letters, digits, _ and $ only, and do not start with a digit'
|
|
571
|
+
);
|
|
410
572
|
err.code = 'INVALID_CONFIG';
|
|
411
573
|
throw err;
|
|
412
574
|
}
|
|
413
575
|
|
|
576
|
+
// Address policy (ADR-030), before any socket opens. `localhost` is
|
|
577
|
+
// refused. An IPv6 literal is refused for the ILP path. A name gets
|
|
578
|
+
// one warning per field. See header sub-cases (f), (g) and the
|
|
579
|
+
// ADDRESS_IS_NAME console token.
|
|
580
|
+
const ilpAddress = assertNotLocalhost( 'ilpUrl', ilpUrl, 'QUESTDB_ILP_URL' );
|
|
581
|
+
const pgAddress = assertNotLocalhost( 'pgUrl', pgUrl, 'QUESTDB_PG_URL' );
|
|
582
|
+
assertIlpNotIPv6( ilpAddress );
|
|
583
|
+
warnIfName( 'ilpUrl', ilpAddress );
|
|
584
|
+
warnIfName( 'pgUrl', pgAddress );
|
|
585
|
+
|
|
414
586
|
// Injectable dependencies with defaults
|
|
415
587
|
const {
|
|
416
588
|
SenderClass = Sender,
|
|
417
|
-
PgClientClass = pg.Client
|
|
589
|
+
PgClientClass = pg.Client,
|
|
590
|
+
probeFn = probeAddress,
|
|
591
|
+
createAgent = createKeepAliveAgent
|
|
418
592
|
} = deps;
|
|
419
593
|
|
|
420
|
-
// Build persist plans (pre-compiled closures)
|
|
594
|
+
// Build persist plans (pre-compiled closures). The callbacks go in
|
|
595
|
+
// RAW: buildPersistPlans validates them itself and arms its own
|
|
596
|
+
// guarded copy for the row-append site. Handing it a pre-wrapped
|
|
597
|
+
// function would defeat that validation (the guard turns a
|
|
598
|
+
// non-function into null instead of the fail-fast INVALID_CONFIG).
|
|
599
|
+
// The engine wraps its own copy after this validation (ADR-027:
|
|
600
|
+
// wrap once, after the site's own validation).
|
|
421
601
|
const persistPlans = buildPersistPlans( assetClass, tablePrefix, { onWarning, onDeliveryFailure } );
|
|
422
602
|
|
|
603
|
+
// Setup probe, PostgreSQL side (ADR-030 item 4): every address the
|
|
604
|
+
// endpoint resolves to must answer before the client opens. The
|
|
605
|
+
// ILP side is probed below, before the sender is built.
|
|
606
|
+
await assertReachable( 'pgUrl', pgAddress, probeFn );
|
|
607
|
+
|
|
423
608
|
// Ensure tables exist via PostgreSQL wire protocol
|
|
424
|
-
const
|
|
609
|
+
const { host: pgHost, port: pgPort } = pgConnectionTarget( pgUrl, pgAddress );
|
|
425
610
|
const pgClient = new PgClientClass( {
|
|
426
611
|
host: pgHost,
|
|
427
|
-
port:
|
|
612
|
+
port: pgPort,
|
|
428
613
|
database: ENV_VARS.questdbDatabase,
|
|
429
614
|
user: ENV_VARS.questdbUser,
|
|
430
615
|
password: ENV_VARS.questdbPassword
|
|
@@ -432,20 +617,21 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
432
617
|
|
|
433
618
|
// Wrap the PostgreSQL connect so the failure carries a classified
|
|
434
619
|
// `err.code` an operator can route on. The ADR-018 error vocabulary
|
|
435
|
-
// mandates one split here
|
|
436
|
-
// code like ECONNREFUSED
|
|
437
|
-
//
|
|
438
|
-
// network, the firewall, whether the service is
|
|
439
|
-
// connection string may be
|
|
440
|
-
//
|
|
441
|
-
//
|
|
442
|
-
//
|
|
443
|
-
//
|
|
620
|
+
// mandates one split here. An endpoint that does not answer is
|
|
621
|
+
// TRANSPORT_UNREACHABLE: a Node syscall code like ECONNREFUSED (see
|
|
622
|
+
// NETWORK_ERROR_CODES in address-policy.js). The remediation there
|
|
623
|
+
// is "check the network, the firewall, whether the service is
|
|
624
|
+
// running", and the connection string may be fine.
|
|
625
|
+
//
|
|
626
|
+
// Everything else stays INVALID_CONFIG: an auth failure (the host
|
|
627
|
+
// answered), a protocol error, an unclassified throw. The fix there
|
|
628
|
+
// is the supplied config. The underlying error is preserved on
|
|
629
|
+
// `err.cause` for diagnostics either way.
|
|
444
630
|
try {
|
|
445
631
|
await pgClient.connect();
|
|
446
632
|
} catch ( connErr ) {
|
|
447
633
|
const err = new Error(
|
|
448
|
-
`
|
|
634
|
+
`winkComposer/questdb: could not connect to PostgreSQL at ${pgUrl} — ${connErr.message}`
|
|
449
635
|
);
|
|
450
636
|
err.code = NETWORK_ERROR_CODES.has( connErr.code ) ?
|
|
451
637
|
'TRANSPORT_UNREACHABLE' :
|
|
@@ -460,539 +646,52 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
460
646
|
await pgClient.end();
|
|
461
647
|
}
|
|
462
648
|
|
|
463
|
-
//
|
|
649
|
+
// The sender config. The client trigger is off and the transport is
|
|
650
|
+
// stated: see buildSenderConfig.
|
|
464
651
|
const senderConfig = buildSenderConfig( {
|
|
465
|
-
ilpUrl,
|
|
466
|
-
flushMode,
|
|
467
|
-
autoFlushRows,
|
|
468
|
-
autoFlushIntervalMs,
|
|
469
|
-
maxBufSize,
|
|
470
|
-
retryTimeout
|
|
652
|
+
ilpUrl, stdlibHttp, initBufSize, maxBufSize, requestTimeout, retryTimeout
|
|
471
653
|
} );
|
|
472
654
|
|
|
473
|
-
//
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
// The client's flush() copies its rows out of the buffer synchronously
|
|
478
|
-
// (copy-out) before any network I/O, so the moment a flush is called
|
|
479
|
-
// its rows are no longer buffered — they move to the in-flight tally
|
|
480
|
-
// below. Conflating the two quantities in one counter let shutdown
|
|
481
|
-
// report clean over a hung idle flush.
|
|
482
|
-
let lastWriteTime = 0;
|
|
483
|
-
let idleFlushTimer = null;
|
|
484
|
-
let bufferedRows = 0;
|
|
485
|
-
|
|
486
|
-
// Rows inside flush copies that have not settled yet, plus the flush
|
|
487
|
-
// promises carrying them. getPressure() adds this to bufferedRows (a
|
|
488
|
-
// hung flush is undelivered data and must read as pressure), and
|
|
489
|
-
// shutdown() settles these entries instead of firing a blind flush
|
|
490
|
-
// at a buffer the copy-out already emptied.
|
|
491
|
-
let inFlightRows = 0;
|
|
492
|
-
const inFlightFlushes = new Set();
|
|
493
|
-
|
|
494
|
-
// Shutdown outcome, latched on the first call. A lossy shutdown's
|
|
495
|
-
// failed flush already emptied the buffer via copy-out, so a re-run
|
|
496
|
-
// would find nothing to flush and resolve clean — contradicting the
|
|
497
|
-
// recorded loss. Every caller gets the first call's promise instead.
|
|
498
|
-
let shutdownPromise = null;
|
|
499
|
-
|
|
500
|
-
// Health state — drives getHealth()'s status/connected derivation.
|
|
501
|
-
// shuttingDown flips true at the start of shutdown(); never resets.
|
|
502
|
-
// consecutiveWriteErrors increments on every write() catch and resets to 0
|
|
503
|
-
// on every successful persist plan call (recovery). Pure derived signal —
|
|
504
|
-
// no extra hot-path allocation; just two integer/boolean operations.
|
|
505
|
-
let shuttingDown = false;
|
|
506
|
-
let consecutiveWriteErrors = 0;
|
|
507
|
-
|
|
508
|
-
// One idle flush in flight at a time. Against an unreachable server a
|
|
509
|
-
// flush can never settle (the client's retry loop — see the mid-row
|
|
510
|
-
// recovery section in the file header); without this guard every check
|
|
511
|
-
// interval would start ANOTHER stuck flush. A hung flush leaves the
|
|
512
|
-
// flag set, the buffer keeps filling, and the condition surfaces as
|
|
513
|
-
// rising pressure in getHealth() — visible, not multiplied.
|
|
514
|
-
let idleFlushInFlight = false;
|
|
515
|
-
|
|
516
|
-
/**
|
|
517
|
-
* Registers a flush the moment it is called. Copy-out means the rows
|
|
518
|
-
* leave the buffer NOW, delivered or not, so the caller hands them
|
|
519
|
-
* over in the same breath. Settlement — either way — removes them
|
|
520
|
-
* from the in-flight tally: resolved means delivered; rejected means
|
|
521
|
-
* lost, and loss REPORTING stays at the call site (the idle flush
|
|
522
|
-
* logs or routes to onDeliveryFailure, recovery routes to
|
|
523
|
-
* onDeliveryFailure, shutdown throws classified).
|
|
524
|
-
*
|
|
525
|
-
* @param {Promise} flushPromise - the just-fired sender.flush()
|
|
526
|
-
* @param {number} rows - row count the flush copy carries
|
|
527
|
-
* @returns {{promise: Promise, rows: number}} the tracked entry
|
|
528
|
-
*/
|
|
529
|
-
const trackFlush = function ( flushPromise, rows ) {
|
|
530
|
-
const entry = { promise: flushPromise, rows };
|
|
531
|
-
inFlightRows += rows;
|
|
532
|
-
inFlightFlushes.add( entry );
|
|
533
|
-
const settle = function () {
|
|
534
|
-
inFlightFlushes.delete( entry );
|
|
535
|
-
inFlightRows -= rows;
|
|
536
|
-
};
|
|
537
|
-
flushPromise.then( settle, settle );
|
|
538
|
-
return entry;
|
|
539
|
-
}; // trackFlush()
|
|
540
|
-
|
|
541
|
-
/**
|
|
542
|
-
* Check and flush if idle for idleFlushAfterMs.
|
|
543
|
-
*
|
|
544
|
-
* Runs in BOTH modes (see longer comment block at the setInterval call
|
|
545
|
-
* below). The rows move to the in-flight tally the moment the flush
|
|
546
|
-
* fires; a successful settle clears them, which also bounds the
|
|
547
|
-
* worst-case pressure-counter lag in auto mode (when QuestDB's silent
|
|
548
|
-
* `auto_flush_interval` trigger fires without us observing it, this
|
|
549
|
-
* is the self-healing point).
|
|
550
|
-
*/
|
|
551
|
-
const checkIdleFlush = async function () {
|
|
552
|
-
if ( idleFlushInFlight || bufferedRows === 0 ) {
|
|
553
|
-
return;
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
const now = Date.now();
|
|
557
|
-
const idleTime = now - lastWriteTime;
|
|
558
|
-
|
|
559
|
-
if ( idleTime >= idleFlushAfterMs ) {
|
|
560
|
-
idleFlushInFlight = true;
|
|
561
|
-
const rows = bufferedRows;
|
|
562
|
-
bufferedRows = 0;
|
|
563
|
-
const entry = trackFlush( sender.flush(), rows );
|
|
564
|
-
try {
|
|
565
|
-
await entry.promise;
|
|
566
|
-
} catch ( err ) {
|
|
567
|
-
// Copy-out: the failed flush carried the rows with it —
|
|
568
|
-
// the buffer no longer holds them and a retry cannot
|
|
569
|
-
// resend them. They are lost, and this is the report
|
|
570
|
-
// (the settle handler already cleared them from
|
|
571
|
-
// pressure). Same routing policy as the recovery flush:
|
|
572
|
-
// the caller owns the response when it asked to.
|
|
573
|
-
if ( onDeliveryFailure ) {
|
|
574
|
-
onDeliveryFailure( err, { idleFlush: true, rowsLost: rows } );
|
|
575
|
-
} else {
|
|
576
|
-
console.error(
|
|
577
|
-
`WinkComposer/questdb: idle flush failed; ${rows} buffered row(s) lost: ${err.message}`
|
|
578
|
-
);
|
|
579
|
-
}
|
|
580
|
-
} finally {
|
|
581
|
-
idleFlushInFlight = false;
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
};
|
|
585
|
-
|
|
586
|
-
// Start idle flush timer as safety net for "data stopped flowing" case.
|
|
587
|
-
// QuestDB's auto_flush_interval only checks elapsed time when NEW data is added -
|
|
588
|
-
// it is NOT a real background timer. If no more data arrives, the buffer sits
|
|
589
|
-
// indefinitely. QuestDB docs recommend: "implement your own timer-based logic."
|
|
590
|
-
// See: https://py-questdb-client.readthedocs.io/en/stable/conf.html
|
|
591
|
-
//
|
|
592
|
-
// This timer runs in BOTH modes:
|
|
593
|
-
// - Auto mode: Safety net for when data stops flowing (QuestDB won't flush on its own)
|
|
594
|
-
// - Manual mode: Primary flush mechanism (QuestDB auto-flush is disabled)
|
|
595
|
-
idleFlushTimer = setInterval( checkIdleFlush, idleFlushCheckMs );
|
|
596
|
-
// Don't block process exit
|
|
597
|
-
idleFlushTimer.unref();
|
|
598
|
-
|
|
599
|
-
// ========================================================================
|
|
600
|
-
// STORAGE INTERFACE
|
|
601
|
-
// ========================================================================
|
|
602
|
-
|
|
603
|
-
/**
|
|
604
|
-
* Cancels a half-written ILP row after a mid-row throw, so the NEXT write
|
|
605
|
-
* starts clean (see "Mid-row recovery" in the file header for the full
|
|
606
|
-
* account and the upstream issue link).
|
|
607
|
-
*
|
|
608
|
-
* flush() copies every completed row out of the buffer synchronously and
|
|
609
|
-
* sends them in the background; the unfinished row stays behind. reset()
|
|
610
|
-
* then clears the buffer — only the broken stub remains at that point —
|
|
611
|
-
* and lowers the client's row-in-progress flags. Net effect: the broken
|
|
612
|
-
* row vanishes, every good row is on its way, the sender accepts the
|
|
613
|
-
* next write.
|
|
614
|
-
*
|
|
615
|
-
* The early flush carries real data. If its send ultimately fails (after
|
|
616
|
-
* the client's own retries), that loss is reported like any other
|
|
617
|
-
* delivery failure: routed to `onDeliveryFailure` when provided,
|
|
618
|
-
* otherwise thrown as a classified unhandled rejection — the same policy
|
|
619
|
-
* as persist-plan's sender.at() failure path. Never silent.
|
|
620
|
-
*/
|
|
621
|
-
const recoverSender = function () {
|
|
622
|
-
try {
|
|
623
|
-
// flush() is an async function: it can never throw synchronously,
|
|
624
|
-
// and the rows it sends live in its own copy of the buffer.
|
|
625
|
-
// Tracked like every flush, so shutdown settles it and its
|
|
626
|
-
// rows stay visible as pressure until it settles.
|
|
627
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
628
|
-
sender.reset();
|
|
629
|
-
bufferedRows = 0;
|
|
630
|
-
|
|
631
|
-
entry.promise.catch( ( flushErr ) => {
|
|
632
|
-
if ( onDeliveryFailure ) {
|
|
633
|
-
onDeliveryFailure( flushErr, { recovery: true } );
|
|
634
|
-
return;
|
|
635
|
-
}
|
|
636
|
-
const failure = new Error(
|
|
637
|
-
`WinkComposer/questdb: recovery flush failed after a mid-row write error: ${flushErr.message}. ` +
|
|
638
|
-
'Completed rows in that batch were dropped. Provide an `onDeliveryFailure` callback ' +
|
|
639
|
-
'in the storage config to handle these explicitly.'
|
|
640
|
-
);
|
|
641
|
-
failure.code = 'DELIVERY_FAILED';
|
|
642
|
-
failure.cause = flushErr;
|
|
643
|
-
throw failure;
|
|
644
|
-
} );
|
|
645
|
-
} catch ( recoveryErr ) {
|
|
646
|
-
// Defensive: with the 4.2.0 client neither call can throw here
|
|
647
|
-
// (flush is async, reset is trivial buffer bookkeeping). If a
|
|
648
|
-
// future client changes that, the sender may stay wedged — but
|
|
649
|
-
// write() must still return its classified result rather than
|
|
650
|
-
// throw (ADR-018: the hot path never throws), and the
|
|
651
|
-
// failure must be visible.
|
|
652
|
-
console.error( `WinkComposer/questdb: sender recovery failed: ${recoveryErr.message}` );
|
|
653
|
-
}
|
|
654
|
-
}; // recoverSender()
|
|
655
|
-
|
|
656
|
-
/**
|
|
657
|
-
* Write a message to QuestDB for a given insightType.
|
|
658
|
-
*
|
|
659
|
-
* Sync hot-path return per ADR-018 / ADR-013:
|
|
660
|
-
* - `RESULT_OK` (singleton) on successful enqueue.
|
|
661
|
-
* - `{ ok: false, error: { code: 'INVALID_INSIGHT_TYPE', message } }`
|
|
662
|
-
* when the insightType has no persist plan (config error).
|
|
663
|
-
* - `{ ok: false, error: { code: 'SEND_FAILED', message } }` when the
|
|
664
|
-
* persist plan throws (sender method failure: ILP buffer overflow,
|
|
665
|
-
* type coercion failure, etc.).
|
|
666
|
-
*
|
|
667
|
-
* Never throws. Async tryFlush rejections inside `sender.at()` are
|
|
668
|
-
* contained at the call site in `persist-plan.js` (see file header).
|
|
669
|
-
*
|
|
670
|
-
* @param {string} insightType - SignalType name (must exist in assetClass)
|
|
671
|
-
* @param {Object} message - Message with column values
|
|
672
|
-
* @param {string} partitionId - Partition identifier (stored as SYMBOL)
|
|
673
|
-
* @returns {{ok: true} | {ok: false, error: {code: string, message: string}}}
|
|
674
|
-
*/
|
|
675
|
-
const write = function ( insightType, message, partitionId ) {
|
|
676
|
-
if ( shuttingDown ) {
|
|
677
|
-
// The idle timer is already stopped and the final flush may
|
|
678
|
-
// have run: a row accepted now would have no flusher left.
|
|
679
|
-
// Refuse loudly instead of stranding it — uniform with the
|
|
680
|
-
// MQTT emitter's SHUTTING_DOWN refusal.
|
|
681
|
-
return {
|
|
682
|
-
ok: false,
|
|
683
|
-
error: {
|
|
684
|
-
code: 'SHUTTING_DOWN',
|
|
685
|
-
message: 'WinkComposer/questdb: write rejected — storage is shutting down'
|
|
686
|
-
}
|
|
687
|
-
};
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
const persistPlan = persistPlans[ insightType ];
|
|
691
|
-
|
|
692
|
-
if ( !persistPlan ) {
|
|
693
|
-
return {
|
|
694
|
-
ok: false,
|
|
695
|
-
error: {
|
|
696
|
-
code: 'INVALID_INSIGHT_TYPE',
|
|
697
|
-
message: `No persist plan for insightType '${insightType}'`
|
|
698
|
-
}
|
|
699
|
-
};
|
|
700
|
-
}
|
|
701
|
-
|
|
702
|
-
try {
|
|
703
|
-
// The plan reports whether it actually opened a row. A row
|
|
704
|
-
// skipped in phase 1 (bad designated timestamp) never touched
|
|
705
|
-
// the sender and must not count as buffered — it would inflate
|
|
706
|
-
// pressure and shutdown's dropped count.
|
|
707
|
-
const written = persistPlan( sender, message, partitionId );
|
|
708
|
-
consecutiveWriteErrors = 0; // recovery — health flips back to green
|
|
709
|
-
if ( written ) {
|
|
710
|
-
lastWriteTime = Date.now();
|
|
711
|
-
bufferedRows += 1;
|
|
712
|
-
// Auto mode: QuestDB triggers an internal flush on the sender.at()
|
|
713
|
-
// call that brings pendingRowCount to autoFlushRows (verified
|
|
714
|
-
// against @questdb/nodejs-client v4.2.0 dist/cjs/index.js
|
|
715
|
-
// tryFlush: pendingRowCount >= autoFlushRows). Mirror that reset
|
|
716
|
-
// so getPressure() reflects QuestDB's actual buffer state.
|
|
717
|
-
//
|
|
718
|
-
// Guarded on `autoFlushRows > 0` because the value is optional in
|
|
719
|
-
// ENV_VARS / config; when unset, QuestDB falls back to its own
|
|
720
|
-
// internal default which we cannot observe. In that case the
|
|
721
|
-
// heuristic does not fire and QuestDB contributes no pressure
|
|
722
|
-
// signal — `getPressure()` (below) returns 0.
|
|
723
|
-
if ( flushMode === 'auto' && autoFlushRows > 0 && bufferedRows >= autoFlushRows ) {
|
|
724
|
-
bufferedRows = 0;
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
return RESULT_OK;
|
|
728
|
-
} catch ( err ) {
|
|
729
|
-
consecutiveWriteErrors += 1; // health degradation signal
|
|
730
|
-
// The throw may have left a half-written row in the sender;
|
|
731
|
-
// cancel it so this failure costs one row, not the rest of the
|
|
732
|
-
// run (ADR-018). Safe to call even when no row was open: flushing
|
|
733
|
-
// early is harmless and reset() on a consistent buffer is a no-op.
|
|
734
|
-
recoverSender();
|
|
735
|
-
return {
|
|
736
|
-
ok: false,
|
|
737
|
-
error: {
|
|
738
|
-
code: 'SEND_FAILED',
|
|
739
|
-
message: err.message
|
|
740
|
-
}
|
|
741
|
-
};
|
|
742
|
-
}
|
|
743
|
-
};
|
|
744
|
-
|
|
745
|
-
/**
|
|
746
|
-
* Flush pending rows to QuestDB.
|
|
747
|
-
* In auto mode, this is a no-op (QuestDB handles it).
|
|
748
|
-
* In manual mode, this forces an immediate flush. The rows move to
|
|
749
|
-
* the in-flight tally at the call (copy-out); a failure rejects to
|
|
750
|
-
* the caller exactly as before, and the settle handler keeps the
|
|
751
|
-
* pressure accounting straight either way.
|
|
752
|
-
*
|
|
753
|
-
* @returns {Promise<void>}
|
|
754
|
-
*/
|
|
755
|
-
const flush = async function () {
|
|
756
|
-
if ( flushMode === 'manual' && bufferedRows > 0 ) {
|
|
757
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
758
|
-
bufferedRows = 0;
|
|
759
|
-
await entry.promise;
|
|
760
|
-
}
|
|
761
|
-
// Auto mode: no-op, QuestDB handles flushing
|
|
762
|
-
};
|
|
763
|
-
|
|
764
|
-
/**
|
|
765
|
-
* Backpressure metric for the partition manager. Returns the buffer fill
|
|
766
|
-
* ratio in [0, 1] per ADR-018 (sync, O(1), allocation-free). Clamped
|
|
767
|
-
* to 1.0 — the codebase invariant for getPressure across adapters.
|
|
768
|
-
*
|
|
769
|
-
* Accuracy by mode:
|
|
770
|
-
* - Manual mode: exact (we own every flush).
|
|
771
|
-
* - Auto mode, heavy flow: exact (autoFlushRows heuristic reset matches).
|
|
772
|
-
* - Auto mode, slow flow + auto_flush_interval triggers: over-states by
|
|
773
|
-
* up to `idleFlushAfterMs`, self-heals via `checkIdleFlush`.
|
|
774
|
-
*
|
|
775
|
-
* Rows inside unsettled flush copies count as pressure: a hung flush
|
|
776
|
-
* is undelivered data, and hiding it is what let shutdown report
|
|
777
|
-
* clean over it. They leave the tally when their flush settles —
|
|
778
|
-
* delivered or reported lost.
|
|
779
|
-
*
|
|
780
|
-
* When `autoFlushRows` is not configured (left to QuestDB's internal
|
|
781
|
-
* default), we have no observable capacity reference and return
|
|
782
|
-
* 0 — QuestDB then contributes nothing to the flow's pressure signal.
|
|
783
|
-
* The time-based yield (ADR-024) and message-count safety nets remain
|
|
784
|
-
* effective.
|
|
785
|
-
*
|
|
786
|
-
* @returns {number} Pressure value in [0, 1]
|
|
787
|
-
*/
|
|
788
|
-
const getPressure = function () {
|
|
789
|
-
if ( !( autoFlushRows > 0 ) ) {
|
|
790
|
-
return 0;
|
|
791
|
-
}
|
|
792
|
-
return Math.min( 1.0, ( bufferedRows + inFlightRows ) / autoFlushRows );
|
|
793
|
-
};
|
|
655
|
+
// Setup probe, ILP side (ADR-030 item 4). Before this change the
|
|
656
|
+
// write path was never opened at setup; the first sign of a dead
|
|
657
|
+
// endpoint was the first flush.
|
|
658
|
+
await assertReachable( 'ilpUrl', ilpAddress, probeFn );
|
|
794
659
|
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
* load-bearing institutional knowledge):
|
|
802
|
-
* - `red` if `!connected` (shutting down or sustained write failure)
|
|
803
|
-
* - `yellow` if `pressure >= HEALTH_PRESSURE_YELLOW_THRESHOLD` OR
|
|
804
|
-
* `consecutiveWriteErrors >= HEALTH_ERROR_YELLOW_THRESHOLD`
|
|
805
|
-
* - `green` otherwise
|
|
806
|
-
*
|
|
807
|
-
* `connected` here is *derived* — QuestDB's ILP sender is fire-and-forget
|
|
808
|
-
* with no observable socket state, so we infer transport health from
|
|
809
|
-
* recent write success.
|
|
810
|
-
*
|
|
811
|
-
* @returns {{status: 'green'|'yellow'|'red', connected: boolean, pressure: number, consecutiveWriteErrors: number, bufferedRows: number, flushMode: string}}
|
|
812
|
-
*/
|
|
813
|
-
const getHealth = function () {
|
|
814
|
-
const pressure = getPressure();
|
|
815
|
-
const connected = !shuttingDown && consecutiveWriteErrors < HEALTH_ERROR_RED_THRESHOLD;
|
|
660
|
+
// The agent, when the standard-library transport is selected, then
|
|
661
|
+
// the sender. A failure is classified, cause attached (see
|
|
662
|
+
// buildSender), and leaves no agent behind (see openTransport).
|
|
663
|
+
const { sender, closeTransport } = await openTransport( {
|
|
664
|
+
SenderClass, senderConfig, ilpUrl, stdlibHttp, createAgent
|
|
665
|
+
} );
|
|
816
666
|
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
667
|
+
// The probe the engine runs after a failed flush (ADR-029 hold and
|
|
668
|
+
// probe), bound to the ILP address so the engine knows nothing
|
|
669
|
+
// about addresses. Setup ran the same probe on both endpoints.
|
|
670
|
+
const probe = {
|
|
671
|
+
run: function () {
|
|
672
|
+
return probeFn( ilpAddress );
|
|
673
|
+
},
|
|
674
|
+
describe: function ( outcome ) {
|
|
675
|
+
return describeProbe( outcome, 'ilpUrl', ilpAddress );
|
|
824
676
|
}
|
|
825
|
-
|
|
826
|
-
return {
|
|
827
|
-
// Required health floor (ADR-018)
|
|
828
|
-
status,
|
|
829
|
-
connected,
|
|
830
|
-
pressure,
|
|
831
|
-
// Adapter-specific diagnostics
|
|
832
|
-
consecutiveWriteErrors,
|
|
833
|
-
bufferedRows,
|
|
834
|
-
flushMode
|
|
835
|
-
};
|
|
836
677
|
};
|
|
837
678
|
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
*/
|
|
843
|
-
const closeQuietly = function () {
|
|
844
|
-
return sender.close().catch( function ( closeErr ) {
|
|
845
|
-
console.error( `WinkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
|
|
846
|
-
} );
|
|
847
|
-
}; // closeQuietly()
|
|
848
|
-
|
|
849
|
-
/**
|
|
850
|
-
* The real shutdown body. `shutdown` below latches its promise so
|
|
851
|
-
* every caller — including re-entrant and post-failure callers —
|
|
852
|
-
* receives this one outcome.
|
|
853
|
-
*
|
|
854
|
-
* A clean resolve is a delivery statement (ADR-018): everything
|
|
855
|
-
* buffered OR in flight was delivered. Shutdown therefore settles
|
|
856
|
-
* every unsettled flush (idle, recovery) plus one final flush for
|
|
857
|
-
* whatever is still buffered, all raced against the caller's
|
|
858
|
-
* `{ timeout }` (ADR-018 drain-then-close). It never fires a blind flush at a
|
|
859
|
-
* buffer an earlier copy-out emptied — that is what let it report
|
|
860
|
-
* clean over a hung idle flush.
|
|
861
|
-
*
|
|
862
|
-
* On loss it rejects classified, `dropped: { count }` exact:
|
|
863
|
-
* - any awaited flush fails → `DELIVERY_FAILED`, first flush error
|
|
864
|
-
* on `cause`, count = rows on the flushes that failed;
|
|
865
|
-
* - the combined wait does not settle in time → `SHUTDOWN_TIMEOUT`,
|
|
866
|
-
* count = rows not confirmed delivered. A send against an
|
|
867
|
-
* unreachable server never settles (the client's retry loop — see
|
|
868
|
-
* the file header), so the bound is what keeps shutdown finite.
|
|
869
|
-
* `dropped` is a statement about THIS session: those rows were not
|
|
870
|
-
* confirmed delivered before close. An abandoned flush keeps
|
|
871
|
-
* retrying and may still land its rows later if the server
|
|
872
|
-
* recovers — the count is a floor on uncertainty, not a proof of
|
|
873
|
-
* loss.
|
|
874
|
-
*
|
|
875
|
-
* The transport close is attempted in both loss paths, but on the
|
|
876
|
-
* HTTP transport the client's `close()` is an empty function
|
|
877
|
-
* (verified against @questdb/nodejs-client 4.2.0), so nothing can
|
|
878
|
-
* abort an abandoned flush's retry timers from here; they keep the
|
|
879
|
-
* event loop alive. Process exit is the final backstop — in a flow,
|
|
880
|
-
* the shutdown manager's `SHUTDOWN_FORCE_TIMEOUT_MS` exit covers
|
|
881
|
-
* this. No timeout supplied = no enforcement (unbounded await),
|
|
882
|
-
* preserving direct-caller behavior.
|
|
883
|
-
*/
|
|
884
|
-
const doShutdown = async function ( timeout ) {
|
|
885
|
-
// Flip the health flag first so any concurrent getHealth() call
|
|
886
|
-
// immediately sees the shutdown and returns red/disconnected —
|
|
887
|
-
// and write() starts refusing new rows (SHUTTING_DOWN).
|
|
888
|
-
shuttingDown = true;
|
|
889
|
-
|
|
890
|
-
// Stop idle flush timer
|
|
891
|
-
if ( idleFlushTimer ) {
|
|
892
|
-
clearInterval( idleFlushTimer );
|
|
893
|
-
idleFlushTimer = null;
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
// Everything delivery still owes: flushes already in flight
|
|
897
|
-
// (their rows left the buffer at their call) plus one final
|
|
898
|
-
// flush for whatever is still buffered. Each wait records its
|
|
899
|
-
// outcome into the tallies below; the mapped promises never
|
|
900
|
-
// reject, so the only rejection the race can surface is the
|
|
901
|
-
// timeout itself.
|
|
902
|
-
let totalRows = 0;
|
|
903
|
-
let deliveredRows = 0;
|
|
904
|
-
let failedRows = 0;
|
|
905
|
-
let firstFailure = null;
|
|
906
|
-
const waits = [];
|
|
907
|
-
|
|
908
|
-
const awaitDelivery = function ( entry ) {
|
|
909
|
-
totalRows += entry.rows;
|
|
910
|
-
waits.push( entry.promise.then(
|
|
911
|
-
function () {
|
|
912
|
-
deliveredRows += entry.rows;
|
|
913
|
-
},
|
|
914
|
-
function ( err ) {
|
|
915
|
-
failedRows += entry.rows;
|
|
916
|
-
if ( !firstFailure ) {
|
|
917
|
-
firstFailure = err;
|
|
918
|
-
}
|
|
919
|
-
}
|
|
920
|
-
) );
|
|
921
|
-
}; // awaitDelivery()
|
|
922
|
-
|
|
923
|
-
inFlightFlushes.forEach( awaitDelivery );
|
|
924
|
-
if ( bufferedRows > 0 ) {
|
|
925
|
-
const entry = trackFlush( sender.flush(), bufferedRows );
|
|
926
|
-
bufferedRows = 0;
|
|
927
|
-
awaitDelivery( entry );
|
|
928
|
-
}
|
|
929
|
-
|
|
930
|
-
if ( waits.length > 0 ) {
|
|
931
|
-
try {
|
|
932
|
-
await raceFlushTimeout( Promise.all( waits ), timeout );
|
|
933
|
-
} catch ( err ) {
|
|
934
|
-
await closeQuietly();
|
|
935
|
-
const dropped = totalRows - deliveredRows;
|
|
936
|
-
const timedOut = new Error(
|
|
937
|
-
`WinkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
|
|
938
|
-
);
|
|
939
|
-
timedOut.code = 'SHUTDOWN_TIMEOUT';
|
|
940
|
-
timedOut.dropped = { count: dropped };
|
|
941
|
-
throw timedOut;
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
if ( failedRows > 0 ) {
|
|
945
|
-
await closeQuietly();
|
|
946
|
-
const failure = new Error(
|
|
947
|
-
`WinkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
|
|
948
|
-
);
|
|
949
|
-
failure.code = 'DELIVERY_FAILED';
|
|
950
|
-
failure.dropped = { count: failedRows };
|
|
951
|
-
failure.cause = firstFailure;
|
|
952
|
-
throw failure;
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
|
-
|
|
956
|
-
// Close sender
|
|
957
|
-
await sender.close();
|
|
958
|
-
}; // doShutdown()
|
|
959
|
-
|
|
960
|
-
/**
|
|
961
|
-
* Shutdown the storage adapter gracefully.
|
|
962
|
-
* Flushes pending data and closes connections.
|
|
963
|
-
*
|
|
964
|
-
* Called by wire-storages.shutdown() which is invoked during:
|
|
965
|
-
* - Pipeline shutdown (flowHandle.shutdown())
|
|
966
|
-
* - Process signal handlers (SIGINT, SIGTERM)
|
|
967
|
-
*
|
|
968
|
-
* The outcome is latched: the first call runs the shutdown, every
|
|
969
|
-
* later call returns the same promise. A lossy shutdown's failed
|
|
970
|
-
* flush already emptied the buffer (copy-out), so a re-run would
|
|
971
|
-
* find nothing to flush and resolve clean, contradicting the
|
|
972
|
-
* recorded loss. One consequence: the first caller's `{ timeout }`
|
|
973
|
-
* governs; a later caller's is ignored.
|
|
974
|
-
*
|
|
975
|
-
* @param {{timeout?: number}} [options]
|
|
976
|
-
* @returns {Promise<void>}
|
|
977
|
-
*/
|
|
978
|
-
const shutdown = function ( { timeout = 0 } = {} ) {
|
|
979
|
-
if ( !shutdownPromise ) {
|
|
980
|
-
shutdownPromise = doShutdown( timeout );
|
|
981
|
-
}
|
|
982
|
-
return shutdownPromise;
|
|
983
|
-
}; // shutdown()
|
|
679
|
+
// Everything after setup is the engine's: the hot path, the
|
|
680
|
+
// flushes, the counters, the drain. The transport close stays the
|
|
681
|
+
// factory's, because the factory owns the agent.
|
|
682
|
+
const engine = createFlushEngine( { sender, persistPlans, settings, onDeliveryFailure, probe, closeTransport } );
|
|
984
683
|
|
|
985
684
|
return {
|
|
986
|
-
write,
|
|
987
|
-
flush,
|
|
988
|
-
shutdown,
|
|
989
|
-
getPressure,
|
|
990
|
-
getHealth,
|
|
685
|
+
write: engine.write,
|
|
686
|
+
flush: engine.flush,
|
|
687
|
+
shutdown: engine.shutdown,
|
|
688
|
+
getPressure: engine.getPressure,
|
|
689
|
+
getHealth: engine.getHealth,
|
|
991
690
|
// Expose for testing/debugging
|
|
992
691
|
_sender: sender,
|
|
993
692
|
_persistPlans: persistPlans
|
|
994
693
|
};
|
|
995
|
-
};
|
|
694
|
+
}; // createQuestDBStorage()
|
|
996
695
|
|
|
997
696
|
// ============================================================================
|
|
998
697
|
// CONFIG SCHEMA
|
|
@@ -1022,65 +721,116 @@ const configSchema = {
|
|
|
1022
721
|
'idleFlushCheckMs',
|
|
1023
722
|
'autoFlushRows',
|
|
1024
723
|
'autoFlushIntervalMs',
|
|
1025
|
-
'
|
|
724
|
+
'flushRows',
|
|
725
|
+
'flushIntervalMs',
|
|
726
|
+
'bufferCeilingRows',
|
|
727
|
+
'flushDeadlineMs',
|
|
728
|
+
'stdlibHttp',
|
|
729
|
+
'requestTimeout',
|
|
1026
730
|
'retryTimeout',
|
|
731
|
+
'initBufSize',
|
|
732
|
+
'maxBufSize',
|
|
1027
733
|
'partitionBy',
|
|
1028
734
|
'onWarning',
|
|
1029
735
|
'onDeliveryFailure'
|
|
1030
736
|
],
|
|
737
|
+
// Both addresses refuse `localhost` at flow definition (ADR-030).
|
|
738
|
+
// The non-empty rule lives inside the validator, so one static
|
|
739
|
+
// error string covers every refusal.
|
|
1031
740
|
ilpUrl: {
|
|
1032
741
|
type: 'string',
|
|
1033
742
|
required: false,
|
|
1034
|
-
|
|
1035
|
-
error: 'ilpUrl must be a
|
|
743
|
+
validator: isAllowedIlpUrl,
|
|
744
|
+
error: 'ilpUrl must be host:port with a literal address or a name, never localhost and never ' +
|
|
745
|
+
'an IPv6 literal (the QuestDB client cannot read one); e.g., 127.0.0.1:9000'
|
|
1036
746
|
},
|
|
1037
747
|
pgUrl: {
|
|
1038
748
|
type: 'string',
|
|
1039
749
|
required: false,
|
|
1040
|
-
|
|
1041
|
-
error: 'pgUrl must be a
|
|
750
|
+
validator: isAllowedPgUrl,
|
|
751
|
+
error: 'pgUrl must be host:port with a literal address or a name, never localhost; e.g., 127.0.0.1:8812'
|
|
1042
752
|
},
|
|
1043
753
|
tablePrefix: {
|
|
1044
754
|
type: 'string',
|
|
1045
755
|
required: false,
|
|
1046
756
|
minLength: 1,
|
|
1047
|
-
|
|
757
|
+
// The prefix opens every table name, written unquoted in CREATE
|
|
758
|
+
// TABLE. See header sub-case (j) for the rule and its sources.
|
|
759
|
+
validator: validators.identifier,
|
|
760
|
+
error: 'tablePrefix must use letters, digits, _ and $ only, and not start with a digit ' +
|
|
761
|
+
'(defaults to assetClass.name when omitted)'
|
|
1048
762
|
},
|
|
763
|
+
// The five legacy keys (deprecated, ADR-029; removed in 0.8.0). They
|
|
764
|
+
// keep their old validation so an existing flow still passes the
|
|
765
|
+
// schema. The option resolver maps or ignores them and setup prints
|
|
766
|
+
// one DEPRECATED_OPTION line.
|
|
1049
767
|
flushMode: {
|
|
1050
768
|
type: 'string',
|
|
1051
769
|
required: false,
|
|
1052
770
|
validator: validators.oneOf( [ 'auto', 'manual' ] ),
|
|
1053
|
-
error: 'flushMode must be "auto" or "manual"'
|
|
771
|
+
error: 'flushMode must be "auto" or "manual" (deprecated: composer owns every flush)'
|
|
1054
772
|
},
|
|
1055
773
|
idleFlushAfterMs: {
|
|
1056
774
|
type: 'number',
|
|
1057
775
|
required: false,
|
|
1058
776
|
validator: validators.positiveInteger,
|
|
1059
|
-
error: 'idleFlushAfterMs must be a positive integer'
|
|
777
|
+
error: 'idleFlushAfterMs must be a positive integer (deprecated: ignored)'
|
|
1060
778
|
},
|
|
1061
779
|
idleFlushCheckMs: {
|
|
1062
780
|
type: 'number',
|
|
1063
781
|
required: false,
|
|
1064
782
|
validator: validators.positiveInteger,
|
|
1065
|
-
error: 'idleFlushCheckMs must be a positive integer'
|
|
783
|
+
error: 'idleFlushCheckMs must be a positive integer (deprecated: use flushIntervalMs)'
|
|
1066
784
|
},
|
|
1067
785
|
autoFlushRows: {
|
|
1068
786
|
type: 'number',
|
|
1069
787
|
required: false,
|
|
1070
788
|
validator: validators.positiveInteger,
|
|
1071
|
-
error: 'autoFlushRows must be a positive integer'
|
|
789
|
+
error: 'autoFlushRows must be a positive integer (deprecated: use flushRows)'
|
|
1072
790
|
},
|
|
1073
791
|
autoFlushIntervalMs: {
|
|
1074
792
|
type: 'number',
|
|
1075
793
|
required: false,
|
|
1076
794
|
validator: validators.positiveInteger,
|
|
1077
|
-
error: 'autoFlushIntervalMs must be a positive integer'
|
|
795
|
+
error: 'autoFlushIntervalMs must be a positive integer (deprecated: ignored)'
|
|
1078
796
|
},
|
|
1079
|
-
|
|
797
|
+
// The flush settings composer owns (ADR-029). The relation between
|
|
798
|
+
// the ceiling and the threshold is checked by the option resolver
|
|
799
|
+
// at setup, where both values are known.
|
|
800
|
+
flushRows: {
|
|
1080
801
|
type: 'number',
|
|
1081
802
|
required: false,
|
|
1082
803
|
validator: validators.positiveInteger,
|
|
1083
|
-
error: '
|
|
804
|
+
error: 'flushRows must be a positive integer'
|
|
805
|
+
},
|
|
806
|
+
flushIntervalMs: {
|
|
807
|
+
type: 'number',
|
|
808
|
+
required: false,
|
|
809
|
+
validator: validators.positiveInteger,
|
|
810
|
+
error: 'flushIntervalMs must be a positive integer'
|
|
811
|
+
},
|
|
812
|
+
bufferCeilingRows: {
|
|
813
|
+
type: 'number',
|
|
814
|
+
required: false,
|
|
815
|
+
validator: validators.positiveInteger,
|
|
816
|
+
error: 'bufferCeilingRows must be a positive integer'
|
|
817
|
+
},
|
|
818
|
+
flushDeadlineMs: {
|
|
819
|
+
type: 'number',
|
|
820
|
+
required: false,
|
|
821
|
+
validator: validators.positiveInteger,
|
|
822
|
+
error: 'flushDeadlineMs must be a positive integer'
|
|
823
|
+
},
|
|
824
|
+
stdlibHttp: {
|
|
825
|
+
type: 'boolean',
|
|
826
|
+
required: false,
|
|
827
|
+
error: 'stdlibHttp must be a boolean'
|
|
828
|
+
},
|
|
829
|
+
requestTimeout: {
|
|
830
|
+
type: 'number',
|
|
831
|
+
required: false,
|
|
832
|
+
validator: validators.positiveInteger,
|
|
833
|
+
error: 'requestTimeout must be a positive integer'
|
|
1084
834
|
},
|
|
1085
835
|
retryTimeout: {
|
|
1086
836
|
type: 'number',
|
|
@@ -1088,6 +838,18 @@ const configSchema = {
|
|
|
1088
838
|
validator: validators.positiveInteger,
|
|
1089
839
|
error: 'retryTimeout must be a positive integer'
|
|
1090
840
|
},
|
|
841
|
+
initBufSize: {
|
|
842
|
+
type: 'number',
|
|
843
|
+
required: false,
|
|
844
|
+
validator: validators.positiveInteger,
|
|
845
|
+
error: 'initBufSize must be a positive integer'
|
|
846
|
+
},
|
|
847
|
+
maxBufSize: {
|
|
848
|
+
type: 'number',
|
|
849
|
+
required: false,
|
|
850
|
+
validator: validators.positiveInteger,
|
|
851
|
+
error: 'maxBufSize must be a positive integer'
|
|
852
|
+
},
|
|
1091
853
|
partitionBy: {
|
|
1092
854
|
type: 'string',
|
|
1093
855
|
required: false,
|
|
@@ -1128,23 +890,24 @@ const durabilityClass = 'in-memory';
|
|
|
1128
890
|
|
|
1129
891
|
// Capability declaration per ADR-018.
|
|
1130
892
|
//
|
|
1131
|
-
// QuestDB needs the asset class
|
|
1132
|
-
// `columns` and `insightTypes` to issue CREATE TABLE statements
|
|
1133
|
-
//
|
|
893
|
+
// QuestDB needs the asset class for two reasons. `ensure-tables.js`
|
|
894
|
+
// reads `columns` and `insightTypes` to issue CREATE TABLE statements.
|
|
895
|
+
// `persist-plan.js` reads the same to dispatch column writers and
|
|
1134
896
|
// pre-compile resolution-aware quantization. `name` drives the default
|
|
1135
897
|
// `tablePrefix` when the caller did not pass one.
|
|
1136
898
|
//
|
|
1137
899
|
// Declared `required: true` because there is no useful behaviour for
|
|
1138
|
-
// QuestDB without an assetClass
|
|
1139
|
-
// MISSING_ASSET_CLASS
|
|
900
|
+
// QuestDB without an assetClass. The wiring layer fails fast with
|
|
901
|
+
// MISSING_ASSET_CLASS instead of calling `createStorage` and letting
|
|
1140
902
|
// the failure surface deeper.
|
|
1141
903
|
//
|
|
1142
904
|
// `fields` lists exactly the top-level fields read. The slicing is
|
|
1143
|
-
// top-level only
|
|
905
|
+
// top-level only. Column-internal fields (`type`, `resolution`) are
|
|
1144
906
|
// read directly from the `columns` slice and validated by Layer 2
|
|
1145
|
-
// assertions inside `createStorage` below.
|
|
1146
|
-
//
|
|
1147
|
-
//
|
|
907
|
+
// assertions inside `createStorage` below.
|
|
908
|
+
//
|
|
909
|
+
// A declarative shape for column-internal capability is deferred
|
|
910
|
+
// until a second adapter needs it.
|
|
1148
911
|
const semanticsRequirement = {
|
|
1149
912
|
assetClass: {
|
|
1150
913
|
required: true,
|
|
@@ -1158,17 +921,20 @@ const semanticsRequirement = {
|
|
|
1158
921
|
* @param {Object} config - Configuration object
|
|
1159
922
|
* @param {Object} config.assetClass - Asset class definition with columns and insightTypes
|
|
1160
923
|
* @param {string} config.tablePrefix - Prefix for table names (defaults to assetClass.name)
|
|
1161
|
-
* @param {string} config.ilpUrl - ILP endpoint (host:port)
|
|
1162
|
-
* @param {string} config.pgUrl - PostgreSQL endpoint (host:port)
|
|
1163
|
-
* @param {
|
|
1164
|
-
* @param {number} [config.
|
|
1165
|
-
* @param {number} [config.
|
|
1166
|
-
* @param {number} [config.
|
|
1167
|
-
* @param {
|
|
1168
|
-
* @param {number} [config.
|
|
1169
|
-
* @param {number} [config.retryTimeout] -
|
|
924
|
+
* @param {string} [config.ilpUrl] - ILP endpoint (host:port)
|
|
925
|
+
* @param {string} [config.pgUrl] - PostgreSQL endpoint (host:port)
|
|
926
|
+
* @param {number} [config.flushRows=5000] - Rows that start a flush from inside write()
|
|
927
|
+
* @param {number} [config.flushIntervalMs=1000] - Period of the flush timer
|
|
928
|
+
* @param {number} [config.bufferCeilingRows] - Most rows held; default ten times flushRows
|
|
929
|
+
* @param {number} [config.flushDeadlineMs] - Fixed deadline per flush
|
|
930
|
+
* @param {boolean} [config.stdlibHttp=true] - The standard-library transport; false selects undici
|
|
931
|
+
* @param {number} [config.requestTimeout] - Client request timeout in ms
|
|
932
|
+
* @param {number} [config.retryTimeout] - Client retry window in ms
|
|
933
|
+
* @param {number} [config.initBufSize] - Initial client buffer size in bytes
|
|
934
|
+
* @param {number} [config.maxBufSize] - Byte ceiling of the client buffer
|
|
1170
935
|
* @param {string} [config.partitionBy='DAY'] - Table partition interval
|
|
1171
|
-
* @param {function} [config.onWarning] - Warning callback for
|
|
936
|
+
* @param {function} [config.onWarning] - Warning callback for skipped values
|
|
937
|
+
* @param {function} [config.onDeliveryFailure] - Called once per lost flush
|
|
1172
938
|
* @param {Object} [config._deps] - Injectable dependencies (for testing)
|
|
1173
939
|
* @returns {Promise<Object>} Storage instance with write, flush, shutdown methods
|
|
1174
940
|
*/
|
|
@@ -1183,22 +949,21 @@ const createStorage = function ( config ) {
|
|
|
1183
949
|
// remediation differs — they need to add .assetClass() to the
|
|
1184
950
|
// flow definition, not edit env vars or storage config.
|
|
1185
951
|
const err = new Error(
|
|
1186
|
-
'
|
|
952
|
+
'winkComposer/questdb: assetClass is required - add .assetClass(assetClassDef) to flow before .storage()'
|
|
1187
953
|
);
|
|
1188
954
|
err.code = 'MISSING_ASSET_CLASS';
|
|
1189
955
|
throw err;
|
|
1190
956
|
}
|
|
1191
957
|
|
|
1192
958
|
// Adapter-side defensive validation of column-internal facts
|
|
1193
|
-
// QuestDB consumes
|
|
1194
|
-
//
|
|
1195
|
-
//
|
|
1196
|
-
//
|
|
1197
|
-
//
|
|
1198
|
-
//
|
|
1199
|
-
//
|
|
1200
|
-
//
|
|
1201
|
-
// full rationale.
|
|
959
|
+
// QuestDB consumes (the ADR-018 column-internal facts pattern). The
|
|
960
|
+
// universal semantics schema validates the top-level structure at
|
|
961
|
+
// load and at .assetClass() time: type required, type in
|
|
962
|
+
// COLUMN_TYPES. This layer catches bypasses, such as a direct
|
|
963
|
+
// createStorage call from test code. It also checks adapter-specific
|
|
964
|
+
// requirements, such as "a float64 column with a declared resolution
|
|
965
|
+
// must have a positive value". See `assert-columns.js` for the full
|
|
966
|
+
// rationale.
|
|
1202
967
|
assertColumnFacts( assetClass );
|
|
1203
968
|
|
|
1204
969
|
// Default `tablePrefix` to the asset class name when the caller
|
|
@@ -1226,8 +991,8 @@ const createStorage = function ( config ) {
|
|
|
1226
991
|
* .assetClass( assetClassDef ) // wire-storages injects the slice
|
|
1227
992
|
* .storage( questdbAdapter, {
|
|
1228
993
|
* tablePrefix: 'myPrefix',
|
|
1229
|
-
* ilpUrl: '
|
|
1230
|
-
* pgUrl: '
|
|
994
|
+
* ilpUrl: '127.0.0.1:9000',
|
|
995
|
+
* pgUrl: '127.0.0.1:8812'
|
|
1231
996
|
* } )
|
|
1232
997
|
*
|
|
1233
998
|
* @type {Object}
|