@winkjs/composer 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/emitter.js +44 -14
- package/src/core/emitter-manager/mqtt/index.js +2 -1
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +4 -4
- package/src/core/env-vars.js +44 -3
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +95 -36
- package/src/core/source-manager/mqtt/dedup.js +3 -3
- package/src/core/source-manager/mqtt/index.js +1 -1
- package/src/core/source-manager/mqtt/status.js +67 -14
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/index.js +55 -17
- package/src/core/storage-manager/questdb/persist-plan.js +32 -8
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/callback-guard/index.js +155 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +4 -3
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +4 -3
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +41 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +176 -38
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/emitter-manager/mqtt/mqtt-store.js +0 -538
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
|
@@ -25,6 +25,10 @@
|
|
|
25
25
|
* dedupHits, dedupMisses, dedupBypassed, dedupCacheSize}`, emitted
|
|
26
26
|
* on every tick() (the client's 1 Hz cadence) and on every health
|
|
27
27
|
* transition. Consumers diff snapshots; nothing is ever reset.
|
|
28
|
+
* `delivered` means handed to the flow: a message the pipeline
|
|
29
|
+
* later failed on still counts here. Failure visibility belongs
|
|
30
|
+
* to the flow's MESSAGE_HANDLER_FAILED reports (ADR-018), not to
|
|
31
|
+
* this counter.
|
|
28
32
|
*
|
|
29
33
|
* Health rules (evaluated on events and on tick):
|
|
30
34
|
* - RED — a subscribe failure (connected but deaf — no retry
|
|
@@ -84,6 +88,8 @@ import {
|
|
|
84
88
|
DISCONNECT_RED_MS,
|
|
85
89
|
DECODE_RING_SIZE
|
|
86
90
|
} from './constants.js';
|
|
91
|
+
import { wrapCallback } from '../../utils/callback-guard/index.js';
|
|
92
|
+
import { logger } from '../../logger/index.js';
|
|
87
93
|
|
|
88
94
|
// ============================================================================
|
|
89
95
|
// VALIDATION HELPERS
|
|
@@ -98,7 +104,7 @@ import {
|
|
|
98
104
|
*/
|
|
99
105
|
const assertOptionalFunction = function ( value, name ) {
|
|
100
106
|
if ( value !== undefined && value !== null && typeof value !== 'function' ) {
|
|
101
|
-
const err = new Error( `
|
|
107
|
+
const err = new Error( `winkComposer/mqttSource: ${name} must be a function` );
|
|
102
108
|
err.code = 'INVALID_CONFIG';
|
|
103
109
|
throw err;
|
|
104
110
|
}
|
|
@@ -124,7 +130,7 @@ const assertOptionalFunction = function ( value, name ) {
|
|
|
124
130
|
*/
|
|
125
131
|
const createStatusReporter = function ( options = {} ) {
|
|
126
132
|
if ( typeof options !== 'object' || options === null ) {
|
|
127
|
-
const err = new Error( '
|
|
133
|
+
const err = new Error( 'winkComposer/mqttSource: status reporter options must be an object' );
|
|
128
134
|
err.code = 'INVALID_CONFIG';
|
|
129
135
|
throw err;
|
|
130
136
|
}
|
|
@@ -141,7 +147,7 @@ const createStatusReporter = function ( options = {} ) {
|
|
|
141
147
|
assertOptionalFunction( onMetrics, 'onMetrics' );
|
|
142
148
|
assertOptionalFunction( dedupSizeFn, 'dedupSizeFn' );
|
|
143
149
|
if ( typeof nowFn !== 'function' ) {
|
|
144
|
-
const err = new Error( '
|
|
150
|
+
const err = new Error( 'winkComposer/mqttSource: nowFn must be a function' );
|
|
145
151
|
err.code = 'INVALID_CONFIG';
|
|
146
152
|
throw err;
|
|
147
153
|
}
|
|
@@ -149,7 +155,7 @@ const createStatusReporter = function ( options = {} ) {
|
|
|
149
155
|
( typeof expectedQuietPeriodMs !== 'number' ||
|
|
150
156
|
expectedQuietPeriodMs < 1 ||
|
|
151
157
|
!Number.isInteger( expectedQuietPeriodMs ) ) ) {
|
|
152
|
-
const err = new Error( '
|
|
158
|
+
const err = new Error( 'winkComposer/mqttSource: expectedQuietPeriodMs must be a positive integer' );
|
|
153
159
|
err.code = 'INVALID_CONFIG';
|
|
154
160
|
throw err;
|
|
155
161
|
}
|
|
@@ -220,14 +226,20 @@ const createStatusReporter = function ( options = {} ) {
|
|
|
220
226
|
};
|
|
221
227
|
};
|
|
222
228
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
+
// Both user callbacks are armed by the shared callback guard.
|
|
230
|
+
// They were validated raw above and are wrapped here, once
|
|
231
|
+
// (ADR-018: a misbehaving user callback never reaches transport
|
|
232
|
+
// code and never fails silently). A broken onStatus reports to
|
|
233
|
+
// the console in this adapter's classified line family. It
|
|
234
|
+
// cannot report through itself. A broken onMetrics reports on
|
|
235
|
+
// both channels; the comment at its wrap below explains why.
|
|
236
|
+
const safeOnStatus = wrapCallback( onStatus, {
|
|
237
|
+
name: 'onStatus',
|
|
238
|
+
severity: 'red',
|
|
239
|
+
report: function ( severity, name, detail ) {
|
|
240
|
+
logger.error( `winkComposer/mqttSource: user callback ${name} failed [CALLBACK_FAILED]: ${detail}` );
|
|
229
241
|
}
|
|
230
|
-
};
|
|
242
|
+
} );
|
|
231
243
|
|
|
232
244
|
/**
|
|
233
245
|
* Route a status payload: to the caller's handler when supplied;
|
|
@@ -238,10 +250,51 @@ const createStatusReporter = function ( options = {} ) {
|
|
|
238
250
|
* @param {Object} payload - The structured status payload
|
|
239
251
|
*/
|
|
240
252
|
const emitStatus = function ( payload ) {
|
|
241
|
-
if (
|
|
242
|
-
|
|
253
|
+
if ( safeOnStatus ) {
|
|
254
|
+
safeOnStatus( payload );
|
|
243
255
|
} else if ( payload.error ) {
|
|
244
|
-
|
|
256
|
+
logger.error( `winkComposer/mqttSource: source error [${payload.error.code}]: ${payload.error.message}` );
|
|
257
|
+
}
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
// The metrics fault report allocates on the failure path only.
|
|
261
|
+
// `connected` and `phase` read the reporter's live state at fault
|
|
262
|
+
// time, so the payload tells the truth about the moment it fired.
|
|
263
|
+
//
|
|
264
|
+
// The console line comes first and always fires. It is the
|
|
265
|
+
// guaranteed audience: inside a flow the runtime installs its own
|
|
266
|
+
// onStatus wrapper, which forwards only red payloads when the
|
|
267
|
+
// user gave no handler, so the yellow payload alone could vanish
|
|
268
|
+
// (fresh-eyes find, 2026-08-28). The payload then additionally
|
|
269
|
+
// reaches a listening handler, so a health view sees the
|
|
270
|
+
// degradation. With no handler at all, emitStatus would print
|
|
271
|
+
// the same line a second time — skip the payload there.
|
|
272
|
+
const safeOnMetrics = wrapCallback( onMetrics, {
|
|
273
|
+
name: 'onMetrics',
|
|
274
|
+
severity: 'yellow',
|
|
275
|
+
report: function ( severity, name, detail ) {
|
|
276
|
+
const message = `user callback ${name} failed: ${detail}`;
|
|
277
|
+
logger.error( `winkComposer/mqttSource: user callback ${name} failed [CALLBACK_FAILED]: ${detail}` );
|
|
278
|
+
if ( safeOnStatus ) {
|
|
279
|
+
emitStatus( {
|
|
280
|
+
status: severity,
|
|
281
|
+
connected,
|
|
282
|
+
phase,
|
|
283
|
+
error: {
|
|
284
|
+
code: 'CALLBACK_FAILED',
|
|
285
|
+
message
|
|
286
|
+
}
|
|
287
|
+
} );
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
} );
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Emit a metrics snapshot when the caller asked for one.
|
|
294
|
+
*/
|
|
295
|
+
const emitMetrics = function () {
|
|
296
|
+
if ( safeOnMetrics ) {
|
|
297
|
+
safeOnMetrics( snapshot() );
|
|
245
298
|
}
|
|
246
299
|
};
|
|
247
300
|
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// core/source-manager/record-shape.js
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @fileoverview One definition of "usable record" for every source.
|
|
5
|
+
*
|
|
6
|
+
* A source hands the pipeline record objects. A decoded payload or a
|
|
7
|
+
* transform return can instead be a scalar, null, undefined, or a
|
|
8
|
+
* bare array. All of those are valid JSON documents, so decoding
|
|
9
|
+
* alone cannot reject them. Attaching metadata to them throws in
|
|
10
|
+
* strict mode. This module gives every source the same two tools:
|
|
11
|
+
* the guard predicate, and the plain name of the rejected shape for
|
|
12
|
+
* the per-record report (skip-classify-continue, ADR-018).
|
|
13
|
+
*
|
|
14
|
+
* Kept as one shared module so the MQTT and CSV sources cannot
|
|
15
|
+
* drift apart on what a usable record is (the cross-source
|
|
16
|
+
* transform contract).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Tell whether a decoded value can carry pipeline fields: a real
|
|
21
|
+
* object, not null, not an array. Hot-path cost is one comparison,
|
|
22
|
+
* one typeof, and one Array.isArray — no allocation.
|
|
23
|
+
*
|
|
24
|
+
* @param {*} value - Decoded payload or transform return
|
|
25
|
+
* @returns {boolean} True when the value is a usable record
|
|
26
|
+
*/
|
|
27
|
+
const isUsableRecord = function ( value ) {
|
|
28
|
+
return ( value !== null ) && ( typeof value === 'object' ) && !Array.isArray( value );
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Name a rejected value's shape for an operator-facing report.
|
|
33
|
+
* Called on the failure path only, so the string cost is fine.
|
|
34
|
+
*
|
|
35
|
+
* @param {*} value - The rejected value
|
|
36
|
+
* @returns {string} Plain shape name, e.g. 'a number', 'an array'
|
|
37
|
+
*/
|
|
38
|
+
const describeShape = function ( value ) {
|
|
39
|
+
if ( value === null ) {
|
|
40
|
+
return 'null';
|
|
41
|
+
}
|
|
42
|
+
if ( value === undefined ) {
|
|
43
|
+
return 'undefined';
|
|
44
|
+
}
|
|
45
|
+
if ( Array.isArray( value ) ) {
|
|
46
|
+
return 'an array';
|
|
47
|
+
}
|
|
48
|
+
return `a ${typeof value}`;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
// ============================================================================
|
|
52
|
+
// EXPORTS
|
|
53
|
+
// ============================================================================
|
|
54
|
+
|
|
55
|
+
export { isUsableRecord, describeShape };
|
|
@@ -80,6 +80,6 @@ export const generateField = function ( fieldSpec, prng, msgIndex, intervalMs )
|
|
|
80
80
|
default:
|
|
81
81
|
// Unreachable in practice — validate.js rejects unknown
|
|
82
82
|
// types at startup. Kept for clarity at the boundary.
|
|
83
|
-
throw new Error( `testHarness: unknown field type "${fieldSpec.type}"` );
|
|
83
|
+
throw new Error( `winkComposer/testHarness: unknown field type "${fieldSpec.type}"` );
|
|
84
84
|
}
|
|
85
85
|
};
|
|
@@ -37,7 +37,20 @@
|
|
|
37
37
|
* class. Thrown synchronously from start().
|
|
38
38
|
* - `GENERATOR_ERROR` — runtime; a downstream `onMessage` call
|
|
39
39
|
* threw inside the loop. Routed through
|
|
40
|
-
* onStatus, not re-thrown.
|
|
40
|
+
* onStatus, not re-thrown. Narrowed by the
|
|
41
|
+
* flow's dispatch guard (ADR-018): in a
|
|
42
|
+
* flow, a pipeline fault is contained at the
|
|
43
|
+
* flow's own chokepoint and reported as
|
|
44
|
+
* MESSAGE_HANDLER_FAILED, so it never
|
|
45
|
+
* reaches this loop. The code remains live
|
|
46
|
+
* for direct callers of start() whose own
|
|
47
|
+
* onMessage throws.
|
|
48
|
+
* - `CALLBACK_FAILED` — runtime; the user's `onStatus` itself threw
|
|
49
|
+
* or rejected. The shared callback guard
|
|
50
|
+
* contains the fault (ADR-018): generation
|
|
51
|
+
* continues and each fault becomes one
|
|
52
|
+
* classified console line. Fix the onStatus
|
|
53
|
+
* handler; the line carries the fault detail.
|
|
41
54
|
*
|
|
42
55
|
* Design decisions date from 2026-04-29. The `messageTemplate` shape
|
|
43
56
|
* is enforced at startup by `validate.js`; the per-field spec shape
|
|
@@ -48,6 +61,17 @@ import { createPrng } from './prng.js';
|
|
|
48
61
|
import { generateField } from './field-generator.js';
|
|
49
62
|
import { applyFuzz, FUZZ_PATTERN_NAMES } from './fuzz.js';
|
|
50
63
|
import { validateMessageTemplate, validateAssetClass } from './validate.js';
|
|
64
|
+
import { wrapCallback } from '../../utils/callback-guard/index.js';
|
|
65
|
+
import { logger } from '../../logger/index.js';
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Console channel for the callback guard: one classified line in this
|
|
69
|
+
* source's family. Receives an already-safe detail string, never the
|
|
70
|
+
* raw thrown value.
|
|
71
|
+
*/
|
|
72
|
+
const reportCallbackFault = function ( severity, name, detail ) {
|
|
73
|
+
logger.error( `winkComposer/testHarness: user callback ${name} failed [CALLBACK_FAILED]: ${detail}` );
|
|
74
|
+
}; // reportCallbackFault()
|
|
51
75
|
|
|
52
76
|
const DEFAULT_MESSAGE_COUNT = 1000;
|
|
53
77
|
const DEFAULT_INTERVAL_MS = 0;
|
|
@@ -83,7 +107,16 @@ export const start = function ( config ) {
|
|
|
83
107
|
validateAssetClass( assetClass );
|
|
84
108
|
|
|
85
109
|
if ( typeof onMessage !== 'function' ) {
|
|
86
|
-
const err = new Error( 'testHarness: onMessage must be a function' );
|
|
110
|
+
const err = new Error( 'winkComposer/testHarness: onMessage must be a function' );
|
|
111
|
+
err.code = 'INVALID_CONFIG';
|
|
112
|
+
throw err;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// The guard below turns a non-function into null (absent). So a
|
|
116
|
+
// misconfigured handler must be rejected here, loudly, before the
|
|
117
|
+
// wrap can silently erase it. Null stays legal as "no handler".
|
|
118
|
+
if ( onStatus !== null && typeof onStatus !== 'function' ) {
|
|
119
|
+
const err = new Error( 'winkComposer/testHarness: onStatus must be a function' );
|
|
87
120
|
err.code = 'INVALID_CONFIG';
|
|
88
121
|
throw err;
|
|
89
122
|
}
|
|
@@ -108,7 +141,16 @@ export const start = function ( config ) {
|
|
|
108
141
|
resolveFinished = resolve;
|
|
109
142
|
} );
|
|
110
143
|
|
|
111
|
-
|
|
144
|
+
// The user's onStatus is armed once by the shared callback guard
|
|
145
|
+
// (ADR-018). A throw or rejection inside it becomes one classified
|
|
146
|
+
// CALLBACK_FAILED console line, and generation continues. Absent
|
|
147
|
+
// stays null, so the no-handler console fallback below keeps its
|
|
148
|
+
// exact meaning.
|
|
149
|
+
const safeOnStatus = wrapCallback( onStatus, {
|
|
150
|
+
name: 'onStatus', severity: 'red', report: reportCallbackFault
|
|
151
|
+
} );
|
|
152
|
+
|
|
153
|
+
if ( safeOnStatus ) safeOnStatus( {
|
|
112
154
|
status: 'green',
|
|
113
155
|
connected: true,
|
|
114
156
|
phase: 'starting',
|
|
@@ -117,7 +159,7 @@ export const start = function ( config ) {
|
|
|
117
159
|
} );
|
|
118
160
|
|
|
119
161
|
const run = async function () {
|
|
120
|
-
if (
|
|
162
|
+
if ( safeOnStatus ) safeOnStatus( {
|
|
121
163
|
status: 'green',
|
|
122
164
|
connected: true,
|
|
123
165
|
phase: 'generating'
|
|
@@ -152,7 +194,7 @@ export const start = function ( config ) {
|
|
|
152
194
|
|
|
153
195
|
// Completion travels onStatus with the uniform `count` field
|
|
154
196
|
// (per ADR-018 there is no onComplete callback).
|
|
155
|
-
if (
|
|
197
|
+
if ( safeOnStatus ) safeOnStatus( {
|
|
156
198
|
status: 'green',
|
|
157
199
|
connected: false,
|
|
158
200
|
phase: 'complete',
|
|
@@ -171,17 +213,17 @@ export const start = function ( config ) {
|
|
|
171
213
|
// wait on it.
|
|
172
214
|
run().catch( ( err ) => {
|
|
173
215
|
const message = ( err && err.message ) ? err.message : String( err );
|
|
174
|
-
if (
|
|
216
|
+
if ( safeOnStatus ) {
|
|
175
217
|
// Terminal red: generation stopped. Uniform payload with
|
|
176
218
|
// phase 'errored' per the ADR-018 two-tier rule.
|
|
177
|
-
|
|
219
|
+
safeOnStatus( {
|
|
178
220
|
status: 'red',
|
|
179
221
|
connected: false,
|
|
180
222
|
phase: 'errored',
|
|
181
223
|
error: { code: 'GENERATOR_ERROR', message }
|
|
182
224
|
} );
|
|
183
225
|
} else {
|
|
184
|
-
|
|
226
|
+
logger.error( `winkComposer/testHarness: generator failed [GENERATOR_ERROR]: ${message}` );
|
|
185
227
|
}
|
|
186
228
|
} ).finally( () => {
|
|
187
229
|
resolveFinished();
|
|
@@ -215,8 +257,8 @@ export const start = function ( config ) {
|
|
|
215
257
|
resolve();
|
|
216
258
|
};
|
|
217
259
|
forceTimer = setTimeout( () => {
|
|
218
|
-
if (
|
|
219
|
-
|
|
260
|
+
if ( safeOnStatus ) {
|
|
261
|
+
safeOnStatus( {
|
|
220
262
|
status: 'yellow',
|
|
221
263
|
connected: false,
|
|
222
264
|
phase: 'stopped',
|
|
@@ -173,7 +173,7 @@ export const ASSET_CLASS_SCHEMA = {
|
|
|
173
173
|
};
|
|
174
174
|
|
|
175
175
|
const throwInvalidConfig = function ( errors ) {
|
|
176
|
-
const err = new Error( `testHarness: ${errors.join( '; ' )}` );
|
|
176
|
+
const err = new Error( `winkComposer/testHarness: ${errors.join( '; ' )}` );
|
|
177
177
|
err.code = 'INVALID_CONFIG';
|
|
178
178
|
throw err;
|
|
179
179
|
};
|
|
@@ -89,7 +89,7 @@ const assertColumnFacts = function ( assetClass ) {
|
|
|
89
89
|
const supportedTypes = Object.keys( DDL_TYPES ).join( ', ' );
|
|
90
90
|
const actualType = colSpec ? String( colSpec.type ) : 'undefined';
|
|
91
91
|
const err = new Error(
|
|
92
|
-
`
|
|
92
|
+
`winkComposer/questdb: column '${colName}' has invalid or missing type '${actualType}'; ` +
|
|
93
93
|
`expected one of: ${supportedTypes}`
|
|
94
94
|
);
|
|
95
95
|
err.code = 'INVALID_CONFIG';
|
|
@@ -106,7 +106,7 @@ const assertColumnFacts = function ( assetClass ) {
|
|
|
106
106
|
!Number.isFinite( colSpec.resolution ) ||
|
|
107
107
|
colSpec.resolution <= 0 ) {
|
|
108
108
|
const err = new Error(
|
|
109
|
-
`
|
|
109
|
+
`winkComposer/questdb: column '${colName}' has type 'float64' ` +
|
|
110
110
|
`but resolution is '${colSpec.resolution}'; ` +
|
|
111
111
|
'expected positive finite number (or omit for passthrough)'
|
|
112
112
|
);
|
|
@@ -162,7 +162,7 @@ const ensureTables = async function ( pgClient, assetClass, tablePrefix, options
|
|
|
162
162
|
// originates from QuestDB's PG-wire DDL response, not from
|
|
163
163
|
// our pre-validation of caller-supplied config.
|
|
164
164
|
const wrapped = new Error(
|
|
165
|
-
`Failed to create table '${tableName}': ${err.message}`
|
|
165
|
+
`winkComposer/questdb: Failed to create table '${tableName}': ${err.message}`
|
|
166
166
|
);
|
|
167
167
|
wrapped.code = 'SCHEMA_ERROR';
|
|
168
168
|
throw wrapped;
|
|
@@ -110,6 +110,16 @@
|
|
|
110
110
|
* `{ timeout }` (a send against an unreachable server never settles —
|
|
111
111
|
* see the mid-row recovery section below).
|
|
112
112
|
*
|
|
113
|
+
* Runtime console classification:
|
|
114
|
+
* - `CALLBACK_FAILED` — the user's `onDeliveryFailure` itself threw or
|
|
115
|
+
* rejected. The shared callback guard contains the fault (ADR-018).
|
|
116
|
+
* The adapter keeps writing and flushing, and each fault becomes one
|
|
117
|
+
* classified console line carrying the detail. `onWarning` is
|
|
118
|
+
* deliberately NOT guarded. A throwing `onWarning` is strict mode:
|
|
119
|
+
* the throw is the instruction that rejects the row. Wrapping it
|
|
120
|
+
* would erase that contract (the ADR-027 exclusion, pinned by the
|
|
121
|
+
* strict-mode specs).
|
|
122
|
+
*
|
|
113
123
|
* Column-internal facts consumed (the ADR-018 column-internal facts
|
|
114
124
|
* pattern):
|
|
115
125
|
*
|
|
@@ -197,7 +207,9 @@ import { Sender } from '@questdb/nodejs-client';
|
|
|
197
207
|
import pg from 'pg';
|
|
198
208
|
|
|
199
209
|
import { ENV_VARS } from '../../env-vars.js';
|
|
210
|
+
import { logger } from '../../logger/index.js';
|
|
200
211
|
import { validators } from '../../utils/validate/index.js';
|
|
212
|
+
import { wrapCallback } from '../../utils/callback-guard/index.js';
|
|
201
213
|
import { buildPersistPlans } from './persist-plan.js';
|
|
202
214
|
import { ensureTables } from './ensure-tables.js';
|
|
203
215
|
import { assertColumnFacts } from './assert-columns.js';
|
|
@@ -214,6 +226,17 @@ import { assertColumnFacts } from './assert-columns.js';
|
|
|
214
226
|
*/
|
|
215
227
|
const RESULT_OK = { ok: true };
|
|
216
228
|
|
|
229
|
+
/**
|
|
230
|
+
* Console channel for the callback guard: one classified line in this
|
|
231
|
+
* adapter's family. Receives an already-safe detail string, never the
|
|
232
|
+
* raw thrown value.
|
|
233
|
+
*/
|
|
234
|
+
const reportCallbackFault = function ( severity, name, detail ) {
|
|
235
|
+
logger.error(
|
|
236
|
+
`winkComposer/questdb: user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
|
|
237
|
+
);
|
|
238
|
+
}; // reportCallbackFault()
|
|
239
|
+
|
|
217
240
|
// Error results (INVALID_INSIGHT_TYPE, SEND_FAILED) are constructed per-call
|
|
218
241
|
// because each carries dynamic content (the offending insightType name and the
|
|
219
242
|
// underlying err.message respectively). These paths are rare; per-occurrence
|
|
@@ -401,12 +424,12 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
401
424
|
// Runtime validation — required from either DSL config or ENV_VARS.
|
|
402
425
|
// Per ADR-018, setup-time throws carry classified err.code.
|
|
403
426
|
if ( !ilpUrl ) {
|
|
404
|
-
const err = new Error( '
|
|
427
|
+
const err = new Error( 'winkComposer/questdb: ilpUrl required — set in .storage() config or QUESTDB_ILP_URL env var' );
|
|
405
428
|
err.code = 'INVALID_CONFIG';
|
|
406
429
|
throw err;
|
|
407
430
|
}
|
|
408
431
|
if ( !pgUrl ) {
|
|
409
|
-
const err = new Error( '
|
|
432
|
+
const err = new Error( 'winkComposer/questdb: pgUrl required — set in .storage() config or QUESTDB_PG_URL env var' );
|
|
410
433
|
err.code = 'INVALID_CONFIG';
|
|
411
434
|
throw err;
|
|
412
435
|
}
|
|
@@ -417,9 +440,24 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
417
440
|
PgClientClass = pg.Client
|
|
418
441
|
} = deps;
|
|
419
442
|
|
|
420
|
-
// Build persist plans (pre-compiled closures)
|
|
443
|
+
// Build persist plans (pre-compiled closures). The callbacks go in
|
|
444
|
+
// RAW: buildPersistPlans validates them itself and arms its own
|
|
445
|
+
// guarded copy for the at-flush site. Handing it a pre-wrapped
|
|
446
|
+
// function would defeat that validation (the guard turns a
|
|
447
|
+
// non-function into null instead of the fail-fast INVALID_CONFIG).
|
|
421
448
|
const persistPlans = buildPersistPlans( assetClass, tablePrefix, { onWarning, onDeliveryFailure } );
|
|
422
449
|
|
|
450
|
+
// Arm the delivery-failure callback for this module's own report
|
|
451
|
+
// sites: the idle-flush timer and the mid-row recovery flush. Both
|
|
452
|
+
// fire inside promise chains, where a broken handler used to become
|
|
453
|
+
// an unhandled rejection. The guard classifies the fault instead
|
|
454
|
+
// (ADR-018) and its cost stays inside the handler. Absent stays
|
|
455
|
+
// null, so the no-handler console/throw fallbacks keep their
|
|
456
|
+
// meaning.
|
|
457
|
+
const safeOnDeliveryFailure = wrapCallback( onDeliveryFailure, {
|
|
458
|
+
name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
|
|
459
|
+
} );
|
|
460
|
+
|
|
423
461
|
// Ensure tables exist via PostgreSQL wire protocol
|
|
424
462
|
const [ pgHost, pgPort ] = pgUrl.split( ':' );
|
|
425
463
|
const pgClient = new PgClientClass( {
|
|
@@ -445,7 +483,7 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
445
483
|
await pgClient.connect();
|
|
446
484
|
} catch ( connErr ) {
|
|
447
485
|
const err = new Error(
|
|
448
|
-
`
|
|
486
|
+
`winkComposer/questdb: could not connect to PostgreSQL at ${pgUrl} — ${connErr.message}`
|
|
449
487
|
);
|
|
450
488
|
err.code = NETWORK_ERROR_CODES.has( connErr.code ) ?
|
|
451
489
|
'TRANSPORT_UNREACHABLE' :
|
|
@@ -570,11 +608,11 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
570
608
|
// (the settle handler already cleared them from
|
|
571
609
|
// pressure). Same routing policy as the recovery flush:
|
|
572
610
|
// the caller owns the response when it asked to.
|
|
573
|
-
if (
|
|
574
|
-
|
|
611
|
+
if ( safeOnDeliveryFailure ) {
|
|
612
|
+
safeOnDeliveryFailure( err, { idleFlush: true, rowsLost: rows } );
|
|
575
613
|
} else {
|
|
576
|
-
|
|
577
|
-
`
|
|
614
|
+
logger.error(
|
|
615
|
+
`winkComposer/questdb: idle flush failed; ${rows} buffered row(s) lost: ${err.message}`
|
|
578
616
|
);
|
|
579
617
|
}
|
|
580
618
|
} finally {
|
|
@@ -629,12 +667,12 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
629
667
|
bufferedRows = 0;
|
|
630
668
|
|
|
631
669
|
entry.promise.catch( ( flushErr ) => {
|
|
632
|
-
if (
|
|
633
|
-
|
|
670
|
+
if ( safeOnDeliveryFailure ) {
|
|
671
|
+
safeOnDeliveryFailure( flushErr, { recovery: true } );
|
|
634
672
|
return;
|
|
635
673
|
}
|
|
636
674
|
const failure = new Error(
|
|
637
|
-
`
|
|
675
|
+
`winkComposer/questdb: recovery flush failed after a mid-row write error: ${flushErr.message}. ` +
|
|
638
676
|
'Completed rows in that batch were dropped. Provide an `onDeliveryFailure` callback ' +
|
|
639
677
|
'in the storage config to handle these explicitly.'
|
|
640
678
|
);
|
|
@@ -649,7 +687,7 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
649
687
|
// write() must still return its classified result rather than
|
|
650
688
|
// throw (ADR-018: the hot path never throws), and the
|
|
651
689
|
// failure must be visible.
|
|
652
|
-
|
|
690
|
+
logger.error( `winkComposer/questdb: sender recovery failed: ${recoveryErr.message}` );
|
|
653
691
|
}
|
|
654
692
|
}; // recoverSender()
|
|
655
693
|
|
|
@@ -682,7 +720,7 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
682
720
|
ok: false,
|
|
683
721
|
error: {
|
|
684
722
|
code: 'SHUTTING_DOWN',
|
|
685
|
-
message: '
|
|
723
|
+
message: 'winkComposer/questdb: write rejected — storage is shutting down'
|
|
686
724
|
}
|
|
687
725
|
};
|
|
688
726
|
}
|
|
@@ -842,7 +880,7 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
842
880
|
*/
|
|
843
881
|
const closeQuietly = function () {
|
|
844
882
|
return sender.close().catch( function ( closeErr ) {
|
|
845
|
-
|
|
883
|
+
logger.error( `winkComposer/questdb: transport close failed during lossy shutdown: ${closeErr.message}` );
|
|
846
884
|
} );
|
|
847
885
|
}; // closeQuietly()
|
|
848
886
|
|
|
@@ -934,7 +972,7 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
934
972
|
await closeQuietly();
|
|
935
973
|
const dropped = totalRows - deliveredRows;
|
|
936
974
|
const timedOut = new Error(
|
|
937
|
-
`
|
|
975
|
+
`winkComposer/questdb: ${err.message}; ${dropped} buffered row(s) dropped`
|
|
938
976
|
);
|
|
939
977
|
timedOut.code = 'SHUTDOWN_TIMEOUT';
|
|
940
978
|
timedOut.dropped = { count: dropped };
|
|
@@ -944,7 +982,7 @@ const createQuestDBStorage = async function ( assetClass, tablePrefix, options,
|
|
|
944
982
|
if ( failedRows > 0 ) {
|
|
945
983
|
await closeQuietly();
|
|
946
984
|
const failure = new Error(
|
|
947
|
-
`
|
|
985
|
+
`winkComposer/questdb: flush failed during shutdown: ${firstFailure.message}; ${failedRows} buffered row(s) dropped`
|
|
948
986
|
);
|
|
949
987
|
failure.code = 'DELIVERY_FAILED';
|
|
950
988
|
failure.dropped = { count: failedRows };
|
|
@@ -1183,7 +1221,7 @@ const createStorage = function ( config ) {
|
|
|
1183
1221
|
// remediation differs — they need to add .assetClass() to the
|
|
1184
1222
|
// flow definition, not edit env vars or storage config.
|
|
1185
1223
|
const err = new Error(
|
|
1186
|
-
'
|
|
1224
|
+
'winkComposer/questdb: assetClass is required - add .assetClass(assetClassDef) to flow before .storage()'
|
|
1187
1225
|
);
|
|
1188
1226
|
err.code = 'MISSING_ASSET_CLASS';
|
|
1189
1227
|
throw err;
|
|
@@ -83,6 +83,19 @@
|
|
|
83
83
|
import { SenderBufferV1 } from '@questdb/nodejs-client';
|
|
84
84
|
|
|
85
85
|
import { QUEST_WRITERS, writeAsString, createFloat64Writer } from './writers.js';
|
|
86
|
+
import { wrapCallback } from '../../utils/callback-guard/index.js';
|
|
87
|
+
import { logger } from '../../logger/index.js';
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Console channel for the callback guard: one classified line in this
|
|
91
|
+
* adapter's family. Receives an already-safe detail string, never the
|
|
92
|
+
* raw thrown value.
|
|
93
|
+
*/
|
|
94
|
+
const reportCallbackFault = function ( severity, name, detail ) {
|
|
95
|
+
logger.error(
|
|
96
|
+
`winkComposer/questdb: user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
|
|
97
|
+
);
|
|
98
|
+
}; // reportCallbackFault()
|
|
86
99
|
|
|
87
100
|
// ============================================================================
|
|
88
101
|
// DEFAULT WARNING HANDLER
|
|
@@ -90,7 +103,7 @@ import { QUEST_WRITERS, writeAsString, createFloat64Writer } from './writers.js'
|
|
|
90
103
|
|
|
91
104
|
/**
|
|
92
105
|
* Default warning handler for invalid column values.
|
|
93
|
-
* Logs to console in
|
|
106
|
+
* Logs to console in winkComposer format.
|
|
94
107
|
*
|
|
95
108
|
* Validation behavior:
|
|
96
109
|
* - null/undefined columns: skip column only (QuestDB stores NULL)
|
|
@@ -108,7 +121,7 @@ import { QUEST_WRITERS, writeAsString, createFloat64Writer } from './writers.js'
|
|
|
108
121
|
* @param {string} message - Warning message describing the issue
|
|
109
122
|
*/
|
|
110
123
|
const defaultOnWarning = function ( message ) {
|
|
111
|
-
|
|
124
|
+
logger.warn( `winkComposer/questdb: ${message}` );
|
|
112
125
|
};
|
|
113
126
|
|
|
114
127
|
/**
|
|
@@ -213,7 +226,7 @@ const assertIlpNames = function ( tableName, columnNames ) {
|
|
|
213
226
|
}
|
|
214
227
|
} catch ( err ) {
|
|
215
228
|
const wrapped = new Error(
|
|
216
|
-
`
|
|
229
|
+
`winkComposer/questdb: invalid ILP ${kind} '${current}' — ${err.message}`
|
|
217
230
|
);
|
|
218
231
|
wrapped.code = 'INVALID_CONFIG';
|
|
219
232
|
wrapped.cause = err;
|
|
@@ -264,18 +277,29 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
|
|
|
264
277
|
|
|
265
278
|
if ( providedOnWarning !== undefined && typeof providedOnWarning !== 'function' ) {
|
|
266
279
|
// Per ADR-018, setup-time throws carry classified err.code.
|
|
267
|
-
const err = new Error( '
|
|
280
|
+
const err = new Error( 'winkComposer/questdb: onWarning must be a function' );
|
|
268
281
|
err.code = 'INVALID_CONFIG';
|
|
269
282
|
throw err;
|
|
270
283
|
}
|
|
271
284
|
if ( providedOnDeliveryFailure !== undefined && typeof providedOnDeliveryFailure !== 'function' ) {
|
|
272
|
-
const err = new Error( '
|
|
285
|
+
const err = new Error( 'winkComposer/questdb: onDeliveryFailure must be a function' );
|
|
273
286
|
err.code = 'INVALID_CONFIG';
|
|
274
287
|
throw err;
|
|
275
288
|
}
|
|
276
289
|
|
|
277
290
|
const onWarning = providedOnWarning || defaultOnWarning;
|
|
278
|
-
|
|
291
|
+
// onWarning stays raw on purpose. A strict-mode onWarning throws,
|
|
292
|
+
// and that throw is the instruction that rejects the row. ADR-027
|
|
293
|
+
// keeps such callbacks — ones whose throw or return the adapter
|
|
294
|
+
// acts on — out of the guard's scope. onDeliveryFailure only
|
|
295
|
+
// notifies, so the shared guard arms it. It was validated raw
|
|
296
|
+
// above and is wrapped once here (ADR-018: a broken handler costs
|
|
297
|
+
// its own output, never the flush chain). Absent stays null, so
|
|
298
|
+
// the no-handler DELIVERY_FAILED escape hatch below keeps its
|
|
299
|
+
// exact meaning.
|
|
300
|
+
const onDeliveryFailure = wrapCallback( providedOnDeliveryFailure || null, {
|
|
301
|
+
name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
|
|
302
|
+
} );
|
|
279
303
|
const plansByInsightType = Object.create( null );
|
|
280
304
|
const insightTypes = assetClass.insightTypes || Object.create( null );
|
|
281
305
|
const columns = assetClass.columns;
|
|
@@ -298,7 +322,7 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
|
|
|
298
322
|
// 'assetId' that no insightType persists stays legal.
|
|
299
323
|
if ( persistedColumnNames.includes( 'assetId' ) || designatedTimestamp === 'assetId' ) {
|
|
300
324
|
const err = new Error(
|
|
301
|
-
`
|
|
325
|
+
`winkComposer/questdb: insightType '${insightTypeName}' uses reserved column name 'assetId' — ` +
|
|
302
326
|
'composer writes this column automatically from the flow\'s .assetId() field (the partition id); ' +
|
|
303
327
|
'rename the semantics column or remove it from this insightType'
|
|
304
328
|
);
|
|
@@ -464,7 +488,7 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
|
|
|
464
488
|
return;
|
|
465
489
|
}
|
|
466
490
|
const failure = new Error(
|
|
467
|
-
`
|
|
491
|
+
`winkComposer/questdb: silent data loss — sender.at() async flush failed for table '${tableName}': ${err.message}. ` +
|
|
468
492
|
'Rows in this batch were dropped. Per the no-silent-failures contract this is a hard failure. ' +
|
|
469
493
|
'Provide an `onDeliveryFailure` callback in the storage config to handle these explicitly.'
|
|
470
494
|
);
|
|
@@ -72,7 +72,7 @@ export const lookupByField = function ( field, map, defaultVal ) {
|
|
|
72
72
|
export const pickByField = function ( map ) {
|
|
73
73
|
const fn = () => {
|
|
74
74
|
throw new Error(
|
|
75
|
-
'
|
|
75
|
+
'winkComposer/flow: pickByField is resolved at build time inside forEach; ' +
|
|
76
76
|
'it has no runtime value. Use it only for a forEach option.'
|
|
77
77
|
);
|
|
78
78
|
};
|