@winkjs/composer 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -3
- package/src/composer.js +1 -1
- package/src/core/batch/aoo-encoder.js +4 -4
- package/src/core/batch/sliding-window.js +2 -2
- package/src/core/codec/index.js +4 -4
- package/src/core/emitter-manager/mqtt/connection.js +209 -0
- package/src/core/emitter-manager/mqtt/constants.js +43 -29
- package/src/core/emitter-manager/mqtt/emitter.js +108 -633
- package/src/core/emitter-manager/mqtt/health.js +151 -0
- package/src/core/emitter-manager/mqtt/index.js +68 -4
- package/src/core/emitter-manager/mqtt/publish.js +292 -0
- package/src/core/emitter-manager/mqtt/resolve-config.js +312 -0
- package/src/core/emitter-manager/mqtt/shutdown-drain.js +247 -0
- package/src/core/emitter-manager/terminal/assert-columns.js +1 -1
- package/src/core/emitter-manager/terminal/index.js +20 -5
- package/src/core/env-vars.js +143 -16
- package/src/core/logger/index.js +174 -0
- package/src/core/mqtt-protocol.js +2 -2
- package/src/core/partition-manager/get-stats.js +31 -0
- package/src/core/partition-manager/index.js +1 -0
- package/src/core/partition-manager/init.js +15 -0
- package/src/core/partition-manager/update.js +143 -87
- package/src/core/partition-manager/validate-flow.js +1 -1
- package/src/core/semantics/loader.js +1 -1
- package/src/core/semantics/warnings.js +5 -3
- package/src/core/shutdown-manager/index.js +71 -21
- package/src/core/source-manager/csv/start.js +109 -35
- package/src/core/source-manager/mqtt/client.js +276 -83
- package/src/core/source-manager/mqtt/dedup.js +19 -14
- package/src/core/source-manager/mqtt/index.js +33 -6
- package/src/core/source-manager/mqtt/status.js +211 -46
- package/src/core/source-manager/record-shape.js +55 -0
- package/src/core/source-manager/test-harness/field-generator.js +1 -1
- package/src/core/source-manager/test-harness/start.js +52 -10
- package/src/core/source-manager/test-harness/validate.js +1 -1
- package/src/core/storage-manager/questdb/address-policy.js +264 -0
- package/src/core/storage-manager/questdb/assert-columns.js +2 -2
- package/src/core/storage-manager/questdb/delivery-gate.js +218 -0
- package/src/core/storage-manager/questdb/ensure-tables.js +1 -1
- package/src/core/storage-manager/questdb/flush-engine.js +852 -0
- package/src/core/storage-manager/questdb/flush-tracker.js +331 -0
- package/src/core/storage-manager/questdb/index.js +592 -827
- package/src/core/storage-manager/questdb/persist-plan.js +147 -162
- package/src/core/storage-manager/questdb/resolve-options.js +403 -0
- package/src/core/storage-manager/questdb/shutdown-drain.js +181 -0
- package/src/core/storage-manager/questdb/skip-warnings.js +180 -0
- package/src/core/test-utils/tcp-proxy.js +194 -1
- package/src/core/tunable/helpers.js +1 -1
- package/src/core/utils/address/index.js +362 -0
- package/src/core/utils/address/probe.js +210 -0
- package/src/core/utils/callback-guard/index.js +176 -0
- package/src/core/utils/clock/index.js +34 -0
- package/src/core/utils/half-life/index.js +4 -4
- package/src/core/utils/jitter/index.js +51 -0
- package/src/core/utils/line-rate/index.js +104 -0
- package/src/core/utils/quantize/index.js +43 -19
- package/src/core/utils/validate/composer-validators.js +2 -2
- package/src/core/utils/validate/index.js +1 -1
- package/src/core/utils/validate/validate.js +1 -1
- package/src/core/wiring/assert-handle.js +5 -5
- package/src/core/wiring/assert-module.js +2 -2
- package/src/core/wiring/wire-emitters.js +7 -4
- package/src/core/wiring/wire-linear-graph.js +1 -1
- package/src/core/wiring/wire-node.js +6 -6
- package/src/core/wiring/wire-semantics.js +1 -1
- package/src/core/wiring/wire-storages.js +14 -5
- package/src/flow/driver.js +31 -6
- package/src/flow/expand-single-input-array.js +4 -4
- package/src/flow/flow-api-schemas.js +4 -4
- package/src/flow/flow.js +62 -41
- package/src/flow/for-each.js +12 -11
- package/src/flow/get-target-array.js +2 -2
- package/src/flow/load-node-module.js +1 -1
- package/src/flow/make-node-method.js +4 -4
- package/src/flow/resolve-pick-by-field.js +1 -1
- package/src/flow/run.js +200 -40
- package/src/flow/serialize.js +1 -1
- package/src/flow/spec-builder.js +1 -1
- package/src/flow/validate.js +1 -1
- package/src/nodes/butterworth-filter/init.js +5 -4
- package/src/nodes/categorize/update.js +3 -1
- package/src/nodes/controller/update.js +2 -1
- package/src/nodes/dwell-time-tracker/update.js +2 -1
- package/src/nodes/emit-if/helpers.js +57 -11
- package/src/nodes/emit-if/init.js +1 -1
- package/src/nodes/emit-if/update.js +8 -11
- package/src/nodes/page-hinkley/update.js +2 -1
- package/src/nodes/pass-if/update.js +3 -1
- package/src/nodes/persist-if/helpers.js +60 -4
- package/src/nodes/persist-if/init.js +1 -1
- package/src/nodes/persist-if/update.js +8 -16
- package/src/nodes/persistence-check/update.js +2 -1
- package/src/nodes/sanitize/update.js +4 -2
- package/src/nodes/swing-watch/update.js +3 -2
- package/src/nodes/threshold/update.js +2 -1
- package/src/nodes/trend/compute-confidence.js +4 -4
- package/src/nodes/trend/update.js +2 -1
- package/src/nodes/winnow/update.js +3 -1
- package/src/tools/training/features.js +13 -13
- package/src/tools/training/logistic.js +1 -1
- package/src/tools/training/metrics.js +2 -2
- package/src/tools/training/scale.js +2 -2
- package/src/tools/training/validate.js +7 -7
- package/src/core/utils/console-formatter/index.js +0 -207
- package/src/nodes/archive/csv/index.js +0 -6
- package/src/nodes/archive/csv/init.js +0 -36
- package/src/nodes/archive/csv/load.js +0 -70
- package/src/nodes/archive/csv/reset.js +0 -14
- package/src/nodes/archive/csv/simple-parser.js +0 -216
- package/src/nodes/archive/csv/test.js +0 -173
- package/src/nodes/archive/csv/update.js +0 -63
- package/src/nodes/sse-emitter/index.js +0 -7
- package/src/nodes/sse-emitter/init.js +0 -105
- package/src/nodes/sse-emitter/introspect.js +0 -134
- package/src/nodes/sse-emitter/publish-to.js +0 -6
- package/src/nodes/sse-emitter/recompute.js +0 -16
- package/src/nodes/sse-emitter/reset.js +0 -19
- package/src/nodes/sse-emitter/update.js +0 -69
- package/src/nodes/sse-emitter/utils/broadcast-heartbeat.js +0 -36
- package/src/nodes/sse-emitter/utils/build-message.js +0 -31
- package/src/nodes/sse-emitter/utils/connection-manager.js +0 -45
- package/src/nodes/sse-emitter/utils/endpoint-initializer.js +0 -20
- package/src/nodes/sse-emitter/utils/health-monitor.js +0 -38
- package/src/nodes/sse-emitter/utils/qos-handler.js +0 -41
package/src/flow/for-each.js
CHANGED
|
@@ -31,6 +31,7 @@ import { getSignaturePattern } from './get-signature-pattern.js';
|
|
|
31
31
|
import { specBuilder } from './spec-builder.js';
|
|
32
32
|
import { getTargetArray } from './get-target-array.js';
|
|
33
33
|
import { resolvePickByField } from './resolve-pick-by-field.js';
|
|
34
|
+
import { logger } from '../core/logger/index.js';
|
|
34
35
|
|
|
35
36
|
/**
|
|
36
37
|
* Locate the index of a named argument slot in a signature schema (the source of
|
|
@@ -90,7 +91,7 @@ export const makeForEach = function ( api, flowState ) {
|
|
|
90
91
|
// rather than emit a silently-wrong per-field controller.
|
|
91
92
|
if ( pattern === SIGNATURE_PATTERNS.nameLogic ) {
|
|
92
93
|
throw new Error(
|
|
93
|
-
`
|
|
94
|
+
`winkComposer/flow: forEach cannot fan a controller ('${args[ 0 ]}'); ` +
|
|
94
95
|
'fan data-processing nodes only.'
|
|
95
96
|
);
|
|
96
97
|
}
|
|
@@ -109,7 +110,7 @@ export const makeForEach = function ( api, flowState ) {
|
|
|
109
110
|
// Node-name collision check before building stats (matches make-node-method),
|
|
110
111
|
// so a rejected node never registers its output params in the global checker.
|
|
111
112
|
if ( isNodeNameDuplicate( t[ 0 ] ) ) {
|
|
112
|
-
throw new Error( `
|
|
113
|
+
throw new Error( `winkComposer/flow: duplicate node ${t[ 0 ]} found.` );
|
|
113
114
|
}
|
|
114
115
|
|
|
115
116
|
// Outputs: store each labelled output as `${field}_${label}`, and record the
|
|
@@ -121,8 +122,8 @@ export const makeForEach = function ( api, flowState ) {
|
|
|
121
122
|
for ( let i = 0; i < outKeys.length; i += 1 ) {
|
|
122
123
|
const label = rawOutputs[ outKeys[ i ] ];
|
|
123
124
|
if ( Object.prototype.hasOwnProperty.call( recorded, label ) ) {
|
|
124
|
-
|
|
125
|
-
`
|
|
125
|
+
logger.warn(
|
|
126
|
+
`winkComposer/flow: forEach output label '${label}' written by ` +
|
|
126
127
|
`more than one step on field '${field}'; each.out will return the last.`
|
|
127
128
|
);
|
|
128
129
|
}
|
|
@@ -146,7 +147,7 @@ export const makeForEach = function ( api, flowState ) {
|
|
|
146
147
|
targetArray.push( spec );
|
|
147
148
|
} catch ( error ) {
|
|
148
149
|
throw new Error(
|
|
149
|
-
`
|
|
150
|
+
`winkComposer/flow: Failed to process "${callContext}"\n` +
|
|
150
151
|
` Reason: ${error.message}\n`
|
|
151
152
|
);
|
|
152
153
|
}
|
|
@@ -170,7 +171,7 @@ export const makeForEach = function ( api, flowState ) {
|
|
|
170
171
|
each.out = function ( label ) {
|
|
171
172
|
if ( !Object.prototype.hasOwnProperty.call( recorded, label ) ) {
|
|
172
173
|
throw new Error(
|
|
173
|
-
`
|
|
174
|
+
`winkComposer/flow: each.out( '${label}' ) on field '${field}': ` +
|
|
174
175
|
`no earlier step in this chain wrote an output labelled '${label}'.`
|
|
175
176
|
);
|
|
176
177
|
}
|
|
@@ -191,25 +192,25 @@ export const makeForEach = function ( api, flowState ) {
|
|
|
191
192
|
// that only resolves at .endGroup(), so the callback cannot run where written.
|
|
192
193
|
if ( groupByState.active ) {
|
|
193
194
|
throw new Error(
|
|
194
|
-
'
|
|
195
|
+
'winkComposer/flow: forEach inside groupBy is not supported yet; ' +
|
|
195
196
|
'the field list must be a concrete array at the call site.'
|
|
196
197
|
);
|
|
197
198
|
}
|
|
198
199
|
if ( !Array.isArray( fields ) || fields.length === 0 ) {
|
|
199
|
-
throw new Error( '
|
|
200
|
+
throw new Error( 'winkComposer/flow: forEach requires a non-empty array of field names.' );
|
|
200
201
|
}
|
|
201
202
|
if ( typeof callback !== 'function' ) {
|
|
202
|
-
throw new Error( '
|
|
203
|
+
throw new Error( 'winkComposer/flow: forEach requires a callback function.' );
|
|
203
204
|
}
|
|
204
205
|
|
|
205
206
|
const seen = Object.create( null );
|
|
206
207
|
for ( let i = 0; i < fields.length; i += 1 ) {
|
|
207
208
|
const field = fields[ i ];
|
|
208
209
|
if ( typeof field !== 'string' || field.length === 0 ) {
|
|
209
|
-
throw new Error( '
|
|
210
|
+
throw new Error( 'winkComposer/flow: forEach field names must be non-empty strings.' );
|
|
210
211
|
}
|
|
211
212
|
if ( Object.prototype.hasOwnProperty.call( seen, field ) ) {
|
|
212
|
-
throw new Error( `
|
|
213
|
+
throw new Error( `winkComposer/flow: forEach duplicate field '${field}' in field list.` );
|
|
213
214
|
}
|
|
214
215
|
seen[ field ] = 1;
|
|
215
216
|
}
|
|
@@ -29,13 +29,13 @@ export const getTargetArray = function ( switchState, groupByState, flowDefiniti
|
|
|
29
29
|
if ( switchState && switchState.active ) {
|
|
30
30
|
if ( switchState.currentCase === null ) {
|
|
31
31
|
throw new Error(
|
|
32
|
-
'
|
|
32
|
+
'winkComposer/flow: node called outside of a .case() block - ' +
|
|
33
33
|
'after .switch(), you must call .case() before adding nodes'
|
|
34
34
|
);
|
|
35
35
|
}
|
|
36
36
|
if ( switchState.caseEnded ) {
|
|
37
37
|
throw new Error(
|
|
38
|
-
'
|
|
38
|
+
'winkComposer/flow: node called after .break() - ' +
|
|
39
39
|
'start a new .case() before adding more nodes'
|
|
40
40
|
);
|
|
41
41
|
}
|
|
@@ -31,7 +31,7 @@ const moduleCache = new Map();
|
|
|
31
31
|
export const loadNodeModule = async function ( nodeName ) {
|
|
32
32
|
// Validate node name
|
|
33
33
|
if ( !isValidNodeName( nodeName ) ) {
|
|
34
|
-
throw new Error( `Unknown node type: "${nodeName}"` );
|
|
34
|
+
throw new Error( `winkComposer/flow: Unknown node type: "${nodeName}"` );
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
// Return cached module if available
|
|
@@ -8,7 +8,7 @@ import { specBuilder } from './spec-builder.js';
|
|
|
8
8
|
import { getTargetArray } from './get-target-array.js';
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* Creates a chainable DSL method for a
|
|
11
|
+
* Creates a chainable DSL method for a winkComposer node.
|
|
12
12
|
*
|
|
13
13
|
* @param {Object} api - The fluent API object to extend
|
|
14
14
|
* @param {string} node - Node type name (e.g., 'esMean')
|
|
@@ -79,9 +79,9 @@ export const makeNodeMethod = function ( api, node, meta, flowState ) {
|
|
|
79
79
|
} else {
|
|
80
80
|
// Canonical builder path
|
|
81
81
|
const instName = args[ 0 ];
|
|
82
|
-
if ( effectiveNodeChecker( instName ) ) throw Error( `
|
|
82
|
+
if ( effectiveNodeChecker( instName ) ) throw Error( `winkComposer/flow: duplicate node ${instName} found.` );
|
|
83
83
|
const pattern = getSignaturePattern( schema );
|
|
84
|
-
if ( pattern === SIGNATURE_PATTERNS.unknown ) throw Error( `
|
|
84
|
+
if ( pattern === SIGNATURE_PATTERNS.unknown ) throw Error( `winkComposer/flow: unknown signature pattern found for ${instName}.` );
|
|
85
85
|
const status = validateWithSchema( schemas[ pattern ], [ ...args ], 'Arg' );
|
|
86
86
|
status.throwIfInvalid( 'argument' );
|
|
87
87
|
built = buildSpecFor[ pattern ]( [ ...args ] );
|
|
@@ -101,7 +101,7 @@ export const makeNodeMethod = function ( api, node, meta, flowState ) {
|
|
|
101
101
|
}
|
|
102
102
|
} catch ( error ) {
|
|
103
103
|
const enhancedError = new Error(
|
|
104
|
-
`
|
|
104
|
+
`winkComposer/flow: Failed to process "${callContext}"\n` +
|
|
105
105
|
` Reason: ${error.message}\n`
|
|
106
106
|
);
|
|
107
107
|
|
|
@@ -35,7 +35,7 @@ export const resolvePickByField = function ( value, field ) {
|
|
|
35
35
|
const map = value.semantics.map;
|
|
36
36
|
if ( !Object.prototype.hasOwnProperty.call( map, field ) ) {
|
|
37
37
|
throw new Error(
|
|
38
|
-
`
|
|
38
|
+
`winkComposer/flow: pickByField has no entry for field '${field}' ` +
|
|
39
39
|
`(map keys: ${Object.keys( map ).join( ', ' )}).`
|
|
40
40
|
);
|
|
41
41
|
}
|
package/src/flow/run.js
CHANGED
|
@@ -17,6 +17,8 @@ import * as partitionManager from '../core/partition-manager/index.js';
|
|
|
17
17
|
import { wireLinearGraph, emitters, storages, assertModuleDurability } from '../core/wiring/index.js';
|
|
18
18
|
import shutdownManager from '../core/shutdown-manager/index.js';
|
|
19
19
|
import { ENV_VARS } from '../core/env-vars.js';
|
|
20
|
+
import { wrapCallback } from '../core/utils/callback-guard/index.js';
|
|
21
|
+
import { logger } from '../core/logger/index.js';
|
|
20
22
|
|
|
21
23
|
/**
|
|
22
24
|
* Wires and starts the pipeline for direct execution.
|
|
@@ -27,7 +29,8 @@ import { ENV_VARS } from '../core/env-vars.js';
|
|
|
27
29
|
* @param {Set} importSet - Set of node names used
|
|
28
30
|
* @param {Object} runtime - Runtime configuration
|
|
29
31
|
* @param {Array} [caseOrder] - Order of case keys (only for multi-specialization)
|
|
30
|
-
* @returns {Promise<Object>} Pipeline handle
|
|
32
|
+
* @returns {Promise<Object>} Pipeline handle: processMessage, shutdown,
|
|
33
|
+
* whenComplete, getStats, flowName, composerState
|
|
31
34
|
*/
|
|
32
35
|
export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet, runtime, caseOrder ) {
|
|
33
36
|
// Determine if this is multi-specialization mode
|
|
@@ -156,10 +159,16 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
156
159
|
// caller that ignores the Promise (a push source such as the MQTT
|
|
157
160
|
// subscriber) can never see messages update a partition out of order.
|
|
158
161
|
//
|
|
159
|
-
// The caller contract (ADR-013) is unchanged: `undefined` on
|
|
160
|
-
// path, a Promise on the yield tick; `await` works with both,
|
|
161
|
-
// Promise never rejects
|
|
162
|
-
//
|
|
162
|
+
// The caller contract (ADR-013) is unchanged in shape: `undefined` on
|
|
163
|
+
// the hot path, a Promise on the yield tick; `await` works with both,
|
|
164
|
+
// and the Promise never rejects. Fault delivery depends on the flow
|
|
165
|
+
// kind (ADR-018 — the runtime owns per-message dispatch failure). With
|
|
166
|
+
// a source present, a pipeline fault is contained here: the message is
|
|
167
|
+
// skipped, one red MESSAGE_HANDLER_FAILED travels the status channel,
|
|
168
|
+
// and a run of consecutive failures stops the flow in the terminal
|
|
169
|
+
// 'errored' phase. A headless flow (no source) has no status channel,
|
|
170
|
+
// so the fault throws synchronously to the caller — the headless
|
|
171
|
+
// driver's catch and `failed` counter are that path's containment.
|
|
163
172
|
//
|
|
164
173
|
// The default threshold comes from YIELD_TIME_THRESHOLD_MS (500 ms);
|
|
165
174
|
// `.yield( { threshold } )` overrides it per flow, and Infinity turns
|
|
@@ -187,26 +196,10 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
187
196
|
|
|
188
197
|
const partitionState = composerState.partitionState;
|
|
189
198
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
// (or null for a dropped message, which never sets the flag).
|
|
195
|
-
if ( graph ) {
|
|
196
|
-
runPipeline( graph, msg );
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
// Breathe after (yield tick, rare): hand the awaiting caller a
|
|
200
|
-
// Promise that resolves once the event loop has had a full turn.
|
|
201
|
-
if ( partitionState.yieldPending ) {
|
|
202
|
-
partitionState.yieldPending = false;
|
|
203
|
-
return new Promise( ( resolve ) => {
|
|
204
|
-
setImmediate( resolve );
|
|
205
|
-
} );
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
return undefined;
|
|
209
|
-
};
|
|
199
|
+
// The dispatch function itself is defined AFTER the shutdown
|
|
200
|
+
// closure below — its escalation path drains the flow, so it
|
|
201
|
+
// needs `shutdown` in scope. No message can arrive before then:
|
|
202
|
+
// the source only receives processMessage at start, further down.
|
|
210
203
|
|
|
211
204
|
// 6. Setup the flow's shutdown closure.
|
|
212
205
|
//
|
|
@@ -272,13 +265,28 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
272
265
|
await stage();
|
|
273
266
|
} catch ( err ) {
|
|
274
267
|
stageErrors.push( err );
|
|
275
|
-
|
|
276
|
-
`
|
|
268
|
+
logger.error(
|
|
269
|
+
`winkComposer/flow: ${label} drain stage failed [${( err && err.code ) || 'UNKNOWN'}]: ` +
|
|
277
270
|
`${( err && err.message ) || String( err )} — later drain stages still ran`
|
|
278
271
|
);
|
|
279
272
|
}
|
|
280
273
|
}; // runDrainStage()
|
|
281
274
|
|
|
275
|
+
// A sink stage returns `Promise.allSettled` results. The wiring
|
|
276
|
+
// logs each adapter whose drain rejected and lets its siblings
|
|
277
|
+
// finish, so one failing sink never strands another's data. The
|
|
278
|
+
// loss must still reach this handle's caller. A rejected drain is
|
|
279
|
+
// how the shutdown manager knows to exit 1 (ADR-018 §7). So once
|
|
280
|
+
// every adapter in the stage has drained, the stage throws the
|
|
281
|
+
// first rejected reason, code and dropped count intact.
|
|
282
|
+
const throwFirstRejection = function ( results ) {
|
|
283
|
+
for ( let i = 0; i < results.length; i += 1 ) {
|
|
284
|
+
if ( results[ i ].status === 'rejected' ) {
|
|
285
|
+
throw results[ i ].reason;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}; // throwFirstRejection()
|
|
289
|
+
|
|
282
290
|
const drainAll = async function () {
|
|
283
291
|
const stageErrors = [];
|
|
284
292
|
try {
|
|
@@ -288,9 +296,9 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
288
296
|
() => stopSource( { timeout: STAGE_TIMEOUT_MS } ) );
|
|
289
297
|
}
|
|
290
298
|
await runDrainStage( 'emitter', stageErrors,
|
|
291
|
-
() => emitters.shutdown( { timeout: STAGE_TIMEOUT_MS } ) );
|
|
299
|
+
async () => throwFirstRejection( await emitters.shutdown( { timeout: STAGE_TIMEOUT_MS } ) ) );
|
|
292
300
|
await runDrainStage( 'storage', stageErrors,
|
|
293
|
-
() => storages.shutdown( { timeout: STAGE_TIMEOUT_MS } ) );
|
|
301
|
+
async () => throwFirstRejection( await storages.shutdown( { timeout: STAGE_TIMEOUT_MS } ) ) );
|
|
294
302
|
} finally {
|
|
295
303
|
// Whatever the path here — clean drain or a stage that threw —
|
|
296
304
|
// the source has stopped producing, so unblock any
|
|
@@ -311,6 +319,100 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
311
319
|
return shutdownPromise;
|
|
312
320
|
};
|
|
313
321
|
|
|
322
|
+
// Run the drain with its rejection observed. Both fire-and-forget
|
|
323
|
+
// call sites use this: the natural-completion trigger and the
|
|
324
|
+
// consecutive-failure escalation. drainAll rethrows the first
|
|
325
|
+
// stage error; with no awaiting caller that rejection would end
|
|
326
|
+
// the process as unhandled. The classified log keeps the loss
|
|
327
|
+
// loud without the crash, and a caller that later awaits
|
|
328
|
+
// `handle.shutdown()` still receives the memoized rejection.
|
|
329
|
+
//
|
|
330
|
+
// The exit code carries the loss too. This drain has no awaiting
|
|
331
|
+
// caller, and the flow left the shutdown manager's list when the
|
|
332
|
+
// drain began, so no other path can set it. A batch script that
|
|
333
|
+
// replays a file and loses rows at the end must exit 1, the same
|
|
334
|
+
// as the signal path does (ADR-018 §7).
|
|
335
|
+
const observedShutdown = function () {
|
|
336
|
+
shutdown().catch( function ( err ) {
|
|
337
|
+
process.exitCode = 1;
|
|
338
|
+
logger.error(
|
|
339
|
+
`winkComposer/flow '${flowName}': shutdown failed ` +
|
|
340
|
+
`[${( err && err.code ) || 'UNKNOWN'}]: ${( err && err.message ) || String( err )}`
|
|
341
|
+
);
|
|
342
|
+
} );
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
// ── Message dispatch with fault containment (ADR-018: the flow
|
|
346
|
+
// runtime owns per-message dispatch failure at this chokepoint) ──
|
|
347
|
+
// All containment state is flow-scoped policy, kept in this
|
|
348
|
+
// closure beside the shutdown state — never on the handle-exposed
|
|
349
|
+
// composerState. `reportDispatchFailure` stays null for a headless
|
|
350
|
+
// flow (no source); the source block below assigns it before the
|
|
351
|
+
// source ever receives processMessage.
|
|
352
|
+
let consecutiveFailures = 0;
|
|
353
|
+
let errored = false;
|
|
354
|
+
let erroredDropReported = false;
|
|
355
|
+
let reportDispatchFailure = null;
|
|
356
|
+
|
|
357
|
+
const processMessage = function ( msg ) {
|
|
358
|
+
// Terminal 'errored': drop everything. The first drop is
|
|
359
|
+
// reported once, classified; a flood of per-drop reports
|
|
360
|
+
// would bury the one terminal red that matters.
|
|
361
|
+
if ( errored ) {
|
|
362
|
+
if ( !erroredDropReported ) {
|
|
363
|
+
erroredDropReported = true;
|
|
364
|
+
logger.error(
|
|
365
|
+
`winkComposer/flow '${flowName}': flow is in the terminal 'errored' ` +
|
|
366
|
+
'phase [MESSAGE_HANDLER_FAILED] — dropping this and all further messages'
|
|
367
|
+
);
|
|
368
|
+
}
|
|
369
|
+
return undefined;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Process first — pmUpdate always returns the graph synchronously
|
|
373
|
+
// (or null for a dropped message, which never sets the flag).
|
|
374
|
+
// The guard covers pmUpdate too: a first message for a new asset
|
|
375
|
+
// runs every node's init lazily, so an init throw surfaces here.
|
|
376
|
+
try {
|
|
377
|
+
const graph = partitionManager.update( composerState, msg );
|
|
378
|
+
if ( graph ) {
|
|
379
|
+
runPipeline( graph, msg );
|
|
380
|
+
// Reset on success only — a dropped message is neither
|
|
381
|
+
// a success nor a failure. Branch-guarded so the
|
|
382
|
+
// healthy steady state never writes.
|
|
383
|
+
if ( consecutiveFailures !== 0 ) {
|
|
384
|
+
consecutiveFailures = 0;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
} catch ( err ) {
|
|
388
|
+
if ( reportDispatchFailure === null ) {
|
|
389
|
+
// Headless: the caller is the reporting channel — the
|
|
390
|
+
// driver's catch and `failed` counter own the fault.
|
|
391
|
+
throw err;
|
|
392
|
+
}
|
|
393
|
+
consecutiveFailures += 1;
|
|
394
|
+
const isTerminal = consecutiveFailures >= ENV_VARS.messageFailureThreshold;
|
|
395
|
+
reportDispatchFailure( err, isTerminal );
|
|
396
|
+
if ( isTerminal ) {
|
|
397
|
+
errored = true;
|
|
398
|
+
observedShutdown();
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
// Breathe after (yield tick, rare): hand the awaiting caller a
|
|
403
|
+
// Promise that resolves once the event loop has had a full turn.
|
|
404
|
+
// Runs on the catch path too — pmUpdate may have set the flag
|
|
405
|
+
// before the pipeline threw, and the breath must not be lost.
|
|
406
|
+
if ( partitionState.yieldPending ) {
|
|
407
|
+
partitionState.yieldPending = false;
|
|
408
|
+
return new Promise( ( resolve ) => {
|
|
409
|
+
setImmediate( resolve );
|
|
410
|
+
} );
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
return undefined;
|
|
414
|
+
};
|
|
415
|
+
|
|
314
416
|
// 7. Start source (singleton).
|
|
315
417
|
if ( runtime.source ) {
|
|
316
418
|
const { adapter, config } = runtime.source;
|
|
@@ -321,17 +423,42 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
321
423
|
// too (they bypass the DSL entirely).
|
|
322
424
|
assertModuleDurability( adapter.id || 'source', adapter );
|
|
323
425
|
|
|
426
|
+
// The callback guard below turns a non-function into null
|
|
427
|
+
// (absent). Validate raw first, so a misconfigured handler
|
|
428
|
+
// fails loudly at setup instead of silently vanishing. Placed
|
|
429
|
+
// here for the same reason as the durability check above:
|
|
430
|
+
// direct runFlow() callers bypass the DSL schema.
|
|
431
|
+
if ( config.onStatus !== undefined && config.onStatus !== null && typeof config.onStatus !== 'function' ) {
|
|
432
|
+
const err = new Error( `winkComposer/flow '${flowName}': onStatus must be a function` );
|
|
433
|
+
err.code = 'INVALID_CONFIG';
|
|
434
|
+
throw err;
|
|
435
|
+
}
|
|
436
|
+
|
|
324
437
|
// Wrap user's onStatus so the runtime can spot the natural-
|
|
325
438
|
// completion signal without taking it away from the user.
|
|
326
439
|
// When the source signals `phase: 'complete'`:
|
|
327
440
|
// 1. Resolve the whenComplete Promise so callers can stop
|
|
328
441
|
// waiting for natural completion.
|
|
329
|
-
// 2. Trigger the drain
|
|
330
|
-
//
|
|
331
|
-
|
|
442
|
+
// 2. Trigger the drain via observedShutdown — fire-and-forget
|
|
443
|
+
// with its rejection logged; shutdown is concurrent-safe
|
|
444
|
+
// and stages have their own timeouts.
|
|
445
|
+
// The user's handler is armed by the shared callback guard
|
|
446
|
+
// (ADR-018: a misbehaving user callback never interrupts the
|
|
447
|
+
// runtime that called it). Null when the flow has no handler,
|
|
448
|
+
// so the classified fallback branch below keeps red statuses
|
|
449
|
+
// loud.
|
|
450
|
+
const safeUserOnStatus = wrapCallback( config.onStatus, {
|
|
451
|
+
name: 'onStatus',
|
|
452
|
+
severity: 'red',
|
|
453
|
+
report: function ( severity, name, detail ) {
|
|
454
|
+
logger.error(
|
|
455
|
+
`winkComposer/flow '${flowName}': user callback ${name} failed [CALLBACK_FAILED]: ${detail}`
|
|
456
|
+
);
|
|
457
|
+
}
|
|
458
|
+
} );
|
|
332
459
|
const wrappedOnStatus = function ( s ) {
|
|
333
|
-
if (
|
|
334
|
-
|
|
460
|
+
if ( safeUserOnStatus ) {
|
|
461
|
+
safeUserOnStatus( s );
|
|
335
462
|
} else if ( s && s.status === 'red' ) {
|
|
336
463
|
// No user handler: log the failure here, classified. The
|
|
337
464
|
// source's own console fallback fires only when NO handler
|
|
@@ -341,13 +468,39 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
341
468
|
const code = ( s.error && s.error.code ) || 'UNKNOWN';
|
|
342
469
|
const message = ( s.error && s.error.message ) ||
|
|
343
470
|
'source reported status red with no error detail';
|
|
344
|
-
|
|
471
|
+
logger.error( `winkComposer/flow '${flowName}': source error [${code}]: ${message}` );
|
|
345
472
|
}
|
|
473
|
+
// Completion bookkeeping runs after — and independently of —
|
|
474
|
+
// the user's reporter. A throwing onStatus must never swallow
|
|
475
|
+
// the flow's own completion: whenComplete waiters would hang
|
|
476
|
+
// and the source would never stop.
|
|
346
477
|
if ( s && s.phase === 'complete' ) {
|
|
347
478
|
resolveWhenComplete();
|
|
348
|
-
|
|
479
|
+
observedShutdown();
|
|
349
480
|
}
|
|
350
481
|
};
|
|
482
|
+
|
|
483
|
+
// The dispatch guard's reporting channel. Routing through
|
|
484
|
+
// wrappedOnStatus keeps the two-party rule: a flow without a
|
|
485
|
+
// user handler still gets the classified console fallback for
|
|
486
|
+
// red statuses. No local guard is needed here: the callback
|
|
487
|
+
// guard above contains a throwing user handler, and every other
|
|
488
|
+
// branch of wrappedOnStatus is throw-free.
|
|
489
|
+
reportDispatchFailure = function ( err, isTerminal ) {
|
|
490
|
+
const payload = {
|
|
491
|
+
status: 'red',
|
|
492
|
+
connected: true,
|
|
493
|
+
phase: isTerminal ? 'errored' : 'running',
|
|
494
|
+
error: {
|
|
495
|
+
code: 'MESSAGE_HANDLER_FAILED',
|
|
496
|
+
message: isTerminal ?
|
|
497
|
+
`${ENV_VARS.messageFailureThreshold} consecutive message failures — ` +
|
|
498
|
+
`flow stopped. Last: ${err.message}` :
|
|
499
|
+
`${err.message} — message skipped`
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
wrappedOnStatus( payload );
|
|
503
|
+
};
|
|
351
504
|
stopSource = adapter.start( {
|
|
352
505
|
...config,
|
|
353
506
|
// Override any source-level auto-shutdown setting. The
|
|
@@ -362,10 +515,12 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
362
515
|
onShutdown: shutdown
|
|
363
516
|
} );
|
|
364
517
|
} else {
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
//
|
|
368
|
-
//
|
|
518
|
+
// The headless branch: a flow with no source is a documented
|
|
519
|
+
// DSL path (the headless driver feeds it). No source means
|
|
520
|
+
// "nothing to wait for" — resolve immediately so any
|
|
521
|
+
// `await handle.whenComplete()` is a no-op. And with no
|
|
522
|
+
// status channel, `reportDispatchFailure` stays null, so the
|
|
523
|
+
// dispatch guard rethrows pipeline faults to the caller.
|
|
369
524
|
resolveWhenComplete();
|
|
370
525
|
}
|
|
371
526
|
|
|
@@ -384,6 +539,11 @@ export const runFlow = async function ( flowName, specsOrSpecsByCase, importSet,
|
|
|
384
539
|
composerState,
|
|
385
540
|
shutdown,
|
|
386
541
|
processMessage,
|
|
542
|
+
// Cold-path counter snapshot (drops, partitions). Allocates one
|
|
543
|
+
// small object per call; never called by the pipeline itself.
|
|
544
|
+
getStats: function () {
|
|
545
|
+
return partitionManager.getStats( composerState );
|
|
546
|
+
},
|
|
387
547
|
whenComplete: function () {
|
|
388
548
|
return whenCompletePromise;
|
|
389
549
|
}
|
package/src/flow/serialize.js
CHANGED
|
@@ -59,7 +59,7 @@ const serializeValue = function ( value, indent ) {
|
|
|
59
59
|
|
|
60
60
|
if ( t === 'function' ) {
|
|
61
61
|
const funcStr = value.toString();
|
|
62
|
-
if ( funcStr.includes( '[native code]' ) ) throw Error( '
|
|
62
|
+
if ( funcStr.includes( '[native code]' ) ) throw Error( 'winkComposer/flow: only regular or arrow functions are supported.' );
|
|
63
63
|
return funcStr;
|
|
64
64
|
}
|
|
65
65
|
|
package/src/flow/spec-builder.js
CHANGED
|
@@ -11,7 +11,7 @@ const buildStatsFromOutputs = function ( outputs ) {
|
|
|
11
11
|
// overwrite case, caught at build time by check-output-collisions.js.
|
|
12
12
|
if ( seenStoreAs[ storeAs ] ) {
|
|
13
13
|
throw new Error(
|
|
14
|
-
`
|
|
14
|
+
`winkComposer/flow: Node has duplicate storeAs '${storeAs}' ` +
|
|
15
15
|
`for stats '${seenStoreAs[ storeAs ]}' and '${stat}'`
|
|
16
16
|
);
|
|
17
17
|
}
|
package/src/flow/validate.js
CHANGED
|
@@ -254,7 +254,7 @@ export const validateFlowOrThrow = function ( flowName, specs, nodeModules, regi
|
|
|
254
254
|
if ( !result.valid ) {
|
|
255
255
|
const errorList = result.errors.map( ( e ) => ` - ${e}` ).join( '\n' );
|
|
256
256
|
throw new Error(
|
|
257
|
-
`
|
|
257
|
+
`winkComposer/flow '${flowName}': validation failed:\n${errorList}`
|
|
258
258
|
);
|
|
259
259
|
}
|
|
260
260
|
};
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import { validateSpec } from '../../core/utils/node/index.js';
|
|
11
11
|
import * as introspect from './introspect.js';
|
|
12
12
|
import { resolveScalar } from '../../core/utils/options/resolve-field-keyed.js';
|
|
13
|
+
import { logger } from '../../core/logger/index.js';
|
|
13
14
|
|
|
14
15
|
const init = function ( spec ) {
|
|
15
16
|
validateSpec( spec, introspect );
|
|
@@ -74,14 +75,14 @@ const init = function ( spec ) {
|
|
|
74
75
|
// Validate the computed cutoff
|
|
75
76
|
if ( cutoffHz <= 0 || cutoffHz >= nyquistHz ) {
|
|
76
77
|
throw new RangeError(
|
|
77
|
-
`Computed cutoff ${cutoffHz.toFixed(1)}Hz is outside valid range (0, ${nyquistHz}Hz)`
|
|
78
|
+
`winkComposer/butterworthFilter: Computed cutoff ${cutoffHz.toFixed(1)}Hz is outside valid range (0, ${nyquistHz}Hz)`
|
|
78
79
|
);
|
|
79
80
|
}
|
|
80
81
|
|
|
81
82
|
// Warn about potential numerical issues
|
|
82
83
|
const normalizedCutoff = cutoffHz / nyquistHz;
|
|
83
84
|
if ( normalizedCutoff < 0.001 && !state.acceptNumericalRisk ) {
|
|
84
|
-
|
|
85
|
+
logger.warn( `winkComposer/butterworthFilter: Very low cutoff (${cutoffHz.toFixed(3)}Hz) may cause numerical instability` );
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
// Store configuration for reference
|
|
@@ -127,14 +128,14 @@ const init = function ( spec ) {
|
|
|
127
128
|
/* c8 ignore next 5 -- defensive: bilinear transform produces stable coefficients for valid Nyquist-bounded cutoff */
|
|
128
129
|
if ( Math.abs( state.a2 ) >= 1 ) {
|
|
129
130
|
throw new Error(
|
|
130
|
-
`Filter unstable: coefficient a2 (${state.a2.toFixed(4)}) is outside stability bounds`
|
|
131
|
+
`winkComposer/butterworthFilter: Filter unstable: coefficient a2 (${state.a2.toFixed(4)}) is outside stability bounds`
|
|
131
132
|
);
|
|
132
133
|
}
|
|
133
134
|
|
|
134
135
|
/* c8 ignore next 5 -- defensive: same Nyquist guard prevents a1 stability violation */
|
|
135
136
|
if ( Math.abs( state.a1 ) >= 1 + state.a2 ) {
|
|
136
137
|
throw new Error(
|
|
137
|
-
`Filter unstable: coefficient a1 (${state.a1.toFixed(4)}) violates stability condition |a1| < 1 + a2`
|
|
138
|
+
`winkComposer/butterworthFilter: Filter unstable: coefficient a1 (${state.a1.toFixed(4)}) violates stability condition |a1| < 1 + a2`
|
|
138
139
|
);
|
|
139
140
|
}
|
|
140
141
|
|
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
* threshold counts (3–5), linear search outperforms binary search.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { logger } from '../../core/logger/index.js';
|
|
10
|
+
|
|
9
11
|
const update = function ( state, msg ) {
|
|
10
12
|
if ( state.disable || state.pause ) return state;
|
|
11
13
|
const xVal = msg[ state.x ];
|
|
@@ -25,7 +27,7 @@ const update = function ( state, msg ) {
|
|
|
25
27
|
} catch ( error ) {
|
|
26
28
|
if ( !state.tunableErrorLogged ) {
|
|
27
29
|
state.tunableErrorLogged = true;
|
|
28
|
-
|
|
30
|
+
logger.error( `winkComposer/${state.nodeType}: tunable threw: ${error.message}` );
|
|
29
31
|
}
|
|
30
32
|
}
|
|
31
33
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// nodes/controller/update.js
|
|
2
2
|
|
|
3
3
|
import { executeTriggers } from '../../core/utils/node/index.js';
|
|
4
|
+
import { logger } from '../../core/logger/index.js';
|
|
4
5
|
|
|
5
6
|
const update = function ( state, msg ) {
|
|
6
7
|
// Track whether any predicate threw during this pass
|
|
@@ -22,7 +23,7 @@ const update = function ( state, msg ) {
|
|
|
22
23
|
// Log first error per episode; suppress subsequent until recovery
|
|
23
24
|
if ( !state.predicateErrorLogged ) {
|
|
24
25
|
state.predicateErrorLogged = true;
|
|
25
|
-
|
|
26
|
+
logger.error( `winkComposer/controller: predicate threw exception: ${error.message}` );
|
|
26
27
|
}
|
|
27
28
|
// Skip to next condition
|
|
28
29
|
continue; // eslint-disable-line no-continue
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// nodes/dwell-time-tracker/update.js
|
|
2
2
|
|
|
3
3
|
import { executeTriggers } from '../../core/utils/node/index.js';
|
|
4
|
+
import { logger } from '../../core/logger/index.js';
|
|
4
5
|
|
|
5
6
|
const update = function ( state, msg ) {
|
|
6
7
|
if ( state.disable || state.pause ) return state;
|
|
@@ -20,7 +21,7 @@ const update = function ( state, msg ) {
|
|
|
20
21
|
// Log first error per episode; suppress subsequent until recovery
|
|
21
22
|
if ( !state.predicateErrorLogged ) {
|
|
22
23
|
state.predicateErrorLogged = true;
|
|
23
|
-
|
|
24
|
+
logger.error( `winkComposer/dwellTimeTracker: predicate threw exception: ${error.message}` );
|
|
24
25
|
}
|
|
25
26
|
return state;
|
|
26
27
|
}
|