@winkjs/composer 0.0.2 → 0.4.1

package/src/nodes/winnow/introspect.js

@@ -0,0 +1,253 @@
+/**
+ * @fileoverview Introspection metadata for the winnow node.
+ *
+ * Winnow is a trajectory-aware significance detector. It tracks a
+ * projected trajectory from the last significant point and fires when
+ * the signal deviates beyond an adaptive, self-tightening threshold.
+ *
+ * The name comes from agriculture: winnowing separates the grain
+ * (significant samples) from the chaff (redundant ones). Pair with
+ * passIf for compression, emitIf for edge-to-cloud gating, or
+ * threshold on deviation for adaptive alarming.
+ *
+ * Reads slope, noise, direction, and gate from upstream nodes via
+ * configurable field names — does not compute them itself.
+ *
+ * Validated through 10 compression experiments on synthetic signals
+ * and real 20 kHz bearing vibration data.
+ */
+
+import { validators } from '../../core/utils/validate/index.js';
+
+// ── Supported Features ──────────────────────────────────────────────────────
+
+const SUPPORTED_STATS = [ 'deviation', 'predicted', 'significant', 'xPrev', 'tPrev' ];
+
+const STAT_DESCRIPTIONS = {
+    deviation: 'Distance from projected trajectory (always published)',
+    predicted: 'Where the trajectory says the signal should be',
+    significant: 'Boolean: did the signal stray beyond the adaptive threshold?',
+    xPrev: 'Previous tick input value (from 1-sample buffer). Non-NaN only on chi-squared gate-fire keeps. Requires bufferPrev: true.',
+    tPrev: 'Previous tick timestamp (from 1-sample buffer). Non-NaN only on chi-squared gate-fire keeps. Requires bufferPrev: true and timestampField.'
+};
+
+const SUPPORTED_CONTROL_METHODS = {
+    reset: 'Clears anchor, counter, and accumulated state',
+    enable: 'Enables node processing',
+    disable: 'Disables node processing',
+    pause: 'Pauses node processing while keeping state visible',
+    unpause: 'Resumes node processing after pause'
+};
+
+const NODE_TYPE = 'Winnow';
+
+const CAPABILITIES = {
+    description: 'Trajectory-aware significance detector — separates the grain from the chaff',
+    features: [
+        'Slope-aware deadband: projects anchor forward using the rate of change',
+        'Progressive tightening: threshold narrows as segment grows',
+        'Step detection via chi-squared innovation gate from upstream kalman1d',
+        'Trend reversal detection from upstream trend node',
+        'Adaptive threshold: K × noise floor (zero per-signal tuning)',
+        'Gap prevention: forces a point through after maxGap samples',
+        'Graceful degradation: works without slope/gate/direction (reduces to flat deadband)',
+        'Publishes deviation and predicted for downstream analytics',
+        'Optional 1-sample buffer (bufferPrev): publishes xPrev/tPrev on gate-fire keeps for spike-region anchoring'
+    ]
+};
+
+// ── Default Options ─────────────────────────────────────────────────────────
+
+export const DEFAULT_OPTIONS = {
+    K: 2,
+    tightenBase: 100,
+    maxGap: 500,
+    slopeField: 'roc',
+    noiseField: 'stdev',
+    dirField: 'trendDir',
+    gateField: 'gate',
+    chi2Threshold: 6.63,
+    bufferPrev: false
+};
+
+// ── Tunable Parameters ──────────────────────────────────────────────────────
+
+export const TUNABLE_PARAMS = [ 'K' ];
+
+// ── Getter Functions (defensive copies) ─────────────────────────────────────
+
+export const getSupportedStats = () => SUPPORTED_STATS.slice();
+export const getStatDescriptions = () => ( { ...STAT_DESCRIPTIONS } );
+export const getSupportedControlMethods = () => ( { ...SUPPORTED_CONTROL_METHODS } );
+export const getNodeType = () => NODE_TYPE;
+export const getCapabilities = () => ( {
+    ...CAPABILITIES,
+    features: CAPABILITIES.features.slice()
+} );
+
+// ── DSL Metadata ────────────────────────────────────────────────────────────
+
+const DSL_METADATA = {
+    specSchema: {
+        nodeType: {
+            type: 'string',
+            required: true,
+            value: NODE_TYPE
+        },
+        name: {
+            type: 'string',
+            required: true,
+            validator: validators.identifier,
+            error: 'name must be a valid identifier'
+        },
+        from: {
+            type: 'object',
+            required: true,
+            properties: {
+                x: {
+                    type: 'string',
+                    required: true,
+                    validator: validators.noSpaces,
+                    error: 'from.x must not contain spaces'
+                }
+            }
+        },
+        stats: {
+            type: 'object',
+            required: true,
+            minProperties: 1,
+            propertyNames: SUPPORTED_STATS,
+            propertySchema: {
+                type: 'object',
+                required: true,
+                properties: {
+                    storeAs: {
+                        type: 'string',
+                        required: true,
+                        validator: validators.identifier,
+                        error: 'storeAs must be a valid identifier'
+                    }
+                }
+            }
+        },
+
+        // ── Sensitivity ─────────────────────────────────────────
+        K: {
+            type: 'numberOrFunctionOrFieldKeyed',
+            required: false,
+            default: DEFAULT_OPTIONS.K,
+            validator: validators.positiveOrFunction,
+            error: 'K must be positive or a function'
+        },
+
+        // ── Tightening ──────────────────────────────────────────
+        tightenBase: {
+            type: 'numberOrFieldKeyed',
+            required: false,
+            default: DEFAULT_OPTIONS.tightenBase,
+            validator: validators.positive,
+            error: 'tightenBase must be positive'
+        },
+
+        // ── Gap prevention ──────────────────────────────────────
+        maxGap: {
+            type: 'numberOrFieldKeyed',
+            required: false,
+            default: DEFAULT_OPTIONS.maxGap,
+            integer: true,
+            validator: validators.positiveInteger,
+            error: 'maxGap must be a positive integer'
+        },
+
+        // ── Upstream field names ─────────────────────────────────
+        slopeField: {
+            type: 'string',
+            required: false,
+            default: DEFAULT_OPTIONS.slopeField,
+            validator: validators.noSpaces,
+            error: 'slopeField must not contain spaces'
+        },
+        noiseField: {
+            type: 'string',
+            required: false,
+            default: DEFAULT_OPTIONS.noiseField,
+            validator: validators.noSpaces,
+            error: 'noiseField must not contain spaces'
+        },
+        dirField: {
+            type: 'string',
+            required: false,
+            default: DEFAULT_OPTIONS.dirField,
+            validator: validators.noSpaces,
+            error: 'dirField must not contain spaces'
+        },
+        gateField: {
+            type: 'string',
+            required: false,
+            default: DEFAULT_OPTIONS.gateField,
+            validator: validators.noSpaces,
+            error: 'gateField must not contain spaces'
+        },
+
+        // ── Step detection threshold ─────────────────────────────
+        // numberOrFieldKeyed matches its siblings tightenBase/maxGap and the
+        // resolveScalar call in init.js, so a per-field value is accepted.
+        chi2Threshold: {
+            type: 'numberOrFieldKeyed',
+            required: false,
+            default: DEFAULT_OPTIONS.chi2Threshold,
+            validator: validators.positive,
+            error: 'chi2Threshold must be positive'
+        },
+
+        // ── 1-sample buffer for spike-region anchoring ──────────
+        bufferPrev: {
+            type: 'boolean',
+            required: false,
+            default: DEFAULT_OPTIONS.bufferPrev
+        },
+        timestampField: {
+            type: 'string',
+            required: false,
+            validator: validators.noSpaces,
+            error: 'timestampField must not contain spaces'
+        }
+    },
+
+    crossFieldValidators: [
+        {
+            fields: [ 'stats', 'bufferPrev' ],
+            validator: ( spec ) => {
+                // xPrev and tPrev stats require bufferPrev to be true
+                if ( ( spec.stats?.xPrev !== undefined || spec.stats?.tPrev !== undefined ) &&
+                     !spec.bufferPrev ) {
+                    return false;
+                }
+                return true;
+            },
+            error: 'xPrev and tPrev stats require bufferPrev: true'
+        },
+        {
+            fields: [ 'stats', 'timestampField' ],
+            validator: ( spec ) => {
+                // tPrev stat requires timestampField
+                if ( spec.stats?.tPrev !== undefined && spec.timestampField === undefined ) {
+                    return false;
+                }
+                return true;
+            },
+            error: 'tPrev stat requires timestampField to be specified'
+        }
+    ],
+
+    // Pattern: NAME_X_OUTPUTS_OPTIONS -> ( name, x, stats, options )
+    buildSpec: ( name, x, stats, options ) => ( {
+        nodeType: NODE_TYPE,
+        name,
+        from: { x },
+        stats,
+        ...options
+    } )
+};
+
+export const getDSLMetadata = () => ( { ...DSL_METADATA } );

package/src/nodes/winnow/publish-to.js

@@ -0,0 +1,51 @@
+/**
+ * @fileoverview Publish winnow outputs to the message.
+ *
+ * Copies deviation, predicted, and significant to the message for
+ * downstream consumption. All three are always computed by update()
+ * — publishTo iterates only over stats the user configured.
+ *
+ * When input validation has failed, publishes NaN for numeric stats
+ * and false for significant — downstream nodes see invalid data,
+ * not stale values.
+ */
+
+import { publishNaN } from '../../core/utils/node/index.js';
+
+const publishTo = function ( state, msg ) {
+    if ( state.disable ) return;
+
+    if ( state.inputValidationFailed ) {
+        publishNaN( state, msg );
+        return;
+    }
+
+    const stats = state.stats;
+
+    if ( stats.deviation ) {
+        msg[ stats.deviation.storeAs ] = state.deviation;
+    }
+
+    if ( stats.predicted ) {
+        msg[ stats.predicted.storeAs ] = state.predicted;
+    }
+
+    if ( stats.significant ) {
+        msg[ stats.significant.storeAs ] = state.significant;
+    }
+
+    // ── Buffer-backed stats (conditional on gate-fire keep) ─────────
+    // xPrev/tPrev are published as the buffered (k-1) values only when
+    // the current sample was kept by Check 2 (chi-squared gate fire).
+    // On non-gate keeps and on non-kept messages, they publish NaN —
+    // signalling "no lookback anchor for this sample." This prevents
+    // noisy non-gate prevValues from degrading the kfSmoothed trajectory.
+    if ( state.hasXPrev ) {
+        msg[ stats.xPrev.storeAs ] = state.keptByGate ? state.xPrev : NaN;
+    }
+    if ( state.hasTPrev ) {
+        msg[ stats.tPrev.storeAs ] = state.keptByGate ? state.tPrev : NaN;
+    }
+}; // publishTo()
+
+export default publishTo;

package/src/nodes/winnow/recompute.js

@@ -0,0 +1,14 @@
+/**
+ * @fileoverview Recompute for winnow — no-op.
+ *
+ * Winnow's state is a reference trajectory (anchor + slope) and a
+ * counter. No accumulated sums or running averages that could drift
+ * due to floating-point error. Counter overflow is safe to 2^53
+ * (at 20 kHz, that is 14 million years).
+ */
+
+const recompute = function () {
+    return true;
+}; // recompute()
+
+export default recompute;

package/src/nodes/winnow/reset.js

@@ -0,0 +1,34 @@
+/**
+ * @fileoverview Reset winnow to initial state.
+ *
+ * Clears the anchor, counter, and all accumulated state. The next
+ * message will trigger the warmup path (anchor is null), establishing
+ * a fresh reference trajectory.
+ *
+ * Called by the controller node when a downstream condition fires
+ * (e.g., after a step change detected by kalman1d's innovation gate).
+ */
+
+const reset = function ( state ) {
+    state.anchor = null;
+    state.anchorSlope = 0;
+    state.anchorTime = 0;
+    state.lastPassedAt = 0;
+    state.prevDirection = null;
+    state.counter = 0;
+    state.deviation = 0;
+    state.predicted = 0;
+    state.significant = false;
+    state.tunableErrorLogged = false;
+
+    // ── Buffer state ───────────────────────────────────────────────
+    state.bufferedX = NaN;
+    state.bufferedT = NaN;
+    state.keptByGate = false;
+    state.xPrev = NaN;
+    state.tPrev = NaN;
+
+    return true;
+}; // reset()
+
+export default reset;

package/src/nodes/winnow/update.js

@@ -0,0 +1,203 @@
+/**
+ * @fileoverview Update function for the winnow node (hot path).
+ *
+ * Winnow answers one question per message: "Has this signal strayed
+ * from where it was heading?" Five checks run in order — the first
+ * one that fires wins. If none fire, the signal is on trajectory
+ * and `significant` is set to false.
+ *
+ * The anchor is the last significant point — its value, slope, and
+ * timestamp. Between significant points, the anchor projects forward
+ * using its slope: `predicted = anchor + slope × elapsed`. The
+ * deviation from this projection, measured against the local noise
+ * floor, is the core compression signal.
+ *
+ * Zero allocation. All state mutations are scalar assignments.
+ * prevDirection updates unconditionally on every message — not only
+ * when significant (design audit finding: the closure does this on
+ * line 86, before the deadband check; missing it causes trend
+ * reversals during non-significant periods to be lost).
+ */
+
+// Extracted helper: set anchor and mark significant.
+// Called from each of the five checks. Zero allocation — scalar
+// assignments only. Reduces update() cyclomatic complexity.
+const setAnchor = function ( state, xVal, slope ) {
+    state.anchor = xVal;
+    state.anchorSlope = Number.isFinite( slope ) ? slope : 0;
+    state.anchorTime = state.counter;
+    state.lastPassedAt = state.counter;
+    state.significant = true;
+};
+
+const update = function ( state, msg ) {
+    // ── Guards ──────────────────────────────────────────────────────
+    // Disabled: skip everything. Paused: skip update, publishTo still
+    // runs (last-known values stay visible downstream).
+    if ( state.disable || state.pause ) return state;
+
+    // Every message increments the counter — used for elapsed time
+    // computation and gap prevention.
+    state.counter += 1;
+
+    // ── Read primary input ──────────────────────────────────────────
+    // The signal being tracked. Upstream median3 or sanitize should
+    // have cleaned it, but we guard anyway.
+    const xVal = msg[ state.x ];
+
+    if ( !Number.isFinite( xVal ) ) {
+        state.inputValidationFailed = true;
+        return state;
+    }
+
+    state.inputValidationFailed = false;
+
+    // ── Buffer previous tick (when enabled) ─────────────────────────
+    // Copy buffer → output, then overwrite buffer with current values.
+    // O(1), zero allocation. The output (xPrev/tPrev) holds the k-1
+    // value until publishTo decides whether to publish it.
+    if ( state.bufferPrev ) {
+        state.xPrev = state.bufferedX;
+        state.tPrev = state.bufferedT;
+        state.bufferedX = xVal;
+        if ( state.timestampField ) {
+            state.bufferedT = msg[ state.timestampField ];
+        }
+        state.keptByGate = false;
+    }
+
+    // ── Resolve tunable K ───────────────────────────────────────────
+    // K is the sensitivity multiplier: "store when deviation exceeds
+    // K × noise floor." It can be a static number or a function that
+    // reads from the message (e.g., scaleBy('signalQuality', 2)).
+    // On error, the last good value is retained (assignment fails
+    // because RHS throws before LHS is written).
+    try {
+        state.K = state.KFn( msg ); // eslint-disable-line new-cap
+        if ( state.tunableErrorLogged ) {
+            state.tunableErrorLogged = false;
+        }
+    } catch ( error ) {
+        if ( !state.tunableErrorLogged ) {
+            state.tunableErrorLogged = true;
+            console.error( `WinkComposer/${state.nodeType}: tunable threw: ${error.message}` );
+        }
+    }
+
+    // ── Read supporting fields from upstream nodes ───────────────────
+    // These are optional — winnow degrades gracefully without them:
+    //   noise undefined → warmup fires forever (safe, visible)
+    //   slope undefined → anchorSlope stays 0 (flat deadband)
+    //   direction undefined → no trend reversal triggers
+    //   gate undefined → no step detection
+    const noise = msg[ state.noiseField ];
+    const slope = msg[ state.slopeField ];
+    const direction = msg[ state.dirField ];
+    const gate = msg[ state.gateField ];
+
+    // ── Detect trend reversal ───────────────────────────────────────
+    // Check BEFORE updating prevDirection — we need to compare the
+    // current direction against what we saw last time.
+    // Exclude 'learning' transitions (trend node warmup) — these are
+    // not genuine reversals.
+    const dirChanged = (
+        state.prevDirection !== null &&
+        direction !== state.prevDirection &&
+        direction !== 'learning' &&
+        state.prevDirection !== 'learning'
+    );
+
+    // Update unconditionally — on EVERY message, not only significant
+    // ones. If we only update on significant events, we miss reversals
+    // that develop during quiet (non-significant) periods. This was
+    // caught in the design audit: the original closure updates
+    // prevTrendDir on every call at line 86, before the deadband check.
+    state.prevDirection = direction;
+
+    // ── Compute trajectory projection ───────────────────────────────
+    // The anchor projects forward using its slope at the time it was
+    // set. For a linear ramp, the projection matches the signal →
+    // deviation ≈ 0 → high compression. For a sinusoidal peak, the
+    // projection overshoots → deviation grows → significant fires.
+    const elapsed = state.counter - state.anchorTime;
+    const predicted = state.anchor === null ?
+        xVal :
+        state.anchor + ( state.anchorSlope * elapsed );
+    const deviation = Math.abs( xVal - predicted );
+
+    // Always publish deviation and predicted — useful for downstream
+    // analytics (adaptive alarming, process monitoring) regardless of
+    // whether the point is significant.
+    state.deviation = deviation;
+    state.predicted = predicted;
+
+    // ── Check 1: Warmup ─────────────────────────────────────────────
+    // Anchor not yet set (first message) or noise estimate not
+    // available (esStats needs 3 samples to publish). Accept the point
+    // and establish the anchor. Every partition starts here.
+    if ( state.anchor === null || !Number.isFinite( noise ) ) {
+        setAnchor( state, xVal, slope );
+        return state;
+    }
+
+    // ── Check 2: Step change ────────────────────────────────────────
+    // The upstream kalman1d publishes an innovation gate — a
+    // chi-squared(1) statistic that measures how surprising the
+    // current sample is relative to the model's prediction. When
+    // it exceeds the threshold (6.63 = 99% confidence), the signal
+    // did something the model could not predict. This is a structural
+    // break — accept immediately and reset the anchor.
+    if ( Number.isFinite( gate ) && gate > state.chi2Threshold ) {
+        state.keptByGate = true;
+        setAnchor( state, xVal, slope );
+        return state;
+    }
+
+    // ── Check 3: Trend reversal ─────────────────────────────────────
+    // The upstream trend node classifies the signal as learning,
+    // stable, rising, or falling. When the direction changes (rising →
+    // falling at a peak, stable → rising at a ramp onset), the signal
+    // reached an inflection point. Store this point to preserve the
+    // shape. The dirChanged flag was computed above, before
+    // prevDirection was updated.
+    if ( dirChanged ) {
+        setAnchor( state, xVal, slope );
+        return state;
+    }
+
+    // ── Check 4: Slope-aware deadband with tightening ───────────────
+    // The threshold starts at K × noise and narrows as elapsed grows.
+    // At elapsed = tightenBase: full threshold (K × noise).
+    // At elapsed = 2 × tightenBase: half threshold.
+    // At elapsed = 4 × tightenBase: quarter threshold.
+    // This progressive tightening ensures that long segments without
+    // a stored point become increasingly likely to trigger — placing
+    // intermediate points in regions that would otherwise be long gaps.
+    const tightenFactor = state.tightenBase / Math.max( state.tightenBase, elapsed );
+    const threshold = state.K * noise * tightenFactor;
+
+    if ( deviation > threshold ) {
+        setAnchor( state, xVal, slope );
+        return state;
+    }
+
+    // ── Check 5: Gap prevention ─────────────────────────────────────
+    // If none of the above fired and it has been maxGap samples since
+    // the last significant point, force one through. This prevents
+    // data gaps in storage or transmission. In streaming mode there is
+    // no "last point"; maxGap serves as the tail anchor.
+    if ( ( state.counter - state.lastPassedAt ) >= state.maxGap ) {
+        setAnchor( state, xVal, slope );
+        return state;
+    }
+
+    // ── Not significant ─────────────────────────────────────────────
+    // The signal is on trajectory. deviation and predicted are already
+    // set above — publishTo will write them downstream. The point is
+    // redundant: it can be reconstructed by linear interpolation
+    // between the previous and next significant points.
+    state.significant = false;
+    return state;
+}; // update()
+
+export default update;