@winkjs/composer 0.6.0 → 0.7.0

package/src/core/storage-manager/questdb/skip-warnings.js

@@ -0,0 +1,180 @@
+// core/storage-manager/questdb/skip-warnings.js
+
+/**
+ * @fileoverview How a skipped value is reported.
+ *
+ * The persist plan checks every value before a row opens. A value that
+ * fails its check is skipped, and the skip is reported through
+ * `onWarning`: a column skip lands the column as NULL, a bad designated
+ * timestamp skips the whole row. This module builds the functions the
+ * plan calls at those two sites. It owns the wording of the lines, the
+ * default handler, and the bound on that default.
+ *
+ * Why the default is bounded (ADR-029, the same rule as the flush loss
+ * line). A dead sensor publishes NaN in one column on every row. That
+ * is one warning line per row for as long as the sensor is dead: one
+ * line a second for a whole night, in a log nobody can read. So the
+ * default handler is bounded per column. The first two skips of an
+ * episode print in full. Later skips are counted, and one summary
+ * line prints per minute with the count, the latest reason, and the
+ * latest asset. A quiet minute on that column ends the episode. Rows
+ * skipped for a bad designated timestamp share one bound per insight
+ * type.
+ *
+ * The bound is on the default only. A user `onWarning` hears every
+ * skip with the full line, because the flow chose to listen. A user
+ * handler that throws is strict mode (ADR-027): the throw is the
+ * instruction that rejects the row, so it must reach the plan
+ * unchanged. Neither is bounded.
+ *
+ * Nothing here runs on the happy path. The plan calls a warning only
+ * when a value failed its check, and a counted skip allocates nothing:
+ * the line is rendered only when it prints.
+ *
+ * @see ADR-027
+ * @see ADR-029
+ */
+
+import { createLineBound } from '../../utils/line-rate/index.js';
+import { logger } from '../../logger/index.js';
+
+/**
+ * The bound on the default warning (see the file header). The values
+ * match the flush loss line in flush-engine.js, so an operator reads
+ * one rhythm.
+ */
+const FULL_WARNING_LINES_PER_EPISODE = 2;
+const WARNING_SUMMARY_INTERVAL_MS = 60000;
+
+/**
+ * Default warning handler for invalid column values: one `logger.warn`
+ * line in winkComposer format. The skip sites reach it through the
+ * per-column line bound (see the file header), so under a streak it
+ * prints the first two skips in full and then one summary per minute.
+ *
+ * Validation behavior:
+ * - null/undefined columns: skip column only (QuestDB stores NULL)
+ * - NaN/Infinity in numeric columns: skip column only (QuestDB stores NULL)
+ * - Wrong-typed column values (e.g. a number where the column is string-typed):
+ *   skip column only, never coerce — the warning names the expected and
+ *   received types
+ * - Invalid designatedTimestamp: skip entire row
+ *
+ * For strict mode (throw on any invalid), provide:
+ *   { onWarning: (msg) => { throw new Error(msg); } }
+ *
+ * Future extension: return value may control skip-row vs skip-column behavior.
+ *
+ * @param {string} message - Warning message describing the issue
+ */
+const defaultOnWarning = function ( message ) {
+    logger.warn( `winkComposer/questdb: ${message}` );
+}; // defaultOnWarning()
+
+/**
+ * Builds the reason text for a value that failed its phase-1 acceptance
+ * check. Called only on the rare skip path, so its allocations are
+ * acceptable (same budget as the error returns in index.js).
+ *
+ * @param {*} rawValue - the rejected value
+ * @param {boolean} isNumeric - whether the column type is numeric
+ * @param {string} expectedType - declared column type, named in the message
+ * @returns {string} plain reason text for the skip warning
+ */
+const skipReason = function ( rawValue, isNumeric, expectedType ) {
+    if ( rawValue === null ) return 'null';
+    if ( rawValue === undefined ) return 'undefined';
+    if ( isNumeric && typeof rawValue === 'number' ) {
+        if ( Number.isNaN( rawValue ) ) return 'NaN';
+        // A finite number can only be rejected by the integer-required
+        // types (int64/timestamp/designated timestamp) — float64 accepts
+        // every finite number.
+        return Number.isFinite( rawValue ) ? 'non-integer' : 'non-finite';
+    }
+    return `wrong-typed (expected ${expectedType}, received ${typeof rawValue})`;
+}; // skipReason()
+
+/**
+ * Builds the warning function for one skip site. With a user
+ * `onWarning`, every skip reaches it with the full line. With the
+ * default, the line bound applies. Both renderers run only when a line
+ * prints, so a counted skip allocates nothing.
+ *
+ * @param {Function|undefined} providedOnWarning - The user's handler, or undefined for the default
+ * @param {Function} fullLine - `( rawValue, partitionId ) => string`, without the brand prefix
+ * @param {Function} summaryLine - `( count, seconds, rawValue, partitionId ) => string`, without the brand prefix
+ * @returns {Function} `( rawValue, partitionId ) => void`
+ */
+const makeSkipWarning = function ( providedOnWarning, fullLine, summaryLine ) {
+    if ( providedOnWarning !== undefined ) {
+        return function ( rawValue, partitionId ) {
+            providedOnWarning( fullLine( rawValue, partitionId ) );
+        };
+    }
+    return createLineBound( {
+        fullLines: FULL_WARNING_LINES_PER_EPISODE,
+        intervalMs: WARNING_SUMMARY_INTERVAL_MS,
+        printFull: function ( rawValue, partitionId ) {
+            defaultOnWarning( fullLine( rawValue, partitionId ) );
+        },
+        printSummary: function ( count, seconds, rawValue, partitionId ) {
+            defaultOnWarning( summaryLine( count, seconds, rawValue, partitionId ) );
+        }
+    } );
+}; // makeSkipWarning()
+
+/**
+ * Builds the skip warning for one column of one insight type.
+ *
+ * @param {Function|undefined} providedOnWarning - The user's handler, or undefined for the default
+ * @param {string} insightTypeName - Named in every line
+ * @param {string} columnName - The column the skip is about
+ * @param {boolean} isNumeric - Whether the column type is numeric, for the reason text
+ * @param {string} columnType - The declared column type, for the reason text
+ * @returns {Function} `( rawValue, partitionId ) => void`
+ */
+const makeColumnWarning = function ( providedOnWarning, insightTypeName, columnName, isNumeric, columnType ) {
+    return makeSkipWarning(
+        providedOnWarning,
+        ( rawValue, partitionId ) => (
+            `column '${columnName}' is ${skipReason( rawValue, isNumeric, columnType )} ` +
+            `in insightType '${insightTypeName}' (asset: ${partitionId}) — column skipped`
+        ),
+        ( count, seconds, rawValue, partitionId ) => (
+            `column '${columnName}' skipped in ${count} more row(s) of insightType '${insightTypeName}' ` +
+            `in the last ${seconds} s (latest: ${skipReason( rawValue, isNumeric, columnType )}, asset: ${partitionId})`
+        )
+    );
+}; // makeColumnWarning()
+
+/**
+ * Builds the row-skip warning of one insight type, for a designated
+ * timestamp that is missing or not an integer.
+ *
+ * @param {Function|undefined} providedOnWarning - The user's handler, or undefined for the default
+ * @param {string} insightTypeName - Named in every line
+ * @param {string} designatedTimestamp - The timestamp column, named in every line
+ * @returns {Function} `( tsValue, partitionId ) => void`
+ */
+const makeRowWarning = function ( providedOnWarning, insightTypeName, designatedTimestamp ) {
+    return makeSkipWarning(
+        providedOnWarning,
+        ( tsValue, partitionId ) => (
+            `designatedTimestamp '${designatedTimestamp}' is ${skipReason( tsValue, true, 'timestamp' )} ` +
+            `in insightType '${insightTypeName}' (asset: ${partitionId}) - row skipped`
+        ),
+        ( count, seconds, tsValue, partitionId ) => (
+            `${count} more row(s) of insightType '${insightTypeName}' skipped for designatedTimestamp ` +
+            `'${designatedTimestamp}' in the last ${seconds} s (latest: ${skipReason( tsValue, true, 'timestamp' )}, ` +
+            `asset: ${partitionId})`
+        )
+    );
+}; // makeRowWarning()
+
+export {
+    defaultOnWarning,
+    makeColumnWarning,
+    makeRowWarning,
+    FULL_WARNING_LINES_PER_EPISODE,
+    WARNING_SUMMARY_INTERVAL_MS
+};

package/src/core/test-utils/tcp-proxy.js

@@ -14,6 +14,15 @@
  * "service unreachable" without coupling tests to docker daemon
  * access. The pattern is identical for any TCP-based service.
  *
+ * `startBlackHole()` models the other dead-endpoint shape: a server
+ * that accepts the connection and never answers. The QuestDB exit test
+ * uses it to hold a request on the wire while the process shuts down.
+ *
+ * `startHttpResponder()` models a service that is up but answers an
+ * error. It forwards to the real service or answers a status code,
+ * switched on the live server so no request is reset. The HTTP-error
+ * hardening spec uses it.
+ *
  * Cross-cutting test infrastructure mirrors `src/core/source-manager/test-harness/`
  * — both live under `src/core/` because they're shared across adapter
  * modules.
@@ -27,6 +36,7 @@
  * @module test-utils/tcp-proxy
  */
 
+import http from 'node:http';
 import net from 'node:net';
 
 /**
@@ -58,6 +68,14 @@ export const startProxy = function ( listenPort, targetPort ) {
             clientSocket.pipe( targetSocket );
             targetSocket.pipe( clientSocket );
         } );
+        // The target side is tracked too. A destroyed client socket does
+        // not end its pipe partner. Without this, the proxy's own sockets
+        // to the service outlive `stopProxy()` and hold the process open.
+        // The QuestDB exit test found that.
+        sockets.add( targetSocket );
+        targetSocket.on( 'close', function () {
+            sockets.delete( targetSocket );
+        } );
         // See file-header note on swallowed errors.
         clientSocket.on( 'error', function () {} );
         targetSocket.on( 'error', function () {} );
@@ -76,7 +94,8 @@ export const startProxy = function ( listenPort, targetPort ) {
 
 /**
  * Stop a proxy started via `startProxy()`. Force-destroys every
- * tracked client socket first (sending RST so clients see the
+ * tracked socket first, both the client side and the proxy's own
+ * side toward the service (sending RST so clients see the
  * disconnect immediately, not at next keepalive timeout), then
  * waits for the server to close cleanly.
  *
@@ -104,3 +123,177 @@ export const stopProxy = function ( server ) {
         } );
     } );
 };
+
+/**
+ * Start a black hole on `port`: a server that accepts every connection
+ * and never answers. It reads and discards whatever the client sends,
+ * so the client's write completes and the client then waits for a
+ * response that never comes. That is the shape of an endpoint whose
+ * process is alive but wedged. A closed port has a different shape:
+ * it refuses at once.
+ *
+ * The hole has a second mode. After `forwardTo( targetPort )` every
+ * new connection is relayed to that port, like `startProxy()`. After
+ * `swallow()` the hole is back: new connections are swallowed, and a
+ * connection that was being relayed goes quiet from that instant. Its
+ * service side is dropped and its client side stays open, so a
+ * request already on a kept-alive socket waits for an answer that
+ * never comes. The switch happens on the live server, so the port
+ * never closes and no request on the wire is reset. A port bounce
+ * would reset one, and the client reports a reset as a socket error,
+ * not as a hang.
+ *
+ * Returns the `net.Server` once it is listening, with `forwardTo()`
+ * and `swallow()` attached; pass it to `stopProxy()` to tear down,
+ * sockets included.
+ *
+ * @param {number} port
+ * @returns {Promise<net.Server>}
+ */
+export const startBlackHole = function ( port ) {
+    const sockets = new Set();
+    // Client socket → service socket, for the connections being relayed.
+    const relayed = new Map();
+    let targetPort = null;
+
+    const track = function ( socket ) {
+        sockets.add( socket );
+        socket.on( 'close', function () {
+            sockets.delete( socket );
+        } );
+        // See file-header note on swallowed errors.
+        socket.on( 'error', function () {} );
+    };
+
+    const relay = function ( clientSocket, servicePort ) {
+        const targetSocket = net.connect( servicePort, '127.0.0.1', function () {
+            clientSocket.pipe( targetSocket );
+            targetSocket.pipe( clientSocket );
+        } );
+        track( targetSocket );
+        relayed.set( clientSocket, targetSocket );
+        clientSocket.on( 'close', function () {
+            relayed.delete( clientSocket );
+            targetSocket.destroy();
+        } );
+    };
+
+    const server = net.createServer( function ( clientSocket ) {
+        track( clientSocket );
+        if ( targetPort === null ) {
+            // Consume the request bytes so the client's write completes.
+            clientSocket.resume();
+            return;
+        }
+        relay( clientSocket, targetPort );
+    } );
+    server.forwardTo = function ( servicePort ) {
+        targetPort = servicePort;
+    };
+    server.swallow = function () {
+        targetPort = null;
+        for ( const [ clientSocket, targetSocket ] of relayed ) {
+            clientSocket.unpipe( targetSocket );
+            targetSocket.unpipe( clientSocket );
+            targetSocket.destroy();
+            clientSocket.resume();
+        }
+        relayed.clear();
+    };
+    // eslint-disable-next-line no-underscore-dangle
+    server._sockets = sockets;
+    return new Promise( function ( resolve ) {
+        server.listen( port, '127.0.0.1', function () {
+            resolve( server );
+        } );
+    } );
+};
+
+/**
+ * Start an HTTP responder on `port`. It has two modes, switched on the
+ * live server, so the port never closes between them. In answer mode
+ * it reads every request and answers it with `statusCode` and a short
+ * plain-text body. That is the shape of a service that is up but
+ * refuses the request, such as a 400 for a malformed row or a 500 for
+ * an internal failure. In forward mode it relays each request to a
+ * real service on `127.0.0.1:<upstreamPort>` and returns that
+ * service's answer. Every answer closes the connection.
+ *
+ * Why a mode switch and not a port bounce: closing a server destroys
+ * its sockets, so a request on the wire at that instant reads a reset
+ * instead of an answer. A switch on a live server resets nothing. Each
+ * request gets the mode that was active when it arrived.
+ *
+ * The QuestDB client treats some codes as retryable and the rest as
+ * final; the HTTP-error hardening spec pins both against this
+ * responder. Returns the server once it is listening, with two extra
+ * methods: `answerWith( statusCode )` and `forwardTo( upstreamPort )`.
+ * Pass the server to `stopProxy()` to tear down, sockets included.
+ *
+ * @param {number} port
+ * @param {number} statusCode - answered until `forwardTo()` is called
+ * @returns {Promise<http.Server>}
+ */
+export const startHttpResponder = function ( port, statusCode ) {
+    const sockets = new Set();
+    let mode = { answer: statusCode, forward: null };
+
+    const answer = function ( req, res, code ) {
+        req.resume();
+        req.on( 'end', function () {
+            res.writeHead( code, { 'Content-Type': 'text/plain', Connection: 'close' } );
+            res.end( `responder answered ${code}` );
+        } );
+    };
+
+    const forward = function ( req, res, upstreamPort ) {
+        const headers = Object.assign( {}, req.headers );
+        delete headers.connection;
+        const upstream = http.request( {
+            host: '127.0.0.1',
+            port: upstreamPort,
+            method: req.method,
+            path: req.url,
+            headers,
+            agent: false
+        }, function ( upstreamRes ) {
+            res.writeHead( upstreamRes.statusCode, Object.assign( {}, upstreamRes.headers, { connection: 'close' } ) );
+            upstreamRes.pipe( res );
+        } );
+        // See file-header note on swallowed errors.
+        upstream.on( 'error', function () {
+            res.destroy();
+        } );
+        req.pipe( upstream );
+    };
+
+    const server = http.createServer( function ( req, res ) {
+        const current = mode;
+        if ( current.forward === null ) {
+            answer( req, res, current.answer );
+            return;
+        }
+        forward( req, res, current.forward );
+    } );
+    server.answerWith = function ( code ) {
+        mode = { answer: code, forward: null };
+    };
+    server.forwardTo = function ( upstreamPort ) {
+        mode = { answer: null, forward: upstreamPort };
+    };
+    server.on( 'connection', function ( socket ) {
+        sockets.add( socket );
+        socket.on( 'close', function () {
+            sockets.delete( socket );
+        } );
+        // See file-header note on swallowed errors.
+        socket.on( 'error', function () {} );
+    } );
+    // eslint-disable-next-line no-underscore-dangle
+    server._sockets = sockets;
+    return new Promise( function ( resolve ) {
+        server.listen( port, '127.0.0.1', function () {
+            resolve( server );
+        } );
+    } );
+};