@winkjs/composer 0.6.0 → 0.8.0

package/src/core/storage-manager/questdb/persist-plan.js

@@ -38,19 +38,28 @@
  *   strict mode (an `onWarning` that throws) rejects a bad row with the sender
  *   untouched.
  *
- * Async-flush failures — the no-silent-failures contract:
- * - @questdb/nodejs-client v4 declares sender.at() as async; the buffer
- *   mutation is sync but the trailing `await this.tryFlush()` can fire a
- *   network flush. When that flush fails (HTTP timeout, buffer overflow,
- *   QDB unreachable), the rows in that batch are dropped. Per composer's
- *   "no silent failures" contract, those drops MUST surface loudly.
+ * Row append rejections — the no-silent-failures contract:
+ * - @questdb/nodejs-client v4 declares sender.at() as async. The append
+ *   runs first and synchronously; the trailing `await this.tryFlush()`
+ *   does nothing, because the adapter turns the client's own flush
+ *   trigger off (ADR-029). So the promise rejects only when the append
+ *   itself threw, which for a validated row means the client's byte
+ *   ceiling (`max_buf_size`). The row never completed. Per composer's
+ *   "no silent failures" contract, that loss MUST surface.
  *
- *   Failures are routed through the `onDeliveryFailure` callback. When
- *   a caller provides one, they own the response (log, retry, alert,
- *   stop the flow). When none is provided, the catch handler **throws**
- *   the failure so it surfaces as an unhandled rejection and the
- *   process crashes loudly — better than running on with silently
- *   missing data.
+ *   The rejection is routed through the `onDeliveryFailure` callback,
+ *   with a context of the same shape as a flush report:
+ *   `{ trigger: 'append', rowsLost: 1, abandoned: false, probe: null,
+ *   tableName }`. The context is one object per insight type, shared
+ *   by every rejection, so a handler must not keep it.
+ *   When a caller provides one, they own the response (log, alert,
+ *   stop the flow). When none is provided, one classified
+ *   `DELIVERY_FAILED` console line reports it and the process keeps
+ *   running: an unattended deployment must report a lost row, not
+ *   stop on it. One handler exists per insight type and is attached
+ *   to each row's promise. The derived promise that `.catch()` creates
+ *   is the one per-row allocation this file cannot avoid while `at()`
+ *   is async.
  *
  *   This separates two concerns:
  *     - `onWarning` — soft, per-row data quality (NaN in float column,
@@ -62,8 +71,13 @@
  *       configuration, not in the data, so every row would repeat it.
  *       The field is ignored either way; the column stores the
  *       partition id.
- *     - `onDeliveryFailure` — hard, batch-level data loss. Default
- *       behaviour: throw, so production cannot run with silent drops.
+ *       The wording of the lines, the default handler, and the bound
+ *       on that default live in `skip-warnings.js`. The default is
+ *       bounded per column (ADR-029): two full lines per episode,
+ *       then one summary a minute. A user `onWarning` hears every
+ *       skip, and a throwing one is strict mode (ADR-027).
+ *     - `onDeliveryFailure` — hard data loss: the row the client
+ *       refused. Default behaviour: one DELIVERY_FAILED console line.
  *
  * `err.code` (setup-time throws per ADR-018 fail-fast setup):
  * - `INVALID_CONFIG` — buildPersistPlans called with non-function
@@ -72,9 +86,9 @@
  *   name through a throwaway client buffer — a name is not a value, so the
  *   per-message phase 1 below cannot catch it).
  *
- * `err.code` (runtime throws / unhandled rejections):
- * - `DELIVERY_FAILED` — sender.at() async flush rejected and no
- *   `onDeliveryFailure` callback was provided.
+ * Runtime console classification:
+ * - `DELIVERY_FAILED` — sender.at() rejected and no `onDeliveryFailure`
+ *   callback was provided. One `logger.error` line naming the table.
  *
  * @see docs/architecture/storage-layer.md
  * @see ADR-018
@@ -83,6 +97,7 @@
 import { SenderBufferV1 } from '@questdb/nodejs-client';
 
 import { QUEST_WRITERS, writeAsString, createFloat64Writer } from './writers.js';
+import { defaultOnWarning, makeColumnWarning, makeRowWarning } from './skip-warnings.js';
 import { wrapCallback } from '../../utils/callback-guard/index.js';
 import { logger } from '../../logger/index.js';
 
@@ -97,33 +112,6 @@ const reportCallbackFault = function ( severity, name, detail ) {
     );
 }; // reportCallbackFault()
 
-// ============================================================================
-// DEFAULT WARNING HANDLER
-// ============================================================================
-
-/**
- * Default warning handler for invalid column values.
- * Logs to console in winkComposer format.
- *
- * Validation behavior:
- * - null/undefined columns: skip column only (QuestDB stores NULL)
- * - NaN/Infinity in numeric columns: skip column only (QuestDB stores NULL)
- * - Wrong-typed column values (e.g. a number where the column is string-typed):
- *   skip column only, never coerce — the warning names the expected and
- *   received types
- * - Invalid designatedTimestamp: skip entire row
- *
- * For strict mode (throw on any invalid), provide:
- *   { onWarning: (msg) => { throw new Error(msg); } }
- *
- * Future extension: return value may control skip-row vs skip-column behavior.
- *
- * @param {string} message - Warning message describing the issue
- */
-const defaultOnWarning = function ( message ) {
-    logger.warn( `winkComposer/questdb: ${message}` );
-};
-
 /**
  * Check if column type requires numeric validation (NaN/Infinity check).
  *
@@ -166,29 +154,6 @@ ACCEPTS.bool = ( value ) => typeof value === 'boolean';
  */
 const acceptAny = () => true;
 
-/**
- * Builds the reason text for a value that failed its phase-1 acceptance
- * check. Called only on the rare skip path, so its allocations are
- * acceptable (same budget as the error returns in index.js).
- *
- * @param {*} rawValue - the rejected value
- * @param {boolean} isNumeric - whether the column type is numeric
- * @param {string} expectedType - declared column type, named in the message
- * @returns {string} plain reason text for the skip warning
- */
-const skipReason = function ( rawValue, isNumeric, expectedType ) {
-    if ( rawValue === null ) return 'null';
-    if ( rawValue === undefined ) return 'undefined';
-    if ( isNumeric && typeof rawValue === 'number' ) {
-        if ( Number.isNaN( rawValue ) ) return 'NaN';
-        // A finite number can only be rejected by the integer-required
-        // types (int64/timestamp/designated timestamp) — float64 accepts
-        // every finite number.
-        return Number.isFinite( rawValue ) ? 'non-integer' : 'non-finite';
-    }
-    return `wrong-typed (expected ${expectedType}, received ${typeof rawValue})`;
-}; // skipReason()
-
 /**
  * Asserts every ILP name a plan will write — the table name and each column
  * name — by driving them once through a throwaway client buffer, so the
@@ -264,11 +229,11 @@ const assertIlpNames = function ( tableName, columnNames ) {
  * @param {string} tablePrefix - Prefix for table names (typically assetClass.name)
  * @param {Object} [options] - Optional configuration
  * @param {function} [options.onWarning] - Soft per-row warning callback for
- *   invalid values (NaN, null, invalid timestamp). Default: console.warn.
- * @param {function} [options.onDeliveryFailure] - Hard batch-level failure
- *   callback for `sender.at()` async flush rejection (HTTP timeout, buffer
- *   overflow, QDB unreachable). Default: throw — surfaces as an unhandled
- *   rejection so production cannot run with silent row drops.
+ *   invalid values (NaN, null, invalid timestamp). Default: one
+ *   `logger.warn` line per skip, bounded per column (see the file header).
+ * @param {function} [options.onDeliveryFailure] - Called as
+ *   `( err, { tableName } )` when `sender.at()` rejects (the client's
+ *   byte ceiling). Default: one classified DELIVERY_FAILED console line.
  * @returns {Object.<string, function>} Map of insightType to persistRow function
  */
 const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
@@ -287,16 +252,19 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
         throw err;
     }
 
-    const onWarning = providedOnWarning || defaultOnWarning;
     // onWarning stays raw on purpose. A strict-mode onWarning throws,
     // and that throw is the instruction that rejects the row. ADR-027
     // keeps such callbacks — ones whose throw or return the adapter
-    // acts on — out of the guard's scope. onDeliveryFailure only
-    // notifies, so the shared guard arms it. It was validated raw
-    // above and is wrapped once here (ADR-018: a broken handler costs
-    // its own output, never the flush chain). Absent stays null, so
-    // the no-handler DELIVERY_FAILED escape hatch below keeps its
-    // exact meaning.
+    // acts on — out of the guard's scope. The skip sites reach it
+    // through the pre-built warnings (`skip-warnings.js`); this raw
+    // handler serves the once-per-type mismatch warning below.
+    const onWarning = providedOnWarning || defaultOnWarning;
+    //
+    // onDeliveryFailure only notifies, so the shared guard arms it. It
+    // was validated raw above and is wrapped once here (ADR-018: a
+    // broken handler costs its own output, never the flush chain).
+    // Absent stays null, so the no-handler DELIVERY_FAILED escape
+    // hatch below keeps its exact meaning.
     const onDeliveryFailure = wrapCallback( providedOnDeliveryFailure || null, {
         name: 'onDeliveryFailure', severity: 'red', report: reportCallbackFault
     } );
@@ -316,10 +284,12 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
         // column itself, from the partition id. A persisted column with
         // the same name would make the CREATE TABLE ask for 'assetId'
         // twice. That happens whether the name appears in the columns
-        // list or as the designated timestamp. Fail fast here and name
-        // the fix; otherwise QuestDB answers at table creation with its
-        // raw "Duplicate column" error. A dictionary column named
-        // 'assetId' that no insightType persists stays legal.
+        // list or as the designated timestamp.
+        //
+        // Fail fast here and name the fix. Otherwise QuestDB answers at
+        // table creation with its raw "Duplicate column" error. A
+        // dictionary column named 'assetId' that no insightType
+        // persists stays legal.
         if ( persistedColumnNames.includes( 'assetId' ) || designatedTimestamp === 'assetId' ) {
             const err = new Error(
                 `winkComposer/questdb: insightType '${insightTypeName}' uses reserved column name 'assetId' — ` +
@@ -330,31 +300,34 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
             throw err;
         }
 
-        // Pre-compile writer steps
-        // designatedTimestamp is handled separately via sender.at()
-        // Other columns use their type-specific writers
+        // Pre-compile the writer steps. The designated timestamp is
+        // handled separately, via sender.at(). Every other column uses
+        // its type-specific writer.
         const stepNames = [];
         const stepWriters = [];
-        const stepIsNumeric = [];
-        const stepTypes = [];
         const stepAccepts = [];
+        // One skip warning per column, pre-built here so the per-row
+        // path only calls it (see the file header for the bound).
+        const stepWarnings = [];
 
         for ( let j = 0; j < persistedColumnNames.length; j += 1 ) {
             const columnName = persistedColumnNames[ j ];
 
-            // Skip designatedTimestamp - handled separately via sender.at()
+            // Skip the designated timestamp. It is handled separately, via
+            // sender.at().
             if ( columnName !== designatedTimestamp ) {
                 const columnSpec = columns[ columnName ];
                 const columnType = columnSpec ? columnSpec.type : 'string';
 
                 stepNames.push( columnName );
-                stepIsNumeric.push( isNumericType( columnType ) );
-                stepTypes.push( columnType );
                 stepAccepts.push( ACCEPTS[ columnType ] || acceptAny );
+                stepWarnings.push( makeColumnWarning(
+                    providedOnWarning, insightTypeName, columnName, isNumericType( columnType ), columnType
+                ) );
 
-                // float64 columns use resolution-aware writer factory
-                // Note: columnSpec is guaranteed to exist here since columnType='float64'
-                // requires columnSpec.type to be 'float64' (line 65)
+                // float64 columns use the resolution-aware writer factory.
+                // columnSpec exists here: columnType is 'float64' only when
+                // columnSpec.type is 'float64'.
                 if ( columnType === 'float64' ) {
                     stepWriters.push( createFloat64Writer( columnSpec.resolution ) );
                 } else {
@@ -365,6 +338,28 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
 
         const stepCount = stepNames.length;
         const tableName = tablePrefix + '_' + insightTypeName;
+        const warnRowSkipped = makeRowWarning( providedOnWarning, insightTypeName, designatedTimestamp );
+
+        // One rejection handler and one shared context per insight
+        // type, built here so the per-row path allocates nothing. See
+        // the file header for the context's shape and what a rejection
+        // means.
+        const appendContext = {
+            trigger: 'append',
+            rowsLost: 1,
+            abandoned: false,
+            probe: null,
+            tableName
+        };
+        const onAppendRejected = function ( err ) {
+            if ( onDeliveryFailure ) {
+                onDeliveryFailure( err, appendContext );
+                return;
+            }
+            logger.error(
+                `winkComposer/questdb: row append failed for table '${tableName}' [DELIVERY_FAILED]: ${err.message}`
+            );
+        }; // onAppendRejected()
         // Fail-fast at startup: a bad table or column name would otherwise
         // throw inside the client mid-row at write time.
         assertIlpNames( tableName, stepNames );
@@ -378,36 +373,31 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
         // One report carries all the information.
         let assetIdMismatchWarned = false;
 
-        // Create closure that captures pre-resolved references
-        // Interface: persistRow(sender, message, partitionId) -> boolean
-        // Returns true when a row was opened and completed on the sender,
-        // false when phase 1 skipped the whole row (the sender was never
-        // touched). The caller's buffered-row accounting keys off this —
-        // a skipped row must not count as buffered.
-        // Note: partitionId is internal name, written as 'assetId' column to QuestDB
+        // Create the closure that captures the pre-resolved references.
+        // Interface: persistRow(sender, message, partitionId) -> boolean.
+        // It returns true when a row was opened and completed on the
+        // sender. It returns false when phase 1 skipped the whole row,
+        // so the sender was never touched.
+        //
+        // The caller's buffered-row accounting keys off that return. A
+        // skipped row must not count as buffered. Note that partitionId
+        // is the internal name. It is written as the 'assetId' column
+        // in QuestDB.
         plansByInsightType[ insightTypeName ] = function ( sender, message, partitionId ) {
             // ---- Phase 1: validate. No sender calls — nothing irreversible
             // happens until every value has been checked (see file header).
 
-            // designatedTimestamp first - if invalid, skip entire row
+            // The designated timestamp first. When it is missing or not
+            // an integer, skip the entire row. Integer or bigint matches
+            // the client's own .at() validation. Probe-verified: at(
+            // ...000.5, 'ms' ) throws "Designated timestamp must be an
+            // integer or BigInt". It throws AFTER the whole row is
+            // written, so catching it here is the only place the row
+            // survives intact. A null or undefined value fails the same
+            // check, and the warning names which it was.
             const tsValue = message[ designatedTimestamp ];
-            if ( tsValue === undefined || tsValue === null ) {
-                onWarning(
-                    `designatedTimestamp '${designatedTimestamp}' is ${tsValue === null ? 'null' : 'undefined'} ` +
-                    `in insightType '${insightTypeName}' (asset: ${partitionId}) - row skipped`
-                );
-                return false;
-            }
-            // Integer-or-bigint, matching the client's own .at() validation
-            // (probe-verified: at( ...000.5, 'ms' ) throws "Designated
-            // timestamp must be an integer or BigInt" — and it throws AFTER
-            // the whole row is written, so catching it here is the only
-            // place the row survives intact).
             if ( !Number.isInteger( tsValue ) && typeof tsValue !== 'bigint' ) {
-                onWarning(
-                    `designatedTimestamp '${designatedTimestamp}' is ${skipReason( tsValue, true, 'timestamp' )} ` +
-                    `in insightType '${insightTypeName}' (asset: ${partitionId}) - row skipped`
-                );
+                warnRowSkipped( tsValue, partitionId );
                 return false;
             }
 
@@ -415,9 +405,10 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
             // That column always stores the partition id. A record value
             // that differs is almost always one mistake: trying to relabel
             // identity in the record. It is reported once per insightType,
-            // never silently dropped (ADR-018 no-silent-failures). The
-            // flag advances only after onWarning returns. So under strict
-            // mode, where onWarning throws, every mismatched row is
+            // never silently dropped (ADR-018 no-silent-failures).
+            //
+            // The flag advances only after onWarning returns. So under
+            // strict mode, where onWarning throws, every mismatched row is
             // rejected here with the sender untouched.
             if ( !assetIdMismatchWarned && message.assetId !== undefined && message.assetId !== partitionId ) {
                 onWarning(
@@ -429,22 +420,20 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
             }
 
             // Check every column value against its declared type. A failed
-            // check marks the column for a silent skip in phase 2 (QuestDB
-            // stores NULL) — the same treatment null already gets — and
-            // warns HERE, before the row opens, so an onWarning that throws
-            // (documented strict mode) rejects the row with the sender
-            // untouched instead of wedging it mid-row. Composer's NaN
-            // propagation ends here exactly as before: a NaN in a numeric
-            // column fails its acceptance check and lands as a NULL column
-            // while the row survives.
+            // check marks the column for a silent skip in phase 2, where
+            // QuestDB stores NULL. That is the treatment null already gets.
+            // The warning fires HERE, before the row opens. So an onWarning
+            // that throws (documented strict mode) rejects the row with the
+            // sender untouched instead of wedging it mid-row.
+            //
+            // Composer's NaN propagation ends here exactly as before. A
+            // NaN in a numeric column fails its acceptance check and lands
+            // as a NULL column while the row survives.
             for ( let k = 0; k < stepCount; k += 1 ) {
                 const rawValue = message[ stepNames[ k ] ];
                 stepValueOk[ k ] = ( rawValue !== null ) && ( rawValue !== undefined ) && stepAccepts[ k ]( rawValue );
                 if ( !stepValueOk[ k ] ) {
-                    onWarning(
-                        `column '${stepNames[ k ]}' is ${skipReason( rawValue, stepIsNumeric[ k ], stepTypes[ k ] )} ` +
-                        `in insightType '${insightTypeName}' (asset: ${partitionId}) — column skipped`
-                    );
+                    stepWarnings[ k ]( rawValue, partitionId );
                 }
             }
 
@@ -461,41 +450,13 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
                 // about in phase 1 — before the row opened.
             }
 
-            // Designated timestamp (ends the row in ILP).
-            //
-            // Async flush failure handling — the no-silent-failures contract:
-            //
-            // sender.at() is async — the buffer mutation is sync but the
-            // trailing `await this.tryFlush()` may fire a network flush.
-            // When that flush fails (HTTP timeout, buffer overflow at the
-            // client's max_buf_size, QDB unreachable), the rows in that
-            // batch are dropped. Composer must NOT silently lose data.
-            //
-            // Routing rules:
-            //   - If a caller provided `onDeliveryFailure`, route the
-            //     failure to it. The caller owns the response (log,
-            //     retry-budget, alert, stop the flow).
-            //   - Otherwise, throw a classified `DELIVERY_FAILED` error.
-            //     Inside a .catch() handler this surfaces as an
-            //     unhandled rejection — Node logs it loudly and (15+)
-            //     terminates the process. Loud failure beats silent
-            //     loss every time.
+            // Designated timestamp (ends the row in ILP). sender.at() is
+            // async; its promise rejects when the append itself threw
+            // (see the file header). The prebuilt handler routes the
+            // rejection; a mock at() that returns no promise is skipped.
             const atResult = sender.at( tsValue, 'ms' );
             if ( atResult && typeof atResult.catch === 'function' ) {
-                atResult.catch( ( err ) => {
-                    if ( onDeliveryFailure ) {
-                        onDeliveryFailure( err, { tableName } );
-                        return;
-                    }
-                    const failure = new Error(
-                        `winkComposer/questdb: silent data loss — sender.at() async flush failed for table '${tableName}': ${err.message}. ` +
-                        'Rows in this batch were dropped. Per the no-silent-failures contract this is a hard failure. ' +
-                        'Provide an `onDeliveryFailure` callback in the storage config to handle these explicitly.'
-                    );
-                    failure.code = 'DELIVERY_FAILED';
-                    failure.cause = err;
-                    throw failure;
-                } );
+                atResult.catch( onAppendRejected );
             }
 
             return true;
@@ -509,4 +470,4 @@ const buildPersistPlans = function ( assetClass, tablePrefix, options ) {
 // EXPORTS
 // ============================================================================
 
-export { buildPersistPlans, defaultOnWarning };
+export { buildPersistPlans };