ga4-export-fixer 0.5.2-dev.3 → 0.5.2-dev.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.5.2-dev.3",
+  "version": "0.5.2-dev.5",
   "description": "",
   "main": "index.js",
   "files": [
@@ -17,13 +17,14 @@
     "createTable.js"
   ],
   "scripts": {
-    "test": "node tests/ga4EventsEnhanced.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js",
+    "test": "node tests/ga4EventsEnhanced.test.js && node tests/assertions.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js",
     "test:summary": "node tests/testRunner.js",
     "test:docs": "node tests/documentation.test.js",
     "test:preops": "node tests/preOperations.test.js",
     "test:events": "node tests/ga4EventsEnhanced.test.js",
     "test:merge": "node tests/mergeSQLConfigurations.test.js",
     "test:validation": "node tests/inputValidation.test.js",
+    "test:assertions": "node tests/assertions.test.js",
     "test:createTable": "node tests/createTable.test.js",
     "test:integration": "node tests/integration/integration.test.js",
     "release:dev": "./scripts/release-dev.sh",

package/tables/ga4EventsEnhanced/assertions/index.js

@@ -0,0 +1,5 @@
+const { generateItemRevenueAssertionSql } = require('./itemRevenue.js');
+
+module.exports = {
+    itemRevenue: generateItemRevenueAssertionSql,
+};

package/tables/ga4EventsEnhanced/assertions/itemRevenue.js

@@ -0,0 +1,135 @@
+const helpers = require('../../../helpers/index.js');
+const utils = require('../../../utils.js');
+const { ga4EventsEnhancedConfig } = require('../config.js');
+const { validateEnhancedEventsConfig } = require('../validation.js');
+
+const defaultConfig = { ...ga4EventsEnhancedConfig };
+
+// Ecommerce events that carry item data (excluding refund — refunds reverse revenue
+// and are handled separately in some pipelines, but item_revenue on refund rows
+// should still reconcile 1:1 between enhanced and raw).
+const ecommerceEvents = helpers.ga4EcommerceEvents.map(e => `'${e}'`).join(', ');
+
+/**
+ * Builds a _table_suffix date filter for the assertion's raw-side query.
+ *
+ * Uses the low-level ga4ExportDateFilter() helper per enabled export type
+ * with a fixed 5-day lookback window. This is intentionally separate from
+ * the pipeline's ga4ExportDateFilters() which depends on incremental state
+ * and BigQuery pre-operation variables.
+ *
+ * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
+ * @returns {string} SQL fragment for a WHERE clause
+ */
+const buildAssertionDateFilter = (includedExportTypes) => {
+    const start = 'date_sub(current_date(), interval 5 day)';
+    const end = 'current_date()';
+
+    const filters = [
+        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
+        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
+        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
+    ].filter(Boolean);
+
+    return filters.join(' or ');
+};
+
+/**
+ * Generates a SQL assertion query that reconciles item_revenue between the
+ * enhanced events table and the raw GA4 export data.
+ *
+ * The query compares item_revenue grouped by (event_date, item_id) for the
+ * last 5 days of final data. Returns mismatched rows — 0 rows means the
+ * assertion passes.
+ *
+ * @param {string} tableRef - Fully qualified reference to the enhanced table
+ *   (e.g., ctx.ref('ga4_events_enhanced_123456789') in Dataform, or a backtick-quoted string).
+ * @param {Object} mergedConfig - Merged table configuration (after merge + validation).
+ * @returns {string} SQL query returning violating rows
+ */
+const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
+    // excluded events filter (same logic as the enhanced table pipeline)
+    const excludedEvents = mergedConfig.excludedEvents;
+    const excludedEventsSQL = excludedEvents.length > 0
+        ? `and event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
+        : '';
+
+    // data_is_final condition for the raw side
+    const dataIsFinalCondition = helpers.isFinalData(
+        mergedConfig.dataIsFinal.detectionMethod,
+        mergedConfig.dataIsFinal.dayThreshold
+    );
+
+    // date filter for the raw side (per-export-type, fixed 5-day window)
+    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes);
+
+    return `with enhanced_revenue as (
+    select
+        event_date,
+        item.item_id,
+        sum(item.item_revenue) as total_item_revenue,
+        count(*) as item_count
+    from
+        ${tableRef},
+        unnest(items) as item
+    where
+        data_is_final = true
+        and event_date >= date_sub(current_date(), interval 5 day)
+        and event_name in (${ecommerceEvents})
+    group by event_date, item.item_id
+),
+raw_revenue as (
+    select
+        cast(event_date as date format 'YYYYMMDD') as event_date,
+        item.item_id,
+        sum(item.item_revenue) as total_item_revenue,
+        count(*) as item_count
+    from
+        ${mergedConfig.sourceTable},
+        unnest(items) as item
+    where
+        (${dateFilter})
+        ${excludedEventsSQL}
+        and event_name in (${ecommerceEvents})
+        and ${dataIsFinalCondition}
+        and cast(event_date as date format 'YYYYMMDD') >= date_sub(current_date(), interval 5 day)
+    group by event_date, item.item_id
+)
+select
+    coalesce(e.event_date, r.event_date) as event_date,
+    coalesce(e.item_id, r.item_id) as item_id,
+    e.total_item_revenue as enhanced_revenue,
+    r.total_item_revenue as raw_revenue,
+    e.item_count as enhanced_count,
+    r.item_count as raw_count
+from
+    enhanced_revenue e
+full outer join
+    raw_revenue r using(event_date, item_id)
+where
+    round(coalesce(e.total_item_revenue, 0), 2) != round(coalesce(r.total_item_revenue, 0), 2)
+    or e.item_count != r.item_count
+    or e.event_date is null
+    or r.event_date is null`;
+};
+
+/**
+ * Generates an item_revenue reconciliation assertion SQL query.
+ *
+ * Merges the provided config with defaults, validates, then generates a SQL
+ * query comparing item_revenue between the enhanced table and raw export data.
+ *
+ * @param {string} tableRef - Fully qualified reference to the enhanced table.
+ * @param {Object} config - User-provided table configuration.
+ * @returns {string} SQL query returning violating rows (0 rows = pass)
+ */
+const generateItemRevenueAssertionSql = (tableRef, config) => {
+    if (!tableRef || typeof tableRef !== 'string' || !tableRef.trim()) {
+        throw new Error('assertions.itemRevenue: tableRef is required and must be a non-empty string (e.g., ctx.ref(\'table_name\') or \'`project.dataset.table`\').');
+    }
+    const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
+    validateEnhancedEventsConfig(mergedConfig, { skipDataformContextFields: true });
+    return _generateItemRevenueAssertionSql(tableRef, mergedConfig);
+};
+
+module.exports = { generateItemRevenueAssertionSql };

package/tables/ga4EventsEnhanced/index.js

@@ -7,6 +7,7 @@ const { validateEnhancedEventsConfig } = require('./validation.js');
 const documentation = require('../../documentation.js');
 const { createTable } = require('../../createTable.js');
 const { getTableDescriptionSections } = require('./tableDescription.js');
+const assertions = require('./assertions/index.js');
 
 // Column metadata for the GA4 Events Enhanced table
 const columnMetadata = {
@@ -161,6 +162,9 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
 
     // item list attribution config
     const itemListAttribution = mergedConfig.itemListAttribution;
+    const ecommerceEventsFilter = itemListAttribution
+        ? helpers.ga4EcommerceEvents.filter(e => e !== 'refund').map(e => `'${e}'`).join(', ')
+        : null;
 
     // auto-adjust bufferDays for time-based item list attribution lookback
     const effectiveBufferDays = (itemListAttribution && itemListAttribution.lookbackType === 'TIME')
@@ -225,11 +229,12 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
             // ecommerce
             ecommerce: helpers.fixEcommerceStruct('ecommerce'),
             items: 'items',
-            // unique row id for item list attribution join.
-            // row_number() over() breaks hash collisions for batched events with identical data.
+            // unique row id for item list attribution join. Only computed for ecommerce events.
+            // row_number() breaks hash collisions for batched events with identical data.
+            // partition by event_name avoids a single-partition bottleneck in the window function.
             // Non-determinism is safe: colliding rows have identical items (to_json_string(items) is in the hash),
             // so swapping row numbers between them produces the same final result.
-            _event_row_id: itemListAttribution ? `farm_fingerprint(concat(user_pseudo_id, cast(event_timestamp as string), event_name, to_json_string(items), cast(row_number() over() as string)))` : undefined,
+            _item_list_attribution_row_id: itemListAttribution ? `if(event_name in (${ecommerceEventsFilter}), farm_fingerprint(concat(user_pseudo_id, cast(event_timestamp as string), event_name, to_json_string(items), cast(row_number() over(partition by event_name, user_pseudo_id) as string))), null)` : undefined,
             // flag if the data is "final" and is not expected to change anymore
             data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
             export_type: helpers.getGa4ExportType('_table_suffix'),
@@ -272,12 +277,11 @@ ${excludedEventsSQL}`,
             itemListAttribution.lookbackTimeMs
         );
         const passthroughEvents = `event_name in ('view_item_list', 'select_item', 'view_promotion', 'select_promotion')`;
-        const ecommerceFilter = helpers.ga4EcommerceEvents.filter(e => e !== 'refund').map(e => `'${e}'`).join(', ');
 
         return {
             name: 'item_list_data',
             columns: {
-                '_event_row_id': '_event_row_id',
+                '_item_list_attribution_row_id': '_item_list_attribution_row_id',
                 'items': `array_agg(
       (select as struct item.* replace(
         coalesce(if(${passthroughEvents}, item.item_list_name, _item_list_attr.item_list_name), '(not set)') as item_list_name,
@@ -286,19 +290,19 @@ ${excludedEventsSQL}`,
       ))
     )`,
             },
-            from: `(select _event_row_id, event_name, item, ${attrExpr} as _item_list_attr from event_data, unnest(items) as item where event_name in (${ecommerceFilter}))`,
-            groupBy: ['_event_row_id'],
+            from: `(select _item_list_attribution_row_id, event_name, item, ${attrExpr} as _item_list_attr from event_data, unnest(items) as item where event_name in (${ecommerceEventsFilter}))`,
+            groupBy: ['_item_list_attribution_row_id'],
         };
     })() : null;
 
     const finalColumnOrder = getFinalColumnOrder(eventDataStep, sessionDataStep);
 
-    // When item list attribution is enabled, override the items column and exclude _event_row_id
+    // When item list attribution is enabled, override the items column and exclude _item_list_attribution_row_id
     // COALESCE handles events without items (not in ecommerce filter) where the LEFT JOIN returns NULL
     const itemListOverrides = itemListDataStep ? {
         items: 'coalesce(item_list_data.items, event_data.items)',
     } : {};
-    const itemListExcludedColumns = itemListDataStep ? ['_event_row_id'] : [];
+    const itemListExcludedColumns = itemListDataStep ? ['_item_list_attribution_row_id'] : [];
 
     // Join event_data and session_data, include additional logic
     const finalStep = {
@@ -334,7 +338,7 @@ ${excludedEventsSQL}`,
         leftJoin: [
             ...(itemListDataStep ? [{
                 table: 'item_list_data',
-                condition: 'using(_event_row_id)'
+                condition: 'using(_item_list_attribution_row_id)'
             }] : []),
             {
                 table: 'session_data',
@@ -418,5 +422,6 @@ module.exports = {
     generateSql: generateEnhancedEventsSQL,
     setPreOperations: setPreOperations,
     getColumnDescriptions: getColumnDescriptions,
-    getTableDescription: getTableDescription
+    getTableDescription: getTableDescription,
+    assertions,
 }