ga4-export-fixer 0.5.2-dev.6 → 0.5.2-dev.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.5.2-dev.6",
+  "version": "0.5.2-dev.8",
   "description": "",
   "main": "index.js",
   "files": [

package/tables/ga4EventsEnhanced/assertions/dailyQuality.js

@@ -0,0 +1,168 @@
+const helpers = require('../../../helpers/index.js');
+const utils = require('../../../utils.js');
+const { ga4EventsEnhancedConfig } = require('../config.js');
+const { validateEnhancedEventsConfig } = require('../validation.js');
+
+const defaultConfig = { ...ga4EventsEnhancedConfig };
+
+/**
+ * Builds a _table_suffix date filter for the assertion's raw-side query.
+ *
+ * Uses the low-level ga4ExportDateFilter() helper per enabled export type
+ * with a fixed 5-day lookback window. This is intentionally separate from
+ * the pipeline's ga4ExportDateFilters() which depends on incremental state
+ * and BigQuery pre-operation variables.
+ *
+ * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
+ * @returns {string} SQL fragment for a WHERE clause
+ */
+const buildAssertionDateFilter = (includedExportTypes) => {
+    const start = 'date_sub(current_date(), interval 5 day)';
+    const end = 'current_date()';
+
+    const filters = [
+        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
+        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
+        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
+    ].filter(Boolean);
+
+    return filters.join(' or ');
+};
+
+/**
+ * Generates a SQL assertion query that validates daily data quality between the
+ * enhanced events table and the raw GA4 export data.
+ *
+ * The query compares session count, event count, and total item_revenue
+ * aggregated per (event_date, data_is_final) for the last 5 days.
+ * Returns violating rows -- 0 rows means the assertion passes.
+ *
+ * Five violation types are detected:
+ * - MISSING_DAY: Raw data has events but enhanced table has none for this day
+ * - SESSION_COUNT_MISMATCH: Final data session count differs
+ * - EVENT_COUNT_MISMATCH: Final data event count differs
+ * - REVENUE_MISMATCH: Final data total item_revenue differs
+ * - NON_FINAL_EXCESS_EVENTS: Non-final enhanced data has more events than raw
+ *
+ * @param {string} tableRef - Fully qualified reference to the enhanced table
+ * @param {Object} mergedConfig - Merged table configuration (after merge + validation)
+ * @returns {string} SQL query returning violating rows
+ */
+const _generateDailyQualityAssertionSql = (tableRef, mergedConfig) => {
+    const excludedEvents = mergedConfig.excludedEvents;
+    const excludedEventsSQL = excludedEvents.length > 0
+        ? `and event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
+        : '';
+
+    const dataIsFinalCondition = helpers.isFinalData(
+        mergedConfig.dataIsFinal.detectionMethod,
+        mergedConfig.dataIsFinal.dayThreshold
+    );
+
+    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes);
+
+    return `with enhanced_daily as (
+    select
+        event_date,
+        data_is_final,
+        count(distinct session_id) as session_count,
+        count(*) as event_count,
+        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue
+    from
+        ${tableRef}
+    where
+        event_date >= date_sub(current_date(), interval 5 day)
+    group by event_date, data_is_final
+),
+raw_daily as (
+    select
+        cast(event_date as date format 'YYYYMMDD') as event_date,
+        ${dataIsFinalCondition} as data_is_final,
+        count(distinct concat(user_pseudo_id, cast((select value.int_value from unnest(event_params) where key = 'ga_session_id') as string))) as session_count,
+        count(*) as event_count,
+        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue
+    from
+        ${mergedConfig.sourceTable}
+    where
+        (${dateFilter})
+        ${excludedEventsSQL}
+        and cast(event_date as date format 'YYYYMMDD') >= date_sub(current_date(), interval 5 day)
+    group by event_date, data_is_final
+)
+select
+    coalesce(e.event_date, r.event_date) as event_date,
+    coalesce(e.data_is_final, r.data_is_final) as data_is_final,
+    e.session_count as enhanced_sessions,
+    r.session_count as raw_sessions,
+    e.event_count as enhanced_events,
+    r.event_count as raw_events,
+    round(e.total_item_revenue, 2) as enhanced_revenue,
+    round(r.total_item_revenue, 2) as raw_revenue,
+    case
+        when e.event_date is null and r.event_count > 0
+            then 'MISSING_DAY'
+        when coalesce(e.data_is_final, r.data_is_final) = true
+            and e.session_count != r.session_count
+            then 'SESSION_COUNT_MISMATCH'
+        when coalesce(e.data_is_final, r.data_is_final) = true
+            and e.event_count != r.event_count
+            then 'EVENT_COUNT_MISMATCH'
+        when coalesce(e.data_is_final, r.data_is_final) = true
+            and round(coalesce(e.total_item_revenue, 0), 2) != round(coalesce(r.total_item_revenue, 0), 2)
+            then 'REVENUE_MISMATCH'
+        when coalesce(e.data_is_final, r.data_is_final) = false
+            and coalesce(e.event_count, 0) > coalesce(r.event_count, 0)
+            then 'NON_FINAL_EXCESS_EVENTS'
+    end as violation_type
+from
+    enhanced_daily e
+full outer join
+    raw_daily r using(event_date, data_is_final)
+where
+    (coalesce(e.data_is_final, r.data_is_final) = true and (
+        e.session_count != r.session_count
+        or e.event_count != r.event_count
+        or round(coalesce(e.total_item_revenue, 0), 2) != round(coalesce(r.total_item_revenue, 0), 2)
+        or e.event_date is null
+    ))
+    or
+    (e.event_date is null and r.event_count > 0)
+    or
+    (coalesce(e.data_is_final, r.data_is_final) = false
+        and coalesce(e.event_count, 0) > coalesce(r.event_count, 0))`;
+};
+
+/**
+ * Generates a daily quality assertion SQL query.
+ *
+ * Merges the provided config with defaults, validates, then generates a SQL
+ * query comparing daily aggregates (session count, event count, item_revenue)
+ * between the enhanced table and raw export data, and checks for missing days
+ * and non-final data inflation.
+ *
+ * @param {string} tableRef - Fully qualified reference to the enhanced table.
+ * @param {Object} config - User-provided table configuration.
+ * @returns {string} SQL query returning violating rows (0 rows = pass)
+ */
+const generateDailyQualityAssertionSql = (tableRef, config) => {
+    if (!tableRef || typeof tableRef !== 'string' || !tableRef.trim()) {
+        throw new Error('assertions.dailyQuality: tableRef is required and must be a non-empty string (e.g., ctx.ref(\'table_name\') or \'`project.dataset.table`\').');
+    }
+    const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
+
+    if (utils.isDataformTableReferenceObject(mergedConfig.sourceTable)) {
+        throw new Error(
+            'assertions.dailyQuality: config.sourceTable is a Dataform table reference object, but assertions do not have access to Dataform context to resolve it. ' +
+            'Resolve it with ctx.ref() before passing it to the assertion:\n\n' +
+            '  .query(ctx => ga4EventsEnhanced.assertions.dailyQuality(\n' +
+            '    ctx.ref(\'enhanced_table_name\'),\n' +
+            '    { ...config, sourceTable: ctx.ref(config.sourceTable) }\n' +
+            '  ))'
+        );
+    }
+
+    validateEnhancedEventsConfig(mergedConfig, { skipDataformContextFields: true });
+    return _generateDailyQualityAssertionSql(tableRef, mergedConfig);
+};
+
+module.exports = { generateDailyQualityAssertionSql };

package/tables/ga4EventsEnhanced/assertions/index.js

@@ -1,5 +1,7 @@
 const { generateItemRevenueAssertionSql } = require('./itemRevenue.js');
+const { generateDailyQualityAssertionSql } = require('./dailyQuality.js');
 
 module.exports = {
     itemRevenue: generateItemRevenueAssertionSql,
+    dailyQuality: generateDailyQualityAssertionSql,
 };

package/tables/ga4EventsEnhanced/index.js

@@ -269,8 +269,10 @@ ${excludedEventsSQL}`,
         groupBy: ['session_id']
     };
 
-    // item list attribution CTE: unnest items, attribute via window function, re-aggregate
-    const itemListDataStep = itemListAttribution ? (() => {
+    // item list attribution CTEs:
+    // 1. item_list_unnest: unnest items from ecommerce events, compute attribution via window function
+    // 2. item_list_data: re-aggregate items with attributed list fields
+    const itemListSteps = itemListAttribution ? (() => {
         const attrExpr = helpers.itemListAttributionExpr(
             itemListAttribution.lookbackType,
             timestampColumn,
@@ -278,7 +280,19 @@ ${excludedEventsSQL}`,
         );
         const passthroughEvents = `event_name in ('view_item_list', 'select_item', 'view_promotion', 'select_promotion')`;
 
-        return {
+        const attributionStep = {
+            name: 'item_list_attribution',
+            columns: {
+                '_item_list_attribution_row_id': '_item_list_attribution_row_id',
+                'event_name': 'event_name',
+                'item': 'item',
+                '_item_list_attr': attrExpr,
+            },
+            from: 'event_data, unnest(items) as item',
+            where: `event_name in (${ecommerceEventsFilter})`,
+        };
+
+        const dataStep = {
             name: 'item_list_data',
             columns: {
                 '_item_list_attribution_row_id': '_item_list_attribution_row_id',
@@ -290,19 +304,21 @@ ${excludedEventsSQL}`,
       ))
     )`,
             },
-            from: `(select _item_list_attribution_row_id, event_name, item, ${attrExpr} as _item_list_attr from event_data, unnest(items) as item where event_name in (${ecommerceEventsFilter}))`,
+            from: 'item_list_attribution',
             groupBy: ['_item_list_attribution_row_id'],
         };
+
+        return [attributionStep, dataStep];
     })() : null;
 
     const finalColumnOrder = getFinalColumnOrder(eventDataStep, sessionDataStep);
 
     // When item list attribution is enabled, override the items column and exclude _item_list_attribution_row_id
     // COALESCE handles events without items (not in ecommerce filter) where the LEFT JOIN returns NULL
-    const itemListOverrides = itemListDataStep ? {
+    const itemListOverrides = itemListSteps ? {
         items: 'coalesce(item_list_data.items, event_data.items)',
     } : {};
-    const itemListExcludedColumns = itemListDataStep ? ['_item_list_attribution_row_id'] : [];
+    const itemListExcludedColumns = itemListSteps ? ['_item_list_attribution_row_id'] : [];
 
     // Join event_data and session_data, include additional logic
     const finalStep = {
@@ -336,7 +352,7 @@ ${excludedEventsSQL}`,
         },
         from: 'event_data',
         leftJoin: [
-            ...(itemListDataStep ? [{
+            ...(itemListSteps ? [{
                 table: 'item_list_data',
                 condition: 'using(_item_list_attribution_row_id)'
             }] : []),
@@ -350,7 +366,7 @@ ${excludedEventsSQL}`,
 
     const steps = [
         eventDataStep,
-        ...(itemListDataStep ? [itemListDataStep] : []),
+        ...(itemListSteps ?? []),
         sessionDataStep,
         finalStep,
     ];