ga4-export-fixer 0.6.2-dev.1 → 0.6.2-dev.3

package/README.md

@@ -457,7 +457,7 @@ This creates the table along with the default-enabled assertions, using the same
 
 | Assertion | Name | Enabled by default | Description |
 | --------- | ---- | ------------------ | ----------- |
-| `dailyQuality` | `{tableName}_daily_quality` | Yes | Compares session count, event count, and item revenue per day between the enhanced table and raw export. Detects missing days, count mismatches, and non-final data inflation |
+| `dailyQuality` | `{tableName}_daily_quality` | Yes | Compares session count, event count, item revenue, and ecommerce purchase revenue per day between the enhanced table and raw export. Detects missing days, count mismatches, and non-final data inflation |
 | `itemRevenue` | `{tableName}_item_revenue` | No (opt-in) | Reconciles item_revenue at the (event_date, item_id) grain between the enhanced table and raw export |
 
 Assertions inherit the table's schema and tags from `dataformTableConfig`. Each assertion queries the last 5 days of data.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.6.2-dev.1",
+  "version": "0.6.2-dev.3",
   "description": "",
   "main": "index.js",
   "files": [

package/tables/ga4EventsEnhanced/assertions/dailyQuality.js

@@ -2,46 +2,27 @@ const helpers = require('../../../helpers/index.js');
 const utils = require('../../../utils.js');
 const { ga4EventsEnhancedConfig } = require('../config.js');
 const { validateEnhancedEventsConfig } = require('../validation.js');
+const { buildDedupedRawSource } = require('./shared.js');
 
 const defaultConfig = { ...ga4EventsEnhancedConfig };
 
-/**
- * Builds a _table_suffix date filter for the assertion's raw-side query.
- *
- * Uses the low-level ga4ExportDateFilter() helper per enabled export type
- * with a fixed 5-day lookback window. This is intentionally separate from
- * the pipeline's ga4ExportDateFilters() which depends on incremental state
- * and BigQuery pre-operation variables.
- *
- * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
- * @returns {string} SQL fragment for a WHERE clause
- */
-const buildAssertionDateFilter = (includedExportTypes) => {
-    const start = 'date_sub(current_date(), interval 5 day)';
-    const end = 'current_date()';
-
-    const filters = [
-        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
-        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
-        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
-    ].filter(Boolean);
-
-    return filters.join(' or ');
-};
+const ASSERTION_LOOKBACK_DAYS = 5;
 
 /**
  * Generates a SQL assertion query that validates daily data quality between the
  * enhanced events table and the raw GA4 export data.
  *
- * The query compares session count, event count, and total item_revenue
- * aggregated per (event_date, data_is_final) for the last 5 days.
- * Returns violating rows -- 0 rows means the assertion passes.
+ * The query compares session count, event count, total item_revenue, and total
+ * purchase_revenue aggregated per (event_date, data_is_final) for the last 5
+ * days. Returns violating rows -- 0 rows means the assertion passes.
  *
- * Five violation types are detected:
+ * Six violation types are detected:
  * - MISSING_DAY: Raw data has events but enhanced table has none for this day
  * - SESSION_COUNT_MISMATCH: Final data session count differs
  * - EVENT_COUNT_MISMATCH: Final data event count differs
- * - REVENUE_MISMATCH: Final data total item_revenue differs
+ * - ITEM_REVENUE_MISMATCH: Final data total item_revenue differs
+ * - PURCHASE_REVENUE_MISMATCH: Final data total ecommerce.purchase_revenue differs
+ *   (raw side applies fixEcommerceStruct() to mirror the enhanced pipeline's fix)
  * - NON_FINAL_EXCESS_EVENTS: Non-final enhanced data has more events than raw
  *
  * @param {string} tableRef - Fully qualified reference to the enhanced table
@@ -51,15 +32,15 @@ const buildAssertionDateFilter = (includedExportTypes) => {
 const _generateDailyQualityAssertionSql = (tableRef, mergedConfig) => {
     const excludedEvents = mergedConfig.excludedEvents;
     const excludedEventsSQL = excludedEvents.length > 0
-        ? `and event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
-        : '';
+        ? `event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
+        : 'true';
 
     const dataIsFinalCondition = helpers.isFinalData(
         mergedConfig.dataIsFinal.detectionMethod,
         mergedConfig.dataIsFinal.dayThreshold
     );
 
-    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes);
+    const dedupedRawSource = buildDedupedRawSource(mergedConfig, ASSERTION_LOOKBACK_DAYS);
 
     return `with enhanced_daily as (
     select
@@ -67,11 +48,12 @@ const _generateDailyQualityAssertionSql = (tableRef, mergedConfig) => {
         data_is_final,
         count(distinct session_id) as session_count,
         count(*) as event_count,
-        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue
+        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue,
+        coalesce(sum(ecommerce.purchase_revenue), 0) as total_purchase_revenue
     from
         ${tableRef}
     where
-        event_date >= date_sub(current_date(), interval 5 day)
+        event_date >= date_sub(current_date(), interval ${ASSERTION_LOOKBACK_DAYS} day)
     group by event_date, data_is_final
 ),
 raw_daily as (
@@ -80,13 +62,12 @@ raw_daily as (
         ${dataIsFinalCondition} as data_is_final,
         count(distinct concat(user_pseudo_id, cast((select value.int_value from unnest(event_params) where key = 'ga_session_id') as string))) as session_count,
         count(*) as event_count,
-        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue
+        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue,
+        coalesce(sum(${helpers.fixEcommerceStruct()}.purchase_revenue), 0) as total_purchase_revenue
     from
-        ${mergedConfig.sourceTable}
+        ${dedupedRawSource}
     where
-        (${dateFilter})
         ${excludedEventsSQL}
-        and cast(event_date as date format 'YYYYMMDD') >= date_sub(current_date(), interval 5 day)
     group by event_date, data_is_final
 ),
 daily_comparison as (
@@ -97,8 +78,10 @@ daily_comparison as (
         r.session_count as raw_sessions,
         e.event_count as enhanced_events,
         r.event_count as raw_events,
-        round(e.total_item_revenue, 2) as enhanced_revenue,
-        round(r.total_item_revenue, 2) as raw_revenue
+        round(e.total_item_revenue, 2) as enhanced_item_revenue,
+        round(r.total_item_revenue, 2) as raw_item_revenue,
+        round(e.total_purchase_revenue, 2) as enhanced_purchase_revenue,
+        round(r.total_purchase_revenue, 2) as raw_purchase_revenue
     from
         enhanced_daily e
     full outer join
@@ -111,8 +94,10 @@ select
     raw_sessions,
     enhanced_events,
     raw_events,
-    enhanced_revenue,
-    raw_revenue,
+    enhanced_item_revenue,
+    raw_item_revenue,
+    enhanced_purchase_revenue,
+    raw_purchase_revenue,
     violation_type
 from
     daily_comparison,
@@ -120,7 +105,8 @@ from
         if(enhanced_events is null and raw_events > 0, 'MISSING_DAY', null),
         if(data_is_final = true and enhanced_sessions != raw_sessions, 'SESSION_COUNT_MISMATCH', null),
         if(data_is_final = true and enhanced_events != raw_events, 'EVENT_COUNT_MISMATCH', null),
-        if(data_is_final = true and enhanced_revenue != raw_revenue, 'REVENUE_MISMATCH', null),
+        if(data_is_final = true and enhanced_item_revenue != raw_item_revenue, 'ITEM_REVENUE_MISMATCH', null),
+        if(data_is_final = true and enhanced_purchase_revenue != raw_purchase_revenue, 'PURCHASE_REVENUE_MISMATCH', null),
         if(data_is_final = false and coalesce(enhanced_events, 0) > coalesce(raw_events, 0), 'NON_FINAL_EXCESS_EVENTS', null)
     ]) as violation_type
 where
@@ -131,9 +117,9 @@ where
  * Generates a daily quality assertion SQL query.
  *
  * Merges the provided config with defaults, validates, then generates a SQL
- * query comparing daily aggregates (session count, event count, item_revenue)
- * between the enhanced table and raw export data, and checks for missing days
- * and non-final data inflation.
+ * query comparing daily aggregates (session count, event count, item_revenue,
+ * ecommerce.purchase_revenue) between the enhanced table and raw export data,
+ * and checks for missing days and non-final data inflation.
  *
  * @param {string} tableRef - Fully qualified reference to the enhanced table.
  * @param {Object} config - User-provided table configuration.

package/tables/ga4EventsEnhanced/assertions/itemRevenue.js

@@ -2,38 +2,17 @@ const helpers = require('../../../helpers/index.js');
 const utils = require('../../../utils.js');
 const { ga4EventsEnhancedConfig } = require('../config.js');
 const { validateEnhancedEventsConfig } = require('../validation.js');
+const { buildDedupedRawSource } = require('./shared.js');
 
 const defaultConfig = { ...ga4EventsEnhancedConfig };
 
+const ASSERTION_LOOKBACK_DAYS = 5;
+
 // Ecommerce events that carry item data (excluding refund — refunds reverse revenue
 // and are handled separately in some pipelines, but item_revenue on refund rows
 // should still reconcile 1:1 between enhanced and raw).
 const ecommerceEvents = helpers.ga4EcommerceEvents.map(e => `'${e}'`).join(', ');
 
-/**
- * Builds a _table_suffix date filter for the assertion's raw-side query.
- *
- * Uses the low-level ga4ExportDateFilter() helper per enabled export type
- * with a fixed 5-day lookback window. This is intentionally separate from
- * the pipeline's ga4ExportDateFilters() which depends on incremental state
- * and BigQuery pre-operation variables.
- *
- * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
- * @returns {string} SQL fragment for a WHERE clause
- */
-const buildAssertionDateFilter = (includedExportTypes) => {
-    const start = 'date_sub(current_date(), interval 5 day)';
-    const end = 'current_date()';
-
-    const filters = [
-        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
-        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
-        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
-    ].filter(Boolean);
-
-    return filters.join(' or ');
-};
-
 /**
  * Generates a SQL assertion query that reconciles item_revenue between the
  * enhanced events table and the raw GA4 export data.
@@ -51,8 +30,8 @@ const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
     // excluded events filter (same logic as the enhanced table pipeline)
     const excludedEvents = mergedConfig.excludedEvents;
     const excludedEventsSQL = excludedEvents.length > 0
-        ? `and event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
-        : '';
+        ? `event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
+        : 'true';
 
     // data_is_final condition for the raw side
     const dataIsFinalCondition = helpers.isFinalData(
@@ -60,8 +39,8 @@ const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
         mergedConfig.dataIsFinal.dayThreshold
     );
 
-    // date filter for the raw side (per-export-type, fixed 5-day window)
-    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes);
+    // deduplicated raw-source subquery (mirrors pipeline setPreOperations dedup)
+    const dedupedRawSource = buildDedupedRawSource(mergedConfig, ASSERTION_LOOKBACK_DAYS);
 
     return `with enhanced_revenue as (
     select
@@ -74,7 +53,7 @@ const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
         unnest(items) as item
     where
         data_is_final = true
-        and event_date >= date_sub(current_date(), interval 5 day)
+        and event_date >= date_sub(current_date(), interval ${ASSERTION_LOOKBACK_DAYS} day)
         and event_name in (${ecommerceEvents})
     group by event_date, item.item_id
 ),
@@ -85,14 +64,12 @@ raw_revenue as (
         sum(item.item_revenue) as total_item_revenue,
         count(*) as item_count
     from
-        ${mergedConfig.sourceTable},
+        ${dedupedRawSource},
         unnest(items) as item
     where
-        (${dateFilter})
         ${excludedEventsSQL}
         and event_name in (${ecommerceEvents})
         and ${dataIsFinalCondition}
-        and cast(event_date as date format 'YYYYMMDD') >= date_sub(current_date(), interval 5 day)
     group by event_date, item.item_id
 )
 select

package/tables/ga4EventsEnhanced/assertions/shared.js

@@ -0,0 +1,83 @@
+const helpers = require('../../../helpers/index.js');
+
+/**
+ * Builds a _table_suffix date filter for the assertion's raw-side query.
+ *
+ * Uses the low-level ga4ExportDateFilter() helper per enabled export type
+ * over a caller-provided lookback window. Intentionally separate from the
+ * pipeline's ga4ExportDateFilters() which depends on incremental state
+ * and BigQuery pre-operation variables.
+ *
+ * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
+ * @param {number} lookbackDays - Number of days to look back from current_date().
+ * @returns {string} SQL fragment for a WHERE clause
+ */
+const buildAssertionDateFilter = (includedExportTypes, lookbackDays) => {
+    const start = `date_sub(current_date(), interval ${lookbackDays} day)`;
+    const end = 'current_date()';
+
+    const filters = [
+        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
+        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
+        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
+    ].filter(Boolean);
+
+    return filters.join(' or ');
+};
+
+/**
+ * Builds a deduplicated raw-source subquery for assertion use.
+ *
+ * Replicates what setPreOperations() does at pipeline time, without access
+ * to its BigQuery variables. Covers all seven combinations of
+ * includedExportTypes {daily, fresh, intraday}:
+ *
+ *   - qualify dense_rank() over (partition by date, order by _table_suffix) = 1
+ *     picks the highest-priority table per day. Alphabetical order gives
+ *     daily ('20260115') < fresh ('fresh_20260115') < intraday ('intraday_20260115'),
+ *     matching the pipeline's daily > fresh > intraday priority.
+ *   - When fresh and intraday are both enabled, intraday rows with
+ *     event_timestamp > max(fresh.event_timestamp) for the same date are
+ *     additionally admitted — matching the FRESH_MAX_EVENT_TIMESTAMP boundary.
+ *
+ * @param {Object} mergedConfig - Merged table configuration.
+ * @param {number} lookbackDays - Number of days to look back from current_date().
+ * @returns {string} SQL fragment: a parenthesized subquery usable in a FROM clause.
+ */
+const buildDedupedRawSource = (mergedConfig, lookbackDays) => {
+    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes, lookbackDays);
+    const freshAndIntraday = mergedConfig.includedExportTypes.fresh && mergedConfig.includedExportTypes.intraday;
+
+    const intradayException = freshAndIntraday
+        ? `
+        or (
+            starts_with(_table_suffix, 'intraday_')
+            and dense_rank() over (
+                partition by regexp_extract(_table_suffix, r'[0-9]+')
+                order by _table_suffix
+            ) = 2
+            and event_timestamp > max(if(starts_with(_table_suffix, 'fresh_'), event_timestamp, null)) over (
+                partition by regexp_extract(_table_suffix, r'[0-9]+')
+            )
+        )`
+        : '';
+
+    // _table_suffix is a pseudo-column and not propagated by SELECT *; select it
+    // explicitly so downstream CTEs (e.g., isFinalData('EXPORT_TYPE')) can still reference it.
+    return `(
+    select
+        *,
+        _table_suffix
+    from
+        ${mergedConfig.sourceTable}
+    where
+        (${dateFilter})
+    qualify
+        dense_rank() over (
+            partition by regexp_extract(_table_suffix, r'[0-9]+')
+            order by _table_suffix
+        ) = 1${intradayException}
+)`;
+};
+
+module.exports = { buildAssertionDateFilter, buildDedupedRawSource };