ga4-export-fixer 0.6.2-dev.1 → 0.6.2-dev.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.6.2-dev.1",
+  "version": "0.6.2-dev.2",
   "description": "",
   "main": "index.js",
   "files": [

package/tables/ga4EventsEnhanced/assertions/dailyQuality.js

@@ -2,32 +2,11 @@ const helpers = require('../../../helpers/index.js');
 const utils = require('../../../utils.js');
 const { ga4EventsEnhancedConfig } = require('../config.js');
 const { validateEnhancedEventsConfig } = require('../validation.js');
+const { buildDedupedRawSource } = require('./shared.js');
 
 const defaultConfig = { ...ga4EventsEnhancedConfig };
 
-/**
- * Builds a _table_suffix date filter for the assertion's raw-side query.
- *
- * Uses the low-level ga4ExportDateFilter() helper per enabled export type
- * with a fixed 5-day lookback window. This is intentionally separate from
- * the pipeline's ga4ExportDateFilters() which depends on incremental state
- * and BigQuery pre-operation variables.
- *
- * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
- * @returns {string} SQL fragment for a WHERE clause
- */
-const buildAssertionDateFilter = (includedExportTypes) => {
-    const start = 'date_sub(current_date(), interval 5 day)';
-    const end = 'current_date()';
-
-    const filters = [
-        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
-        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
-        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
-    ].filter(Boolean);
-
-    return filters.join(' or ');
-};
+const ASSERTION_LOOKBACK_DAYS = 5;
 
 /**
  * Generates a SQL assertion query that validates daily data quality between the
@@ -51,15 +30,15 @@ const buildAssertionDateFilter = (includedExportTypes) => {
 const _generateDailyQualityAssertionSql = (tableRef, mergedConfig) => {
     const excludedEvents = mergedConfig.excludedEvents;
     const excludedEventsSQL = excludedEvents.length > 0
-        ? `and event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
-        : '';
+        ? `event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
+        : 'true';
 
     const dataIsFinalCondition = helpers.isFinalData(
         mergedConfig.dataIsFinal.detectionMethod,
         mergedConfig.dataIsFinal.dayThreshold
     );
 
-    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes);
+    const dedupedRawSource = buildDedupedRawSource(mergedConfig, ASSERTION_LOOKBACK_DAYS);
 
     return `with enhanced_daily as (
     select
@@ -71,7 +50,7 @@ const _generateDailyQualityAssertionSql = (tableRef, mergedConfig) => {
     from
         ${tableRef}
     where
-        event_date >= date_sub(current_date(), interval 5 day)
+        event_date >= date_sub(current_date(), interval ${ASSERTION_LOOKBACK_DAYS} day)
     group by event_date, data_is_final
 ),
 raw_daily as (
@@ -82,11 +61,9 @@ raw_daily as (
         count(*) as event_count,
         coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue
     from
-        ${mergedConfig.sourceTable}
+        ${dedupedRawSource}
     where
-        (${dateFilter})
         ${excludedEventsSQL}
-        and cast(event_date as date format 'YYYYMMDD') >= date_sub(current_date(), interval 5 day)
     group by event_date, data_is_final
 ),
 daily_comparison as (

package/tables/ga4EventsEnhanced/assertions/itemRevenue.js

@@ -2,38 +2,17 @@ const helpers = require('../../../helpers/index.js');
 const utils = require('../../../utils.js');
 const { ga4EventsEnhancedConfig } = require('../config.js');
 const { validateEnhancedEventsConfig } = require('../validation.js');
+const { buildDedupedRawSource } = require('./shared.js');
 
 const defaultConfig = { ...ga4EventsEnhancedConfig };
 
+const ASSERTION_LOOKBACK_DAYS = 5;
+
 // Ecommerce events that carry item data (excluding refund — refunds reverse revenue
 // and are handled separately in some pipelines, but item_revenue on refund rows
 // should still reconcile 1:1 between enhanced and raw).
 const ecommerceEvents = helpers.ga4EcommerceEvents.map(e => `'${e}'`).join(', ');
 
-/**
- * Builds a _table_suffix date filter for the assertion's raw-side query.
- *
- * Uses the low-level ga4ExportDateFilter() helper per enabled export type
- * with a fixed 5-day lookback window. This is intentionally separate from
- * the pipeline's ga4ExportDateFilters() which depends on incremental state
- * and BigQuery pre-operation variables.
- *
- * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
- * @returns {string} SQL fragment for a WHERE clause
- */
-const buildAssertionDateFilter = (includedExportTypes) => {
-    const start = 'date_sub(current_date(), interval 5 day)';
-    const end = 'current_date()';
-
-    const filters = [
-        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
-        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
-        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
-    ].filter(Boolean);
-
-    return filters.join(' or ');
-};
-
 /**
  * Generates a SQL assertion query that reconciles item_revenue between the
  * enhanced events table and the raw GA4 export data.
@@ -51,8 +30,8 @@ const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
     // excluded events filter (same logic as the enhanced table pipeline)
     const excludedEvents = mergedConfig.excludedEvents;
     const excludedEventsSQL = excludedEvents.length > 0
-        ? `and event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
-        : '';
+        ? `event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
+        : 'true';
 
     // data_is_final condition for the raw side
     const dataIsFinalCondition = helpers.isFinalData(
@@ -60,8 +39,8 @@ const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
         mergedConfig.dataIsFinal.dayThreshold
     );
 
-    // date filter for the raw side (per-export-type, fixed 5-day window)
-    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes);
+    // deduplicated raw-source subquery (mirrors pipeline setPreOperations dedup)
+    const dedupedRawSource = buildDedupedRawSource(mergedConfig, ASSERTION_LOOKBACK_DAYS);
 
     return `with enhanced_revenue as (
     select
@@ -74,7 +53,7 @@ const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
         unnest(items) as item
     where
         data_is_final = true
-        and event_date >= date_sub(current_date(), interval 5 day)
+        and event_date >= date_sub(current_date(), interval ${ASSERTION_LOOKBACK_DAYS} day)
         and event_name in (${ecommerceEvents})
     group by event_date, item.item_id
 ),
@@ -85,14 +64,12 @@ raw_revenue as (
         sum(item.item_revenue) as total_item_revenue,
         count(*) as item_count
     from
-        ${mergedConfig.sourceTable},
+        ${dedupedRawSource},
         unnest(items) as item
     where
-        (${dateFilter})
         ${excludedEventsSQL}
         and event_name in (${ecommerceEvents})
         and ${dataIsFinalCondition}
-        and cast(event_date as date format 'YYYYMMDD') >= date_sub(current_date(), interval 5 day)
     group by event_date, item.item_id
 )
 select

package/tables/ga4EventsEnhanced/assertions/shared.js

@@ -0,0 +1,83 @@
+const helpers = require('../../../helpers/index.js');
+
+/**
+ * Builds a _table_suffix date filter for the assertion's raw-side query.
+ *
+ * Uses the low-level ga4ExportDateFilter() helper per enabled export type
+ * over a caller-provided lookback window. Intentionally separate from the
+ * pipeline's ga4ExportDateFilters() which depends on incremental state
+ * and BigQuery pre-operation variables.
+ *
+ * @param {Object} includedExportTypes - { daily: boolean, fresh: boolean, intraday: boolean }
+ * @param {number} lookbackDays - Number of days to look back from current_date().
+ * @returns {string} SQL fragment for a WHERE clause
+ */
+const buildAssertionDateFilter = (includedExportTypes, lookbackDays) => {
+    const start = `date_sub(current_date(), interval ${lookbackDays} day)`;
+    const end = 'current_date()';
+
+    const filters = [
+        includedExportTypes.daily ? helpers.ga4ExportDateFilter('daily', start, end) : null,
+        includedExportTypes.fresh ? helpers.ga4ExportDateFilter('fresh', start, end) : null,
+        includedExportTypes.intraday ? helpers.ga4ExportDateFilter('intraday', start, end) : null,
+    ].filter(Boolean);
+
+    return filters.join(' or ');
+};
+
+/**
+ * Builds a deduplicated raw-source subquery for assertion use.
+ *
+ * Replicates what setPreOperations() does at pipeline time, without access
+ * to its BigQuery variables. Covers all seven combinations of
+ * includedExportTypes {daily, fresh, intraday}:
+ *
+ *   - qualify dense_rank() over (partition by date, order by _table_suffix) = 1
+ *     picks the highest-priority table per day. Alphabetical order gives
+ *     daily ('20260115') < fresh ('fresh_20260115') < intraday ('intraday_20260115'),
+ *     matching the pipeline's daily > fresh > intraday priority.
+ *   - When fresh and intraday are both enabled, intraday rows with
+ *     event_timestamp > max(fresh.event_timestamp) for the same date are
+ *     additionally admitted — matching the FRESH_MAX_EVENT_TIMESTAMP boundary.
+ *
+ * @param {Object} mergedConfig - Merged table configuration.
+ * @param {number} lookbackDays - Number of days to look back from current_date().
+ * @returns {string} SQL fragment: a parenthesized subquery usable in a FROM clause.
+ */
+const buildDedupedRawSource = (mergedConfig, lookbackDays) => {
+    const dateFilter = buildAssertionDateFilter(mergedConfig.includedExportTypes, lookbackDays);
+    const freshAndIntraday = mergedConfig.includedExportTypes.fresh && mergedConfig.includedExportTypes.intraday;
+
+    const intradayException = freshAndIntraday
+        ? `
+        or (
+            starts_with(_table_suffix, 'intraday_')
+            and dense_rank() over (
+                partition by regexp_extract(_table_suffix, r'[0-9]+')
+                order by _table_suffix
+            ) = 2
+            and event_timestamp > max(if(starts_with(_table_suffix, 'fresh_'), event_timestamp, null)) over (
+                partition by regexp_extract(_table_suffix, r'[0-9]+')
+            )
+        )`
+        : '';
+
+    // _table_suffix is a pseudo-column and not propagated by SELECT *; select it
+    // explicitly so downstream CTEs (e.g., isFinalData('EXPORT_TYPE')) can still reference it.
+    return `(
+    select
+        *,
+        _table_suffix
+    from
+        ${mergedConfig.sourceTable}
+    where
+        (${dateFilter})
+    qualify
+        dense_rank() over (
+            partition by regexp_extract(_table_suffix, r'[0-9]+')
+            order by _table_suffix
+        ) = 1${intradayException}
+)`;
+};
+
+module.exports = { buildAssertionDateFilter, buildDedupedRawSource };