ga4-export-fixer 0.6.2-dev.2 → 0.6.2-dev.4

package/README.md

@@ -457,19 +457,18 @@ This creates the table along with the default-enabled assertions, using the same
 
 | Assertion | Name | Enabled by default | Description |
 | --------- | ---- | ------------------ | ----------- |
-| `dailyQuality` | `{tableName}_daily_quality` | Yes | Compares session count, event count, and item revenue per day between the enhanced table and raw export. Detects missing days, count mismatches, and non-final data inflation |
-| `itemRevenue` | `{tableName}_item_revenue` | No (opt-in) | Reconciles item_revenue at the (event_date, item_id) grain between the enhanced table and raw export |
+| `dailyQuality` | `{tableName}_daily_quality` | Yes | Compares session count, event count, item revenue, and ecommerce purchase revenue per day between the enhanced table and raw export. Also reconciles item_revenue at the (event_date, item_id) grain on purchase events for days both sides consider final. Detects missing days, count mismatches, and non-final data inflation |
 
-Assertions inherit the table's schema and tags from `dataformTableConfig`. Each assertion queries the last 5 days of data.
+The assertion inherits the table's schema and tags from `dataformTableConfig` and queries the last 5 days of data.
 
 #### Selective Assertions
 
-Enable opt-in assertions by setting them to `true`, or disable default-enabled ones by setting them to `false`:
+Disable the assertion by setting it to `false`:
 
 ```javascript
 ga4EventsEnhanced.createTable(publish, config, {
     assert,
-    assertions: { dailyQuality: true, itemRevenue: true },
+    assertions: { dailyQuality: false },
 });
 ```
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.6.2-dev.2",
+  "version": "0.6.2-dev.4",
   "description": "",
   "main": "index.js",
   "files": [

package/tables/ga4EventsEnhanced/assertions/dailyQuality.js

@@ -12,16 +12,27 @@ const ASSERTION_LOOKBACK_DAYS = 5;
  * Generates a SQL assertion query that validates daily data quality between the
  * enhanced events table and the raw GA4 export data.
  *
- * The query compares session count, event count, and total item_revenue
- * aggregated per (event_date, data_is_final) for the last 5 days.
+ * The query runs two aggregations for the last 5 days and unions the violations:
+ *   - Day grain: (event_date, data_is_final) -- session/event counts, item_revenue,
+ *     ecommerce.purchase_revenue.
+ *   - Item-id grain: (event_date, item_id) on purchase events for days both sides
+ *     consider final -- validates per-item_id revenue and item-row count.
+ *
  * Returns violating rows -- 0 rows means the assertion passes.
  *
- * Five violation types are detected:
+ * Eight violation types are detected:
  * - MISSING_DAY: Raw data has events but enhanced table has none for this day
  * - SESSION_COUNT_MISMATCH: Final data session count differs
  * - EVENT_COUNT_MISMATCH: Final data event count differs
- * - REVENUE_MISMATCH: Final data total item_revenue differs
+ * - ITEM_REVENUE_MISMATCH: Final data total item_revenue differs
+ * - PURCHASE_REVENUE_MISMATCH: Final data total ecommerce.purchase_revenue differs
+ *   (raw side applies fixEcommerceStruct() to mirror the enhanced pipeline's fix)
  * - NON_FINAL_EXCESS_EVENTS: Non-final enhanced data has more events than raw
+ * - ITEM_REVENUE_MISMATCH_BY_ID: Per-item_id item_revenue differs on a shared-final day
+ * - ITEM_COUNT_MISMATCH_BY_ID: Per-item_id purchase item-row count differs on a shared-final day
+ *
+ * Day-level rows leave item_id / enhanced_item_count / raw_item_count NULL.
+ * Item-id-level rows leave session / event / purchase_revenue columns NULL.
  *
  * @param {string} tableRef - Fully qualified reference to the enhanced table
  * @param {Object} mergedConfig - Merged table configuration (after merge + validation)
@@ -46,7 +57,8 @@ const _generateDailyQualityAssertionSql = (tableRef, mergedConfig) => {
         data_is_final,
         count(distinct session_id) as session_count,
         count(*) as event_count,
-        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue
+        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue,
+        coalesce(sum(ecommerce.purchase_revenue), 0) as total_purchase_revenue
     from
         ${tableRef}
     where
@@ -59,7 +71,8 @@ raw_daily as (
         ${dataIsFinalCondition} as data_is_final,
         count(distinct concat(user_pseudo_id, cast((select value.int_value from unnest(event_params) where key = 'ga_session_id') as string))) as session_count,
         count(*) as event_count,
-        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue
+        coalesce(sum((select sum(item.item_revenue) from unnest(items) as item)), 0) as total_item_revenue,
+        coalesce(sum(${helpers.fixEcommerceStruct()}.purchase_revenue), 0) as total_purchase_revenue
     from
         ${dedupedRawSource}
     where
@@ -74,22 +87,81 @@ daily_comparison as (
         r.session_count as raw_sessions,
         e.event_count as enhanced_events,
         r.event_count as raw_events,
-        round(e.total_item_revenue, 2) as enhanced_revenue,
-        round(r.total_item_revenue, 2) as raw_revenue
+        round(e.total_item_revenue, 2) as enhanced_item_revenue,
+        round(r.total_item_revenue, 2) as raw_item_revenue,
+        round(e.total_purchase_revenue, 2) as enhanced_purchase_revenue,
+        round(r.total_purchase_revenue, 2) as raw_purchase_revenue
     from
         enhanced_daily e
     full outer join
         raw_daily r using(event_date, data_is_final)
+),
+enhanced_items as (
+    select
+        event_date,
+        item.item_id,
+        sum(item.item_revenue) as total_item_revenue,
+        count(*) as item_count
+    from
+        ${tableRef},
+        unnest(items) as item
+    where
+        data_is_final = true
+        and event_date >= date_sub(current_date(), interval ${ASSERTION_LOOKBACK_DAYS} day)
+        and event_name = 'purchase'
+    group by event_date, item.item_id
+),
+raw_items as (
+    select
+        cast(event_date as date format 'YYYYMMDD') as event_date,
+        item.item_id,
+        sum(item.item_revenue) as total_item_revenue,
+        count(*) as item_count
+    from
+        ${dedupedRawSource},
+        unnest(items) as item
+    where
+        ${excludedEventsSQL}
+        and event_name = 'purchase'
+        and ${dataIsFinalCondition}
+    group by event_date, item.item_id
+),
+shared_final_days as (
+    select event_date
+    from daily_comparison
+    where data_is_final = true
+        and enhanced_events is not null
+        and raw_events is not null
+),
+item_comparison as (
+    select
+        coalesce(e.event_date, r.event_date) as event_date,
+        coalesce(e.item_id, r.item_id) as item_id,
+        round(e.total_item_revenue, 2) as enhanced_item_revenue,
+        round(r.total_item_revenue, 2) as raw_item_revenue,
+        e.item_count as enhanced_item_count,
+        r.item_count as raw_item_count
+    from
+        enhanced_items e
+    full outer join
+        raw_items r using(event_date, item_id)
+    where
+        coalesce(e.event_date, r.event_date) in (select event_date from shared_final_days)
 )
 select
     event_date,
     data_is_final,
+    null as item_id,
     enhanced_sessions,
     raw_sessions,
     enhanced_events,
     raw_events,
-    enhanced_revenue,
-    raw_revenue,
+    enhanced_item_revenue,
+    raw_item_revenue,
+    enhanced_purchase_revenue,
+    raw_purchase_revenue,
+    cast(null as int64) as enhanced_item_count,
+    cast(null as int64) as raw_item_count,
     violation_type
 from
     daily_comparison,
@@ -97,9 +169,34 @@ from
         if(enhanced_events is null and raw_events > 0, 'MISSING_DAY', null),
         if(data_is_final = true and enhanced_sessions != raw_sessions, 'SESSION_COUNT_MISMATCH', null),
         if(data_is_final = true and enhanced_events != raw_events, 'EVENT_COUNT_MISMATCH', null),
-        if(data_is_final = true and enhanced_revenue != raw_revenue, 'REVENUE_MISMATCH', null),
+        if(data_is_final = true and enhanced_item_revenue != raw_item_revenue, 'ITEM_REVENUE_MISMATCH', null),
+        if(data_is_final = true and enhanced_purchase_revenue != raw_purchase_revenue, 'PURCHASE_REVENUE_MISMATCH', null),
         if(data_is_final = false and coalesce(enhanced_events, 0) > coalesce(raw_events, 0), 'NON_FINAL_EXCESS_EVENTS', null)
     ]) as violation_type
+where
+    violation_type is not null
+union all
+select
+    event_date,
+    true as data_is_final,
+    item_id,
+    cast(null as int64) as enhanced_sessions,
+    cast(null as int64) as raw_sessions,
+    cast(null as int64) as enhanced_events,
+    cast(null as int64) as raw_events,
+    enhanced_item_revenue,
+    raw_item_revenue,
+    cast(null as float64) as enhanced_purchase_revenue,
+    cast(null as float64) as raw_purchase_revenue,
+    enhanced_item_count,
+    raw_item_count,
+    violation_type
+from
+    item_comparison,
+    unnest([
+        if(round(coalesce(enhanced_item_revenue, 0), 2) != round(coalesce(raw_item_revenue, 0), 2), 'ITEM_REVENUE_MISMATCH_BY_ID', null),
+        if(coalesce(enhanced_item_count, 0) != coalesce(raw_item_count, 0), 'ITEM_COUNT_MISMATCH_BY_ID', null)
+    ]) as violation_type
 where
     violation_type is not null`;
 };
@@ -108,9 +205,10 @@ where
  * Generates a daily quality assertion SQL query.
  *
  * Merges the provided config with defaults, validates, then generates a SQL
- * query comparing daily aggregates (session count, event count, item_revenue)
- * between the enhanced table and raw export data, and checks for missing days
- * and non-final data inflation.
+ * query comparing daily aggregates (session count, event count, item_revenue,
+ * ecommerce.purchase_revenue) between the enhanced table and raw export data,
+ * plus a per-item_id revenue/row-count check on purchase events for shared-final
+ * days. Also checks for missing days and non-final data inflation.
  *
  * @param {string} tableRef - Fully qualified reference to the enhanced table.
  * @param {Object} config - User-provided table configuration.

package/tables/ga4EventsEnhanced/assertions/index.js

@@ -1,11 +1,8 @@
-const { generateItemRevenueAssertionSql, _generateItemRevenueAssertionSql } = require('./itemRevenue.js');
 const { generateDailyQualityAssertionSql, _generateDailyQualityAssertionSql } = require('./dailyQuality.js');
 
 module.exports = {
-    itemRevenue: generateItemRevenueAssertionSql,
     dailyQuality: generateDailyQualityAssertionSql,
     _internal: {
         dailyQuality: { generate: _generateDailyQualityAssertionSql, defaultName: 'daily_quality' },
-        itemRevenue: { generate: _generateItemRevenueAssertionSql, defaultName: 'item_revenue', enabledByDefault: false },
     },
 };

package/tables/ga4EventsEnhanced/assertions/itemRevenue.js

@@ -1,126 +0,0 @@
-const helpers = require('../../../helpers/index.js');
-const utils = require('../../../utils.js');
-const { ga4EventsEnhancedConfig } = require('../config.js');
-const { validateEnhancedEventsConfig } = require('../validation.js');
-const { buildDedupedRawSource } = require('./shared.js');
-
-const defaultConfig = { ...ga4EventsEnhancedConfig };
-
-const ASSERTION_LOOKBACK_DAYS = 5;
-
-// Ecommerce events that carry item data (excluding refund — refunds reverse revenue
-// and are handled separately in some pipelines, but item_revenue on refund rows
-// should still reconcile 1:1 between enhanced and raw).
-const ecommerceEvents = helpers.ga4EcommerceEvents.map(e => `'${e}'`).join(', ');
-
-/**
- * Generates a SQL assertion query that reconciles item_revenue between the
- * enhanced events table and the raw GA4 export data.
- *
- * The query compares item_revenue grouped by (event_date, item_id) for the
- * last 5 days of final data. Returns mismatched rows — 0 rows means the
- * assertion passes.
- *
- * @param {string} tableRef - Fully qualified reference to the enhanced table
- *   (e.g., ctx.ref('ga4_events_enhanced_123456789') in Dataform, or a backtick-quoted string).
- * @param {Object} mergedConfig - Merged table configuration (after merge + validation).
- * @returns {string} SQL query returning violating rows
- */
-const _generateItemRevenueAssertionSql = (tableRef, mergedConfig) => {
-    // excluded events filter (same logic as the enhanced table pipeline)
-    const excludedEvents = mergedConfig.excludedEvents;
-    const excludedEventsSQL = excludedEvents.length > 0
-        ? `event_name not in (${excludedEvents.map(e => `'${e}'`).join(', ')})`
-        : 'true';
-
-    // data_is_final condition for the raw side
-    const dataIsFinalCondition = helpers.isFinalData(
-        mergedConfig.dataIsFinal.detectionMethod,
-        mergedConfig.dataIsFinal.dayThreshold
-    );
-
-    // deduplicated raw-source subquery (mirrors pipeline setPreOperations dedup)
-    const dedupedRawSource = buildDedupedRawSource(mergedConfig, ASSERTION_LOOKBACK_DAYS);
-
-    return `with enhanced_revenue as (
-    select
-        event_date,
-        item.item_id,
-        sum(item.item_revenue) as total_item_revenue,
-        count(*) as item_count
-    from
-        ${tableRef},
-        unnest(items) as item
-    where
-        data_is_final = true
-        and event_date >= date_sub(current_date(), interval ${ASSERTION_LOOKBACK_DAYS} day)
-        and event_name in (${ecommerceEvents})
-    group by event_date, item.item_id
-),
-raw_revenue as (
-    select
-        cast(event_date as date format 'YYYYMMDD') as event_date,
-        item.item_id,
-        sum(item.item_revenue) as total_item_revenue,
-        count(*) as item_count
-    from
-        ${dedupedRawSource},
-        unnest(items) as item
-    where
-        ${excludedEventsSQL}
-        and event_name in (${ecommerceEvents})
-        and ${dataIsFinalCondition}
-    group by event_date, item.item_id
-)
-select
-    coalesce(e.event_date, r.event_date) as event_date,
-    coalesce(e.item_id, r.item_id) as item_id,
-    e.total_item_revenue as enhanced_revenue,
-    r.total_item_revenue as raw_revenue,
-    e.item_count as enhanced_count,
-    r.item_count as raw_count
-from
-    enhanced_revenue e
-full outer join
-    raw_revenue r using(event_date, item_id)
-where
-    round(coalesce(e.total_item_revenue, 0), 2) != round(coalesce(r.total_item_revenue, 0), 2)
-    or e.item_count != r.item_count
-    or e.event_date is null
-    or r.event_date is null`;
-};
-
-/**
- * Generates an item_revenue reconciliation assertion SQL query.
- *
- * Merges the provided config with defaults, validates, then generates a SQL
- * query comparing item_revenue between the enhanced table and raw export data.
- *
- * @param {string} tableRef - Fully qualified reference to the enhanced table.
- * @param {Object} config - User-provided table configuration.
- * @returns {string} SQL query returning violating rows (0 rows = pass)
- */
-const generateItemRevenueAssertionSql = (tableRef, config) => {
-    if (!tableRef || typeof tableRef !== 'string' || !tableRef.trim()) {
-        throw new Error('assertions.itemRevenue: tableRef is required and must be a non-empty string (e.g., ctx.ref(\'table_name\') or \'`project.dataset.table`\').');
-    }
-    const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
-
-    // The assertion interpolates sourceTable directly into SQL (no Dataform ctx available).
-    // If sourceTable is still a Dataform reference object, it would render as [object Object].
-    if (utils.isDataformTableReferenceObject(mergedConfig.sourceTable)) {
-        throw new Error(
-            'assertions.itemRevenue: config.sourceTable is a Dataform table reference object, but assertions do not have access to Dataform context to resolve it. ' +
-            'Resolve it with ctx.ref() before passing it to the assertion:\n\n' +
-            '  .query(ctx => ga4EventsEnhanced.assertions.itemRevenue(\n' +
-            '    ctx.ref(\'enhanced_table_name\'),\n' +
-            '    { ...config, sourceTable: ctx.ref(config.sourceTable) }\n' +
-            '  ))'
-        );
-    }
-
-    validateEnhancedEventsConfig(mergedConfig, { skipDataformContextFields: true });
-    return _generateItemRevenueAssertionSql(tableRef, mergedConfig);
-};
-
-module.exports = { generateItemRevenueAssertionSql, _generateItemRevenueAssertionSql };