ga4-export-fixer 0.1.5-dev.0 → 0.1.5-dev.1

package/README.md

@@ -33,7 +33,7 @@ Include the package in the package.json file in your Dataform repository.
 {
   "dependencies": {
     "@dataform/core": "3.0.42",
-    "ga4-export-fixer": "0.1.5-dev.0"
+    "ga4-export-fixer": "0.1.4"
   }
 }
 ```

package/helpers.js

@@ -139,66 +139,90 @@ const ga4ExportDateFilter = (exportType, start, end) => {
   }
 };
 
-// Filter the export tables by date range for both intraday and daily exports
 /**
- * Generates a SQL filter condition for GA4 export tables based on the provided configuration.
- *
- * This function produces a composite filter condition that constrains which BigQuery GA4 export tables
- * (daily and, optionally, intraday) are included based on start and end dates, the operational mode (test, incremental, or full refresh),
- * and any configured buffer days for session overlap.
- *
- * Logic:
- *   - In test mode: Uses `testConfig.dateRangeStart` and `testConfig.dateRangeEnd` for the table date filters.
- *   - In incremental refresh mode: Uses dynamic variable placeholders for efficient incremental logic (`constants.DATE_RANGE_START_VARIABLE`, etc).
- *   - Otherwise (full refresh): Uses static date values from the configuration's preOperations.
- *   - If `includedExportTypes.intraday` is true, includes a filter for intraday tables with their own start date variable.
- *   - Applies `bufferDays` to the daily export's start boundary to ensure session completeness across date boundaries.
- *
- * Example output (SQL snippet):
- *   (
- *     (_table_suffix >= '20240101-1' and _table_suffix <= '20240105')
- *     or (_table_suffix >= 'intraday_20240101' and _table_suffix <= 'intraday_20240105')
- *   )
- *
- * @param {Object} config Configuration object governing the date filtering logic. Expected properties:
- *   @param {boolean} [config.test] Whether to use test configuration dates.
- *   @param {Object} [config.testConfig] Contains `dateRangeStart` and `dateRangeEnd` for tests.
- *   @param {boolean} [config.incremental] Whether to use incremental variable placeholders.
- *   @param {Object} [config.preOperations] Contains `dateRangeStartFullRefresh` and `dateRangeEnd` for full refresh.
- *   @param {Object} [config.includedExportTypes] Should contain `intraday` (boolean).
- *   @param {number} [config.bufferDays] Number of buffer days to extend date range for daily exports.
- * @returns {string} SQL condition as a string that can be injected into a WHERE clause.
+ * Builds a `_table_suffix` WHERE clause for GA4 BigQuery export tables (daily and/or intraday).
+ *
+ * Date boundaries are resolved differently depending on the mode:
+ *   - **test** -- literal dates from `config.testConfig`
+ *   - **incremental** -- BigQuery variable placeholders set by pre-operations
+ *   - **full refresh** -- static dates from `config.preOperations`
+ *
+ * `bufferDays` is subtracted from the daily start date so sessions that span
+ * midnight are not partially excluded.
+ *
+ * When both daily and intraday exports are enabled, the intraday start date
+ * comes from a dedicated variable (`INTRADAY_DATE_RANGE_START_VARIABLE`) so
+ * intraday tables that already have a corresponding daily table are excluded.
+ * When only intraday is enabled, the daily start-date logic (including buffer
+ * days) is reused instead.
+ *
+ * @param {Object} config
+ * @param {boolean}  config.test                      - Use literal test dates.
+ * @param {Object}   config.testConfig                - `{ dateRangeStart, dateRangeEnd }`.
+ * @param {boolean}  config.incremental               - Use BigQuery variable placeholders.
+ * @param {Object}   config.preOperations             - `{ dateRangeStartFullRefresh, dateRangeEnd }`.
+ * @param {Object}   config.includedExportTypes       - `{ daily: boolean, intraday: boolean }`.
+ * @param {number}  [config.bufferDays=0]             - Extra days subtracted from the start date.
+ * @returns {string} SQL fragment for a WHERE clause.
  */
 const ga4ExportDateFilters = (config) => {
   const bufferDays = config.bufferDays || 0;
 
   const getStartDate = () => {
+    //test mode
     if (config.test) {
       return config.testConfig.dateRangeStart;
     }
     if (config.incremental) {
       return constants.DATE_RANGE_START_VARIABLE;
     }
+    // full refresh
     return config.preOperations.dateRangeStartFullRefresh;
   };
 
   const getEndDate = () => {
+    // test mode, avoid using a BigQuery variable
     if (config.test) {
       return config.testConfig.dateRangeEnd;
     }
+    // use checkpoint variable with incremental refresh -> allows pre processing any part of the table without having to do a full refresh
     if (config.incremental) {
       return constants.DATE_RANGE_END_VARIABLE;
     }
+    // full refresh
     return config.preOperations.dateRangeEnd;
   };
 
-  const start = getStartDate();
+  const getIntradayStartDate = () => {
+    // In test mode, skip pre-operations even though intraday and daily tables may temporarily overlap.
+    if (config.test) {
+      return config.testConfig.dateRangeStart;
+    }
+    // Dedicated variable excludes intraday tables that overlap with already-processed daily tables.
+    if (config.includedExportTypes.intraday && config.includedExportTypes.daily) {
+      return constants.INTRADAY_DATE_RANGE_START_VARIABLE;
+    }
+    // Without daily export, reuse the daily start-date logic and apply bufferDays
+    // (buffer is normally only applied to the daily start date).
+    if (config.includedExportTypes.intraday && !config.includedExportTypes.daily) {
+      // use the same start date as if daily export was in use
+      // include the buffer days as well (not included otherwise for intraday data)
+      return `${getStartDate()}-${bufferDays}`;
+    }
+  };
+
+  const dailyStart = `${getStartDate()}-${bufferDays}`;
+  const intradayStart = getIntradayStartDate();
   const end = getEndDate();
-  const intradayStart = config.test ? config.testConfig.dateRangeStart : constants.INTRADAY_DATE_RANGE_START_VARIABLE;
+  
+
+  const dateFilters = [
+    config.includedExportTypes.daily ? ga4ExportDateFilter('daily', dailyStart, end) : null,
+    config.includedExportTypes.intraday ? ga4ExportDateFilter('intraday', intradayStart, end) : null,
+  ];
 
   return `(
-    ${ga4ExportDateFilter('daily', `${start}-${bufferDays}`, end)}
-    ${config.includedExportTypes.intraday ? `or ${ga4ExportDateFilter('intraday', intradayStart, end)}` : ''}
+    ${dateFilters.filter(filter => !!filter).join(' or ')}
   )`;
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.1.5-dev.0",
+  "version": "0.1.5-dev.1",
   "description": "",
   "main": "index.js",
   "files": [
@@ -14,6 +14,7 @@
   "scripts": {
     "test": "node tests/ga4EventsEnhanced.test.js",
     "test:events": "node tests/ga4EventsEnhanced.test.js",
+    "test:merge": "node tests/mergeSQLConfigurations.test.js",
     "prepublishOnly": "node scripts/updateReadme.js"
   },
   "repository": {

package/preOperations.js

@@ -140,9 +140,9 @@ const setPreOperations = (config) => {
     {
       type: 'variable',
       name: constants.INTRADAY_DATE_RANGE_START_VARIABLE,
-      // variable only needed with incremental refresh and intraday export tables
-      value: config.includedExportTypes.intraday ? getDateRangeStartIntraday(config) : undefined,
-      comment: 'Define the date range start for intraday export tables. Avoid returning intraday data if it overlaps with daily export data.',
+      // variable only needed if intraday export tables are included together with daily export tables
+      value: config.includedExportTypes.intraday && config.includedExportTypes.daily ? getDateRangeStartIntraday(config) : undefined,
+      comment: 'Define the date range start for intraday export tables. Avoid returning intraday data if it overlaps with daily export data. Only needed if intraday export tables are included together with daily export tables.',
     },
     {
       type: 'variable',

package/tables/ga4EventsEnhanced.js

@@ -216,7 +216,7 @@ const generateEnhancedEventsSQL = (config) => {
     const mainTimestampColumn = mergedConfig.customTimestampParam ? 'event_custom_timestamp' : 'event_timestamp';
 
     // exlude these events from the table
-    const excludedEvents = utils.mergeUniqueArrays(mergedConfig.defaultExcludedEvents, mergedConfig.excludedEvents);
+    const excludedEvents = mergedConfig.excludedEvents;
     const excludedEventsSQL = excludedEvents.length > 0 ? `and event_name not in (${excludedEvents.map(event => `'${event}'`).join(',')})` : '';
 
     // promote these event parameters to columns
@@ -230,7 +230,7 @@ const generateEnhancedEventsSQL = (config) => {
     };
 
     const getExcludedColumns = () => {
-        const allExcludedColumns = utils.mergeUniqueArrays(mergedConfig.defaultExcludedColumns, mergedConfig.excludedColumns);
+        const allExcludedColumns = mergedConfig.excludedColumns;
         const excludedColumns = {};
         allExcludedColumns.forEach(c => {
             excludedColumns[c] = undefined;
@@ -261,10 +261,7 @@ const generateEnhancedEventsSQL = (config) => {
             page: helpers.extractPageDetails(),
             // event parameters and user properties
             ...promotedEventParameters(),
-            event_params: helpers.filterEventParams(utils.mergeUniqueArrays(
-                mergedConfig.defaultExcludedEventParams, mergedConfig.excludedEventParams),
-                'exclude'
-            ),
+            event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
             user_properties: 'user_properties',
             // traffic source
             collected_traffic_source: 'collected_traffic_source',

package/utils.js

@@ -87,7 +87,10 @@ ${groupByClause}`;
  * 
  * Rules:
  * - Nested objects are merged recursively key by key
- * - Arrays are overwritten by user input
+ * - Arrays with a "default" counterpart (e.g. excludedEvents + defaultExcludedEvents)
+ *   are merged with mergeUniqueArrays, with user values taking precedence
+ * - Arrays that are themselves a "default" version, or have no default counterpart,
+ *   are overwritten by user input
  * - Default values are preserved unless explicitly overridden (including with undefined)
  * - Explicitly setting a value to undefined in inputConfig will override the default
  * - Date fields: after merging, specific date fields (listed in dateFields) are processed via processDate().
@@ -120,9 +123,16 @@ const mergeSQLConfigurations = (defaultConfig, inputConfig = {}) => {
             continue;
         }
 
-        // Handle arrays: overwrite with user input
+        // Handle arrays: merge with default counterpart if one exists, otherwise overwrite
         if (Array.isArray(defaultValue) && Array.isArray(inputValue)) {
-            result[key] = inputValue;
+            // check if the array has a "default" counterpart
+            // for example, excludedEvents and defaultExcludedEvents
+            const defaultKey = 'default' + key.charAt(0).toUpperCase() + key.slice(1);
+            if (!key.startsWith('default') && defaultKey in result) {
+                result[key] = mergeUniqueArrays(inputValue, result[defaultKey]);
+            } else {
+                result[key] = inputValue;
+            }
             continue;
         }