ga4-export-fixer 0.1.4 → 0.1.5-dev.1

package/README.md

@@ -2,6 +2,20 @@
 
 **ga4-export-fixer** is a **Dataform NPM package** that transforms raw GA4 BigQuery export data into a cleaner, more queryable incremental table. It combines daily and intraday exports so the best available version of each event is always in use, adds session-level fields like `session_id` and `landing_page`, promotes key event parameters to columns, and fixes known GA4 export issues — handling the boilerplate transformations that are otherwise tedious to include in every GA4 query.
 
+The goal of the package is to **speed up development** when building data models and pipelines on top of GA4 export data, allowing you to focus on your use case instead of wrestling with the raw export format.
+
+### Planned Features
+
+- Support for using only intraday export data
+- Tools for building on top of the enhanced events table
+- Column descriptions
+- Default configurations for app and web properties
+- Item list attribution
+- Data enrichment (item-level, session-level, event-level)
+- Support for fresh export (GA4 360)
+- Custom processing steps (additional CTEs)
+- Custom traffic source attribution
+
 ## Installation
 
 ### Bash
@@ -19,7 +33,7 @@ Include the package in the package.json file in your Dataform repository.
 {
   "dependencies": {
     "@dataform/core": "3.0.42",
-    "ga4-export-fixer": "0.1.2"
+    "ga4-export-fixer": "0.1.4"
   }
 }
 ```

package/helpers.js

@@ -139,66 +139,90 @@ const ga4ExportDateFilter = (exportType, start, end) => {
   }
 };
 
-// Filter the export tables by date range for both intraday and daily exports
 /**
- * Generates a SQL filter condition for GA4 export tables based on the provided configuration.
- *
- * This function produces a composite filter condition that constrains which BigQuery GA4 export tables
- * (daily and, optionally, intraday) are included based on start and end dates, the operational mode (test, incremental, or full refresh),
- * and any configured buffer days for session overlap.
- *
- * Logic:
- *   - In test mode: Uses `testConfig.dateRangeStart` and `testConfig.dateRangeEnd` for the table date filters.
- *   - In incremental refresh mode: Uses dynamic variable placeholders for efficient incremental logic (`constants.DATE_RANGE_START_VARIABLE`, etc).
- *   - Otherwise (full refresh): Uses static date values from the configuration's preOperations.
- *   - If `includedExportTypes.intraday` is true, includes a filter for intraday tables with their own start date variable.
- *   - Applies `bufferDays` to the daily export's start boundary to ensure session completeness across date boundaries.
- *
- * Example output (SQL snippet):
- *   (
- *     (_table_suffix >= '20240101-1' and _table_suffix <= '20240105')
- *     or (_table_suffix >= 'intraday_20240101' and _table_suffix <= 'intraday_20240105')
- *   )
- *
- * @param {Object} config Configuration object governing the date filtering logic. Expected properties:
- *   @param {boolean} [config.test] Whether to use test configuration dates.
- *   @param {Object} [config.testConfig] Contains `dateRangeStart` and `dateRangeEnd` for tests.
- *   @param {boolean} [config.incremental] Whether to use incremental variable placeholders.
- *   @param {Object} [config.preOperations] Contains `dateRangeStartFullRefresh` and `dateRangeEnd` for full refresh.
- *   @param {Object} [config.includedExportTypes] Should contain `intraday` (boolean).
- *   @param {number} [config.bufferDays] Number of buffer days to extend date range for daily exports.
- * @returns {string} SQL condition as a string that can be injected into a WHERE clause.
+ * Builds a `_table_suffix` WHERE clause for GA4 BigQuery export tables (daily and/or intraday).
+ *
+ * Date boundaries are resolved differently depending on the mode:
+ *   - **test** -- literal dates from `config.testConfig`
+ *   - **incremental** -- BigQuery variable placeholders set by pre-operations
+ *   - **full refresh** -- static dates from `config.preOperations`
+ *
+ * `bufferDays` is subtracted from the daily start date so sessions that span
+ * midnight are not partially excluded.
+ *
+ * When both daily and intraday exports are enabled, the intraday start date
+ * comes from a dedicated variable (`INTRADAY_DATE_RANGE_START_VARIABLE`) so
+ * intraday tables that already have a corresponding daily table are excluded.
+ * When only intraday is enabled, the daily start-date logic (including buffer
+ * days) is reused instead.
+ *
+ * @param {Object} config
+ * @param {boolean}  config.test                      - Use literal test dates.
+ * @param {Object}   config.testConfig                - `{ dateRangeStart, dateRangeEnd }`.
+ * @param {boolean}  config.incremental               - Use BigQuery variable placeholders.
+ * @param {Object}   config.preOperations             - `{ dateRangeStartFullRefresh, dateRangeEnd }`.
+ * @param {Object}   config.includedExportTypes       - `{ daily: boolean, intraday: boolean }`.
+ * @param {number}  [config.bufferDays=0]             - Extra days subtracted from the start date.
+ * @returns {string} SQL fragment for a WHERE clause.
  */
 const ga4ExportDateFilters = (config) => {
   const bufferDays = config.bufferDays || 0;
 
   const getStartDate = () => {
+    //test mode
     if (config.test) {
       return config.testConfig.dateRangeStart;
     }
     if (config.incremental) {
       return constants.DATE_RANGE_START_VARIABLE;
     }
+    // full refresh
     return config.preOperations.dateRangeStartFullRefresh;
   };
 
   const getEndDate = () => {
+    // test mode, avoid using a BigQuery variable
     if (config.test) {
       return config.testConfig.dateRangeEnd;
     }
+    // use checkpoint variable with incremental refresh -> allows pre processing any part of the table without having to do a full refresh
     if (config.incremental) {
       return constants.DATE_RANGE_END_VARIABLE;
     }
+    // full refresh
     return config.preOperations.dateRangeEnd;
   };
 
-  const start = getStartDate();
+  const getIntradayStartDate = () => {
+    // In test mode, skip pre-operations even though intraday and daily tables may temporarily overlap.
+    if (config.test) {
+      return config.testConfig.dateRangeStart;
+    }
+    // Dedicated variable excludes intraday tables that overlap with already-processed daily tables.
+    if (config.includedExportTypes.intraday && config.includedExportTypes.daily) {
+      return constants.INTRADAY_DATE_RANGE_START_VARIABLE;
+    }
+    // Without daily export, reuse the daily start-date logic and apply bufferDays
+    // (buffer is normally only applied to the daily start date).
+    if (config.includedExportTypes.intraday && !config.includedExportTypes.daily) {
+      // use the same start date as if daily export was in use
+      // include the buffer days as well (not included otherwise for intraday data)
+      return `${getStartDate()}-${bufferDays}`;
+    }
+  };
+
+  const dailyStart = `${getStartDate()}-${bufferDays}`;
+  const intradayStart = getIntradayStartDate();
   const end = getEndDate();
-  const intradayStart = config.test ? config.testConfig.dateRangeStart : constants.INTRADAY_DATE_RANGE_START_VARIABLE;
+  
+
+  const dateFilters = [
+    config.includedExportTypes.daily ? ga4ExportDateFilter('daily', dailyStart, end) : null,
+    config.includedExportTypes.intraday ? ga4ExportDateFilter('intraday', intradayStart, end) : null,
+  ];
 
   return `(
-    ${ga4ExportDateFilter('daily', `${start}-${bufferDays}`, end)}
-    ${config.includedExportTypes.intraday ? `or ${ga4ExportDateFilter('intraday', intradayStart, end)}` : ''}
+    ${dateFilters.filter(filter => !!filter).join(' or ')}
   )`;
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.1.4",
+  "version": "0.1.5-dev.1",
   "description": "",
   "main": "index.js",
   "files": [
@@ -14,11 +14,12 @@
   "scripts": {
     "test": "node tests/ga4EventsEnhanced.test.js",
     "test:events": "node tests/ga4EventsEnhanced.test.js",
+    "test:merge": "node tests/mergeSQLConfigurations.test.js",
     "prepublishOnly": "node scripts/updateReadme.js"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tanelytics/ga4-export-fixer"
+    "url": "git+https://github.com/tanelytics/ga4-export-fixer.git"
   },
   "author": "Taneli Salonen",
   "license": "MIT",

package/preOperations.js

@@ -140,9 +140,9 @@ const setPreOperations = (config) => {
     {
       type: 'variable',
       name: constants.INTRADAY_DATE_RANGE_START_VARIABLE,
-      // variable only needed with incremental refresh and intraday export tables
-      value: config.includedExportTypes.intraday ? getDateRangeStartIntraday(config) : undefined,
-      comment: 'Define the date range start for intraday export tables. Avoid returning intraday data if it overlaps with daily export data.',
+      // variable only needed if intraday export tables are included together with daily export tables
+      value: config.includedExportTypes.intraday && config.includedExportTypes.daily ? getDateRangeStartIntraday(config) : undefined,
+      comment: 'Define the date range start for intraday export tables. Avoid returning intraday data if it overlaps with daily export data. Only needed if intraday export tables are included together with daily export tables.',
     },
     {
       type: 'variable',

package/tables/ga4EventsEnhanced.js

@@ -52,11 +52,13 @@ const defaultConfig = {
         //{name: 'page_location', type: 'string', columnName: 'page_location2'},
     ],
     sessionParams: [],
-    defaultExcludedEvents: [
+    defaultExcludedEvents: [],
+    // session_start and first_visit are excluded via the excludedEvents array
+    // this allows the user to include them if needed
+    excludedEvents: [
         'session_start',
         'first_visit'
     ],
-    excludedEvents: [],
     defaultExcludedColumns: [
         'event_dimensions', // legacy column, not needed
         'traffic_source', // renamed to user_traffic_source
@@ -214,7 +216,7 @@ const generateEnhancedEventsSQL = (config) => {
     const mainTimestampColumn = mergedConfig.customTimestampParam ? 'event_custom_timestamp' : 'event_timestamp';
 
     // exlude these events from the table
-    const excludedEvents = utils.mergeUniqueArrays(mergedConfig.defaultExcludedEvents, mergedConfig.excludedEvents);
+    const excludedEvents = mergedConfig.excludedEvents;
     const excludedEventsSQL = excludedEvents.length > 0 ? `and event_name not in (${excludedEvents.map(event => `'${event}'`).join(',')})` : '';
 
     // promote these event parameters to columns
@@ -228,7 +230,7 @@ const generateEnhancedEventsSQL = (config) => {
     };
 
     const getExcludedColumns = () => {
-        const allExcludedColumns = utils.mergeUniqueArrays(mergedConfig.defaultExcludedColumns, mergedConfig.excludedColumns);
+        const allExcludedColumns = mergedConfig.excludedColumns;
         const excludedColumns = {};
         allExcludedColumns.forEach(c => {
             excludedColumns[c] = undefined;
@@ -259,10 +261,7 @@ const generateEnhancedEventsSQL = (config) => {
             page: helpers.extractPageDetails(),
             // event parameters and user properties
             ...promotedEventParameters(),
-            event_params: helpers.filterEventParams(utils.mergeUniqueArrays(
-                mergedConfig.defaultExcludedEventParams, mergedConfig.excludedEventParams),
-                'exclude'
-            ),
+            event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
             user_properties: 'user_properties',
             // traffic source
             collected_traffic_source: 'collected_traffic_source',

package/utils.js

@@ -87,7 +87,10 @@ ${groupByClause}`;
  * 
  * Rules:
  * - Nested objects are merged recursively key by key
- * - Arrays are concatenated and deduplicated
+ * - Arrays with a "default" counterpart (e.g. excludedEvents + defaultExcludedEvents)
+ *   are merged with mergeUniqueArrays, with user values taking precedence
+ * - Arrays that are themselves a "default" version, or have no default counterpart,
+ *   are overwritten by user input
  * - Default values are preserved unless explicitly overridden (including with undefined)
  * - Explicitly setting a value to undefined in inputConfig will override the default
  * - Date fields: after merging, specific date fields (listed in dateFields) are processed via processDate().
@@ -120,9 +123,16 @@ const mergeSQLConfigurations = (defaultConfig, inputConfig = {}) => {
             continue;
         }
 
-        // Handle arrays: concatenate and remove duplicates
+        // Handle arrays: merge with default counterpart if one exists, otherwise overwrite
         if (Array.isArray(defaultValue) && Array.isArray(inputValue)) {
-            result[key] = mergeUniqueArrays(defaultValue, inputValue);
+            // check if the array has a "default" counterpart
+            // for example, excludedEvents and defaultExcludedEvents
+            const defaultKey = 'default' + key.charAt(0).toUpperCase() + key.slice(1);
+            if (!key.startsWith('default') && defaultKey in result) {
+                result[key] = mergeUniqueArrays(inputValue, result[defaultKey]);
+            } else {
+                result[key] = inputValue;
+            }
             continue;
         }