ga4-export-fixer 0.9.0-dev.3 → 0.9.0-dev.4

package/helpers/ga4Transforms.js

@@ -95,50 +95,50 @@ const isFinalData = (detectionMethod, dayThreshold) => {
 };
 
 /**
- * Checks whether a given column name is part of the standard/expected GA4 BigQuery export columns.
+ * The standard GA4 BigQuery export top-level column names, based on the official schema.
  *
- * The list of recognized GA4 export columns is based on the official schema as of 2026-02-18.
- * This function can be used to filter or validate column names when processing GA4 data exports.
+ * list updated 2026-02-18
+ */
+const ga4ExportColumns = [
+  "event_date",
+  "event_timestamp",
+  "event_name",
+  "event_params",
+  "event_previous_timestamp",
+  "event_value_in_usd",
+  "event_bundle_sequence_id",
+  "event_server_timestamp_offset",
+  "user_id",
+  "user_pseudo_id",
+  "privacy_info",
+  "user_properties",
+  "user_first_touch_timestamp",
+  "user_ltv",
+  "device",
+  "geo",
+  "app_info",
+  "traffic_source",
+  "stream_id",
+  "platform",
+  "event_dimensions",
+  "ecommerce",
+  "items",
+  "collected_traffic_source",
+  "is_active_user",
+  "batch_event_index",
+  "batch_page_id",
+  "batch_ordering_id",
+  "session_traffic_source_last_click",
+  "publisher"
+];
+
+/**
+ * Checks whether a given column name is part of the standard GA4 BigQuery export columns.
  *
  * @param {string} columnName - The name of the column to check.
  * @returns {boolean} True if the column name is a GA4 export column, otherwise false.
  */
-const isGa4ExportColumn = (columnName) => {
-  // list updated 2026-02-18
-  const ga4ExportColumns = [
-    "event_date",
-    "event_timestamp",
-    "event_name",
-    "event_params",
-    "event_previous_timestamp",
-    "event_value_in_usd",
-    "event_bundle_sequence_id",
-    "event_server_timestamp_offset",
-    "user_id",
-    "user_pseudo_id",
-    "privacy_info",
-    "user_properties",
-    "user_first_touch_timestamp",
-    "user_ltv",
-    "device",
-    "geo",
-    "app_info",
-    "traffic_source",
-    "stream_id",
-    "platform",
-    "event_dimensions",
-    "ecommerce",
-    "items",
-    "collected_traffic_source",
-    "is_active_user",
-    "batch_event_index",
-    "batch_page_id",
-    "batch_ordering_id",
-    "session_traffic_source_last_click",
-    "publisher"
-  ];
-  return ga4ExportColumns.includes(columnName);
-};
+const isGa4ExportColumn = (columnName) => ga4ExportColumns.includes(columnName);
 
 /**
  * Generates a SQL CASE expression that determines the GA4 export type from a table suffix.
@@ -255,6 +255,7 @@ module.exports = {
   sessionId,
   fixEcommerceStruct,
   isFinalData,
+  ga4ExportColumns,
   isGa4ExportColumn,
   getGa4ExportType,
   itemListAttributionExpr,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.9.0-dev.3",
+  "version": "0.9.0-dev.4",
   "description": "",
   "main": "index.js",
   "files": [
@@ -17,7 +17,7 @@
     "createTable.js"
   ],
   "scripts": {
-    "test": "node tests/ga4EventsEnhanced.test.js && node tests/assertions.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js && node tests/queryBuilder.test.js && node tests/customSteps.test.js && node tests/enrichments.test.js",
+    "test": "node tests/ga4EventsEnhanced.test.js && node tests/assertions.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js && node tests/queryBuilder.test.js && node tests/customSteps.test.js && node tests/enrichments.test.js && node tests/eventDataColumns.test.js && node tests/utils.test.js",
     "test:summary": "node tests/testRunner.js",
     "test:docs": "node tests/documentation.test.js",
     "test:preops": "node tests/preOperations.test.js",
@@ -29,6 +29,8 @@
     "test:queryBuilder": "node tests/queryBuilder.test.js",
     "test:customSteps": "node tests/customSteps.test.js",
     "test:enrichments": "node tests/enrichments.test.js",
+    "test:eventDataColumns": "node tests/eventDataColumns.test.js",
+    "test:utils": "node tests/utils.test.js",
     "test:integration": "node tests/integration/integration.test.js",
     "release:dev": "./scripts/release-dev.sh",
     "readme": "node scripts/updateReadme.js",

package/tables/ga4EventsEnhanced/index.js

@@ -197,51 +197,56 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
         return excludedColumns;
     };
 
-    // initial step: extract data from the export tables
+    // initial step: extract data from the export tables.
+    // Explicit columns first (transforms + package-promoted + user-excluded sentinels);
+    // then pass-through entries for every GA4 export column not already accounted for.
+    // After this, Object.keys(eventDataStep.select.columns) is the complete column set of event_data.
+    const eventDataExplicitColumns = {
+        // exclude default export columns that are not needed
+        // do this first so that the columns defined later are not excluded
+        ...getExcludedColumns(),
+        // date and time
+        event_date: helpers.eventDate,
+        event_datetime: `extract(datetime from timestamp_micros(${helpers.getEventTimestampMicros(mergedConfig.customTimestampParam)}) at time zone '${mergedConfig.timezone}')`,
+        event_timestamp: 'event_timestamp',
+        event_custom_timestamp: mergedConfig.customTimestampParam ? helpers.getEventTimestampMicros(mergedConfig.customTimestampParam) : undefined,
+        // event name
+        event_name: 'event_name',
+        // identifiers
+        session_id: helpers.sessionId,
+        user_pseudo_id: 'user_pseudo_id',
+        user_id: 'user_id',
+        // page
+        page_location: helpers.unnestEventParam('page_location', 'string'),
+        page: helpers.extractPageDetails(),
+        // event parameters and user properties
+        ...promotedEventParameters(),
+        event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
+        user_properties: 'user_properties',
+        // traffic source
+        collected_traffic_source: 'collected_traffic_source',
+        session_traffic_source_last_click: 'session_traffic_source_last_click',
+        user_traffic_source: 'traffic_source',
+        // ecommerce
+        ecommerce: helpers.fixEcommerceStruct('ecommerce'),
+        items: 'items',
+        _item_row_id: itemListAttribution ? helpers.itemRowId(ecommerceEventsFilter) : undefined,
+        // flag if the data is "final" and is not expected to change anymore
+        data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
+        export_type: helpers.getGa4ExportType('_table_suffix'),
+        // prep columns for later steps
+        entrances: helpers.unnestEventParam('entrances', 'int'),
+        session_params_prep: mergedConfig.sessionParams.length > 0 ? helpers.filterEventParams(mergedConfig.sessionParams, 'include') : undefined,
+    };
+    // Pass through every GA4 export column not already covered by an explicit transform,
+    // promotion, exclusion sentinel, or value-side rename in eventDataExplicitColumns.
+    const eventDataPassThroughs = utils.buildPassThroughs(eventDataExplicitColumns, helpers.ga4ExportColumns);
     const eventDataStep = {
         name: 'event_data',
         select: {
             columns: {
-                // exclude default export columns that are not needed
-                // do this first so that the columns defined later are not excluded
-                ...getExcludedColumns(),
-                // date and time
-                event_date: helpers.eventDate,
-                event_datetime: `extract(datetime from timestamp_micros(${helpers.getEventTimestampMicros(mergedConfig.customTimestampParam)}) at time zone '${mergedConfig.timezone}')`,
-                event_timestamp: 'event_timestamp',
-                event_custom_timestamp: mergedConfig.customTimestampParam ? helpers.getEventTimestampMicros(mergedConfig.customTimestampParam) : undefined,
-                // event name
-                event_name: 'event_name',
-                // identifiers
-                session_id: helpers.sessionId,
-                user_pseudo_id: 'user_pseudo_id',
-                user_id: 'user_id',
-                // page
-                page_location: helpers.unnestEventParam('page_location', 'string'),
-                page: helpers.extractPageDetails(),
-                // event parameters and user properties
-                ...promotedEventParameters(),
-                event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
-                user_properties: 'user_properties',
-                // traffic source
-                collected_traffic_source: 'collected_traffic_source',
-                session_traffic_source_last_click: 'session_traffic_source_last_click',
-                user_traffic_source: 'traffic_source',
-                // ecommerce
-                ecommerce: helpers.fixEcommerceStruct('ecommerce'),
-                items: 'items',
-                _item_row_id: itemListAttribution ? helpers.itemRowId(ecommerceEventsFilter) : undefined,
-                // flag if the data is "final" and is not expected to change anymore
-                data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
-                export_type: helpers.getGa4ExportType('_table_suffix'),
-                // prep columns for later steps
-                entrances: helpers.unnestEventParam('entrances', 'int'),
-                session_params_prep: mergedConfig.sessionParams.length > 0 ? helpers.filterEventParams(mergedConfig.sessionParams, 'include') : undefined,
-                // include all other columns from the export data
-                get '[sql]other_columns'() {
-                    const definedColumns = Object.keys(this);
-                    return `* except (${definedColumns.filter(column => helpers.isGa4ExportColumn(column)).join(', ')})`;
-                },
+                ...eventDataExplicitColumns,
+                ...eventDataPassThroughs,
             },
         },
         from: mergedConfig.sourceTable,
@@ -385,20 +390,14 @@ ${excludedEventsSQL}`,
     }
     const enrichmentExcludedColumns = [...enrichmentColumnNames];
 
-    // Only forward enrichment columns to each wildcard's EXCEPT input if they would actually
-    // appear in that wildcard's source CTE. Otherwise BigQuery rejects with "Column X in
-    // SELECT * EXCEPT list does not exist". event_data.* expands to its explicit columns plus
-    // GA4 export columns minus user-excluded ones; session_data.* expands only to its explicit
-    // columns. selectOtherColumns dedupes between internalExcept and externalExcept.
+    // Only forward enrichment columns to each wildcard's EXCEPT input if they actually exist
+    // in that wildcard's source CTE. Otherwise BigQuery rejects with "Column X in SELECT *
+    // EXCEPT list does not exist". After M1, Object.keys(step.select.columns) is the complete
+    // column set of both event_data and session_data — so the same predicate works for both.
     const eventDataExplicit = new Set(Object.keys(eventDataStep.select.columns));
     const sessionDataExplicit = new Set(Object.keys(sessionDataStep.select.columns));
-    const userExcluded = new Set(mergedConfig.excludedColumns);
-    const eventDataEnrichmentExcept = enrichmentExcludedColumns.filter(c =>
-        eventDataExplicit.has(c) || (helpers.isGa4ExportColumn(c) && !userExcluded.has(c))
-    );
-    const sessionDataEnrichmentExcept = enrichmentExcludedColumns.filter(c =>
-        sessionDataExplicit.has(c)
-    );
+    const eventDataEnrichmentExcept = enrichmentExcludedColumns.filter(c => eventDataExplicit.has(c));
+    const sessionDataEnrichmentExcept = enrichmentExcludedColumns.filter(c => sessionDataExplicit.has(c));
 
     // Join event_data and session_data, include additional logic
     // Named 'enhanced_events' so user-supplied customSteps can reference it as a stable handle.

package/utils.js

@@ -521,6 +521,45 @@ const selectOtherColumns = (step, alreadyDefinedColumns = [], excludedColumns =
 };
 
 
+/**
+ * Builds a queryBuilder `select.columns` fragment that passes through every source column
+ * not already covered by an explicit columns object.
+ *
+ * A source column is considered "covered" — and skipped from pass-throughs — when it appears as:
+ *   - a KEY in `explicitColumns` (a transform, package promotion, or undefined-valued exclusion
+ *     sentinel like `{ event_dimensions: undefined }`), OR
+ *   - a VALUE in `explicitColumns` (a bare source-column identifier referenced by a value-side
+ *     rename, e.g. `{ user_traffic_source: 'traffic_source' }` covers 'traffic_source').
+ *
+ * Values that are SQL expressions, function calls, or non-strings never count as coverage —
+ * they reference the source column internally but the column itself is still available as a
+ * pass-through. (`.includes()` compares by strict equality, so 'extract(datetime from ...)'
+ * never matches a bare column name.)
+ *
+ * @param {Object} explicitColumns - A queryBuilder step's explicit `select.columns` entries.
+ * @param {Iterable<string>} sourceColumns - Column names available on the source schema.
+ * @returns {Object} A map of `{ column: column }` entries for every source column not covered.
+ *
+ * @example
+ *   buildPassThroughs(
+ *     { event_name: 'event_name', user_traffic_source: 'traffic_source' },
+ *     ['event_name', 'traffic_source', 'device', 'geo']
+ *   );
+ *   // → { device: 'device', geo: 'geo' }
+ */
+const buildPassThroughs = (explicitColumns, sourceColumns) => {
+    const explicitKeys = Object.keys(explicitColumns);
+    const explicitValues = Object.values(explicitColumns);
+    const passThroughs = {};
+    for (const column of sourceColumns) {
+        if (!explicitKeys.includes(column) && !explicitValues.includes(column)) {
+            passThroughs[column] = column;
+        }
+    }
+    return passThroughs;
+};
+
+
 /**
  * Processes a date input string and returns a corresponding SQL date casting expression,
  * or passes through BigQuery SQL statements as-is.
@@ -596,6 +635,7 @@ module.exports = {
     isDataformTableReferenceObject,
     setDataformContext,
     selectOtherColumns,
+    buildPassThroughs,
     processDate,
     getDatasetName
 };