ga4-export-fixer 0.7.1 → 0.8.0-dev.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.7.1",
+  "version": "0.8.0-dev.2",
   "description": "",
   "main": "index.js",
   "files": [
@@ -17,7 +17,7 @@
     "createTable.js"
   ],
   "scripts": {
-    "test": "node tests/ga4EventsEnhanced.test.js && node tests/assertions.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js",
+    "test": "node tests/ga4EventsEnhanced.test.js && node tests/assertions.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js && node tests/queryBuilder.test.js && node tests/customSteps.test.js",
     "test:summary": "node tests/testRunner.js",
     "test:docs": "node tests/documentation.test.js",
     "test:preops": "node tests/preOperations.test.js",
@@ -26,6 +26,8 @@
     "test:validation": "node tests/inputValidation.test.js",
     "test:assertions": "node tests/assertions.test.js",
     "test:createTable": "node tests/createTable.test.js",
+    "test:queryBuilder": "node tests/queryBuilder.test.js",
+    "test:customSteps": "node tests/customSteps.test.js",
     "test:integration": "node tests/integration/integration.test.js",
     "release:dev": "./scripts/release-dev.sh",
     "readme": "node scripts/updateReadme.js",

package/tables/ga4EventsEnhanced/config.js

@@ -1,70 +1,73 @@
-const { baseConfig } = require('../../defaultConfig.js');
-
-/*
-The default configuration for the GA4 Events Enhanced table.
-*/
-const ga4EventsEnhancedConfig = {
-    ...baseConfig,
-    sourceTable: undefined,
-    sourceTableType: 'GA4_EXPORT', // used with pre operations to detect if ga4 export specific pre operations are needed
-    // optional but recommended
-    schemaLock: undefined,
-    // only used with js tables
-    dataformTableConfig: {
-        type: 'incremental',
-        bigquery: {
-            partitionBy: 'event_date',
-            clusterBy: ['event_name', 'session_id', 'page_location', 'data_is_final'],
-            labels: {
-                'ga4_export_fixer': 'true'
-            }
-        },
-        onSchemaChange: 'EXTEND',
-        tags: ['ga4_export_fixer'],
-    },
-    // optional
-    includedExportTypes: {
-        daily: true,
-        fresh: false,
-        intraday: true,
-    },
-    timezone: 'Etc/UTC',
-    customTimestampParam: undefined,
-    dataIsFinal: {
-        detectionMethod: 'DAY_THRESHOLD', // 'EXPORT_TYPE' or 'DAY_THRESHOLD'
-        dayThreshold: 3 // only used if detectionMethod is 'DAY_THRESHOLD'
-        // according to GA4 documentation, the data up to 72 hours old is subject to possible changes
-        // in reality, there have been cases where the data has changed even after 72 hours (4 day window would have covered these)
-    },
-    // optional item list attribution - disabled by default (compute-heavy, only useful for ecommerce sites)
-    itemListAttribution: undefined,
-    // number of additional days to take in for taking into account sessions that overlap days
-    bufferDays: 1,
-    // these parameters are excluded by default because they've been made available in other columns
-    defaultExcludedEventParams: [
-        'page_location',
-        'ga_session_id',
-        //'custom_event_timestamp', // removed if customTimestampParam is used
-    ],
-    excludedEventParams: [],
-    eventParamsToColumns: [
-        //{name: 'page_location', type: 'string', columnName: 'page_location2'},
-    ],
-    sessionParams: [],
-    defaultExcludedEvents: [],
-    // session_start and first_visit are excluded via the excludedEvents array
-    // this allows the user to include them if needed
-    excludedEvents: [
-        'session_start',
-        'first_visit'
-    ],
-    defaultExcludedColumns: [
-        'event_dimensions', // legacy column, not needed
-        'traffic_source', // renamed to user_traffic_source
-        'session_id'
-    ],
-    // exclude these columns when extracting raw data from the export tables
-    excludedColumns: [],
-};
-
-module.exports = { ga4EventsEnhancedConfig };
+const { baseConfig } = require('../../defaultConfig.js');
+
+/*
+The default configuration for the GA4 Events Enhanced table.
+*/
+const ga4EventsEnhancedConfig = {
+    ...baseConfig,
+    sourceTable: undefined,
+    sourceTableType: 'GA4_EXPORT', // used with pre operations to detect if ga4 export specific pre operations are needed
+    // optional but recommended
+    schemaLock: undefined,
+    // only used with js tables
+    dataformTableConfig: {
+        type: 'incremental',
+        bigquery: {
+            partitionBy: 'event_date',
+            clusterBy: ['event_name', 'session_id', 'page_location', 'data_is_final'],
+            labels: {
+                'ga4_export_fixer': 'true'
+            }
+        },
+        onSchemaChange: 'EXTEND',
+        tags: ['ga4_export_fixer'],
+    },
+    // optional
+    includedExportTypes: {
+        daily: true,
+        fresh: false,
+        intraday: true,
+    },
+    timezone: 'Etc/UTC',
+    customTimestampParam: undefined,
+    dataIsFinal: {
+        detectionMethod: 'DAY_THRESHOLD', // 'EXPORT_TYPE' or 'DAY_THRESHOLD'
+        dayThreshold: 3 // only used if detectionMethod is 'DAY_THRESHOLD'
+        // according to GA4 documentation, the data up to 72 hours old is subject to possible changes
+        // in reality, there have been cases where the data has changed even after 72 hours (4 day window would have covered these)
+    },
+    // optional item list attribution - disabled by default (compute-heavy, only useful for ecommerce sites)
+    itemListAttribution: undefined,
+    // number of additional days to take in for taking into account sessions that overlap days
+    bufferDays: 1,
+    // these parameters are excluded by default because they've been made available in other columns
+    defaultExcludedEventParams: [
+        'page_location',
+        'ga_session_id',
+        //'custom_event_timestamp', // removed if customTimestampParam is used
+    ],
+    excludedEventParams: [],
+    eventParamsToColumns: [
+        //{name: 'page_location', type: 'string', columnName: 'page_location2'},
+    ],
+    sessionParams: [],
+    defaultExcludedEvents: [],
+    // session_start and first_visit are excluded via the excludedEvents array
+    // this allows the user to include them if needed
+    excludedEvents: [
+        'session_start',
+        'first_visit'
+    ],
+    defaultExcludedColumns: [
+        'event_dimensions', // legacy column, not needed
+        'traffic_source', // renamed to user_traffic_source
+        'session_id'
+    ],
+    // exclude these columns when extracting raw data from the export tables
+    excludedColumns: [],
+    // user-defined CTEs appended to the pipeline after enhanced_events
+    // each entry is a queryBuilder step (raw {name, query} or structured {name, select, from, ...})
+    customSteps: [],
+};
+
+module.exports = { ga4EventsEnhancedConfig };

package/tables/ga4EventsEnhanced/index.js

@@ -106,9 +106,9 @@ const getFinalColumnOrder = (eventDataStep, sessionDataStep) => {
     // Construct the columns object: key is column name, value is {step.name}.{column}
     const columnOrder = {};
     for (const col of finalColumnOrder) {
-        if (sessionDataStep?.columns?.hasOwnProperty(col) && sessionDataStep.columns[col] !== undefined) {
+        if (sessionDataStep?.select?.columns?.hasOwnProperty(col) && sessionDataStep.select.columns[col] !== undefined) {
             columnOrder[col] = `${sessionDataStep.name}.${col}`;
-        } else if (eventDataStep?.columns?.hasOwnProperty(col) && eventDataStep.columns[col] !== undefined) {
+        } else if (eventDataStep?.select?.columns?.hasOwnProperty(col) && eventDataStep.select.columns[col] !== undefined) {
             columnOrder[col] = `${eventDataStep.name}.${col}`;
         }
     }
@@ -200,46 +200,48 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
     // initial step: extract data from the export tables
     const eventDataStep = {
         name: 'event_data',
-        columns: {
-            // exclude default export columns that are not needed
-            // do this first so that the columns defined later are not excluded
-            ...getExcludedColumns(),
-            // date and time
-            event_date: helpers.eventDate,
-            event_datetime: `extract(datetime from timestamp_micros(${helpers.getEventTimestampMicros(mergedConfig.customTimestampParam)}) at time zone '${mergedConfig.timezone}')`,
-            event_timestamp: 'event_timestamp',
-            event_custom_timestamp: mergedConfig.customTimestampParam ? helpers.getEventTimestampMicros(mergedConfig.customTimestampParam) : undefined,
-            // event name
-            event_name: 'event_name',
-            // identifiers
-            session_id: helpers.sessionId,
-            user_pseudo_id: 'user_pseudo_id',
-            user_id: 'user_id',
-            // page
-            page_location: helpers.unnestEventParam('page_location', 'string'),
-            page: helpers.extractPageDetails(),
-            // event parameters and user properties
-            ...promotedEventParameters(),
-            event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
-            user_properties: 'user_properties',
-            // traffic source
-            collected_traffic_source: 'collected_traffic_source',
-            session_traffic_source_last_click: 'session_traffic_source_last_click',
-            user_traffic_source: 'traffic_source',
-            // ecommerce
-            ecommerce: helpers.fixEcommerceStruct('ecommerce'),
-            items: 'items',
-            _item_list_attribution_row_id: itemListAttribution ? helpers.itemListAttributionRowId(ecommerceEventsFilter) : undefined,
-            // flag if the data is "final" and is not expected to change anymore
-            data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
-            export_type: helpers.getGa4ExportType('_table_suffix'),
-            // prep columns for later steps
-            entrances: helpers.unnestEventParam('entrances', 'int'),
-            session_params_prep: mergedConfig.sessionParams.length > 0 ? helpers.filterEventParams(mergedConfig.sessionParams, 'include') : undefined,
-            // include all other columns from the export data
-            get '[sql]other_columns'() {
-                const definedColumns = Object.keys(this);
-                return `* except (${definedColumns.filter(column => helpers.isGa4ExportColumn(column)).join(', ')})`;
+        select: {
+            columns: {
+                // exclude default export columns that are not needed
+                // do this first so that the columns defined later are not excluded
+                ...getExcludedColumns(),
+                // date and time
+                event_date: helpers.eventDate,
+                event_datetime: `extract(datetime from timestamp_micros(${helpers.getEventTimestampMicros(mergedConfig.customTimestampParam)}) at time zone '${mergedConfig.timezone}')`,
+                event_timestamp: 'event_timestamp',
+                event_custom_timestamp: mergedConfig.customTimestampParam ? helpers.getEventTimestampMicros(mergedConfig.customTimestampParam) : undefined,
+                // event name
+                event_name: 'event_name',
+                // identifiers
+                session_id: helpers.sessionId,
+                user_pseudo_id: 'user_pseudo_id',
+                user_id: 'user_id',
+                // page
+                page_location: helpers.unnestEventParam('page_location', 'string'),
+                page: helpers.extractPageDetails(),
+                // event parameters and user properties
+                ...promotedEventParameters(),
+                event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
+                user_properties: 'user_properties',
+                // traffic source
+                collected_traffic_source: 'collected_traffic_source',
+                session_traffic_source_last_click: 'session_traffic_source_last_click',
+                user_traffic_source: 'traffic_source',
+                // ecommerce
+                ecommerce: helpers.fixEcommerceStruct('ecommerce'),
+                items: 'items',
+                _item_list_attribution_row_id: itemListAttribution ? helpers.itemListAttributionRowId(ecommerceEventsFilter) : undefined,
+                // flag if the data is "final" and is not expected to change anymore
+                data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
+                export_type: helpers.getGa4ExportType('_table_suffix'),
+                // prep columns for later steps
+                entrances: helpers.unnestEventParam('entrances', 'int'),
+                session_params_prep: mergedConfig.sessionParams.length > 0 ? helpers.filterEventParams(mergedConfig.sessionParams, 'include') : undefined,
+                // include all other columns from the export data
+                get '[sql]other_columns'() {
+                    const definedColumns = Object.keys(this);
+                    return `* except (${definedColumns.filter(column => helpers.isGa4ExportColumn(column)).join(', ')})`;
+                },
             },
         },
         from: mergedConfig.sourceTable,
@@ -250,18 +252,20 @@ ${excludedEventsSQL}`,
     // Do session-level data aggregation
     const sessionDataStep = {
         name: 'session_data',
-        columns: {
-            session_id: 'session_id',
-            user_id: helpers.aggregateValue('user_id', 'last', timestampColumn),
-            merged_user_id: `ifnull(${helpers.aggregateValue('user_id', 'last', timestampColumn)}, any_value(user_pseudo_id))`,
-            session_params: helpers.aggregateSessionParams(mergedConfig.sessionParams, 'session_params_prep', timestampColumn),
-            session_traffic_source_last_click: helpers.aggregateValue('session_traffic_source_last_click', 'first', timestampColumn),
-            session_first_traffic_source: `array_agg(collected_traffic_source order by ${timestampColumn} limit 1)[safe_offset(0)]`, // don't ignore nulls
-            landing_page: helpers.aggregateValue(`if(entrances > 0, page, null)`, 'first', timestampColumn),
+        select: {
+            columns: {
+                session_id: 'session_id',
+                user_id: helpers.aggregateValue('user_id', 'last', timestampColumn),
+                merged_user_id: `ifnull(${helpers.aggregateValue('user_id', 'last', timestampColumn)}, any_value(user_pseudo_id))`,
+                session_params: helpers.aggregateSessionParams(mergedConfig.sessionParams, 'session_params_prep', timestampColumn),
+                session_traffic_source_last_click: helpers.aggregateValue('session_traffic_source_last_click', 'first', timestampColumn),
+                session_first_traffic_source: `array_agg(collected_traffic_source order by ${timestampColumn} limit 1)[safe_offset(0)]`, // don't ignore nulls
+                landing_page: helpers.aggregateValue(`if(entrances > 0, page, null)`, 'first', timestampColumn),
+            },
         },
         from: 'event_data',
         where: `session_id is not null`,
-        groupBy: ['session_id']
+        'group by': 'session_id',
     };
 
     // item list attribution CTEs:
@@ -277,11 +281,13 @@ ${excludedEventsSQL}`,
 
         const attributionStep = {
             name: 'item_list_attribution',
-            columns: {
-                '_item_list_attribution_row_id': '_item_list_attribution_row_id',
-                'event_name': 'event_name',
-                'item': 'item',
-                '_item_list_attr': attrExpr,
+            select: {
+                columns: {
+                    '_item_list_attribution_row_id': '_item_list_attribution_row_id',
+                    'event_name': 'event_name',
+                    'item': 'item',
+                    '_item_list_attr': attrExpr,
+                },
             },
             from: 'event_data, unnest(items) as item',
             where: `event_name in (${ecommerceEventsFilter})`,
@@ -289,18 +295,20 @@ ${excludedEventsSQL}`,
 
         const dataStep = {
             name: 'item_list_data',
-            columns: {
-                '_item_list_attribution_row_id': '_item_list_attribution_row_id',
-                'items': `array_agg(
+            select: {
+                columns: {
+                    '_item_list_attribution_row_id': '_item_list_attribution_row_id',
+                    'items': `array_agg(
       (select as struct item.* replace(
         coalesce(if(${passthroughEvents}, item.item_list_name, _item_list_attr.item_list_name), '(not set)') as item_list_name,
         coalesce(if(${passthroughEvents}, item.item_list_id, _item_list_attr.item_list_id), '(not set)') as item_list_id,
         coalesce(if(${passthroughEvents}, item.item_list_index, _item_list_attr.item_list_index)) as item_list_index
       ))
     )`,
+                },
             },
             from: 'item_list_attribution',
-            groupBy: ['_item_list_attribution_row_id'],
+            'group by': '_item_list_attribution_row_id',
         };
 
         return [attributionStep, dataStep];
@@ -316,56 +324,79 @@ ${excludedEventsSQL}`,
     const itemListExcludedColumns = itemListSteps ? ['_item_list_attribution_row_id'] : [];
 
     // Join event_data and session_data, include additional logic
-    const finalStep = {
-        name: 'final',
-        columns: {
-            // get the most important columns in the correct order
-            ...finalColumnOrder,
-            ...itemListOverrides,
-            // get the rest of the event_data columns
-            '[sql]event_data': utils.selectOtherColumns(
-                eventDataStep,
-                Object.keys(finalColumnOrder),
-                [
-                    'entrances',
-                    mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
-                    'data_is_final',
-                    'export_type',
-                    ...itemListExcludedColumns,
-                ]
-            ),
-            // get the rest of the session_data columns
-            '[sql]session_data': utils.selectOtherColumns(
-                sessionDataStep,
-                Object.keys(finalColumnOrder),
-                []
-            ),
-            // include additional columns
-            row_inserted_timestamp: 'current_timestamp()',
-            data_is_final: 'data_is_final',
-            export_type: 'export_type',
+    // Named 'enhanced_events' so user-supplied customSteps can reference it as a stable handle.
+    const enhancedEventsStep = {
+        name: 'enhanced_events',
+        select: {
+            columns: {
+                // get the most important columns in the correct order
+                ...finalColumnOrder,
+                ...itemListOverrides,
+                // get the rest of the event_data columns
+                '[sql]event_data': utils.selectOtherColumns(
+                    eventDataStep,
+                    Object.keys(finalColumnOrder),
+                    [
+                        'entrances',
+                        mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
+                        'data_is_final',
+                        'export_type',
+                        ...itemListExcludedColumns,
+                    ]
+                ),
+                // get the rest of the session_data columns
+                '[sql]session_data': utils.selectOtherColumns(
+                    sessionDataStep,
+                    Object.keys(finalColumnOrder),
+                    []
+                ),
+                // include additional columns
+                row_inserted_timestamp: 'current_timestamp()',
+                data_is_final: 'data_is_final',
+                export_type: 'export_type',
+            },
         },
         from: 'event_data',
-        leftJoin: [
+        joins: [
             ...(itemListSteps ? [{
+                type: 'left',
                 table: 'item_list_data',
-                condition: 'using(_item_list_attribution_row_id)'
+                on: 'using(_item_list_attribution_row_id)'
             }] : []),
             {
+                type: 'left',
                 table: 'session_data',
-                condition: 'using(session_id)'
+                on: 'using(session_id)'
             }
         ],
         where: helpers.incrementalDateFilter(mergedConfig)
     };
 
-    const steps = [
+    const packageSteps = [
         eventDataStep,
         ...(itemListSteps ?? []),
         sessionDataStep,
-        finalStep,
+        enhancedEventsStep,
     ];
 
+    // Layer 2 validation: customSteps name must not collide with package step names.
+    // Reserved set is derived from packageSteps at runtime (single source of truth) — what
+    // is reserved depends on config (e.g. item_list_* exist only when itemListAttribution is on).
+    const customSteps = mergedConfig.customSteps ?? [];
+    if (customSteps.length > 0) {
+        const reservedNames = new Set(packageSteps.map(s => s.name));
+        for (const [i, step] of customSteps.entries()) {
+            if (reservedNames.has(step.name)) {
+                throw new Error(
+                    `config.customSteps[${i}].name '${step.name}' collides with a reserved package CTE name. ` +
+                    `Reserved names (active for this config): ${[...reservedNames].join(', ')}. Choose a different name.`
+                );
+            }
+        }
+    }
+
+    const steps = [...packageSteps, ...customSteps];
+
     return utils.queryBuilder(steps);
 };