ga4-export-fixer 0.8.0 → 0.9.0-dev.2

package/tables/ga4EventsEnhanced/index.js

@@ -230,7 +230,7 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
                 // ecommerce
                 ecommerce: helpers.fixEcommerceStruct('ecommerce'),
                 items: 'items',
-                _item_list_attribution_row_id: itemListAttribution ? helpers.itemListAttributionRowId(ecommerceEventsFilter) : undefined,
+                _item_row_id: itemListAttribution ? helpers.itemRowId(ecommerceEventsFilter) : undefined,
                 // flag if the data is "final" and is not expected to change anymore
                 data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
                 export_type: helpers.getGa4ExportType('_table_suffix'),
@@ -268,9 +268,9 @@ ${excludedEventsSQL}`,
         'group by': 'session_id',
     };
 
-    // item list attribution CTEs:
-    // 1. item_list_unnest: unnest items from ecommerce events, compute attribution via window function
-    // 2. item_list_data: re-aggregate items with attributed list fields
+    // Shared item-array CTEs:
+    // 1. items_unnested: unnest items from ecommerce events, compute attribution via window function
+    // 2. items_rebuilt: re-aggregate items with attributed list fields
     const itemListSteps = itemListAttribution ? (() => {
         const attrExpr = helpers.itemListAttributionExpr(
             itemListAttribution.lookbackType,
@@ -279,12 +279,14 @@ ${excludedEventsSQL}`,
         );
         const passthroughEvents = `event_name in ('view_item_list', 'select_item', 'view_promotion', 'select_promotion')`;
 
-        const attributionStep = {
-            name: 'item_list_attribution',
+        const unnestedStep = {
+            name: 'items_unnested',
             select: {
                 columns: {
-                    '_item_list_attribution_row_id': '_item_list_attribution_row_id',
+                    '_item_row_id': '_item_row_id',
                     'event_name': 'event_name',
+                    // event_date is carried forward for ability to use it in data enrichment joins
+                    'event_date': 'event_date',
                     'item': 'item',
                     '_item_list_attr': attrExpr,
                 },
@@ -293,11 +295,11 @@ ${excludedEventsSQL}`,
             where: `event_name in (${ecommerceEventsFilter})`,
         };
 
-        const dataStep = {
-            name: 'item_list_data',
+        const rebuiltStep = {
+            name: 'items_rebuilt',
             select: {
                 columns: {
-                    '_item_list_attribution_row_id': '_item_list_attribution_row_id',
+                    '_item_row_id': '_item_row_id',
                     'items': `array_agg(
       (select as struct item.* replace(
         coalesce(if(${passthroughEvents}, item.item_list_name, _item_list_attr.item_list_name), '(not set)') as item_list_name,
@@ -307,21 +309,81 @@ ${excludedEventsSQL}`,
     )`,
                 },
             },
-            from: 'item_list_attribution',
-            'group by': '_item_list_attribution_row_id',
+            from: 'items_unnested',
+            'group by': '_item_row_id',
         };
 
-        return [attributionStep, dataStep];
+        return [unnestedStep, rebuiltStep];
     })() : null;
 
     const finalColumnOrder = getFinalColumnOrder(eventDataStep, sessionDataStep);
 
-    // When item list attribution is enabled, override the items column and exclude _item_list_attribution_row_id
+    // When item list attribution is enabled, override the items column and exclude _item_row_id
     // COALESCE handles events without items (not in ecommerce filter) where the LEFT JOIN returns NULL
     const itemListOverrides = itemListSteps ? {
-        items: 'coalesce(item_list_data.items, event_data.items)',
+        items: 'coalesce(items_rebuilt.items, event_data.items)',
     } : {};
-    const itemListExcludedColumns = itemListSteps ? ['_item_list_attribution_row_id'] : [];
+    const itemListExcludedColumns = itemListSteps ? ['_item_row_id'] : [];
+
+    // Build enrichment-source CTEs and gather event-level join/column data.
+    // Item-level enrichments throw "not yet supported" — they will arrive in a later release.
+    const enrichments = mergedConfig.enrichments ?? [];
+    const enrichmentSteps = [];
+    const enrichmentJoins = [];
+    const enrichmentColumns = {};         // column name → SQL expression for select.columns
+    const enrichmentColumnNames = new Set();  // column names for excludedColumns of wildcards
+    const enrichmentColumnOwner = {};     // column name → { i, name } for collision errors
+    for (const [i, e] of enrichments.entries()) {
+        const level = e.level ?? 'event';
+        if (level === 'item') {
+            throw new Error(
+                `config.enrichments[${i}] uses level: 'item', which is not yet supported in this version. ` +
+                `Item-level enrichments will ship in a future release; see design_docs/planned/data-enrichments.md.`
+            );
+        }
+        const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
+        const cteName = `enrich_${e.name}`;
+        // Source CTE selects joinKey columns plus the requested columns. key === value
+        // shape skips the alias clause in queryBuilder's columnsToSQL.
+        const cteCols = {};
+        for (const k of joinKeys) cteCols[k] = k;
+        for (const c of e.columns) cteCols[c] = c;
+        const sourceStep = {
+            name: cteName,
+            select: { columns: cteCols },
+            from: e.source,
+        };
+        // Opt-in dedupe: which row wins is non-deterministic — users with strict needs
+        // pre-aggregate in their source SQL.
+        if (e.dedupe) {
+            sourceStep.qualify = `row_number() over (partition by ${joinKeys.join(', ')}) = 1`;
+        }
+        enrichmentSteps.push(sourceStep);
+
+        enrichmentJoins.push({
+            type: 'left',
+            table: cteName,
+            on: `using(${joinKeys.join(', ')})`,
+        });
+
+        // Replace-or-add: each enrichment column overrides explicit select columns via JS object
+        // spread, AND joins the excludedColumns set so it suppresses overlap with the wildcard
+        // event_data.* / session_data.* expansions below.
+        for (const c of e.columns) {
+            if (enrichmentColumnNames.has(c)) {
+                const owner = enrichmentColumnOwner[c];
+                throw new Error(
+                    `config.enrichments[${i}] (name: '${e.name}') and config.enrichments[${owner.i}] ` +
+                    `(name: '${owner.name}') both target column '${c}'. ` +
+                    `Two enrichments cannot write the same column; rename one in source SQL or pick a different name.`
+                );
+            }
+            enrichmentColumns[c] = `${cteName}.${c}`;
+            enrichmentColumnNames.add(c);
+            enrichmentColumnOwner[c] = { i, name: e.name };
+        }
+    }
+    const enrichmentExcludedColumns = [...enrichmentColumnNames];
 
     // Join event_data and session_data, include additional logic
     // Named 'enhanced_events' so user-supplied customSteps can reference it as a stable handle.
@@ -332,6 +394,9 @@ ${excludedEventsSQL}`,
                 // get the most important columns in the correct order
                 ...finalColumnOrder,
                 ...itemListOverrides,
+                // event-level enrichment columns: override matching explicit columns; new columns added.
+                // Wildcard-column overlap is handled below via excludedColumns.
+                ...enrichmentColumns,
                 // get the rest of the event_data columns
                 '[sql]event_data': utils.selectOtherColumns(
                     eventDataStep,
@@ -342,13 +407,14 @@ ${excludedEventsSQL}`,
                         'data_is_final',
                         'export_type',
                         ...itemListExcludedColumns,
+                        ...enrichmentExcludedColumns,
                     ]
                 ),
                 // get the rest of the session_data columns
                 '[sql]session_data': utils.selectOtherColumns(
                     sessionDataStep,
                     Object.keys(finalColumnOrder),
-                    []
+                    [...enrichmentExcludedColumns],
                 ),
                 // include additional columns
                 row_inserted_timestamp: 'current_timestamp()',
@@ -360,19 +426,22 @@ ${excludedEventsSQL}`,
         joins: [
             ...(itemListSteps ? [{
                 type: 'left',
-                table: 'item_list_data',
-                on: 'using(_item_list_attribution_row_id)'
+                table: 'items_rebuilt',
+                on: 'using(_item_row_id)'
             }] : []),
             {
                 type: 'left',
                 table: 'session_data',
                 on: 'using(session_id)'
-            }
+            },
+            // Event-level enrichment joins go last so they apply on top of the package's own joins.
+            ...enrichmentJoins,
         ],
         where: helpers.incrementalDateFilter(mergedConfig)
     };
 
     const packageSteps = [
+        ...enrichmentSteps,
         eventDataStep,
         ...(itemListSteps ?? []),
         sessionDataStep,
@@ -381,7 +450,8 @@ ${excludedEventsSQL}`,
 
     // Layer 2 validation: customSteps name must not collide with package step names.
     // Reserved set is derived from packageSteps at runtime (single source of truth) — what
-    // is reserved depends on config (e.g. item_list_* exist only when itemListAttribution is on).
+    // is reserved depends on config (e.g. item_list_* exist only when itemListAttribution is on,
+    // and enrich_* names exist only when enrichments are configured).
     const customSteps = mergedConfig.customSteps ?? [];
     if (customSteps.length > 0) {
         const reservedNames = new Set(packageSteps.map(s => s.name));

package/tables/ga4EventsEnhanced/validation.js

@@ -225,6 +225,101 @@ const validateEnhancedEventsConfig = (config, options = {}) => {
             seenNames.add(step.name);
         }
     }
+
+    // enrichments - optional array of declarative external-data enrichment specs.
+    // This block performs Layer 1 (config-shape) checks. Layer 2 checks (reserved-name collision
+    // + item-level deferral throw) live in _generateEnhancedEventsSQL — the reserved set is
+    // config-dependent and the item-level deferral throws there once the SQL is built.
+    if (config.enrichments !== undefined) {
+        if (!Array.isArray(config.enrichments)) {
+            throw new Error(`config.enrichments must be an array. Received: ${JSON.stringify(config.enrichments)}`);
+        }
+        const validLevels = ['event', 'item'];
+        const seenNames = new Set();
+        for (let i = 0; i < config.enrichments.length; i++) {
+            const entry = config.enrichments[i];
+            if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
+                throw new Error(`config.enrichments[${i}] must be a non-null object. Received: ${JSON.stringify(entry)}`);
+            }
+            if (typeof entry.name !== 'string' || !entry.name.trim()) {
+                throw new Error(`config.enrichments[${i}].name must be a non-empty string. Received: ${JSON.stringify(entry.name)}`);
+            }
+            if (seenNames.has(entry.name)) {
+                throw new Error(`config.enrichments contains duplicate name '${entry.name}'. Each enrichments entry must have a unique name.`);
+            }
+            seenNames.add(entry.name);
+            if (entry.level !== undefined && !validLevels.includes(entry.level)) {
+                throw new Error(`config.enrichments[${i}].level must be one of: ${validLevels.join(', ')}. Received: ${JSON.stringify(entry.level)}`);
+            }
+            // source: Dataform table reference object or backtick-quoted string
+            if (entry.source === undefined || entry.source === null) {
+                throw new Error(`config.enrichments[${i}].source is required.`);
+            }
+            if (isDataformTableReferenceObject(entry.source)) {
+                // Valid Dataform reference
+            } else if (typeof entry.source === 'string') {
+                if (!entry.source.trim()) {
+                    throw new Error(`config.enrichments[${i}].source must be a non-empty string. Received empty string.`);
+                }
+                if (!/^`[^\.]+\.[^\.]+\.[^\.]+`$/.test(entry.source.trim())) {
+                    throw new Error(`config.enrichments[${i}].source must be in the format '\`project.dataset.table\`' (with backticks) or a Dataform table reference. Received: ${JSON.stringify(entry.source)}`);
+                }
+            } else {
+                throw new Error(`config.enrichments[${i}].source must be a Dataform table reference object or a string in format '\`project.dataset.table\`'. Received: ${JSON.stringify(entry.source)}`);
+            }
+            // joinKey: required, plain SQL identifier OR non-empty array of plain SQL identifiers.
+            // Plain identifier = ^[a-zA-Z_][a-zA-Z0-9_]*$ — no aliases (`id as user_id`), no backticks,
+            // no dotted paths. Users with mismatched dim-column names alias in an upstream Dataform view.
+            const sqlIdentifier = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+            const aliasingHint = ' Aliases like \'id as user_id\' are not supported here; alias in an upstream Dataform view if your dim has a different column name.';
+            if (entry.joinKey === undefined || entry.joinKey === null) {
+                throw new Error(`config.enrichments[${i}].joinKey is required.`);
+            }
+            if (typeof entry.joinKey === 'string') {
+                if (!entry.joinKey.trim()) {
+                    throw new Error(`config.enrichments[${i}].joinKey must be a non-empty string. Received empty string.`);
+                }
+                if (!sqlIdentifier.test(entry.joinKey)) {
+                    throw new Error(`config.enrichments[${i}].joinKey must be a plain SQL identifier. Received: ${JSON.stringify(entry.joinKey)}.${aliasingHint}`);
+                }
+            } else if (Array.isArray(entry.joinKey)) {
+                if (entry.joinKey.length === 0) {
+                    throw new Error(`config.enrichments[${i}].joinKey must be a non-empty array when provided as an array.`);
+                }
+                for (let j = 0; j < entry.joinKey.length; j++) {
+                    const k = entry.joinKey[j];
+                    if (typeof k !== 'string' || !k.trim()) {
+                        throw new Error(`config.enrichments[${i}].joinKey[${j}] must be a non-empty string. Received: ${JSON.stringify(k)}`);
+                    }
+                    if (!sqlIdentifier.test(k)) {
+                        throw new Error(`config.enrichments[${i}].joinKey[${j}] must be a plain SQL identifier. Received: ${JSON.stringify(k)}.${aliasingHint}`);
+                    }
+                }
+            } else {
+                throw new Error(`config.enrichments[${i}].joinKey must be a string or a non-empty array of strings. Received: ${JSON.stringify(entry.joinKey)}`);
+            }
+            // columns: required, non-empty array of plain SQL identifiers (no aliasing).
+            if (!Array.isArray(entry.columns)) {
+                throw new Error(`config.enrichments[${i}].columns must be an array. Received: ${JSON.stringify(entry.columns)}`);
+            }
+            if (entry.columns.length === 0) {
+                throw new Error(`config.enrichments[${i}].columns must be non-empty. List the source columns to add to the output (excluding joinKey).`);
+            }
+            for (let j = 0; j < entry.columns.length; j++) {
+                const c = entry.columns[j];
+                if (typeof c !== 'string' || !c.trim()) {
+                    throw new Error(`config.enrichments[${i}].columns[${j}] must be a non-empty string. Received: ${JSON.stringify(c)}`);
+                }
+                if (!sqlIdentifier.test(c)) {
+                    throw new Error(`config.enrichments[${i}].columns[${j}] must be a plain SQL identifier. Received: ${JSON.stringify(c)}.${aliasingHint}`);
+                }
+            }
+            // dedupe: optional boolean
+            if (entry.dedupe !== undefined && typeof entry.dedupe !== 'boolean') {
+                throw new Error(`config.enrichments[${i}].dedupe must be a boolean when provided. Received: ${JSON.stringify(entry.dedupe)}`);
+            }
+        }
+    }
   } catch (e) {
     e.message = `Config validation: ${e.message}`;
     throw e;

package/utils.js

@@ -389,6 +389,16 @@ const setDataformContext = (ctx, config) => {
         }
     }
 
+    // resolve Dataform refs in enrichments[].source the same way as sourceTable
+    if (Array.isArray(config.enrichments)) {
+        config.enrichments = config.enrichments.map(e => {
+            if (isDataformTableReferenceObject(e.source)) {
+                return { ...e, source: ctx.ref(e.source) };
+            }
+            return e;
+        });
+    }
+
     config.self = ctx.self();
     config.incremental = ctx.incremental();
 
@@ -479,23 +489,35 @@ const selectOtherColumns = (step, alreadyDefinedColumns = [], excludedColumns =
     const stepName = step.name;
     const stepColumns = Object.keys(step.select.columns);
 
-    // Determine which columns to exclude: those already defined or explicitly excluded
-    const exceptColumns = stepColumns.filter(
+    // Columns in step.select.columns that should be excluded (already-defined or explicitly listed)
+    const internalExcept = stepColumns.filter(
         column => alreadyDefinedColumns.includes(column) || excludedColumns.includes(column)
     );
 
-    // If none of the columns have been defined or excluded, select them all
-    if (exceptColumns.length === 0) {
+    // Columns in excludedColumns that aren't enumerated in step.select.columns. These are
+    // wildcard-sourced columns (e.g. default GA4 export columns coming through `event_data.*`
+    // inside event_data's own select). The caller knows what to exclude; trust them.
+    // BigQuery throws at dry-run if the column doesn't exist in the source — surfaces typos.
+    // Filter out undefined/null entries (callers can pass conditional values like
+    // `cond ? 'col' : undefined` for ergonomics).
+    const externalExcept = excludedColumns.filter(
+        c => typeof c === 'string' && c.length > 0 && !stepColumns.includes(c)
+    );
+
+    const allExcept = [...internalExcept, ...externalExcept];
+
+    // If nothing is excluded, select everything
+    if (allExcept.length === 0) {
         return `${stepName}.*`;
     }
 
-    // If all columns have been defined or excluded, do not select any
-    if (exceptColumns.length === stepColumns.length) {
+    // If every enumerated column is excluded and there are no external excepts to apply,
+    // there's nothing to select via the wildcard
+    if (internalExcept.length === stepColumns.length && externalExcept.length === 0) {
         return;
     }
 
-    // Otherwise, select all except the excluded/defined ones
-    return `${stepName}.* except (${exceptColumns.join(', ')})`;
+    return `${stepName}.* except (${allExcept.join(', ')})`;
 };