ga4-export-fixer 0.9.0-dev.7 → 0.9.0-dev.9

package/README.md

@@ -537,10 +537,10 @@ For typical use cases this is the right tool; reach for `customSteps` only when
 | `level` | `'event'` | No, defaults to `'event'` | Join grain. Currently only `'event'` is supported (item-level enrichments will arrive in a later release). |
 | `source` | Dataform ref / object / string | Yes | Source dim table. Inside an SQLX `js { }` block use `ref(...)`. From a `.js` definition file use a `{ schema, name }` ref object (resolved later via `ctx.ref()`) or a backtick-quoted ``` `project.dataset.table` ``` string for an external table. |
 | `joinKey` | string / string[] | Yes | Column name(s) on `enhanced_events` to join on. Composite keys (array) compile to `USING(col1, col2, ...)`. |
-| `columns` | string[] | Yes | Source columns to add to the output (excluding `joinKey`). Names matching existing columns REPLACE them. |
+| `columns` | string[] | Yes | Source columns to add to the output (excluding `joinKey`). Names matching existing columns are coalesced with the original (`coalesce(enrich.col, original)`) so missed JOINs fall back to the existing value. |
 | `dedupe` | boolean | No, defaults to `false` | When `true`, wraps the source CTE in `qualify row_number() over (partition by <joinKey>) = 1` for non-unique-key dim sources. Non-deterministic which row wins; for strict needs, pre-aggregate in source SQL. |
 
-**Replace-or-add semantics.** If an enrichment column name matches an existing column on `enhanced_events` (a column promoted via `eventParamsToColumns`, a package-generated column, or a default GA4 column from the export), the enrichment value REPLACES it. If there is no overlap, the column is added.
+**Coalesce-or-add semantics.** If an enrichment column name matches an existing column on `enhanced_events` (a column promoted via `eventParamsToColumns`, a package-generated column, or a default GA4 column from the export), the enrichment value is coalesced with the original: `coalesce(enrich_<name>.<col>, <original>) as <col>`. Rows where the JOIN matches get the enrichment value; rows where it misses fall back to the existing value rather than going NULL. If there is no overlap, the column is added as a plain `enrich_<name>.<col>`. There is no opt-out — for hard-replace semantics (NULL on missed JOIN), pre-aggregate or sentinel-fill in your source SQL.
 
 **Example** — attach user cohort labels by `user_pseudo_id` (Dataform-declared table referenced by `{ schema, name }`):
 
@@ -571,7 +571,7 @@ enrichments: [
 ],
 ```
 
-**Example** — fix a promoted event parameter via enrichment (replacement case):
+**Example** — fix a promoted event parameter via enrichment (coalesce case: enrichment value wins where the JOIN matches, original kept where it doesn't):
 
 ```javascript
 {
@@ -582,7 +582,7 @@ enrichments: [
             level: 'event',
             source: { schema: 'analytics', name: 'page_title_overrides' },
             joinKey: 'page_location',
-            columns: ['page_title'],   // overlaps the promoted column → replaces it
+            columns: ['page_title'],   // overlaps the promoted column → coalesce(enrich.page_title, event_data.page_title)
         },
     ],
 }
@@ -590,7 +590,7 @@ enrichments: [
 
 > **Note:** Each enrichment generates a CTE named `enrich_<name>` at the top of the pipeline. The `enrich_*` namespace is part of the reserved-names contract — `customSteps` cannot use these names. The active reserved set includes only the names of enrichments actually configured.
 
-> **Note:** Enrichment columns get auto-generated descriptions (`Added by enrichment '<name>' (joined on <joinKey> from <source>).` for new columns; `Replaced by enrichment '<name>' (...). Original: <description>` for replacements). User-supplied `dataformTableConfig.columns` overrides win — the auto-generated description is the default.
+> **Note:** Enrichment columns get auto-generated descriptions (`Added by enrichment '<name>' (joined on <joinKey> from <source>).` for new columns; `Coalesced by enrichment '<name>' (...; falls back to original on missed JOIN). Original: <description>` for overlapping columns). User-supplied `dataformTableConfig.columns` overrides win — the auto-generated description is the default.
 
 > **Note:** `joinKey` and `columns` entries must be plain SQL identifiers — inline aliases like `'id as user_id'` are rejected at validation time. If your dim source uses a different column name, alias it in an upstream Dataform view and point `source` at that view.
 

package/documentation.js

@@ -186,7 +186,7 @@ const getColumnDescriptions = (config, columnMetadata) => {
                         ? existing.description
                         : null;
                 const newDesc = existingText
-                    ? `Replaced by enrichment '${e.name}' (joined on ${joinKeyText} from ${sourceText}). Original: ${existingText}`
+                    ? `Coalesced by enrichment '${e.name}' (joined on ${joinKeyText} from ${sourceText}; falls back to original on missed JOIN). Original: ${existingText}`
                     : `Added by enrichment '${e.name}' (joined on ${joinKeyText} from ${sourceText}).`;
                 // If the original was a struct-shaped entry, preserve the structure but replace the description.
                 // Otherwise, set as a plain string.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.9.0-dev.7",
+  "version": "0.9.0-dev.9",
   "description": "",
   "main": "index.js",
   "files": [

package/tables/ga4EventsEnhanced/index.js

@@ -324,16 +324,40 @@ ${excludedEventsSQL}`,
     // enrichments throw "not yet supported" inside the utility — they will arrive in a later release.
     const { steps: enrichmentSteps, joins: enrichmentJoins, columns: enrichmentColumns,
             columnNames: enrichmentColumnNames } = utils.buildEnrichments(mergedConfig.enrichments);
-    const enrichmentExcludedColumns = [...enrichmentColumnNames];
 
-    // Only forward enrichment columns to each wildcard's EXCEPT input if they actually exist
-    // in that wildcard's source CTE. Otherwise BigQuery rejects with "Column X in SELECT *
-    // EXCEPT list does not exist". After M1, Object.keys(step.select.columns) is the complete
-    // column set of both event_data and session_data — so the same predicate works for both.
-    const eventDataExplicit = new Set(Object.keys(eventDataStep.select.columns));
-    const sessionDataExplicit = new Set(Object.keys(sessionDataStep.select.columns));
-    const eventDataEnrichmentExcept = enrichmentExcludedColumns.filter(c => eventDataExplicit.has(c));
-    const sessionDataEnrichmentExcept = enrichmentExcludedColumns.filter(c => sessionDataExplicit.has(c));
+    // Wrap overlapping enrichment columns in coalesce(enrich_<name>.<col>, <original>) so a
+    // missed JOIN falls back to the existing value. Purely additive columns (no overlap)
+    // pass through unchanged. Source-of-original precedence matches the final SELECT's spread
+    // order: itemListOverrides first (overrides finalColumnOrder for `items`), then
+    // session_data (wins over event_data in getFinalColumnOrder when both have the column).
+    // See design_docs/planned/data-enrichments.md Q13.
+    const wrappedEnrichmentColumns = {};
+    for (const [col, enrichExpr] of Object.entries(enrichmentColumns)) {
+        let originalExpr;
+        if (col in itemListOverrides) {
+            originalExpr = itemListOverrides[col];
+        } else if (col in sessionDataStep.select.columns) {
+            originalExpr = `session_data.${col}`;
+        } else if (col in eventDataStep.select.columns && eventDataStep.select.columns[col] !== undefined) {
+            originalExpr = `event_data.${col}`;
+        }
+        wrappedEnrichmentColumns[col] = originalExpr
+            ? `coalesce(${enrichExpr}, ${originalExpr})`
+            : enrichExpr;
+    }
+
+    // List all column names that have already been defined or should be left out
+    // Used for the final pass-through: include the rest of the coulumns that haven't been explicitly listed yet
+    const alreadyMapped = [
+        ...Object.keys(finalColumnOrder),
+        ...Object.keys(itemListOverrides),
+        ...enrichmentColumnNames,
+        'entrances',
+        mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
+        'data_is_final',
+        'export_type',
+        ...itemListExcludedColumns,
+    ];
 
     // Join event_data and session_data, include additional logic
     // Named 'enhanced_events' so user-supplied customSteps can reference it as a stable handle.
@@ -344,28 +368,11 @@ ${excludedEventsSQL}`,
                 // get the most important columns in the correct order
                 ...finalColumnOrder,
                 ...itemListOverrides,
-                // event-level enrichment columns: override matching explicit columns; new columns added.
-                // Wildcard-column overlap is handled below via excludedColumns.
-                ...enrichmentColumns,
-                // get the rest of the event_data columns
-                '[sql]event_data': utils.selectOtherColumns(
-                    eventDataStep,
-                    Object.keys(finalColumnOrder),
-                    [
-                        'entrances',
-                        mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
-                        'data_is_final',
-                        'export_type',
-                        ...itemListExcludedColumns,
-                        ...eventDataEnrichmentExcept,
-                    ]
-                ),
-                // get the rest of the session_data columns
-                '[sql]session_data': utils.selectOtherColumns(
-                    sessionDataStep,
-                    Object.keys(finalColumnOrder),
-                    sessionDataEnrichmentExcept,
-                ),
+                // event-level enrichment columns: coalesce with the original when overlapping; otherwise add.
+                ...wrappedEnrichmentColumns,
+                // explicit pass-throughs for the rest of event_data and session_data
+                ...utils.buildQualifiedPassThroughs(eventDataStep, alreadyMapped),
+                ...utils.buildQualifiedPassThroughs(sessionDataStep, alreadyMapped),
                 // include additional columns
                 row_inserted_timestamp: 'current_timestamp()',
                 data_is_final: 'data_is_final',
@@ -384,7 +391,7 @@ ${excludedEventsSQL}`,
                 table: 'session_data',
                 on: 'using(session_id)'
             },
-            // Event-level enrichment joins go last so they apply on top of the package's own joins.
+            // The left joins for the enrichment ctes
             ...enrichmentJoins,
         ],
         where: helpers.incrementalDateFilter(mergedConfig)
@@ -398,10 +405,7 @@ ${excludedEventsSQL}`,
         enhancedEventsStep,
     ];
 
-    // Layer 2 validation: customSteps name must not collide with package step names.
-    // Reserved set is derived from packageSteps at runtime (single source of truth) — what
-    // is reserved depends on config (e.g. item_list_* exist only when itemListAttribution is on,
-    // and enrich_* names exist only when enrichments are configured).
+    // Ensure that the custom step names don't collide with the default or data enrichment step names
     const customSteps = mergedConfig.customSteps ?? [];
     if (customSteps.length > 0) {
         const reservedNames = new Set(packageSteps.map(s => s.name));
@@ -415,6 +419,7 @@ ${excludedEventsSQL}`,
         }
     }
 
+    // Include custom steps last in the list
     const steps = [...packageSteps, ...customSteps];
 
     return utils.queryBuilder(steps);

package/utils.js

@@ -474,53 +474,6 @@ const mergeDataformTableConfigurations = (defaultConfig, inputConfig = {}) => {
     return deepMerge(defaultConfig, inputConfig);
 };
 
-/**
- * Generates a SQL selection string for a given query step, excluding columns already defined elsewhere
- * or columns that should be excluded.
- *
- * This utility is helpful when joining tables/CTEs to avoid selecting duplicate or already-present columns.
- * 
- * @param {Object} step - A queryBuilder structured step containing a `name` (CTE/table alias) and a `select.columns` object.
- * @param {string[]} [alreadyDefinedColumns=[]] - Columns that have already been defined and should be excluded from selection.
- * @param {string[]} [excludedColumns=[]] - Additional columns to explicitly exclude from selection.
- * @returns {string|undefined} A SQL select string (e.g. 'stepName.*' or 'stepName.* except (col1, col2)'), or undefined if all columns are excluded.
- */
-const selectOtherColumns = (step, alreadyDefinedColumns = [], excludedColumns = []) => {
-    const stepName = step.name;
-    const stepColumns = Object.keys(step.select.columns);
-
-    // Columns in step.select.columns that should be excluded (already-defined or explicitly listed)
-    const internalExcept = stepColumns.filter(
-        column => alreadyDefinedColumns.includes(column) || excludedColumns.includes(column)
-    );
-
-    // Columns in excludedColumns that aren't enumerated in step.select.columns. These are
-    // wildcard-sourced columns (e.g. default GA4 export columns coming through `event_data.*`
-    // inside event_data's own select). The caller knows what to exclude; trust them.
-    // BigQuery throws at dry-run if the column doesn't exist in the source — surfaces typos.
-    // Filter out undefined/null entries (callers can pass conditional values like
-    // `cond ? 'col' : undefined` for ergonomics).
-    const externalExcept = excludedColumns.filter(
-        c => typeof c === 'string' && c.length > 0 && !stepColumns.includes(c)
-    );
-
-    const allExcept = [...internalExcept, ...externalExcept];
-
-    // If nothing is excluded, select everything
-    if (allExcept.length === 0) {
-        return `${stepName}.*`;
-    }
-
-    // If every enumerated column is excluded and there are no external excepts to apply,
-    // there's nothing to select via the wildcard
-    if (internalExcept.length === stepColumns.length && externalExcept.length === 0) {
-        return;
-    }
-
-    return `${stepName}.* except (${allExcept.join(', ')})`;
-};
-
-
 /**
  * Builds a queryBuilder `select.columns` fragment that passes through every source column
  * not already covered by an explicit columns object.
@@ -641,6 +594,38 @@ const buildEnrichments = (enrichments) => {
 };
 
 
+/**
+ * Builds a qualified pass-through fragment for spreading into a downstream SELECT's
+ * `select.columns`. For each column in `step.select.columns` not already in `alreadyCovered`,
+ * emits an entry of the form `{ <col>: '<step.name>.<col>' }`.
+ *
+ * Columns whose values in `step.select.columns` are `undefined` (the user-exclusion sentinel
+ * shape from getExcludedColumns) are skipped. Names in `alreadyCovered` that don't exist in
+ * `step.select.columns` are silently ignored — the loop only iterates `step.select.columns`,
+ * so unknown names cause no harm. This is the safety property that lets callers pass
+ * "everything that might collide" without pre-filtering.
+ *
+ * @param {Object} step - A queryBuilder step with a `name` and `select.columns` object.
+ * @param {Iterable<string>} alreadyCovered - Column names already mapped elsewhere in the
+ *   downstream SELECT, plus any internal-only columns the downstream SELECT shouldn't re-emit.
+ * @returns {Object} A map of `{ <col>: '<step.name>.<col>' }` entries.
+ *
+ * @example
+ *   buildQualifiedPassThroughs(eventDataStep, ['event_date', 'session_id', 'entrances']);
+ *   // → { event_name: 'event_data.event_name', user_pseudo_id: 'event_data.user_pseudo_id', ... }
+ */
+const buildQualifiedPassThroughs = (step, alreadyCovered) => {
+    const covered = new Set(alreadyCovered);
+    const passThroughs = {};
+    for (const [col, expr] of Object.entries(step.select.columns)) {
+        if (expr === undefined) continue;
+        if (covered.has(col)) continue;
+        passThroughs[col] = `${step.name}.${col}`;
+    }
+    return passThroughs;
+};
+
+
 /**
  * Processes a date input string and returns a corresponding SQL date casting expression,
  * or passes through BigQuery SQL statements as-is.
@@ -715,9 +700,9 @@ module.exports = {
     queryBuilder,
     isDataformTableReferenceObject,
     setDataformContext,
-    selectOtherColumns,
     buildPassThroughs,
     buildEnrichments,
+    buildQualifiedPassThroughs,
     processDate,
     getDatasetName
 };