ga4-export-fixer 0.8.0 → 0.9.0-dev.10

package/utils.js

@@ -389,6 +389,16 @@ const setDataformContext = (ctx, config) => {
         }
     }
 
+    // resolve Dataform refs in enrichments[].source the same way as sourceTable
+    if (Array.isArray(config.enrichments)) {
+        config.enrichments = config.enrichments.map(e => {
+            if (isDataformTableReferenceObject(e.source)) {
+                return { ...e, source: ctx.ref(e.source) };
+            }
+            return e;
+        });
+    }
+
     config.self = ctx.self();
     config.incremental = ctx.incremental();
 
@@ -465,37 +475,162 @@ const mergeDataformTableConfigurations = (defaultConfig, inputConfig = {}) => {
 };
 
 /**
- * Generates a SQL selection string for a given query step, excluding columns already defined elsewhere
- * or columns that should be excluded.
+ * Builds a queryBuilder `select.columns` fragment that passes through every source column
+ * not already covered by an explicit columns object.
  *
- * This utility is helpful when joining tables/CTEs to avoid selecting duplicate or already-present columns.
- * 
- * @param {Object} step - A queryBuilder structured step containing a `name` (CTE/table alias) and a `select.columns` object.
- * @param {string[]} [alreadyDefinedColumns=[]] - Columns that have already been defined and should be excluded from selection.
- * @param {string[]} [excludedColumns=[]] - Additional columns to explicitly exclude from selection.
- * @returns {string|undefined} A SQL select string (e.g. 'stepName.*' or 'stepName.* except (col1, col2)'), or undefined if all columns are excluded.
+ * A source column is considered "covered" — and skipped from pass-throughs — when it appears as:
+ *   - a KEY in `explicitColumns` (a transform, package promotion, or undefined-valued exclusion
+ *     sentinel like `{ event_dimensions: undefined }`), OR
+ *   - a VALUE in `explicitColumns` (a bare source-column identifier referenced by a value-side
+ *     rename, e.g. `{ user_traffic_source: 'traffic_source' }` covers 'traffic_source').
+ *
+ * Values that are SQL expressions, function calls, or non-strings never count as coverage —
+ * they reference the source column internally but the column itself is still available as a
+ * pass-through. (`.includes()` compares by strict equality, so 'extract(datetime from ...)'
+ * never matches a bare column name.)
+ *
+ * @param {Object} explicitColumns - A queryBuilder step's explicit `select.columns` entries.
+ * @param {Iterable<string>} sourceColumns - Column names available on the source schema.
+ * @returns {Object} A map of `{ column: column }` entries for every source column not covered.
+ *
+ * @example
+ *   buildPassThroughs(
+ *     { event_name: 'event_name', user_traffic_source: 'traffic_source' },
+ *     ['event_name', 'traffic_source', 'device', 'geo']
+ *   );
+ *   // → { device: 'device', geo: 'geo' }
  */
-const selectOtherColumns = (step, alreadyDefinedColumns = [], excludedColumns = []) => {
-    const stepName = step.name;
-    const stepColumns = Object.keys(step.select.columns);
-
-    // Determine which columns to exclude: those already defined or explicitly excluded
-    const exceptColumns = stepColumns.filter(
-        column => alreadyDefinedColumns.includes(column) || excludedColumns.includes(column)
-    );
-
-    // If none of the columns have been defined or excluded, select them all
-    if (exceptColumns.length === 0) {
-        return `${stepName}.*`;
+const buildPassThroughs = (explicitColumns, sourceColumns) => {
+    const explicitKeys = Object.keys(explicitColumns);
+    const explicitValues = Object.values(explicitColumns);
+    const passThroughs = {};
+    for (const column of sourceColumns) {
+        if (!explicitKeys.includes(column) && !explicitValues.includes(column)) {
+            passThroughs[column] = column;
+        }
     }
+    return passThroughs;
+};
 
-    // If all columns have been defined or excluded, do not select any
-    if (exceptColumns.length === stepColumns.length) {
-        return;
+
+/**
+ * Builds the per-enrichment CTE definitions, JOIN clauses, and column-name mappings for the
+ * declarative `enrichments` feature. Routes event-level and item-level entries through
+ * separate output channels so the caller can attach them to different downstream CTEs.
+ *
+ * Pure config-to-data mapping. No knowledge of downstream CTEs or specific table modules —
+ * intended to be called by any table module that exposes an `enrichments` config field.
+ *
+ * Encapsulates one generation-time throw:
+ *   - Same-level enrichment-vs-enrichment column collisions (two event-level enrichments or
+ *     two item-level enrichments targeting the same column). Cross-level same-name is allowed —
+ *     the two columns target structurally distinct slots (`enhanced_events.<col>` vs
+ *     `items[].<col>`).
+ *
+ * @param {Array<Object>} enrichments - Validated enrichment entries. Each entry has fields:
+ *   { name, level, source, joinKey, columns, dedupe? }. `level` is 'event' (default) or 'item'.
+ * @returns {Object} A struct with four fields:
+ *   - `steps` — array of queryBuilder source-CTE step definitions (one `enrich_<name>` per
+ *     entry, regardless of level — all source CTEs go to the top of the pipeline).
+ *   - `event` — { joins, columns, columnNames } for event-level enrichments. Caller attaches
+ *     `joins` to the event-grained downstream CTE (e.g. `enhanced_events`) and spreads `columns`
+ *     into that CTE's `select.columns`.
+ *   - `item` — { joins, columns, columnNames } for item-level enrichments. Caller attaches
+ *     `joins` to the item-grained downstream CTE (e.g. `items_rebuilt`) and folds `columns`
+ *     into that CTE's struct construction.
+ *   - `columnOwner` — map of `{ <column>: { i, name, level } }` recording which enrichment
+ *     owns each column. The `level` field distinguishes cross-level same-name entries.
+ *
+ * @throws {Error} If two same-level enrichments target the same column name (with both
+ *   enrichment names and the conflicting column in the error message).
+ *
+ * @example
+ *   const { steps, event, item } = buildEnrichments(config.enrichments);
+ *   // event.joins → attach to enhanced_events; event.columns → spread into enhanced_events
+ *   // item.joins → attach to items_rebuilt; item.columns → fold into items struct
+ */
+const buildEnrichments = (enrichments) => {
+    const steps = [];
+    const channels = {
+        event: { joins: [], columns: {}, columnNames: new Set() },
+        item: { joins: [], columns: {}, columnNames: new Set() },
+    };
+    const columnOwner = {};
+
+    for (const [i, e] of (enrichments ?? []).entries()) {
+        const level = e.level ?? 'event';
+        const channel = channels[level];
+        const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
+        const cteName = `enrich_${e.name}`;
+
+        // Source CTE selects joinKey columns plus the requested columns. key === value
+        // shape skips the alias clause in queryBuilder's columnsToSQL.
+        const cteCols = {};
+        for (const k of joinKeys) cteCols[k] = k;
+        for (const c of e.columns) cteCols[c] = c;
+        const sourceStep = { name: cteName, select: { columns: cteCols }, from: e.source };
+        // Opt-in dedupe: which row wins is non-deterministic — users with strict needs
+        // pre-aggregate in their source SQL.
+        if (e.dedupe) {
+            sourceStep.qualify = `row_number() over (partition by ${joinKeys.join(', ')}) = 1`;
+        }
+        steps.push(sourceStep);
+
+        channel.joins.push({ type: 'left', table: cteName, on: `using(${joinKeys.join(', ')})` });
+
+        for (const c of e.columns) {
+            // Same-level collision throw. Cross-level same-name is allowed because the two
+            // columns target structurally distinct output slots (event_data vs items[]).
+            if (channel.columnNames.has(c)) {
+                const owner = columnOwner[c];
+                throw new Error(
+                    `config.enrichments[${i}] (name: '${e.name}') and config.enrichments[${owner.i}] ` +
+                    `(name: '${owner.name}') both target column '${c}' at level '${level}'. ` +
+                    `Two enrichments cannot write the same column at the same level; rename one in source SQL or pick a different name.`
+                );
+            }
+            channel.columns[c] = `${cteName}.${c}`;
+            channel.columnNames.add(c);
+            // columnOwner is keyed by column name; if the same name appears at different
+            // levels, the second-writer entry wins, but we record level so diagnostics
+            // distinguish them. Same-level collisions throw above before reaching here.
+            columnOwner[c] = { i, name: e.name, level };
+        }
     }
 
-    // Otherwise, select all except the excluded/defined ones
-    return `${stepName}.* except (${exceptColumns.join(', ')})`;
+    return { steps, event: channels.event, item: channels.item, columnOwner };
+};
+
+
+/**
+ * Builds a qualified pass-through fragment for spreading into a downstream SELECT's
+ * `select.columns`. For each column in `step.select.columns` not already in `alreadyCovered`,
+ * emits an entry of the form `{ <col>: '<step.name>.<col>' }`.
+ *
+ * Columns whose values in `step.select.columns` are `undefined` (the user-exclusion sentinel
+ * shape from getExcludedColumns) are skipped. Names in `alreadyCovered` that don't exist in
+ * `step.select.columns` are silently ignored — the loop only iterates `step.select.columns`,
+ * so unknown names cause no harm. This is the safety property that lets callers pass
+ * "everything that might collide" without pre-filtering.
+ *
+ * @param {Object} step - A queryBuilder step with a `name` and `select.columns` object.
+ * @param {Iterable<string>} alreadyCovered - Column names already mapped elsewhere in the
+ *   downstream SELECT, plus any internal-only columns the downstream SELECT shouldn't re-emit.
+ * @returns {Object} A map of `{ <col>: '<step.name>.<col>' }` entries.
+ *
+ * @example
+ *   buildQualifiedPassThroughs(eventDataStep, ['event_date', 'session_id', 'entrances']);
+ *   // → { event_name: 'event_data.event_name', user_pseudo_id: 'event_data.user_pseudo_id', ... }
+ */
+const buildQualifiedPassThroughs = (step, alreadyCovered) => {
+    const covered = new Set(alreadyCovered);
+    const passThroughs = {};
+    for (const [col, expr] of Object.entries(step.select.columns)) {
+        if (expr === undefined) continue;
+        if (covered.has(col)) continue;
+        passThroughs[col] = `${step.name}.${col}`;
+    }
+    return passThroughs;
 };
 
 
@@ -573,7 +708,9 @@ module.exports = {
     queryBuilder,
     isDataformTableReferenceObject,
     setDataformContext,
-    selectOtherColumns,
+    buildPassThroughs,
+    buildEnrichments,
+    buildQualifiedPassThroughs,
     processDate,
     getDatasetName
 };