ga4-export-fixer 0.9.0-dev.6 → 0.9.0-dev.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.9.0-dev.6",
+  "version": "0.9.0-dev.8",
   "description": "",
   "main": "index.js",
   "files": [

package/tables/ga4EventsEnhanced/index.js

@@ -320,74 +320,23 @@ ${excludedEventsSQL}`,
     } : {};
     const itemListExcludedColumns = itemListSteps ? ['_item_row_id'] : [];
 
-    // Build enrichment-source CTEs and gather event-level join/column data.
-    // Item-level enrichments throw "not yet supported" — they will arrive in a later release.
-    const enrichments = mergedConfig.enrichments ?? [];
-    const enrichmentSteps = [];
-    const enrichmentJoins = [];
-    const enrichmentColumns = {};         // column name → SQL expression for select.columns
-    const enrichmentColumnNames = new Set();  // column names for excludedColumns of wildcards
-    const enrichmentColumnOwner = {};     // column name → { i, name } for collision errors
-    for (const [i, e] of enrichments.entries()) {
-        const level = e.level ?? 'event';
-        if (level === 'item') {
-            throw new Error(
-                `config.enrichments[${i}] uses level: 'item', which is not yet supported in this version. ` +
-                `Item-level enrichments will ship in a future release; see design_docs/planned/data-enrichments.md.`
-            );
-        }
-        const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
-        const cteName = `enrich_${e.name}`;
-        // Source CTE selects joinKey columns plus the requested columns. key === value
-        // shape skips the alias clause in queryBuilder's columnsToSQL.
-        const cteCols = {};
-        for (const k of joinKeys) cteCols[k] = k;
-        for (const c of e.columns) cteCols[c] = c;
-        const sourceStep = {
-            name: cteName,
-            select: { columns: cteCols },
-            from: e.source,
-        };
-        // Opt-in dedupe: which row wins is non-deterministic — users with strict needs
-        // pre-aggregate in their source SQL.
-        if (e.dedupe) {
-            sourceStep.qualify = `row_number() over (partition by ${joinKeys.join(', ')}) = 1`;
-        }
-        enrichmentSteps.push(sourceStep);
-
-        enrichmentJoins.push({
-            type: 'left',
-            table: cteName,
-            on: `using(${joinKeys.join(', ')})`,
-        });
-
-        // Replace-or-add: each enrichment column overrides explicit select columns via JS object
-        // spread, AND joins the excludedColumns set so it suppresses overlap with the wildcard
-        // event_data.* / session_data.* expansions below.
-        for (const c of e.columns) {
-            if (enrichmentColumnNames.has(c)) {
-                const owner = enrichmentColumnOwner[c];
-                throw new Error(
-                    `config.enrichments[${i}] (name: '${e.name}') and config.enrichments[${owner.i}] ` +
-                    `(name: '${owner.name}') both target column '${c}'. ` +
-                    `Two enrichments cannot write the same column; rename one in source SQL or pick a different name.`
-                );
-            }
-            enrichmentColumns[c] = `${cteName}.${c}`;
-            enrichmentColumnNames.add(c);
-            enrichmentColumnOwner[c] = { i, name: e.name };
-        }
-    }
-    const enrichmentExcludedColumns = [...enrichmentColumnNames];
-
-    // Only forward enrichment columns to each wildcard's EXCEPT input if they actually exist
-    // in that wildcard's source CTE. Otherwise BigQuery rejects with "Column X in SELECT *
-    // EXCEPT list does not exist". After M1, Object.keys(step.select.columns) is the complete
-    // column set of both event_data and session_data — so the same predicate works for both.
-    const eventDataExplicit = new Set(Object.keys(eventDataStep.select.columns));
-    const sessionDataExplicit = new Set(Object.keys(sessionDataStep.select.columns));
-    const eventDataEnrichmentExcept = enrichmentExcludedColumns.filter(c => eventDataExplicit.has(c));
-    const sessionDataEnrichmentExcept = enrichmentExcludedColumns.filter(c => sessionDataExplicit.has(c));
+    // Build enrichment-source CTEs and gather event-level join/column data. Item-level
+    // enrichments throw "not yet supported" inside the utility — they will arrive in a later release.
+    const { steps: enrichmentSteps, joins: enrichmentJoins, columns: enrichmentColumns,
+            columnNames: enrichmentColumnNames } = utils.buildEnrichments(mergedConfig.enrichments);
+
+    // Build the set of columns the outer SELECT already maps explicitly (so wildcards skip them)
+    // plus internal-only columns that should never reach enhanced_events.
+    const alreadyMapped = [
+        ...Object.keys(finalColumnOrder),
+        ...Object.keys(itemListOverrides),
+        ...enrichmentColumnNames,
+        'entrances',
+        mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
+        'data_is_final',
+        'export_type',
+        ...itemListExcludedColumns,
+    ];
 
     // Join event_data and session_data, include additional logic
     // Named 'enhanced_events' so user-supplied customSteps can reference it as a stable handle.
@@ -399,27 +348,10 @@ ${excludedEventsSQL}`,
                 ...finalColumnOrder,
                 ...itemListOverrides,
                 // event-level enrichment columns: override matching explicit columns; new columns added.
-                // Wildcard-column overlap is handled below via excludedColumns.
                 ...enrichmentColumns,
-                // get the rest of the event_data columns
-                '[sql]event_data': utils.selectOtherColumns(
-                    eventDataStep,
-                    Object.keys(finalColumnOrder),
-                    [
-                        'entrances',
-                        mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
-                        'data_is_final',
-                        'export_type',
-                        ...itemListExcludedColumns,
-                        ...eventDataEnrichmentExcept,
-                    ]
-                ),
-                // get the rest of the session_data columns
-                '[sql]session_data': utils.selectOtherColumns(
-                    sessionDataStep,
-                    Object.keys(finalColumnOrder),
-                    sessionDataEnrichmentExcept,
-                ),
+                // explicit pass-throughs for the rest of event_data and session_data
+                ...utils.buildQualifiedPassThroughs(eventDataStep, alreadyMapped),
+                ...utils.buildQualifiedPassThroughs(sessionDataStep, alreadyMapped),
                 // include additional columns
                 row_inserted_timestamp: 'current_timestamp()',
                 data_is_final: 'data_is_final',

package/utils.js

@@ -474,53 +474,6 @@ const mergeDataformTableConfigurations = (defaultConfig, inputConfig = {}) => {
     return deepMerge(defaultConfig, inputConfig);
 };
 
-/**
- * Generates a SQL selection string for a given query step, excluding columns already defined elsewhere
- * or columns that should be excluded.
- *
- * This utility is helpful when joining tables/CTEs to avoid selecting duplicate or already-present columns.
- * 
- * @param {Object} step - A queryBuilder structured step containing a `name` (CTE/table alias) and a `select.columns` object.
- * @param {string[]} [alreadyDefinedColumns=[]] - Columns that have already been defined and should be excluded from selection.
- * @param {string[]} [excludedColumns=[]] - Additional columns to explicitly exclude from selection.
- * @returns {string|undefined} A SQL select string (e.g. 'stepName.*' or 'stepName.* except (col1, col2)'), or undefined if all columns are excluded.
- */
-const selectOtherColumns = (step, alreadyDefinedColumns = [], excludedColumns = []) => {
-    const stepName = step.name;
-    const stepColumns = Object.keys(step.select.columns);
-
-    // Columns in step.select.columns that should be excluded (already-defined or explicitly listed)
-    const internalExcept = stepColumns.filter(
-        column => alreadyDefinedColumns.includes(column) || excludedColumns.includes(column)
-    );
-
-    // Columns in excludedColumns that aren't enumerated in step.select.columns. These are
-    // wildcard-sourced columns (e.g. default GA4 export columns coming through `event_data.*`
-    // inside event_data's own select). The caller knows what to exclude; trust them.
-    // BigQuery throws at dry-run if the column doesn't exist in the source — surfaces typos.
-    // Filter out undefined/null entries (callers can pass conditional values like
-    // `cond ? 'col' : undefined` for ergonomics).
-    const externalExcept = excludedColumns.filter(
-        c => typeof c === 'string' && c.length > 0 && !stepColumns.includes(c)
-    );
-
-    const allExcept = [...internalExcept, ...externalExcept];
-
-    // If nothing is excluded, select everything
-    if (allExcept.length === 0) {
-        return `${stepName}.*`;
-    }
-
-    // If every enumerated column is excluded and there are no external excepts to apply,
-    // there's nothing to select via the wildcard
-    if (internalExcept.length === stepColumns.length && externalExcept.length === 0) {
-        return;
-    }
-
-    return `${stepName}.* except (${allExcept.join(', ')})`;
-};
-
-
 /**
  * Builds a queryBuilder `select.columns` fragment that passes through every source column
  * not already covered by an explicit columns object.
@@ -560,6 +513,119 @@ const buildPassThroughs = (explicitColumns, sourceColumns) => {
 };
 
 
+/**
+ * Builds the per-enrichment CTE definitions, JOIN clauses, and column-name mappings for the
+ * declarative `enrichments` feature.
+ *
+ * Pure config-to-data mapping. No knowledge of downstream CTEs or specific table modules —
+ * intended to be called by any table module that exposes an `enrichments` config field.
+ *
+ * Encapsulates two generation-time throws:
+ *   - level: 'item' (not yet supported; deferred per design_docs/planned/data-enrichments.md Q15).
+ *   - Enrichment-vs-enrichment column collisions (two enrichments targeting the same column).
+ *
+ * @param {Array<Object>} enrichments - Validated enrichment entries. Each entry has fields:
+ *   { name, level, source, joinKey, columns, dedupe? } per data-enrichments.md Q8.
+ * @returns {Object} A struct with five fields:
+ *   - `steps` — array of queryBuilder source-CTE step definitions (one `enrich_<name>` per entry).
+ *   - `joins` — array of LEFT JOIN clauses to attach downstream (one per entry).
+ *   - `columns` — map of `{ <enrichmentColumn>: 'enrich_<name>.<col>' }` for spreading into a
+ *     downstream SELECT's `select.columns`.
+ *   - `columnNames` — Set of all enrichment column names (used by callers for overlap detection
+ *     against downstream CTEs).
+ *   - `columnOwner` — map of `{ <column>: { i, name } }` recording which enrichment owns each
+ *     column; preserved for diagnostics.
+ *
+ * @throws {Error} If any entry has `level: 'item'` (with a pointer to data-enrichments.md).
+ * @throws {Error} If two enrichments target the same column name (with both enrichment names).
+ *
+ * @example
+ *   const { steps, joins, columns, columnNames } = buildEnrichments(config.enrichments);
+ */
+const buildEnrichments = (enrichments) => {
+    const steps = [];
+    const joins = [];
+    const columns = {};
+    const columnNames = new Set();
+    const columnOwner = {};
+
+    for (const [i, e] of (enrichments ?? []).entries()) {
+        const level = e.level ?? 'event';
+        if (level === 'item') {
+            throw new Error(
+                `config.enrichments[${i}] uses level: 'item', which is not yet supported in this version. ` +
+                `Item-level enrichments will ship in a future release; see design_docs/planned/data-enrichments.md.`
+            );
+        }
+        const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
+        const cteName = `enrich_${e.name}`;
+
+        // Source CTE selects joinKey columns plus the requested columns. key === value
+        // shape skips the alias clause in queryBuilder's columnsToSQL.
+        const cteCols = {};
+        for (const k of joinKeys) cteCols[k] = k;
+        for (const c of e.columns) cteCols[c] = c;
+        const sourceStep = { name: cteName, select: { columns: cteCols }, from: e.source };
+        // Opt-in dedupe: which row wins is non-deterministic — users with strict needs
+        // pre-aggregate in their source SQL.
+        if (e.dedupe) {
+            sourceStep.qualify = `row_number() over (partition by ${joinKeys.join(', ')}) = 1`;
+        }
+        steps.push(sourceStep);
+
+        joins.push({ type: 'left', table: cteName, on: `using(${joinKeys.join(', ')})` });
+
+        for (const c of e.columns) {
+            if (columnNames.has(c)) {
+                const owner = columnOwner[c];
+                throw new Error(
+                    `config.enrichments[${i}] (name: '${e.name}') and config.enrichments[${owner.i}] ` +
+                    `(name: '${owner.name}') both target column '${c}'. ` +
+                    `Two enrichments cannot write the same column; rename one in source SQL or pick a different name.`
+                );
+            }
+            columns[c] = `${cteName}.${c}`;
+            columnNames.add(c);
+            columnOwner[c] = { i, name: e.name };
+        }
+    }
+
+    return { steps, joins, columns, columnNames, columnOwner };
+};
+
+
+/**
+ * Builds a qualified pass-through fragment for spreading into a downstream SELECT's
+ * `select.columns`. For each column in `step.select.columns` not already in `alreadyCovered`,
+ * emits an entry of the form `{ <col>: '<step.name>.<col>' }`.
+ *
+ * Columns whose values in `step.select.columns` are `undefined` (the user-exclusion sentinel
+ * shape from getExcludedColumns) are skipped. Names in `alreadyCovered` that don't exist in
+ * `step.select.columns` are silently ignored — the loop only iterates `step.select.columns`,
+ * so unknown names cause no harm. This is the safety property that lets callers pass
+ * "everything that might collide" without pre-filtering.
+ *
+ * @param {Object} step - A queryBuilder step with a `name` and `select.columns` object.
+ * @param {Iterable<string>} alreadyCovered - Column names already mapped elsewhere in the
+ *   downstream SELECT, plus any internal-only columns the downstream SELECT shouldn't re-emit.
+ * @returns {Object} A map of `{ <col>: '<step.name>.<col>' }` entries.
+ *
+ * @example
+ *   buildQualifiedPassThroughs(eventDataStep, ['event_date', 'session_id', 'entrances']);
+ *   // → { event_name: 'event_data.event_name', user_pseudo_id: 'event_data.user_pseudo_id', ... }
+ */
+const buildQualifiedPassThroughs = (step, alreadyCovered) => {
+    const covered = new Set(alreadyCovered);
+    const passThroughs = {};
+    for (const [col, expr] of Object.entries(step.select.columns)) {
+        if (expr === undefined) continue;
+        if (covered.has(col)) continue;
+        passThroughs[col] = `${step.name}.${col}`;
+    }
+    return passThroughs;
+};
+
+
 /**
  * Processes a date input string and returns a corresponding SQL date casting expression,
  * or passes through BigQuery SQL statements as-is.
@@ -634,8 +700,9 @@ module.exports = {
     queryBuilder,
     isDataformTableReferenceObject,
     setDataformContext,
-    selectOtherColumns,
     buildPassThroughs,
+    buildEnrichments,
+    buildQualifiedPassThroughs,
     processDate,
     getDatasetName
 };