ga4-export-fixer 0.9.0-dev.1 → 0.9.0-dev.10

package/tables/ga4EventsEnhanced/validation.js

@@ -201,11 +201,11 @@ const validateEnhancedEventsConfig = (config, options = {}) => {
         }
     }
 
-    // customSteps - optional array of queryBuilder step objects appended to the pipeline
-    // Layer 1 (config shape): array, objects with non-empty name, no duplicates within customSteps.
+    // customSteps - optional array of queryBuilder step objects appended to the pipeline.
+    // Config-shape checks only: array, objects with non-empty name, no duplicates within customSteps.
     // Step-shape validation (clause keys, etc.) deferred to queryBuilder.
-    // Collision-with-package-names check deferred to _generateEnhancedEventsSQL (Layer 2),
-    // since the reserved set is config-dependent (e.g. item_list_* only exist when itemListAttribution is on).
+    // Collision-with-package-names check deferred to _generateEnhancedEventsSQL, since the
+    // reserved set is config-dependent (e.g. item_list_* only exist when itemListAttribution is on).
     if (config.customSteps !== undefined) {
         if (!Array.isArray(config.customSteps)) {
             throw new Error(`config.customSteps must be an array. Received: ${JSON.stringify(config.customSteps)}`);
@@ -225,6 +225,101 @@ const validateEnhancedEventsConfig = (config, options = {}) => {
             seenNames.add(step.name);
         }
     }
+
+    // enrichments - optional array of declarative external-data enrichment specs.
+    // Config-shape checks only. Reserved-name collision and item-level joinKey resolution
+    // happen in _generateEnhancedEventsSQL, where the reserved set and item-level join targets
+    // are derived from the resolved config.
+    if (config.enrichments !== undefined) {
+        if (!Array.isArray(config.enrichments)) {
+            throw new Error(`config.enrichments must be an array. Received: ${JSON.stringify(config.enrichments)}`);
+        }
+        const validLevels = ['event', 'item'];
+        const seenNames = new Set();
+        for (let i = 0; i < config.enrichments.length; i++) {
+            const entry = config.enrichments[i];
+            if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
+                throw new Error(`config.enrichments[${i}] must be a non-null object. Received: ${JSON.stringify(entry)}`);
+            }
+            if (typeof entry.name !== 'string' || !entry.name.trim()) {
+                throw new Error(`config.enrichments[${i}].name must be a non-empty string. Received: ${JSON.stringify(entry.name)}`);
+            }
+            if (seenNames.has(entry.name)) {
+                throw new Error(`config.enrichments contains duplicate name '${entry.name}'. Each enrichments entry must have a unique name.`);
+            }
+            seenNames.add(entry.name);
+            if (entry.level !== undefined && !validLevels.includes(entry.level)) {
+                throw new Error(`config.enrichments[${i}].level must be one of: ${validLevels.join(', ')}. Received: ${JSON.stringify(entry.level)}`);
+            }
+            // source: Dataform table reference object or backtick-quoted string
+            if (entry.source === undefined || entry.source === null) {
+                throw new Error(`config.enrichments[${i}].source is required.`);
+            }
+            if (isDataformTableReferenceObject(entry.source)) {
+                // Valid Dataform reference
+            } else if (typeof entry.source === 'string') {
+                if (!entry.source.trim()) {
+                    throw new Error(`config.enrichments[${i}].source must be a non-empty string. Received empty string.`);
+                }
+                if (!/^`[^\.]+\.[^\.]+\.[^\.]+`$/.test(entry.source.trim())) {
+                    throw new Error(`config.enrichments[${i}].source must be in the format '\`project.dataset.table\`' (with backticks) or a Dataform table reference. Received: ${JSON.stringify(entry.source)}`);
+                }
+            } else {
+                throw new Error(`config.enrichments[${i}].source must be a Dataform table reference object or a string in format '\`project.dataset.table\`'. Received: ${JSON.stringify(entry.source)}`);
+            }
+            // joinKey: required, plain SQL identifier OR non-empty array of plain SQL identifiers.
+            // Plain identifier = ^[a-zA-Z_][a-zA-Z0-9_]*$ — no aliases (`id as user_id`), no backticks,
+            // no dotted paths. Users with mismatched dim-column names alias in an upstream Dataform view.
+            const sqlIdentifier = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+            const aliasingHint = ' Aliases like \'id as user_id\' are not supported here; alias in an upstream Dataform view if your dim has a different column name.';
+            if (entry.joinKey === undefined || entry.joinKey === null) {
+                throw new Error(`config.enrichments[${i}].joinKey is required.`);
+            }
+            if (typeof entry.joinKey === 'string') {
+                if (!entry.joinKey.trim()) {
+                    throw new Error(`config.enrichments[${i}].joinKey must be a non-empty string. Received empty string.`);
+                }
+                if (!sqlIdentifier.test(entry.joinKey)) {
+                    throw new Error(`config.enrichments[${i}].joinKey must be a plain SQL identifier. Received: ${JSON.stringify(entry.joinKey)}.${aliasingHint}`);
+                }
+            } else if (Array.isArray(entry.joinKey)) {
+                if (entry.joinKey.length === 0) {
+                    throw new Error(`config.enrichments[${i}].joinKey must be a non-empty array when provided as an array.`);
+                }
+                for (let j = 0; j < entry.joinKey.length; j++) {
+                    const k = entry.joinKey[j];
+                    if (typeof k !== 'string' || !k.trim()) {
+                        throw new Error(`config.enrichments[${i}].joinKey[${j}] must be a non-empty string. Received: ${JSON.stringify(k)}`);
+                    }
+                    if (!sqlIdentifier.test(k)) {
+                        throw new Error(`config.enrichments[${i}].joinKey[${j}] must be a plain SQL identifier. Received: ${JSON.stringify(k)}.${aliasingHint}`);
+                    }
+                }
+            } else {
+                throw new Error(`config.enrichments[${i}].joinKey must be a string or a non-empty array of strings. Received: ${JSON.stringify(entry.joinKey)}`);
+            }
+            // columns: required, non-empty array of plain SQL identifiers (no aliasing).
+            if (!Array.isArray(entry.columns)) {
+                throw new Error(`config.enrichments[${i}].columns must be an array. Received: ${JSON.stringify(entry.columns)}`);
+            }
+            if (entry.columns.length === 0) {
+                throw new Error(`config.enrichments[${i}].columns must be non-empty. List the source columns to add to the output (excluding joinKey).`);
+            }
+            for (let j = 0; j < entry.columns.length; j++) {
+                const c = entry.columns[j];
+                if (typeof c !== 'string' || !c.trim()) {
+                    throw new Error(`config.enrichments[${i}].columns[${j}] must be a non-empty string. Received: ${JSON.stringify(c)}`);
+                }
+                if (!sqlIdentifier.test(c)) {
+                    throw new Error(`config.enrichments[${i}].columns[${j}] must be a plain SQL identifier. Received: ${JSON.stringify(c)}.${aliasingHint}`);
+                }
+            }
+            // dedupe: optional boolean
+            if (entry.dedupe !== undefined && typeof entry.dedupe !== 'boolean') {
+                throw new Error(`config.enrichments[${i}].dedupe must be a boolean when provided. Received: ${JSON.stringify(entry.dedupe)}`);
+            }
+        }
+    }
   } catch (e) {
     e.message = `Config validation: ${e.message}`;
     throw e;

package/utils.js

@@ -389,6 +389,16 @@ const setDataformContext = (ctx, config) => {
         }
     }
 
+    // resolve Dataform refs in enrichments[].source the same way as sourceTable
+    if (Array.isArray(config.enrichments)) {
+        config.enrichments = config.enrichments.map(e => {
+            if (isDataformTableReferenceObject(e.source)) {
+                return { ...e, source: ctx.ref(e.source) };
+            }
+            return e;
+        });
+    }
+
     config.self = ctx.self();
     config.incremental = ctx.incremental();
 
@@ -465,37 +475,162 @@ const mergeDataformTableConfigurations = (defaultConfig, inputConfig = {}) => {
 };
 
 /**
- * Generates a SQL selection string for a given query step, excluding columns already defined elsewhere
- * or columns that should be excluded.
+ * Builds a queryBuilder `select.columns` fragment that passes through every source column
+ * not already covered by an explicit columns object.
  *
- * This utility is helpful when joining tables/CTEs to avoid selecting duplicate or already-present columns.
- * 
- * @param {Object} step - A queryBuilder structured step containing a `name` (CTE/table alias) and a `select.columns` object.
- * @param {string[]} [alreadyDefinedColumns=[]] - Columns that have already been defined and should be excluded from selection.
- * @param {string[]} [excludedColumns=[]] - Additional columns to explicitly exclude from selection.
- * @returns {string|undefined} A SQL select string (e.g. 'stepName.*' or 'stepName.* except (col1, col2)'), or undefined if all columns are excluded.
+ * A source column is considered "covered" — and skipped from pass-throughs — when it appears as:
+ *   - a KEY in `explicitColumns` (a transform, package promotion, or undefined-valued exclusion
+ *     sentinel like `{ event_dimensions: undefined }`), OR
+ *   - a VALUE in `explicitColumns` (a bare source-column identifier referenced by a value-side
+ *     rename, e.g. `{ user_traffic_source: 'traffic_source' }` covers 'traffic_source').
+ *
+ * Values that are SQL expressions, function calls, or non-strings never count as coverage —
+ * they reference the source column internally but the column itself is still available as a
+ * pass-through. (`.includes()` compares by strict equality, so 'extract(datetime from ...)'
+ * never matches a bare column name.)
+ *
+ * @param {Object} explicitColumns - A queryBuilder step's explicit `select.columns` entries.
+ * @param {Iterable<string>} sourceColumns - Column names available on the source schema.
+ * @returns {Object} A map of `{ column: column }` entries for every source column not covered.
+ *
+ * @example
+ *   buildPassThroughs(
+ *     { event_name: 'event_name', user_traffic_source: 'traffic_source' },
+ *     ['event_name', 'traffic_source', 'device', 'geo']
+ *   );
+ *   // → { device: 'device', geo: 'geo' }
  */
-const selectOtherColumns = (step, alreadyDefinedColumns = [], excludedColumns = []) => {
-    const stepName = step.name;
-    const stepColumns = Object.keys(step.select.columns);
-
-    // Determine which columns to exclude: those already defined or explicitly excluded
-    const exceptColumns = stepColumns.filter(
-        column => alreadyDefinedColumns.includes(column) || excludedColumns.includes(column)
-    );
-
-    // If none of the columns have been defined or excluded, select them all
-    if (exceptColumns.length === 0) {
-        return `${stepName}.*`;
+const buildPassThroughs = (explicitColumns, sourceColumns) => {
+    const explicitKeys = Object.keys(explicitColumns);
+    const explicitValues = Object.values(explicitColumns);
+    const passThroughs = {};
+    for (const column of sourceColumns) {
+        if (!explicitKeys.includes(column) && !explicitValues.includes(column)) {
+            passThroughs[column] = column;
+        }
     }
+    return passThroughs;
+};
 
-    // If all columns have been defined or excluded, do not select any
-    if (exceptColumns.length === stepColumns.length) {
-        return;
+
+/**
+ * Builds the per-enrichment CTE definitions, JOIN clauses, and column-name mappings for the
+ * declarative `enrichments` feature. Routes event-level and item-level entries through
+ * separate output channels so the caller can attach them to different downstream CTEs.
+ *
+ * Pure config-to-data mapping. No knowledge of downstream CTEs or specific table modules —
+ * intended to be called by any table module that exposes an `enrichments` config field.
+ *
+ * Encapsulates one generation-time throw:
+ *   - Same-level enrichment-vs-enrichment column collisions (two event-level enrichments or
+ *     two item-level enrichments targeting the same column). Cross-level same-name is allowed —
+ *     the two columns target structurally distinct slots (`enhanced_events.<col>` vs
+ *     `items[].<col>`).
+ *
+ * @param {Array<Object>} enrichments - Validated enrichment entries. Each entry has fields:
+ *   { name, level, source, joinKey, columns, dedupe? }. `level` is 'event' (default) or 'item'.
+ * @returns {Object} A struct with four fields:
+ *   - `steps` — array of queryBuilder source-CTE step definitions (one `enrich_<name>` per
+ *     entry, regardless of level — all source CTEs go to the top of the pipeline).
+ *   - `event` — { joins, columns, columnNames } for event-level enrichments. Caller attaches
+ *     `joins` to the event-grained downstream CTE (e.g. `enhanced_events`) and spreads `columns`
+ *     into that CTE's `select.columns`.
+ *   - `item` — { joins, columns, columnNames } for item-level enrichments. Caller attaches
+ *     `joins` to the item-grained downstream CTE (e.g. `items_rebuilt`) and folds `columns`
+ *     into that CTE's struct construction.
+ *   - `columnOwner` — map of `{ <column>: { i, name, level } }` recording which enrichment
+ *     owns each column. The `level` field distinguishes cross-level same-name entries.
+ *
+ * @throws {Error} If two same-level enrichments target the same column name (with both
+ *   enrichment names and the conflicting column in the error message).
+ *
+ * @example
+ *   const { steps, event, item } = buildEnrichments(config.enrichments);
+ *   // event.joins → attach to enhanced_events; event.columns → spread into enhanced_events
+ *   // item.joins → attach to items_rebuilt; item.columns → fold into items struct
+ */
+const buildEnrichments = (enrichments) => {
+    const steps = [];
+    const channels = {
+        event: { joins: [], columns: {}, columnNames: new Set() },
+        item: { joins: [], columns: {}, columnNames: new Set() },
+    };
+    const columnOwner = {};
+
+    for (const [i, e] of (enrichments ?? []).entries()) {
+        const level = e.level ?? 'event';
+        const channel = channels[level];
+        const joinKeys = Array.isArray(e.joinKey) ? e.joinKey : [e.joinKey];
+        const cteName = `enrich_${e.name}`;
+
+        // Source CTE selects joinKey columns plus the requested columns. key === value
+        // shape skips the alias clause in queryBuilder's columnsToSQL.
+        const cteCols = {};
+        for (const k of joinKeys) cteCols[k] = k;
+        for (const c of e.columns) cteCols[c] = c;
+        const sourceStep = { name: cteName, select: { columns: cteCols }, from: e.source };
+        // Opt-in dedupe: which row wins is non-deterministic — users with strict needs
+        // pre-aggregate in their source SQL.
+        if (e.dedupe) {
+            sourceStep.qualify = `row_number() over (partition by ${joinKeys.join(', ')}) = 1`;
+        }
+        steps.push(sourceStep);
+
+        channel.joins.push({ type: 'left', table: cteName, on: `using(${joinKeys.join(', ')})` });
+
+        for (const c of e.columns) {
+            // Same-level collision throw. Cross-level same-name is allowed because the two
+            // columns target structurally distinct output slots (event_data vs items[]).
+            if (channel.columnNames.has(c)) {
+                const owner = columnOwner[c];
+                throw new Error(
+                    `config.enrichments[${i}] (name: '${e.name}') and config.enrichments[${owner.i}] ` +
+                    `(name: '${owner.name}') both target column '${c}' at level '${level}'. ` +
+                    `Two enrichments cannot write the same column at the same level; rename one in source SQL or pick a different name.`
+                );
+            }
+            channel.columns[c] = `${cteName}.${c}`;
+            channel.columnNames.add(c);
+            // columnOwner is keyed by column name; if the same name appears at different
+            // levels, the second-writer entry wins, but we record level so diagnostics
+            // distinguish them. Same-level collisions throw above before reaching here.
+            columnOwner[c] = { i, name: e.name, level };
+        }
     }
 
-    // Otherwise, select all except the excluded/defined ones
-    return `${stepName}.* except (${exceptColumns.join(', ')})`;
+    return { steps, event: channels.event, item: channels.item, columnOwner };
+};
+
+
+/**
+ * Builds a qualified pass-through fragment for spreading into a downstream SELECT's
+ * `select.columns`. For each column in `step.select.columns` not already in `alreadyCovered`,
+ * emits an entry of the form `{ <col>: '<step.name>.<col>' }`.
+ *
+ * Columns whose values in `step.select.columns` are `undefined` (the user-exclusion sentinel
+ * shape from getExcludedColumns) are skipped. Names in `alreadyCovered` that don't exist in
+ * `step.select.columns` are silently ignored — the loop only iterates `step.select.columns`,
+ * so unknown names cause no harm. This is the safety property that lets callers pass
+ * "everything that might collide" without pre-filtering.
+ *
+ * @param {Object} step - A queryBuilder step with a `name` and `select.columns` object.
+ * @param {Iterable<string>} alreadyCovered - Column names already mapped elsewhere in the
+ *   downstream SELECT, plus any internal-only columns the downstream SELECT shouldn't re-emit.
+ * @returns {Object} A map of `{ <col>: '<step.name>.<col>' }` entries.
+ *
+ * @example
+ *   buildQualifiedPassThroughs(eventDataStep, ['event_date', 'session_id', 'entrances']);
+ *   // → { event_name: 'event_data.event_name', user_pseudo_id: 'event_data.user_pseudo_id', ... }
+ */
+const buildQualifiedPassThroughs = (step, alreadyCovered) => {
+    const covered = new Set(alreadyCovered);
+    const passThroughs = {};
+    for (const [col, expr] of Object.entries(step.select.columns)) {
+        if (expr === undefined) continue;
+        if (covered.has(col)) continue;
+        passThroughs[col] = `${step.name}.${col}`;
+    }
+    return passThroughs;
 };
 
 
@@ -573,7 +708,9 @@ module.exports = {
     queryBuilder,
     isDataformTableReferenceObject,
     setDataformContext,
-    selectOtherColumns,
+    buildPassThroughs,
+    buildEnrichments,
+    buildQualifiedPassThroughs,
     processDate,
     getDatasetName
 };