ga4-export-fixer 0.7.1-dev.0 → 0.8.0-dev.1

package/README.md

@@ -163,7 +163,7 @@ Include the package in the package.json file in your Dataform repository.
 {
   "dependencies": {
     "@dataform/core": "3.0.42",
-    "ga4-export-fixer": "0.7.0"
+    "ga4-export-fixer": "0.7.1"
   }
 }
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.7.1-dev.0",
+  "version": "0.8.0-dev.1",
   "description": "",
   "main": "index.js",
   "files": [
@@ -17,7 +17,7 @@
     "createTable.js"
   ],
   "scripts": {
-    "test": "node tests/ga4EventsEnhanced.test.js && node tests/assertions.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js",
+    "test": "node tests/ga4EventsEnhanced.test.js && node tests/assertions.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js && node tests/inputValidation.test.js && node tests/createTable.test.js && node tests/queryBuilder.test.js",
     "test:summary": "node tests/testRunner.js",
     "test:docs": "node tests/documentation.test.js",
     "test:preops": "node tests/preOperations.test.js",
@@ -26,6 +26,7 @@
     "test:validation": "node tests/inputValidation.test.js",
     "test:assertions": "node tests/assertions.test.js",
     "test:createTable": "node tests/createTable.test.js",
+    "test:queryBuilder": "node tests/queryBuilder.test.js",
     "test:integration": "node tests/integration/integration.test.js",
     "release:dev": "./scripts/release-dev.sh",
     "readme": "node scripts/updateReadme.js",

package/tables/ga4EventsEnhanced/index.js

@@ -106,9 +106,9 @@ const getFinalColumnOrder = (eventDataStep, sessionDataStep) => {
     // Construct the columns object: key is column name, value is {step.name}.{column}
     const columnOrder = {};
     for (const col of finalColumnOrder) {
-        if (sessionDataStep?.columns?.hasOwnProperty(col) && sessionDataStep.columns[col] !== undefined) {
+        if (sessionDataStep?.select?.columns?.hasOwnProperty(col) && sessionDataStep.select.columns[col] !== undefined) {
             columnOrder[col] = `${sessionDataStep.name}.${col}`;
-        } else if (eventDataStep?.columns?.hasOwnProperty(col) && eventDataStep.columns[col] !== undefined) {
+        } else if (eventDataStep?.select?.columns?.hasOwnProperty(col) && eventDataStep.select.columns[col] !== undefined) {
             columnOrder[col] = `${eventDataStep.name}.${col}`;
         }
     }
@@ -200,46 +200,48 @@ const _generateEnhancedEventsSQL = (mergedConfig) => {
     // initial step: extract data from the export tables
     const eventDataStep = {
         name: 'event_data',
-        columns: {
-            // exclude default export columns that are not needed
-            // do this first so that the columns defined later are not excluded
-            ...getExcludedColumns(),
-            // date and time
-            event_date: helpers.eventDate,
-            event_datetime: `extract(datetime from timestamp_micros(${helpers.getEventTimestampMicros(mergedConfig.customTimestampParam)}) at time zone '${mergedConfig.timezone}')`,
-            event_timestamp: 'event_timestamp',
-            event_custom_timestamp: mergedConfig.customTimestampParam ? helpers.getEventTimestampMicros(mergedConfig.customTimestampParam) : undefined,
-            // event name
-            event_name: 'event_name',
-            // identifiers
-            session_id: helpers.sessionId,
-            user_pseudo_id: 'user_pseudo_id',
-            user_id: 'user_id',
-            // page
-            page_location: helpers.unnestEventParam('page_location', 'string'),
-            page: helpers.extractPageDetails(),
-            // event parameters and user properties
-            ...promotedEventParameters(),
-            event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
-            user_properties: 'user_properties',
-            // traffic source
-            collected_traffic_source: 'collected_traffic_source',
-            session_traffic_source_last_click: 'session_traffic_source_last_click',
-            user_traffic_source: 'traffic_source',
-            // ecommerce
-            ecommerce: helpers.fixEcommerceStruct('ecommerce'),
-            items: 'items',
-            _item_list_attribution_row_id: itemListAttribution ? helpers.itemListAttributionRowId(ecommerceEventsFilter) : undefined,
-            // flag if the data is "final" and is not expected to change anymore
-            data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
-            export_type: helpers.getGa4ExportType('_table_suffix'),
-            // prep columns for later steps
-            entrances: helpers.unnestEventParam('entrances', 'int'),
-            session_params_prep: mergedConfig.sessionParams.length > 0 ? helpers.filterEventParams(mergedConfig.sessionParams, 'include') : undefined,
-            // include all other columns from the export data
-            get '[sql]other_columns'() {
-                const definedColumns = Object.keys(this);
-                return `* except (${definedColumns.filter(column => helpers.isGa4ExportColumn(column)).join(', ')})`;
+        select: {
+            columns: {
+                // exclude default export columns that are not needed
+                // do this first so that the columns defined later are not excluded
+                ...getExcludedColumns(),
+                // date and time
+                event_date: helpers.eventDate,
+                event_datetime: `extract(datetime from timestamp_micros(${helpers.getEventTimestampMicros(mergedConfig.customTimestampParam)}) at time zone '${mergedConfig.timezone}')`,
+                event_timestamp: 'event_timestamp',
+                event_custom_timestamp: mergedConfig.customTimestampParam ? helpers.getEventTimestampMicros(mergedConfig.customTimestampParam) : undefined,
+                // event name
+                event_name: 'event_name',
+                // identifiers
+                session_id: helpers.sessionId,
+                user_pseudo_id: 'user_pseudo_id',
+                user_id: 'user_id',
+                // page
+                page_location: helpers.unnestEventParam('page_location', 'string'),
+                page: helpers.extractPageDetails(),
+                // event parameters and user properties
+                ...promotedEventParameters(),
+                event_params: helpers.filterEventParams(mergedConfig.excludedEventParams, 'exclude'),
+                user_properties: 'user_properties',
+                // traffic source
+                collected_traffic_source: 'collected_traffic_source',
+                session_traffic_source_last_click: 'session_traffic_source_last_click',
+                user_traffic_source: 'traffic_source',
+                // ecommerce
+                ecommerce: helpers.fixEcommerceStruct('ecommerce'),
+                items: 'items',
+                _item_list_attribution_row_id: itemListAttribution ? helpers.itemListAttributionRowId(ecommerceEventsFilter) : undefined,
+                // flag if the data is "final" and is not expected to change anymore
+                data_is_final: helpers.isFinalData(mergedConfig.dataIsFinal.detectionMethod, mergedConfig.dataIsFinal.dayThreshold),
+                export_type: helpers.getGa4ExportType('_table_suffix'),
+                // prep columns for later steps
+                entrances: helpers.unnestEventParam('entrances', 'int'),
+                session_params_prep: mergedConfig.sessionParams.length > 0 ? helpers.filterEventParams(mergedConfig.sessionParams, 'include') : undefined,
+                // include all other columns from the export data
+                get '[sql]other_columns'() {
+                    const definedColumns = Object.keys(this);
+                    return `* except (${definedColumns.filter(column => helpers.isGa4ExportColumn(column)).join(', ')})`;
+                },
             },
         },
         from: mergedConfig.sourceTable,
@@ -250,18 +252,20 @@ ${excludedEventsSQL}`,
     // Do session-level data aggregation
     const sessionDataStep = {
         name: 'session_data',
-        columns: {
-            session_id: 'session_id',
-            user_id: helpers.aggregateValue('user_id', 'last', timestampColumn),
-            merged_user_id: `ifnull(${helpers.aggregateValue('user_id', 'last', timestampColumn)}, any_value(user_pseudo_id))`,
-            session_params: helpers.aggregateSessionParams(mergedConfig.sessionParams, 'session_params_prep', timestampColumn),
-            session_traffic_source_last_click: helpers.aggregateValue('session_traffic_source_last_click', 'first', timestampColumn),
-            session_first_traffic_source: `array_agg(collected_traffic_source order by ${timestampColumn} limit 1)[safe_offset(0)]`, // don't ignore nulls
-            landing_page: helpers.aggregateValue(`if(entrances > 0, page, null)`, 'first', timestampColumn),
+        select: {
+            columns: {
+                session_id: 'session_id',
+                user_id: helpers.aggregateValue('user_id', 'last', timestampColumn),
+                merged_user_id: `ifnull(${helpers.aggregateValue('user_id', 'last', timestampColumn)}, any_value(user_pseudo_id))`,
+                session_params: helpers.aggregateSessionParams(mergedConfig.sessionParams, 'session_params_prep', timestampColumn),
+                session_traffic_source_last_click: helpers.aggregateValue('session_traffic_source_last_click', 'first', timestampColumn),
+                session_first_traffic_source: `array_agg(collected_traffic_source order by ${timestampColumn} limit 1)[safe_offset(0)]`, // don't ignore nulls
+                landing_page: helpers.aggregateValue(`if(entrances > 0, page, null)`, 'first', timestampColumn),
+            },
         },
         from: 'event_data',
         where: `session_id is not null`,
-        groupBy: ['session_id']
+        'group by': 'session_id',
     };
 
     // item list attribution CTEs:
@@ -277,11 +281,13 @@ ${excludedEventsSQL}`,
 
         const attributionStep = {
             name: 'item_list_attribution',
-            columns: {
-                '_item_list_attribution_row_id': '_item_list_attribution_row_id',
-                'event_name': 'event_name',
-                'item': 'item',
-                '_item_list_attr': attrExpr,
+            select: {
+                columns: {
+                    '_item_list_attribution_row_id': '_item_list_attribution_row_id',
+                    'event_name': 'event_name',
+                    'item': 'item',
+                    '_item_list_attr': attrExpr,
+                },
             },
             from: 'event_data, unnest(items) as item',
             where: `event_name in (${ecommerceEventsFilter})`,
@@ -289,18 +295,20 @@ ${excludedEventsSQL}`,
 
         const dataStep = {
             name: 'item_list_data',
-            columns: {
-                '_item_list_attribution_row_id': '_item_list_attribution_row_id',
-                'items': `array_agg(
+            select: {
+                columns: {
+                    '_item_list_attribution_row_id': '_item_list_attribution_row_id',
+                    'items': `array_agg(
       (select as struct item.* replace(
         coalesce(if(${passthroughEvents}, item.item_list_name, _item_list_attr.item_list_name), '(not set)') as item_list_name,
         coalesce(if(${passthroughEvents}, item.item_list_id, _item_list_attr.item_list_id), '(not set)') as item_list_id,
         coalesce(if(${passthroughEvents}, item.item_list_index, _item_list_attr.item_list_index)) as item_list_index
       ))
     )`,
+                },
             },
             from: 'item_list_attribution',
-            groupBy: ['_item_list_attribution_row_id'],
+            'group by': '_item_list_attribution_row_id',
         };
 
         return [attributionStep, dataStep];
@@ -318,42 +326,46 @@ ${excludedEventsSQL}`,
     // Join event_data and session_data, include additional logic
     const finalStep = {
         name: 'final',
-        columns: {
-            // get the most important columns in the correct order
-            ...finalColumnOrder,
-            ...itemListOverrides,
-            // get the rest of the event_data columns
-            '[sql]event_data': utils.selectOtherColumns(
-                eventDataStep,
-                Object.keys(finalColumnOrder),
-                [
-                    'entrances',
-                    mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
-                    'data_is_final',
-                    'export_type',
-                    ...itemListExcludedColumns,
-                ]
-            ),
-            // get the rest of the session_data columns
-            '[sql]session_data': utils.selectOtherColumns(
-                sessionDataStep,
-                Object.keys(finalColumnOrder),
-                []
-            ),
-            // include additional columns
-            row_inserted_timestamp: 'current_timestamp()',
-            data_is_final: 'data_is_final',
-            export_type: 'export_type',
+        select: {
+            columns: {
+                // get the most important columns in the correct order
+                ...finalColumnOrder,
+                ...itemListOverrides,
+                // get the rest of the event_data columns
+                '[sql]event_data': utils.selectOtherColumns(
+                    eventDataStep,
+                    Object.keys(finalColumnOrder),
+                    [
+                        'entrances',
+                        mergedConfig.sessionParams.length > 0 ? 'session_params_prep' : undefined,
+                        'data_is_final',
+                        'export_type',
+                        ...itemListExcludedColumns,
+                    ]
+                ),
+                // get the rest of the session_data columns
+                '[sql]session_data': utils.selectOtherColumns(
+                    sessionDataStep,
+                    Object.keys(finalColumnOrder),
+                    []
+                ),
+                // include additional columns
+                row_inserted_timestamp: 'current_timestamp()',
+                data_is_final: 'data_is_final',
+                export_type: 'export_type',
+            },
         },
         from: 'event_data',
-        leftJoin: [
+        joins: [
             ...(itemListSteps ? [{
+                type: 'left',
                 table: 'item_list_data',
-                condition: 'using(_item_list_attribution_row_id)'
+                on: 'using(_item_list_attribution_row_id)'
             }] : []),
             {
+                type: 'left',
                 table: 'session_data',
-                condition: 'using(session_id)'
+                on: 'using(session_id)'
             }
         ],
         where: helpers.incrementalDateFilter(mergedConfig)

package/utils.js

@@ -20,15 +20,34 @@ const mergeUniqueArrays = (...arrays) => {
 
 /**
  * Build SQL query from an array of steps with support for CTEs.
- * 
- * Each step should have:
- * - name: CTE name (required for CTEs)
- * - columns: Object with column definitions { alias: 'expression' }
- * - from: Source table/CTE
- * - where: Optional WHERE clause
- * - groupBy: Optional array of GROUP BY columns
- * - leftJoin: Optional array of join definitions { table, condition }
- * 
+ *
+ * Each step is one of two shapes — structured (clause-keyed) or raw (`{name, query}`).
+ *
+ * STRUCTURED SHAPE:
+ *   { name, select, from, joins?, where?, 'group by'?, having?, qualify?, 'order by'?, limit? }
+ *
+ *   - name: CTE name (required for non-final steps)
+ *   - select: Either a string (raw select list) or { columns?: {alias: expr}, sql?: string }
+ *       - columns: alias -> expression map. `key === value` skips the alias;
+ *         keys starting with `[sql]` emit raw values with no alias; `undefined` values are filtered out.
+ *       - sql: optional raw column-list tail appended after columns
+ *   - from: Source table/CTE (string)
+ *   - joins: Either an array of { type, table, on? } or a string fallback
+ *       - type: 'left', 'inner', 'cross', 'right', 'full'
+ *       - on omitted for cross joins
+ *   - where, having, qualify, 'order by', 'group by', limit: string clauses (limit may be a number)
+ *
+ *   Clauses are emitted in canonical SQL order regardless of input key order.
+ *
+ * RAW SHAPE:
+ *   { name, query }
+ *
+ *   - name: CTE name
+ *   - query: Entire CTE body as raw SQL, emitted verbatim
+ *
+ * Detection: a step is raw iff it has a top-level `query` key. Mixing raw and structured
+ * keys within a single step throws.
+ *
  * @param {Array<Object>} steps - Array of step objects defining the query structure
  * @returns {string} Generated SQL query
  */
@@ -62,7 +81,7 @@ const queryBuilder = (steps) => {
             .join('\n');
     };
 
-    // Helper function to turn step.columns into SQL string
+    // Helper function to turn step.select.columns into SQL string
     const columnsToSQL = (columns) => {
         return Object.entries(columns)
             // exclude all columns that have been explicitly set to undefined
@@ -83,37 +102,115 @@ const queryBuilder = (steps) => {
             .join(',\n' + pad);
     };
 
-    const selectSQL = (step) => {
-        const parts = [`select\n${pad}${columnsToSQL(step.columns)}`];
-        parts.push(`from\n${pad}${step.from}`);
-
-        if (step.leftJoin) {
-            step.leftJoin.forEach(join => {
-                parts.push(`left join\n${pad}${join.table} ${join.condition}`);
-            });
+    // Renderer for the SELECT clause. Accepts a string (sugar for {sql: <string>})
+    // or an object with columns and/or sql fields.
+    const renderSelect = (value) => {
+        const v = typeof value === 'string' ? { sql: value } : value;
+        const hasColumns = v.columns !== undefined && Object.keys(v.columns).length > 0;
+        const hasSql = typeof v.sql === 'string' && v.sql.length > 0;
+        if (!hasColumns && !hasSql) {
+            throw new Error('queryBuilder: select must include at least one of `columns` or `sql`');
         }
+        const parts = [];
+        if (hasColumns) parts.push(columnsToSQL(v.columns));
+        if (hasSql) parts.push(reindent(v.sql, INDENT));
+        return `select\n${pad}${parts.join(',\n' + pad)}`;
+    };
 
-        if (step.where) {
-            parts.push(`where\n${pad}${reindent(step.where, INDENT)}`);
+    // Renderer for the JOINS clause. Accepts an array of {type, table, on}
+    // entries (rendered in array order) or a string fallback.
+    const renderJoins = (value) => {
+        if (typeof value === 'string') {
+            return reindent(value, 0);
         }
+        return value
+            .map(j => j.type === 'cross'
+                ? `cross join\n${pad}${j.table}`
+                : `${j.type} join\n${pad}${j.table} ${j.on}`)
+            .join('\n');
+    };
 
-        if (step.groupBy) {
-            parts.push(`group by\n${pad}${step.groupBy.join(', ')}`);
+    // Renderer factory for inline string clauses (where, having, group by, ...).
+    // Coerces non-strings (e.g. limit: 100) via String().
+    const renderInline = (keyword) => (value) =>
+        `${keyword}\n${pad}${reindent(String(value), INDENT)}`;
+
+    // Registry of clause renderers. Declaration order is the canonical SQL order
+    // — clauses are always emitted in this order regardless of input key order.
+    const CLAUSE_RENDERERS = [
+        { key: 'select',   render: renderSelect             },
+        { key: 'from',     render: renderInline('from')     },
+        { key: 'joins',    render: renderJoins              },
+        { key: 'where',    render: renderInline('where')    },
+        { key: 'group by', render: renderInline('group by') },
+        { key: 'having',   render: renderInline('having')   },
+        { key: 'qualify',  render: renderInline('qualify')  },
+        { key: 'order by', render: renderInline('order by') },
+        { key: 'limit',    render: renderInline('limit')    },
+    ];
+
+    const STRUCTURED_KEYS = new Set(['name', ...CLAUSE_RENDERERS.map(c => c.key)]);
+    const RAW_KEYS = new Set(['name', 'query']);
+
+    const validateStep = (step) => {
+        if (!step || typeof step !== 'object' || Array.isArray(step)) {
+            throw new Error(`queryBuilder: each step must be a non-null object, received: ${JSON.stringify(step)}`);
         }
+        const isRaw = 'query' in step;
+        const allowed = isRaw ? RAW_KEYS : STRUCTURED_KEYS;
+        const allowedList = [...allowed].map(k => `\`${k}\``).join(', ');
+        const stepLabel = step.name ? `\`${step.name}\`` : '<unnamed>';
+        for (const key of Object.keys(step)) {
+            if (!allowed.has(key)) {
+                if (isRaw && STRUCTURED_KEYS.has(key) && key !== 'name') {
+                    throw new Error(
+                        `queryBuilder: step ${stepLabel} has both \`query\` (raw shape) and \`${key}\` (structured key). ` +
+                        `Raw and structured shapes are mutually exclusive within a single step. ` +
+                        `Allowed raw-shape keys: ${allowedList}.`
+                    );
+                }
+                throw new Error(
+                    `queryBuilder: unknown key \`${key}\` in ${isRaw ? 'raw' : 'structured'} step ${stepLabel}. ` +
+                    `Allowed keys: ${allowedList}.`
+                );
+            }
+        }
+        if (isRaw) {
+            if (typeof step.query !== 'string' || step.query.length === 0) {
+                throw new Error(`queryBuilder: raw step ${stepLabel} requires a non-empty \`query\` string`);
+            }
+        } else {
+            if (step.select === undefined) {
+                throw new Error(`queryBuilder: structured step ${stepLabel} requires \`select\``);
+            }
+            if (step.from === undefined) {
+                throw new Error(`queryBuilder: structured step ${stepLabel} requires \`from\``);
+            }
+        }
+    };
 
-        return parts.join('\n');
+    const renderStep = (step) => {
+        validateStep(step);
+        if ('query' in step) {
+            // Raw shape: emit body verbatim, normalized to col 0 of the step.
+            return reindent(step.query, 0);
+        }
+        return CLAUSE_RENDERERS
+            .filter(c => step[c.key] !== undefined)
+            .map(c => c.render(step[c.key]))
+            .join('\n');
     };
 
     if (steps.length === 1) {
-        return selectSQL(steps[0]);
+        return renderStep(steps[0]);
     }
 
     const ctes = steps.slice(0, -1).map(step => {
-        const body = indentBlock(selectSQL(step), INDENT);
+        const body = indentBlock(renderStep(step), INDENT);
         return `${step.name} as (\n${body}\n)`;
     });
     const lastStep = steps[steps.length - 1];
-    return `with ${ctes.join(',\n')}\n${selectSQL(lastStep)}`;
+    return `with ${ctes.join(',\n')}\n${renderStep(lastStep)}`;
 };
 
 /**
@@ -373,14 +470,14 @@ const mergeDataformTableConfigurations = (defaultConfig, inputConfig = {}) => {
  *
  * This utility is helpful when joining tables/CTEs to avoid selecting duplicate or already-present columns.
  * 
- * @param {Object} step - The step object containing a `name` (CTE/table alias) and a `columns` object.
+ * @param {Object} step - A queryBuilder structured step containing a `name` (CTE/table alias) and a `select.columns` object.
  * @param {string[]} [alreadyDefinedColumns=[]] - Columns that have already been defined and should be excluded from selection.
  * @param {string[]} [excludedColumns=[]] - Additional columns to explicitly exclude from selection.
  * @returns {string|undefined} A SQL select string (e.g. 'stepName.*' or 'stepName.* except (col1, col2)'), or undefined if all columns are excluded.
  */
 const selectOtherColumns = (step, alreadyDefinedColumns = [], excludedColumns = []) => {
     const stepName = step.name;
-    const stepColumns = Object.keys(step.columns);
+    const stepColumns = Object.keys(step.select.columns);
 
     // Determine which columns to exclude: those already defined or explicitly excluded
     const exceptColumns = stepColumns.filter(