ga4-export-fixer 0.1.6-dev.0 → 0.1.6-dev.10

package/README.md

@@ -220,6 +220,64 @@ All fields are optional except `sourceTable`. Default values are applied automat
 
 Date fields (`dateRangeStart`, `dateRangeEnd`, etc.) accept string dates in `YYYYMMDD` or `YYYY-MM-DD` format, or BigQuery SQL expressions (e.g. `'current_date()'`, `'date(2026, 1, 1)'`).
 
+### Building on top of the ga4_events_enhanced table
+
+**`definitions/ga4/ga4_sessions.sqlx`**
+```javascript
+config {
+  type: "incremental",
+  description: "GA4 sessions table",
+  schema: "ga4_export_fixer",
+  bigquery: {
+    partitionBy: "event_date",
+    clusterBy: ['session_id', 'data_is_final'],
+  },
+  tags: ['ga4_export_fixer']
+}
+
+js {
+  const { setPreOperations, helpers } = require('ga4-export-fixer');
+
+  const config = {
+    self: self(),
+    incremental: incremental(),
+    test: false,
+    testConfig: {
+        dateRangeStart: 'current_date()-1',
+        dateRangeEnd: 'current_date()',
+    },
+    preOperations: {
+        dateRangeStartFullRefresh: 'date(2000, 1, 1)',
+        dateRangeEnd: 'current_date()',
+        //incrementalStartOverride: undefined,
+        //incrementalEndOverride: undefined,
+        //numberOfPreviousDaysToScan: 10,
+    },
+  };
+}
+
+select
+  event_date,
+  session_id,
+  user_pseudo_id,
+  any_value(session_traffic_source_last_click.cross_channel_campaign) as session_traffic_source,
+  any_value(landing_page) as landing_page,
+  current_datetime() as row_inserted_timestamp,
+  min(data_is_final) as data_is_final
+from
+  ${ref('ga4_events_enhanced_298233330')}
+where
+  ${helpers.incrementalDateFilter(config)}
+group by 
+  event_date,
+  session_id,
+  user_pseudo_id
+
+pre_operations {
+  ${setPreOperations(config)}
+}
+```
+
 ### Helpers
 
 The helpers contain templates for common SQL expressions. The functions are referenced by **ga4EventsEnhanced** but can also be imported as utility functions for working with GA4 data.

package/defaultConfig.js

@@ -0,0 +1,85 @@
+/*
+These are the configuration defaults that can be extended.
+
+For example, load the defaults in ga4EventsEnhanced.js and then extend them with whatever is psecific to the table.
+After that, extend the configuration further with the user's configuration.
+*/
+
+/*
+The base configuration. Input config validation should always check these fields.
+*/
+const baseConfig = {
+    self: undefined,
+    incremental: undefined,
+    test: false,
+    testConfig: {
+        dateRangeStart: 'current_date()-1',
+        dateRangeEnd: 'current_date()',
+    },
+    preOperations: {
+        dateRangeStartFullRefresh: 'date(2000, 1, 1)',
+        dateRangeEnd: 'current_date()',
+        // incrementalStartOverride and incrementalEndOverride are used to override the date range start and end for incremental refresh
+        // this is useful if you want to re-process only a specific date range
+        incrementalStartOverride: undefined,
+        incrementalEndOverride: undefined,
+        numberOfPreviousDaysToScan: 10,
+    },
+};
+
+/*
+The default configuration for the GA4 Events Enhanced table.
+*/
+const ga4EventsEnhancedConfig = {
+    ...baseConfig,
+    sourceTable: undefined,
+    sourceTableType: 'GA4_EXPORT', // used with pre operations to detect if ga4 export specific pre operations are needed
+    // optional but recommended
+    schemaLock: undefined,
+    // only used with js tables
+    // dataformTableConfig: {},
+    // optional
+    includedExportTypes: {
+        daily: true,
+        intraday: true,
+        fresh: false,
+    },
+    timezone: 'Etc/UTC',
+    customTimestampParam: undefined,
+    dataIsFinal: {
+        detectionMethod: 'EXPORT_TYPE', // or 'DAY_THRESHOLD'
+        dayThreshold: 4 // only used if detectionMethod is 'DAY_THRESHOLD'
+    },
+    // number of additional days to take in for taking into account sessions that overlap days
+    bufferDays: 1,
+    // these parameters are excluded by default because they've been made available in other columns
+    defaultExcludedEventParams: [
+        'page_location',
+        'ga_session_id',
+        //'custom_event_timestamp', // removed if customTimestampParam is used
+    ],
+    excludedEventParams: [],
+    eventParamsToColumns: [
+        //{name: 'page_location', type: 'string', columnName: 'page_location2'},
+    ],
+    sessionParams: [],
+    defaultExcludedEvents: [],
+    // session_start and first_visit are excluded via the excludedEvents array
+    // this allows the user to include them if needed
+    excludedEvents: [
+        'session_start',
+        'first_visit'
+    ],
+    defaultExcludedColumns: [
+        'event_dimensions', // legacy column, not needed
+        'traffic_source', // renamed to user_traffic_source
+        'session_id'
+    ],
+    // exclude these columns when extracting raw data from the export tables
+    excludedColumns: [],
+};
+
+module.exports = {
+    baseConfig,
+    ga4EventsEnhancedConfig,
+};

package/helpers.js

@@ -1,4 +1,5 @@
 const constants = require('./constants');
+const { baseConfig } = require('./defaultConfig');
 
 /*
 Unnesting parameters
@@ -248,19 +249,29 @@ const ga4ExportDateFilters = (config) => {
  *   @param {Object} [config.preOperations] - Contains full refresh date range values.
  * @returns {string} - SQL condition string to filter the query by date range.
  */
-const finalDataFilter = (config) => {
+const incrementalDateFilter = (config) => {
   const setDateRange = (start, end) => {
     return `(event_date >= ${start} and event_date <= ${end})`;
   };
 
+  // test mode
   if (config.test) {
-    return setDateRange(config.testConfig.dateRangeStart, config.testConfig.dateRangeEnd);
+    const testStart = config?.testConfig?.dateRangeStart || baseConfig.testConfig.dateRangeStart;
+    const testEnd = config?.testConfig?.dateRangeEnd || baseConfig.testConfig.dateRangeEnd;
+
+    return setDateRange(testStart, testEnd);
   }
+
+  // incremental mode
   if (config.incremental) {
     return setDateRange(constants.DATE_RANGE_START_VARIABLE, constants.DATE_RANGE_END_VARIABLE);
   }
-    
-  return setDateRange(config.preOperations.dateRangeStartFullRefresh, config.preOperations.dateRangeEnd);
+
+  // full refresh mode
+  const fullRefreshStart = config?.preOperations?.dateRangeStartFullRefresh || baseConfig.preOperations.dateRangeStartFullRefresh;
+  const fullRefreshEnd = config?.preOperations?.dateRangeEnd || baseConfig.preOperations.dateRangeEnd;
+
+  return setDateRange(fullRefreshStart, fullRefreshEnd);
 };
 
 /*
@@ -552,8 +563,14 @@ Aggregation
  *   // => SQL expression for the last user_id by event_timestamp.
  */
 const aggregateValue = (column, aggregateType, timestampColumn) => {
-  if (typeof column === 'undefined' || typeof timestampColumn === 'undefined') {
-    throw new Error("aggregateValue: 'column' and 'timestampColumn' are required parameters and must be defined.");
+  if (typeof column === 'undefined') {
+    throw new Error("aggregateValue: 'column' is a required parameter and must be defined.");
+  }
+  if (typeof aggregateType === 'undefined') {
+    throw new Error("aggregateValue: 'aggregateType' is a required parameter and must be defined.");
+  }
+  if ((aggregateType === 'first' || aggregateType === 'last') && typeof timestampColumn === 'undefined') {
+    throw new Error(`aggregateValue: 'timestampColumn' is required when aggregateType is '${aggregateType}'.`);
   }
   
   if (aggregateType === 'max') {
@@ -587,6 +604,17 @@ const aggregateValue = (column, aggregateType, timestampColumn) => {
   throw new Error(`aggregateValue: Unsupported aggregateType '${aggregateType}'. Supported values are 'max', 'min', 'first', 'last', and 'any'.`);
 };
 
+// perform aggregations on an array of values
+const aggregateValues = (values) => {
+  if (Array.isArray(values)) {
+    return values.map(value => {
+      const sqlExpression = aggregateValue(value.column, value.aggregateType, value.timestampColumn)
+      return `${sqlExpression}${value.alias ? ` as ${value.alias}` : ''}`;
+    }).join(',\n ');
+  }
+  throw new Error("aggregateValues: 'values' must be an array of objects with 'column', 'aggregateType', and 'timestampColumn' properties.");
+};
+
 /*
 Ecommerce
 */
@@ -737,6 +765,7 @@ module.exports = {
   unnestEventParam,
   sessionId,
   aggregateValue,
+  aggregateValues,
   fixEcommerceStruct,
   isFinalData,
   ga4ExportDateFilter,
@@ -744,7 +773,7 @@ module.exports = {
   filterEventParams,
   aggregateSessionParams,
   excludeNullSessionParams,
-  finalDataFilter,
+  incrementalDateFilter,
   extractPageDetails,
   extractUrlHostname,
   extractUrlPath,

package/index.js

@@ -1,26 +1,25 @@
 const helpers = require('./helpers.js');
 const ga4EventsEnhanced = require('./tables/ga4EventsEnhanced.js');
 const preOperations = require('./preOperations.js');
+const { validateBaseConfig, validateEnhancedEventsConfig } = require('./inputValidation.js');
+const { mergeSQLConfigurations } = require('./utils.js');
+const { baseConfig } = require('./defaultConfig.js');
 
 // export setPreOperations with default configuration for usage with downstream tables
-const setPreOperations = (self, incremental, preOperationsConfig) => {
-  const defaultPreOperationsConfig = {
-    dateRangeStartFullRefresh: 'date(2000, 1, 1)',
-    dateRangeEnd: 'current_date()',
-    numberOfPreviousDaysToScan: 10,
-  };
-  
-  const config = {
-    self,
-    incremental,
-    preOperations: {...defaultPreOperationsConfig, ...preOperationsConfig}
-  };
+const setPreOperations = (config) => {
+  // merge the input config with the defaults
+  const mergedConfig = mergeSQLConfigurations(baseConfig, config);
 
-  return preOperations.setPreOperations(config);
+  // do input validation on the merged config
+  validateBaseConfig(mergedConfig);
+
+  return preOperations.setPreOperations(mergedConfig);
 };
 
 module.exports = {
   helpers,
   ga4EventsEnhanced,
-  setPreOperations
+  setPreOperations,
+  validateBaseConfig,
+  validateEnhancedEventsConfig
 };

package/inputValidation.js

@@ -0,0 +1,267 @@
+const { isDataformTableReferenceObject } = require('./utils.js');
+
+/**
+ * Validates the base configuration fields shared across all table types.
+ * These correspond to the fields defined in baseConfig (defaultConfig.js):
+ * self, incremental, test, testConfig, and preOperations.
+ *
+ * @param {Object} config - The merged configuration object to validate.
+ * @throws {Error} If any base configuration value is invalid or missing.
+ */
+const validateBaseConfig = (config) => {
+    if (!config || typeof config !== 'object' || Array.isArray(config)) {
+        throw new Error(`config must be a non-null object. Received: ${JSON.stringify(config)}`);
+    }
+
+    // self - required, must be valid format
+    if (config.test !== true) {
+        if (typeof config.self !== 'string' || !config.self.trim() || !/^`[^`]+`$/.test(config.self.trim())) {
+            throw new Error(`config.self is required when config.test !== true and must be a non-empty string in format '\`project.dataset.table\`' (using the ref() function). Received: ${JSON.stringify(config.self)}`);
+        }
+    }
+
+    // incremental - required, must be boolean
+    if (typeof config.incremental !== 'boolean') {
+        throw new Error(`config.incremental must be a boolean. Received: ${JSON.stringify(config.incremental)}`);
+    }
+
+    // test - optional; when defined, must be a boolean
+    if (typeof config.test !== 'undefined' && typeof config.test !== 'boolean') {
+        throw new Error(`config.test must be a boolean when defined. Received: ${JSON.stringify(config.test)}`);
+    }
+
+    // testConfig - optional; when included, must be an object with optional dateRangeStart and dateRangeEnd
+    if (typeof config.testConfig !== 'undefined') {
+        if (!config.testConfig || typeof config.testConfig !== 'object' || Array.isArray(config.testConfig)) {
+            throw new Error(`config.testConfig must be an object when included. Received: ${JSON.stringify(config.testConfig)}`);
+        }
+        if (config.testConfig.dateRangeStart !== undefined && (typeof config.testConfig.dateRangeStart !== 'string' || !config.testConfig.dateRangeStart.trim())) {
+            throw new Error(`config.testConfig.dateRangeStart must be a non-empty string (SQL date expression) when provided. Received: ${JSON.stringify(config.testConfig.dateRangeStart)}`);
+        }
+        if (config.testConfig.dateRangeEnd !== undefined && (typeof config.testConfig.dateRangeEnd !== 'string' || !config.testConfig.dateRangeEnd.trim())) {
+            throw new Error(`config.testConfig.dateRangeEnd must be a non-empty string (SQL date expression) when provided. Received: ${JSON.stringify(config.testConfig.dateRangeEnd)}`);
+        }
+    }
+
+    // preOperations - required
+    if (config.preOperations === undefined) {
+        throw new Error("config.preOperations is required.");
+    }
+    if (!config.preOperations || typeof config.preOperations !== 'object' || Array.isArray(config.preOperations)) {
+        throw new Error(`config.preOperations must be an object. Received: ${JSON.stringify(config.preOperations)}`);
+    }
+    if (config.preOperations.numberOfPreviousDaysToScan === undefined) {
+        throw new Error("config.preOperations.numberOfPreviousDaysToScan is required.");
+    }
+    const v = config.preOperations.numberOfPreviousDaysToScan;
+    if (typeof v !== 'number' || isNaN(v) || !Number.isInteger(v) || v < 0) {
+        throw new Error(`config.preOperations.numberOfPreviousDaysToScan must be a non-negative integer. Received: ${JSON.stringify(v)}`);
+    }
+    if (config.preOperations.dateRangeStartFullRefresh === undefined || config.preOperations.dateRangeStartFullRefresh === null) {
+        throw new Error("config.preOperations.dateRangeStartFullRefresh is required.");
+    }
+    if (typeof config.preOperations.dateRangeStartFullRefresh !== 'string' || !config.preOperations.dateRangeStartFullRefresh.trim()) {
+        throw new Error(`config.preOperations.dateRangeStartFullRefresh must be a non-empty string (SQL date expression). Received: ${JSON.stringify(config.preOperations.dateRangeStartFullRefresh)}`);
+    }
+    if (config.preOperations.dateRangeEnd === undefined || config.preOperations.dateRangeEnd === null) {
+        throw new Error("config.preOperations.dateRangeEnd is required.");
+    }
+    if (typeof config.preOperations.dateRangeEnd !== 'string' || !config.preOperations.dateRangeEnd.trim()) {
+        throw new Error(`config.preOperations.dateRangeEnd must be a non-empty string (SQL date expression). Received: ${JSON.stringify(config.preOperations.dateRangeEnd)}`);
+    }
+    if (config.preOperations.incrementalStartOverride !== undefined && config.preOperations.incrementalStartOverride !== null && config.preOperations.incrementalStartOverride !== '') {
+        if (typeof config.preOperations.incrementalStartOverride !== 'string' || !config.preOperations.incrementalStartOverride.trim()) {
+            throw new Error(`config.preOperations.incrementalStartOverride must be a non-empty string when provided. Received: ${JSON.stringify(config.preOperations.incrementalStartOverride)}`);
+        }
+    }
+    if (config.preOperations.incrementalEndOverride !== undefined && config.preOperations.incrementalEndOverride !== null && config.preOperations.incrementalEndOverride !== '') {
+        if (typeof config.preOperations.incrementalEndOverride !== 'string' || !config.preOperations.incrementalEndOverride.trim()) {
+            throw new Error(`config.preOperations.incrementalEndOverride must be a non-empty string when provided. Received: ${JSON.stringify(config.preOperations.incrementalEndOverride)}`);
+        }
+    }
+};
+
+/**
+ * Validates a GA4 export fixer configuration object.
+ * Validation is performed on mergedConfig (default values merged with user input).
+ * All fields are required in the merged config; optional fields are only optional for user input
+ * and receive their values from the default configuration during merge.
+ *
+ * @param {Object} config - The merged configuration object to validate.
+ * @throws {Error} If any configuration value is invalid or missing.
+ */
+const validateEnhancedEventsConfig = (config) => {
+  try {
+    if (!config || typeof config !== 'object' || Array.isArray(config)) {
+        throw new Error(`config must be a non-null object. Received: ${JSON.stringify(config)}`);
+    }
+
+    // base config fields (self, incremental, test, testConfig, preOperations)
+    validateBaseConfig(config);
+
+    /*
+    Rest of the validations are related to ga4_events_enhanced table specific fields
+    */
+
+    // sourceTable - required; string or Dataform table reference
+    if (config.sourceTable === undefined || config.sourceTable === null) {
+        throw new Error("config.sourceTable is required. Provide a Dataform table reference (using the ref() function) or a string in format '`project.dataset.table`'.");
+    }
+    if (isDataformTableReferenceObject(config.sourceTable)) {
+        // Valid Dataform reference
+    } else if (typeof config.sourceTable === 'string') {
+        if (!config.sourceTable.trim()) {
+            throw new Error("config.sourceTable must be a non-empty string. Received empty string.");
+        }
+        if (!/^`[^\.]+\.[^\.]+\.[^\.]+`$/.test(config.sourceTable.trim())) {
+            throw new Error(`config.sourceTable must be in the format '\`project.dataset.table\`' (with backticks). Received: ${JSON.stringify(config.sourceTable)}`);
+        }
+    } else {
+        throw new Error(`config.sourceTable must be a Dataform table reference object or a string in format '\`project.dataset.table\`'. Received: ${JSON.stringify(config.sourceTable)}`);
+    }
+
+    // schemaLock - optional; must be undefined or a string in "YYYYMMDD" format (e.g., "20260101")
+    if (typeof config.schemaLock !== 'undefined') {
+        if (typeof config.schemaLock !== 'string' || !/^\d{8}$/.test(config.schemaLock)) {
+            throw new Error(`config.schemaLock must be a string in "YYYYMMDD" format (e.g., "20260101"). Received: ${JSON.stringify(config.schemaLock)}`);
+        }
+        // Must be a valid date
+        const year = parseInt(config.schemaLock.slice(0, 4), 10);
+        const month = parseInt(config.schemaLock.slice(4, 6), 10);
+        const day = parseInt(config.schemaLock.slice(6, 8), 10);
+        const date = new Date(year, month - 1, day);
+        if (date.getFullYear() !== year || date.getMonth() !== month - 1 || date.getDate() !== day) {
+            throw new Error(`config.schemaLock must be a valid date. Received: ${JSON.stringify(config.schemaLock)}`);
+        }
+        // Must be at least 20241009
+        if (config.schemaLock < "20241009") {
+            throw new Error(`config.schemaLock must be a date string equal to or greater than "20241009". Received: ${JSON.stringify(config.schemaLock)}`);
+        }
+    }
+
+    // includedExportTypes - required
+    if (typeof config.includedExportTypes === 'undefined') {
+        throw new Error("config.includedExportTypes is required.");
+    }
+    if (!config.includedExportTypes || typeof config.includedExportTypes !== 'object' || Array.isArray(config.includedExportTypes)) {
+        throw new Error(`config.includedExportTypes must be an object. Received: ${JSON.stringify(config.includedExportTypes)}`);
+    }
+    for (const key of ['daily', 'intraday']) {
+        if (!(key in config.includedExportTypes)) {
+            throw new Error(`config.includedExportTypes.${key} is required.`);
+        }
+        if (typeof config.includedExportTypes[key] !== 'boolean') {
+            throw new Error(`config.includedExportTypes.${key} must be a boolean. Received: ${JSON.stringify(config.includedExportTypes[key])}`);
+        }
+    }
+    if (!config.includedExportTypes.daily && !config.includedExportTypes.intraday) {
+        throw new Error("At least one of config.includedExportTypes.daily or config.includedExportTypes.intraday must be true.");
+    }
+
+    // timezone - required
+    if (typeof config.timezone === 'undefined') {
+        throw new Error("config.timezone is required.");
+    }
+    if (typeof config.timezone !== 'string' || !config.timezone.trim()) {
+        throw new Error(`config.timezone must be a non-empty string (e.g. 'Etc/UTC', 'Europe/Helsinki'). Received: ${JSON.stringify(config.timezone)}`);
+    }
+
+    // customTimestampParam - optional; must be undefined or a non-empty string
+    if (typeof config.customTimestampParam !== 'undefined') {
+        if (typeof config.customTimestampParam !== 'string' || !config.customTimestampParam.trim()) {
+            throw new Error(`config.customTimestampParam must be a non-empty string when provided. Received: ${JSON.stringify(config.customTimestampParam)}`);
+        }
+    }
+
+    // dataIsFinal - required
+    if (typeof config.dataIsFinal === 'undefined') {
+        throw new Error("config.dataIsFinal is required.");
+    }
+    if (typeof config.dataIsFinal !== 'object' || Array.isArray(config.dataIsFinal)) {
+        throw new Error(`config.dataIsFinal must be an object. Received: ${JSON.stringify(config.dataIsFinal)}`);
+    }
+    if (typeof config.dataIsFinal.detectionMethod === 'undefined') {
+        throw new Error("config.dataIsFinal.detectionMethod is required.");
+    }
+    if (typeof config.dataIsFinal.detectionMethod !== 'string' || (config.dataIsFinal.detectionMethod !== 'EXPORT_TYPE' && config.dataIsFinal.detectionMethod !== 'DAY_THRESHOLD')) {
+        throw new Error(`config.dataIsFinal.detectionMethod must be 'EXPORT_TYPE' or 'DAY_THRESHOLD'. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
+    }
+    if (
+        config.dataIsFinal.detectionMethod === 'DAY_THRESHOLD' && 
+        typeof config.dataIsFinal.dayThreshold === 'undefined'
+    ) {
+        throw new Error("config.dataIsFinal.dayThreshold is required when detectionMethod is 'DAY_THRESHOLD'.");
+    }
+    if (
+        config.dataIsFinal.detectionMethod === 'DAY_THRESHOLD' && 
+        (typeof config.dataIsFinal.dayThreshold !== 'number' || !Number.isInteger(config.dataIsFinal.dayThreshold) || config.dataIsFinal.dayThreshold < 0)
+    ) {
+        throw new Error(`config.dataIsFinal.dayThreshold must be a non-negative integer. Received: ${JSON.stringify(config.dataIsFinal.dayThreshold)}`);
+    }
+    // EXPORT_TYPE detection relies on daily export metadata; intraday-only requires DAY_THRESHOLD instead.
+    if (
+        config.includedExportTypes.intraday &&
+        !config.includedExportTypes.daily &&
+        config.dataIsFinal.detectionMethod !== 'DAY_THRESHOLD'
+    ) {
+        throw new Error(`config.dataIsFinal.detectionMethod must be 'DAY_THRESHOLD' when only intraday export is enabled (config.includedExportTypes.daily is false). A dayThreshold of 1 is recommended for intraday-only configurations. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
+    }
+
+    // bufferDays - required
+    if (typeof config.bufferDays !== 'number' || !Number.isInteger(config.bufferDays) || config.bufferDays < 0) {
+        throw new Error(`config.bufferDays must be a non-negative integer. Received: ${JSON.stringify(config.bufferDays)}`);
+    }
+
+    // Array fields - all required
+    const stringArrayKeys = ['defaultExcludedEventParams', 'excludedEventParams', 'sessionParams', 'defaultExcludedEvents', 'excludedEvents', 'excludedColumns'];
+    for (const key of stringArrayKeys) {
+        if (config[key] === undefined) {
+            throw new Error(`config.${key} is required.`);
+        }
+        if (!Array.isArray(config[key])) {
+            throw new Error(`config.${key} must be an array. Received: ${JSON.stringify(config[key])}`);
+        }
+        for (let i = 0; i < config[key].length; i++) {
+            if (typeof config[key][i] !== 'string' || !config[key][i].trim()) {
+                throw new Error(`config.${key}[${i}] must be a non-empty string. Received: ${JSON.stringify(config[key][i])}`);
+            }
+        }
+    }
+
+    // eventParamsToColumns - required
+    if (config.eventParamsToColumns === undefined) {
+        throw new Error("config.eventParamsToColumns is required.");
+    }
+    if (!Array.isArray(config.eventParamsToColumns)) {
+        throw new Error(`config.eventParamsToColumns must be an array. Received: ${JSON.stringify(config.eventParamsToColumns)}`);
+    }
+    const validEventParamTypes = ['string', 'int', 'int64', 'double', 'float', 'float64'];
+    for (let i = 0; i < config.eventParamsToColumns.length; i++) {
+        const item = config.eventParamsToColumns[i];
+        if (!item || typeof item !== 'object' || Array.isArray(item)) {
+            throw new Error(`config.eventParamsToColumns[${i}] must be an object with 'name' and 'type' properties. Received: ${JSON.stringify(item)}`);
+        }
+        if (!item.name || typeof item.name !== 'string' || !item.name.trim()) {
+            throw new Error(`config.eventParamsToColumns[${i}].name must be a non-empty string. Received: ${JSON.stringify(item.name)}`);
+        }
+        if (item.type !== undefined && item.type !== null) {
+            if (!validEventParamTypes.includes(item.type)) {
+                throw new Error(`config.eventParamsToColumns[${i}].type must be one of: ${validEventParamTypes.join(', ')}. Received: ${JSON.stringify(item.type)}`);
+            }
+        }
+        if (item.columnName !== undefined && item.columnName !== null && item.columnName !== '') {
+            if (typeof item.columnName !== 'string' || !item.columnName.trim()) {
+                throw new Error(`config.eventParamsToColumns[${i}].columnName must be a non-empty string when provided. Received: ${JSON.stringify(item.columnName)}`);
+            }
+        }
+    }
+  } catch (e) {
+    e.message = `Config validation: ${e.message}`;
+    throw e;
+  }
+};
+
+module.exports = {
+    validateBaseConfig,
+    validateEnhancedEventsConfig
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.1.6-dev.0",
+  "version": "0.1.6-dev.10",
   "description": "",
   "main": "index.js",
   "files": [
@@ -9,7 +9,10 @@
     "utils.js",
     "preOperations.js",
     "constants.js",
-    "tables"
+    "tables",
+    "inputValidation.js",
+    "defaultConfig.js",
+    "config.js"
   ],
   "scripts": {
     "test": "node tests/ga4EventsEnhanced.test.js",

package/preOperations.js

@@ -85,7 +85,7 @@ const getDateRangeStartIntraday = (config) => {
       date
   )
   select
-    max(
+    min(
       if(
         intraday = true and daily = false,
         date,

package/tables/ga4EventsEnhanced.js

@@ -1,72 +1,13 @@
 const helpers = require('../helpers.js');
 const utils = require('../utils.js');
+const inputValidation = require('../inputValidation.js');
 const constants = require('../constants.js');
 const preOperations = require('../preOperations.js');
+const { ga4EventsEnhancedConfig } = require('../defaultConfig.js'); // config defaults
 
 // default configuration for the GA4 Events Enhanced table
 const defaultConfig = {
-    // required
-    sourceTable: undefined,
-    sourceTableType: 'GA4_EXPORT', // used with pre operations to detect if ga4 export specific pre operations are needed
-    self: undefined,
-    incremental: undefined,
-    // optional but recommended
-    schemaLock: undefined,
-    // only used with js tables
-    // dataformTableConfig: {},
-    // optional
-    includedExportTypes: {
-        daily: true,
-        intraday: true,
-        fresh: false,
-    },
-    timezone: 'Etc/UTC',
-    customTimestampParam: undefined,
-    dataIsFinal: {
-        detectionMethod: 'EXPORT_TYPE', // or 'DAY_THRESHOLD'
-        dayThreshold: 4 // only used if detectionMethod is 'DAY_THRESHOLD'
-    },
-    test: false,
-    testConfig: {
-        dateRangeStart: 'current_date()-1',
-        dateRangeEnd: 'current_date()',
-    },
-    // number of additional days to take in for taking into account sessions that overlap days
-    bufferDays: 1,
-    preOperations: {
-        dateRangeStartFullRefresh: 'date(2000, 1, 1)',
-        dateRangeEnd: 'current_date()',
-        // incrementalStartOverride and incrementalEndOverride are used to override the date range start and end for incremental refresh
-        // this is useful if you want to re-process only a specific date range
-        incrementalStartOverride: undefined,
-        incrementalEndOverride: undefined,
-        numberOfPreviousDaysToScan: 10,
-    },
-    // these parameters are excluded by default because they've been made available in other columns
-    defaultExcludedEventParams: [
-        'page_location',
-        'ga_session_id',
-        //'custom_event_timestamp', // removed if customTimestampParam is used
-    ],
-    excludedEventParams: [],
-    eventParamsToColumns: [
-        //{name: 'page_location', type: 'string', columnName: 'page_location2'},
-    ],
-    sessionParams: [],
-    defaultExcludedEvents: [],
-    // session_start and first_visit are excluded via the excludedEvents array
-    // this allows the user to include them if needed
-    excludedEvents: [
-        'session_start',
-        'first_visit'
-    ],
-    defaultExcludedColumns: [
-        'event_dimensions', // legacy column, not needed
-        'traffic_source', // renamed to user_traffic_source
-        'session_id'
-    ],
-    // exclude these columns when extracting raw data from the export tables
-    excludedColumns: [],
+    ...ga4EventsEnhancedConfig,
 };
 
 // List the columns in the order they should be in the final table
@@ -207,7 +148,7 @@ const generateEnhancedEventsSQL = (config) => {
     const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
 
     // validate the config and throw an error if it's invalid
-    utils.validateConfig(mergedConfig);
+    inputValidation.validateEnhancedEventsConfig(mergedConfig);
 
     if (!mergedConfig.sourceTable || typeof mergedConfig.sourceTable !== 'string' || mergedConfig.sourceTable.trim() === '') {
         throw new Error("generateEnhancedEventsSQL: 'sourceTable' is a required parameter in config and must be a non-empty string.");
@@ -341,7 +282,7 @@ ${excludedEventsSQL}`,
                 condition: 'using(session_id)'
             }
         ],
-        where: helpers.finalDataFilter(mergedConfig)
+        where: helpers.incrementalDateFilter(mergedConfig)
     };
 
     const steps = [

package/utils.js

@@ -404,247 +404,6 @@ const processDate = (dateInput) => {
     throw new Error(`processDate: Unsupported date input format: ${JSON.stringify(dateInput)}. Expected formats are: YYYYMMDD, YYYY-MM-DD, or BigQuery SQL statement.`);
 };
 
-/**
- * Validates a GA4 export fixer configuration object.
- * Validation is performed on mergedConfig (default values merged with user input).
- * All fields are required in the merged config; optional fields are only optional for user input
- * and receive their values from the default configuration during merge.
- *
- * @param {Object} config - The merged configuration object to validate.
- * @throws {Error} If any configuration value is invalid or missing.
- */
-const validateConfig = (config) => {
-  try {
-    if (!config || typeof config !== 'object' || Array.isArray(config)) {
-        throw new Error(`config must be a non-null object. Received: ${JSON.stringify(config)}`);
-    }
-
-    // sourceTable - required; string or Dataform table reference
-    if (config.sourceTable === undefined || config.sourceTable === null) {
-        throw new Error("config.sourceTable is required. Provide a Dataform table reference (using the ref() function) or a string in format '`project.dataset.table`'.");
-    }
-    if (isDataformTableReferenceObject(config.sourceTable)) {
-        // Valid Dataform reference
-    } else if (typeof config.sourceTable === 'string') {
-        if (!config.sourceTable.trim()) {
-            throw new Error("config.sourceTable must be a non-empty string. Received empty string.");
-        }
-        if (!/^`[^\.]+\.[^\.]+\.[^\.]+`$/.test(config.sourceTable.trim())) {
-            throw new Error(`config.sourceTable must be in the format '\`project.dataset.table\`' (with backticks). Received: ${JSON.stringify(config.sourceTable)}`);
-        }
-    } else {
-        throw new Error(`config.sourceTable must be a Dataform table reference object or a string in format '\`project.dataset.table\`'. Received: ${JSON.stringify(config.sourceTable)}`);
-    }
-
-    // self - required when using Dataform; must be valid format
-    // config.self is required when config.test === true and must be a non-empty string in format '\`project.dataset.table\`' (using the ref() function)
-    if (config.test !== true) {
-        if (typeof config.self !== 'string' || !config.self.trim() || !/^`[^`]+`$/.test(config.self.trim())) {
-            throw new Error(`config.self is required when config.test === true and must be a non-empty string in format '\`project.dataset.table\`' (using the ref() function). Received: ${JSON.stringify(config.self)}`);
-        }
-    }
-
-    // incremental - required when using Dataform; must be boolean
-    if (typeof config.incremental !== 'boolean') {
-        throw new Error(`config.incremental must be a boolean. Received: ${JSON.stringify(config.incremental)}`);
-    }
-
-    // schemaLock - optional; must be undefined or a string in "YYYYMMDD" format (e.g., "20260101")
-    if (typeof config.schemaLock !== 'undefined') {
-        if (typeof config.schemaLock !== 'string' || !/^\d{8}$/.test(config.schemaLock)) {
-            throw new Error(`config.schemaLock must be a string in "YYYYMMDD" format (e.g., "20260101"). Received: ${JSON.stringify(config.schemaLock)}`);
-        }
-        // Must be a valid date
-        const year = parseInt(config.schemaLock.slice(0, 4), 10);
-        const month = parseInt(config.schemaLock.slice(4, 6), 10);
-        const day = parseInt(config.schemaLock.slice(6, 8), 10);
-        const date = new Date(year, month - 1, day);
-        if (date.getFullYear() !== year || date.getMonth() !== month - 1 || date.getDate() !== day) {
-            throw new Error(`config.schemaLock must be a valid date. Received: ${JSON.stringify(config.schemaLock)}`);
-        }
-        // Must be at least 20241009
-        if (config.schemaLock < "20241009") {
-            throw new Error(`config.schemaLock must be a date string equal to or greater than "20241009". Received: ${JSON.stringify(config.schemaLock)}`);
-        }
-    }
-
-    // includedExportTypes - required
-    if (typeof config.includedExportTypes === 'undefined') {
-        throw new Error("config.includedExportTypes is required.");
-    }
-    if (!config.includedExportTypes || typeof config.includedExportTypes !== 'object' || Array.isArray(config.includedExportTypes)) {
-        throw new Error(`config.includedExportTypes must be an object. Received: ${JSON.stringify(config.includedExportTypes)}`);
-    }
-    for (const key of ['daily', 'intraday']) {
-        if (!(key in config.includedExportTypes)) {
-            throw new Error(`config.includedExportTypes.${key} is required.`);
-        }
-        if (typeof config.includedExportTypes[key] !== 'boolean') {
-            throw new Error(`config.includedExportTypes.${key} must be a boolean. Received: ${JSON.stringify(config.includedExportTypes[key])}`);
-        }
-    }
-    if (!config.includedExportTypes.daily && !config.includedExportTypes.intraday) {
-        throw new Error("At least one of config.includedExportTypes.daily or config.includedExportTypes.intraday must be true.");
-    }
-
-    // timezone - required
-    if (typeof config.timezone === 'undefined') {
-        throw new Error("config.timezone is required.");
-    }
-    if (typeof config.timezone !== 'string' || !config.timezone.trim()) {
-        throw new Error(`config.timezone must be a non-empty string (e.g. 'Etc/UTC', 'Europe/Helsinki'). Received: ${JSON.stringify(config.timezone)}`);
-    }
-
-    // customTimestampParam - optional; must be undefined or a non-empty string
-    if (typeof config.customTimestampParam !== 'undefined') {
-        if (typeof config.customTimestampParam !== 'string' || !config.customTimestampParam.trim()) {
-            throw new Error(`config.customTimestampParam must be a non-empty string when provided. Received: ${JSON.stringify(config.customTimestampParam)}`);
-        }
-    }
-
-    // dataIsFinal - required
-    if (typeof config.dataIsFinal === 'undefined') {
-        throw new Error("config.dataIsFinal is required.");
-    }
-    if (typeof config.dataIsFinal !== 'object' || Array.isArray(config.dataIsFinal)) {
-        throw new Error(`config.dataIsFinal must be an object. Received: ${JSON.stringify(config.dataIsFinal)}`);
-    }
-    if (typeof config.dataIsFinal.detectionMethod === 'undefined') {
-        throw new Error("config.dataIsFinal.detectionMethod is required.");
-    }
-    if (typeof config.dataIsFinal.detectionMethod !== 'string' || (config.dataIsFinal.detectionMethod !== 'EXPORT_TYPE' && config.dataIsFinal.detectionMethod !== 'DAY_THRESHOLD')) {
-        throw new Error(`config.dataIsFinal.detectionMethod must be 'EXPORT_TYPE' or 'DAY_THRESHOLD'. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
-    }
-    if (
-        config.dataIsFinal.detectionMethod === 'DAY_THRESHOLD' && 
-        typeof config.dataIsFinal.dayThreshold === 'undefined'
-    ) {
-        throw new Error("config.dataIsFinal.dayThreshold is required when detectionMethod is 'DAY_THRESHOLD'.");
-    }
-    if (
-        config.dataIsFinal.detectionMethod === 'DAY_THRESHOLD' && 
-        (typeof config.dataIsFinal.dayThreshold !== 'number' || !Number.isInteger(config.dataIsFinal.dayThreshold) || config.dataIsFinal.dayThreshold < 0)
-    ) {
-        throw new Error(`config.dataIsFinal.dayThreshold must be a non-negative integer. Received: ${JSON.stringify(config.dataIsFinal.dayThreshold)}`);
-    }
-    // EXPORT_TYPE detection relies on daily export metadata; intraday-only requires DAY_THRESHOLD instead.
-    if (
-        config.includedExportTypes.intraday &&
-        !config.includedExportTypes.daily &&
-        config.dataIsFinal.detectionMethod !== 'DAY_THRESHOLD'
-    ) {
-        throw new Error(`config.dataIsFinal.detectionMethod must be 'DAY_THRESHOLD' when only intraday export is enabled (config.includedExportTypes.daily is false). A dayThreshold of 1 is recommended for intraday-only configurations. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
-    }
-
-    // test - optional; when defined, must be a boolean
-    if (typeof config.test !== 'undefined' && typeof config.test !== 'boolean') {
-        throw new Error(`config.test must be a boolean when defined. Received: ${JSON.stringify(config.test)}`);
-    }
-
-    // testConfig - optional; when included, must be an object with optional dateRangeStart and dateRangeEnd
-    if (typeof config.testConfig !== 'undefined') {
-        if (!config.testConfig || typeof config.testConfig !== 'object' || Array.isArray(config.testConfig)) {
-            throw new Error(`config.testConfig must be an object when included. Received: ${JSON.stringify(config.testConfig)}`);
-        }
-        if (config.testConfig.dateRangeStart !== undefined && (typeof config.testConfig.dateRangeStart !== 'string' || !config.testConfig.dateRangeStart.trim())) {
-            throw new Error(`config.testConfig.dateRangeStart must be a non-empty string (SQL date expression) when provided. Received: ${JSON.stringify(config.testConfig.dateRangeStart)}`);
-        }
-        if (config.testConfig.dateRangeEnd !== undefined && (typeof config.testConfig.dateRangeEnd !== 'string' || !config.testConfig.dateRangeEnd.trim())) {
-            throw new Error(`config.testConfig.dateRangeEnd must be a non-empty string (SQL date expression) when provided. Received: ${JSON.stringify(config.testConfig.dateRangeEnd)}`);
-        }
-    }
-
-    // bufferDays - required
-    if (typeof config.bufferDays !== 'number' || !Number.isInteger(config.bufferDays) || config.bufferDays < 0) {
-        throw new Error(`config.bufferDays must be a non-negative integer. Received: ${JSON.stringify(config.bufferDays)}`);
-    }
-
-    // preOperations - required
-    if (config.preOperations === undefined) {
-        throw new Error("config.preOperations is required.");
-    }
-    if (!config.preOperations || typeof config.preOperations !== 'object' || Array.isArray(config.preOperations)) {
-        throw new Error(`config.preOperations must be an object. Received: ${JSON.stringify(config.preOperations)}`);
-    }
-    if (config.preOperations.numberOfPreviousDaysToScan === undefined) {
-        throw new Error("config.preOperations.numberOfPreviousDaysToScan is required.");
-    }
-    const v = config.preOperations.numberOfPreviousDaysToScan;
-    if (typeof v !== 'number' || isNaN(v) || !Number.isInteger(v) || v < 0) {
-        throw new Error(`config.preOperations.numberOfPreviousDaysToScan must be a non-negative integer. Received: ${JSON.stringify(v)}`);
-    }
-    if (config.preOperations.dateRangeStartFullRefresh === undefined || config.preOperations.dateRangeStartFullRefresh === null) {
-        throw new Error("config.preOperations.dateRangeStartFullRefresh is required.");
-    }
-    if (typeof config.preOperations.dateRangeStartFullRefresh !== 'string' || !config.preOperations.dateRangeStartFullRefresh.trim()) {
-        throw new Error(`config.preOperations.dateRangeStartFullRefresh must be a non-empty string (SQL date expression). Received: ${JSON.stringify(config.preOperations.dateRangeStartFullRefresh)}`);
-    }
-    if (config.preOperations.dateRangeEnd === undefined || config.preOperations.dateRangeEnd === null) {
-        throw new Error("config.preOperations.dateRangeEnd is required.");
-    }
-    if (typeof config.preOperations.dateRangeEnd !== 'string' || !config.preOperations.dateRangeEnd.trim()) {
-        throw new Error(`config.preOperations.dateRangeEnd must be a non-empty string (SQL date expression). Received: ${JSON.stringify(config.preOperations.dateRangeEnd)}`);
-    }
-    if (config.preOperations.incrementalStartOverride !== undefined && config.preOperations.incrementalStartOverride !== null && config.preOperations.incrementalStartOverride !== '') {
-        if (typeof config.preOperations.incrementalStartOverride !== 'string' || !config.preOperations.incrementalStartOverride.trim()) {
-            throw new Error(`config.preOperations.incrementalStartOverride must be a non-empty string when provided. Received: ${JSON.stringify(config.preOperations.incrementalStartOverride)}`);
-        }
-    }
-    if (config.preOperations.incrementalEndOverride !== undefined && config.preOperations.incrementalEndOverride !== null && config.preOperations.incrementalEndOverride !== '') {
-        if (typeof config.preOperations.incrementalEndOverride !== 'string' || !config.preOperations.incrementalEndOverride.trim()) {
-            throw new Error(`config.preOperations.incrementalEndOverride must be a non-empty string when provided. Received: ${JSON.stringify(config.preOperations.incrementalEndOverride)}`);
-        }
-    }
-
-    // Array fields - all required
-    const stringArrayKeys = ['defaultExcludedEventParams', 'excludedEventParams', 'sessionParams', 'defaultExcludedEvents', 'excludedEvents', 'excludedColumns'];
-    for (const key of stringArrayKeys) {
-        if (config[key] === undefined) {
-            throw new Error(`config.${key} is required.`);
-        }
-        if (!Array.isArray(config[key])) {
-            throw new Error(`config.${key} must be an array. Received: ${JSON.stringify(config[key])}`);
-        }
-        for (let i = 0; i < config[key].length; i++) {
-            if (typeof config[key][i] !== 'string' || !config[key][i].trim()) {
-                throw new Error(`config.${key}[${i}] must be a non-empty string. Received: ${JSON.stringify(config[key][i])}`);
-            }
-        }
-    }
-
-    // eventParamsToColumns - required
-    if (config.eventParamsToColumns === undefined) {
-        throw new Error("config.eventParamsToColumns is required.");
-    }
-    if (!Array.isArray(config.eventParamsToColumns)) {
-        throw new Error(`config.eventParamsToColumns must be an array. Received: ${JSON.stringify(config.eventParamsToColumns)}`);
-    }
-    const validEventParamTypes = ['string', 'int', 'int64', 'double', 'float', 'float64'];
-    for (let i = 0; i < config.eventParamsToColumns.length; i++) {
-        const item = config.eventParamsToColumns[i];
-        if (!item || typeof item !== 'object' || Array.isArray(item)) {
-            throw new Error(`config.eventParamsToColumns[${i}] must be an object with 'name' and 'type' properties. Received: ${JSON.stringify(item)}`);
-        }
-        if (!item.name || typeof item.name !== 'string' || !item.name.trim()) {
-            throw new Error(`config.eventParamsToColumns[${i}].name must be a non-empty string. Received: ${JSON.stringify(item.name)}`);
-        }
-        if (item.type !== undefined && item.type !== null) {
-            if (!validEventParamTypes.includes(item.type)) {
-                throw new Error(`config.eventParamsToColumns[${i}].type must be one of: ${validEventParamTypes.join(', ')}. Received: ${JSON.stringify(item.type)}`);
-            }
-        }
-        if (item.columnName !== undefined && item.columnName !== null && item.columnName !== '') {
-            if (typeof item.columnName !== 'string' || !item.columnName.trim()) {
-                throw new Error(`config.eventParamsToColumns[${i}].columnName must be a non-empty string when provided. Received: ${JSON.stringify(item.columnName)}`);
-            }
-        }
-    }
-  } catch (e) {
-    e.message = `Config validation: ${e.message}`;
-    throw e;
-  }
-};
-
 module.exports = {
     mergeUniqueArrays,
     mergeSQLConfigurations,
@@ -653,6 +412,5 @@ module.exports = {
     isDataformTableReferenceObject,
     setDataformContext,
     selectOtherColumns,
-    processDate,
-    validateConfig
+    processDate
 };