ga4-export-fixer 0.4.6 → 0.4.7-dev.0

package/inputValidation.js

@@ -1,5 +1,3 @@
-const { isDataformTableReferenceObject } = require('./utils.js');
-
 /**
  * Validates the base configuration fields shared across all table types.
  * These correspond to the fields defined in baseConfig (defaultConfig.js):
@@ -92,188 +90,6 @@ const validateBaseConfig = (config, options = {}) => {
     }
 };
 
-/**
- * Validates a GA4 export fixer configuration object.
- * Validation is performed on mergedConfig (default values merged with user input).
- * All fields are required in the merged config; optional fields are only optional for user input
- * and receive their values from the default configuration during merge.
- *
- * @param {Object} config - The merged configuration object to validate.
- * @throws {Error} If any configuration value is invalid or missing.
- */
-const validateEnhancedEventsConfig = (config, options = {}) => {
-  try {
-    if (!config || typeof config !== 'object' || Array.isArray(config)) {
-        throw new Error(`config must be a non-null object. Received: ${JSON.stringify(config)}`);
-    }
-
-    // base config fields (self, incremental, test, testConfig, preOperations)
-    validateBaseConfig(config, options);
-
-    /*
-    Rest of the validations are related to ga4_events_enhanced table specific fields
-    */
-
-    // sourceTable - required; string or Dataform table reference
-    if (config.sourceTable === undefined || config.sourceTable === null) {
-        throw new Error("config.sourceTable is required. Provide a Dataform table reference (using the ref() function) or a string in format '`project.dataset.table`'.");
-    }
-    if (isDataformTableReferenceObject(config.sourceTable)) {
-        // Valid Dataform reference
-    } else if (typeof config.sourceTable === 'string') {
-        if (!config.sourceTable.trim()) {
-            throw new Error("config.sourceTable must be a non-empty string. Received empty string.");
-        }
-        if (!/^`[^\.]+\.[^\.]+\.[^\.]+`$/.test(config.sourceTable.trim())) {
-            throw new Error(`config.sourceTable must be in the format '\`project.dataset.table\`' (with backticks). Received: ${JSON.stringify(config.sourceTable)}`);
-        }
-    } else {
-        throw new Error(`config.sourceTable must be a Dataform table reference object or a string in format '\`project.dataset.table\`'. Received: ${JSON.stringify(config.sourceTable)}`);
-    }
-
-    // schemaLock - optional; must be undefined or a string in "YYYYMMDD" format (e.g., "20260101")
-    if (typeof config.schemaLock !== 'undefined') {
-        if (typeof config.schemaLock !== 'string' || !/^\d{8}$/.test(config.schemaLock)) {
-            throw new Error(`config.schemaLock must be a string in "YYYYMMDD" format (e.g., "20260101"). Received: ${JSON.stringify(config.schemaLock)}`);
-        }
-        // Must be a valid date
-        const year = parseInt(config.schemaLock.slice(0, 4), 10);
-        const month = parseInt(config.schemaLock.slice(4, 6), 10);
-        const day = parseInt(config.schemaLock.slice(6, 8), 10);
-        const date = new Date(year, month - 1, day);
-        if (date.getFullYear() !== year || date.getMonth() !== month - 1 || date.getDate() !== day) {
-            throw new Error(`config.schemaLock must be a valid date. Received: ${JSON.stringify(config.schemaLock)}`);
-        }
-        // Must be at least 20241009
-        if (config.schemaLock < "20241009") {
-            throw new Error(`config.schemaLock must be a date string equal to or greater than "20241009". Received: ${JSON.stringify(config.schemaLock)}`);
-        }
-    }
-
-    // includedExportTypes - required
-    if (typeof config.includedExportTypes === 'undefined') {
-        throw new Error("config.includedExportTypes is required.");
-    }
-    if (!config.includedExportTypes || typeof config.includedExportTypes !== 'object' || Array.isArray(config.includedExportTypes)) {
-        throw new Error(`config.includedExportTypes must be an object. Received: ${JSON.stringify(config.includedExportTypes)}`);
-    }
-    for (const key of ['daily', 'fresh', 'intraday']) {
-        if (!(key in config.includedExportTypes)) {
-            throw new Error(`config.includedExportTypes.${key} is required.`);
-        }
-        if (typeof config.includedExportTypes[key] !== 'boolean') {
-            throw new Error(`config.includedExportTypes.${key} must be a boolean. Received: ${JSON.stringify(config.includedExportTypes[key])}`);
-        }
-    }
-    if (!config.includedExportTypes.daily && !config.includedExportTypes.fresh && !config.includedExportTypes.intraday) {
-        throw new Error("At least one of config.includedExportTypes.daily, config.includedExportTypes.fresh, or config.includedExportTypes.intraday must be true.");
-    }
-
-    // timezone - required
-    if (typeof config.timezone === 'undefined') {
-        throw new Error("config.timezone is required.");
-    }
-    if (typeof config.timezone !== 'string' || !config.timezone.trim()) {
-        throw new Error(`config.timezone must be a non-empty string (e.g. 'Etc/UTC', 'Europe/Helsinki'). Received: ${JSON.stringify(config.timezone)}`);
-    }
-
-    // customTimestampParam - optional; must be undefined or a non-empty string
-    if (typeof config.customTimestampParam !== 'undefined') {
-        if (typeof config.customTimestampParam !== 'string' || !config.customTimestampParam.trim()) {
-            throw new Error(`config.customTimestampParam must be a non-empty string when provided. Received: ${JSON.stringify(config.customTimestampParam)}`);
-        }
-    }
-
-    // dataIsFinal - required
-    if (typeof config.dataIsFinal === 'undefined') {
-        throw new Error("config.dataIsFinal is required.");
-    }
-    if (typeof config.dataIsFinal !== 'object' || Array.isArray(config.dataIsFinal)) {
-        throw new Error(`config.dataIsFinal must be an object. Received: ${JSON.stringify(config.dataIsFinal)}`);
-    }
-    if (typeof config.dataIsFinal.detectionMethod === 'undefined') {
-        throw new Error("config.dataIsFinal.detectionMethod is required.");
-    }
-    if (typeof config.dataIsFinal.detectionMethod !== 'string' || (config.dataIsFinal.detectionMethod !== 'EXPORT_TYPE' && config.dataIsFinal.detectionMethod !== 'DAY_THRESHOLD')) {
-        throw new Error(`config.dataIsFinal.detectionMethod must be 'EXPORT_TYPE' or 'DAY_THRESHOLD'. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
-    }
-    if (
-        config.dataIsFinal.detectionMethod === 'DAY_THRESHOLD' && 
-        typeof config.dataIsFinal.dayThreshold === 'undefined'
-    ) {
-        throw new Error("config.dataIsFinal.dayThreshold is required when detectionMethod is 'DAY_THRESHOLD'.");
-    }
-    if (
-        config.dataIsFinal.detectionMethod === 'DAY_THRESHOLD' && 
-        (typeof config.dataIsFinal.dayThreshold !== 'number' || !Number.isInteger(config.dataIsFinal.dayThreshold) || config.dataIsFinal.dayThreshold < 0)
-    ) {
-        throw new Error(`config.dataIsFinal.dayThreshold must be a non-negative integer. Received: ${JSON.stringify(config.dataIsFinal.dayThreshold)}`);
-    }
-    // EXPORT_TYPE detection relies on daily export tables to mark data as final.
-    // When daily is not enabled, all data would be marked as not final under EXPORT_TYPE,
-    // so DAY_THRESHOLD must be used instead.
-    if (
-        !config.includedExportTypes.daily &&
-        config.dataIsFinal.detectionMethod !== 'DAY_THRESHOLD'
-    ) {
-        throw new Error(`config.dataIsFinal.detectionMethod must be 'DAY_THRESHOLD' when daily export is not enabled (config.includedExportTypes.daily is false). A dayThreshold of 1 is recommended for intraday only setups. With fresh export, the GA4 data is subject to possible changes for up to 72 hours. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
-    }
-
-    // bufferDays - required
-    if (typeof config.bufferDays !== 'number' || !Number.isInteger(config.bufferDays) || config.bufferDays < 0) {
-        throw new Error(`config.bufferDays must be a non-negative integer. Received: ${JSON.stringify(config.bufferDays)}`);
-    }
-
-    // Array fields - all required
-    const stringArrayKeys = ['defaultExcludedEventParams', 'excludedEventParams', 'sessionParams', 'defaultExcludedEvents', 'excludedEvents', 'excludedColumns'];
-    for (const key of stringArrayKeys) {
-        if (config[key] === undefined) {
-            throw new Error(`config.${key} is required.`);
-        }
-        if (!Array.isArray(config[key])) {
-            throw new Error(`config.${key} must be an array. Received: ${JSON.stringify(config[key])}`);
-        }
-        for (let i = 0; i < config[key].length; i++) {
-            if (typeof config[key][i] !== 'string' || !config[key][i].trim()) {
-                throw new Error(`config.${key}[${i}] must be a non-empty string. Received: ${JSON.stringify(config[key][i])}`);
-            }
-        }
-    }
-
-    // eventParamsToColumns - required
-    if (config.eventParamsToColumns === undefined) {
-        throw new Error("config.eventParamsToColumns is required.");
-    }
-    if (!Array.isArray(config.eventParamsToColumns)) {
-        throw new Error(`config.eventParamsToColumns must be an array. Received: ${JSON.stringify(config.eventParamsToColumns)}`);
-    }
-    const validEventParamTypes = ['string', 'int', 'int64', 'double', 'float', 'float64'];
-    for (let i = 0; i < config.eventParamsToColumns.length; i++) {
-        const item = config.eventParamsToColumns[i];
-        if (!item || typeof item !== 'object' || Array.isArray(item)) {
-            throw new Error(`config.eventParamsToColumns[${i}] must be an object with 'name' and 'type' properties. Received: ${JSON.stringify(item)}`);
-        }
-        if (!item.name || typeof item.name !== 'string' || !item.name.trim()) {
-            throw new Error(`config.eventParamsToColumns[${i}].name must be a non-empty string. Received: ${JSON.stringify(item.name)}`);
-        }
-        if (item.type !== undefined && item.type !== null) {
-            if (!validEventParamTypes.includes(item.type)) {
-                throw new Error(`config.eventParamsToColumns[${i}].type must be one of: ${validEventParamTypes.join(', ')}. Received: ${JSON.stringify(item.type)}`);
-            }
-        }
-        if (item.columnName !== undefined && item.columnName !== null && item.columnName !== '') {
-            if (typeof item.columnName !== 'string' || !item.columnName.trim()) {
-                throw new Error(`config.eventParamsToColumns[${i}].columnName must be a non-empty string when provided. Received: ${JSON.stringify(item.columnName)}`);
-            }
-        }
-    }
-  } catch (e) {
-    e.message = `Config validation: ${e.message}`;
-    throw e;
-  }
-};
-
 module.exports = {
     validateBaseConfig,
-    validateEnhancedEventsConfig
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.4.6",
+  "version": "0.4.7-dev.0",
   "description": "",
   "main": "index.js",
   "files": [
@@ -13,8 +13,8 @@
     "inputValidation.js",
     "defaultConfig.js",
     "config.js",
-    "columns",
-    "documentation.js"
+    "documentation.js",
+    "createTable.js"
   ],
   "scripts": {
     "test": "node tests/ga4EventsEnhanced.test.js && node tests/mergeSQLConfigurations.test.js && node tests/preOperations.test.js && node tests/documentation.test.js",

package/tables/ga4EventsEnhanced/config.js

@@ -0,0 +1,68 @@
+const { baseConfig } = require('../../defaultConfig.js');
+
+/*
+The default configuration for the GA4 Events Enhanced table.
+*/
+const ga4EventsEnhancedConfig = {
+    ...baseConfig,
+    sourceTable: undefined,
+    sourceTableType: 'GA4_EXPORT', // used with pre operations to detect if ga4 export specific pre operations are needed
+    // optional but recommended
+    schemaLock: undefined,
+    // only used with js tables
+    dataformTableConfig: {
+        type: 'incremental',
+        bigquery: {
+            partitionBy: 'event_date',
+            clusterBy: ['event_name', 'session_id', 'page_location', 'data_is_final'],
+            labels: {
+                'ga4_export_fixer': 'true'
+            }
+        },
+        onSchemaChange: 'EXTEND',
+        tags: ['ga4_export_fixer'],
+    },
+    // optional
+    includedExportTypes: {
+        daily: true,
+        fresh: false,
+        intraday: true,
+    },
+    timezone: 'Etc/UTC',
+    customTimestampParam: undefined,
+    dataIsFinal: {
+        detectionMethod: 'DAY_THRESHOLD', // 'EXPORT_TYPE' or 'DAY_THRESHOLD'
+        dayThreshold: 3 // only used if detectionMethod is 'DAY_THRESHOLD'
+        // according to GA4 documentation, the data up to 72 hours old is subject to possible changes
+        // in reality, there have been cases where the data has changed even after 72 hours (4 day window would have covered these)
+    },
+    // number of additional days to take in for taking into account sessions that overlap days
+    bufferDays: 1,
+    // these parameters are excluded by default because they've been made available in other columns
+    defaultExcludedEventParams: [
+        'page_location',
+        'ga_session_id',
+        //'custom_event_timestamp', // removed if customTimestampParam is used
+    ],
+    excludedEventParams: [],
+    eventParamsToColumns: [
+        //{name: 'page_location', type: 'string', columnName: 'page_location2'},
+    ],
+    sessionParams: [],
+    defaultExcludedEvents: [],
+    // session_start and first_visit are excluded via the excludedEvents array
+    // this allows the user to include them if needed
+    excludedEvents: [
+        'session_start',
+        'first_visit'
+    ],
+    defaultExcludedColumns: [
+        'event_dimensions', // legacy column, not needed
+        'traffic_source', // renamed to user_traffic_source
+        'session_id'
+    ],
+    // exclude these columns when extracting raw data from the export tables
+    excludedColumns: [],
+};
+
+module.exports = { ga4EventsEnhancedConfig };

package/tables/{ga4EventsEnhanced.js → ga4EventsEnhanced/index.js}

@@ -1,10 +1,18 @@
-const helpers = require('../helpers/index.js');
-const utils = require('../utils.js');
-const inputValidation = require('../inputValidation.js');
-const constants = require('../constants.js');
-const preOperations = require('../preOperations.js');
-const { ga4EventsEnhancedConfig } = require('../defaultConfig.js'); // config defaults
-const documentation = require('../documentation.js');
+const helpers = require('../../helpers/index.js');
+const utils = require('../../utils.js');
+const constants = require('../../constants.js');
+const { ga4EventsEnhancedConfig } = require('./config.js');
+const { validateEnhancedEventsConfig } = require('./validation.js');
+const documentation = require('../../documentation.js');
+const { createTable } = require('../../createTable.js');
+const { getTableDescriptionSections } = require('./tableDescription.js');
+
+// Column metadata for the GA4 Events Enhanced table
+const columnMetadata = {
+    descriptions: require('./columns/columnDescriptions.json'),
+    lineage: require('./columns/columnLineage.json'),
+    typicalUse: require('./columns/columnTypicalUse.json'),
+};
 
 // default configuration for the GA4 Events Enhanced table
 const defaultConfig = {
@@ -291,7 +299,7 @@ ${excludedEventsSQL}`,
 // Exported wrapper: merge config, validate, then delegate to the internal function
 const generateEnhancedEventsSQL = (config) => {
     const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
-    inputValidation.validateEnhancedEventsConfig(mergedConfig);
+    validateEnhancedEventsConfig(mergedConfig);
     return _generateEnhancedEventsSQL(mergedConfig);
 };
 
@@ -316,58 +324,24 @@ const generateEnhancedEventsSQL = (config) => {
  *
  * @returns {Object} The Dataform publish() object for the enhanced events table, supporting chaining (e.g. .preOps, .query).
  */
-const createEnhancedEventsTable = (dataformPublish, config) => {
-    const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
-    inputValidation.validateEnhancedEventsConfig(mergedConfig, { skipDataformContextFields: true });
-
-    // Compute dynamic fields from merged SQL config
-    const getDatasetName = (sourceTable) => {
-        if (utils.isDataformTableReferenceObject(sourceTable)) {
-            return sourceTable.dataset || sourceTable.schema;
-        }
-        if (typeof sourceTable === 'string' && /^`[^\.]+\.[^\.]+\.[^\.]+`$/.test(sourceTable)) {
-            return sourceTable.split('.')[1];
-        }
-        throw new Error(`Unable to extract the dataset name from sourceTable, received: ${JSON.stringify(sourceTable)}`);
-    };
-
-    const dataset = getDatasetName(mergedConfig.sourceTable);
-
-    // Build dataformTableConfig: static defaults (from defaultConfig.js) → dynamic fields → user overrides.
-    // Deep-clone defaults to prevent Dataform's publish() from mutating nested objects (e.g. bigquery)
-    // across multiple createTable calls in the same process.
-    const dataformTableConfig = utils.mergeDataformTableConfigurations(
-        {
-            ...JSON.parse(JSON.stringify(defaultConfig.dataformTableConfig || {})),
-            name: `${constants.DEFAULT_EVENTS_TABLE_NAME}_${dataset.replace('analytics_', '')}`,
-            schema: dataset,
-            columns: documentation.getColumnDescriptions(mergedConfig),
-        },
-        config.dataformTableConfig
-    );
-
-    // Pass dataformTableConfig to getTableDescription via a new object to avoid mutating mergedConfig
-    // (Dataform's sandboxed runtime may freeze objects returned by mergeSQLConfigurations)
-    const tableDescription = documentation.getTableDescription({ ...mergedConfig, dataformTableConfig });
-
-    // Set description (user override from the merge wins if provided)
-    if (!dataformTableConfig.description) {
-        dataformTableConfig.description = tableDescription;
-    }
-
-    // create the table using Dataform publish()
-    return dataformPublish(dataformTableConfig.name, dataformTableConfig).preOps(ctx => {
-        return preOperations.setPreOperations(utils.setDataformContext(ctx, mergedConfig));
-    }).query(ctx => {
-        return _generateEnhancedEventsSQL(utils.setDataformContext(ctx, mergedConfig));
-    });
+// Table module definition — conforms to the shared createTable interface
+const tableModule = {
+    defaultConfig,
+    defaultTableName: constants.DEFAULT_EVENTS_TABLE_NAME,
+    validate: validateEnhancedEventsConfig,
+    generateSql: _generateEnhancedEventsSQL,
+    getColumnDescriptions: (config) => documentation.getColumnDescriptions(config, columnMetadata),
+    getTableDescription: (config) => documentation.buildTableDescription(config, getTableDescriptionSections(config)),
+};
 
+const createEnhancedEventsTable = (dataformPublish, config) => {
+    return createTable(dataformPublish, config, tableModule);
 };
 
 // Exported wrapper: merge config, validate, then delegate to preOperations module
 const setPreOperations = (config) => {
     const mergedConfig = utils.mergeSQLConfigurations(defaultConfig, config);
-    inputValidation.validateEnhancedEventsConfig(mergedConfig);
+    validateEnhancedEventsConfig(mergedConfig);
     return preOperations.setPreOperations(mergedConfig);
 };
 

package/tables/ga4EventsEnhanced/tableDescription.js

@@ -0,0 +1,103 @@
+const { isExcluded } = require('../../documentation.js');
+const tableAgentInstructions = require('./columns/tableAgentInstructions.json');
+
+/**
+ * Builds the GA4-specific table description sections for ga4_events_enhanced.
+ * These are passed to buildTableDescription() which adds shared sections
+ * (package attribution, config JSON dump).
+ *
+ * @param {Object} config - The merged configuration object.
+ * @returns {string[]} Array of description section strings.
+ */
+const getTableDescriptionSections = (config) => {
+    // Only use user-configured excludedColumns for filtering AI instructions.
+    // defaultExcludedColumns refers to raw GA4 export columns excluded during extraction
+    // (e.g. session_id is excluded from the raw export but exists as a derived column in the final table).
+    const excludedColumns = config.excludedColumns || [];
+
+    const excludedEvents = [
+        ...(config.defaultExcludedEvents || []),
+        ...(config.excludedEvents || []),
+    ];
+
+    const sections = [];
+
+    // 1. Overview
+    const overviewLines = [
+        'GA4 Events Enhanced',
+        '',
+        'An enhanced version of the GA4 BigQuery export. Each row is one event.',
+    ];
+    if (config.timezone) {
+        overviewLines.push(`Timezone: ${config.timezone}.`);
+    }
+    sections.push(overviewLines.join('\n'));
+
+    // 2. Key Fields
+    const keyFieldLines = tableAgentInstructions.keyFields
+        .filter(kf => !isExcluded(kf.dependsOn, excludedColumns))
+        .map(kf => `- ${kf.field}: ${kf.note}`);
+
+    // Add promoted event params
+    if (config.eventParamsToColumns && config.eventParamsToColumns.length > 0) {
+        config.eventParamsToColumns.forEach(p => {
+            const columnName = p.columnName || p.name;
+            keyFieldLines.push(`- ${columnName}: Promoted event parameter '${p.name}'. Available as a top-level column for direct filtering.`);
+        });
+    }
+
+    if (keyFieldLines.length > 0) {
+        sections.push('KEY FIELDS:\n' + keyFieldLines.join('\n'));
+    }
+
+    // 3. Synonyms
+    const synonymLines = tableAgentInstructions.synonyms
+        .filter(s => !isExcluded(s.dependsOn, excludedColumns))
+        .map(s => `- "${s.terms.join('" / "')}" → ${s.sql}`);
+
+    if (synonymLines.length > 0) {
+        sections.push('SYNONYMS:\n' + synonymLines.join('\n'));
+    }
+
+    // 4. Filtering and Grouping
+    const guidanceLines = tableAgentInstructions.filteringGuidance
+        .filter(g => !isExcluded(g.dependsOn, excludedColumns))
+        .map(g => `- ${g.text}`);
+
+    if (guidanceLines.length > 0) {
+        sections.push('FILTERING AND GROUPING:\n' + guidanceLines.join('\n'));
+    }
+
+    // 5. Event Vocabulary
+    const vocabParts = [];
+    const autoEvents = tableAgentInstructions.eventVocabulary.autoCollectedAndEnhanced
+        .filter(e => !excludedEvents.includes(e));
+    if (autoEvents.length > 0) {
+        vocabParts.push(`Auto-collected and enhanced measurement: ${autoEvents.join(', ')}`);
+    }
+
+    if (!isExcluded(['ecommerce'], excludedColumns)) {
+        const ecomEvents = tableAgentInstructions.eventVocabulary.ecommerce
+            .filter(e => !excludedEvents.includes(e));
+        if (ecomEvents.length > 0) {
+            vocabParts.push(`Ecommerce (recommended): ${ecomEvents.join(', ')}`);
+        }
+    }
+
+    if (vocabParts.length > 0) {
+        sections.push('COMMON EVENT NAMES:\n' + vocabParts.join('\n'));
+    }
+
+    // 6. Table Features
+    const featureLines = [
+        'Combines daily, intraday, and fresh exports; the best available version of each event is used.',
+        'Incremental updates: non-final data is replaced with the latest available data on every run.',
+        'Promotes key fields (e.g. page_location, session_id) to top-level columns for faster queries.',
+        'Session-level fields: landing_page, user_id resolution, and configurable session parameters.',
+    ];
+    sections.push('TABLE FEATURES:\n' + featureLines.map(f => `- ${f}`).join('\n'));
+
+    return sections;
+};
+
+module.exports = { getTableDescriptionSections };