ga4-export-fixer 0.1.1 → 0.1.3-dev.0

package/README.md

@@ -17,13 +17,15 @@ Include the package in the package.json file in your Dataform repository.
 **`package.json`**
 ```json
 {
-  "name": "my_dataform_repo",
   "dependencies": {
-    "@dataform/core": "3.0.39",
-    "ga4-export-fixer": "0.1.0"
+    "@dataform/core": "3.0.42",
+    "ga4-export-fixer": "0.1.2"
   }
 }
 ```
+
+**Note:** The best practice is to specify the package version explicitly (e.g. `"0.1.2"`) rather than using `"latest"` or `"*"`, to avoid unexpected breaking changes when the package is updated.
+
 In Google Cloud Dataform, click "Install Packages" to install it in your development workspace.
 
 If your Dataform repository does not have a package.json file, see this guide: https://docs.cloud.google.com/dataform/docs/manage-repository#move-to-package-json
@@ -46,6 +48,8 @@ The main features include:
 
 Create a new **ga4_events_enhanced** table using a **.js** file in your repository's **definitions** folder.
 
+##### Using Defaults
+
 **`definitions/ga4/ga4_events_enhanced.js`**
 ```javascript
 const { ga4EventsEnhanced } = require('ga4-export-fixer');
@@ -57,6 +61,53 @@ const config = {
 ga4EventsEnhanced.createTable(publish, config);
 ```
 
+##### With Custom Configuration
+
+**`definitions/ga4/ga4_events_enhanced.js`**
+```javascript
+const { ga4EventsEnhanced } = require('ga4-export-fixer');
+
+const config = {
+  sourceTable: constants.GA4_TABLES.MY_GA4_EXPORT,
+  schemaLock: '20260101', // prevent possible issues from updates to the export schema
+  customTimestampParam: 'custom_event_timestamp', // custom timestamp collected as an event param
+  timezone: 'Europe/Helsinki',
+  // not needed data
+  excludedColumns: [
+    'app_info',
+    'publisher'
+  ],
+  // not needed events
+  excludedEvents: [
+    'user_engagement'
+  ],
+  // transform to session-level
+  sessionParams: [
+    'user_agent'
+  ],
+  // promote as columns
+  eventParamsToColumns: [
+    {name: 'session_engaged'},
+    {name: 'ga_session_number', type: 'int'},
+    {name: 'page_type', type: 'string'},
+  ],
+  // not needed in the event_params array
+  excludedEventParams: [
+    'session_engaged',
+    'ga_session_number',
+    'page_type',
+    'user_agent'
+  ],
+  // use day threshold for data_is_final
+  dataIsFinal: {
+    detectionMethod: 'DAY_THRESHOLD',
+    dayThreshold: 4
+  },
+};
+
+ga4EventsEnhanced.createTable(publish, config);
+```
+
 #### SQLX Deployment
 
 Alternatively, you can create the **ga4_events_enhanced** table using a .SQLX file.
@@ -91,19 +142,132 @@ pre_operations {
 }
 ```
 
+
+
+#### Configuration Object
+
+All fields are optional except `sourceTable`. Default values are applied automatically, so you only need to specify the fields you want to override.
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `sourceTable` | Dataform ref() / string | **required** | Source GA4 export table. Use `ref()` in Dataform or a string in format `` `project.dataset.table` `` |
+| `self` | Dataform self() | **required for .SQLX deployment** | Reference to the table itself. Use `self()` in Dataform |
+| `incremental` | Dataform incremental() | **required for .SQLX deployment** | Switch between incremental and full refresh logic. Use `incremental()` in Dataform |
+| `schemaLock` | string (YYYYMMDD) | `undefined` | Lock the table schema to a specific date. Must be a valid date >= `"20241009"` |
+| `timezone` | string | `'Etc/UTC'` | IANA timezone for event datetime (e.g. `'Europe/Helsinki'`) |
+| `customTimestampParam` | string | `undefined` | Name of a custom event parameter containing a JS timestamp in milliseconds (e.g. collected via `Date.now()`) |
+| `bufferDays` | integer | `1` | Extra days to include for sessions that span midnight |
+| `test` | boolean | `false` | Enable test mode (uses `testConfig` date range instead of pre-operations) |
+| `excludedEventParams` | string[] | `[]` | Event parameter names to exclude from the `event_params` array |
+| `excludedEvents` | string[] | `[]` | Event names to exclude from the table |
+| `excludedColumns` | string[] | `[]` | Default GA4 export columns to exclude from the final table, for example `'app_info'` or `'publisher'` |
+| `sessionParams` | string[] | `[]` | Event parameter names to aggregate as session-level parameters |
+
+**`includedExportTypes`** — which GA4 export types to include:
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `includedExportTypes.daily` | boolean | `true` | Include daily (processed) export |
+| `includedExportTypes.intraday` | boolean | `true` | Include intraday export |
+
+**`dataIsFinal`** — how to determine whether data is final (not expected to change):
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `dataIsFinal.detectionMethod` | string | `'EXPORT_TYPE'` | `'EXPORT_TYPE'` (uses table suffix, all data from the daily export is considered final) or `'DAY_THRESHOLD'` (uses days since event) |
+| `dataIsFinal.dayThreshold` | integer | `4` | Days after which data is considered final. Required when `detectionMethod` is `'DAY_THRESHOLD'` |
+
+**`testConfig`** — date range used when `test` is `true`:
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `testConfig.dateRangeStart` | string (SQL date) | `'current_date()-1'` | Start date for test queries |
+| `testConfig.dateRangeEnd` | string (SQL date) | `'current_date()'` | End date for test queries |
+
+**`preOperations`** — date range and incremental refresh configuration:
+
+| Field | Type | Default | Description |
+|-------|------|---------|-------------|
+| `preOperations.dateRangeStartFullRefresh` | string (SQL date) | `'date(2000, 1, 1)'` | Start date for full refresh |
+| `preOperations.dateRangeEnd` | string (SQL date) | `'current_date()'` | End date for queries |
+| `preOperations.numberOfPreviousDaysToScan` | integer | `10` | Number of previous days to scan from the result table when determining the incremental refresh start checkpoint. A higher value is required if the table updates have fallen behind for some reason |
+| `preOperations.incrementalStartOverride` | string (SQL date) | `undefined` | Override the incremental start date to re-process a specific range |
+| `preOperations.incrementalEndOverride` | string (SQL date) | `undefined` | Override the incremental end date to re-process a specific range |
+
+**`eventParamsToColumns`** — each item in the array is an object:
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `name` | string | Yes | Event parameter name |
+| `type` | string | No | Data type: `'string'`, `'int'`, `'int64'`, `'double'`, `'float'`, or `'float64'`. If omitted, returns the value converted to a string |
+| `columnName` | string | No | Column name in the output. Defaults to the parameter `name` |
+
+Date fields (`dateRangeStart`, `dateRangeEnd`, etc.) accept string dates in `YYYYMMDD` or `YYYY-MM-DD` format, or BigQuery SQL expressions (e.g. `'current_date()'`, `'date(2026, 1, 1)'`).
+
 ### Helpers
 
-The helpers contain templates for common SQL expression needed when working with GA4 data.
+The helpers contain templates for common SQL expressions. The functions are referenced by **ga4EventsEnhanced** but can also be imported as utility functions for working with GA4 data.
 
 ```javascript
 const { helpers } = require('ga4-export-fixer');
-
-// Unnest event parameters, date filters, URL extraction, session aggregation, etc.
-helpers.unnestEventParam('page_location', 'string');
-helpers.ga4ExportDateFilter('daily', 'current_date()-7', 'current_date()');
-helpers.extractPageDetails();
 ```
 
+#### SQL Templates
+
+| Name | Example | Description |
+|------|---------|-------------|
+| `eventDate` | `helpers.eventDate` | Casts `event_date` string to a DATE using YYYYMMDD format |
+| `sessionId` | `helpers.sessionId` | Builds a session ID by concatenating `user_pseudo_id` and `ga_session_id` |
+
+#### Functions
+
+**Unnesting parameters**
+
+| Function | Example | Description |
+|----------|---------|-------------|
+| `unnestEventParam` | `unnestEventParam('page_location', 'string')` | Extracts a value from the `event_params` array by key. Supported types: `'string'`, `'int'`, `'int64'`, `'double'`, `'float'`, `'float64'`. Omit type to get the value converted as a string |
+
+**Date and time**
+
+| Function | Example | Description |
+|----------|---------|-------------|
+| `getEventTimestampMicros` | `getEventTimestampMicros('custom_ts')` | Returns SQL for event timestamp in microseconds. With a custom parameter, uses it (converted from ms) with fallback to `event_timestamp` |
+| `getEventDateTime` | `getEventDateTime({ timezone: 'Europe/Helsinki' })` | Returns SQL for event datetime in the given timezone. Defaults to `'Etc/UTC'` |
+
+**Date filters**
+
+| Function | Example | Description |
+|----------|---------|-------------|
+| `ga4ExportDateFilter` | `ga4ExportDateFilter('daily', 'current_date()-7', 'current_date()')` | Generates a `_table_suffix` filter for a single export type (`'daily'` or `'intraday'`) and date range |
+
+**Page details**
+
+| Function | Example | Description |
+|----------|---------|-------------|
+| `extractUrlHostname` | `extractUrlHostname('page_location')` | Extracts hostname from a URL column |
+| `extractUrlPath` | `extractUrlPath('page_location')` | Extracts the path component from a URL column |
+| `extractUrlQuery` | `extractUrlQuery('page_location')` | Extracts the query string (including `?`) from a URL column |
+| `extractUrlQueryParams` | `extractUrlQueryParams('page_location')` | Parses URL query parameters into `ARRAY<STRUCT<key STRING, value STRING>>` |
+| `extractPageDetails` | `extractPageDetails()` | Returns a struct with `hostname`, `path`, `query`, and `query_params`. Defaults to `page_location` event parameter |
+
+**Aggregation**
+
+| Function | Example | Description |
+|----------|---------|-------------|
+| `aggregateValue` | `aggregateValue('user_id', 'last', 'event_timestamp')` | Aggregates a column using `'max'`, `'min'`, `'first'`, `'last'`, or `'any'`. `'first'` and `'last'` use the timestamp column for ordering |
+
+**Ecommerce**
+
+| Function | Example | Description |
+|----------|---------|-------------|
+| `fixEcommerceStruct` | `fixEcommerceStruct()` | Cleans the ecommerce struct: sets `transaction_id` to null when `'(not set)'`, and fixes missing/NaN `purchase_revenue` for purchase events |
+
+**Data freshness**
+
+| Function | Example | Description |
+|----------|---------|-------------|
+| `isFinalData` | `isFinalData('DAY_THRESHOLD', 4)` | Returns SQL that evaluates to `true` when data is final. `'EXPORT_TYPE'` checks table suffix; `'DAY_THRESHOLD'` uses days since event (`dayThreshold` is required and must be a non-negative integer) |
+
 ## License
 
 MIT

package/helpers.js

@@ -634,19 +634,21 @@ const isFinalData = (detectionMethod, dayThreshold) => {
     throw new Error(`isFinalData: Unsupported detectionMethod '${detectionMethod}'. Supported values are 'EXPORT_TYPE' and 'DAY_THRESHOLD'.`);
   }
 
-  if (typeof dayThreshold !== 'undefined' && (typeof dayThreshold !== 'number' || isNaN(dayThreshold))) {
-    throw new Error("isFinalData: 'dayThreshold' must be a number if provided.");
+  if (detectionMethod === 'DAY_THRESHOLD') {
+    if (typeof dayThreshold === 'undefined') {
+      throw new Error("isFinalData: 'dayThreshold' is required when using 'DAY_THRESHOLD' detectionMethod.");
+    }
+    if (!Number.isInteger(dayThreshold) || dayThreshold < 0) {
+      throw new Error("isFinalData: 'dayThreshold' must be an integer greater than or equal to 0 when using 'DAY_THRESHOLD' detectionMethod.");
+    }
   }
 
-  const defaultDayThreshold = 3;
-  const threshold = typeof dayThreshold !== 'undefined' ? dayThreshold : defaultDayThreshold;
-
   if (detectionMethod === 'EXPORT_TYPE') {
     return 'if(_table_suffix like \'intraday_%\' or _table_suffix like \'fresh_%\', false, true)';
   }
 
   if (detectionMethod === 'DAY_THRESHOLD') {
-    return `if(date_diff(current_date(), cast(event_date as date format 'YYYYMMDD'), day) > ${threshold}, true, false)`;
+    return `if(date_diff(current_date(), cast(event_date as date format 'YYYYMMDD'), day) > ${dayThreshold}, true, false)`;
   }
 };
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.1.1",
+  "version": "0.1.3-dev.0",
   "description": "",
   "main": "index.js",
   "files": [
@@ -13,7 +13,8 @@
   ],
   "scripts": {
     "test": "node tests/ga4EventsEnhanced.test.js",
-    "test:events": "node tests/ga4EventsEnhanced.test.js"
+    "test:events": "node tests/ga4EventsEnhanced.test.js",
+    "prepublishOnly": "node scripts/updateReadme.js"
   },
   "repository": {
     "type": "git",

package/tables/ga4EventsEnhanced.js

@@ -39,13 +39,13 @@ const defaultConfig = {
         // this is useful if you want to re-process only a specific date range
         incrementalStartOverride: undefined,
         incrementalEndOverride: undefined,
-        numberOfPreviousDaysToScan: 5,
+        numberOfPreviousDaysToScan: 10,
     },
     // these parameters are excluded by default because they've been made available in other columns
     defaultExcludedEventParams: [
         'page_location',
         'ga_session_id',
-        //'custom_event_timestamp', // poistetaan, jos käytössä
+        //'custom_event_timestamp', // removed if customTimestampParam is used
     ],
     excludedEventParams: [],
     eventParamsToColumns: [
@@ -57,12 +57,13 @@ const defaultConfig = {
         'first_visit'
     ],
     excludedEvents: [],
-    // exclude these columns when extracting raw data from the export tables
-    excludedColumns: [
+    defaultExcludedColumns: [
         'event_dimensions', // legacy column, not needed
         'traffic_source', // renamed to user_traffic_source
         'session_id'
     ],
+    // exclude these columns when extracting raw data from the export tables
+    excludedColumns: [],
 };
 
 // List the columns in the order they should be in the final table
@@ -227,8 +228,9 @@ const generateEnhancedEventsSQL = (config) => {
     };
 
     const getExcludedColumns = () => {
+        const allExcludedColumns = utils.mergeUniqueArrays(mergedConfig.defaultExcludedColumns, mergedConfig.excludedColumns);
         const excludedColumns = {};
-        mergedConfig.excludedColumns.forEach(c => {
+        allExcludedColumns.forEach(c => {
             excludedColumns[c] = undefined;
         });
         return excludedColumns;

package/utils.js

@@ -105,6 +105,7 @@ const mergeSQLConfigurations = (defaultConfig, inputConfig = {}) => {
         return defaultConfig;
     }
 
+    // the merged configuration object
     const result = { ...defaultConfig };
 
     for (const key in inputConfig) {
@@ -170,6 +171,27 @@ const mergeSQLConfigurations = (defaultConfig, inputConfig = {}) => {
         }
     }
 
+    // support different formats for passing the sourceTable path
+    const fixSourceTable = (sourceTable) => {
+        if (isDataformTableReferenceObject(sourceTable)) {
+            return sourceTable;
+        }
+        if (typeof sourceTable === 'string') {
+            const tablePath = sourceTable.replace(/[`"']/g, '').trim();
+            if (/^[a-zA-Z0-9-]+\.[a-zA-Z0-9_]+(\.[^\.]+)?$/.test(tablePath)) {
+                const project = tablePath.split('.')[0];
+                const dataset = tablePath.split('.')[1];
+                return `\`${project}.${dataset}.events_*\``;
+            }
+        }
+        throw new Error(`sourceTable must be a Dataform table reference or a string in the format '\`project.dataset.table\`'. Received: ${JSON.stringify(sourceTable)}`);
+    };
+
+    // process the sourceTable to support different formats
+    if (result.sourceTable) {
+        result.sourceTable = fixSourceTable(result.sourceTable);
+    }
+
     return result;
 };
 
@@ -178,15 +200,15 @@ const mergeSQLConfigurations = (defaultConfig, inputConfig = {}) => {
  *
  * A Dataform table reference object is expected to have the properties: 'name', 'project', and 'dataset'.
  *
- * @param {Object} table - The object to check.
+ * @param {Object} obj - The object to check.
  * @returns {boolean} True if the object is a Dataform table reference, false otherwise.
  */
-const isDataformTableReferenceObject = (table) => {
-    return table &&
-        typeof table === 'object' &&
-        Object.hasOwn(table, 'name') &&
-        Object.hasOwn(table, 'project') &&
-        Object.hasOwn(table, 'dataset');
+const isDataformTableReferenceObject = (obj) => {
+    return obj &&
+        typeof obj === 'object' &&
+        Object.hasOwn(obj, 'name') &&
+        Object.hasOwn(obj, 'project') &&
+        Object.hasOwn(obj, 'dataset');
 };