ga4-export-fixer 0.4.1 → 0.4.2-dev.0

package/helpers/ga4Transforms.js

@@ -0,0 +1,158 @@
+const { unnestEventParam } = require('./params');
+
+/*
+Common identifiers
+*/
+
+const sessionId = `concat(user_pseudo_id, (select value.int_value from unnest(event_params) where key = 'ga_session_id'))`;
+
+/*
+Ecommerce
+*/
+
+/**
+ * Fixes and normalizes the ecommerce struct extracted from GA4 event data.
+ *
+ * This helper returns a SQL expression that:
+ *   - Ensures `ecommerce.transaction_id` is set to NULL if it has the placeholder string '(not set)';
+ *   - For 'purchase' events, normalizes `ecommerce.purchase_revenue` by:
+ *       * Removing NaN values;
+ *       * Filling missing purchase revenue (an old GA4 bug) with the event parameter 'value', safely cast as FLOAT64;
+ *   - Leaves other fields in the ecommerce struct unchanged.
+ *
+ * The result is a new struct with the same shape as 'ecommerce' but with cleaned transaction_id and purchase_revenue.
+ *
+ * @returns {string} A SQL snippet for SELECT AS STRUCT ... REPLACE to normalize ecommerce fields.
+ *
+ * @example
+ *   fixEcommerceStruct()
+ *   // => SQL string that can be used in a SELECT list to normalize ecommerce columns
+ */
+const fixEcommerceStruct = () => {
+  return `(select as struct ecommerce.* replace(
+    if(ecommerce.transaction_id <> '(not set)', ecommerce.transaction_id, null)  as transaction_id,
+    if(
+      event_name = 'purchase',
+      coalesce(
+        -- fix possible NaN values
+        if(is_nan(ecommerce.purchase_revenue), null, ecommerce.purchase_revenue),
+        -- fix an old ga4 bug where purchase_revenue was missing
+        safe_cast(${unnestEventParam('value')} as float64)
+      ),
+      null
+    ) as purchase_revenue
+  ))`;
+};
+
+/*
+Check if GA4 data is "final" and is not expected to change anymore
+*/
+
+/**
+ * Generates a SQL expression to determine whether GA4 export data can be considered "final" (not subject to further change).
+ *
+ * Two detection methods are supported:
+ *   - 'EXPORT_TYPE': Checks the table suffix; returns FALSE for intraday or "fresh" tables, TRUE for finalized data.
+ *   - 'DAY_THRESHOLD': Considers data final if a configurable number of days has passed since event_date.
+ *
+ * @param {'EXPORT_TYPE'|'DAY_THRESHOLD'} detectionMethod - The method to use for finality determination.
+ *        'EXPORT_TYPE': Uses patterns in _table_suffix (e.g., 'intraday_%', 'fresh_%').
+ *        'DAY_THRESHOLD': Uses date difference between the current date and event_date.
+ * @param {number} [dayThreshold=3] - (Only for 'DAY_THRESHOLD') Number of days after which data is considered final.
+ * @returns {string} SQL expression that evaluates to TRUE if the data is final, otherwise FALSE.
+ *
+ * @throws {Error} If an unsupported detectionMethod is provided.
+ *
+ * @example
+ *   // Checks based on export type
+ *   isFinalData('EXPORT_TYPE')
+ *   // => "if(_table_suffix like 'intraday_%' or _table_suffix like 'fresh_%', false, true)"
+ *
+ *   // Checks using a custom day threshold
+ *   isFinalData('DAY_THRESHOLD', 5)
+ *   // => "if(date_diff(current_date(), cast(event_date as date format 'YYYYMMDD'), day) > 5, true, false)"
+ */
+const isFinalData = (detectionMethod, dayThreshold) => {
+  if (detectionMethod !== 'EXPORT_TYPE' && detectionMethod !== 'DAY_THRESHOLD') {
+    throw new Error(`isFinalData: Unsupported detectionMethod '${detectionMethod}'. Supported values are 'EXPORT_TYPE' and 'DAY_THRESHOLD'.`);
+  }
+
+  if (detectionMethod === 'DAY_THRESHOLD') {
+    if (typeof dayThreshold === 'undefined') {
+      throw new Error("isFinalData: 'dayThreshold' is required when using 'DAY_THRESHOLD' detectionMethod.");
+    }
+    if (!Number.isInteger(dayThreshold) || dayThreshold < 0) {
+      throw new Error("isFinalData: 'dayThreshold' must be an integer greater than or equal to 0 when using 'DAY_THRESHOLD' detectionMethod.");
+    }
+  }
+
+  if (detectionMethod === 'EXPORT_TYPE') {
+    return 'if(_table_suffix like \'intraday_%\' or _table_suffix like \'fresh_%\', false, true)';
+  }
+
+  if (detectionMethod === 'DAY_THRESHOLD') {
+    return `if(date_diff(current_date(), cast(event_date as date format 'YYYYMMDD'), day) > ${dayThreshold}, true, false)`;
+  }
+};
+
+/**
+ * Checks whether a given column name is part of the standard/expected GA4 BigQuery export columns.
+ *
+ * The list of recognized GA4 export columns is based on the official schema as of 2026-02-18.
+ * This function can be used to filter or validate column names when processing GA4 data exports.
+ *
+ * @param {string} columnName - The name of the column to check.
+ * @returns {boolean} True if the column name is a GA4 export column, otherwise false.
+ */
+const isGa4ExportColumn = (columnName) => {
+  // list updated 2026-02-18
+  const ga4ExportColumns = [
+    "event_date",
+    "event_timestamp",
+    "event_name",
+    "event_params",
+    "event_previous_timestamp",
+    "event_value_in_usd",
+    "event_bundle_sequence_id",
+    "event_server_timestamp_offset",
+    "user_id",
+    "user_pseudo_id",
+    "privacy_info",
+    "user_properties",
+    "user_first_touch_timestamp",
+    "user_ltv",
+    "device",
+    "geo",
+    "app_info",
+    "traffic_source",
+    "stream_id",
+    "platform",
+    "event_dimensions",
+    "ecommerce",
+    "items",
+    "collected_traffic_source",
+    "is_active_user",
+    "batch_event_index",
+    "batch_page_id",
+    "batch_ordering_id",
+    "session_traffic_source_last_click",
+    "publisher"
+  ];
+  return ga4ExportColumns.includes(columnName);
+};
+
+const getGa4ExportType = (tableSuffix) => {
+  return `case
+      when ${tableSuffix} like 'intraday_%' then 'intraday'
+      when ${tableSuffix} like 'fresh_%' then 'fresh'
+      when regexp_contains(${tableSuffix}, r'^\\d{8}$') then 'daily'
+    end`;
+};
+
+module.exports = {
+  sessionId,
+  fixEcommerceStruct,
+  isFinalData,
+  isGa4ExportColumn,
+  getGa4ExportType
+};

package/helpers/index.js

@@ -0,0 +1,8 @@
+module.exports = {
+  ...require('./params'),
+  ...require('./dateTime'),
+  ...require('./dateFilters'),
+  ...require('./urlParsing'),
+  ...require('./aggregation'),
+  ...require('./ga4Transforms')
+};

package/helpers/params.js

@@ -0,0 +1,48 @@
+/*
+Unnesting parameters
+*/
+
+// unnest any parameter from the selected params array
+const unnestParam = (keyName, paramsArray, dataType) => {
+  if (typeof keyName !== 'string' || keyName.trim() === '') {
+    throw new Error("unnestParam: 'keyName' is required and must be a non-empty string.");
+  }
+  if (typeof paramsArray !== 'string' || paramsArray.trim() === '') {
+    throw new Error("unnestParam: 'paramsArray' is required and must be a non-empty string.");
+  }
+
+  if (dataType) {
+    // return the value from the selected column
+    if (dataType === 'string') {
+      return `(select value.string_value from unnest(${paramsArray}) where key = '${keyName}')`;
+    } else if (dataType === 'int' || dataType === 'int64') {
+      return `(select value.int_value from unnest(${paramsArray}) where key = '${keyName}')`;
+    } else if (dataType === 'double') {
+      return `(select value.double_value from unnest(${paramsArray}) where key = '${keyName}')`;
+    } else if (dataType === 'float' || dataType === 'float64') {
+      return `(select value.float_value from unnest(${paramsArray}) where key = '${keyName}')`;
+    }
+
+    throw new Error(`unnestParam: Unsupported dataType '${dataType}'. Supported values are 'string', 'int', 'int64', 'double', 'float', and 'float64'.`);
+  } else {
+    // return the value from the column that has data, cast as string
+    return `(select coalesce(value.string_value, cast(value.int_value as string), cast(value.double_value as string), cast(value.float_value as string)) from unnest(${paramsArray}) where key = '${keyName}')`;
+  }
+};
+
+// event_params and session_params
+
+// unnest a param from the event_params array
+const unnestEventParam = (keyName, dataType) => {
+  return unnestParam(keyName, 'event_params', dataType);
+};
+
+// unnest a param from the session_params array
+const unnestSessionParam = (keyName, dataType) => {
+  return unnestParam(keyName, 'session_params', dataType);
+};
+
+module.exports = {
+  unnestEventParam,
+  unnestSessionParam
+};

package/helpers/urlParsing.js

@@ -0,0 +1,155 @@
+const { unnestEventParam } = require('./params');
+
+/*
+Page details
+*/
+
+/**
+ * Generates a SQL expression to extract the hostname from a URL.
+ *
+ * This function returns a BigQuery SQL string that:
+ *   1. Removes the HTTP or HTTPS scheme from the start of the URL using regexp_replace.
+ *   2. Extracts the hostname (the first part before the next '/') using regexp_extract.
+ *
+ * Example usage (in SQL context):
+ *   SELECT ${extractUrlHostname('my_url_column')} AS hostname
+ *
+ * @param {string} url - The SQL expression or column reference containing the URL.
+ * @returns {string} - BigQuery SQL expression for extracting the hostname from the input URL.
+ */
+const extractUrlHostname = (url) => {
+  return `regexp_extract(
+    regexp_replace(
+      ${url},
+      r'^https?://',
+      ''
+    ),
+    r'^[^/]+'
+  )`;
+};
+
+/**
+ * Generates a SQL expression to extract the path component from a URL.
+ *
+ * This function returns a BigQuery SQL string that:
+ *   1. Removes the scheme and hostname (e.g., http(s)://domain) from the URL using regexp_replace.
+ *   2. Removes any query ('?') or fragment ('#') from the resulting string.
+ *   3. Trims whitespace from the result.
+ *
+ * Example usage (in SQL context):
+ *   SELECT ${extractUrlPath('my_url_column')} AS path
+ *
+ * @param {string} url - The SQL expression or column reference containing the URL.
+ * @returns {string} - BigQuery SQL expression for extracting the path component from the input URL.
+ */
+const extractUrlPath = (url) => {
+  return `trim(
+    regexp_replace(
+      regexp_replace(
+        ${url},
+        r'^https?://[^/]+',
+        ''
+      ),
+      r'[\\?#].*',
+      ''
+    )
+  )`;
+};
+
+/**
+ * Generates a SQL expression to extract the query component from a URL.
+ *
+ * This function returns a BigQuery SQL string that:
+ *   1. Uses regexp_extract to retrieve the query string (the part starting with '?', up to but not including a fragment '#', if present) from the input URL.
+ *   2. Trims leading/trailing whitespace from the extracted query string.
+ *
+ * Example usage (in SQL context):
+ *   SELECT ${extractUrlQuery('my_url_column')} AS url_query
+ *
+ * @param {string} url - The SQL expression or column reference containing the URL.
+ * @returns {string} - BigQuery SQL expression for extracting the query string from the input URL, including the leading '?' if present.
+ */
+const extractUrlQuery = (url) => {
+  return `trim(regexp_extract(${url}, r'\\?[^#]+'))`;
+};
+
+/**
+ * Generates a SQL expression to parse the query parameters of a URL into an array of structs (key-value pairs).
+ *
+ * This function:
+ *   1. Extracts the query string from the given URL using {@link extractUrlQuery}.
+ *   2. Splits the query string on '&' to separate individual key-value pairs.
+ *   3. Splits each pair on '=' to extract the parameter key and value.
+ *   4. Returns an array of STRUCTs with fields "key" and "value".
+ *
+ * Example usage (in SQL context):
+ *   SELECT ${extractUrlQueryParams('my_url_column')} AS query_params
+ *
+ * Output schema:
+ *   ARRAY<STRUCT<key STRING, value STRING>>
+ *
+ * @param {string} url - The SQL expression or column reference containing the URL.
+ * @returns {string} - BigQuery SQL expression producing an array of key/value structs for the query parameters.
+ */
+const extractUrlQueryParams = (url) => {
+  return `array(
+        (
+          select
+            as struct split(keyval, '=') [safe_offset(0)] as key,
+            split(keyval, '=') [safe_offset(1)] as value
+          from
+            unnest(
+              split(
+                ${extractUrlQuery(url)},
+                '&'
+              )
+            ) as keyval
+        )
+      )`;
+};
+
+/**
+ * Generates a SQL expression that extracts detailed page information from a given URL.
+ *
+ * This function produces a BigQuery SQL struct containing the following fields:
+ *   - hostname: The hostname part of the URL (e.g., 'www.example.com')
+ *   - path: The path portion of the URL (e.g., '/about/team')
+ *   - query: The raw query string from the URL, including the leading '?', if present (e.g., '?id=123')
+ *   - query_params: An array of STRUCT<key STRING, value STRING> representing parsed key/value pairs from the query string
+ *
+ * If no URL is provided, the function defaults to extracting the URL from the `page_location` event parameter.
+ * All fields are derived via helper functions that generate appropriate BigQuery SQL expressions.
+ *
+ * Example usage (in SQL context):
+ *   SELECT ${extractPageDetails('my_url_column')} AS page_details
+ *
+ * Output schema (STRUCT):
+ *   {
+ *     hostname: STRING,
+ *     path: STRING,
+ *     query: STRING,
+ *     query_params: ARRAY<STRUCT<key STRING, value STRING>>
+ *   }
+ *
+ * @param {string} [url] - (Optional) SQL expression or column reference for the URL to extract details from.
+ *                         If not provided, defaults to unnesting the 'page_location' event parameter as a string.
+ * @returns {string} BigQuery SQL expression yielding a STRUCT of hostname, path, query, and query_params from the URL.
+ */
+const extractPageDetails = (url) => {
+  url = url || `${unnestEventParam('page_location', 'string')}`;
+
+  return `(select as struct
+    ${extractUrlHostname(url)} as hostname,
+    ${extractUrlPath(url)} as path,
+    ${extractUrlQuery(url)} as query,
+    ${extractUrlQueryParams(url)} as query_params
+  )`;
+};
+
+module.exports = {
+  extractUrlHostname,
+  extractUrlPath,
+  extractUrlQuery,
+  extractUrlQueryParams,
+  extractPageDetails
+};

package/index.js

@@ -1,4 +1,4 @@
-const helpers = require('./helpers.js');
+const helpers = require('./helpers/index.js');
 const ga4EventsEnhanced = require('./tables/ga4EventsEnhanced.js');
 const preOperations = require('./preOperations.js');
 const { validateBaseConfig, validateEnhancedEventsConfig } = require('./inputValidation.js');

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.4.1",
+  "version": "0.4.2-dev.0",
   "description": "",
   "main": "index.js",
   "files": [
     "index.js",
-    "helpers.js",
+    "helpers",
     "utils.js",
     "preOperations.js",
     "constants.js",

package/tables/ga4EventsEnhanced.js

@@ -1,4 +1,4 @@
-const helpers = require('../helpers.js');
+const helpers = require('../helpers/index.js');
 const utils = require('../utils.js');
 const inputValidation = require('../inputValidation.js');
 const constants = require('../constants.js');