ga4-export-fixer 0.2.5-dev.2 → 0.2.6-dev.0

package/README.md

@@ -43,8 +43,10 @@ The **ga4_events_enhanced** table comes with features such as these:
 - **Schema lock** – Lock the table schema to a specific GA4 export date to prevent schema drift
 - **Data freshness tracking** – `data_is_final` flag and `export_type` label on every row
 - **Selective date range re-processing** – Re-process a subset of data without a full table rebuild, using `incrementalStartOverride` and `incrementalEndOverride`
+- **Batch processing** – Process large GA4 exports in smaller batches using the `numberOfDaysToProcess` configuration setting
 - **Timezone-aware datetime** – `event_datetime` converted to a configurable IANA timezone
 - **Column descriptions** – Full column-level documentation included in the Dataform table configuration, reflecting the specific configuration used to build the table
+- **Zero dependencies** – The package has no external dependencies and will not add any additional packages to your Dataform repository
 
 ### Planned, Upcoming Features
 
@@ -77,7 +79,7 @@ Include the package in the package.json file in your Dataform repository.
 {
   "dependencies": {
     "@dataform/core": "3.0.42",
-    "ga4-export-fixer": "0.2.4"
+    "ga4-export-fixer": "0.2.5"
   }
 }
 ```
@@ -269,18 +271,28 @@ The `onSchemaChange: "EXTEND"` setting updates the result table schema on increm
 | Field                          | Type    | Default | Description                      |
 | ------------------------------ | ------- | ------- | -------------------------------- |
 | `includedExportTypes.daily`    | boolean | `true`  | Include daily (processed) export |
+| `includedExportTypes.fresh`    | boolean | `false` | Include fresh (hourly-updated) export |
 | `includedExportTypes.intraday` | boolean | `true`  | Include intraday export          |
 
 
-> **Intraday-only mode:** Set `daily` to `false` and `intraday` to `true` to use only intraday export tables. When using intraday-only mode, `dataIsFinal.detectionMethod` must be set to `'DAY_THRESHOLD'`.
+Export priority: **daily > fresh > intraday**. Each lower-priority export only provides data not already covered by a higher-priority one. All seven combinations of the three export types are supported.
+
+When all three exports are enabled, the package:
+1. Gets all data from daily export tables
+2. Gets fresh export data for days not yet covered by a daily table
+3. Gets intraday export data for events after the latest fresh event timestamp
+
+The boundary between fresh and intraday is timestamp-based because the fresh export is updated hourly, so within the same day some events come from the fresh export and the rest from intraday.
+
+> **Without daily export:** When `daily` is `false`, `dataIsFinal.detectionMethod` must be set to `'DAY_THRESHOLD'`, because `EXPORT_TYPE` detection relies on daily tables to mark data as final.
 
 **`dataIsFinal`** — how to determine whether data is final (not expected to change):
 
 
-| Field                         | Type    | Default         | Description                                                                                                                                                                                          |
-| ----------------------------- | ------- | --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `dataIsFinal.detectionMethod` | string  | `'EXPORT_TYPE'` | `'EXPORT_TYPE'` (uses table suffix; all data from the daily export is considered final) or `'DAY_THRESHOLD'` (uses days since event). Must be `'DAY_THRESHOLD'` when only intraday export is enabled |
-| `dataIsFinal.dayThreshold`    | integer | `4`             | Days after which data is considered final. Required when `detectionMethod` is `'DAY_THRESHOLD'`                                                                                                      |
+| Field                         | Type    | Default         | Description                                                                                                                                                                                     |
+| ----------------------------- | ------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `dataIsFinal.detectionMethod` | string  | `'EXPORT_TYPE'` | `'EXPORT_TYPE'` (uses table suffix; all data from the daily export is considered final) or `'DAY_THRESHOLD'` (uses days since event). Must be `'DAY_THRESHOLD'` when daily export is not enabled |
+| `dataIsFinal.dayThreshold`    | integer | `4`             | Days after which data is considered final. Required when `detectionMethod` is `'DAY_THRESHOLD'`                                                                                                  |
 
 
 **`testConfig`** — date range used when `test` is `true`:

package/constants.js

@@ -1,6 +1,8 @@
 const constants = {
     DATE_RANGE_START_VARIABLE: 'date_range_start',
     INTRADAY_DATE_RANGE_START_VARIABLE: 'intraday_date_range_start',
+    FRESH_DATE_RANGE_START_VARIABLE: 'fresh_date_range_start',
+    FRESH_MAX_EVENT_TIMESTAMP_VARIABLE: 'fresh_max_event_timestamp',
     DATE_RANGE_END_VARIABLE: 'date_range_end',
     LAST_PARTITION_DATE_VARIABLE: 'last_partition_date',
     DATE_COLUMN: 'event_date',

package/defaultConfig.js

@@ -42,8 +42,8 @@ const ga4EventsEnhancedConfig = {
     // optional
     includedExportTypes: {
         daily: true,
-        intraday: true,
         fresh: false,
+        intraday: true,
     },
     timezone: 'Etc/UTC',
     customTimestampParam: undefined,

package/helpers.js

@@ -105,48 +105,37 @@ const getEventDateTime = (config) => {
 
 // Filter the export tables by date range
 /**
- * Generates a SQL filter condition for selecting GA4 export tables based on the export type ('intraday' or 'daily') and a date range.
+ * Generates a SQL filter condition for selecting GA4 export tables based on the export type and a date range.
  *
  * This helper produces SQL snippets to be used in WHERE clauses, ensuring only tables within the provided date range and export type are included.
  * 
  * - For 'daily' exports: Matches table suffixes formatted as YYYYMMDD (e.g., 20240101).
+ * - For 'fresh' exports: Matches table suffixes prefixed with 'fresh_' followed by the date (e.g., fresh_20240101).
  * - For 'intraday' exports: Matches table suffixes prefixed with 'intraday_' followed by the date (e.g., intraday_20240101).
- * - Throws an error for unsupported export types or if start/end dates are undefined.
  *
- * @param {'intraday'|'daily'} exportType - The type of export table; either 'intraday' or 'daily'.
+ * @param {'daily'|'fresh'|'intraday'} exportType - The type of export table.
  * @param {string} start - The start date value as a SQL date expression (e.g. 'current_date()-1').
  * @param {string} end - The end date value as a SQL date expression (e.g. 'current_date()').
  * @returns {string} SQL condition to restrict tables by _table_suffix to the appropriate date range and export type.
  *
- * @throws {Error} If exportType is not 'intraday' or 'daily', or if start/end are not defined.
- *
- * @example
- *   ga4ExportDateFilter('daily', 'current_date()-1', 'current_date()')
- *   // => "(_table_suffix >= cast(current_date()-1 as string format \"YYYYMMDD\") and _table_suffix <= cast(current_date() as string format \"YYYYMMDD\"))"
- *
- *   ga4ExportDateFilter('intraday', 'current_date()-1', 'current_date()')
- *   // => "(_table_suffix >= 'intraday_' || cast(current_date()-1 as string format \"YYYYMMDD\") and _table_suffix <= 'intraday_' || cast(current_date() as string format \"YYYYMMDD\"))"
+ * @throws {Error} If exportType is not supported, or if start/end are not defined.
  */
 const ga4ExportDateFilter = (exportType, start, end) => {
-  if (exportType !== 'intraday' && exportType !== 'daily') {
+  if (exportType !== 'intraday' && exportType !== 'daily' && exportType !== 'fresh') {
     throw new Error(
-      `ga4ExportDateFilter: Unsupported exportType '${exportType}'. Supported values are 'intraday' and 'daily'.`
+      `ga4ExportDateFilter: Unsupported exportType '${exportType}'. Supported values are 'daily', 'fresh', and 'intraday'.`
     );
   }
   if (typeof start === 'undefined' || typeof end === 'undefined') {
     throw new Error("ga4ExportDateFilter: 'start' and 'end' parameters must be defined.");
   }
   
-  if (exportType === 'intraday') {
-    return `(_table_suffix >= 'intraday_' || cast(${start} as string format "YYYYMMDD") and _table_suffix <= 'intraday_' || cast(${end} as string format "YYYYMMDD"))`;
-  }
-  if (exportType === 'daily') {
-    return `(_table_suffix >= cast(${start} as string format "YYYYMMDD") and _table_suffix <= cast(${end} as string format "YYYYMMDD"))`;
-  }
+  const prefix = exportType === 'daily' ? '' : `'${exportType}_' || `;
+  return `(_table_suffix >= ${prefix}cast(${start} as string format "YYYYMMDD") and _table_suffix <= ${prefix}cast(${end} as string format "YYYYMMDD"))`;
 };
 
 /**
- * Builds a `_table_suffix` WHERE clause for GA4 BigQuery export tables (daily and/or intraday).
+ * Builds a `_table_suffix` WHERE clause for GA4 BigQuery export tables (daily, fresh, and/or intraday).
  *
  * Date boundaries are resolved differently depending on the mode:
  *   - **test** -- literal dates from `config.testConfig`
@@ -156,18 +145,24 @@ const ga4ExportDateFilter = (exportType, start, end) => {
  * `bufferDays` is subtracted from the daily start date so sessions that span
  * midnight are not partially excluded.
  *
- * When both daily and intraday exports are enabled, the intraday start date
- * comes from a dedicated variable (`INTRADAY_DATE_RANGE_START_VARIABLE`) so
- * intraday tables that already have a corresponding daily table are excluded.
- * When only intraday is enabled, the daily start-date logic (including buffer
- * days) is reused instead.
+ * Export priority: daily > fresh > intraday. Each lower-priority export only
+ * provides data not already covered by a higher-priority one.
+ *
+ * When fresh and daily are both enabled, the fresh start date comes from
+ * `FRESH_DATE_RANGE_START_VARIABLE` (first day with fresh but no daily table).
+ *
+ * When fresh and intraday are both enabled, intraday rows are filtered by
+ * `event_timestamp > fresh_max_event_timestamp` to avoid duplicating fresh data.
+ *
+ * When only daily and intraday are enabled (no fresh), the existing
+ * `INTRADAY_DATE_RANGE_START_VARIABLE` checkpoint logic is preserved.
  *
  * @param {Object} config
  * @param {boolean}  config.test                      - Use literal test dates.
  * @param {Object}   config.testConfig                - `{ dateRangeStart, dateRangeEnd }`.
  * @param {boolean}  config.incremental               - Use BigQuery variable placeholders.
  * @param {Object}   config.preOperations             - `{ dateRangeStartFullRefresh, dateRangeEnd }`.
- * @param {Object}   config.includedExportTypes       - `{ daily: boolean, intraday: boolean }`.
+ * @param {Object}   config.includedExportTypes       - `{ daily: boolean, fresh: boolean, intraday: boolean }`.
  * @param {number}  [config.bufferDays=0]             - Extra days subtracted from the start date.
  * @returns {string} SQL fragment for a WHERE clause.
  */
@@ -175,59 +170,81 @@ const ga4ExportDateFilters = (config) => {
   const bufferDays = config.bufferDays || 0;
 
   const getStartDate = () => {
-    //test mode
     if (config.test) {
       return config.testConfig.dateRangeStart;
     }
     if (config.incremental) {
       return constants.DATE_RANGE_START_VARIABLE;
     }
-    // full refresh
     return config.preOperations.dateRangeStartFullRefresh;
   };
 
   const getEndDate = () => {
-    // test mode, avoid using a BigQuery variable
     if (config.test) {
       return config.testConfig.dateRangeEnd;
     }
-    // use checkpoint variable with incremental refresh -> allows pre processing any part of the table without having to do a full refresh
     if (config.incremental) {
       return constants.DATE_RANGE_END_VARIABLE;
     }
-    // full refresh
     if (config.preOperations.numberOfDaysToProcess !== undefined) {
       return `least(${config.preOperations.dateRangeStartFullRefresh}+${config.preOperations.numberOfDaysToProcess}-1, current_date())`;
     }
     return config.preOperations.dateRangeEnd;
   };
 
+  const getFreshStartDate = () => {
+    // Fresh tables persist alongside daily tables (unlike intraday which gets deleted),
+    // so the checkpoint variable is needed even in test mode to avoid duplicate data.
+    if (config.includedExportTypes.fresh && config.includedExportTypes.daily) {
+      return constants.FRESH_DATE_RANGE_START_VARIABLE;
+    }
+    if (config.includedExportTypes.fresh && !config.includedExportTypes.daily) {
+      return getStartDate();
+    }
+  };
+
   const getIntradayStartDate = () => {
-    // In test mode, skip pre-operations even though intraday and daily tables may temporarily overlap.
+    // When fresh is enabled: intraday starts from the same point as fresh.
+    // Fresh tables persist alongside intraday tables, so the checkpoint is
+    // needed even in test mode to avoid duplicate data.
+    if (config.includedExportTypes.fresh) {
+      return getFreshStartDate();
+    }
+    // For non-fresh paths, test mode skips pre-operation variables.
     if (config.test) {
       return config.testConfig.dateRangeStart;
     }
-    // Dedicated variable excludes intraday tables that overlap with already-processed daily tables.
+    // When daily+intraday without fresh: use the existing date-based checkpoint
     if (config.includedExportTypes.intraday && config.includedExportTypes.daily) {
       return constants.INTRADAY_DATE_RANGE_START_VARIABLE;
     }
-    // Without daily export, reuse the daily start-date logic and apply bufferDays
-    // (buffer is normally only applied to the daily start date).
+    // Intraday-only: reuse the daily start-date logic with bufferDays
     if (config.includedExportTypes.intraday && !config.includedExportTypes.daily) {
-      // use the same start date as if daily export was in use
-      // include the buffer days as well (not included otherwise for intraday data)
       return `${getStartDate()}-${bufferDays}`;
     }
   };
 
+  const getIntradayFilter = () => {
+    const intradayStart = getIntradayStartDate();
+    const suffixFilter = ga4ExportDateFilter('intraday', intradayStart, end);
+
+    // When fresh is also enabled, add timestamp condition to avoid duplicating fresh data.
+    // Applied even in test mode because fresh and intraday tables coexist for the same days.
+    if (config.includedExportTypes.fresh) {
+      return `(${suffixFilter} and event_timestamp > coalesce(${constants.FRESH_MAX_EVENT_TIMESTAMP_VARIABLE}, 0))`;
+    }
+
+    return suffixFilter;
+  };
+
   const dailyStart = `${getStartDate()}-${bufferDays}`;
-  const intradayStart = getIntradayStartDate();
+  const freshStart = getFreshStartDate();
   const end = getEndDate();
-  
 
   const dateFilters = [
     config.includedExportTypes.daily ? ga4ExportDateFilter('daily', dailyStart, end) : null,
-    config.includedExportTypes.intraday ? ga4ExportDateFilter('intraday', intradayStart, end) : null,
+    config.includedExportTypes.fresh ? ga4ExportDateFilter('fresh', freshStart, end) : null,
+    config.includedExportTypes.intraday ? getIntradayFilter() : null,
   ];
 
   return `(

package/inputValidation.js

@@ -152,7 +152,7 @@ const validateEnhancedEventsConfig = (config) => {
     if (!config.includedExportTypes || typeof config.includedExportTypes !== 'object' || Array.isArray(config.includedExportTypes)) {
         throw new Error(`config.includedExportTypes must be an object. Received: ${JSON.stringify(config.includedExportTypes)}`);
     }
-    for (const key of ['daily', 'intraday']) {
+    for (const key of ['daily', 'fresh', 'intraday']) {
         if (!(key in config.includedExportTypes)) {
             throw new Error(`config.includedExportTypes.${key} is required.`);
         }
@@ -160,8 +160,8 @@ const validateEnhancedEventsConfig = (config) => {
             throw new Error(`config.includedExportTypes.${key} must be a boolean. Received: ${JSON.stringify(config.includedExportTypes[key])}`);
         }
     }
-    if (!config.includedExportTypes.daily && !config.includedExportTypes.intraday) {
-        throw new Error("At least one of config.includedExportTypes.daily or config.includedExportTypes.intraday must be true.");
+    if (!config.includedExportTypes.daily && !config.includedExportTypes.fresh && !config.includedExportTypes.intraday) {
+        throw new Error("At least one of config.includedExportTypes.daily, config.includedExportTypes.fresh, or config.includedExportTypes.intraday must be true.");
     }
 
     // timezone - required
@@ -204,13 +204,14 @@ const validateEnhancedEventsConfig = (config) => {
     ) {
         throw new Error(`config.dataIsFinal.dayThreshold must be a non-negative integer. Received: ${JSON.stringify(config.dataIsFinal.dayThreshold)}`);
     }
-    // EXPORT_TYPE detection relies on daily export metadata; intraday-only requires DAY_THRESHOLD instead.
+    // EXPORT_TYPE detection relies on daily export tables to mark data as final.
+    // When daily is not enabled, all data would be marked as not final under EXPORT_TYPE,
+    // so DAY_THRESHOLD must be used instead.
     if (
-        config.includedExportTypes.intraday &&
         !config.includedExportTypes.daily &&
         config.dataIsFinal.detectionMethod !== 'DAY_THRESHOLD'
     ) {
-        throw new Error(`config.dataIsFinal.detectionMethod must be 'DAY_THRESHOLD' when only intraday export is enabled (config.includedExportTypes.daily is false). A dayThreshold of 1 is recommended for intraday-only configurations. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
+        throw new Error(`config.dataIsFinal.detectionMethod must be 'DAY_THRESHOLD' when daily export is not enabled (config.includedExportTypes.daily is false). A dayThreshold of 1 is recommended for intraday only setups. With fresh export, the GA4 data is subject to possible changes for up to 72 hours. Received: ${JSON.stringify(config.dataIsFinal.detectionMethod)}`);
     }
 
     // bufferDays - required

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.2.5-dev.2",
+  "version": "0.2.6-dev.0",
   "description": "",
   "main": "index.js",
   "files": [

package/preOperations.js

@@ -114,6 +114,79 @@ const getDateRangeStartIntraday = (config) => {
   return undefined;
 };
 
+// Define the date range start for fresh export tables
+// Uses INFORMATION_SCHEMA.TABLES to find the first day with a fresh table but no daily table
+const getDateRangeStartFresh = (config) => {
+  const getStartDate = () => {
+    if (config.incremental) {
+      return `greatest(${constants.DATE_RANGE_START_VARIABLE}, current_date()-5)`;
+    }
+    return 'current_date()-5';
+  };
+
+  const startDate = getStartDate();
+
+  if (config.includedExportTypes.fresh) {
+    const informationSchemaPath = config.sourceTable.replace(
+      /`?([^`]+)\.([^`]+)\.[^`]+`?$/,
+      '`$1.$2.INFORMATION_SCHEMA.TABLES`'
+    );
+
+    return `with export_statuses as (
+    select
+      safe_cast(regexp_extract(table_name, r'\\d+') as date format 'YYYYMMDD') as date,
+      case
+        when table_name like 'events_fresh_%' then 'fresh'
+        else 'daily'
+      end as export_type
+    from
+      ${informationSchemaPath}
+    where
+      regexp_contains(table_name, r'^events_(fresh_)?\\d{8}$')
+      and safe_cast(regexp_extract(table_name, r'\\d+') as date format 'YYYYMMDD')
+        between ${startDate} and current_date()
+  ),
+  statuses_by_day as (
+    select
+      date,
+      max(if(export_type = 'daily', true, false)) as daily,
+      max(if(export_type = 'fresh', true, false)) as fresh
+    from
+      export_statuses
+    group by 
+      date
+  )
+  select
+    min(
+      if(
+        fresh = true and daily = false,
+        date,
+        null
+      )
+    )
+  from
+    statuses_by_day`;
+  }
+
+  return undefined;
+};
+
+// Get the maximum event_timestamp from fresh export tables
+// Used as the boundary between fresh and intraday data
+const getFreshMaxEventTimestamp = (config) => {
+  if (config.includedExportTypes.fresh && config.includedExportTypes.intraday) {
+    const freshStartRef = config.includedExportTypes.daily
+      ? constants.FRESH_DATE_RANGE_START_VARIABLE
+      : (config.incremental ? constants.DATE_RANGE_START_VARIABLE : config.preOperations.dateRangeStartFullRefresh);
+
+    return `select max(event_timestamp) from ${config.sourceTable}
+  where _table_suffix >= 'fresh_' || cast(${freshStartRef} as string format 'YYYYMMDD')
+    and _table_suffix <= 'fresh_' || cast(current_date() as string format 'YYYYMMDD')`;
+  }
+
+  return undefined;
+};
+
 const getDateRangeEnd = (config) => {
   // if an incremental end override is provided, use it
   if (config.incremental && config.preOperations.incrementalEndOverride) {
@@ -147,8 +220,12 @@ const createSchemaLockTable = (config) => {
 
 // Set the pre operations for the query
 const setPreOperations = (config) => {
-  // if in test mode, avoid setting BigQuery variables to make query dry run estimation accurate
-  if (config.test) {
+  // In test mode, most BigQuery variables are skipped to keep dry-run estimation accurate.
+  // Fresh checkpoint variables are the exception: fresh tables persist alongside daily and
+  // intraday tables, so the checkpoints are needed even in test mode to avoid duplicate data.
+  const freshNeedsCheckpoint = config.includedExportTypes.fresh &&
+    (config.includedExportTypes.daily || config.includedExportTypes.intraday);
+  if (config.test && !freshNeedsCheckpoint) {
     return '';
   }
 
@@ -170,9 +247,24 @@ const setPreOperations = (config) => {
     {
       type: 'variable',
       name: constants.INTRADAY_DATE_RANGE_START_VARIABLE,
-      // variable only needed if intraday export tables are included together with daily export tables
-      value: config.sourceTableType === 'GA4_EXPORT' && config.includedExportTypes.intraday && config.includedExportTypes.daily ? getDateRangeStartIntraday(config) : undefined,
-      comment: 'Define the date range start for intraday export tables. Avoid returning intraday data if it overlaps with daily export data. Only needed if intraday export tables are included together with daily export tables.',
+      // only needed when daily+intraday WITHOUT fresh (the two-way intraday checkpoint)
+      // when fresh is also enabled, intraday uses fresh_date_range_start instead
+      value: !config.test && config.sourceTableType === 'GA4_EXPORT' && config.includedExportTypes.intraday && config.includedExportTypes.daily && !config.includedExportTypes.fresh ? getDateRangeStartIntraday(config) : undefined,
+      comment: 'Define the date range start for intraday export tables. Avoid returning intraday data if it overlaps with daily export data. Only needed if intraday and daily export tables are included without fresh.',
+    },
+    {
+      type: 'variable',
+      name: constants.FRESH_DATE_RANGE_START_VARIABLE,
+      // needed when fresh and daily are both enabled, to avoid fresh data overlapping with daily
+      value: config.sourceTableType === 'GA4_EXPORT' && config.includedExportTypes.fresh && config.includedExportTypes.daily ? getDateRangeStartFresh(config) : undefined,
+      comment: 'Define the date range start for fresh export tables. Returns the first day with a fresh table but no daily table.',
+    },
+    {
+      type: 'variable',
+      name: constants.FRESH_MAX_EVENT_TIMESTAMP_VARIABLE,
+      // needed when fresh and intraday are both enabled, to set the timestamp boundary
+      value: config.sourceTableType === 'GA4_EXPORT' && config.includedExportTypes.fresh && config.includedExportTypes.intraday ? getFreshMaxEventTimestamp(config) : undefined,
+      comment: 'Get the latest event timestamp from fresh export tables. Used as the boundary between fresh and intraday data.',
     },
     {
       type: 'variable',
@@ -190,7 +282,7 @@ const setPreOperations = (config) => {
     {
       type: 'create',
       // create table statement only needed with schema lock
-      value: config.sourceTableType === 'GA4_EXPORT' && config.schemaLock ? createSchemaLockTable(config) : undefined,
+      value: !config.test && config.sourceTableType === 'GA4_EXPORT' && config.schemaLock ? createSchemaLockTable(config) : undefined,
       comment: 'Lock the schema to a specific version by creating a table copy from the selected day\'s export.'
     },
   ];