ga4-export-fixer 0.2.5-dev.1 → 0.2.5

package/README.md

@@ -43,6 +43,7 @@ The **ga4_events_enhanced** table comes with features such as these:
 - **Schema lock** – Lock the table schema to a specific GA4 export date to prevent schema drift
 - **Data freshness tracking** – `data_is_final` flag and `export_type` label on every row
 - **Selective date range re-processing** – Re-process a subset of data without a full table rebuild, using `incrementalStartOverride` and `incrementalEndOverride`
+- **Batch processing** – Process large GA4 exports in smaller batches using the `numberOfDaysToProcess` configuration setting
 - **Timezone-aware datetime** – `event_datetime` converted to a configurable IANA timezone
 - **Column descriptions** – Full column-level documentation included in the Dataform table configuration, reflecting the specific configuration used to build the table
 
@@ -77,7 +78,7 @@ Include the package in the package.json file in your Dataform repository.
 {
   "dependencies": {
     "@dataform/core": "3.0.42",
-    "ga4-export-fixer": "0.2.4"
+    "ga4-export-fixer": "0.2.5"
   }
 }
 ```
@@ -299,9 +300,10 @@ The `onSchemaChange: "EXTEND"` setting updates the result table schema on increm
 | ------------------------------------------ | ----------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `preOperations.dateRangeStartFullRefresh`  | string (SQL date) | `'date(2000, 1, 1)'` | Start date for full refresh                                                                                                                                                                         |
 | `preOperations.dateRangeEnd`               | string (SQL date) | `'current_date()'`   | End date for queries                                                                                                                                                                                |
-| `preOperations.numberOfPreviousDaysToScan` | integer           | `10`                 | Number of previous days to scan from the result table when determining the incremental refresh start checkpoint. A higher value is required if the table updates have fallen behind for some reason |
+| `preOperations.numberOfPreviousDaysToScan` | integer           | `10`                 | Number of days to scan backwards from the result table's last partition when determining the incremental refresh start checkpoint. Needs to cover the number of days that can still contain not final `(data_is_final = false)` data                                                                    |
 | `preOperations.incrementalStartOverride`   | string (SQL date) | `undefined`          | Override the incremental start date to re-process a specific range                                                                                                                                  |
 | `preOperations.incrementalEndOverride`     | string (SQL date) | `undefined`          | Override the incremental end date to re-process a specific range                                                                                                                                    |
+| `preOperations.numberOfDaysToProcess`      | integer           | `undefined`          | Limit each run to N days of data. When set, the end date becomes `start + N - 1` (capped at `current_date()`). When `undefined`, `dateRangeEnd` is used as-is. `incrementalEndOverride` takes priority  |
 
 
 **`eventParamsToColumns`** — each item in the array is an object:

package/defaultConfig.js

@@ -24,6 +24,7 @@ const baseConfig = {
         incrementalStartOverride: undefined,
         incrementalEndOverride: undefined,
         numberOfPreviousDaysToScan: 10,
+        numberOfDaysToProcess: undefined,
     },
 };
 

package/helpers.js

@@ -196,6 +196,9 @@ const ga4ExportDateFilters = (config) => {
       return constants.DATE_RANGE_END_VARIABLE;
     }
     // full refresh
+    if (config.preOperations.numberOfDaysToProcess !== undefined) {
+      return `least(${config.preOperations.dateRangeStartFullRefresh}+${config.preOperations.numberOfDaysToProcess}-1, current_date())`;
+    }
     return config.preOperations.dateRangeEnd;
   };
 
@@ -274,7 +277,9 @@ const incrementalDateFilter = (config) => {
 
   // full refresh mode
   const fullRefreshStart = config?.preOperations?.dateRangeStartFullRefresh || baseConfig.preOperations.dateRangeStartFullRefresh;
-  const fullRefreshEnd = config?.preOperations?.dateRangeEnd || baseConfig.preOperations.dateRangeEnd;
+  const fullRefreshEnd = config?.preOperations?.numberOfDaysToProcess !== undefined
+    ? `least(${fullRefreshStart}+${config.preOperations.numberOfDaysToProcess}-1, current_date())`
+    : (config?.preOperations?.dateRangeEnd || baseConfig.preOperations.dateRangeEnd);
 
   return setDateRange(fullRefreshStart, fullRefreshEnd);
 };

package/inputValidation.js

@@ -69,6 +69,12 @@ const validateBaseConfig = (config) => {
     if (typeof config.preOperations.dateRangeEnd !== 'string' || !config.preOperations.dateRangeEnd.trim()) {
         throw new Error(`config.preOperations.dateRangeEnd must be a non-empty string (SQL date expression). Received: ${JSON.stringify(config.preOperations.dateRangeEnd)}`);
     }
+    if (config.preOperations.numberOfDaysToProcess !== undefined) {
+        const nd = config.preOperations.numberOfDaysToProcess;
+        if (typeof nd !== 'number' || isNaN(nd) || !Number.isInteger(nd) || nd < 1) {
+            throw new Error(`config.preOperations.numberOfDaysToProcess must be a positive integer when defined. Received: ${JSON.stringify(nd)}`);
+        }
+    }
     if (config.preOperations.incrementalStartOverride !== undefined && config.preOperations.incrementalStartOverride !== null && config.preOperations.incrementalStartOverride !== '') {
         if (typeof config.preOperations.incrementalStartOverride !== 'string' || !config.preOperations.incrementalStartOverride.trim()) {
             throw new Error(`config.preOperations.incrementalStartOverride must be a non-empty string when provided. Received: ${JSON.stringify(config.preOperations.incrementalStartOverride)}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.2.5-dev.1",
+  "version": "0.2.5",
   "description": "",
   "main": "index.js",
   "files": [

package/preOperations.js

@@ -15,11 +15,11 @@ const getLastPartitionDate = (config) => {
   const tableName = config.self.replace(/`/g, '').split('.').pop();
 
   return `select 
-  max(parse_date('%Y%m%d', partition_id))
-from 
-  ${informationSchemaPath}
-where 
-  table_name = '${tableName}' and partition_id != '__NULL__'`;
+    max(parse_date('%Y%m%d', partition_id))
+  from 
+    ${informationSchemaPath}
+  where 
+    table_name = '${tableName}' and partition_id != '__NULL__'`;
 };
 
 // Define the date range start for incremental and full refresh
@@ -38,6 +38,8 @@ const getDateRangeStart = (config) => {
   from
     ${config.self}
   where
+    -- the scan is relative to the last partition date in the table
+    -- takes into account cases where table updates have fallen behind
     ${constants.DATE_COLUMN} > ${constants.LAST_PARTITION_DATE_VARIABLE}-${config.preOperations.numberOfPreviousDaysToScan}
   group by
     ${constants.DATE_COLUMN}
@@ -118,7 +120,11 @@ const getDateRangeEnd = (config) => {
     return `select ${config.preOperations.incrementalEndOverride}`;
   }
 
-  // otherwise, use the default logic
+  // if a number of days to process is capped, adjust the end date accordingly
+  if (config.preOperations.numberOfDaysToProcess !== undefined) {
+    return `select least(${constants.DATE_RANGE_START_VARIABLE}+${config.preOperations.numberOfDaysToProcess}-1, current_date())`;
+  }
+
   return `select ${config.preOperations.dateRangeEnd}`;
 };
 
@@ -152,7 +158,7 @@ const setPreOperations = (config) => {
       type: 'variable',
       name: constants.LAST_PARTITION_DATE_VARIABLE,
       value: config.incremental ? getLastPartitionDate(config) : undefined,
-      comment: 'Get the last partition date from the result table. Used to anchor the incremental date checkpoint scan window to the table\'s actual data.',
+      comment: `Get the last partition date from the result table. Reduces the number of rows scanned when defining the ${constants.DATE_RANGE_START_VARIABLE} variable.`,
     },
     {
       type: 'variable',