ga4-export-fixer 0.2.5-dev.0 → 0.2.5-dev.2

package/README.md

@@ -299,9 +299,10 @@ The `onSchemaChange: "EXTEND"` setting updates the result table schema on increm
 | ------------------------------------------ | ----------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `preOperations.dateRangeStartFullRefresh`  | string (SQL date) | `'date(2000, 1, 1)'` | Start date for full refresh                                                                                                                                                                         |
 | `preOperations.dateRangeEnd`               | string (SQL date) | `'current_date()'`   | End date for queries                                                                                                                                                                                |
-| `preOperations.numberOfPreviousDaysToScan` | integer           | `10`                 | Number of previous days to scan from the result table when determining the incremental refresh start checkpoint. A higher value is required if the table updates have fallen behind for some reason |
+| `preOperations.numberOfPreviousDaysToScan` | integer           | `10`                 | Number of days to scan backwards from the result table's last partition when determining the incremental refresh start checkpoint. Needs to cover the number of days that can still contain not final `(data_is_final = false)` data                                                                    |
 | `preOperations.incrementalStartOverride`   | string (SQL date) | `undefined`          | Override the incremental start date to re-process a specific range                                                                                                                                  |
 | `preOperations.incrementalEndOverride`     | string (SQL date) | `undefined`          | Override the incremental end date to re-process a specific range                                                                                                                                    |
+| `preOperations.numberOfDaysToProcess`      | integer           | `undefined`          | Limit each run to N days of data. When set, the end date becomes `start + N - 1` (capped at `current_date()`). When `undefined`, `dateRangeEnd` is used as-is. `incrementalEndOverride` takes priority  |
 
 
 **`eventParamsToColumns`** — each item in the array is an object:

package/constants.js

@@ -2,6 +2,7 @@ const constants = {
     DATE_RANGE_START_VARIABLE: 'date_range_start',
     INTRADAY_DATE_RANGE_START_VARIABLE: 'intraday_date_range_start',
     DATE_RANGE_END_VARIABLE: 'date_range_end',
+    LAST_PARTITION_DATE_VARIABLE: 'last_partition_date',
     DATE_COLUMN: 'event_date',
     DEFAULT_EVENTS_TABLE_NAME: 'ga4_events_enhanced',
     TABLE_DESCRIPTION_SUFFIX: 'Created by the ga4-export-fixer package.',

package/defaultConfig.js

@@ -24,6 +24,7 @@ const baseConfig = {
         incrementalStartOverride: undefined,
         incrementalEndOverride: undefined,
         numberOfPreviousDaysToScan: 10,
+        numberOfDaysToProcess: undefined,
     },
 };
 

package/helpers.js

@@ -196,6 +196,9 @@ const ga4ExportDateFilters = (config) => {
       return constants.DATE_RANGE_END_VARIABLE;
     }
     // full refresh
+    if (config.preOperations.numberOfDaysToProcess !== undefined) {
+      return `least(${config.preOperations.dateRangeStartFullRefresh}+${config.preOperations.numberOfDaysToProcess}-1, current_date())`;
+    }
     return config.preOperations.dateRangeEnd;
   };
 
@@ -274,7 +277,9 @@ const incrementalDateFilter = (config) => {
 
   // full refresh mode
   const fullRefreshStart = config?.preOperations?.dateRangeStartFullRefresh || baseConfig.preOperations.dateRangeStartFullRefresh;
-  const fullRefreshEnd = config?.preOperations?.dateRangeEnd || baseConfig.preOperations.dateRangeEnd;
+  const fullRefreshEnd = config?.preOperations?.numberOfDaysToProcess !== undefined
+    ? `least(${fullRefreshStart}+${config.preOperations.numberOfDaysToProcess}-1, current_date())`
+    : (config?.preOperations?.dateRangeEnd || baseConfig.preOperations.dateRangeEnd);
 
   return setDateRange(fullRefreshStart, fullRefreshEnd);
 };

package/inputValidation.js

@@ -69,6 +69,12 @@ const validateBaseConfig = (config) => {
     if (typeof config.preOperations.dateRangeEnd !== 'string' || !config.preOperations.dateRangeEnd.trim()) {
         throw new Error(`config.preOperations.dateRangeEnd must be a non-empty string (SQL date expression). Received: ${JSON.stringify(config.preOperations.dateRangeEnd)}`);
     }
+    if (config.preOperations.numberOfDaysToProcess !== undefined) {
+        const nd = config.preOperations.numberOfDaysToProcess;
+        if (typeof nd !== 'number' || isNaN(nd) || !Number.isInteger(nd) || nd < 1) {
+            throw new Error(`config.preOperations.numberOfDaysToProcess must be a positive integer when defined. Received: ${JSON.stringify(nd)}`);
+        }
+    }
     if (config.preOperations.incrementalStartOverride !== undefined && config.preOperations.incrementalStartOverride !== null && config.preOperations.incrementalStartOverride !== '') {
         if (typeof config.preOperations.incrementalStartOverride !== 'string' || !config.preOperations.incrementalStartOverride.trim()) {
             throw new Error(`config.preOperations.incrementalStartOverride must be a non-empty string when provided. Received: ${JSON.stringify(config.preOperations.incrementalStartOverride)}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.2.5-dev.0",
+  "version": "0.2.5-dev.2",
   "description": "",
   "main": "index.js",
   "files": [

package/preOperations.js

@@ -6,6 +6,22 @@ const declareVariable = (variable, value) => {
 );`;
 };
 
+// Get the last partition date from the result table
+const getLastPartitionDate = (config) => {
+  const informationSchemaPath = config.self.replace(
+    /`?([^`]+)\.([^`]+)\.[^`]+`?$/,
+    '`$1.$2.INFORMATION_SCHEMA.PARTITIONS`'
+  );
+  const tableName = config.self.replace(/`/g, '').split('.').pop();
+
+  return `select 
+    max(parse_date('%Y%m%d', partition_id))
+  from 
+    ${informationSchemaPath}
+  where 
+    table_name = '${tableName}' and partition_id != '__NULL__'`;
+};
+
 // Define the date range start for incremental and full refresh
 const getDateRangeStart = (config) => {
   if (config.incremental) {
@@ -22,7 +38,9 @@ const getDateRangeStart = (config) => {
   from
     ${config.self}
   where
-    ${constants.DATE_COLUMN} > current_date()-${config.preOperations.numberOfPreviousDaysToScan}
+    -- the scan is relative to the last partition date in the table
+    -- takes into account cases where table updates have fallen behind
+    ${constants.DATE_COLUMN} > ${constants.LAST_PARTITION_DATE_VARIABLE}-${config.preOperations.numberOfPreviousDaysToScan}
   group by
     ${constants.DATE_COLUMN}
 )
@@ -102,7 +120,11 @@ const getDateRangeEnd = (config) => {
     return `select ${config.preOperations.incrementalEndOverride}`;
   }
 
-  // otherwise, use the default logic
+  // if a number of days to process is capped, adjust the end date accordingly
+  if (config.preOperations.numberOfDaysToProcess !== undefined) {
+    return `select least(${constants.DATE_RANGE_START_VARIABLE}+${config.preOperations.numberOfDaysToProcess}-1, current_date())`;
+  }
+
   return `select ${config.preOperations.dateRangeEnd}`;
 };
 
@@ -132,6 +154,12 @@ const setPreOperations = (config) => {
 
   // define the pre operations
   const preOperations = [
+    {
+      type: 'variable',
+      name: constants.LAST_PARTITION_DATE_VARIABLE,
+      value: config.incremental ? getLastPartitionDate(config) : undefined,
+      comment: `Get the last partition date from the result table. Reduces the number of rows scanned when defining the ${constants.DATE_RANGE_START_VARIABLE} variable.`,
+    },
     {
       type: 'variable',
       name: constants.DATE_RANGE_START_VARIABLE,