ga4-export-fixer 0.2.4 → 0.2.5-dev.1

package/README.md

@@ -42,6 +42,7 @@ The **ga4_events_enhanced** table comes with features such as these:
 - **Custom timestamp support** – Optionally use a custom event parameter as the primary timestamp, with automatic fallback to `event_timestamp`
 - **Schema lock** – Lock the table schema to a specific GA4 export date to prevent schema drift
 - **Data freshness tracking** – `data_is_final` flag and `export_type` label on every row
+- **Selective date range re-processing** – Re-process a subset of data without a full table rebuild, using `incrementalStartOverride` and `incrementalEndOverride`
 - **Timezone-aware datetime** – `event_datetime` converted to a configurable IANA timezone
 - **Column descriptions** – Full column-level documentation included in the Dataform table configuration, reflecting the specific configuration used to build the table
 

package/constants.js

@@ -2,6 +2,7 @@ const constants = {
     DATE_RANGE_START_VARIABLE: 'date_range_start',
     INTRADAY_DATE_RANGE_START_VARIABLE: 'intraday_date_range_start',
     DATE_RANGE_END_VARIABLE: 'date_range_end',
+    LAST_PARTITION_DATE_VARIABLE: 'last_partition_date',
     DATE_COLUMN: 'event_date',
     DEFAULT_EVENTS_TABLE_NAME: 'ga4_events_enhanced',
     TABLE_DESCRIPTION_SUFFIX: 'Created by the ga4-export-fixer package.',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.2.4",
+  "version": "0.2.5-dev.1",
   "description": "",
   "main": "index.js",
   "files": [

package/preOperations.js

@@ -6,6 +6,22 @@ const declareVariable = (variable, value) => {
 );`;
 };
 
+// Get the last partition date from the result table
+const getLastPartitionDate = (config) => {
+  const informationSchemaPath = config.self.replace(
+    /`?([^`]+)\.([^`]+)\.[^`]+`?$/,
+    '`$1.$2.INFORMATION_SCHEMA.PARTITIONS`'
+  );
+  const tableName = config.self.replace(/`/g, '').split('.').pop();
+
+  return `select 
+  max(parse_date('%Y%m%d', partition_id))
+from 
+  ${informationSchemaPath}
+where 
+  table_name = '${tableName}' and partition_id != '__NULL__'`;
+};
+
 // Define the date range start for incremental and full refresh
 const getDateRangeStart = (config) => {
   if (config.incremental) {
@@ -22,7 +38,7 @@ const getDateRangeStart = (config) => {
   from
     ${config.self}
   where
-    ${constants.DATE_COLUMN} > current_date()-${config.preOperations.numberOfPreviousDaysToScan}
+    ${constants.DATE_COLUMN} > ${constants.LAST_PARTITION_DATE_VARIABLE}-${config.preOperations.numberOfPreviousDaysToScan}
   group by
     ${constants.DATE_COLUMN}
 )
@@ -40,39 +56,36 @@ where
 };
 
 // Define the date range start for incremental refresh with intraday tables
+// Uses INFORMATION_SCHEMA.TABLES to avoid scanning actual table data
 const getDateRangeStartIntraday = (config) => {
   const getStartDate = () => {
-    // with incremental refresh, use the checkpoint variable and check tables created starting from that date
     if (config.incremental) {
       return `greatest(${constants.DATE_RANGE_START_VARIABLE}, current_date()-5)`;
     }
-    // otherwise, just scan the last 5 days
     return 'current_date()-5';
   };
 
   const startDate = getStartDate();
 
   if (config.includedExportTypes.intraday) {
+    const informationSchemaPath = config.sourceTable.replace(
+      /`?([^`]+)\.([^`]+)\.[^`]+`?$/,
+      '`$1.$2.INFORMATION_SCHEMA.TABLES`'
+    );
+
     return `with export_statuses as (
     select
-      safe_cast(regexp_extract(_table_suffix, r'\\d+') as date format 'YYYYMMDD') as date,
+      safe_cast(regexp_extract(table_name, r'\\d+') as date format 'YYYYMMDD') as date,
       case
-        when _table_suffix like 'intraday_%' then 'intraday'
+        when table_name like 'events_intraday_%' then 'intraday'
         else 'daily'
-      end as export_type,
+      end as export_type
     from
-      ${config.sourceTable}
+      ${informationSchemaPath}
     where
-      -- check tables that are newer than the date range start or the last 5 days (if not incremental refresh)
-      (
-        regexp_extract(_table_suffix, r'\\d+') between 
-        cast(${startDate} as string format 'YYYYMMDD') 
-        and cast(current_date() as string format 'YYYYMMDD')
-      )
-      -- only include tables that are from the daily or intraday exports
-      and regexp_contains(_table_suffix, r'^(intraday_)?\\d{8}$')
-    group by 
-      date, export_type
+      regexp_contains(table_name, r'^events_(intraday_)?\\d{8}$')
+      and safe_cast(regexp_extract(table_name, r'\\d+') as date format 'YYYYMMDD')
+        between ${startDate} and current_date()
   ),
   statuses_by_day as (
     select
@@ -135,6 +148,12 @@ const setPreOperations = (config) => {
 
   // define the pre operations
   const preOperations = [
+    {
+      type: 'variable',
+      name: constants.LAST_PARTITION_DATE_VARIABLE,
+      value: config.incremental ? getLastPartitionDate(config) : undefined,
+      comment: 'Get the last partition date from the result table. Used to anchor the incremental date checkpoint scan window to the table\'s actual data.',
+    },
     {
       type: 'variable',
       name: constants.DATE_RANGE_START_VARIABLE,