ga4-export-fixer 0.2.5-dev.0 → 0.2.5-dev.1

package/constants.js

@@ -2,6 +2,7 @@ const constants = {
     DATE_RANGE_START_VARIABLE: 'date_range_start',
     INTRADAY_DATE_RANGE_START_VARIABLE: 'intraday_date_range_start',
     DATE_RANGE_END_VARIABLE: 'date_range_end',
+    LAST_PARTITION_DATE_VARIABLE: 'last_partition_date',
     DATE_COLUMN: 'event_date',
     DEFAULT_EVENTS_TABLE_NAME: 'ga4_events_enhanced',
     TABLE_DESCRIPTION_SUFFIX: 'Created by the ga4-export-fixer package.',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ga4-export-fixer",
-  "version": "0.2.5-dev.0",
+  "version": "0.2.5-dev.1",
   "description": "",
   "main": "index.js",
   "files": [

package/preOperations.js

@@ -6,6 +6,22 @@ const declareVariable = (variable, value) => {
 );`;
 };
 
+// Get the last partition date from the result table
+const getLastPartitionDate = (config) => {
+  const informationSchemaPath = config.self.replace(
+    /`?([^`]+)\.([^`]+)\.[^`]+`?$/,
+    '`$1.$2.INFORMATION_SCHEMA.PARTITIONS`'
+  );
+  const tableName = config.self.replace(/`/g, '').split('.').pop();
+
+  return `select 
+  max(parse_date('%Y%m%d', partition_id))
+from 
+  ${informationSchemaPath}
+where 
+  table_name = '${tableName}' and partition_id != '__NULL__'`;
+};
+
 // Define the date range start for incremental and full refresh
 const getDateRangeStart = (config) => {
   if (config.incremental) {
@@ -22,7 +38,7 @@ const getDateRangeStart = (config) => {
   from
     ${config.self}
   where
-    ${constants.DATE_COLUMN} > current_date()-${config.preOperations.numberOfPreviousDaysToScan}
+    ${constants.DATE_COLUMN} > ${constants.LAST_PARTITION_DATE_VARIABLE}-${config.preOperations.numberOfPreviousDaysToScan}
   group by
     ${constants.DATE_COLUMN}
 )
@@ -132,6 +148,12 @@ const setPreOperations = (config) => {
 
   // define the pre operations
   const preOperations = [
+    {
+      type: 'variable',
+      name: constants.LAST_PARTITION_DATE_VARIABLE,
+      value: config.incremental ? getLastPartitionDate(config) : undefined,
+      comment: 'Get the last partition date from the result table. Used to anchor the incremental date checkpoint scan window to the table\'s actual data.',
+    },
     {
       type: 'variable',
       name: constants.DATE_RANGE_START_VARIABLE,