@ken-e/dataform-youtube 0.0.2 → 0.0.4

package/includes/definitions/ytc_list_combined.js

@@ -3,44 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_combined", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Playlist Combined Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_combined"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -65,7 +46,13 @@ with int_with_lookup_names as (
     using (playback_location_type)
   left join ${ctx.ref("stg_ytc_lu_traffic_sources")} as ts
     using (traffic_source_type)
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_video_titles as (

package/includes/definitions/ytc_list_device_os.js

@@ -3,45 +3,26 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_device_os", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "device_type", "operating_system"],
     },
     columns: column_descriptions.column_descriptions,
     description:
       "YT Channel Playlist Device and OS Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_device_os"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -58,7 +39,13 @@ with int_with_device_os_names as (
     using (device_type)
   left join ${ctx.ref("stg_ytc_lu_operating_systems")} as os
     using (operating_system)
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_video_titles as (

package/includes/definitions/ytc_list_playback.js

@@ -3,45 +3,26 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_playback", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "playback_location_type"],
     },
     columns: column_descriptions.column_descriptions,
     description:
       "YT Channel Playlist Playback Location Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_playback"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -54,7 +35,13 @@ with int_with_playback_names as (
   from ${ctx.ref("stg_ytc_list_playback")} as base
   left join ${ctx.ref("stg_ytc_lu_playback_location")} as pl
     using (playback_location_type)
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_video_titles as (

package/includes/definitions/ytc_list_province.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_province", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Playlist Province Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_province"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     playlist_saves_added - playlist_saves_removed as playlist_saves_net
   from ${ctx.ref("stg_ytc_list_province")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_video_titles as (

package/includes/definitions/ytc_list_traffic_source.js

@@ -3,45 +3,26 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_traffic_source", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "traffic_source_type"],
     },
     columns: column_descriptions.column_descriptions,
     description:
       "YT Channel Playlist Traffic Source Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_traffic_source"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -50,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     playlist_saves_added - playlist_saves_removed as playlist_saves_net
   from ${ctx.ref("stg_ytc_list_traffic_source")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_traffic_source as (

package/includes/definitions/ytc_playback.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_playback", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "playback_location_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Playback Location Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_playback"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_playback")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_province.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_province", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Province Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_province"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_province")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_share_platform.js

@@ -7,45 +7,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_share_platform", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "sharing_service"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Share Platform Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_share_platform"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -54,7 +34,13 @@ with int_ex_titles as (
   select 
     * except (source_partition_date)
   from ${ctx.ref("stg_ytc_share_platform")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_subtitles.js

@@ -3,44 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_subtitles", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Subtitles Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_subtitles"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -49,7 +30,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_subtitles")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_traffic_source.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_traffic_source", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "traffic_source_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Traffic Source Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_traffic_source"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_traffic_source")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select