@ken-e/dataform-youtube 0.0.2 → 0.0.4

package/includes/definitions/ytc_annotation.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_annotation", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "annotation_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Annotation Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_annotation"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -58,8 +38,13 @@ left join ${ctx.ref(
         config.titlesTable,
       )} as titles
 using (video_id)
-where interaction_date > interaction_date_checkpoint
-
+${ctx.when(
+  ctx.incremental(),
+  `where
+    data_date in unnest(partitions_to_update)`,
+  `where
+    data_date > date '${config.startDate}'`,
+)}
 `,
     );
 };

package/includes/definitions/ytc_basic.js

@@ -3,47 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_basic", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Basic Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
--- Calculate date checkpoint for incremental updates.
-
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_basic"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -54,7 +32,13 @@ with int_ex_titles as (
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds,
     subscribers_gained - subscribers_lost as subscribers_net,
   from ${ctx.ref("stg_ytc_basic")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_cards.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_cards", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "card_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Cards Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_cards"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -58,8 +38,13 @@ left join ${ctx.ref(
         config.titlesTable,
       )} as titles
 using (video_id)
-where interaction_date > interaction_date_checkpoint
-
+${ctx.when(
+  ctx.incremental(),
+  `where
+    data_date in unnest(partitions_to_update)`,
+  `where
+    data_date > date '${config.startDate}'`,
+)}
 
 `,
     );

package/includes/definitions/ytc_combined.js

@@ -3,48 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_combined", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Combined Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
--- Calculate date checkpoint for incremental updates.
-
-
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_combined"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -53,7 +30,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds,
   from ${ctx.ref("stg_ytc_combined")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_titles as (

package/includes/definitions/ytc_demographics.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_demographics", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "age_group", "gender"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Demographics Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_demographics"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -50,7 +30,13 @@ with int_ex_titles as (
   select 
     * except (source_partition_date)
   from ${ctx.ref("stg_ytc_demographics")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_demographics_views.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_demographics_views", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "age_group", "gender"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Demographics Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_demographics"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -54,14 +34,20 @@ with int_ex_titles as (
   from ${ctx.ref("stg_ytc_demographics")} as stg
   left join ${ctx.ref("ytc_basic")} as ytc_basic
     using (
-      interaction_date,
+      data_date,
       channel_id,
       video_id,
       live_or_on_demand,
       subscribed_status,
       country_code
     )
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_device_os.js

@@ -7,45 +7,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_device_os", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "device_type", "operating_system"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Device and OS Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_device_os"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -55,7 +35,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_device_os")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_end_screens.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_end_screens", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "end_screen_element_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel End Screen Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_end_screens"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -58,7 +38,13 @@ left join ${ctx.ref(
         config.titlesTable,
       )} as titles
 using (video_id)
-where interaction_date > interaction_date_checkpoint
+${ctx.when(
+  ctx.incremental(),
+  `where
+    data_date in unnest(partitions_to_update)`,
+  `where
+    data_date > date '${config.startDate}'`,
+)}
 
   `,
     );

package/includes/definitions/ytc_list_basic.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_basic", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.target.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Playlist Basic Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_basic"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     playlist_saves_added - playlist_saves_removed as playlist_saves_net
   from ${ctx.ref("stg_ytc_list_basic")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_video_titles as (