@ken-e/dataform-youtube 0.0.2 → 0.0.3

package/includes/definitions/sources/stg_ytc_list_traffic_source.js

@@ -2,6 +2,8 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_list_traffic_source", {
@@ -11,13 +13,13 @@ module.exports = (config) => {
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "traffic_source_type"],
     },
     assertions: {
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "playlist_id",
           "video_id",
@@ -28,42 +30,22 @@ module.exports = (config) => {
           "traffic_source_detail",
         ],
       ],
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description:
       "Staging table for YouTube Channel Playlist Traffic Source data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_list_traffic_source")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -81,7 +63,11 @@ select
   playlist_saves_added,
   playlist_saves_removed
 from ${ctx.ref(t.database, t.schema, "p_playlist_traffic_source_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_lu_operating_systems.js

@@ -13,7 +13,7 @@ module.exports = (config) => {
     description:
       "Lookup table for operating system types and their names from YouTube Analytics API.",
   }).query(
-    (ctx) =>
+    () =>
       `
 
 select 

package/includes/definitions/sources/stg_ytc_playback.js

@@ -2,6 +2,8 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_playback", {
@@ -11,14 +13,14 @@ module.exports = (config) => {
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "playback_location_type"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,35 +31,14 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Playback Location data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_playback")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
@@ -65,7 +46,7 @@ declare source_date_checkpoint default (
 
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -82,7 +63,11 @@ select
   red_views,
   red_watch_time_minutes
 from ${ctx.ref(t.database, t.schema, "p_channel_playback_location_a2_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_province.js

@@ -3,6 +3,7 @@
  */
 
 const column_descriptions = require("../../column_descriptions");
+const helpers = require("../../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
@@ -13,14 +14,14 @@ module.exports = (config) => {
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -30,43 +31,22 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Province Report Table - Staging",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-  select date("${config.startDate}")
-);
-
---Set the incremental update checkpoint based on current max partition value minus lookback. 
-set source_date_checkpoint = (
-  ${ctx.when(
-    ctx.incremental(),
-    `select
-      least(
-        (select date_sub(current_date(), interval ${config.daysBack} day)),
-        (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-      )`,
-    `select date("${config.startDate}")`,
-  )}
-);
-
-${ctx.when(
-  ctx.incremental(),
-  `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-)}
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_province")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select 
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d',date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -95,7 +75,11 @@ select
   red_views,
   red_watch_time_minutes
 from ${ctx.ref(t.database, t.schema, "p_channel_province_a2_" + t.suffix)} 
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_share_platform.js

@@ -2,6 +2,8 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_share_platform", {
@@ -11,14 +13,14 @@ module.exports = (config) => {
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "sharing_service"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -28,42 +30,21 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Share Platform data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_share_platform")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -74,7 +55,11 @@ select
   cast(sharing_service as int) as sharing_service,
   shares
 from ${ctx.ref(t.database, t.schema, "p_channel_sharing_service_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_subtitles.js

@@ -3,6 +3,7 @@
  */
 
 const column_descriptions = require("../../column_descriptions");
+const helpers = require("../../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
@@ -13,13 +14,13 @@ module.exports = (config) => {
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     assertions: {
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,42 +30,22 @@ module.exports = (config) => {
           "subtitle_language_autotranslated",
         ],
       ],
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Subtitles Report Table - Staging",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_subtitles")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select 
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d',date) as interaction_date,  
+  parse_date('%Y%m%d', date) as data_date,  
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -81,7 +62,11 @@ select
   red_views,
   red_watch_time_minutes
 from ${ctx.ref(t.database, t.schema, "p_channel_subtitles_a2_" + t.suffix)} 
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_traffic_source.js

@@ -2,6 +2,8 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_traffic_source", {
@@ -11,14 +13,14 @@ module.exports = (config) => {
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "traffic_source_type"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,60 +31,21 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
-    },
-    description: "Staging table for YouTube Channel Traffic Source data",
-    assertions: {
-      // make sure rows have unique dimensions
-      uniqueKeys: [
-        [
-          "interaction_date",
-          "channel_id",
-          "video_id",
-          "live_or_on_demand",
-          "subscribed_status",
-          "country_code",
-          "traffic_source_type",
-          "traffic_source_detail",
-        ],
-      ],
-      // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Traffic Source data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_traffic_source")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -99,7 +62,11 @@ select
   red_views,
   red_watch_time_minutes
 from ${ctx.ref(t.database, t.schema, "p_channel_traffic_source_a2_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/ytc_annotation.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_annotation", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "annotation_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Annotation Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_annotation"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -58,8 +38,13 @@ left join ${ctx.ref(
         config.titlesTable,
       )} as titles
 using (video_id)
-where interaction_date > interaction_date_checkpoint
-
+${ctx.when(
+  ctx.incremental(),
+  `where
+    data_date in unnest(partitions_to_update)`,
+  `where
+    data_date > date '${config.startDate}'`,
+)}
 `,
     );
 };

package/includes/definitions/ytc_basic.js

@@ -3,47 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_basic", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Basic Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
--- Calculate date checkpoint for incremental updates.
-
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_basic"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -54,7 +32,13 @@ with int_ex_titles as (
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds,
     subscribers_gained - subscribers_lost as subscribers_net,
   from ${ctx.ref("stg_ytc_basic")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select