@ken-e/dataform-youtube 0.0.2 → 0.0.4

package/includes/definitions/sources/stg_ytc_demographics.js

@@ -2,23 +2,25 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_demographics", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "age_group", "gender"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,42 +31,21 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Demographics data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_demographics")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -76,7 +57,11 @@ select
   gender,
   views_percentage
 from ${ctx.ref(t.database, t.schema, "p_channel_demographics_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_device_os.js

@@ -2,23 +2,25 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_device_os", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "device_type", "operating_system"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,35 +31,14 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Device and OS data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_device_os")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
@@ -65,7 +46,7 @@ declare source_date_checkpoint default (
 
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -82,7 +63,11 @@ select
   red_views,
   red_watch_time_minutes
 from ${ctx.ref(t.database, t.schema, "p_channel_device_os_a2_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_end_screens.js

@@ -2,52 +2,33 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_end_screens", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "end_screen_element_type"],
     },
     description: "Staging table for YouTube Channel End Screen data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_end_screens")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -61,7 +42,11 @@ select
   end_screen_element_impressions,
   end_screen_element_clicks
 from ${ctx.ref(t.database, t.schema, "p_channel_end_screens_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_list_basic.js

@@ -3,24 +3,25 @@
  */
 
 const column_descriptions = require("../../column_descriptions");
+const helpers = require("../../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_list_basic", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "playlist_id",
           "video_id",
@@ -30,36 +31,15 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Playlist Basic Report Table - Staging",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_list_basic")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
@@ -67,7 +47,7 @@ declare source_date_checkpoint default (
 
 select 
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d',date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,  
   channel_id,
@@ -83,7 +63,11 @@ select
   playlist_saves_added,
   playlist_saves_removed
 from ${ctx.ref(t.database, t.schema, "p_playlist_basic_a1_" + t.suffix)} 
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_list_combined.js

@@ -2,22 +2,24 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_list_combined", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "device_type", "operating_system"],
     },
     assertions: {
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "playlist_id",
           "video_id",
@@ -30,41 +32,21 @@ module.exports = (config) => {
           "operating_system",
         ],
       ],
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Playlist Combined data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_list_combined")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -84,7 +66,11 @@ select
   playlist_saves_added,
   playlist_saves_removed
 from ${ctx.ref(t.database, t.schema, "p_playlist_combined_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_list_device_os.js

@@ -2,22 +2,24 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_list_device_os", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "device_type", "operating_system"],
     },
     assertions: {
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "playlist_id",
           "video_id",
@@ -28,35 +30,15 @@ module.exports = (config) => {
           "operating_system",
         ],
       ],
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description:
       "Staging table for YouTube Channel Playlist Device and OS data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_list_device_os")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
@@ -64,7 +46,7 @@ declare source_date_checkpoint default (
 
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -82,7 +64,11 @@ select
   playlist_saves_added,
   playlist_saves_removed
 from ${ctx.ref(t.database, t.schema, "p_playlist_device_os_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_list_playback.js

@@ -2,22 +2,24 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_list_playback", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "playback_location_type"],
     },
     assertions: {
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "playlist_id",
           "video_id",
@@ -28,42 +30,22 @@ module.exports = (config) => {
           "playback_location_detail",
         ],
       ],
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description:
       "Staging table for YouTube Channel Playlist Playback Location data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_list_playback")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -81,7 +63,11 @@ select
   playlist_saves_added,
   playlist_saves_removed
 from ${ctx.ref(t.database, t.schema, "p_playlist_playback_location_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),