@ken-e/dataform-youtube 0.0.2 → 0.0.4

package/includes/column_descriptions.js

@@ -1,95 +1,137 @@
-/**
- * Column Descriptions
- * Sets all variables that should be customized to the project.
- */
-
-const column_descriptions = {
-    ad_impressions: "The number of verified ad impressions served.",
-    ad_type: "The type of ad displayed (e.g., auction_display, auction_instream, reserved_instream).",
-    age_group: "Age group of the logged-in viewers (e.g., 13-17, 18-24, 25-34, etc.).",
-    annotation_click_through_rate: "The ratio of clicked annotations to total clickable annotation impressions (CTR).",
-    annotation_clickable_impressions: "The number of annotation impressions that could be clicked.",
-    annotation_clicks: "The total number of clicks on annotations.",
-    annotation_closable_impressions: "The total number of closable annotation impressions displayed.",
-    annotation_close_rate: "The ratio of closed annotations to total annotation impressions.",
-    annotation_closes: "The number of times annotations were closed by users.",
-    annotation_id: "Unique identifier for the annotation.",
-    annotation_impressions: "The total number of annotation impressions displayed.",
-    annotation_type: "Integer index indicating the type of annotation.",
-    annotation_type_name: "String name of the annotation type.",
-    average_view_duration: "The average duration (in seconds) of video playbacks.",
-    average_view_duration_percentage: "The average percentage of a video watched during a playback.",
-    average_view_duration_seconds: "The average duration in seconds that videos were watched.",
-    card_click_rate: "The ratio of card clicks to card impressions (CTR).",
-    card_clicks: "The number of times cards were clicked.",
-    card_id: "Unique identifier for the card.",
-    card_impressions: "The number of times cards were displayed (impressions).",
-    card_teaser_click_rate: "The ratio of card teaser clicks to card teaser impressions (CTR).",
-    card_teaser_clicks: "The number of times card teasers were clicked.",
-    card_teaser_impressions: "The number of times card teasers were displayed (impressions).",
-    card_type: "Integer index indicating the type of card.",
-    card_type_name: "String name of the card type.",
-    channel_id: "The ID of the YouTube channel associated with the metrics.",
-    claimed_status: "Indicates whether the video content was claimed ('claimed' or 'unclaimed').",
-    comments: "The number of times users commented on videos.",
-    country_code: "The ISO 3166-1 alpha-2 country code where the interactions occurred.",
-    cpm: "Estimated gross revenue per 1000 ad impressions (CPM, USD).",
-    device_name: "String name of the device type.",
-    device_type: "The type of device used by the viewer (e.g., DESKTOP, MOBILE, TABLET, TV, GAME_CONSOLE).",
-    dislikes: "The number of times users disliked videos (negative ratings).",
-    end_screen_element_id: "Unique identifier for the end screen element.",
-    end_screen_element_type: "Integer index indicating the type of end screen element.",
-    end_screen_element_type_name: "String name of the end screen element type.",
-    estimated_ad_revenue: "Estimated net revenue (USD) specifically from Google-sold advertising sources (subject to month-end adjustment).",
-    estimated_minutes_watched: "The estimated total number of minutes users watched videos.",
-    estimated_monetized_playbacks: "The number of playbacks where at least one ad impression was shown (includes pre-rolls even if video wasn't watched).",
-    estimated_red_minutes_watched: "The estimated number of minutes YouTube Premium members watched videos.",
-    estimated_revenue: "Total estimated net revenue (USD) from Google-sold ads and non-ad sources (subject to month-end adjustment).",
-    gender: "Gender of the logged-in viewers.",
-    gross_revenue: "Estimated gross revenue (USD) from Google-sold or DoubleClick-partner-sold ads (before revenue shares/ownership splits, subject to month-end adjustment).",
-    interaction_date: "Date of interaction in YYYY-MM-DD format, based on Pacific Time (PST/PDT).",
-    likes: "The number of times users liked videos (positive ratings).",
-    live_or_on_demand: "Indicates if the interaction was during a live broadcast ('LIVE') or with on-demand content ('ON_DEMAND').",
-    monetized_playbacks: "DEPRECATED. Use estimated_monetized_playbacks instead.",
-    operating_system: "The operating system used by the viewer (e.g., ANDROID, IOS, LINUX, MACINTOSH, WINDOWS).",
-    operating_system_name: "String name of the operating system.",
-    playback_based_cpm: "Estimated gross revenue per 1000 monetized playbacks (Playback-based CPM, USD).",
-    playback_location: "Type of location where playback occured (text).",
-    playback_location_type: "Integer index indicating type of location where playback occured.",
-    playback_location_name: "String name of the playback location.",
-    playlist_id: "Unique identifier for the YouTube playlist.",
-    playlist_saves_net: "The number of times users saved videos to their playlists less the number of times they removed them.",
-    playlist_starts: "Times the playlist was started.",
-    playlist_title: "Title of the YouTube playlist.",
-    province_code: "The ISO code for the province or state where the interactions occurred.",
-    red_views: "The number of times YouTube Premium members viewed videos.",
-    row_max_duration_seconds: "Maximum duration in seconds for the row's if all videos were played to the end. Provided simplify downstream calculation of percentage of videos watched.",
-    shares: "The number of times users shared videos via the 'Share' button.",
-    sharing_service: "The service used to share the video (e.g., FACEBOOK, WHATSAPP, REDDIT, EMAIL).",
-    sharing_service_name: "String name of the sharing service.",
-    subscribed_status: "Indicates whether the viewer was subscribed to the channel ('SUBSCRIBED' or 'UNSUBSCRIBED').",
-    subscribers_gained: "The number of new subscribers gained.",
-    subscribers_lost: "The number of subscribers lost (unsubscribes).",
-    subscribers_net: "Net change in subscribers (subscribers gained minus subscribers lost).",
-    subtitle_language: "The language code of the video subtitle.",
-    subtitle_language_autotranslated: "Boolean indicating if the subtitle was automatically translated.",
-    traffic_source_detail: "Specific details about the traffic source (e.g., search term, related video ID, external URL).",
-    traffic_source_name: "String name of the traffic source type.",
-    traffic_source_type: "The type of referrer through which viewers reached the video (e.g., YT_SEARCH, RELATED_VIDEO, EXT_URL, SUBSCRIBER).",
-    uploader_type: "Indicates if metrics relate to content uploaded by the owner ('SELF') or claimed third-party content ('THIRD_PARTY').",
-    video_id: "The ID of a YouTube video.",
-    video_title: "Title of the YouTube video.",
-    videos_added_to_playlists: "The number of times videos were added to any YouTube playlist (owner's or others').",
-    videos_removed_from_playlists: "The number of times videos were removed from any YouTube playlist (owner's or others').",
-    views: "The total number of times videos were viewed. In playlist reports, counts views within the playlist context.",
-    views_percentage: "Percentage of total views for the time period that were logged-in.",
-    views_with_demographics: "Number of views from users with demographic information available.",
-    watch_time_minutes: "The number of minutes that users watched videos.",
-    youtube_product: "The YouTube property where the interaction occurred (e.g., CORE, GAMING, KIDS, MUSIC)."
-};
-
-
-// Export project vars
-module.exports = {
-    column_descriptions
-};
+/**
+ * Column Descriptions
+ * Sets all variables that should be customized to the project.
+ */
+
+const column_descriptions = {
+  ad_impressions: "The number of verified ad impressions served.",
+  ad_type:
+    "The type of ad displayed (e.g., auction_display, auction_instream, reserved_instream).",
+  age_group:
+    "Age group of the logged-in viewers (e.g., 13-17, 18-24, 25-34, etc.).",
+  annotation_click_through_rate:
+    "The ratio of clicked annotations to total clickable annotation impressions (CTR).",
+  annotation_clickable_impressions:
+    "The number of annotation impressions that could be clicked.",
+  annotation_clicks: "The total number of clicks on annotations.",
+  annotation_closable_impressions:
+    "The total number of closable annotation impressions displayed.",
+  annotation_close_rate:
+    "The ratio of closed annotations to total annotation impressions.",
+  annotation_closes: "The number of times annotations were closed by users.",
+  annotation_id: "Unique identifier for the annotation.",
+  annotation_impressions:
+    "The total number of annotation impressions displayed.",
+  annotation_type: "Integer index indicating the type of annotation.",
+  annotation_type_name: "String name of the annotation type.",
+  average_view_duration:
+    "The average duration (in seconds) of video playbacks.",
+  average_view_duration_percentage:
+    "The average percentage of a video watched during a playback.",
+  average_view_duration_seconds:
+    "The average duration in seconds that videos were watched.",
+  card_click_rate: "The ratio of card clicks to card impressions (CTR).",
+  card_clicks: "The number of times cards were clicked.",
+  card_id: "Unique identifier for the card.",
+  card_impressions: "The number of times cards were displayed (impressions).",
+  card_teaser_click_rate:
+    "The ratio of card teaser clicks to card teaser impressions (CTR).",
+  card_teaser_clicks: "The number of times card teasers were clicked.",
+  card_teaser_impressions:
+    "The number of times card teasers were displayed (impressions).",
+  card_type: "Integer index indicating the type of card.",
+  card_type_name: "String name of the card type.",
+  channel_id: "The ID of the YouTube channel associated with the metrics.",
+  claimed_status:
+    "Indicates whether the video content was claimed ('claimed' or 'unclaimed').",
+  comments: "The number of times users commented on videos.",
+  country_code:
+    "The ISO 3166-1 alpha-2 country code where the interactions occurred.",
+  cpm: "Estimated gross revenue per 1000 ad impressions (CPM, USD).",
+  device_name: "String name of the device type.",
+  device_type:
+    "The type of device used by the viewer (e.g., DESKTOP, MOBILE, TABLET, TV, GAME_CONSOLE).",
+  dislikes: "The number of times users disliked videos (negative ratings).",
+  end_screen_element_id: "Unique identifier for the end screen element.",
+  end_screen_element_type:
+    "Integer index indicating the type of end screen element.",
+  end_screen_element_type_name: "String name of the end screen element type.",
+  estimated_ad_revenue:
+    "Estimated net revenue (USD) specifically from Google-sold advertising sources (subject to month-end adjustment).",
+  estimated_minutes_watched:
+    "The estimated total number of minutes users watched videos.",
+  estimated_monetized_playbacks:
+    "The number of playbacks where at least one ad impression was shown (includes pre-rolls even if video wasn't watched).",
+  estimated_red_minutes_watched:
+    "The estimated number of minutes YouTube Premium members watched videos.",
+  estimated_revenue:
+    "Total estimated net revenue (USD) from Google-sold ads and non-ad sources (subject to month-end adjustment).",
+  gender: "Gender of the logged-in viewers.",
+  gross_revenue:
+    "Estimated gross revenue (USD) from Google-sold or DoubleClick-partner-sold ads (before revenue shares/ownership splits, subject to month-end adjustment).",
+  interaction_date:
+    "Date of interaction in YYYY-MM-DD format, based on Pacific Time (PST/PDT).",
+  likes: "The number of times users liked videos (positive ratings).",
+  live_or_on_demand:
+    "Indicates if the interaction was during a live broadcast ('LIVE') or with on-demand content ('ON_DEMAND').",
+  monetized_playbacks: "DEPRECATED. Use estimated_monetized_playbacks instead.",
+  operating_system:
+    "The operating system used by the viewer (e.g., ANDROID, IOS, LINUX, MACINTOSH, WINDOWS).",
+  operating_system_name: "String name of the operating system.",
+  playback_based_cpm:
+    "Estimated gross revenue per 1000 monetized playbacks (Playback-based CPM, USD).",
+  playback_location: "Type of location where playback occured (text).",
+  playback_location_type:
+    "Integer index indicating type of location where playback occured.",
+  playback_location_name: "String name of the playback location.",
+  playlist_id: "Unique identifier for the YouTube playlist.",
+  playlist_saves_net:
+    "The number of times users saved videos to their playlists less the number of times they removed them.",
+  playlist_starts: "Times the playlist was started.",
+  playlist_title: "Title of the YouTube playlist.",
+  province_code:
+    "The ISO code for the province or state where the interactions occurred.",
+  red_views: "The number of times YouTube Premium members viewed videos.",
+  row_max_duration_seconds:
+    "Maximum duration in seconds for the row's if all videos were played to the end. Provided simplify downstream calculation of percentage of videos watched.",
+  shares: "The number of times users shared videos via the 'Share' button.",
+  sharing_service:
+    "The service used to share the video (e.g., FACEBOOK, WHATSAPP, REDDIT, EMAIL).",
+  sharing_service_name: "String name of the sharing service.",
+  subscribed_status:
+    "Indicates whether the viewer was subscribed to the channel ('SUBSCRIBED' or 'UNSUBSCRIBED').",
+  subscribers_gained: "The number of new subscribers gained.",
+  subscribers_lost: "The number of subscribers lost (unsubscribes).",
+  subscribers_net:
+    "Net change in subscribers (subscribers gained minus subscribers lost).",
+  subtitle_language: "The language code of the video subtitle.",
+  subtitle_language_autotranslated:
+    "Boolean indicating if the subtitle was automatically translated.",
+  traffic_source_detail:
+    "Specific details about the traffic source (e.g., search term, related video ID, external URL).",
+  traffic_source_name: "String name of the traffic source type.",
+  traffic_source_type:
+    "The type of referrer through which viewers reached the video (e.g., YT_SEARCH, RELATED_VIDEO, EXT_URL, SUBSCRIBER).",
+  uploader_type:
+    "Indicates if metrics relate to content uploaded by the owner ('SELF') or claimed third-party content ('THIRD_PARTY').",
+  video_id: "The ID of a YouTube video.",
+  video_title: "Title of the YouTube video.",
+  videos_added_to_playlists:
+    "The number of times videos were added to any YouTube playlist (owner's or others').",
+  videos_removed_from_playlists:
+    "The number of times videos were removed from any YouTube playlist (owner's or others').",
+  views:
+    "The total number of times videos were viewed. In playlist reports, counts views within the playlist context.",
+  views_percentage:
+    "Percentage of total views for the time period that were logged-in.",
+  views_with_demographics:
+    "Number of views from users with demographic information available.",
+  watch_time_minutes: "The number of minutes that users watched videos.",
+  youtube_product:
+    "The YouTube property where the interaction occurred (e.g., CORE, GAMING, KIDS, MUSIC).",
+};
+
+// Export project vars
+module.exports = {
+  column_descriptions,
+};

package/includes/definitions/sources/stg_ytc_annotation.js

@@ -2,23 +2,25 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_annotation", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "annotation_type"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,42 +31,21 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Annotation data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_annotation")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -82,7 +63,11 @@ select
   annotation_clicks,
   annotation_closes
 from ${ctx.ref(t.database, t.schema, "p_channel_annotations_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 
 `;
         })

package/includes/definitions/sources/stg_ytc_basic.js

@@ -3,24 +3,25 @@
  */
 
 const column_descriptions = require("../../column_descriptions");
+const helpers = require("../../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_basic", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,43 +30,22 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Basic Report Table - Staging",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_basic")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select 
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d',date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,  
   channel_id,
@@ -101,7 +81,11 @@ select
   red_views,
   red_watch_time_minutes,
 from ${ctx.ref(t.database, t.schema, "p_channel_basic_a2_" + t.suffix)} 
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_cards.js

@@ -2,23 +2,25 @@
  * Copyright (C) 2025 by KEN-E, LLC
  */
 
+const helpers = require("../../helpers");
+
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_cards", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "card_type"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -29,42 +31,21 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     description: "Staging table for YouTube Channel Cards data",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_cards")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
           return `
 select
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -81,7 +62,11 @@ select
   card_clicks,
   card_teaser_clicks
 from ${ctx.ref(t.database, t.schema, "p_channel_cards_a1_" + t.suffix)}
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),

package/includes/definitions/sources/stg_ytc_combined.js

@@ -3,24 +3,25 @@
  */
 
 const column_descriptions = require("../../column_descriptions");
+const helpers = require("../../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("stg_ytc_combined", {
     type: "incremental",
     database: config.target.database,
-    schema: config.datasetStaging,
+    schema: config.target.datasetStaging,
     protected: config.protected,
     tags: ["youtube", "source", "staging", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     assertions: {
       // make sure rows have unique dimensions
       uniqueKeys: [
         [
-          "interaction_date",
+          "data_date",
           "channel_id",
           "video_id",
           "live_or_on_demand",
@@ -33,36 +34,15 @@ module.exports = (config) => {
         ],
       ],
       // make sure source partition and data dates match
-      rowConditions: ["interaction_date = source_partition_date"],
+      // rowConditions: ["interaction_date = source_partition_date"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Combined Report Table - Staging",
   })
-    .preOps(
-      (ctx) => `
-declare source_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set source_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
-  )}
-  `,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated source partitions
+      return `${helpers.staging_preops(ctx, config, "stg_ytc_combined")}`;
+    })
     .query((ctx) =>
       config.sources
         .map((t) => {
@@ -70,7 +50,7 @@ declare source_date_checkpoint default (
 
 select 
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d',date) as interaction_date,
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,  
   channel_id,
@@ -89,7 +69,11 @@ select
   red_views,
   red_watch_time_minutes,
 from ${ctx.ref(t.database, t.schema, "p_channel_combined_a2_" + t.suffix)} 
-where _PARTITIONDATE > source_date_checkpoint
+    ${ctx.when(
+      ctx.incremental(),
+      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
+      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+    )}
 `;
         })
         .join(" union all "),