@ken-e/dataform-youtube 0.0.9 → 0.0.11

package/includes/definitions/sources/stg_ytc_basic.js

@@ -39,23 +39,23 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_channel_basic_a2_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_channel_basic_a2_", "p_channel_basic_a3_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select 
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
-  current_timestamp() as updated_at,  
+  current_timestamp() as updated_at,
   channel_id,
   video_id,
   live_or_on_demand,
   subscribed_status,
   country_code,
-  views,
+  views,`;
+          const restColumns = `
   comments,
   likes,
   dislikes,
@@ -81,14 +81,32 @@ select
   videos_added_to_playlists,
   videos_removed_from_playlists,
   red_views,
-  red_watch_time_minutes,
-from ${ctx.ref(t.database, t.schema, "p_channel_basic_a2_" + t.suffix)} 
+  red_watch_time_minutes,`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a2 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_basic_a2_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a3 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_basic_a3_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_combined.js

@@ -43,40 +43,57 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_channel_combined_a2_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_channel_combined_a2_", "p_channel_combined_a3_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-
-select 
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
-  current_timestamp() as updated_at,  
+  current_timestamp() as updated_at,
   channel_id,
   video_id,
   live_or_on_demand,
   subscribed_status,
   country_code,
-  playback_location_type, 
-  traffic_source_type, 
-  device_type, 
+  playback_location_type,
+  traffic_source_type,
+  device_type,
   operating_system,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   average_view_duration_percentage,
   red_views,
-  red_watch_time_minutes,
-from ${ctx.ref(t.database, t.schema, "p_channel_combined_a2_" + t.suffix)} 
+  red_watch_time_minutes,`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a2 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_combined_a2_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
+    )}
+`,
+            `
+-- New a3 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_combined_a3_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
     )}
-`;
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_device_os.js

@@ -39,14 +39,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_channel_device_os_a2_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_channel_device_os_a2_", "p_channel_device_os_a3_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-
-select
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -58,19 +56,38 @@ select
   country_code,
   device_type,
   operating_system,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   average_view_duration_percentage,
   red_views,
-  red_watch_time_minutes
+  red_watch_time_minutes`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a2 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
 from ${ctx.ref(t.database, t.schema, "p_channel_device_os_a2_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
+    )}
+`,
+            `
+-- New a3 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_device_os_a3_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
     )}
-`;
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_list_basic.js

@@ -40,37 +40,54 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_playlist_basic_a1_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_playlist_basic_a1_", "p_playlist_basic_a2_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-
-select 
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
-  current_timestamp() as updated_at,  
+  current_timestamp() as updated_at,
   channel_id,
   playlist_id,
   video_id,
   live_or_on_demand,
   subscribed_status,
   country_code,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   playlist_starts,
   playlist_saves_added,
-  playlist_saves_removed
-from ${ctx.ref(t.database, t.schema, "p_playlist_basic_a1_" + t.suffix)} 
+  playlist_saves_removed`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a1 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_basic_a1_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
+    )}
+`,
+            `
+-- New a2 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_basic_a2_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
     )}
-`;
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_list_combined.js

@@ -40,13 +40,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_playlist_combined_a1_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_playlist_combined_a1_", "p_playlist_combined_a2_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -61,19 +60,38 @@ select
   traffic_source_type,
   device_type,
   operating_system,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   playlist_starts,
   playlist_saves_added,
-  playlist_saves_removed
+  playlist_saves_removed`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a1 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
 from ${ctx.ref(t.database, t.schema, "p_playlist_combined_a1_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a2 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_combined_a2_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_list_device_os.js

@@ -39,14 +39,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_playlist_device_os_a1_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_playlist_device_os_a1_", "p_playlist_device_os_a2_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-
-select
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -59,19 +57,38 @@ select
   country_code,
   device_type,
   operating_system,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   playlist_starts,
   playlist_saves_added,
-  playlist_saves_removed
+  playlist_saves_removed`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a1 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
 from ${ctx.ref(t.database, t.schema, "p_playlist_device_os_a1_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
+    )}
+`,
+            `
+-- New a2 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_device_os_a2_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
     )}
-`;
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_list_playback.js

@@ -39,13 +39,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_playlist_playback_location_a1_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_playlist_playback_location_a1_", "p_playlist_playback_location_a2_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -58,19 +57,38 @@ select
   country_code,
   playback_location_type,
   playback_location_detail,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   playlist_starts,
   playlist_saves_added,
-  playlist_saves_removed
+  playlist_saves_removed`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a1 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
 from ${ctx.ref(t.database, t.schema, "p_playlist_playback_location_a1_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a2 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_playback_location_a2_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_list_province.js

@@ -40,17 +40,16 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_playlist_province_a1_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_playlist_province_a1_", "p_playlist_province_a2_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select 
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
-  current_timestamp() as updated_at,  
+  current_timestamp() as updated_at,
   channel_id,
   playlist_id,
   video_id,
@@ -58,19 +57,38 @@ select
   subscribed_status,
   country_code,
   province_code,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   playlist_starts,
   playlist_saves_added,
-  playlist_saves_removed
-from ${ctx.ref(t.database, t.schema, "p_playlist_province_a1_" + t.suffix)} 
+  playlist_saves_removed`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a1 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_province_a1_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a2 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_province_a2_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_list_traffic_source.js

@@ -39,13 +39,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_playlist_traffic_source_a1_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_playlist_traffic_source_a1_", "p_playlist_traffic_source_a2_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -58,19 +57,38 @@ select
   country_code,
   traffic_source_type,
   traffic_source_detail,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   playlist_starts,
   playlist_saves_added,
-  playlist_saves_removed
+  playlist_saves_removed`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a1 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
 from ${ctx.ref(t.database, t.schema, "p_playlist_traffic_source_a1_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a2 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_playlist_traffic_source_a2_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_playback.js

@@ -39,14 +39,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_channel_playback_location_a2_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_channel_playback_location_a2_", "p_channel_playback_location_a3_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-
-select
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -58,19 +56,38 @@ select
   country_code,
   playback_location_type,
   playback_location_detail,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   average_view_duration_percentage,
   red_views,
-  red_watch_time_minutes
+  red_watch_time_minutes`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a2 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
 from ${ctx.ref(t.database, t.schema, "p_channel_playback_location_a2_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
+    )}
+`,
+            `
+-- New a3 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_playback_location_a3_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
     )}
-`;
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_province.js

@@ -40,13 +40,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_channel_province_a2_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_channel_province_a2_", "p_channel_province_a3_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select 
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -57,7 +56,8 @@ select
   subscribed_status,
   country_code,
   province_code,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   average_view_duration_percentage,
@@ -75,14 +75,32 @@ select
   card_clicks,
   card_teaser_clicks,
   red_views,
-  red_watch_time_minutes
-from ${ctx.ref(t.database, t.schema, "p_channel_province_a2_" + t.suffix)} 
+  red_watch_time_minutes`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a2 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_province_a2_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a3 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_province_a3_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_subtitles.js

@@ -39,15 +39,14 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_channel_subtitles_a2_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_channel_subtitles_a2_", "p_channel_subtitles_a3_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select 
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
-  parse_date('%Y%m%d', date) as data_date,  
+  parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
   current_timestamp() as updated_at,
   channel_id,
@@ -57,19 +56,38 @@ select
   country_code,
   subtitle_language,
   subtitle_language_autotranslated,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   average_view_duration_percentage,
   red_views,
-  red_watch_time_minutes
-from ${ctx.ref(t.database, t.schema, "p_channel_subtitles_a2_" + t.suffix)} 
+  red_watch_time_minutes`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a2 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_subtitles_a2_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a3 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_subtitles_a3_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/includes/definitions/sources/stg_ytc_traffic_source.js

@@ -39,13 +39,12 @@ module.exports = (config) => {
   })
     .preOps((ctx) => {
       // Get pre_operations to find updated source partitions
-      return `${helpers.staging_preops(ctx, config, "p_channel_traffic_source_a2_")}`;
+      return `${helpers.staging_preops(ctx, config, ["p_channel_traffic_source_a2_", "p_channel_traffic_source_a3_"])}`;
     })
     .query((ctx) =>
       config.sources
-        .map((t) => {
-          return `
-select
+        .flatMap((t) => {
+          const columns = `
   _PARTITIONDATE as source_partition_date,
   parse_date('%Y%m%d', date) as data_date,
   "${t.schema}" as site_nm,
@@ -57,19 +56,38 @@ select
   country_code,
   traffic_source_type,
   traffic_source_detail,
-  views,
+  views,`;
+          const restColumns = `
   watch_time_minutes,
   average_view_duration_seconds,
   average_view_duration_percentage,
   red_views,
-  red_watch_time_minutes
+  red_watch_time_minutes`;
+          const incrementalWhere = `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`;
+          return [
+            `
+-- Deprecated a2 table (data before cutover)
+select ${columns}
+  cast(null as int64) as engaged_views,${restColumns}
 from ${ctx.ref(t.database, t.schema, "p_channel_traffic_source_a2_" + t.suffix)}
     ${ctx.when(
       ctx.incremental(),
-      `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
-      `where date(_PARTITIONTIME) > date(self_checkpoint)`,
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) < date '2025-09-22'`,
     )}
-`;
+`,
+            `
+-- New a3 table (data from cutover onward)
+select ${columns}
+  engaged_views,${restColumns}
+from ${ctx.ref(t.database, t.schema, "p_channel_traffic_source_a3_" + t.suffix)}
+    ${ctx.when(
+      ctx.incremental(),
+      incrementalWhere,
+      `where date(_PARTITIONTIME) > date(self_checkpoint) and date(_PARTITIONTIME) >= date '2025-09-22'`,
+    )}
+`,
+          ];
         })
         .join(" union all "),
     );

package/index.js

@@ -107,12 +107,13 @@ module.exports = (config) => {
   };
 
   config.sources.map((s) => {
+    // Channel reports - deprecated a2 tables (data before 2025-09-22)
     declare({
       database: s.database,
       schema: s.schema,
       name: "p_channel_basic_a2_" + s.suffix,
       description:
-        "This report provides user activity statistics related to a channel and its videos.",
+        "This report provides user activity statistics related to a channel and its videos. (Deprecated: replaced by a3)",
     });
 
     declare({
@@ -120,7 +121,7 @@ module.exports = (config) => {
       schema: s.schema,
       name: "p_channel_combined_a2_" + s.suffix,
       description:
-        "This report provides user activity statistics related to a channel and its videos.",
+        "This report provides user activity statistics related to a channel and its videos. (Deprecated: replaced by a3)",
     });
 
     declare({
@@ -128,7 +129,7 @@ module.exports = (config) => {
       schema: s.schema,
       name: "p_channel_province_a2_" + s.suffix,
       description:
-        "This report provides user activity statistics for U.S. states and the District of Columbia.",
+        "This report provides user activity statistics for U.S. states and the District of Columbia. (Deprecated: replaced by a3)",
     });
 
     declare({
@@ -136,7 +137,7 @@ module.exports = (config) => {
       schema: s.schema,
       name: "p_channel_playback_location_a2_" + s.suffix,
       description:
-        "This report provides statistics related to the type of page or application where video playbacks occurred.",
+        "This report provides statistics related to the type of page or application where video playbacks occurred. (Deprecated: replaced by a3)",
     });
 
     declare({
@@ -144,17 +145,83 @@ module.exports = (config) => {
       schema: s.schema,
       name: "p_channel_traffic_source_a2_" + s.suffix,
       description:
-        "This report aggregates viewing statistics based on the manner in which viewers reached the channel's video content.",
+        "This report aggregates viewing statistics based on the manner in which viewers reached the channel's video content. (Deprecated: replaced by a3)",
     });
 
     declare({
       database: s.database,
       schema: s.schema,
       name: "p_channel_device_os_a2_" + s.suffix,
+      description:
+        "This report aggregates video viewing statistics based on viewers' operating systems and device types. (Deprecated: replaced by a3)",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_subtitles_a2_" + s.suffix,
+      description:
+        "This report provides statistics about the closed caption language usage during video views. (Deprecated: replaced by a3)",
+    });
+
+    // Channel reports - new a3 tables (data from 2025-09-22 onward)
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_basic_a3_" + s.suffix,
+      description:
+        "This report provides user activity statistics related to a channel and its videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_combined_a3_" + s.suffix,
+      description:
+        "This report provides user activity statistics related to a channel and its videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_province_a3_" + s.suffix,
+      description:
+        "This report provides user activity statistics for U.S. states and the District of Columbia.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_playback_location_a3_" + s.suffix,
+      description:
+        "This report provides statistics related to the type of page or application where video playbacks occurred.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_traffic_source_a3_" + s.suffix,
+      description:
+        "This report aggregates viewing statistics based on the manner in which viewers reached the channel's video content.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_device_os_a3_" + s.suffix,
       description:
         "This report aggregates video viewing statistics based on viewers' operating systems and device types.",
     });
 
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_subtitles_a3_" + s.suffix,
+      description:
+        "This report provides statistics about the closed caption language usage during video views.",
+    });
+
+    // Channel reports - unchanged (no version bump)
     declare({
       database: s.database,
       schema: s.schema,
@@ -195,18 +262,60 @@ module.exports = (config) => {
         "This report provides impressions and click-through statistics for end screen elements that display during a channel's videos.",
     });
 
+    // Playlist reports - deprecated a1 tables (data before 2025-09-22)
     declare({
       database: s.database,
       schema: s.schema,
-      name: "p_channel_subtitles_a2_" + s.suffix,
+      name: "p_playlist_basic_a1_" + s.suffix,
       description:
-        "This report provides statistics about the closed caption language usage during video views.",
+        "This report provides statistics related to users' interactions with a channel's playlists. (Deprecated: replaced by a2)",
     });
 
     declare({
       database: s.database,
       schema: s.schema,
-      name: "p_playlist_basic_a1_" + s.suffix,
+      name: "p_playlist_device_os_a1_" + s.suffix,
+      description:
+        "This report aggregates playlist viewing statistics based on viewers' operating systems and device types. (Deprecated: replaced by a2)",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_province_a1_" + s.suffix,
+      description:
+        "This report provides user activity statistics related to users' interactions with a channel's playlists for U.S. states and the District of Columbia. (Deprecated: replaced by a2)",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_playback_location_a1_" + s.suffix,
+      description:
+        "This report provides statistics related to the type of page or application where playlist playbacks occurred. (Deprecated: replaced by a2)",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_traffic_source_a1_" + s.suffix,
+      description:
+        "This report aggregates viewing statistics based on the manner in which viewers reached a channel's playlist videos. (Deprecated: replaced by a2)",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_combined_a1_" + s.suffix,
+      description:
+        "This report provides fine-grained playlist statistics by combining dimensions used in the playback location, traffic source, and device/OS reports. (Deprecated: replaced by a2)",
+    });
+
+    // Playlist reports - new a2 tables (data from 2025-09-22 onward)
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_basic_a2_" + s.suffix,
       description:
         "This report provides statistics related to users' interactions with a channel's playlists.",
     });
@@ -214,7 +323,7 @@ module.exports = (config) => {
     declare({
       database: s.database,
       schema: s.schema,
-      name: "p_playlist_device_os_a1_" + s.suffix,
+      name: "p_playlist_device_os_a2_" + s.suffix,
       description:
         "This report aggregates playlist viewing statistics based on viewers' operating systems and device types.",
     });
@@ -222,7 +331,7 @@ module.exports = (config) => {
     declare({
       database: s.database,
       schema: s.schema,
-      name: "p_playlist_province_a1_" + s.suffix,
+      name: "p_playlist_province_a2_" + s.suffix,
       description:
         "This report provides user activity statistics related to users' interactions with a channel's playlists for U.S. states and the District of Columbia.",
     });
@@ -230,7 +339,7 @@ module.exports = (config) => {
     declare({
       database: s.database,
       schema: s.schema,
-      name: "p_playlist_playback_location_a1_" + s.suffix,
+      name: "p_playlist_playback_location_a2_" + s.suffix,
       description:
         "This report provides statistics related to the type of page or application where playlist playbacks occurred.",
     });
@@ -238,7 +347,7 @@ module.exports = (config) => {
     declare({
       database: s.database,
       schema: s.schema,
-      name: "p_playlist_traffic_source_a1_" + s.suffix,
+      name: "p_playlist_traffic_source_a2_" + s.suffix,
       description:
         "This report aggregates viewing statistics based on the manner in which viewers reached a channel's playlist videos.",
     });
@@ -246,7 +355,7 @@ module.exports = (config) => {
     declare({
       database: s.database,
       schema: s.schema,
-      name: "p_playlist_combined_a1_" + s.suffix,
+      name: "p_playlist_combined_a2_" + s.suffix,
       description:
         "This report provides fine-grained playlist statistics by combining dimensions used in the playback location, traffic source, and device/OS reports.",
     });

package/package.json

@@ -2,7 +2,7 @@
   "name": "@ken-e/dataform-youtube",
   "dependencies": {
     "@dataform/core": "3.0.9",
-    "@ken-e/dataform-helpers": "0.0.9"
+    "@ken-e/dataform-helpers": "0.0.11"
   },
   "devDependencies": {
     "@eslint/js": "^9.18.0",
@@ -11,7 +11,7 @@
     "globals": "^15.14.0",
     "prettier": "3.4.2"
   },
-  "version": "0.0.9",
+  "version": "0.0.11",
   "description": "A dataform package to process Youtube data from the BQ Data Transfer.",
   "main": "index.js",
   "scripts": {