@ken-e/dataform-youtube 0.0.2

package/includes/definitions/ytc_end_screens.js

@@ -0,0 +1,65 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_end_screens", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "end_screen_element_type"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel End Screen Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+select
+  stg.* except (source_partition_date),
+  et.end_screen_element_type_name,
+  ifnull(titles.video_title, stg.video_id) as video_title
+from ${ctx.ref("stg_ytc_end_screens")} as stg
+left join ${ctx.ref("stg_ytc_lu_end_screen_element_type")} as et
+  using (end_screen_element_type)
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+where interaction_date > interaction_date_checkpoint
+
+  `,
+    );
+};

package/includes/definitions/ytc_list_basic.js

@@ -0,0 +1,84 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_list_basic", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Playlist Basic Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    playlist_saves_added - playlist_saves_removed as playlist_saves_net
+  from ${ctx.ref("stg_ytc_list_basic")}
+  where interaction_date > interaction_date_checkpoint
+),
+
+int_with_video_titles as (
+  select
+    int_ex_titles.*,
+    ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+  from int_ex_titles
+  left join ${ctx.ref(
+    config.titlesProject,
+    config.titlesDataset,
+    config.titlesTable,
+  )} as titles
+  using (video_id)
+)
+
+select
+  int_with_video_titles.*,
+  ifnull(playlists.playlist_title, int_with_video_titles.playlist_id) as playlist_title
+from int_with_video_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.playlistTable,
+      )} as playlists
+using (playlist_id)
+
+
+`,
+    );
+};

package/includes/definitions/ytc_list_combined.js

@@ -0,0 +1,97 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_list_combined", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Playlist Combined Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+with int_with_lookup_names as (
+  select
+    base.* except(source_partition_date, device_type, operating_system, playback_location_type, traffic_source_type),
+    base.device_type,
+    base.operating_system,
+    base.playback_location_type,
+    base.traffic_source_type,
+    dt.device_name,
+    os.operating_system_name,
+    pl.playback_location_name,
+    ts.traffic_source_name,
+    playlist_saves_added - playlist_saves_removed as playlist_saves_net,
+  from ${ctx.ref("stg_ytc_list_combined")} as base
+  left join ${ctx.ref("stg_ytc_lu_device_types")} as dt
+    using (device_type)
+  left join ${ctx.ref("stg_ytc_lu_operating_systems")} as os
+    using (operating_system)
+  left join ${ctx.ref("stg_ytc_lu_playback_location")} as pl
+    using (playback_location_type)
+  left join ${ctx.ref("stg_ytc_lu_traffic_sources")} as ts
+    using (traffic_source_type)
+  where interaction_date > interaction_date_checkpoint
+),
+
+int_with_video_titles as (
+  select
+    int_with_lookup_names.*,
+    ifnull(titles.video_title, int_with_lookup_names.video_id) as video_title
+  from int_with_lookup_names
+  left join ${ctx.ref(
+    config.titlesProject,
+    config.titlesDataset,
+    config.titlesTable,
+  )} as titles
+  using (video_id)
+)
+
+select
+  int_with_video_titles.*,
+  ifnull(playlists.playlist_title, int_with_video_titles.playlist_id) as playlist_title
+from int_with_video_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.playlistTable,
+      )} as playlists
+using (playlist_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_list_device_os.js

@@ -0,0 +1,90 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_list_device_os", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id", "device_type", "operating_system"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description:
+      "YT Channel Playlist Device and OS Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+with int_with_device_os_names as (
+  select
+    base.* except(source_partition_date, device_type, operating_system),
+    base.device_type,
+    base.operating_system,
+    dt.device_name,
+    os.operating_system_name,
+    playlist_saves_added - playlist_saves_removed as playlist_saves_net
+  from ${ctx.ref("stg_ytc_list_device_os")} as base
+  left join ${ctx.ref("stg_ytc_lu_device_types")} as dt
+    using (device_type)
+  left join ${ctx.ref("stg_ytc_lu_operating_systems")} as os
+    using (operating_system)
+  where interaction_date > interaction_date_checkpoint
+),
+
+int_with_video_titles as (
+  select
+    int_with_device_os_names.*,
+    ifnull(titles.video_title, int_with_device_os_names.video_id) as video_title
+  from int_with_device_os_names
+  left join ${ctx.ref(
+    config.titlesProject,
+    config.titlesDataset,
+    config.titlesTable,
+  )} as titles
+  using (video_id)
+)
+
+select
+  int_with_video_titles.*,
+  ifnull(playlists.playlist_title, int_with_video_titles.playlist_id) as playlist_title
+from int_with_video_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.playlistTable,
+      )} as playlists
+using (playlist_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_list_playback.js

@@ -0,0 +1,86 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_list_playback", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id", "playback_location_type"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description:
+      "YT Channel Playlist Playback Location Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+with int_with_playback_names as (
+  select 
+    base.* except(source_partition_date, playback_location_type),
+    base.playback_location_type,
+    pl.playback_location_name,
+    playlist_saves_added - playlist_saves_removed as playlist_saves_net
+  from ${ctx.ref("stg_ytc_list_playback")} as base
+  left join ${ctx.ref("stg_ytc_lu_playback_location")} as pl
+    using (playback_location_type)
+  where interaction_date > interaction_date_checkpoint
+),
+
+int_with_video_titles as (
+  select
+    int_with_playback_names.*,
+    ifnull(titles.video_title, int_with_playback_names.video_id) as video_title
+  from int_with_playback_names
+  left join ${ctx.ref(
+    config.titlesProject,
+    config.titlesDataset,
+    config.titlesTable,
+  )} as titles
+  using (video_id)
+)
+
+select
+  int_with_video_titles.*,
+  ifnull(playlists.playlist_title, int_with_video_titles.playlist_id) as playlist_title
+from int_with_video_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.playlistTable,
+      )} as playlists
+using (playlist_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_list_province.js

@@ -0,0 +1,83 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_list_province", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Playlist Province Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- Add calculated fields and join with title tables
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    playlist_saves_added - playlist_saves_removed as playlist_saves_net
+  from ${ctx.ref("stg_ytc_list_province")}
+  where interaction_date > interaction_date_checkpoint
+),
+
+int_with_video_titles as (
+  select
+    int_ex_titles.*,
+    ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+  from int_ex_titles
+  left join ${ctx.ref(
+    config.titlesProject,
+    config.titlesDataset,
+    config.titlesTable,
+  )} as titles
+  using (video_id)
+)
+
+select
+  int_with_video_titles.*,
+  ifnull(playlists.playlist_title, int_with_video_titles.playlist_id) as playlist_title
+from int_with_video_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.playlistTable,
+      )} as playlists
+using (playlist_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_list_traffic_source.js

@@ -0,0 +1,91 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_list_traffic_source", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id", "traffic_source_type"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description:
+      "YT Channel Playlist Traffic Source Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    playlist_saves_added - playlist_saves_removed as playlist_saves_net
+  from ${ctx.ref("stg_ytc_list_traffic_source")}
+  where interaction_date > interaction_date_checkpoint
+),
+
+int_with_traffic_source as (
+  select
+    int_ex_titles.*,
+    ts.traffic_source_name
+  from int_ex_titles
+  left join ${ctx.ref("stg_ytc_lu_traffic_sources")} as ts
+  using (traffic_source_type)
+),
+
+int_with_video_titles as (
+  select
+    int_with_traffic_source.*,
+    ifnull(titles.video_title, int_with_traffic_source.video_id) as video_title
+  from int_with_traffic_source
+  left join ${ctx.ref(
+    config.titlesProject,
+    config.titlesDataset,
+    config.titlesTable,
+  )} as titles
+  using (video_id)
+)
+
+select
+  int_with_video_titles.*,
+  ifnull(playlists.playlist_title, int_with_video_titles.playlist_id) as playlist_title
+from int_with_video_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.playlistTable,
+      )} as playlists
+using (playlist_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_playback.js

@@ -0,0 +1,73 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_playback", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "playback_location_type"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Playback Location Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
+  from ${ctx.ref("stg_ytc_playback")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  pl.playback_location_name,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref("stg_ytc_lu_playback_location")} as pl
+  using (playback_location_type)
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+
+  `,
+    );
+};