@ken-e/dataform-youtube 0.0.2

package/includes/definitions/sources/stg_ytc_traffic_source.js

@@ -0,0 +1,107 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_traffic_source", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "traffic_source_type"],
+    },
+    assertions: {
+      // make sure rows have unique dimensions
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "country_code",
+          "traffic_source_type",
+          "traffic_source_detail",
+        ],
+      ],
+      // make sure source partition and data dates match
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    description: "Staging table for YouTube Channel Traffic Source data",
+    assertions: {
+      // make sure rows have unique dimensions
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "country_code",
+          "traffic_source_type",
+          "traffic_source_detail",
+        ],
+      ],
+      // make sure source partition and data dates match
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    description: "Staging table for YouTube Channel Traffic Source data",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+select
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d', date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,
+  channel_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  traffic_source_type,
+  traffic_source_detail,
+  views,
+  watch_time_minutes,
+  average_view_duration_seconds,
+  average_view_duration_percentage,
+  red_views,
+  red_watch_time_minutes
+from ${ctx.ref(t.database, t.schema, "p_channel_traffic_source_a2_" + t.suffix)}
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/ytc_annotation.js

@@ -0,0 +1,65 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_annotation", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "annotation_type"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Annotation Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+select
+  stg.*,
+  annot.annotation_type_name,
+  ifnull(titles.video_title, stg.video_id) as video_title
+from ${ctx.ref("stg_ytc_annotation")} as stg
+left join ${ctx.ref("stg_ytc_lu_annotation_type")} as annot
+  using (annotation_type)
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+where interaction_date > interaction_date_checkpoint
+
+`,
+    );
+};

package/includes/definitions/ytc_basic.js

@@ -0,0 +1,74 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_basic", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Basic Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+-- Calculate date checkpoint for incremental updates.
+
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds,
+    subscribers_gained - subscribers_lost as subscribers_net,
+  from ${ctx.ref("stg_ytc_basic")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using
+  (video_id)
+
+`,
+    );
+};

package/includes/definitions/ytc_cards.js

@@ -0,0 +1,66 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_cards", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "card_type"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Cards Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+select
+  stg.* except (source_partition_date),
+  ct.card_type_name,
+  ifnull(titles.video_title, stg.video_id) as video_title
+from ${ctx.ref("stg_ytc_cards")} as stg
+left join ${ctx.ref("stg_ytc_lu_card_type")} as ct
+  using (card_type)
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+where interaction_date > interaction_date_checkpoint
+
+
+`,
+    );
+};

package/includes/definitions/ytc_combined.js

@@ -0,0 +1,113 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_combined", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Combined Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+-- Calculate date checkpoint for incremental updates.
+
+
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds,
+  from ${ctx.ref("stg_ytc_combined")}
+  where interaction_date > interaction_date_checkpoint
+),
+
+int_with_titles as (
+  select
+    int_ex_titles.*,
+    ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+  from int_ex_titles
+  left join ${ctx.ref(
+    config.titlesProject,
+    config.titlesDataset,
+    config.titlesTable,
+  )} as titles
+  using
+    (video_id)
+),
+
+int_with_playback_location as (
+  select
+    int_with_titles.*,
+    playback_location.playback_location_name
+  from int_with_titles
+  left join ${ctx.ref("stg_ytc_lu_playback_location")} as playback_location
+  using
+    (playback_location_type)
+),
+
+int_with_device_type as (
+  select
+    int_with_playback_location.*,
+    device_types.device_name
+  from int_with_playback_location
+  left join ${ctx.ref("stg_ytc_lu_device_types")} as device_types
+  using
+    (device_type)
+),
+
+int_with_traffic_source as (
+  select
+    int_with_device_type.*,
+    traffic_sources.traffic_source_name
+  from int_with_device_type
+  left join ${ctx.ref("stg_ytc_lu_traffic_sources")} as traffic_sources
+  using
+    (traffic_source_type)
+)
+
+select
+  int_with_traffic_source.*,
+  operating_systems.operating_system_name
+from int_with_traffic_source
+left join ${ctx.ref("stg_ytc_lu_operating_systems")} as operating_systems
+using
+  (operating_system)
+
+`,
+    );
+};

package/includes/definitions/ytc_demographics.js

@@ -0,0 +1,69 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_demographics", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "age_group", "gender"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Demographics Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date)
+  from ${ctx.ref("stg_ytc_demographics")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_demographics_views.js

@@ -0,0 +1,80 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_demographics_views", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "age_group", "gender"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Demographics Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    stg.* except (source_partition_date),
+    ytc_basic.views * stg.views_percentage/100 as views_with_demographics,
+    ytc_basic.views,
+  from ${ctx.ref("stg_ytc_demographics")} as stg
+  left join ${ctx.ref("ytc_basic")} as ytc_basic
+    using (
+      interaction_date,
+      channel_id,
+      video_id,
+      live_or_on_demand,
+      subscribed_status,
+      country_code
+    )
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_device_os.js

@@ -0,0 +1,80 @@
+/**
+ * This model has been prepared as part of the MARKETING PROPELLER project.
+ * Copyright (C) 2024 by DiveTeam, LLC
+ *
+ * Marketing Propeller models are not officially maintained, and are distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_device_os", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "device_type", "operating_system"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Device and OS Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
+  from ${ctx.ref("stg_ytc_device_os")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  dt.device_name,
+  os.operating_system_name,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref("stg_ytc_lu_device_types")} as dt
+  using (device_type)
+left join ${ctx.ref("stg_ytc_lu_operating_systems")} as os
+  using (operating_system)
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+
+`,
+    );
+};