@ken-e/dataform-youtube 0.0.2

package/includes/definitions/sources/stg_ytc_end_screens.js

@@ -0,0 +1,69 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_end_screens", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "end_screen_element_type"],
+    },
+    description: "Staging table for YouTube Channel End Screen data",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+select
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d', date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,
+  channel_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  end_screen_element_type,
+  end_screen_element_id,
+  end_screen_element_click_rate,
+  end_screen_element_impressions,
+  end_screen_element_clicks
+from ${ctx.ref(t.database, t.schema, "p_channel_end_screens_a1_" + t.suffix)}
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/sources/stg_ytc_list_basic.js

@@ -0,0 +1,91 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_list_basic", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id"],
+    },
+    assertions: {
+      // make sure rows have unique dimensions
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "playlist_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "country_code",
+        ],
+      ],
+      // make sure source partition and data dates match
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Playlist Basic Report Table - Staging",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+
+select 
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d',date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,  
+  channel_id,
+  playlist_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  views,
+  watch_time_minutes,
+  average_view_duration_seconds,
+  playlist_starts,
+  playlist_saves_added,
+  playlist_saves_removed
+from ${ctx.ref(t.database, t.schema, "p_playlist_basic_a1_" + t.suffix)} 
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/sources/stg_ytc_list_combined.js

@@ -0,0 +1,92 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_list_combined", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id", "device_type", "operating_system"],
+    },
+    assertions: {
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "playlist_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "country_code",
+          "playback_location_type",
+          "traffic_source_type",
+          "device_type",
+          "operating_system",
+        ],
+      ],
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    description: "Staging table for YouTube Channel Playlist Combined data",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+select
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d', date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,
+  channel_id,
+  playlist_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  playback_location_type,
+  traffic_source_type,
+  device_type,
+  operating_system,
+  views,
+  watch_time_minutes,
+  average_view_duration_seconds,
+  playlist_starts,
+  playlist_saves_added,
+  playlist_saves_removed
+from ${ctx.ref(t.database, t.schema, "p_playlist_combined_a1_" + t.suffix)}
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/sources/stg_ytc_list_device_os.js

@@ -0,0 +1,90 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_list_device_os", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id", "device_type", "operating_system"],
+    },
+    assertions: {
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "playlist_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "country_code",
+          "device_type",
+          "operating_system",
+        ],
+      ],
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    description:
+      "Staging table for YouTube Channel Playlist Device and OS data",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+
+select
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d', date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,
+  channel_id,
+  playlist_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  device_type,
+  operating_system,
+  views,
+  watch_time_minutes,
+  average_view_duration_seconds,
+  playlist_starts,
+  playlist_saves_added,
+  playlist_saves_removed
+from ${ctx.ref(t.database, t.schema, "p_playlist_device_os_a1_" + t.suffix)}
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/sources/stg_ytc_list_playback.js

@@ -0,0 +1,89 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_list_playback", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id", "playback_location_type"],
+    },
+    assertions: {
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "playlist_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "country_code",
+          "playback_location_type",
+          "playback_location_detail",
+        ],
+      ],
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    description:
+      "Staging table for YouTube Channel Playlist Playback Location data",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+select
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d', date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,
+  channel_id,
+  playlist_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  playback_location_type,
+  playback_location_detail,
+  views,
+  watch_time_minutes,
+  average_view_duration_seconds,
+  playlist_starts,
+  playlist_saves_added,
+  playlist_saves_removed
+from ${ctx.ref(t.database, t.schema, "p_playlist_playback_location_a1_" + t.suffix)}
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/sources/stg_ytc_list_province.js

@@ -0,0 +1,91 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_list_province", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id"],
+    },
+    assertions: {
+      // make sure rows have unique dimensions
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "playlist_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "province_code",
+        ],
+      ],
+      // make sure source partition and data dates match
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Playlist Province Report Table - Staging",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+select 
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d',date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,  
+  channel_id,
+  playlist_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  province_code,
+  views,
+  watch_time_minutes,
+  average_view_duration_seconds,
+  playlist_starts,
+  playlist_saves_added,
+  playlist_saves_removed
+from ${ctx.ref(t.database, t.schema, "p_playlist_province_a1_" + t.suffix)} 
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/sources/stg_ytc_list_traffic_source.js

@@ -0,0 +1,89 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_list_traffic_source", {
+    type: "incremental",
+    database: config.target.database,
+    schema: config.datasetStaging,
+    protected: config.protected,
+    tags: ["youtube", "source", "staging", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["playlist_id", "video_id", "traffic_source_type"],
+    },
+    assertions: {
+      uniqueKeys: [
+        [
+          "interaction_date",
+          "channel_id",
+          "playlist_id",
+          "video_id",
+          "live_or_on_demand",
+          "subscribed_status",
+          "country_code",
+          "traffic_source_type",
+          "traffic_source_detail",
+        ],
+      ],
+      rowConditions: ["interaction_date = source_partition_date"],
+    },
+    description:
+      "Staging table for YouTube Channel Playlist Traffic Source data",
+  })
+    .preOps(
+      (ctx) => `
+declare source_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set source_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(source_partition_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where source_partition_date > source_date_checkpoint`,
+  )}
+  `,
+    )
+    .query((ctx) =>
+      config.sources
+        .map((t) => {
+          return `
+select
+  _PARTITIONDATE as source_partition_date,
+  parse_date('%Y%m%d', date) as interaction_date,
+  "${t.schema}" as site_nm,
+  current_timestamp() as updated_at,
+  channel_id,
+  playlist_id,
+  video_id,
+  live_or_on_demand,
+  subscribed_status,
+  country_code,
+  traffic_source_type,
+  traffic_source_detail,
+  views,
+  watch_time_minutes,
+  average_view_duration_seconds,
+  playlist_starts,
+  playlist_saves_added,
+  playlist_saves_removed
+from ${ctx.ref(t.database, t.schema, "p_playlist_traffic_source_a1_" + t.suffix)}
+where _PARTITIONDATE > source_date_checkpoint
+`;
+        })
+        .join(" union all "),
+    );
+};

package/includes/definitions/sources/stg_ytc_lu_annotation_type.js

@@ -0,0 +1,37 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_lu_annotation_type", {
+    type: "table",
+    database: config.target.database,
+    schema: config.datasetOutput,
+    protected: config.protected,
+    tags: ["youtube", "lookup", "annotations"],
+    description:
+      "Lookup table for YouTube video annotation types and their names from YouTube Analytics API.",
+  }).query(
+    (ctx) =>
+      `
+
+select 
+  cast(annotation_type as int) as annotation_type,
+  annotation_type_name
+from unnest([
+  struct(0 as annotation_type, "Unknown" as annotation_type_name),
+  struct(1 as annotation_type, "Note" as annotation_type_name),
+  struct(3 as annotation_type, "Spotlight" as annotation_type_name),
+  struct(4 as annotation_type, "Title" as annotation_type_name),
+  struct(8 as annotation_type, "Speech bubble" as annotation_type_name),
+  struct(9 as annotation_type, "Label" as annotation_type_name),
+  struct(10 as annotation_type, "Branding watermark" as annotation_type_name),
+  struct(11 as annotation_type, "Featured video" as annotation_type_name),
+  struct(12 as annotation_type, "Featured playlist" as annotation_type_name),
+  struct(30 as annotation_type, "Call-to-Action" as annotation_type_name)
+]) as annotation_types
+
+`,
+  );
+};

package/includes/definitions/sources/stg_ytc_lu_card_type.js

@@ -0,0 +1,35 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("stg_ytc_lu_card_type", {
+    type: "table",
+    database: config.target.database,
+    schema: config.datasetOutput,
+    protected: config.protected,
+    tags: ["youtube", "lookup", "card_type"],
+    description: "Lookup table for YouTube card types and their descriptions.",
+  }).query(
+    (ctx) =>
+      `
+
+select 
+  cast(card_type as int) as card_type,
+  card_type_name
+from unnest([
+  struct(0 as card_type, "Unknown" as card_type_name),
+  struct(60 as card_type, "Link" as card_type_name),
+  struct(61 as card_type, "Fundraising" as card_type_name),
+  struct(62 as card_type, "Video" as card_type_name),
+  struct(63 as card_type, "Playlist" as card_type_name),
+  struct(65 as card_type, "Fan Funding" as card_type_name),
+  struct(66 as card_type, "Merchandise" as card_type_name),
+  struct(68 as card_type, "Associated website" as card_type_name),
+  struct(69 as card_type, "Channel" as card_type_name)
+]) as card_types
+
+`,
+  );
+};