@ken-e/dataform-youtube 0.0.2

package/includes/definitions/ytc_province.js

@@ -0,0 +1,71 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_province", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Province Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
+  from ${ctx.ref("stg_ytc_province")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using
+  (video_id)
+
+    `,
+    );
+};

package/includes/definitions/ytc_share_platform.js

@@ -0,0 +1,76 @@
+/**
+ * This model has been prepared as part of the MARKETING PROPELLER project.
+ * Copyright (C) 2024 by DiveTeam, LLC
+ *
+ * Marketing Propeller models are not officially maintained, and are distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_share_platform", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "sharing_service"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Share Platform Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date)
+  from ${ctx.ref("stg_ytc_share_platform")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  ss.sharing_service_name,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref("stg_ytc_lu_sharing_services")} as ss
+  using (sharing_service)
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+
+`,
+    );
+};

package/includes/definitions/ytc_subtitles.js

@@ -0,0 +1,69 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_subtitles", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Subtitles Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
+  from ${ctx.ref("stg_ytc_subtitles")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using
+  (video_id)
+
+  `,
+    );
+};

package/includes/definitions/ytc_traffic_source.js

@@ -0,0 +1,73 @@
+/**
+ * Copyright (C) 2025 by KEN-E, LLC
+ */
+
+const column_descriptions = require("../column_descriptions");
+
+module.exports = (config) => {
+  // eslint-disable-next-line no-undef
+  return publish("ytc_traffic_source", {
+    type: "incremental",
+    schema: config.datasetIntermediate,
+    tags: ["youtube", "output", "daily"],
+    bigquery: {
+      partitionBy: "interaction_date",
+      clusterBy: ["video_id", "traffic_source_type"],
+    },
+    columns: column_descriptions.column_descriptions,
+    description: "YT Channel Traffic Source Report Table - Intermediate",
+  })
+    .preOps(
+      (ctx) => `
+  declare interaction_date_checkpoint default (
+    select date("${config.startDate}")
+  );
+
+  --Set the incremental update checkpoint based on current max partition value minus lookback. 
+  set interaction_date_checkpoint = (
+    ${ctx.when(
+      ctx.incremental(),
+      `select
+        least(
+          (select date_sub(current_date(), interval ${config.daysBack} day)),
+          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
+        )`,
+      `select date("${config.startDate}")`,
+    )}
+  );
+
+  ${ctx.when(
+    ctx.incremental(),
+    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
+  )}  
+`,
+    )
+    .query(
+      (ctx) => `
+
+-- determine max possible viewing seconds per row so that percent viewed can be calculated downstream
+with int_ex_titles as (
+  select 
+    * except (source_partition_date),
+    views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
+  from ${ctx.ref("stg_ytc_traffic_source")}
+  where interaction_date > interaction_date_checkpoint
+)
+
+select
+  int_ex_titles.*,
+  ts.traffic_source_name,
+  ifnull(titles.video_title, int_ex_titles.video_id) as video_title
+from int_ex_titles
+left join ${ctx.ref("stg_ytc_lu_traffic_sources")} as ts
+  using (traffic_source_type)
+left join ${ctx.ref(
+        config.titlesProject,
+        config.titlesDataset,
+        config.titlesTable,
+      )} as titles
+using (video_id)
+
+  `,
+    );
+};

package/includes/helpers.js

@@ -0,0 +1,8 @@
+/**
+ * HELPERS
+ * Javascript functions that are referenced throughout the models.
+ */
+
+function addTableSuffix(table) {
+  return table + config.sourceTableSuffix;
+}

package/includes/project_variables.js

@@ -0,0 +1,15 @@
+/**
+ * PROJECT VARIABLES
+ * Sets all variables that should be customized to the project.
+ */
+
+const column_descriptions = {
+    col_1: "",
+    col_2: ""
+};
+
+
+// Export project vars
+module.exports = {
+    column_descriptions
+};

package/index.js

@@ -0,0 +1,312 @@
+const stg_ytc_lu_playback_location = require("./includes/definitions/sources/stg_ytc_lu_playback_location");
+const stg_ytc_lu_traffic_sources = require("./includes/definitions/sources/stg_ytc_lu_traffic_sources");
+const stg_ytc_province = require("./includes/definitions/sources/stg_ytc_province");
+const stg_ytc_list_traffic_source = require("./includes/definitions/sources/stg_ytc_list_traffic_source");
+const stg_ytc_cards = require("./includes/definitions/sources/stg_ytc_cards");
+const stg_ytc_list_device_os = require("./includes/definitions/sources/stg_ytc_list_device_os");
+const stg_ytc_annotation = require("./includes/definitions/sources/stg_ytc_annotation");
+const stg_ytc_traffic_source = require("./includes/definitions/sources/stg_ytc_traffic_source");
+const stg_ytc_lu_annotation_type = require("./includes/definitions/sources/stg_ytc_lu_annotation_type");
+const stg_ytc_lu_sharing_services = require("./includes/definitions/sources/stg_ytc_lu_sharing_services");
+const stg_ytc_lu_end_screen_element_type = require("./includes/definitions/sources/stg_ytc_lu_end_screen_element_type");
+const stg_ytc_lu_device_types = require("./includes/definitions/sources/stg_ytc_lu_device_types");
+const stg_ytc_playback = require("./includes/definitions/sources/stg_ytc_playback");
+const stg_ytc_lu_operating_systems = require("./includes/definitions/sources/stg_ytc_lu_operating_systems");
+const stg_ytc_end_screens = require("./includes/definitions/sources/stg_ytc_end_screens");
+const stg_ytc_list_basic = require("./includes/definitions/sources/stg_ytc_list_basic");
+const stg_ytc_demographics = require("./includes/definitions/sources/stg_ytc_demographics");
+const stg_ytc_lu_card_type = require("./includes/definitions/sources/stg_ytc_lu_card_type");
+const stg_ytc_list_province = require("./includes/definitions/sources/stg_ytc_list_province");
+const stg_ytc_list_combined = require("./includes/definitions/sources/stg_ytc_list_combined");
+const stg_ytc_basic = require("./includes/definitions/sources/stg_ytc_basic");
+const stg_ytc_subtitles = require("./includes/definitions/sources/stg_ytc_subtitles");
+const stg_ytc_combined = require("./includes/definitions/sources/stg_ytc_combined");
+const stg_ytc_device_os = require("./includes/definitions/sources/stg_ytc_device_os");
+const stg_ytc_share_platform = require("./includes/definitions/sources/stg_ytc_share_platform");
+const stg_ytc_list_playback = require("./includes/definitions/sources/stg_ytc_list_playback");
+const ytc_list_province = require("./includes/definitions/ytc_list_province");
+const ytc_list_traffic_source = require("./includes/definitions/ytc_list_traffic_source");
+const ytc_demographics = require("./includes/definitions/ytc_demographics");
+const ytc_traffic_source = require("./includes/definitions/ytc_traffic_source");
+const ytc_combined = require("./includes/definitions/ytc_combined");
+const ytc_demographics_views = require("./includes/definitions/ytc_demographics_views");
+const ytc_list_basic = require("./includes/definitions/ytc_list_basic");
+const ytc_list_combined = require("./includes/definitions/ytc_list_combined");
+const ytc_cards = require("./includes/definitions/ytc_cards");
+const ytc_share_platform = require("./includes/definitions/ytc_share_platform");
+const ytc_playback = require("./includes/definitions/ytc_playback");
+const ytc_list_device_os = require("./includes/definitions/ytc_list_device_os");
+const ytc_device_os = require("./includes/definitions/ytc_device_os");
+const ytc_end_screens = require("./includes/definitions/ytc_end_screens");
+const ytc_annotation = require("./includes/definitions/ytc_annotation");
+const ytc_province = require("./includes/definitions/ytc_province");
+const ytc_subtitles = require("./includes/definitions/ytc_subtitles");
+const ytc_basic = require("./includes/definitions/ytc_basic");
+const ytc_list_playback = require("./includes/definitions/ytc_list_playback");
+/**
+ * Processes data sources configuration.
+ *
+ * @param {Object} config - Configuration object.
+ * @param {Array<Object>} config.sources - An array of data source definitions.
+ * @param {string} config.sources[].schema - The schema name for the source.
+ * @param {string} config.sources[].database - The database name for the source.
+ *
+ * @example
+ * const config = {
+ *   sources: [
+ *     { schema: "schema1", suffix: "11111", database: "db1" },
+ *     { schema: "schema2", suffix: "22222", database: "db2" },
+ *     { schema: "schema3", suffix: "33333", database: "db1" }
+ *   ],
+ *  start_date: "2025-10-01",
+ *  days_back: 10
+ * };
+ * myFunction(config);
+ */
+
+module.exports = (config) => {
+  if (!config.sources || !config.sources.length) {
+    throw new Error("Missing config.sources");
+  }
+
+  if (!config.target) {
+    throw new Error("Missing target");
+  }
+
+  if (!config.target.database) {
+    throw new Error("Missing target database");
+  }
+
+  // TODO: add defaults for your config options
+  config = {
+    sourceTableSuffix: "_drpg",
+    daysBack: "7",
+    startDate: "2024-09-01",
+    titlesProject: "youtube-research-440316",
+    titlesDataset: "yt_channel_development",
+    titlesTable: "video_titles",
+    playlistTable: "playlist_titles",
+    ...config,
+  };
+
+  config.target = {
+    stagingSchema: "propeller_dataform_template",
+    outputSchema: "propeller_dataform_template",
+    protected: false,
+    ...config.target,
+  };
+
+  config.sources.map((s) => {
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_basic_a2_" + s.suffix,
+      description:
+        "This report provides user activity statistics related to a channel and its videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_combined_a2_" + s.suffix,
+      description:
+        "This report provides user activity statistics related to a channel and its videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_province_a2_" + s.suffix,
+      description:
+        "This report provides user activity statistics for U.S. states and the District of Columbia.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_playback_location_a2_" + s.suffix,
+      description:
+        "This report provides statistics related to the type of page or application where video playbacks occurred.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_traffic_source_a2_" + s.suffix,
+      description:
+        "This report aggregates viewing statistics based on the manner in which viewers reached the channel's video content.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_device_os_a2_" + s.suffix,
+      description:
+        "This report aggregates video viewing statistics based on viewers' operating systems and device types.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_demographics_a1_" + s.suffix,
+      description:
+        "This report aggregates viewing statistics based on viewers' age group and gender.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_sharing_service_a1_" + s.suffix,
+      description:
+        "This report provides statistics showing how frequently the channel's videos were shared on different social platforms.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_annotations_a1_" + s.suffix,
+      description:
+        "This report provides statistics for annotations that display during a channel's videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_cards_a1_" + s.suffix,
+      description:
+        "This report provides impressions and click-through statistics for cards that display during a channel's videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_end_screens_a1_" + s.suffix,
+      description:
+        "This report provides impressions and click-through statistics for end screen elements that display during a channel's videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_channel_subtitles_a2_" + s.suffix,
+      description:
+        "This report provides statistics about the closed caption language usage during video views.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_basic_a1_" + s.suffix,
+      description:
+        "This report provides statistics related to users' interactions with a channel's playlists.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_device_os_a1_" + s.suffix,
+      description:
+        "This report aggregates playlist viewing statistics based on viewers' operating systems and device types.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_province_a1_" + s.suffix,
+      description:
+        "This report provides user activity statistics related to users' interactions with a channel's playlists for U.S. states and the District of Columbia.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_playback_location_a1_" + s.suffix,
+      description:
+        "This report provides statistics related to the type of page or application where playlist playbacks occurred.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_traffic_source_a1_" + s.suffix,
+      description:
+        "This report aggregates viewing statistics based on the manner in which viewers reached a channel's playlist videos.",
+    });
+
+    declare({
+      database: s.database,
+      schema: s.schema,
+      name: "p_playlist_combined_a1_" + s.suffix,
+      description:
+        "This report provides fine-grained playlist statistics by combining dimensions used in the playback location, traffic source, and device/OS reports.",
+    });
+
+    declare({
+      database: config.titlesProject,
+      schema: config.titlesDataset,
+      name: config.titlesTable,
+      description:
+        "This table stores video titles retirieved from the data API.",
+    });
+
+    declare({
+      database: config.titlesProject,
+      schema: config.titlesDataset,
+      name: config.playlistTable,
+      description:
+        "This table stores playlist titles retirieved from the data API.",
+    });
+  });
+
+  // Conditionally add models
+  let result = {
+    stg_ytc_lu_playback_location: stg_ytc_lu_playback_location(config),
+    stg_ytc_lu_traffic_sources: stg_ytc_lu_traffic_sources(config),
+    stg_ytc_province: stg_ytc_province(config),
+    stg_ytc_list_traffic_source: stg_ytc_list_traffic_source(config),
+    stg_ytc_cards: stg_ytc_cards(config),
+    stg_ytc_list_device_os: stg_ytc_list_device_os(config),
+    stg_ytc_annotation: stg_ytc_annotation(config),
+    stg_ytc_traffic_source: stg_ytc_traffic_source(config),
+    stg_ytc_lu_annotation_type: stg_ytc_lu_annotation_type(config),
+    stg_ytc_lu_sharing_services: stg_ytc_lu_sharing_services(config),
+    stg_ytc_lu_end_screen_element_type:
+      stg_ytc_lu_end_screen_element_type(config),
+    stg_ytc_lu_device_types: stg_ytc_lu_device_types(config),
+    stg_ytc_playback: stg_ytc_playback(config),
+    stg_ytc_lu_operating_systems: stg_ytc_lu_operating_systems(config),
+    stg_ytc_end_screens: stg_ytc_end_screens(config),
+    stg_ytc_list_basic: stg_ytc_list_basic(config),
+    stg_ytc_demographics: stg_ytc_demographics(config),
+    stg_ytc_lu_card_type: stg_ytc_lu_card_type(config),
+    stg_ytc_list_province: stg_ytc_list_province(config),
+    stg_ytc_list_combined: stg_ytc_list_combined(config),
+    stg_ytc_basic: stg_ytc_basic(config),
+    stg_ytc_subtitles: stg_ytc_subtitles(config),
+    stg_ytc_combined: stg_ytc_combined(config),
+    stg_ytc_device_os: stg_ytc_device_os(config),
+    stg_ytc_share_platform: stg_ytc_share_platform(config),
+    stg_ytc_list_playback: stg_ytc_list_playback(config),
+    ytc_list_province: ytc_list_province(config),
+    ytc_list_traffic_source: ytc_list_traffic_source(config),
+    ytc_demographics: ytc_demographics(config),
+    ytc_traffic_source: ytc_traffic_source(config),
+    ytc_combined: ytc_combined(config),
+    ytc_demographics_views: ytc_demographics_views(config),
+    ytc_list_basic: ytc_list_basic(config),
+    ytc_list_combined: ytc_list_combined(config),
+    ytc_cards: ytc_cards(config),
+    ytc_share_platform: ytc_share_platform(config),
+    ytc_playback: ytc_playback(config),
+    ytc_list_device_os: ytc_list_device_os(config),
+    ytc_device_os: ytc_device_os(config),
+    ytc_end_screens: ytc_end_screens(config),
+    ytc_annotation: ytc_annotation(config),
+    ytc_province: ytc_province(config),
+    ytc_subtitles: ytc_subtitles(config),
+    ytc_basic: ytc_basic(config),
+    ytc_list_playback: ytc_list_playback(config),
+  };
+
+  return result;
+};

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@ken-e/dataform-youtube",
+  "dependencies": {
+    "@dataform/core": "3.0.9",
+    "@ken-e/dataform-helpers": "latest"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.18.0",
+    "eslint": "^9.18.0",
+    "eslint-config-prettier": "^10.0.1",
+    "globals": "^15.14.0",
+    "prettier": "3.4.2"
+  },
+  "version": "0.0.2",
+  "description": "TODO",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/KEN-E-AI/dataform-youtube.git"
+  },
+  "author": "",
+  "license": "UNLICENSED",
+  "bugs": {
+    "url": "https://github.com/KEN-E-AI/dataform-youtube/issues"
+  },
+  "homepage": "https://github.com/KEN-E-AI/dataform-youtube#readme"
+}

package/prettier.config.js

@@ -0,0 +1,14 @@
+// prettier.config.js, .prettierrc.js, prettier.config.mjs, or .prettierrc.mjs
+
+/**
+ * @see https://prettier.io/docs/en/configuration.html
+ * @type {import("prettier").Config}
+ */
+const config = {
+  trailingComma: "es5",
+  tabWidth: 4,
+  semi: true,
+  singleQuote: true,
+};
+
+module.exports = [config];