@ken-e/dataform-youtube 0.0.2 → 0.0.3

package/includes/definitions/ytc_list_province.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_province", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Playlist Province Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_province"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     playlist_saves_added - playlist_saves_removed as playlist_saves_net
   from ${ctx.ref("stg_ytc_list_province")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_video_titles as (

package/includes/definitions/ytc_list_traffic_source.js

@@ -3,45 +3,26 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_list_traffic_source", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["playlist_id", "video_id", "traffic_source_type"],
     },
     columns: column_descriptions.column_descriptions,
     description:
       "YT Channel Playlist Traffic Source Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_list_traffic_source"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -50,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     playlist_saves_added - playlist_saves_removed as playlist_saves_net
   from ${ctx.ref("stg_ytc_list_traffic_source")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 ),
 
 int_with_traffic_source as (

package/includes/definitions/ytc_playback.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_playback", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "playback_location_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Playback Location Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_playback"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_playback")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_province.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_province", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Province Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_province"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_province")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_share_platform.js

@@ -7,45 +7,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_share_platform", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "sharing_service"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Share Platform Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_share_platform"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -54,7 +34,13 @@ with int_ex_titles as (
   select 
     * except (source_partition_date)
   from ${ctx.ref("stg_ytc_share_platform")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_subtitles.js

@@ -3,44 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_subtitles", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Subtitles Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_subtitles"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -49,7 +30,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_subtitles")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/definitions/ytc_traffic_source.js

@@ -3,45 +3,25 @@
  */
 
 const column_descriptions = require("../column_descriptions");
+const helpers = require("../helpers");
 
 module.exports = (config) => {
   // eslint-disable-next-line no-undef
   return publish("ytc_traffic_source", {
     type: "incremental",
-    schema: config.datasetIntermediate,
+    schema: config.outputSchema,
     tags: ["youtube", "output", "daily"],
     bigquery: {
-      partitionBy: "interaction_date",
+      partitionBy: "data_date",
       clusterBy: ["video_id", "traffic_source_type"],
     },
     columns: column_descriptions.column_descriptions,
     description: "YT Channel Traffic Source Report Table - Intermediate",
   })
-    .preOps(
-      (ctx) => `
-  declare interaction_date_checkpoint default (
-    select date("${config.startDate}")
-  );
-
-  --Set the incremental update checkpoint based on current max partition value minus lookback. 
-  set interaction_date_checkpoint = (
-    ${ctx.when(
-      ctx.incremental(),
-      `select
-        least(
-          (select date_sub(current_date(), interval ${config.daysBack} day)),
-          (select date_sub(max(interaction_date), interval ${config.daysBack} day) from ${ctx.self()})
-        )`,
-      `select date("${config.startDate}")`,
-    )}
-  );
-
-  ${ctx.when(
-    ctx.incremental(),
-    `delete ${ctx.self()} where interaction_date > interaction_date_checkpoint`,
-  )}  
-`,
-    )
+    .preOps((ctx) => {
+      // Get pre_operations to find updated output partitions
+      return `${helpers.output_preops(ctx, config, ctx.ref("stg_ytc_traffic_source"))}`;
+    })
     .query(
       (ctx) => `
 
@@ -51,7 +31,13 @@ with int_ex_titles as (
     * except (source_partition_date),
     views * safe_divide(average_view_duration_seconds, average_view_duration_percentage) as row_max_duration_seconds
   from ${ctx.ref("stg_ytc_traffic_source")}
-  where interaction_date > interaction_date_checkpoint
+  ${ctx.when(
+    ctx.incremental(),
+    `where
+      data_date in unnest(partitions_to_update)`,
+    `where
+      data_date > date '${config.startDate}'`,
+  )}
 )
 
 select

package/includes/helpers.js

@@ -3,6 +3,18 @@
  * Javascript functions that are referenced throughout the models.
  */
 
+const { incremental } = require("@ken-e/dataform-helpers");
+
+const {
+  BQ_DATA_TRANSFER_PARTITION_STAGING_PREOPS,
+  BQ_DATA_TRANSFER_PARTITION_OUTPUT_PREOPS,
+} = incremental;
+
+module.exports = {
+  staging_preops: BQ_DATA_TRANSFER_PARTITION_STAGING_PREOPS,
+  output_preops: BQ_DATA_TRANSFER_PARTITION_OUTPUT_PREOPS,
+};
+
 function addTableSuffix(table) {
   return table + config.sourceTableSuffix;
 }

package/index.js

@@ -77,6 +77,16 @@ module.exports = (config) => {
     throw new Error("Missing target database");
   }
 
+  if (
+    !config.target.schema &&
+    !config.target.stagingSchema &&
+    !config.target.outputSchema
+  ) {
+    throw new Error(
+      "Missing output schema.  Supply config.target.schema or individual config.target.stagingSchema and config.target.outputSchema",
+    );
+  }
+
   // TODO: add defaults for your config options
   config = {
     sourceTableSuffix: "_drpg",
@@ -90,8 +100,8 @@ module.exports = (config) => {
   };
 
   config.target = {
-    stagingSchema: "propeller_dataform_template",
-    outputSchema: "propeller_dataform_template",
+    stagingSchema: config.target.stagingSchema || config.target.schema,
+    outputSchema: config.target.outputSchema || config.target.schema,
     protected: false,
     ...config.target,
   };

package/package.json

@@ -2,7 +2,7 @@
   "name": "@ken-e/dataform-youtube",
   "dependencies": {
     "@dataform/core": "3.0.9",
-    "@ken-e/dataform-helpers": "latest"
+    "@ken-e/dataform-helpers": "0.0.6"
   },
   "devDependencies": {
     "@eslint/js": "^9.18.0",
@@ -11,8 +11,8 @@
     "globals": "^15.14.0",
     "prettier": "3.4.2"
   },
-  "version": "0.0.2",
-  "description": "TODO",
+  "version": "0.0.3",
+  "description": "A dataform package to process Youtube data from the BQ Data Transfer.",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"