@ken-e/dataform-youtube 0.0.4 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. package/includes/definitions/sources/stg_ytc_annotation.js +2 -2
  2. package/includes/definitions/sources/stg_ytc_basic.js +2 -2
  3. package/includes/definitions/sources/stg_ytc_cards.js +2 -2
  4. package/includes/definitions/sources/stg_ytc_combined.js +2 -2
  5. package/includes/definitions/sources/stg_ytc_demographics.js +2 -2
  6. package/includes/definitions/sources/stg_ytc_device_os.js +2 -2
  7. package/includes/definitions/sources/stg_ytc_end_screens.js +2 -2
  8. package/includes/definitions/sources/stg_ytc_list_basic.js +2 -2
  9. package/includes/definitions/sources/stg_ytc_list_combined.js +2 -2
  10. package/includes/definitions/sources/stg_ytc_list_device_os.js +2 -2
  11. package/includes/definitions/sources/stg_ytc_list_playback.js +2 -2
  12. package/includes/definitions/sources/stg_ytc_list_province.js +2 -2
  13. package/includes/definitions/sources/stg_ytc_list_traffic_source.js +2 -2
  14. package/includes/definitions/sources/stg_ytc_playback.js +2 -2
  15. package/includes/definitions/sources/stg_ytc_province.js +2 -2
  16. package/includes/definitions/sources/stg_ytc_share_platform.js +2 -2
  17. package/includes/definitions/sources/stg_ytc_subtitles.js +2 -2
  18. package/includes/definitions/sources/stg_ytc_traffic_source.js +2 -2
  19. package/includes/helpers.js +0 -4
  20. package/package.json +2 -2
package/includes/definitions/sources/stg_ytc_annotation.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_annotation")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_annotations_a1_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -65,7 +65,7 @@ select
65
65
  from ${ctx.ref(t.database, t.schema, "p_channel_annotations_a1_" + t.suffix)}
66
66
  ${ctx.when(
67
67
  ctx.incremental(),
68
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
68
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
69
69
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
70
70
  )}
71
71
 
package/includes/definitions/sources/stg_ytc_basic.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_basic")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_basic_a2_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -83,7 +83,7 @@ select
83
83
  from ${ctx.ref(t.database, t.schema, "p_channel_basic_a2_" + t.suffix)}
84
84
  ${ctx.when(
85
85
  ctx.incremental(),
86
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
86
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
87
87
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
88
88
  )}
89
89
  `;
package/includes/definitions/sources/stg_ytc_cards.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_cards")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_cards_a1_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -64,7 +64,7 @@ select
64
64
  from ${ctx.ref(t.database, t.schema, "p_channel_cards_a1_" + t.suffix)}
65
65
  ${ctx.when(
66
66
  ctx.incremental(),
67
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
67
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
68
68
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
69
69
  )}
70
70
  `;
package/includes/definitions/sources/stg_ytc_combined.js CHANGED
@@ -41,7 +41,7 @@ module.exports = (config) => {
41
41
  })
42
42
  .preOps((ctx) => {
43
43
  // Get pre_operations to find updated source partitions
44
- return `${helpers.staging_preops(ctx, config, "stg_ytc_combined")}`;
44
+ return `${helpers.staging_preops(ctx, config, "p_channel_combined_a2_")}`;
45
45
  })
46
46
  .query((ctx) =>
47
47
  config.sources
@@ -71,7 +71,7 @@ select
71
71
  from ${ctx.ref(t.database, t.schema, "p_channel_combined_a2_" + t.suffix)}
72
72
  ${ctx.when(
73
73
  ctx.incremental(),
74
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
74
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
75
75
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
76
76
  )}
77
77
  `;
package/includes/definitions/sources/stg_ytc_demographics.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_demographics")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_demographics_a1_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -59,7 +59,7 @@ select
59
59
  from ${ctx.ref(t.database, t.schema, "p_channel_demographics_a1_" + t.suffix)}
60
60
  ${ctx.when(
61
61
  ctx.incremental(),
62
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
62
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
63
63
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
64
64
  )}
65
65
  `;
package/includes/definitions/sources/stg_ytc_device_os.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_device_os")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_device_os_a2_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -65,7 +65,7 @@ select
65
65
  from ${ctx.ref(t.database, t.schema, "p_channel_device_os_a2_" + t.suffix)}
66
66
  ${ctx.when(
67
67
  ctx.incremental(),
68
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
68
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
69
69
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
70
70
  )}
71
71
  `;
package/includes/definitions/sources/stg_ytc_end_screens.js CHANGED
@@ -20,7 +20,7 @@ module.exports = (config) => {
20
20
  })
21
21
  .preOps((ctx) => {
22
22
  // Get pre_operations to find updated source partitions
23
- return `${helpers.staging_preops(ctx, config, "stg_ytc_end_screens")}`;
23
+ return `${helpers.staging_preops(ctx, config, "p_channel_end_screens_a1_")}`;
24
24
  })
25
25
  .query((ctx) =>
26
26
  config.sources
@@ -44,7 +44,7 @@ select
44
44
  from ${ctx.ref(t.database, t.schema, "p_channel_end_screens_a1_" + t.suffix)}
45
45
  ${ctx.when(
46
46
  ctx.incremental(),
47
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
47
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
48
48
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
49
49
  )}
50
50
  `;
package/includes/definitions/sources/stg_ytc_list_basic.js CHANGED
@@ -38,7 +38,7 @@ module.exports = (config) => {
38
38
  })
39
39
  .preOps((ctx) => {
40
40
  // Get pre_operations to find updated source partitions
41
- return `${helpers.staging_preops(ctx, config, "stg_ytc_list_basic")}`;
41
+ return `${helpers.staging_preops(ctx, config, "p_playlist_basic_a1_")}`;
42
42
  })
43
43
  .query((ctx) =>
44
44
  config.sources
@@ -65,7 +65,7 @@ select
65
65
  from ${ctx.ref(t.database, t.schema, "p_playlist_basic_a1_" + t.suffix)}
66
66
  ${ctx.when(
67
67
  ctx.incremental(),
68
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
68
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
69
69
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
70
70
  )}
71
71
  `;
package/includes/definitions/sources/stg_ytc_list_combined.js CHANGED
@@ -38,7 +38,7 @@ module.exports = (config) => {
38
38
  })
39
39
  .preOps((ctx) => {
40
40
  // Get pre_operations to find updated source partitions
41
- return `${helpers.staging_preops(ctx, config, "stg_ytc_list_combined")}`;
41
+ return `${helpers.staging_preops(ctx, config, "p_playlist_combined_a1_")}`;
42
42
  })
43
43
  .query((ctx) =>
44
44
  config.sources
@@ -68,7 +68,7 @@ select
68
68
  from ${ctx.ref(t.database, t.schema, "p_playlist_combined_a1_" + t.suffix)}
69
69
  ${ctx.when(
70
70
  ctx.incremental(),
71
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
71
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
72
72
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
73
73
  )}
74
74
  `;
package/includes/definitions/sources/stg_ytc_list_device_os.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_list_device_os")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_playlist_device_os_a1_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -66,7 +66,7 @@ select
66
66
  from ${ctx.ref(t.database, t.schema, "p_playlist_device_os_a1_" + t.suffix)}
67
67
  ${ctx.when(
68
68
  ctx.incremental(),
69
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
69
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
70
70
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
71
71
  )}
72
72
  `;
package/includes/definitions/sources/stg_ytc_list_playback.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_list_playback")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_playlist_playback_location_a1_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -65,7 +65,7 @@ select
65
65
  from ${ctx.ref(t.database, t.schema, "p_playlist_playback_location_a1_" + t.suffix)}
66
66
  ${ctx.when(
67
67
  ctx.incremental(),
68
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
68
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
69
69
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
70
70
  )}
71
71
  `;
package/includes/definitions/sources/stg_ytc_list_province.js CHANGED
@@ -38,7 +38,7 @@ module.exports = (config) => {
38
38
  })
39
39
  .preOps((ctx) => {
40
40
  // Get pre_operations to find updated source partitions
41
- return `${helpers.staging_preops(ctx, config, "stg_ytc_list_province")}`;
41
+ return `${helpers.staging_preops(ctx, config, "p_playlist_province_a1_")}`;
42
42
  })
43
43
  .query((ctx) =>
44
44
  config.sources
@@ -65,7 +65,7 @@ select
65
65
  from ${ctx.ref(t.database, t.schema, "p_playlist_province_a1_" + t.suffix)}
66
66
  ${ctx.when(
67
67
  ctx.incremental(),
68
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
68
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
69
69
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
70
70
  )}
71
71
  `;
package/includes/definitions/sources/stg_ytc_list_traffic_source.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_list_traffic_source")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_playlist_traffic_source_a1_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -65,7 +65,7 @@ select
65
65
  from ${ctx.ref(t.database, t.schema, "p_playlist_traffic_source_a1_" + t.suffix)}
66
66
  ${ctx.when(
67
67
  ctx.incremental(),
68
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
68
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
69
69
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
70
70
  )}
71
71
  `;
package/includes/definitions/sources/stg_ytc_playback.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_playback")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_playback_location_a2_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -65,7 +65,7 @@ select
65
65
  from ${ctx.ref(t.database, t.schema, "p_channel_playback_location_a2_" + t.suffix)}
66
66
  ${ctx.when(
67
67
  ctx.incremental(),
68
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
68
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
69
69
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
70
70
  )}
71
71
  `;
package/includes/definitions/sources/stg_ytc_province.js CHANGED
@@ -38,7 +38,7 @@ module.exports = (config) => {
38
38
  })
39
39
  .preOps((ctx) => {
40
40
  // Get pre_operations to find updated source partitions
41
- return `${helpers.staging_preops(ctx, config, "stg_ytc_province")}`;
41
+ return `${helpers.staging_preops(ctx, config, "p_channel_province_a2_")}`;
42
42
  })
43
43
  .query((ctx) =>
44
44
  config.sources
@@ -77,7 +77,7 @@ select
77
77
  from ${ctx.ref(t.database, t.schema, "p_channel_province_a2_" + t.suffix)}
78
78
  ${ctx.when(
79
79
  ctx.incremental(),
80
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
80
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
81
81
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
82
82
  )}
83
83
  `;
package/includes/definitions/sources/stg_ytc_share_platform.js CHANGED
@@ -36,7 +36,7 @@ module.exports = (config) => {
36
36
  })
37
37
  .preOps((ctx) => {
38
38
  // Get pre_operations to find updated source partitions
39
- return `${helpers.staging_preops(ctx, config, "stg_ytc_share_platform")}`;
39
+ return `${helpers.staging_preops(ctx, config, "p_channel_sharing_service_a1_")}`;
40
40
  })
41
41
  .query((ctx) =>
42
42
  config.sources
@@ -57,7 +57,7 @@ select
57
57
  from ${ctx.ref(t.database, t.schema, "p_channel_sharing_service_a1_" + t.suffix)}
58
58
  ${ctx.when(
59
59
  ctx.incremental(),
60
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
60
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
61
61
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
62
62
  )}
63
63
  `;
package/includes/definitions/sources/stg_ytc_subtitles.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_subtitles")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_subtitles_a2_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -64,7 +64,7 @@ select
64
64
  from ${ctx.ref(t.database, t.schema, "p_channel_subtitles_a2_" + t.suffix)}
65
65
  ${ctx.when(
66
66
  ctx.incremental(),
67
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
67
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
68
68
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
69
69
  )}
70
70
  `;
package/includes/definitions/sources/stg_ytc_traffic_source.js CHANGED
@@ -37,7 +37,7 @@ module.exports = (config) => {
37
37
  })
38
38
  .preOps((ctx) => {
39
39
  // Get pre_operations to find updated source partitions
40
- return `${helpers.staging_preops(ctx, config, "stg_ytc_traffic_source")}`;
40
+ return `${helpers.staging_preops(ctx, config, "p_channel_traffic_source_a2_")}`;
41
41
  })
42
42
  .query((ctx) =>
43
43
  config.sources
@@ -64,7 +64,7 @@ select
64
64
  from ${ctx.ref(t.database, t.schema, "p_channel_traffic_source_a2_" + t.suffix)}
65
65
  ${ctx.when(
66
66
  ctx.incremental(),
67
- `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t}"))`,
67
+ `where date(_PARTITIONTIME) in unnest((select partition_dates from unnest(partitions_to_update) where site_nm = "${t.schema}"))`,
68
68
  `where date(_PARTITIONTIME) > date(self_checkpoint)`,
69
69
  )}
70
70
  `;
package/includes/helpers.js CHANGED
@@ -14,7 +14,3 @@ module.exports = {
14
14
  staging_preops: BQ_DATA_TRANSFER_PARTITION_STAGING_PREOPS,
15
15
  output_preops: BQ_DATA_TRANSFER_PARTITION_OUTPUT_PREOPS,
16
16
  };
17
-
18
- function addTableSuffix(table) {
19
- return table + config.sourceTableSuffix;
20
- }
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@ken-e/dataform-youtube",
3
3
  "dependencies": {
4
4
  "@dataform/core": "3.0.9",
5
- "@ken-e/dataform-helpers": "0.0.6"
5
+ "@ken-e/dataform-helpers": "0.0.7"
6
6
  },
7
7
  "devDependencies": {
8
8
  "@eslint/js": "^9.18.0",
@@ -11,7 +11,7 @@
11
11
  "globals": "^15.14.0",
12
12
  "prettier": "3.4.2"
13
13
  },
14
- "version": "0.0.4",
14
+ "version": "0.0.6",
15
15
  "description": "A dataform package to process Youtube data from the BQ Data Transfer.",
16
16
  "main": "index.js",
17
17
  "scripts": {