@ken-e/dataform-youtube 0.0.2 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/includes/column_descriptions.js +137 -95
  2. package/includes/definitions/sources/stg_ytc_annotation.js +16 -31
  3. package/includes/definitions/sources/stg_ytc_basic.js +15 -31
  4. package/includes/definitions/sources/stg_ytc_cards.js +16 -31
  5. package/includes/definitions/sources/stg_ytc_combined.js +15 -31
  6. package/includes/definitions/sources/stg_ytc_demographics.js +16 -31
  7. package/includes/definitions/sources/stg_ytc_device_os.js +16 -31
  8. package/includes/definitions/sources/stg_ytc_end_screens.js +14 -29
  9. package/includes/definitions/sources/stg_ytc_list_basic.js +15 -31
  10. package/includes/definitions/sources/stg_ytc_list_combined.js +16 -30
  11. package/includes/definitions/sources/stg_ytc_list_device_os.js +16 -30
  12. package/includes/definitions/sources/stg_ytc_list_playback.js +16 -30
  13. package/includes/definitions/sources/stg_ytc_list_province.js +15 -31
  14. package/includes/definitions/sources/stg_ytc_list_traffic_source.js +16 -30
  15. package/includes/definitions/sources/stg_ytc_lu_operating_systems.js +1 -1
  16. package/includes/definitions/sources/stg_ytc_playback.js +16 -31
  17. package/includes/definitions/sources/stg_ytc_province.js +15 -31
  18. package/includes/definitions/sources/stg_ytc_share_platform.js +16 -31
  19. package/includes/definitions/sources/stg_ytc_subtitles.js +15 -30
  20. package/includes/definitions/sources/stg_ytc_traffic_source.js +16 -49
  21. package/includes/definitions/ytc_annotation.js +14 -29
  22. package/includes/definitions/ytc_basic.js +14 -30
  23. package/includes/definitions/ytc_cards.js +14 -29
  24. package/includes/definitions/ytc_combined.js +14 -31
  25. package/includes/definitions/ytc_demographics.js +14 -28
  26. package/includes/definitions/ytc_demographics_views.js +15 -29
  27. package/includes/definitions/ytc_device_os.js +14 -28
  28. package/includes/definitions/ytc_end_screens.js +14 -28
  29. package/includes/definitions/ytc_list_basic.js +14 -28
  30. package/includes/definitions/ytc_list_combined.js +14 -27
  31. package/includes/definitions/ytc_list_device_os.js +14 -27
  32. package/includes/definitions/ytc_list_playback.js +14 -27
  33. package/includes/definitions/ytc_list_province.js +14 -28
  34. package/includes/definitions/ytc_list_traffic_source.js +14 -27
  35. package/includes/definitions/ytc_playback.js +14 -28
  36. package/includes/definitions/ytc_province.js +14 -28
  37. package/includes/definitions/ytc_share_platform.js +14 -28
  38. package/includes/definitions/ytc_subtitles.js +14 -27
  39. package/includes/definitions/ytc_traffic_source.js +14 -28
  40. package/includes/helpers.js +12 -0
  41. package/index.js +12 -2
  42. package/package.json +3 -3
package/includes/helpers.js CHANGED
@@ -3,6 +3,18 @@
3
3
  * Javascript functions that are referenced throughout the models.
4
4
  */
5
5
 
6
+ const { incremental } = require("@ken-e/dataform-helpers");
7
+
8
+ const {
9
+ BQ_DATA_TRANSFER_PARTITION_STAGING_PREOPS,
10
+ BQ_DATA_TRANSFER_PARTITION_OUTPUT_PREOPS,
11
+ } = incremental;
12
+
13
+ module.exports = {
14
+ staging_preops: BQ_DATA_TRANSFER_PARTITION_STAGING_PREOPS,
15
+ output_preops: BQ_DATA_TRANSFER_PARTITION_OUTPUT_PREOPS,
16
+ };
17
+
6
18
  function addTableSuffix(table) {
7
19
  return table + config.sourceTableSuffix;
8
20
  }
package/index.js CHANGED
@@ -77,6 +77,16 @@ module.exports = (config) => {
77
77
  throw new Error("Missing target database");
78
78
  }
79
79
 
80
+ if (
81
+ !config.target.schema &&
82
+ !config.target.stagingSchema &&
83
+ !config.target.outputSchema
84
+ ) {
85
+ throw new Error(
86
+ "Missing output schema. Supply config.target.schema or individual config.target.stagingSchema and config.target.outputSchema",
87
+ );
88
+ }
89
+
80
90
  // TODO: add defaults for your config options
81
91
  config = {
82
92
  sourceTableSuffix: "_drpg",
@@ -90,8 +100,8 @@ module.exports = (config) => {
90
100
  };
91
101
 
92
102
  config.target = {
93
- stagingSchema: "propeller_dataform_template",
94
- outputSchema: "propeller_dataform_template",
103
+ stagingSchema: config.target.stagingSchema || config.target.schema,
104
+ outputSchema: config.target.outputSchema || config.target.schema,
95
105
  protected: false,
96
106
  ...config.target,
97
107
  };
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@ken-e/dataform-youtube",
3
3
  "dependencies": {
4
4
  "@dataform/core": "3.0.9",
5
- "@ken-e/dataform-helpers": "latest"
5
+ "@ken-e/dataform-helpers": "0.0.6"
6
6
  },
7
7
  "devDependencies": {
8
8
  "@eslint/js": "^9.18.0",
@@ -11,8 +11,8 @@
11
11
  "globals": "^15.14.0",
12
12
  "prettier": "3.4.2"
13
13
  },
14
- "version": "0.0.2",
15
- "description": "TODO",
14
+ "version": "0.0.4",
15
+ "description": "A dataform package to process Youtube data from the BQ Data Transfer.",
16
16
  "main": "index.js",
17
17
  "scripts": {
18
18
  "test": "echo \"Error: no test specified\" && exit 1"