aws-sdk-glue 1.134.0 → 1.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-glue/client.rb +323 -1940
- data/lib/aws-sdk-glue/client_api.rb +60 -0
- data/lib/aws-sdk-glue/types.rb +278 -9
- data/lib/aws-sdk-glue.rb +1 -1
- metadata +2 -2
data/lib/aws-sdk-glue/client.rb
CHANGED
|
@@ -958,7 +958,7 @@ module Aws::Glue
|
|
|
958
958
|
# resp.dev_endpoints[0].zeppelin_remote_spark_interpreter_port #=> Integer
|
|
959
959
|
# resp.dev_endpoints[0].public_address #=> String
|
|
960
960
|
# resp.dev_endpoints[0].status #=> String
|
|
961
|
-
# resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
961
|
+
# resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
962
962
|
# resp.dev_endpoints[0].glue_version #=> String
|
|
963
963
|
# resp.dev_endpoints[0].number_of_workers #=> Integer
|
|
964
964
|
# resp.dev_endpoints[0].number_of_nodes #=> Integer
|
|
@@ -1032,7 +1032,7 @@ module Aws::Glue
|
|
|
1032
1032
|
# resp.jobs[0].allocated_capacity #=> Integer
|
|
1033
1033
|
# resp.jobs[0].timeout #=> Integer
|
|
1034
1034
|
# resp.jobs[0].max_capacity #=> Float
|
|
1035
|
-
# resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
1035
|
+
# resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
1036
1036
|
# resp.jobs[0].number_of_workers #=> Integer
|
|
1037
1037
|
# resp.jobs[0].security_configuration #=> String
|
|
1038
1038
|
# resp.jobs[0].notification_property.notify_delay_after #=> Integer
|
|
@@ -1543,6 +1543,10 @@ module Aws::Glue
|
|
|
1543
1543
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.function_name #=> String
|
|
1544
1544
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.path #=> String
|
|
1545
1545
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.version #=> String
|
|
1546
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas #=> Array
|
|
1547
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns #=> Array
|
|
1548
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns[0].name #=> String
|
|
1549
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns[0].type #=> String
|
|
1546
1550
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.name #=> String
|
|
1547
1551
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.inputs #=> Array
|
|
1548
1552
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.inputs[0] #=> String
|
|
@@ -1675,6 +1679,102 @@ module Aws::Glue
|
|
|
1675
1679
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
1676
1680
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.table #=> String
|
|
1677
1681
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.database #=> String
|
|
1682
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.name #=> String
|
|
1683
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.access_type #=> String
|
|
1684
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.source_type #=> String
|
|
1685
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.value #=> String
|
|
1686
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.label #=> String
|
|
1687
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.description #=> String
|
|
1688
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.value #=> String
|
|
1689
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.label #=> String
|
|
1690
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.description #=> String
|
|
1691
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.value #=> String
|
|
1692
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.label #=> String
|
|
1693
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.description #=> String
|
|
1694
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.value #=> String
|
|
1695
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.label #=> String
|
|
1696
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.description #=> String
|
|
1697
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.value #=> String
|
|
1698
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.label #=> String
|
|
1699
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.description #=> String
|
|
1700
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_redshift_schema #=> String
|
|
1701
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_redshift_table #=> String
|
|
1702
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.temp_dir #=> String
|
|
1703
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.value #=> String
|
|
1704
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.label #=> String
|
|
1705
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.description #=> String
|
|
1706
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options #=> Array
|
|
1707
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options[0].key #=> String
|
|
1708
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options[0].value #=> String
|
|
1709
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.sample_query #=> String
|
|
1710
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.pre_action #=> String
|
|
1711
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.post_action #=> String
|
|
1712
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.action #=> String
|
|
1713
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_prefix #=> String
|
|
1714
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.upsert #=> Boolean
|
|
1715
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_action #=> String
|
|
1716
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_when_matched #=> String
|
|
1717
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_when_not_matched #=> String
|
|
1718
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_clause #=> String
|
|
1719
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.crawler_connection #=> String
|
|
1720
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema #=> Array
|
|
1721
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].value #=> String
|
|
1722
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].label #=> String
|
|
1723
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].description #=> String
|
|
1724
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.staging_table #=> String
|
|
1725
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns #=> Array
|
|
1726
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].value #=> String
|
|
1727
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].label #=> String
|
|
1728
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].description #=> String
|
|
1729
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.name #=> String
|
|
1730
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.access_type #=> String
|
|
1731
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.source_type #=> String
|
|
1732
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.value #=> String
|
|
1733
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.label #=> String
|
|
1734
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.description #=> String
|
|
1735
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.value #=> String
|
|
1736
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.label #=> String
|
|
1737
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.description #=> String
|
|
1738
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.value #=> String
|
|
1739
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.label #=> String
|
|
1740
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.description #=> String
|
|
1741
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.value #=> String
|
|
1742
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.label #=> String
|
|
1743
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.description #=> String
|
|
1744
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.value #=> String
|
|
1745
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.label #=> String
|
|
1746
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.description #=> String
|
|
1747
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_redshift_schema #=> String
|
|
1748
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_redshift_table #=> String
|
|
1749
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.temp_dir #=> String
|
|
1750
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.value #=> String
|
|
1751
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.label #=> String
|
|
1752
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.description #=> String
|
|
1753
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options #=> Array
|
|
1754
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options[0].key #=> String
|
|
1755
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options[0].value #=> String
|
|
1756
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.sample_query #=> String
|
|
1757
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.pre_action #=> String
|
|
1758
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.post_action #=> String
|
|
1759
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.action #=> String
|
|
1760
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_prefix #=> String
|
|
1761
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.upsert #=> Boolean
|
|
1762
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_action #=> String
|
|
1763
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_when_matched #=> String
|
|
1764
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_when_not_matched #=> String
|
|
1765
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_clause #=> String
|
|
1766
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.crawler_connection #=> String
|
|
1767
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema #=> Array
|
|
1768
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].value #=> String
|
|
1769
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].label #=> String
|
|
1770
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].description #=> String
|
|
1771
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.staging_table #=> String
|
|
1772
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns #=> Array
|
|
1773
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].value #=> String
|
|
1774
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].label #=> String
|
|
1775
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].description #=> String
|
|
1776
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.inputs #=> Array
|
|
1777
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.inputs[0] #=> String
|
|
1678
1778
|
# resp.jobs[0].execution_class #=> String, one of "FLEX", "STANDARD"
|
|
1679
1779
|
# resp.jobs[0].source_control_details.provider #=> String, one of "GITHUB", "AWS_CODE_COMMIT"
|
|
1680
1780
|
# resp.jobs[0].source_control_details.repository #=> String
|
|
@@ -1952,7 +2052,7 @@ module Aws::Glue
|
|
|
1952
2052
|
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
|
|
1953
2053
|
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
|
|
1954
2054
|
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
|
|
1955
|
-
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
2055
|
+
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
1956
2056
|
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
|
|
1957
2057
|
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
|
|
1958
2058
|
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
|
|
@@ -2020,7 +2120,7 @@ module Aws::Glue
|
|
|
2020
2120
|
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
|
|
2021
2121
|
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
|
|
2022
2122
|
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
|
|
2023
|
-
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
2123
|
+
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
2024
2124
|
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
|
|
2025
2125
|
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
|
|
2026
2126
|
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
|
|
@@ -2990,7 +3090,7 @@ module Aws::Glue
|
|
|
2990
3090
|
# public_key: "GenericString",
|
|
2991
3091
|
# public_keys: ["GenericString"],
|
|
2992
3092
|
# number_of_nodes: 1,
|
|
2993
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
|
3093
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
|
|
2994
3094
|
# glue_version: "GlueVersionString",
|
|
2995
3095
|
# number_of_workers: 1,
|
|
2996
3096
|
# extra_python_libs_s3_path: "GenericString",
|
|
@@ -3015,7 +3115,7 @@ module Aws::Glue
|
|
|
3015
3115
|
# resp.yarn_endpoint_address #=> String
|
|
3016
3116
|
# resp.zeppelin_remote_spark_interpreter_port #=> Integer
|
|
3017
3117
|
# resp.number_of_nodes #=> Integer
|
|
3018
|
-
# resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
3118
|
+
# resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
3019
3119
|
# resp.glue_version #=> String
|
|
3020
3120
|
# resp.number_of_workers #=> Integer
|
|
3021
3121
|
# resp.availability_zone #=> String
|
|
@@ -3221,965 +3321,6 @@ module Aws::Glue
|
|
|
3221
3321
|
#
|
|
3222
3322
|
# * {Types::CreateJobResponse#name #name} => String
|
|
3223
3323
|
#
|
|
3224
|
-
# @example Request syntax with placeholder values
|
|
3225
|
-
#
|
|
3226
|
-
# resp = client.create_job({
|
|
3227
|
-
# name: "NameString", # required
|
|
3228
|
-
# description: "DescriptionString",
|
|
3229
|
-
# log_uri: "UriString",
|
|
3230
|
-
# role: "RoleString", # required
|
|
3231
|
-
# execution_property: {
|
|
3232
|
-
# max_concurrent_runs: 1,
|
|
3233
|
-
# },
|
|
3234
|
-
# command: { # required
|
|
3235
|
-
# name: "GenericString",
|
|
3236
|
-
# script_location: "ScriptLocationString",
|
|
3237
|
-
# python_version: "PythonVersionString",
|
|
3238
|
-
# },
|
|
3239
|
-
# default_arguments: {
|
|
3240
|
-
# "GenericString" => "GenericString",
|
|
3241
|
-
# },
|
|
3242
|
-
# non_overridable_arguments: {
|
|
3243
|
-
# "GenericString" => "GenericString",
|
|
3244
|
-
# },
|
|
3245
|
-
# connections: {
|
|
3246
|
-
# connections: ["GenericString"],
|
|
3247
|
-
# },
|
|
3248
|
-
# max_retries: 1,
|
|
3249
|
-
# allocated_capacity: 1,
|
|
3250
|
-
# timeout: 1,
|
|
3251
|
-
# max_capacity: 1.0,
|
|
3252
|
-
# security_configuration: "NameString",
|
|
3253
|
-
# tags: {
|
|
3254
|
-
# "TagKey" => "TagValue",
|
|
3255
|
-
# },
|
|
3256
|
-
# notification_property: {
|
|
3257
|
-
# notify_delay_after: 1,
|
|
3258
|
-
# },
|
|
3259
|
-
# glue_version: "GlueVersionString",
|
|
3260
|
-
# number_of_workers: 1,
|
|
3261
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
|
3262
|
-
# code_gen_configuration_nodes: {
|
|
3263
|
-
# "NodeId" => {
|
|
3264
|
-
# athena_connector_source: {
|
|
3265
|
-
# name: "NodeName", # required
|
|
3266
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
3267
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
3268
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
3269
|
-
# connection_table: "EnclosedInStringPropertyWithQuote",
|
|
3270
|
-
# schema_name: "EnclosedInStringProperty", # required
|
|
3271
|
-
# output_schemas: [
|
|
3272
|
-
# {
|
|
3273
|
-
# columns: [
|
|
3274
|
-
# {
|
|
3275
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3276
|
-
# type: "ColumnTypeString",
|
|
3277
|
-
# },
|
|
3278
|
-
# ],
|
|
3279
|
-
# },
|
|
3280
|
-
# ],
|
|
3281
|
-
# },
|
|
3282
|
-
# jdbc_connector_source: {
|
|
3283
|
-
# name: "NodeName", # required
|
|
3284
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
3285
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
3286
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
3287
|
-
# additional_options: {
|
|
3288
|
-
# filter_predicate: "EnclosedInStringProperty",
|
|
3289
|
-
# partition_column: "EnclosedInStringProperty",
|
|
3290
|
-
# lower_bound: 1,
|
|
3291
|
-
# upper_bound: 1,
|
|
3292
|
-
# num_partitions: 1,
|
|
3293
|
-
# job_bookmark_keys: ["EnclosedInStringProperty"],
|
|
3294
|
-
# job_bookmark_keys_sort_order: "EnclosedInStringProperty",
|
|
3295
|
-
# data_type_mapping: {
|
|
3296
|
-
# "ARRAY" => "DATE", # accepts DATE, STRING, TIMESTAMP, INT, FLOAT, LONG, BIGDECIMAL, BYTE, SHORT, DOUBLE
|
|
3297
|
-
# },
|
|
3298
|
-
# },
|
|
3299
|
-
# connection_table: "EnclosedInStringPropertyWithQuote",
|
|
3300
|
-
# query: "SqlQuery",
|
|
3301
|
-
# output_schemas: [
|
|
3302
|
-
# {
|
|
3303
|
-
# columns: [
|
|
3304
|
-
# {
|
|
3305
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3306
|
-
# type: "ColumnTypeString",
|
|
3307
|
-
# },
|
|
3308
|
-
# ],
|
|
3309
|
-
# },
|
|
3310
|
-
# ],
|
|
3311
|
-
# },
|
|
3312
|
-
# spark_connector_source: {
|
|
3313
|
-
# name: "NodeName", # required
|
|
3314
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
3315
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
3316
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
3317
|
-
# additional_options: {
|
|
3318
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3319
|
-
# },
|
|
3320
|
-
# output_schemas: [
|
|
3321
|
-
# {
|
|
3322
|
-
# columns: [
|
|
3323
|
-
# {
|
|
3324
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3325
|
-
# type: "ColumnTypeString",
|
|
3326
|
-
# },
|
|
3327
|
-
# ],
|
|
3328
|
-
# },
|
|
3329
|
-
# ],
|
|
3330
|
-
# },
|
|
3331
|
-
# catalog_source: {
|
|
3332
|
-
# name: "NodeName", # required
|
|
3333
|
-
# database: "EnclosedInStringProperty", # required
|
|
3334
|
-
# table: "EnclosedInStringProperty", # required
|
|
3335
|
-
# },
|
|
3336
|
-
# redshift_source: {
|
|
3337
|
-
# name: "NodeName", # required
|
|
3338
|
-
# database: "EnclosedInStringProperty", # required
|
|
3339
|
-
# table: "EnclosedInStringProperty", # required
|
|
3340
|
-
# redshift_tmp_dir: "EnclosedInStringProperty",
|
|
3341
|
-
# tmp_dir_iam_role: "EnclosedInStringProperty",
|
|
3342
|
-
# },
|
|
3343
|
-
# s3_catalog_source: {
|
|
3344
|
-
# name: "NodeName", # required
|
|
3345
|
-
# database: "EnclosedInStringProperty", # required
|
|
3346
|
-
# table: "EnclosedInStringProperty", # required
|
|
3347
|
-
# partition_predicate: "EnclosedInStringProperty",
|
|
3348
|
-
# additional_options: {
|
|
3349
|
-
# bounded_size: 1,
|
|
3350
|
-
# bounded_files: 1,
|
|
3351
|
-
# },
|
|
3352
|
-
# },
|
|
3353
|
-
# s3_csv_source: {
|
|
3354
|
-
# name: "NodeName", # required
|
|
3355
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
3356
|
-
# compression_type: "gzip", # accepts gzip, bzip2
|
|
3357
|
-
# exclusions: ["EnclosedInStringProperty"],
|
|
3358
|
-
# group_size: "EnclosedInStringProperty",
|
|
3359
|
-
# group_files: "EnclosedInStringProperty",
|
|
3360
|
-
# recurse: false,
|
|
3361
|
-
# max_band: 1,
|
|
3362
|
-
# max_files_in_band: 1,
|
|
3363
|
-
# additional_options: {
|
|
3364
|
-
# bounded_size: 1,
|
|
3365
|
-
# bounded_files: 1,
|
|
3366
|
-
# enable_sample_path: false,
|
|
3367
|
-
# sample_path: "EnclosedInStringProperty",
|
|
3368
|
-
# },
|
|
3369
|
-
# separator: "comma", # required, accepts comma, ctrla, pipe, semicolon, tab
|
|
3370
|
-
# escaper: "EnclosedInStringPropertyWithQuote",
|
|
3371
|
-
# quote_char: "quote", # required, accepts quote, quillemet, single_quote, disabled
|
|
3372
|
-
# multiline: false,
|
|
3373
|
-
# with_header: false,
|
|
3374
|
-
# write_header: false,
|
|
3375
|
-
# skip_first: false,
|
|
3376
|
-
# optimize_performance: false,
|
|
3377
|
-
# output_schemas: [
|
|
3378
|
-
# {
|
|
3379
|
-
# columns: [
|
|
3380
|
-
# {
|
|
3381
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3382
|
-
# type: "ColumnTypeString",
|
|
3383
|
-
# },
|
|
3384
|
-
# ],
|
|
3385
|
-
# },
|
|
3386
|
-
# ],
|
|
3387
|
-
# },
|
|
3388
|
-
# s3_json_source: {
|
|
3389
|
-
# name: "NodeName", # required
|
|
3390
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
3391
|
-
# compression_type: "gzip", # accepts gzip, bzip2
|
|
3392
|
-
# exclusions: ["EnclosedInStringProperty"],
|
|
3393
|
-
# group_size: "EnclosedInStringProperty",
|
|
3394
|
-
# group_files: "EnclosedInStringProperty",
|
|
3395
|
-
# recurse: false,
|
|
3396
|
-
# max_band: 1,
|
|
3397
|
-
# max_files_in_band: 1,
|
|
3398
|
-
# additional_options: {
|
|
3399
|
-
# bounded_size: 1,
|
|
3400
|
-
# bounded_files: 1,
|
|
3401
|
-
# enable_sample_path: false,
|
|
3402
|
-
# sample_path: "EnclosedInStringProperty",
|
|
3403
|
-
# },
|
|
3404
|
-
# json_path: "EnclosedInStringProperty",
|
|
3405
|
-
# multiline: false,
|
|
3406
|
-
# output_schemas: [
|
|
3407
|
-
# {
|
|
3408
|
-
# columns: [
|
|
3409
|
-
# {
|
|
3410
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3411
|
-
# type: "ColumnTypeString",
|
|
3412
|
-
# },
|
|
3413
|
-
# ],
|
|
3414
|
-
# },
|
|
3415
|
-
# ],
|
|
3416
|
-
# },
|
|
3417
|
-
# s3_parquet_source: {
|
|
3418
|
-
# name: "NodeName", # required
|
|
3419
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
3420
|
-
# compression_type: "snappy", # accepts snappy, lzo, gzip, uncompressed, none
|
|
3421
|
-
# exclusions: ["EnclosedInStringProperty"],
|
|
3422
|
-
# group_size: "EnclosedInStringProperty",
|
|
3423
|
-
# group_files: "EnclosedInStringProperty",
|
|
3424
|
-
# recurse: false,
|
|
3425
|
-
# max_band: 1,
|
|
3426
|
-
# max_files_in_band: 1,
|
|
3427
|
-
# additional_options: {
|
|
3428
|
-
# bounded_size: 1,
|
|
3429
|
-
# bounded_files: 1,
|
|
3430
|
-
# enable_sample_path: false,
|
|
3431
|
-
# sample_path: "EnclosedInStringProperty",
|
|
3432
|
-
# },
|
|
3433
|
-
# output_schemas: [
|
|
3434
|
-
# {
|
|
3435
|
-
# columns: [
|
|
3436
|
-
# {
|
|
3437
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3438
|
-
# type: "ColumnTypeString",
|
|
3439
|
-
# },
|
|
3440
|
-
# ],
|
|
3441
|
-
# },
|
|
3442
|
-
# ],
|
|
3443
|
-
# },
|
|
3444
|
-
# relational_catalog_source: {
|
|
3445
|
-
# name: "NodeName", # required
|
|
3446
|
-
# database: "EnclosedInStringProperty", # required
|
|
3447
|
-
# table: "EnclosedInStringProperty", # required
|
|
3448
|
-
# },
|
|
3449
|
-
# dynamo_db_catalog_source: {
|
|
3450
|
-
# name: "NodeName", # required
|
|
3451
|
-
# database: "EnclosedInStringProperty", # required
|
|
3452
|
-
# table: "EnclosedInStringProperty", # required
|
|
3453
|
-
# },
|
|
3454
|
-
# jdbc_connector_target: {
|
|
3455
|
-
# name: "NodeName", # required
|
|
3456
|
-
# inputs: ["NodeId"], # required
|
|
3457
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
3458
|
-
# connection_table: "EnclosedInStringPropertyWithQuote", # required
|
|
3459
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
3460
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
3461
|
-
# additional_options: {
|
|
3462
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3463
|
-
# },
|
|
3464
|
-
# output_schemas: [
|
|
3465
|
-
# {
|
|
3466
|
-
# columns: [
|
|
3467
|
-
# {
|
|
3468
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3469
|
-
# type: "ColumnTypeString",
|
|
3470
|
-
# },
|
|
3471
|
-
# ],
|
|
3472
|
-
# },
|
|
3473
|
-
# ],
|
|
3474
|
-
# },
|
|
3475
|
-
# spark_connector_target: {
|
|
3476
|
-
# name: "NodeName", # required
|
|
3477
|
-
# inputs: ["NodeId"], # required
|
|
3478
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
3479
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
3480
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
3481
|
-
# additional_options: {
|
|
3482
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3483
|
-
# },
|
|
3484
|
-
# output_schemas: [
|
|
3485
|
-
# {
|
|
3486
|
-
# columns: [
|
|
3487
|
-
# {
|
|
3488
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3489
|
-
# type: "ColumnTypeString",
|
|
3490
|
-
# },
|
|
3491
|
-
# ],
|
|
3492
|
-
# },
|
|
3493
|
-
# ],
|
|
3494
|
-
# },
|
|
3495
|
-
# catalog_target: {
|
|
3496
|
-
# name: "NodeName", # required
|
|
3497
|
-
# inputs: ["NodeId"], # required
|
|
3498
|
-
# database: "EnclosedInStringProperty", # required
|
|
3499
|
-
# table: "EnclosedInStringProperty", # required
|
|
3500
|
-
# },
|
|
3501
|
-
# redshift_target: {
|
|
3502
|
-
# name: "NodeName", # required
|
|
3503
|
-
# inputs: ["NodeId"], # required
|
|
3504
|
-
# database: "EnclosedInStringProperty", # required
|
|
3505
|
-
# table: "EnclosedInStringProperty", # required
|
|
3506
|
-
# redshift_tmp_dir: "EnclosedInStringProperty",
|
|
3507
|
-
# tmp_dir_iam_role: "EnclosedInStringProperty",
|
|
3508
|
-
# upsert_redshift_options: {
|
|
3509
|
-
# table_location: "EnclosedInStringProperty",
|
|
3510
|
-
# connection_name: "EnclosedInStringProperty",
|
|
3511
|
-
# upsert_keys: ["EnclosedInStringProperty"],
|
|
3512
|
-
# },
|
|
3513
|
-
# },
|
|
3514
|
-
# s3_catalog_target: {
|
|
3515
|
-
# name: "NodeName", # required
|
|
3516
|
-
# inputs: ["NodeId"], # required
|
|
3517
|
-
# partition_keys: [
|
|
3518
|
-
# ["EnclosedInStringProperty"],
|
|
3519
|
-
# ],
|
|
3520
|
-
# table: "EnclosedInStringProperty", # required
|
|
3521
|
-
# database: "EnclosedInStringProperty", # required
|
|
3522
|
-
# schema_change_policy: {
|
|
3523
|
-
# enable_update_catalog: false,
|
|
3524
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
3525
|
-
# },
|
|
3526
|
-
# },
|
|
3527
|
-
# s3_glue_parquet_target: {
|
|
3528
|
-
# name: "NodeName", # required
|
|
3529
|
-
# inputs: ["NodeId"], # required
|
|
3530
|
-
# partition_keys: [
|
|
3531
|
-
# ["EnclosedInStringProperty"],
|
|
3532
|
-
# ],
|
|
3533
|
-
# path: "EnclosedInStringProperty", # required
|
|
3534
|
-
# compression: "snappy", # accepts snappy, lzo, gzip, uncompressed, none
|
|
3535
|
-
# schema_change_policy: {
|
|
3536
|
-
# enable_update_catalog: false,
|
|
3537
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
3538
|
-
# table: "EnclosedInStringProperty",
|
|
3539
|
-
# database: "EnclosedInStringProperty",
|
|
3540
|
-
# },
|
|
3541
|
-
# },
|
|
3542
|
-
# s3_direct_target: {
|
|
3543
|
-
# name: "NodeName", # required
|
|
3544
|
-
# inputs: ["NodeId"], # required
|
|
3545
|
-
# partition_keys: [
|
|
3546
|
-
# ["EnclosedInStringProperty"],
|
|
3547
|
-
# ],
|
|
3548
|
-
# path: "EnclosedInStringProperty", # required
|
|
3549
|
-
# compression: "EnclosedInStringProperty",
|
|
3550
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi, delta
|
|
3551
|
-
# schema_change_policy: {
|
|
3552
|
-
# enable_update_catalog: false,
|
|
3553
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
3554
|
-
# table: "EnclosedInStringProperty",
|
|
3555
|
-
# database: "EnclosedInStringProperty",
|
|
3556
|
-
# },
|
|
3557
|
-
# },
|
|
3558
|
-
# apply_mapping: {
|
|
3559
|
-
# name: "NodeName", # required
|
|
3560
|
-
# inputs: ["NodeId"], # required
|
|
3561
|
-
# mapping: [ # required
|
|
3562
|
-
# {
|
|
3563
|
-
# to_key: "EnclosedInStringProperty",
|
|
3564
|
-
# from_path: ["EnclosedInStringProperty"],
|
|
3565
|
-
# from_type: "EnclosedInStringProperty",
|
|
3566
|
-
# to_type: "EnclosedInStringProperty",
|
|
3567
|
-
# dropped: false,
|
|
3568
|
-
# children: {
|
|
3569
|
-
# # recursive Mappings
|
|
3570
|
-
# },
|
|
3571
|
-
# },
|
|
3572
|
-
# ],
|
|
3573
|
-
# },
|
|
3574
|
-
# select_fields: {
|
|
3575
|
-
# name: "NodeName", # required
|
|
3576
|
-
# inputs: ["NodeId"], # required
|
|
3577
|
-
# paths: [ # required
|
|
3578
|
-
# ["EnclosedInStringProperty"],
|
|
3579
|
-
# ],
|
|
3580
|
-
# },
|
|
3581
|
-
# drop_fields: {
|
|
3582
|
-
# name: "NodeName", # required
|
|
3583
|
-
# inputs: ["NodeId"], # required
|
|
3584
|
-
# paths: [ # required
|
|
3585
|
-
# ["EnclosedInStringProperty"],
|
|
3586
|
-
# ],
|
|
3587
|
-
# },
|
|
3588
|
-
# rename_field: {
|
|
3589
|
-
# name: "NodeName", # required
|
|
3590
|
-
# inputs: ["NodeId"], # required
|
|
3591
|
-
# source_path: ["EnclosedInStringProperty"], # required
|
|
3592
|
-
# target_path: ["EnclosedInStringProperty"], # required
|
|
3593
|
-
# },
|
|
3594
|
-
# spigot: {
|
|
3595
|
-
# name: "NodeName", # required
|
|
3596
|
-
# inputs: ["NodeId"], # required
|
|
3597
|
-
# path: "EnclosedInStringProperty", # required
|
|
3598
|
-
# topk: 1,
|
|
3599
|
-
# prob: 1.0,
|
|
3600
|
-
# },
|
|
3601
|
-
# join: {
|
|
3602
|
-
# name: "NodeName", # required
|
|
3603
|
-
# inputs: ["NodeId"], # required
|
|
3604
|
-
# join_type: "equijoin", # required, accepts equijoin, left, right, outer, leftsemi, leftanti
|
|
3605
|
-
# columns: [ # required
|
|
3606
|
-
# {
|
|
3607
|
-
# from: "EnclosedInStringProperty", # required
|
|
3608
|
-
# keys: [ # required
|
|
3609
|
-
# ["EnclosedInStringProperty"],
|
|
3610
|
-
# ],
|
|
3611
|
-
# },
|
|
3612
|
-
# ],
|
|
3613
|
-
# },
|
|
3614
|
-
# split_fields: {
|
|
3615
|
-
# name: "NodeName", # required
|
|
3616
|
-
# inputs: ["NodeId"], # required
|
|
3617
|
-
# paths: [ # required
|
|
3618
|
-
# ["EnclosedInStringProperty"],
|
|
3619
|
-
# ],
|
|
3620
|
-
# },
|
|
3621
|
-
# select_from_collection: {
|
|
3622
|
-
# name: "NodeName", # required
|
|
3623
|
-
# inputs: ["NodeId"], # required
|
|
3624
|
-
# index: 1, # required
|
|
3625
|
-
# },
|
|
3626
|
-
# fill_missing_values: {
|
|
3627
|
-
# name: "NodeName", # required
|
|
3628
|
-
# inputs: ["NodeId"], # required
|
|
3629
|
-
# imputed_path: "EnclosedInStringProperty", # required
|
|
3630
|
-
# filled_path: "EnclosedInStringProperty",
|
|
3631
|
-
# },
|
|
3632
|
-
# filter: {
|
|
3633
|
-
# name: "NodeName", # required
|
|
3634
|
-
# inputs: ["NodeId"], # required
|
|
3635
|
-
# logical_operator: "AND", # required, accepts AND, OR
|
|
3636
|
-
# filters: [ # required
|
|
3637
|
-
# {
|
|
3638
|
-
# operation: "EQ", # required, accepts EQ, LT, GT, LTE, GTE, REGEX, ISNULL
|
|
3639
|
-
# negated: false,
|
|
3640
|
-
# values: [ # required
|
|
3641
|
-
# {
|
|
3642
|
-
# type: "COLUMNEXTRACTED", # required, accepts COLUMNEXTRACTED, CONSTANT
|
|
3643
|
-
# value: ["EnclosedInStringProperty"], # required
|
|
3644
|
-
# },
|
|
3645
|
-
# ],
|
|
3646
|
-
# },
|
|
3647
|
-
# ],
|
|
3648
|
-
# },
|
|
3649
|
-
# custom_code: {
|
|
3650
|
-
# name: "NodeName", # required
|
|
3651
|
-
# inputs: ["NodeId"], # required
|
|
3652
|
-
# code: "ExtendedString", # required
|
|
3653
|
-
# class_name: "EnclosedInStringProperty", # required
|
|
3654
|
-
# output_schemas: [
|
|
3655
|
-
# {
|
|
3656
|
-
# columns: [
|
|
3657
|
-
# {
|
|
3658
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3659
|
-
# type: "ColumnTypeString",
|
|
3660
|
-
# },
|
|
3661
|
-
# ],
|
|
3662
|
-
# },
|
|
3663
|
-
# ],
|
|
3664
|
-
# },
|
|
3665
|
-
# spark_sql: {
|
|
3666
|
-
# name: "NodeName", # required
|
|
3667
|
-
# inputs: ["NodeId"], # required
|
|
3668
|
-
# sql_query: "SqlQuery", # required
|
|
3669
|
-
# sql_aliases: [ # required
|
|
3670
|
-
# {
|
|
3671
|
-
# from: "NodeId", # required
|
|
3672
|
-
# alias: "EnclosedInStringPropertyWithQuote", # required
|
|
3673
|
-
# },
|
|
3674
|
-
# ],
|
|
3675
|
-
# output_schemas: [
|
|
3676
|
-
# {
|
|
3677
|
-
# columns: [
|
|
3678
|
-
# {
|
|
3679
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3680
|
-
# type: "ColumnTypeString",
|
|
3681
|
-
# },
|
|
3682
|
-
# ],
|
|
3683
|
-
# },
|
|
3684
|
-
# ],
|
|
3685
|
-
# },
|
|
3686
|
-
# direct_kinesis_source: {
|
|
3687
|
-
# name: "NodeName", # required
|
|
3688
|
-
# window_size: 1,
|
|
3689
|
-
# detect_schema: false,
|
|
3690
|
-
# streaming_options: {
|
|
3691
|
-
# endpoint_url: "EnclosedInStringProperty",
|
|
3692
|
-
# stream_name: "EnclosedInStringProperty",
|
|
3693
|
-
# classification: "EnclosedInStringProperty",
|
|
3694
|
-
# delimiter: "EnclosedInStringProperty",
|
|
3695
|
-
# starting_position: "latest", # accepts latest, trim_horizon, earliest
|
|
3696
|
-
# max_fetch_time_in_ms: 1,
|
|
3697
|
-
# max_fetch_records_per_shard: 1,
|
|
3698
|
-
# max_record_per_read: 1,
|
|
3699
|
-
# add_idle_time_between_reads: false,
|
|
3700
|
-
# idle_time_between_reads_in_ms: 1,
|
|
3701
|
-
# describe_shard_interval: 1,
|
|
3702
|
-
# num_retries: 1,
|
|
3703
|
-
# retry_interval_ms: 1,
|
|
3704
|
-
# max_retry_interval_ms: 1,
|
|
3705
|
-
# avoid_empty_batches: false,
|
|
3706
|
-
# stream_arn: "EnclosedInStringProperty",
|
|
3707
|
-
# role_arn: "EnclosedInStringProperty",
|
|
3708
|
-
# role_session_name: "EnclosedInStringProperty",
|
|
3709
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3710
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3711
|
-
# },
|
|
3712
|
-
# data_preview_options: {
|
|
3713
|
-
# polling_time: 1,
|
|
3714
|
-
# record_polling_limit: 1,
|
|
3715
|
-
# },
|
|
3716
|
-
# },
|
|
3717
|
-
# direct_kafka_source: {
|
|
3718
|
-
# name: "NodeName", # required
|
|
3719
|
-
# streaming_options: {
|
|
3720
|
-
# bootstrap_servers: "EnclosedInStringProperty",
|
|
3721
|
-
# security_protocol: "EnclosedInStringProperty",
|
|
3722
|
-
# connection_name: "EnclosedInStringProperty",
|
|
3723
|
-
# topic_name: "EnclosedInStringProperty",
|
|
3724
|
-
# assign: "EnclosedInStringProperty",
|
|
3725
|
-
# subscribe_pattern: "EnclosedInStringProperty",
|
|
3726
|
-
# classification: "EnclosedInStringProperty",
|
|
3727
|
-
# delimiter: "EnclosedInStringProperty",
|
|
3728
|
-
# starting_offsets: "EnclosedInStringProperty",
|
|
3729
|
-
# ending_offsets: "EnclosedInStringProperty",
|
|
3730
|
-
# poll_timeout_ms: 1,
|
|
3731
|
-
# num_retries: 1,
|
|
3732
|
-
# retry_interval_ms: 1,
|
|
3733
|
-
# max_offsets_per_trigger: 1,
|
|
3734
|
-
# min_partitions: 1,
|
|
3735
|
-
# include_headers: false,
|
|
3736
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3737
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3738
|
-
# },
|
|
3739
|
-
# window_size: 1,
|
|
3740
|
-
# detect_schema: false,
|
|
3741
|
-
# data_preview_options: {
|
|
3742
|
-
# polling_time: 1,
|
|
3743
|
-
# record_polling_limit: 1,
|
|
3744
|
-
# },
|
|
3745
|
-
# },
|
|
3746
|
-
# catalog_kinesis_source: {
|
|
3747
|
-
# name: "NodeName", # required
|
|
3748
|
-
# window_size: 1,
|
|
3749
|
-
# detect_schema: false,
|
|
3750
|
-
# table: "EnclosedInStringProperty", # required
|
|
3751
|
-
# database: "EnclosedInStringProperty", # required
|
|
3752
|
-
# streaming_options: {
|
|
3753
|
-
# endpoint_url: "EnclosedInStringProperty",
|
|
3754
|
-
# stream_name: "EnclosedInStringProperty",
|
|
3755
|
-
# classification: "EnclosedInStringProperty",
|
|
3756
|
-
# delimiter: "EnclosedInStringProperty",
|
|
3757
|
-
# starting_position: "latest", # accepts latest, trim_horizon, earliest
|
|
3758
|
-
# max_fetch_time_in_ms: 1,
|
|
3759
|
-
# max_fetch_records_per_shard: 1,
|
|
3760
|
-
# max_record_per_read: 1,
|
|
3761
|
-
# add_idle_time_between_reads: false,
|
|
3762
|
-
# idle_time_between_reads_in_ms: 1,
|
|
3763
|
-
# describe_shard_interval: 1,
|
|
3764
|
-
# num_retries: 1,
|
|
3765
|
-
# retry_interval_ms: 1,
|
|
3766
|
-
# max_retry_interval_ms: 1,
|
|
3767
|
-
# avoid_empty_batches: false,
|
|
3768
|
-
# stream_arn: "EnclosedInStringProperty",
|
|
3769
|
-
# role_arn: "EnclosedInStringProperty",
|
|
3770
|
-
# role_session_name: "EnclosedInStringProperty",
|
|
3771
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3772
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3773
|
-
# },
|
|
3774
|
-
# data_preview_options: {
|
|
3775
|
-
# polling_time: 1,
|
|
3776
|
-
# record_polling_limit: 1,
|
|
3777
|
-
# },
|
|
3778
|
-
# },
|
|
3779
|
-
# catalog_kafka_source: {
|
|
3780
|
-
# name: "NodeName", # required
|
|
3781
|
-
# window_size: 1,
|
|
3782
|
-
# detect_schema: false,
|
|
3783
|
-
# table: "EnclosedInStringProperty", # required
|
|
3784
|
-
# database: "EnclosedInStringProperty", # required
|
|
3785
|
-
# streaming_options: {
|
|
3786
|
-
# bootstrap_servers: "EnclosedInStringProperty",
|
|
3787
|
-
# security_protocol: "EnclosedInStringProperty",
|
|
3788
|
-
# connection_name: "EnclosedInStringProperty",
|
|
3789
|
-
# topic_name: "EnclosedInStringProperty",
|
|
3790
|
-
# assign: "EnclosedInStringProperty",
|
|
3791
|
-
# subscribe_pattern: "EnclosedInStringProperty",
|
|
3792
|
-
# classification: "EnclosedInStringProperty",
|
|
3793
|
-
# delimiter: "EnclosedInStringProperty",
|
|
3794
|
-
# starting_offsets: "EnclosedInStringProperty",
|
|
3795
|
-
# ending_offsets: "EnclosedInStringProperty",
|
|
3796
|
-
# poll_timeout_ms: 1,
|
|
3797
|
-
# num_retries: 1,
|
|
3798
|
-
# retry_interval_ms: 1,
|
|
3799
|
-
# max_offsets_per_trigger: 1,
|
|
3800
|
-
# min_partitions: 1,
|
|
3801
|
-
# include_headers: false,
|
|
3802
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3803
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3804
|
-
# },
|
|
3805
|
-
# data_preview_options: {
|
|
3806
|
-
# polling_time: 1,
|
|
3807
|
-
# record_polling_limit: 1,
|
|
3808
|
-
# },
|
|
3809
|
-
# },
|
|
3810
|
-
# drop_null_fields: {
|
|
3811
|
-
# name: "NodeName", # required
|
|
3812
|
-
# inputs: ["NodeId"], # required
|
|
3813
|
-
# null_check_box_list: {
|
|
3814
|
-
# is_empty: false,
|
|
3815
|
-
# is_null_string: false,
|
|
3816
|
-
# is_neg_one: false,
|
|
3817
|
-
# },
|
|
3818
|
-
# null_text_list: [
|
|
3819
|
-
# {
|
|
3820
|
-
# value: "EnclosedInStringProperty", # required
|
|
3821
|
-
# datatype: { # required
|
|
3822
|
-
# id: "GenericLimitedString", # required
|
|
3823
|
-
# label: "GenericLimitedString", # required
|
|
3824
|
-
# },
|
|
3825
|
-
# },
|
|
3826
|
-
# ],
|
|
3827
|
-
# },
|
|
3828
|
-
# merge: {
|
|
3829
|
-
# name: "NodeName", # required
|
|
3830
|
-
# inputs: ["NodeId"], # required
|
|
3831
|
-
# source: "NodeId", # required
|
|
3832
|
-
# primary_keys: [ # required
|
|
3833
|
-
# ["EnclosedInStringProperty"],
|
|
3834
|
-
# ],
|
|
3835
|
-
# },
|
|
3836
|
-
# union: {
|
|
3837
|
-
# name: "NodeName", # required
|
|
3838
|
-
# inputs: ["NodeId"], # required
|
|
3839
|
-
# union_type: "ALL", # required, accepts ALL, DISTINCT
|
|
3840
|
-
# },
|
|
3841
|
-
# pii_detection: {
|
|
3842
|
-
# name: "NodeName", # required
|
|
3843
|
-
# inputs: ["NodeId"], # required
|
|
3844
|
-
# pii_type: "RowAudit", # required, accepts RowAudit, RowMasking, ColumnAudit, ColumnMasking
|
|
3845
|
-
# entity_types_to_detect: ["EnclosedInStringProperty"], # required
|
|
3846
|
-
# output_column_name: "EnclosedInStringProperty",
|
|
3847
|
-
# sample_fraction: 1.0,
|
|
3848
|
-
# threshold_fraction: 1.0,
|
|
3849
|
-
# mask_value: "MaskValue",
|
|
3850
|
-
# },
|
|
3851
|
-
# aggregate: {
|
|
3852
|
-
# name: "NodeName", # required
|
|
3853
|
-
# inputs: ["NodeId"], # required
|
|
3854
|
-
# groups: [ # required
|
|
3855
|
-
# ["EnclosedInStringProperty"],
|
|
3856
|
-
# ],
|
|
3857
|
-
# aggs: [ # required
|
|
3858
|
-
# {
|
|
3859
|
-
# column: ["EnclosedInStringProperty"], # required
|
|
3860
|
-
# agg_func: "avg", # required, accepts avg, countDistinct, count, first, last, kurtosis, max, min, skewness, stddev_samp, stddev_pop, sum, sumDistinct, var_samp, var_pop
|
|
3861
|
-
# },
|
|
3862
|
-
# ],
|
|
3863
|
-
# },
|
|
3864
|
-
# drop_duplicates: {
|
|
3865
|
-
# name: "NodeName", # required
|
|
3866
|
-
# inputs: ["NodeId"], # required
|
|
3867
|
-
# columns: [
|
|
3868
|
-
# ["GenericLimitedString"],
|
|
3869
|
-
# ],
|
|
3870
|
-
# },
|
|
3871
|
-
# governed_catalog_target: {
|
|
3872
|
-
# name: "NodeName", # required
|
|
3873
|
-
# inputs: ["NodeId"], # required
|
|
3874
|
-
# partition_keys: [
|
|
3875
|
-
# ["EnclosedInStringProperty"],
|
|
3876
|
-
# ],
|
|
3877
|
-
# table: "EnclosedInStringProperty", # required
|
|
3878
|
-
# database: "EnclosedInStringProperty", # required
|
|
3879
|
-
# schema_change_policy: {
|
|
3880
|
-
# enable_update_catalog: false,
|
|
3881
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
3882
|
-
# },
|
|
3883
|
-
# },
|
|
3884
|
-
# governed_catalog_source: {
|
|
3885
|
-
# name: "NodeName", # required
|
|
3886
|
-
# database: "EnclosedInStringProperty", # required
|
|
3887
|
-
# table: "EnclosedInStringProperty", # required
|
|
3888
|
-
# partition_predicate: "EnclosedInStringProperty",
|
|
3889
|
-
# additional_options: {
|
|
3890
|
-
# bounded_size: 1,
|
|
3891
|
-
# bounded_files: 1,
|
|
3892
|
-
# },
|
|
3893
|
-
# },
|
|
3894
|
-
# microsoft_sql_server_catalog_source: {
|
|
3895
|
-
# name: "NodeName", # required
|
|
3896
|
-
# database: "EnclosedInStringProperty", # required
|
|
3897
|
-
# table: "EnclosedInStringProperty", # required
|
|
3898
|
-
# },
|
|
3899
|
-
# my_sql_catalog_source: {
|
|
3900
|
-
# name: "NodeName", # required
|
|
3901
|
-
# database: "EnclosedInStringProperty", # required
|
|
3902
|
-
# table: "EnclosedInStringProperty", # required
|
|
3903
|
-
# },
|
|
3904
|
-
# oracle_sql_catalog_source: {
|
|
3905
|
-
# name: "NodeName", # required
|
|
3906
|
-
# database: "EnclosedInStringProperty", # required
|
|
3907
|
-
# table: "EnclosedInStringProperty", # required
|
|
3908
|
-
# },
|
|
3909
|
-
# postgre_sql_catalog_source: {
|
|
3910
|
-
# name: "NodeName", # required
|
|
3911
|
-
# database: "EnclosedInStringProperty", # required
|
|
3912
|
-
# table: "EnclosedInStringProperty", # required
|
|
3913
|
-
# },
|
|
3914
|
-
# microsoft_sql_server_catalog_target: {
|
|
3915
|
-
# name: "NodeName", # required
|
|
3916
|
-
# inputs: ["NodeId"], # required
|
|
3917
|
-
# database: "EnclosedInStringProperty", # required
|
|
3918
|
-
# table: "EnclosedInStringProperty", # required
|
|
3919
|
-
# },
|
|
3920
|
-
# my_sql_catalog_target: {
|
|
3921
|
-
# name: "NodeName", # required
|
|
3922
|
-
# inputs: ["NodeId"], # required
|
|
3923
|
-
# database: "EnclosedInStringProperty", # required
|
|
3924
|
-
# table: "EnclosedInStringProperty", # required
|
|
3925
|
-
# },
|
|
3926
|
-
# oracle_sql_catalog_target: {
|
|
3927
|
-
# name: "NodeName", # required
|
|
3928
|
-
# inputs: ["NodeId"], # required
|
|
3929
|
-
# database: "EnclosedInStringProperty", # required
|
|
3930
|
-
# table: "EnclosedInStringProperty", # required
|
|
3931
|
-
# },
|
|
3932
|
-
# postgre_sql_catalog_target: {
|
|
3933
|
-
# name: "NodeName", # required
|
|
3934
|
-
# inputs: ["NodeId"], # required
|
|
3935
|
-
# database: "EnclosedInStringProperty", # required
|
|
3936
|
-
# table: "EnclosedInStringProperty", # required
|
|
3937
|
-
# },
|
|
3938
|
-
# dynamic_transform: {
|
|
3939
|
-
# name: "EnclosedInStringProperty", # required
|
|
3940
|
-
# transform_name: "EnclosedInStringProperty", # required
|
|
3941
|
-
# inputs: ["NodeId"], # required
|
|
3942
|
-
# parameters: [
|
|
3943
|
-
# {
|
|
3944
|
-
# name: "EnclosedInStringProperty", # required
|
|
3945
|
-
# type: "str", # required, accepts str, int, float, complex, bool, list, null
|
|
3946
|
-
# validation_rule: "EnclosedInStringProperty",
|
|
3947
|
-
# validation_message: "EnclosedInStringProperty",
|
|
3948
|
-
# value: ["EnclosedInStringProperty"],
|
|
3949
|
-
# list_type: "str", # accepts str, int, float, complex, bool, list, null
|
|
3950
|
-
# is_optional: false,
|
|
3951
|
-
# },
|
|
3952
|
-
# ],
|
|
3953
|
-
# function_name: "EnclosedInStringProperty", # required
|
|
3954
|
-
# path: "EnclosedInStringProperty", # required
|
|
3955
|
-
# version: "EnclosedInStringProperty",
|
|
3956
|
-
# },
|
|
3957
|
-
# evaluate_data_quality: {
|
|
3958
|
-
# name: "NodeName", # required
|
|
3959
|
-
# inputs: ["NodeId"], # required
|
|
3960
|
-
# ruleset: "DQDLString", # required
|
|
3961
|
-
# output: "PrimaryInput", # accepts PrimaryInput, EvaluationResults
|
|
3962
|
-
# publishing_options: {
|
|
3963
|
-
# evaluation_context: "GenericLimitedString",
|
|
3964
|
-
# results_s3_prefix: "EnclosedInStringProperty",
|
|
3965
|
-
# cloud_watch_metrics_enabled: false,
|
|
3966
|
-
# results_publishing_enabled: false,
|
|
3967
|
-
# },
|
|
3968
|
-
# stop_job_on_failure_options: {
|
|
3969
|
-
# stop_job_on_failure_timing: "Immediate", # accepts Immediate, AfterDataLoad
|
|
3970
|
-
# },
|
|
3971
|
-
# },
|
|
3972
|
-
# s3_catalog_hudi_source: {
|
|
3973
|
-
# name: "NodeName", # required
|
|
3974
|
-
# database: "EnclosedInStringProperty", # required
|
|
3975
|
-
# table: "EnclosedInStringProperty", # required
|
|
3976
|
-
# additional_hudi_options: {
|
|
3977
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3978
|
-
# },
|
|
3979
|
-
# output_schemas: [
|
|
3980
|
-
# {
|
|
3981
|
-
# columns: [
|
|
3982
|
-
# {
|
|
3983
|
-
# name: "GlueStudioColumnNameString", # required
|
|
3984
|
-
# type: "ColumnTypeString",
|
|
3985
|
-
# },
|
|
3986
|
-
# ],
|
|
3987
|
-
# },
|
|
3988
|
-
# ],
|
|
3989
|
-
# },
|
|
3990
|
-
# catalog_hudi_source: {
|
|
3991
|
-
# name: "NodeName", # required
|
|
3992
|
-
# database: "EnclosedInStringProperty", # required
|
|
3993
|
-
# table: "EnclosedInStringProperty", # required
|
|
3994
|
-
# additional_hudi_options: {
|
|
3995
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3996
|
-
# },
|
|
3997
|
-
# output_schemas: [
|
|
3998
|
-
# {
|
|
3999
|
-
# columns: [
|
|
4000
|
-
# {
|
|
4001
|
-
# name: "GlueStudioColumnNameString", # required
|
|
4002
|
-
# type: "ColumnTypeString",
|
|
4003
|
-
# },
|
|
4004
|
-
# ],
|
|
4005
|
-
# },
|
|
4006
|
-
# ],
|
|
4007
|
-
# },
|
|
4008
|
-
# s3_hudi_source: {
|
|
4009
|
-
# name: "NodeName", # required
|
|
4010
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
4011
|
-
# additional_hudi_options: {
|
|
4012
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4013
|
-
# },
|
|
4014
|
-
# additional_options: {
|
|
4015
|
-
# bounded_size: 1,
|
|
4016
|
-
# bounded_files: 1,
|
|
4017
|
-
# enable_sample_path: false,
|
|
4018
|
-
# sample_path: "EnclosedInStringProperty",
|
|
4019
|
-
# },
|
|
4020
|
-
# output_schemas: [
|
|
4021
|
-
# {
|
|
4022
|
-
# columns: [
|
|
4023
|
-
# {
|
|
4024
|
-
# name: "GlueStudioColumnNameString", # required
|
|
4025
|
-
# type: "ColumnTypeString",
|
|
4026
|
-
# },
|
|
4027
|
-
# ],
|
|
4028
|
-
# },
|
|
4029
|
-
# ],
|
|
4030
|
-
# },
|
|
4031
|
-
# s3_hudi_catalog_target: {
|
|
4032
|
-
# name: "NodeName", # required
|
|
4033
|
-
# inputs: ["NodeId"], # required
|
|
4034
|
-
# partition_keys: [
|
|
4035
|
-
# ["EnclosedInStringProperty"],
|
|
4036
|
-
# ],
|
|
4037
|
-
# table: "EnclosedInStringProperty", # required
|
|
4038
|
-
# database: "EnclosedInStringProperty", # required
|
|
4039
|
-
# additional_options: { # required
|
|
4040
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4041
|
-
# },
|
|
4042
|
-
# schema_change_policy: {
|
|
4043
|
-
# enable_update_catalog: false,
|
|
4044
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
4045
|
-
# },
|
|
4046
|
-
# },
|
|
4047
|
-
# s3_hudi_direct_target: {
|
|
4048
|
-
# name: "NodeName", # required
|
|
4049
|
-
# inputs: ["NodeId"], # required
|
|
4050
|
-
# path: "EnclosedInStringProperty", # required
|
|
4051
|
-
# compression: "gzip", # required, accepts gzip, lzo, uncompressed, snappy
|
|
4052
|
-
# partition_keys: [
|
|
4053
|
-
# ["EnclosedInStringProperty"],
|
|
4054
|
-
# ],
|
|
4055
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi, delta
|
|
4056
|
-
# additional_options: { # required
|
|
4057
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4058
|
-
# },
|
|
4059
|
-
# schema_change_policy: {
|
|
4060
|
-
# enable_update_catalog: false,
|
|
4061
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
4062
|
-
# table: "EnclosedInStringProperty",
|
|
4063
|
-
# database: "EnclosedInStringProperty",
|
|
4064
|
-
# },
|
|
4065
|
-
# },
|
|
4066
|
-
# direct_jdbc_source: {
|
|
4067
|
-
# name: "NodeName", # required
|
|
4068
|
-
# database: "EnclosedInStringProperty", # required
|
|
4069
|
-
# table: "EnclosedInStringProperty", # required
|
|
4070
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
4071
|
-
# connection_type: "sqlserver", # required, accepts sqlserver, mysql, oracle, postgresql, redshift
|
|
4072
|
-
# redshift_tmp_dir: "EnclosedInStringProperty",
|
|
4073
|
-
# },
|
|
4074
|
-
# s3_catalog_delta_source: {
|
|
4075
|
-
# name: "NodeName", # required
|
|
4076
|
-
# database: "EnclosedInStringProperty", # required
|
|
4077
|
-
# table: "EnclosedInStringProperty", # required
|
|
4078
|
-
# additional_delta_options: {
|
|
4079
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4080
|
-
# },
|
|
4081
|
-
# output_schemas: [
|
|
4082
|
-
# {
|
|
4083
|
-
# columns: [
|
|
4084
|
-
# {
|
|
4085
|
-
# name: "GlueStudioColumnNameString", # required
|
|
4086
|
-
# type: "ColumnTypeString",
|
|
4087
|
-
# },
|
|
4088
|
-
# ],
|
|
4089
|
-
# },
|
|
4090
|
-
# ],
|
|
4091
|
-
# },
|
|
4092
|
-
# catalog_delta_source: {
|
|
4093
|
-
# name: "NodeName", # required
|
|
4094
|
-
# database: "EnclosedInStringProperty", # required
|
|
4095
|
-
# table: "EnclosedInStringProperty", # required
|
|
4096
|
-
# additional_delta_options: {
|
|
4097
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4098
|
-
# },
|
|
4099
|
-
# output_schemas: [
|
|
4100
|
-
# {
|
|
4101
|
-
# columns: [
|
|
4102
|
-
# {
|
|
4103
|
-
# name: "GlueStudioColumnNameString", # required
|
|
4104
|
-
# type: "ColumnTypeString",
|
|
4105
|
-
# },
|
|
4106
|
-
# ],
|
|
4107
|
-
# },
|
|
4108
|
-
# ],
|
|
4109
|
-
# },
|
|
4110
|
-
# s3_delta_source: {
|
|
4111
|
-
# name: "NodeName", # required
|
|
4112
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
4113
|
-
# additional_delta_options: {
|
|
4114
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4115
|
-
# },
|
|
4116
|
-
# additional_options: {
|
|
4117
|
-
# bounded_size: 1,
|
|
4118
|
-
# bounded_files: 1,
|
|
4119
|
-
# enable_sample_path: false,
|
|
4120
|
-
# sample_path: "EnclosedInStringProperty",
|
|
4121
|
-
# },
|
|
4122
|
-
# output_schemas: [
|
|
4123
|
-
# {
|
|
4124
|
-
# columns: [
|
|
4125
|
-
# {
|
|
4126
|
-
# name: "GlueStudioColumnNameString", # required
|
|
4127
|
-
# type: "ColumnTypeString",
|
|
4128
|
-
# },
|
|
4129
|
-
# ],
|
|
4130
|
-
# },
|
|
4131
|
-
# ],
|
|
4132
|
-
# },
|
|
4133
|
-
# s3_delta_catalog_target: {
|
|
4134
|
-
# name: "NodeName", # required
|
|
4135
|
-
# inputs: ["NodeId"], # required
|
|
4136
|
-
# partition_keys: [
|
|
4137
|
-
# ["EnclosedInStringProperty"],
|
|
4138
|
-
# ],
|
|
4139
|
-
# table: "EnclosedInStringProperty", # required
|
|
4140
|
-
# database: "EnclosedInStringProperty", # required
|
|
4141
|
-
# additional_options: {
|
|
4142
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4143
|
-
# },
|
|
4144
|
-
# schema_change_policy: {
|
|
4145
|
-
# enable_update_catalog: false,
|
|
4146
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
4147
|
-
# },
|
|
4148
|
-
# },
|
|
4149
|
-
# s3_delta_direct_target: {
|
|
4150
|
-
# name: "NodeName", # required
|
|
4151
|
-
# inputs: ["NodeId"], # required
|
|
4152
|
-
# partition_keys: [
|
|
4153
|
-
# ["EnclosedInStringProperty"],
|
|
4154
|
-
# ],
|
|
4155
|
-
# path: "EnclosedInStringProperty", # required
|
|
4156
|
-
# compression: "uncompressed", # required, accepts uncompressed, snappy
|
|
4157
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi, delta
|
|
4158
|
-
# additional_options: {
|
|
4159
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
4160
|
-
# },
|
|
4161
|
-
# schema_change_policy: {
|
|
4162
|
-
# enable_update_catalog: false,
|
|
4163
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
4164
|
-
# table: "EnclosedInStringProperty",
|
|
4165
|
-
# database: "EnclosedInStringProperty",
|
|
4166
|
-
# },
|
|
4167
|
-
# },
|
|
4168
|
-
# },
|
|
4169
|
-
# },
|
|
4170
|
-
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
|
4171
|
-
# source_control_details: {
|
|
4172
|
-
# provider: "GITHUB", # accepts GITHUB, AWS_CODE_COMMIT
|
|
4173
|
-
# repository: "Generic512CharString",
|
|
4174
|
-
# owner: "Generic512CharString",
|
|
4175
|
-
# branch: "Generic512CharString",
|
|
4176
|
-
# folder: "Generic512CharString",
|
|
4177
|
-
# last_commit_id: "Generic512CharString",
|
|
4178
|
-
# auth_strategy: "PERSONAL_ACCESS_TOKEN", # accepts PERSONAL_ACCESS_TOKEN, AWS_SECRETS_MANAGER
|
|
4179
|
-
# auth_token: "Generic512CharString",
|
|
4180
|
-
# },
|
|
4181
|
-
# })
|
|
4182
|
-
#
|
|
4183
3324
|
# @example Response structure
|
|
4184
3325
|
#
|
|
4185
3326
|
# resp.name #=> String
|
|
@@ -4376,7 +3517,7 @@ module Aws::Glue
|
|
|
4376
3517
|
# role: "RoleString", # required
|
|
4377
3518
|
# glue_version: "GlueVersionString",
|
|
4378
3519
|
# max_capacity: 1.0,
|
|
4379
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
|
3520
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
|
|
4380
3521
|
# number_of_workers: 1,
|
|
4381
3522
|
# timeout: 1,
|
|
4382
3523
|
# max_retries: 1,
|
|
@@ -4950,7 +4091,7 @@ module Aws::Glue
|
|
|
4950
4091
|
# },
|
|
4951
4092
|
# max_capacity: 1.0,
|
|
4952
4093
|
# number_of_workers: 1,
|
|
4953
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
|
4094
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
|
|
4954
4095
|
# security_configuration: "NameString",
|
|
4955
4096
|
# glue_version: "GlueVersionString",
|
|
4956
4097
|
# tags: {
|
|
@@ -7464,7 +6605,7 @@ module Aws::Glue
|
|
|
7464
6605
|
# resp.dev_endpoint.zeppelin_remote_spark_interpreter_port #=> Integer
|
|
7465
6606
|
# resp.dev_endpoint.public_address #=> String
|
|
7466
6607
|
# resp.dev_endpoint.status #=> String
|
|
7467
|
-
# resp.dev_endpoint.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
6608
|
+
# resp.dev_endpoint.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
7468
6609
|
# resp.dev_endpoint.glue_version #=> String
|
|
7469
6610
|
# resp.dev_endpoint.number_of_workers #=> Integer
|
|
7470
6611
|
# resp.dev_endpoint.number_of_nodes #=> Integer
|
|
@@ -7535,7 +6676,7 @@ module Aws::Glue
|
|
|
7535
6676
|
# resp.dev_endpoints[0].zeppelin_remote_spark_interpreter_port #=> Integer
|
|
7536
6677
|
# resp.dev_endpoints[0].public_address #=> String
|
|
7537
6678
|
# resp.dev_endpoints[0].status #=> String
|
|
7538
|
-
# resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
6679
|
+
# resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
7539
6680
|
# resp.dev_endpoints[0].glue_version #=> String
|
|
7540
6681
|
# resp.dev_endpoints[0].number_of_workers #=> Integer
|
|
7541
6682
|
# resp.dev_endpoints[0].number_of_nodes #=> Integer
|
|
@@ -7601,7 +6742,7 @@ module Aws::Glue
|
|
|
7601
6742
|
# resp.job.allocated_capacity #=> Integer
|
|
7602
6743
|
# resp.job.timeout #=> Integer
|
|
7603
6744
|
# resp.job.max_capacity #=> Float
|
|
7604
|
-
# resp.job.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
6745
|
+
# resp.job.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
7605
6746
|
# resp.job.number_of_workers #=> Integer
|
|
7606
6747
|
# resp.job.security_configuration #=> String
|
|
7607
6748
|
# resp.job.notification_property.notify_delay_after #=> Integer
|
|
@@ -8112,6 +7253,10 @@ module Aws::Glue
|
|
|
8112
7253
|
# resp.job.code_gen_configuration_nodes["NodeId"].dynamic_transform.function_name #=> String
|
|
8113
7254
|
# resp.job.code_gen_configuration_nodes["NodeId"].dynamic_transform.path #=> String
|
|
8114
7255
|
# resp.job.code_gen_configuration_nodes["NodeId"].dynamic_transform.version #=> String
|
|
7256
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas #=> Array
|
|
7257
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns #=> Array
|
|
7258
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns[0].name #=> String
|
|
7259
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns[0].type #=> String
|
|
8115
7260
|
# resp.job.code_gen_configuration_nodes["NodeId"].evaluate_data_quality.name #=> String
|
|
8116
7261
|
# resp.job.code_gen_configuration_nodes["NodeId"].evaluate_data_quality.inputs #=> Array
|
|
8117
7262
|
# resp.job.code_gen_configuration_nodes["NodeId"].evaluate_data_quality.inputs[0] #=> String
|
|
@@ -8244,6 +7389,102 @@ module Aws::Glue
|
|
|
8244
7389
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
8245
7390
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.table #=> String
|
|
8246
7391
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.database #=> String
|
|
7392
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.name #=> String
|
|
7393
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.access_type #=> String
|
|
7394
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.source_type #=> String
|
|
7395
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.value #=> String
|
|
7396
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.label #=> String
|
|
7397
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.description #=> String
|
|
7398
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.value #=> String
|
|
7399
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.label #=> String
|
|
7400
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.description #=> String
|
|
7401
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.value #=> String
|
|
7402
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.label #=> String
|
|
7403
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.description #=> String
|
|
7404
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.value #=> String
|
|
7405
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.label #=> String
|
|
7406
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.description #=> String
|
|
7407
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.value #=> String
|
|
7408
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.label #=> String
|
|
7409
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.description #=> String
|
|
7410
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_redshift_schema #=> String
|
|
7411
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_redshift_table #=> String
|
|
7412
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.temp_dir #=> String
|
|
7413
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.value #=> String
|
|
7414
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.label #=> String
|
|
7415
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.description #=> String
|
|
7416
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options #=> Array
|
|
7417
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options[0].key #=> String
|
|
7418
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options[0].value #=> String
|
|
7419
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.sample_query #=> String
|
|
7420
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.pre_action #=> String
|
|
7421
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.post_action #=> String
|
|
7422
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.action #=> String
|
|
7423
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_prefix #=> String
|
|
7424
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.upsert #=> Boolean
|
|
7425
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_action #=> String
|
|
7426
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_when_matched #=> String
|
|
7427
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_when_not_matched #=> String
|
|
7428
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_clause #=> String
|
|
7429
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.crawler_connection #=> String
|
|
7430
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema #=> Array
|
|
7431
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].value #=> String
|
|
7432
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].label #=> String
|
|
7433
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].description #=> String
|
|
7434
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.staging_table #=> String
|
|
7435
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns #=> Array
|
|
7436
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].value #=> String
|
|
7437
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].label #=> String
|
|
7438
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].description #=> String
|
|
7439
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.name #=> String
|
|
7440
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.access_type #=> String
|
|
7441
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.source_type #=> String
|
|
7442
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.value #=> String
|
|
7443
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.label #=> String
|
|
7444
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.description #=> String
|
|
7445
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.value #=> String
|
|
7446
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.label #=> String
|
|
7447
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.description #=> String
|
|
7448
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.value #=> String
|
|
7449
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.label #=> String
|
|
7450
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.description #=> String
|
|
7451
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.value #=> String
|
|
7452
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.label #=> String
|
|
7453
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.description #=> String
|
|
7454
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.value #=> String
|
|
7455
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.label #=> String
|
|
7456
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.description #=> String
|
|
7457
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_redshift_schema #=> String
|
|
7458
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_redshift_table #=> String
|
|
7459
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.temp_dir #=> String
|
|
7460
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.value #=> String
|
|
7461
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.label #=> String
|
|
7462
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.description #=> String
|
|
7463
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options #=> Array
|
|
7464
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options[0].key #=> String
|
|
7465
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options[0].value #=> String
|
|
7466
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.sample_query #=> String
|
|
7467
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.pre_action #=> String
|
|
7468
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.post_action #=> String
|
|
7469
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.action #=> String
|
|
7470
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_prefix #=> String
|
|
7471
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.upsert #=> Boolean
|
|
7472
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_action #=> String
|
|
7473
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_when_matched #=> String
|
|
7474
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_when_not_matched #=> String
|
|
7475
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_clause #=> String
|
|
7476
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.crawler_connection #=> String
|
|
7477
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema #=> Array
|
|
7478
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].value #=> String
|
|
7479
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].label #=> String
|
|
7480
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].description #=> String
|
|
7481
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.staging_table #=> String
|
|
7482
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns #=> Array
|
|
7483
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].value #=> String
|
|
7484
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].label #=> String
|
|
7485
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].description #=> String
|
|
7486
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.inputs #=> Array
|
|
7487
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].amazon_redshift_target.inputs[0] #=> String
|
|
8247
7488
|
# resp.job.execution_class #=> String, one of "FLEX", "STANDARD"
|
|
8248
7489
|
# resp.job.source_control_details.provider #=> String, one of "GITHUB", "AWS_CODE_COMMIT"
|
|
8249
7490
|
# resp.job.source_control_details.repository #=> String
|
|
@@ -8359,7 +7600,7 @@ module Aws::Glue
|
|
|
8359
7600
|
# resp.job_run.execution_time #=> Integer
|
|
8360
7601
|
# resp.job_run.timeout #=> Integer
|
|
8361
7602
|
# resp.job_run.max_capacity #=> Float
|
|
8362
|
-
# resp.job_run.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
7603
|
+
# resp.job_run.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
8363
7604
|
# resp.job_run.number_of_workers #=> Integer
|
|
8364
7605
|
# resp.job_run.security_configuration #=> String
|
|
8365
7606
|
# resp.job_run.log_group_name #=> String
|
|
@@ -8425,7 +7666,7 @@ module Aws::Glue
|
|
|
8425
7666
|
# resp.job_runs[0].execution_time #=> Integer
|
|
8426
7667
|
# resp.job_runs[0].timeout #=> Integer
|
|
8427
7668
|
# resp.job_runs[0].max_capacity #=> Float
|
|
8428
|
-
# resp.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
7669
|
+
# resp.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
8429
7670
|
# resp.job_runs[0].number_of_workers #=> Integer
|
|
8430
7671
|
# resp.job_runs[0].security_configuration #=> String
|
|
8431
7672
|
# resp.job_runs[0].log_group_name #=> String
|
|
@@ -8489,7 +7730,7 @@ module Aws::Glue
|
|
|
8489
7730
|
# resp.jobs[0].allocated_capacity #=> Integer
|
|
8490
7731
|
# resp.jobs[0].timeout #=> Integer
|
|
8491
7732
|
# resp.jobs[0].max_capacity #=> Float
|
|
8492
|
-
# resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
7733
|
+
# resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
8493
7734
|
# resp.jobs[0].number_of_workers #=> Integer
|
|
8494
7735
|
# resp.jobs[0].security_configuration #=> String
|
|
8495
7736
|
# resp.jobs[0].notification_property.notify_delay_after #=> Integer
|
|
@@ -9000,6 +8241,10 @@ module Aws::Glue
|
|
|
9000
8241
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.function_name #=> String
|
|
9001
8242
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.path #=> String
|
|
9002
8243
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.version #=> String
|
|
8244
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas #=> Array
|
|
8245
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns #=> Array
|
|
8246
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns[0].name #=> String
|
|
8247
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].dynamic_transform.output_schemas[0].columns[0].type #=> String
|
|
9003
8248
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.name #=> String
|
|
9004
8249
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.inputs #=> Array
|
|
9005
8250
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.inputs[0] #=> String
|
|
@@ -9132,6 +8377,102 @@ module Aws::Glue
|
|
|
9132
8377
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
9133
8378
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.table #=> String
|
|
9134
8379
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_delta_direct_target.schema_change_policy.database #=> String
|
|
8380
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.name #=> String
|
|
8381
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.access_type #=> String
|
|
8382
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.source_type #=> String
|
|
8383
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.value #=> String
|
|
8384
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.label #=> String
|
|
8385
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.connection.description #=> String
|
|
8386
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.value #=> String
|
|
8387
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.label #=> String
|
|
8388
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.schema.description #=> String
|
|
8389
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.value #=> String
|
|
8390
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.label #=> String
|
|
8391
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table.description #=> String
|
|
8392
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.value #=> String
|
|
8393
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.label #=> String
|
|
8394
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_database.description #=> String
|
|
8395
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.value #=> String
|
|
8396
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.label #=> String
|
|
8397
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_table.description #=> String
|
|
8398
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_redshift_schema #=> String
|
|
8399
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.catalog_redshift_table #=> String
|
|
8400
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.temp_dir #=> String
|
|
8401
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.value #=> String
|
|
8402
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.label #=> String
|
|
8403
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.iam_role.description #=> String
|
|
8404
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options #=> Array
|
|
8405
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options[0].key #=> String
|
|
8406
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.advanced_options[0].value #=> String
|
|
8407
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.sample_query #=> String
|
|
8408
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.pre_action #=> String
|
|
8409
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.post_action #=> String
|
|
8410
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.action #=> String
|
|
8411
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_prefix #=> String
|
|
8412
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.upsert #=> Boolean
|
|
8413
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_action #=> String
|
|
8414
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_when_matched #=> String
|
|
8415
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_when_not_matched #=> String
|
|
8416
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.merge_clause #=> String
|
|
8417
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.crawler_connection #=> String
|
|
8418
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema #=> Array
|
|
8419
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].value #=> String
|
|
8420
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].label #=> String
|
|
8421
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.table_schema[0].description #=> String
|
|
8422
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.staging_table #=> String
|
|
8423
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns #=> Array
|
|
8424
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].value #=> String
|
|
8425
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].label #=> String
|
|
8426
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_source.data.selected_columns[0].description #=> String
|
|
8427
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.name #=> String
|
|
8428
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.access_type #=> String
|
|
8429
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.source_type #=> String
|
|
8430
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.value #=> String
|
|
8431
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.label #=> String
|
|
8432
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.connection.description #=> String
|
|
8433
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.value #=> String
|
|
8434
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.label #=> String
|
|
8435
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.schema.description #=> String
|
|
8436
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.value #=> String
|
|
8437
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.label #=> String
|
|
8438
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table.description #=> String
|
|
8439
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.value #=> String
|
|
8440
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.label #=> String
|
|
8441
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_database.description #=> String
|
|
8442
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.value #=> String
|
|
8443
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.label #=> String
|
|
8444
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_table.description #=> String
|
|
8445
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_redshift_schema #=> String
|
|
8446
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.catalog_redshift_table #=> String
|
|
8447
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.temp_dir #=> String
|
|
8448
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.value #=> String
|
|
8449
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.label #=> String
|
|
8450
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.iam_role.description #=> String
|
|
8451
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options #=> Array
|
|
8452
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options[0].key #=> String
|
|
8453
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.advanced_options[0].value #=> String
|
|
8454
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.sample_query #=> String
|
|
8455
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.pre_action #=> String
|
|
8456
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.post_action #=> String
|
|
8457
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.action #=> String
|
|
8458
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_prefix #=> String
|
|
8459
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.upsert #=> Boolean
|
|
8460
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_action #=> String
|
|
8461
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_when_matched #=> String
|
|
8462
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_when_not_matched #=> String
|
|
8463
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.merge_clause #=> String
|
|
8464
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.crawler_connection #=> String
|
|
8465
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema #=> Array
|
|
8466
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].value #=> String
|
|
8467
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].label #=> String
|
|
8468
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.table_schema[0].description #=> String
|
|
8469
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.staging_table #=> String
|
|
8470
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns #=> Array
|
|
8471
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].value #=> String
|
|
8472
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].label #=> String
|
|
8473
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.data.selected_columns[0].description #=> String
|
|
8474
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.inputs #=> Array
|
|
8475
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].amazon_redshift_target.inputs[0] #=> String
|
|
9135
8476
|
# resp.jobs[0].execution_class #=> String, one of "FLEX", "STANDARD"
|
|
9136
8477
|
# resp.jobs[0].source_control_details.provider #=> String, one of "GITHUB", "AWS_CODE_COMMIT"
|
|
9137
8478
|
# resp.jobs[0].source_control_details.repository #=> String
|
|
@@ -9373,7 +8714,7 @@ module Aws::Glue
|
|
|
9373
8714
|
# resp.role #=> String
|
|
9374
8715
|
# resp.glue_version #=> String
|
|
9375
8716
|
# resp.max_capacity #=> Float
|
|
9376
|
-
# resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
8717
|
+
# resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
9377
8718
|
# resp.number_of_workers #=> Integer
|
|
9378
8719
|
# resp.timeout #=> Integer
|
|
9379
8720
|
# resp.max_retries #=> Integer
|
|
@@ -9483,7 +8824,7 @@ module Aws::Glue
|
|
|
9483
8824
|
# resp.transforms[0].role #=> String
|
|
9484
8825
|
# resp.transforms[0].glue_version #=> String
|
|
9485
8826
|
# resp.transforms[0].max_capacity #=> Float
|
|
9486
|
-
# resp.transforms[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
8827
|
+
# resp.transforms[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
9487
8828
|
# resp.transforms[0].number_of_workers #=> Integer
|
|
9488
8829
|
# resp.transforms[0].timeout #=> Integer
|
|
9489
8830
|
# resp.transforms[0].max_retries #=> Integer
|
|
@@ -11849,7 +11190,7 @@ module Aws::Glue
|
|
|
11849
11190
|
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
|
|
11850
11191
|
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
|
|
11851
11192
|
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
|
|
11852
|
-
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
11193
|
+
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
11853
11194
|
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
|
|
11854
11195
|
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
|
|
11855
11196
|
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
|
|
@@ -11917,7 +11258,7 @@ module Aws::Glue
|
|
|
11917
11258
|
# resp.workflow.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
|
|
11918
11259
|
# resp.workflow.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
|
|
11919
11260
|
# resp.workflow.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
|
|
11920
|
-
# resp.workflow.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
11261
|
+
# resp.workflow.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
11921
11262
|
# resp.workflow.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
|
|
11922
11263
|
# resp.workflow.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
|
|
11923
11264
|
# resp.workflow.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
|
|
@@ -12038,7 +11379,7 @@ module Aws::Glue
|
|
|
12038
11379
|
# resp.run.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
|
|
12039
11380
|
# resp.run.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
|
|
12040
11381
|
# resp.run.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
|
|
12041
|
-
# resp.run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
11382
|
+
# resp.run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
12042
11383
|
# resp.run.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
|
|
12043
11384
|
# resp.run.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
|
|
12044
11385
|
# resp.run.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
|
|
@@ -12199,7 +11540,7 @@ module Aws::Glue
|
|
|
12199
11540
|
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
|
|
12200
11541
|
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
|
|
12201
11542
|
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
|
|
12202
|
-
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X"
|
|
11543
|
+
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
|
|
12203
11544
|
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
|
|
12204
11545
|
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
|
|
12205
11546
|
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
|
|
@@ -14444,7 +13785,7 @@ module Aws::Glue
|
|
|
14444
13785
|
# notification_property: {
|
|
14445
13786
|
# notify_delay_after: 1,
|
|
14446
13787
|
# },
|
|
14447
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
|
13788
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
|
|
14448
13789
|
# number_of_workers: 1,
|
|
14449
13790
|
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
|
14450
13791
|
# })
|
|
@@ -15590,964 +14931,6 @@ module Aws::Glue
|
|
|
15590
14931
|
#
|
|
15591
14932
|
# * {Types::UpdateJobResponse#job_name #job_name} => String
|
|
15592
14933
|
#
|
|
15593
|
-
# @example Request syntax with placeholder values
|
|
15594
|
-
#
|
|
15595
|
-
# resp = client.update_job({
|
|
15596
|
-
# job_name: "NameString", # required
|
|
15597
|
-
# job_update: { # required
|
|
15598
|
-
# description: "DescriptionString",
|
|
15599
|
-
# log_uri: "UriString",
|
|
15600
|
-
# role: "RoleString",
|
|
15601
|
-
# execution_property: {
|
|
15602
|
-
# max_concurrent_runs: 1,
|
|
15603
|
-
# },
|
|
15604
|
-
# command: {
|
|
15605
|
-
# name: "GenericString",
|
|
15606
|
-
# script_location: "ScriptLocationString",
|
|
15607
|
-
# python_version: "PythonVersionString",
|
|
15608
|
-
# },
|
|
15609
|
-
# default_arguments: {
|
|
15610
|
-
# "GenericString" => "GenericString",
|
|
15611
|
-
# },
|
|
15612
|
-
# non_overridable_arguments: {
|
|
15613
|
-
# "GenericString" => "GenericString",
|
|
15614
|
-
# },
|
|
15615
|
-
# connections: {
|
|
15616
|
-
# connections: ["GenericString"],
|
|
15617
|
-
# },
|
|
15618
|
-
# max_retries: 1,
|
|
15619
|
-
# allocated_capacity: 1,
|
|
15620
|
-
# timeout: 1,
|
|
15621
|
-
# max_capacity: 1.0,
|
|
15622
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
|
15623
|
-
# number_of_workers: 1,
|
|
15624
|
-
# security_configuration: "NameString",
|
|
15625
|
-
# notification_property: {
|
|
15626
|
-
# notify_delay_after: 1,
|
|
15627
|
-
# },
|
|
15628
|
-
# glue_version: "GlueVersionString",
|
|
15629
|
-
# code_gen_configuration_nodes: {
|
|
15630
|
-
# "NodeId" => {
|
|
15631
|
-
# athena_connector_source: {
|
|
15632
|
-
# name: "NodeName", # required
|
|
15633
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
15634
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
15635
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
15636
|
-
# connection_table: "EnclosedInStringPropertyWithQuote",
|
|
15637
|
-
# schema_name: "EnclosedInStringProperty", # required
|
|
15638
|
-
# output_schemas: [
|
|
15639
|
-
# {
|
|
15640
|
-
# columns: [
|
|
15641
|
-
# {
|
|
15642
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15643
|
-
# type: "ColumnTypeString",
|
|
15644
|
-
# },
|
|
15645
|
-
# ],
|
|
15646
|
-
# },
|
|
15647
|
-
# ],
|
|
15648
|
-
# },
|
|
15649
|
-
# jdbc_connector_source: {
|
|
15650
|
-
# name: "NodeName", # required
|
|
15651
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
15652
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
15653
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
15654
|
-
# additional_options: {
|
|
15655
|
-
# filter_predicate: "EnclosedInStringProperty",
|
|
15656
|
-
# partition_column: "EnclosedInStringProperty",
|
|
15657
|
-
# lower_bound: 1,
|
|
15658
|
-
# upper_bound: 1,
|
|
15659
|
-
# num_partitions: 1,
|
|
15660
|
-
# job_bookmark_keys: ["EnclosedInStringProperty"],
|
|
15661
|
-
# job_bookmark_keys_sort_order: "EnclosedInStringProperty",
|
|
15662
|
-
# data_type_mapping: {
|
|
15663
|
-
# "ARRAY" => "DATE", # accepts DATE, STRING, TIMESTAMP, INT, FLOAT, LONG, BIGDECIMAL, BYTE, SHORT, DOUBLE
|
|
15664
|
-
# },
|
|
15665
|
-
# },
|
|
15666
|
-
# connection_table: "EnclosedInStringPropertyWithQuote",
|
|
15667
|
-
# query: "SqlQuery",
|
|
15668
|
-
# output_schemas: [
|
|
15669
|
-
# {
|
|
15670
|
-
# columns: [
|
|
15671
|
-
# {
|
|
15672
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15673
|
-
# type: "ColumnTypeString",
|
|
15674
|
-
# },
|
|
15675
|
-
# ],
|
|
15676
|
-
# },
|
|
15677
|
-
# ],
|
|
15678
|
-
# },
|
|
15679
|
-
# spark_connector_source: {
|
|
15680
|
-
# name: "NodeName", # required
|
|
15681
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
15682
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
15683
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
15684
|
-
# additional_options: {
|
|
15685
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15686
|
-
# },
|
|
15687
|
-
# output_schemas: [
|
|
15688
|
-
# {
|
|
15689
|
-
# columns: [
|
|
15690
|
-
# {
|
|
15691
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15692
|
-
# type: "ColumnTypeString",
|
|
15693
|
-
# },
|
|
15694
|
-
# ],
|
|
15695
|
-
# },
|
|
15696
|
-
# ],
|
|
15697
|
-
# },
|
|
15698
|
-
# catalog_source: {
|
|
15699
|
-
# name: "NodeName", # required
|
|
15700
|
-
# database: "EnclosedInStringProperty", # required
|
|
15701
|
-
# table: "EnclosedInStringProperty", # required
|
|
15702
|
-
# },
|
|
15703
|
-
# redshift_source: {
|
|
15704
|
-
# name: "NodeName", # required
|
|
15705
|
-
# database: "EnclosedInStringProperty", # required
|
|
15706
|
-
# table: "EnclosedInStringProperty", # required
|
|
15707
|
-
# redshift_tmp_dir: "EnclosedInStringProperty",
|
|
15708
|
-
# tmp_dir_iam_role: "EnclosedInStringProperty",
|
|
15709
|
-
# },
|
|
15710
|
-
# s3_catalog_source: {
|
|
15711
|
-
# name: "NodeName", # required
|
|
15712
|
-
# database: "EnclosedInStringProperty", # required
|
|
15713
|
-
# table: "EnclosedInStringProperty", # required
|
|
15714
|
-
# partition_predicate: "EnclosedInStringProperty",
|
|
15715
|
-
# additional_options: {
|
|
15716
|
-
# bounded_size: 1,
|
|
15717
|
-
# bounded_files: 1,
|
|
15718
|
-
# },
|
|
15719
|
-
# },
|
|
15720
|
-
# s3_csv_source: {
|
|
15721
|
-
# name: "NodeName", # required
|
|
15722
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
15723
|
-
# compression_type: "gzip", # accepts gzip, bzip2
|
|
15724
|
-
# exclusions: ["EnclosedInStringProperty"],
|
|
15725
|
-
# group_size: "EnclosedInStringProperty",
|
|
15726
|
-
# group_files: "EnclosedInStringProperty",
|
|
15727
|
-
# recurse: false,
|
|
15728
|
-
# max_band: 1,
|
|
15729
|
-
# max_files_in_band: 1,
|
|
15730
|
-
# additional_options: {
|
|
15731
|
-
# bounded_size: 1,
|
|
15732
|
-
# bounded_files: 1,
|
|
15733
|
-
# enable_sample_path: false,
|
|
15734
|
-
# sample_path: "EnclosedInStringProperty",
|
|
15735
|
-
# },
|
|
15736
|
-
# separator: "comma", # required, accepts comma, ctrla, pipe, semicolon, tab
|
|
15737
|
-
# escaper: "EnclosedInStringPropertyWithQuote",
|
|
15738
|
-
# quote_char: "quote", # required, accepts quote, quillemet, single_quote, disabled
|
|
15739
|
-
# multiline: false,
|
|
15740
|
-
# with_header: false,
|
|
15741
|
-
# write_header: false,
|
|
15742
|
-
# skip_first: false,
|
|
15743
|
-
# optimize_performance: false,
|
|
15744
|
-
# output_schemas: [
|
|
15745
|
-
# {
|
|
15746
|
-
# columns: [
|
|
15747
|
-
# {
|
|
15748
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15749
|
-
# type: "ColumnTypeString",
|
|
15750
|
-
# },
|
|
15751
|
-
# ],
|
|
15752
|
-
# },
|
|
15753
|
-
# ],
|
|
15754
|
-
# },
|
|
15755
|
-
# s3_json_source: {
|
|
15756
|
-
# name: "NodeName", # required
|
|
15757
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
15758
|
-
# compression_type: "gzip", # accepts gzip, bzip2
|
|
15759
|
-
# exclusions: ["EnclosedInStringProperty"],
|
|
15760
|
-
# group_size: "EnclosedInStringProperty",
|
|
15761
|
-
# group_files: "EnclosedInStringProperty",
|
|
15762
|
-
# recurse: false,
|
|
15763
|
-
# max_band: 1,
|
|
15764
|
-
# max_files_in_band: 1,
|
|
15765
|
-
# additional_options: {
|
|
15766
|
-
# bounded_size: 1,
|
|
15767
|
-
# bounded_files: 1,
|
|
15768
|
-
# enable_sample_path: false,
|
|
15769
|
-
# sample_path: "EnclosedInStringProperty",
|
|
15770
|
-
# },
|
|
15771
|
-
# json_path: "EnclosedInStringProperty",
|
|
15772
|
-
# multiline: false,
|
|
15773
|
-
# output_schemas: [
|
|
15774
|
-
# {
|
|
15775
|
-
# columns: [
|
|
15776
|
-
# {
|
|
15777
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15778
|
-
# type: "ColumnTypeString",
|
|
15779
|
-
# },
|
|
15780
|
-
# ],
|
|
15781
|
-
# },
|
|
15782
|
-
# ],
|
|
15783
|
-
# },
|
|
15784
|
-
# s3_parquet_source: {
|
|
15785
|
-
# name: "NodeName", # required
|
|
15786
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
15787
|
-
# compression_type: "snappy", # accepts snappy, lzo, gzip, uncompressed, none
|
|
15788
|
-
# exclusions: ["EnclosedInStringProperty"],
|
|
15789
|
-
# group_size: "EnclosedInStringProperty",
|
|
15790
|
-
# group_files: "EnclosedInStringProperty",
|
|
15791
|
-
# recurse: false,
|
|
15792
|
-
# max_band: 1,
|
|
15793
|
-
# max_files_in_band: 1,
|
|
15794
|
-
# additional_options: {
|
|
15795
|
-
# bounded_size: 1,
|
|
15796
|
-
# bounded_files: 1,
|
|
15797
|
-
# enable_sample_path: false,
|
|
15798
|
-
# sample_path: "EnclosedInStringProperty",
|
|
15799
|
-
# },
|
|
15800
|
-
# output_schemas: [
|
|
15801
|
-
# {
|
|
15802
|
-
# columns: [
|
|
15803
|
-
# {
|
|
15804
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15805
|
-
# type: "ColumnTypeString",
|
|
15806
|
-
# },
|
|
15807
|
-
# ],
|
|
15808
|
-
# },
|
|
15809
|
-
# ],
|
|
15810
|
-
# },
|
|
15811
|
-
# relational_catalog_source: {
|
|
15812
|
-
# name: "NodeName", # required
|
|
15813
|
-
# database: "EnclosedInStringProperty", # required
|
|
15814
|
-
# table: "EnclosedInStringProperty", # required
|
|
15815
|
-
# },
|
|
15816
|
-
# dynamo_db_catalog_source: {
|
|
15817
|
-
# name: "NodeName", # required
|
|
15818
|
-
# database: "EnclosedInStringProperty", # required
|
|
15819
|
-
# table: "EnclosedInStringProperty", # required
|
|
15820
|
-
# },
|
|
15821
|
-
# jdbc_connector_target: {
|
|
15822
|
-
# name: "NodeName", # required
|
|
15823
|
-
# inputs: ["NodeId"], # required
|
|
15824
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
15825
|
-
# connection_table: "EnclosedInStringPropertyWithQuote", # required
|
|
15826
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
15827
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
15828
|
-
# additional_options: {
|
|
15829
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15830
|
-
# },
|
|
15831
|
-
# output_schemas: [
|
|
15832
|
-
# {
|
|
15833
|
-
# columns: [
|
|
15834
|
-
# {
|
|
15835
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15836
|
-
# type: "ColumnTypeString",
|
|
15837
|
-
# },
|
|
15838
|
-
# ],
|
|
15839
|
-
# },
|
|
15840
|
-
# ],
|
|
15841
|
-
# },
|
|
15842
|
-
# spark_connector_target: {
|
|
15843
|
-
# name: "NodeName", # required
|
|
15844
|
-
# inputs: ["NodeId"], # required
|
|
15845
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
15846
|
-
# connector_name: "EnclosedInStringProperty", # required
|
|
15847
|
-
# connection_type: "EnclosedInStringProperty", # required
|
|
15848
|
-
# additional_options: {
|
|
15849
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15850
|
-
# },
|
|
15851
|
-
# output_schemas: [
|
|
15852
|
-
# {
|
|
15853
|
-
# columns: [
|
|
15854
|
-
# {
|
|
15855
|
-
# name: "GlueStudioColumnNameString", # required
|
|
15856
|
-
# type: "ColumnTypeString",
|
|
15857
|
-
# },
|
|
15858
|
-
# ],
|
|
15859
|
-
# },
|
|
15860
|
-
# ],
|
|
15861
|
-
# },
|
|
15862
|
-
# catalog_target: {
|
|
15863
|
-
# name: "NodeName", # required
|
|
15864
|
-
# inputs: ["NodeId"], # required
|
|
15865
|
-
# database: "EnclosedInStringProperty", # required
|
|
15866
|
-
# table: "EnclosedInStringProperty", # required
|
|
15867
|
-
# },
|
|
15868
|
-
# redshift_target: {
|
|
15869
|
-
# name: "NodeName", # required
|
|
15870
|
-
# inputs: ["NodeId"], # required
|
|
15871
|
-
# database: "EnclosedInStringProperty", # required
|
|
15872
|
-
# table: "EnclosedInStringProperty", # required
|
|
15873
|
-
# redshift_tmp_dir: "EnclosedInStringProperty",
|
|
15874
|
-
# tmp_dir_iam_role: "EnclosedInStringProperty",
|
|
15875
|
-
# upsert_redshift_options: {
|
|
15876
|
-
# table_location: "EnclosedInStringProperty",
|
|
15877
|
-
# connection_name: "EnclosedInStringProperty",
|
|
15878
|
-
# upsert_keys: ["EnclosedInStringProperty"],
|
|
15879
|
-
# },
|
|
15880
|
-
# },
|
|
15881
|
-
# s3_catalog_target: {
|
|
15882
|
-
# name: "NodeName", # required
|
|
15883
|
-
# inputs: ["NodeId"], # required
|
|
15884
|
-
# partition_keys: [
|
|
15885
|
-
# ["EnclosedInStringProperty"],
|
|
15886
|
-
# ],
|
|
15887
|
-
# table: "EnclosedInStringProperty", # required
|
|
15888
|
-
# database: "EnclosedInStringProperty", # required
|
|
15889
|
-
# schema_change_policy: {
|
|
15890
|
-
# enable_update_catalog: false,
|
|
15891
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
15892
|
-
# },
|
|
15893
|
-
# },
|
|
15894
|
-
# s3_glue_parquet_target: {
|
|
15895
|
-
# name: "NodeName", # required
|
|
15896
|
-
# inputs: ["NodeId"], # required
|
|
15897
|
-
# partition_keys: [
|
|
15898
|
-
# ["EnclosedInStringProperty"],
|
|
15899
|
-
# ],
|
|
15900
|
-
# path: "EnclosedInStringProperty", # required
|
|
15901
|
-
# compression: "snappy", # accepts snappy, lzo, gzip, uncompressed, none
|
|
15902
|
-
# schema_change_policy: {
|
|
15903
|
-
# enable_update_catalog: false,
|
|
15904
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
15905
|
-
# table: "EnclosedInStringProperty",
|
|
15906
|
-
# database: "EnclosedInStringProperty",
|
|
15907
|
-
# },
|
|
15908
|
-
# },
|
|
15909
|
-
# s3_direct_target: {
|
|
15910
|
-
# name: "NodeName", # required
|
|
15911
|
-
# inputs: ["NodeId"], # required
|
|
15912
|
-
# partition_keys: [
|
|
15913
|
-
# ["EnclosedInStringProperty"],
|
|
15914
|
-
# ],
|
|
15915
|
-
# path: "EnclosedInStringProperty", # required
|
|
15916
|
-
# compression: "EnclosedInStringProperty",
|
|
15917
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi, delta
|
|
15918
|
-
# schema_change_policy: {
|
|
15919
|
-
# enable_update_catalog: false,
|
|
15920
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
15921
|
-
# table: "EnclosedInStringProperty",
|
|
15922
|
-
# database: "EnclosedInStringProperty",
|
|
15923
|
-
# },
|
|
15924
|
-
# },
|
|
15925
|
-
# apply_mapping: {
|
|
15926
|
-
# name: "NodeName", # required
|
|
15927
|
-
# inputs: ["NodeId"], # required
|
|
15928
|
-
# mapping: [ # required
|
|
15929
|
-
# {
|
|
15930
|
-
# to_key: "EnclosedInStringProperty",
|
|
15931
|
-
# from_path: ["EnclosedInStringProperty"],
|
|
15932
|
-
# from_type: "EnclosedInStringProperty",
|
|
15933
|
-
# to_type: "EnclosedInStringProperty",
|
|
15934
|
-
# dropped: false,
|
|
15935
|
-
# children: {
|
|
15936
|
-
# # recursive Mappings
|
|
15937
|
-
# },
|
|
15938
|
-
# },
|
|
15939
|
-
# ],
|
|
15940
|
-
# },
|
|
15941
|
-
# select_fields: {
|
|
15942
|
-
# name: "NodeName", # required
|
|
15943
|
-
# inputs: ["NodeId"], # required
|
|
15944
|
-
# paths: [ # required
|
|
15945
|
-
# ["EnclosedInStringProperty"],
|
|
15946
|
-
# ],
|
|
15947
|
-
# },
|
|
15948
|
-
# drop_fields: {
|
|
15949
|
-
# name: "NodeName", # required
|
|
15950
|
-
# inputs: ["NodeId"], # required
|
|
15951
|
-
# paths: [ # required
|
|
15952
|
-
# ["EnclosedInStringProperty"],
|
|
15953
|
-
# ],
|
|
15954
|
-
# },
|
|
15955
|
-
# rename_field: {
|
|
15956
|
-
# name: "NodeName", # required
|
|
15957
|
-
# inputs: ["NodeId"], # required
|
|
15958
|
-
# source_path: ["EnclosedInStringProperty"], # required
|
|
15959
|
-
# target_path: ["EnclosedInStringProperty"], # required
|
|
15960
|
-
# },
|
|
15961
|
-
# spigot: {
|
|
15962
|
-
# name: "NodeName", # required
|
|
15963
|
-
# inputs: ["NodeId"], # required
|
|
15964
|
-
# path: "EnclosedInStringProperty", # required
|
|
15965
|
-
# topk: 1,
|
|
15966
|
-
# prob: 1.0,
|
|
15967
|
-
# },
|
|
15968
|
-
# join: {
|
|
15969
|
-
# name: "NodeName", # required
|
|
15970
|
-
# inputs: ["NodeId"], # required
|
|
15971
|
-
# join_type: "equijoin", # required, accepts equijoin, left, right, outer, leftsemi, leftanti
|
|
15972
|
-
# columns: [ # required
|
|
15973
|
-
# {
|
|
15974
|
-
# from: "EnclosedInStringProperty", # required
|
|
15975
|
-
# keys: [ # required
|
|
15976
|
-
# ["EnclosedInStringProperty"],
|
|
15977
|
-
# ],
|
|
15978
|
-
# },
|
|
15979
|
-
# ],
|
|
15980
|
-
# },
|
|
15981
|
-
# split_fields: {
|
|
15982
|
-
# name: "NodeName", # required
|
|
15983
|
-
# inputs: ["NodeId"], # required
|
|
15984
|
-
# paths: [ # required
|
|
15985
|
-
# ["EnclosedInStringProperty"],
|
|
15986
|
-
# ],
|
|
15987
|
-
# },
|
|
15988
|
-
# select_from_collection: {
|
|
15989
|
-
# name: "NodeName", # required
|
|
15990
|
-
# inputs: ["NodeId"], # required
|
|
15991
|
-
# index: 1, # required
|
|
15992
|
-
# },
|
|
15993
|
-
# fill_missing_values: {
|
|
15994
|
-
# name: "NodeName", # required
|
|
15995
|
-
# inputs: ["NodeId"], # required
|
|
15996
|
-
# imputed_path: "EnclosedInStringProperty", # required
|
|
15997
|
-
# filled_path: "EnclosedInStringProperty",
|
|
15998
|
-
# },
|
|
15999
|
-
# filter: {
|
|
16000
|
-
# name: "NodeName", # required
|
|
16001
|
-
# inputs: ["NodeId"], # required
|
|
16002
|
-
# logical_operator: "AND", # required, accepts AND, OR
|
|
16003
|
-
# filters: [ # required
|
|
16004
|
-
# {
|
|
16005
|
-
# operation: "EQ", # required, accepts EQ, LT, GT, LTE, GTE, REGEX, ISNULL
|
|
16006
|
-
# negated: false,
|
|
16007
|
-
# values: [ # required
|
|
16008
|
-
# {
|
|
16009
|
-
# type: "COLUMNEXTRACTED", # required, accepts COLUMNEXTRACTED, CONSTANT
|
|
16010
|
-
# value: ["EnclosedInStringProperty"], # required
|
|
16011
|
-
# },
|
|
16012
|
-
# ],
|
|
16013
|
-
# },
|
|
16014
|
-
# ],
|
|
16015
|
-
# },
|
|
16016
|
-
# custom_code: {
|
|
16017
|
-
# name: "NodeName", # required
|
|
16018
|
-
# inputs: ["NodeId"], # required
|
|
16019
|
-
# code: "ExtendedString", # required
|
|
16020
|
-
# class_name: "EnclosedInStringProperty", # required
|
|
16021
|
-
# output_schemas: [
|
|
16022
|
-
# {
|
|
16023
|
-
# columns: [
|
|
16024
|
-
# {
|
|
16025
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16026
|
-
# type: "ColumnTypeString",
|
|
16027
|
-
# },
|
|
16028
|
-
# ],
|
|
16029
|
-
# },
|
|
16030
|
-
# ],
|
|
16031
|
-
# },
|
|
16032
|
-
# spark_sql: {
|
|
16033
|
-
# name: "NodeName", # required
|
|
16034
|
-
# inputs: ["NodeId"], # required
|
|
16035
|
-
# sql_query: "SqlQuery", # required
|
|
16036
|
-
# sql_aliases: [ # required
|
|
16037
|
-
# {
|
|
16038
|
-
# from: "NodeId", # required
|
|
16039
|
-
# alias: "EnclosedInStringPropertyWithQuote", # required
|
|
16040
|
-
# },
|
|
16041
|
-
# ],
|
|
16042
|
-
# output_schemas: [
|
|
16043
|
-
# {
|
|
16044
|
-
# columns: [
|
|
16045
|
-
# {
|
|
16046
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16047
|
-
# type: "ColumnTypeString",
|
|
16048
|
-
# },
|
|
16049
|
-
# ],
|
|
16050
|
-
# },
|
|
16051
|
-
# ],
|
|
16052
|
-
# },
|
|
16053
|
-
# direct_kinesis_source: {
|
|
16054
|
-
# name: "NodeName", # required
|
|
16055
|
-
# window_size: 1,
|
|
16056
|
-
# detect_schema: false,
|
|
16057
|
-
# streaming_options: {
|
|
16058
|
-
# endpoint_url: "EnclosedInStringProperty",
|
|
16059
|
-
# stream_name: "EnclosedInStringProperty",
|
|
16060
|
-
# classification: "EnclosedInStringProperty",
|
|
16061
|
-
# delimiter: "EnclosedInStringProperty",
|
|
16062
|
-
# starting_position: "latest", # accepts latest, trim_horizon, earliest
|
|
16063
|
-
# max_fetch_time_in_ms: 1,
|
|
16064
|
-
# max_fetch_records_per_shard: 1,
|
|
16065
|
-
# max_record_per_read: 1,
|
|
16066
|
-
# add_idle_time_between_reads: false,
|
|
16067
|
-
# idle_time_between_reads_in_ms: 1,
|
|
16068
|
-
# describe_shard_interval: 1,
|
|
16069
|
-
# num_retries: 1,
|
|
16070
|
-
# retry_interval_ms: 1,
|
|
16071
|
-
# max_retry_interval_ms: 1,
|
|
16072
|
-
# avoid_empty_batches: false,
|
|
16073
|
-
# stream_arn: "EnclosedInStringProperty",
|
|
16074
|
-
# role_arn: "EnclosedInStringProperty",
|
|
16075
|
-
# role_session_name: "EnclosedInStringProperty",
|
|
16076
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
16077
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
16078
|
-
# },
|
|
16079
|
-
# data_preview_options: {
|
|
16080
|
-
# polling_time: 1,
|
|
16081
|
-
# record_polling_limit: 1,
|
|
16082
|
-
# },
|
|
16083
|
-
# },
|
|
16084
|
-
# direct_kafka_source: {
|
|
16085
|
-
# name: "NodeName", # required
|
|
16086
|
-
# streaming_options: {
|
|
16087
|
-
# bootstrap_servers: "EnclosedInStringProperty",
|
|
16088
|
-
# security_protocol: "EnclosedInStringProperty",
|
|
16089
|
-
# connection_name: "EnclosedInStringProperty",
|
|
16090
|
-
# topic_name: "EnclosedInStringProperty",
|
|
16091
|
-
# assign: "EnclosedInStringProperty",
|
|
16092
|
-
# subscribe_pattern: "EnclosedInStringProperty",
|
|
16093
|
-
# classification: "EnclosedInStringProperty",
|
|
16094
|
-
# delimiter: "EnclosedInStringProperty",
|
|
16095
|
-
# starting_offsets: "EnclosedInStringProperty",
|
|
16096
|
-
# ending_offsets: "EnclosedInStringProperty",
|
|
16097
|
-
# poll_timeout_ms: 1,
|
|
16098
|
-
# num_retries: 1,
|
|
16099
|
-
# retry_interval_ms: 1,
|
|
16100
|
-
# max_offsets_per_trigger: 1,
|
|
16101
|
-
# min_partitions: 1,
|
|
16102
|
-
# include_headers: false,
|
|
16103
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
16104
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
16105
|
-
# },
|
|
16106
|
-
# window_size: 1,
|
|
16107
|
-
# detect_schema: false,
|
|
16108
|
-
# data_preview_options: {
|
|
16109
|
-
# polling_time: 1,
|
|
16110
|
-
# record_polling_limit: 1,
|
|
16111
|
-
# },
|
|
16112
|
-
# },
|
|
16113
|
-
# catalog_kinesis_source: {
|
|
16114
|
-
# name: "NodeName", # required
|
|
16115
|
-
# window_size: 1,
|
|
16116
|
-
# detect_schema: false,
|
|
16117
|
-
# table: "EnclosedInStringProperty", # required
|
|
16118
|
-
# database: "EnclosedInStringProperty", # required
|
|
16119
|
-
# streaming_options: {
|
|
16120
|
-
# endpoint_url: "EnclosedInStringProperty",
|
|
16121
|
-
# stream_name: "EnclosedInStringProperty",
|
|
16122
|
-
# classification: "EnclosedInStringProperty",
|
|
16123
|
-
# delimiter: "EnclosedInStringProperty",
|
|
16124
|
-
# starting_position: "latest", # accepts latest, trim_horizon, earliest
|
|
16125
|
-
# max_fetch_time_in_ms: 1,
|
|
16126
|
-
# max_fetch_records_per_shard: 1,
|
|
16127
|
-
# max_record_per_read: 1,
|
|
16128
|
-
# add_idle_time_between_reads: false,
|
|
16129
|
-
# idle_time_between_reads_in_ms: 1,
|
|
16130
|
-
# describe_shard_interval: 1,
|
|
16131
|
-
# num_retries: 1,
|
|
16132
|
-
# retry_interval_ms: 1,
|
|
16133
|
-
# max_retry_interval_ms: 1,
|
|
16134
|
-
# avoid_empty_batches: false,
|
|
16135
|
-
# stream_arn: "EnclosedInStringProperty",
|
|
16136
|
-
# role_arn: "EnclosedInStringProperty",
|
|
16137
|
-
# role_session_name: "EnclosedInStringProperty",
|
|
16138
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
16139
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
16140
|
-
# },
|
|
16141
|
-
# data_preview_options: {
|
|
16142
|
-
# polling_time: 1,
|
|
16143
|
-
# record_polling_limit: 1,
|
|
16144
|
-
# },
|
|
16145
|
-
# },
|
|
16146
|
-
# catalog_kafka_source: {
|
|
16147
|
-
# name: "NodeName", # required
|
|
16148
|
-
# window_size: 1,
|
|
16149
|
-
# detect_schema: false,
|
|
16150
|
-
# table: "EnclosedInStringProperty", # required
|
|
16151
|
-
# database: "EnclosedInStringProperty", # required
|
|
16152
|
-
# streaming_options: {
|
|
16153
|
-
# bootstrap_servers: "EnclosedInStringProperty",
|
|
16154
|
-
# security_protocol: "EnclosedInStringProperty",
|
|
16155
|
-
# connection_name: "EnclosedInStringProperty",
|
|
16156
|
-
# topic_name: "EnclosedInStringProperty",
|
|
16157
|
-
# assign: "EnclosedInStringProperty",
|
|
16158
|
-
# subscribe_pattern: "EnclosedInStringProperty",
|
|
16159
|
-
# classification: "EnclosedInStringProperty",
|
|
16160
|
-
# delimiter: "EnclosedInStringProperty",
|
|
16161
|
-
# starting_offsets: "EnclosedInStringProperty",
|
|
16162
|
-
# ending_offsets: "EnclosedInStringProperty",
|
|
16163
|
-
# poll_timeout_ms: 1,
|
|
16164
|
-
# num_retries: 1,
|
|
16165
|
-
# retry_interval_ms: 1,
|
|
16166
|
-
# max_offsets_per_trigger: 1,
|
|
16167
|
-
# min_partitions: 1,
|
|
16168
|
-
# include_headers: false,
|
|
16169
|
-
# add_record_timestamp: "EnclosedInStringProperty",
|
|
16170
|
-
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
16171
|
-
# },
|
|
16172
|
-
# data_preview_options: {
|
|
16173
|
-
# polling_time: 1,
|
|
16174
|
-
# record_polling_limit: 1,
|
|
16175
|
-
# },
|
|
16176
|
-
# },
|
|
16177
|
-
# drop_null_fields: {
|
|
16178
|
-
# name: "NodeName", # required
|
|
16179
|
-
# inputs: ["NodeId"], # required
|
|
16180
|
-
# null_check_box_list: {
|
|
16181
|
-
# is_empty: false,
|
|
16182
|
-
# is_null_string: false,
|
|
16183
|
-
# is_neg_one: false,
|
|
16184
|
-
# },
|
|
16185
|
-
# null_text_list: [
|
|
16186
|
-
# {
|
|
16187
|
-
# value: "EnclosedInStringProperty", # required
|
|
16188
|
-
# datatype: { # required
|
|
16189
|
-
# id: "GenericLimitedString", # required
|
|
16190
|
-
# label: "GenericLimitedString", # required
|
|
16191
|
-
# },
|
|
16192
|
-
# },
|
|
16193
|
-
# ],
|
|
16194
|
-
# },
|
|
16195
|
-
# merge: {
|
|
16196
|
-
# name: "NodeName", # required
|
|
16197
|
-
# inputs: ["NodeId"], # required
|
|
16198
|
-
# source: "NodeId", # required
|
|
16199
|
-
# primary_keys: [ # required
|
|
16200
|
-
# ["EnclosedInStringProperty"],
|
|
16201
|
-
# ],
|
|
16202
|
-
# },
|
|
16203
|
-
# union: {
|
|
16204
|
-
# name: "NodeName", # required
|
|
16205
|
-
# inputs: ["NodeId"], # required
|
|
16206
|
-
# union_type: "ALL", # required, accepts ALL, DISTINCT
|
|
16207
|
-
# },
|
|
16208
|
-
# pii_detection: {
|
|
16209
|
-
# name: "NodeName", # required
|
|
16210
|
-
# inputs: ["NodeId"], # required
|
|
16211
|
-
# pii_type: "RowAudit", # required, accepts RowAudit, RowMasking, ColumnAudit, ColumnMasking
|
|
16212
|
-
# entity_types_to_detect: ["EnclosedInStringProperty"], # required
|
|
16213
|
-
# output_column_name: "EnclosedInStringProperty",
|
|
16214
|
-
# sample_fraction: 1.0,
|
|
16215
|
-
# threshold_fraction: 1.0,
|
|
16216
|
-
# mask_value: "MaskValue",
|
|
16217
|
-
# },
|
|
16218
|
-
# aggregate: {
|
|
16219
|
-
# name: "NodeName", # required
|
|
16220
|
-
# inputs: ["NodeId"], # required
|
|
16221
|
-
# groups: [ # required
|
|
16222
|
-
# ["EnclosedInStringProperty"],
|
|
16223
|
-
# ],
|
|
16224
|
-
# aggs: [ # required
|
|
16225
|
-
# {
|
|
16226
|
-
# column: ["EnclosedInStringProperty"], # required
|
|
16227
|
-
# agg_func: "avg", # required, accepts avg, countDistinct, count, first, last, kurtosis, max, min, skewness, stddev_samp, stddev_pop, sum, sumDistinct, var_samp, var_pop
|
|
16228
|
-
# },
|
|
16229
|
-
# ],
|
|
16230
|
-
# },
|
|
16231
|
-
# drop_duplicates: {
|
|
16232
|
-
# name: "NodeName", # required
|
|
16233
|
-
# inputs: ["NodeId"], # required
|
|
16234
|
-
# columns: [
|
|
16235
|
-
# ["GenericLimitedString"],
|
|
16236
|
-
# ],
|
|
16237
|
-
# },
|
|
16238
|
-
# governed_catalog_target: {
|
|
16239
|
-
# name: "NodeName", # required
|
|
16240
|
-
# inputs: ["NodeId"], # required
|
|
16241
|
-
# partition_keys: [
|
|
16242
|
-
# ["EnclosedInStringProperty"],
|
|
16243
|
-
# ],
|
|
16244
|
-
# table: "EnclosedInStringProperty", # required
|
|
16245
|
-
# database: "EnclosedInStringProperty", # required
|
|
16246
|
-
# schema_change_policy: {
|
|
16247
|
-
# enable_update_catalog: false,
|
|
16248
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
16249
|
-
# },
|
|
16250
|
-
# },
|
|
16251
|
-
# governed_catalog_source: {
|
|
16252
|
-
# name: "NodeName", # required
|
|
16253
|
-
# database: "EnclosedInStringProperty", # required
|
|
16254
|
-
# table: "EnclosedInStringProperty", # required
|
|
16255
|
-
# partition_predicate: "EnclosedInStringProperty",
|
|
16256
|
-
# additional_options: {
|
|
16257
|
-
# bounded_size: 1,
|
|
16258
|
-
# bounded_files: 1,
|
|
16259
|
-
# },
|
|
16260
|
-
# },
|
|
16261
|
-
# microsoft_sql_server_catalog_source: {
|
|
16262
|
-
# name: "NodeName", # required
|
|
16263
|
-
# database: "EnclosedInStringProperty", # required
|
|
16264
|
-
# table: "EnclosedInStringProperty", # required
|
|
16265
|
-
# },
|
|
16266
|
-
# my_sql_catalog_source: {
|
|
16267
|
-
# name: "NodeName", # required
|
|
16268
|
-
# database: "EnclosedInStringProperty", # required
|
|
16269
|
-
# table: "EnclosedInStringProperty", # required
|
|
16270
|
-
# },
|
|
16271
|
-
# oracle_sql_catalog_source: {
|
|
16272
|
-
# name: "NodeName", # required
|
|
16273
|
-
# database: "EnclosedInStringProperty", # required
|
|
16274
|
-
# table: "EnclosedInStringProperty", # required
|
|
16275
|
-
# },
|
|
16276
|
-
# postgre_sql_catalog_source: {
|
|
16277
|
-
# name: "NodeName", # required
|
|
16278
|
-
# database: "EnclosedInStringProperty", # required
|
|
16279
|
-
# table: "EnclosedInStringProperty", # required
|
|
16280
|
-
# },
|
|
16281
|
-
# microsoft_sql_server_catalog_target: {
|
|
16282
|
-
# name: "NodeName", # required
|
|
16283
|
-
# inputs: ["NodeId"], # required
|
|
16284
|
-
# database: "EnclosedInStringProperty", # required
|
|
16285
|
-
# table: "EnclosedInStringProperty", # required
|
|
16286
|
-
# },
|
|
16287
|
-
# my_sql_catalog_target: {
|
|
16288
|
-
# name: "NodeName", # required
|
|
16289
|
-
# inputs: ["NodeId"], # required
|
|
16290
|
-
# database: "EnclosedInStringProperty", # required
|
|
16291
|
-
# table: "EnclosedInStringProperty", # required
|
|
16292
|
-
# },
|
|
16293
|
-
# oracle_sql_catalog_target: {
|
|
16294
|
-
# name: "NodeName", # required
|
|
16295
|
-
# inputs: ["NodeId"], # required
|
|
16296
|
-
# database: "EnclosedInStringProperty", # required
|
|
16297
|
-
# table: "EnclosedInStringProperty", # required
|
|
16298
|
-
# },
|
|
16299
|
-
# postgre_sql_catalog_target: {
|
|
16300
|
-
# name: "NodeName", # required
|
|
16301
|
-
# inputs: ["NodeId"], # required
|
|
16302
|
-
# database: "EnclosedInStringProperty", # required
|
|
16303
|
-
# table: "EnclosedInStringProperty", # required
|
|
16304
|
-
# },
|
|
16305
|
-
# dynamic_transform: {
|
|
16306
|
-
# name: "EnclosedInStringProperty", # required
|
|
16307
|
-
# transform_name: "EnclosedInStringProperty", # required
|
|
16308
|
-
# inputs: ["NodeId"], # required
|
|
16309
|
-
# parameters: [
|
|
16310
|
-
# {
|
|
16311
|
-
# name: "EnclosedInStringProperty", # required
|
|
16312
|
-
# type: "str", # required, accepts str, int, float, complex, bool, list, null
|
|
16313
|
-
# validation_rule: "EnclosedInStringProperty",
|
|
16314
|
-
# validation_message: "EnclosedInStringProperty",
|
|
16315
|
-
# value: ["EnclosedInStringProperty"],
|
|
16316
|
-
# list_type: "str", # accepts str, int, float, complex, bool, list, null
|
|
16317
|
-
# is_optional: false,
|
|
16318
|
-
# },
|
|
16319
|
-
# ],
|
|
16320
|
-
# function_name: "EnclosedInStringProperty", # required
|
|
16321
|
-
# path: "EnclosedInStringProperty", # required
|
|
16322
|
-
# version: "EnclosedInStringProperty",
|
|
16323
|
-
# },
|
|
16324
|
-
# evaluate_data_quality: {
|
|
16325
|
-
# name: "NodeName", # required
|
|
16326
|
-
# inputs: ["NodeId"], # required
|
|
16327
|
-
# ruleset: "DQDLString", # required
|
|
16328
|
-
# output: "PrimaryInput", # accepts PrimaryInput, EvaluationResults
|
|
16329
|
-
# publishing_options: {
|
|
16330
|
-
# evaluation_context: "GenericLimitedString",
|
|
16331
|
-
# results_s3_prefix: "EnclosedInStringProperty",
|
|
16332
|
-
# cloud_watch_metrics_enabled: false,
|
|
16333
|
-
# results_publishing_enabled: false,
|
|
16334
|
-
# },
|
|
16335
|
-
# stop_job_on_failure_options: {
|
|
16336
|
-
# stop_job_on_failure_timing: "Immediate", # accepts Immediate, AfterDataLoad
|
|
16337
|
-
# },
|
|
16338
|
-
# },
|
|
16339
|
-
# s3_catalog_hudi_source: {
|
|
16340
|
-
# name: "NodeName", # required
|
|
16341
|
-
# database: "EnclosedInStringProperty", # required
|
|
16342
|
-
# table: "EnclosedInStringProperty", # required
|
|
16343
|
-
# additional_hudi_options: {
|
|
16344
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16345
|
-
# },
|
|
16346
|
-
# output_schemas: [
|
|
16347
|
-
# {
|
|
16348
|
-
# columns: [
|
|
16349
|
-
# {
|
|
16350
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16351
|
-
# type: "ColumnTypeString",
|
|
16352
|
-
# },
|
|
16353
|
-
# ],
|
|
16354
|
-
# },
|
|
16355
|
-
# ],
|
|
16356
|
-
# },
|
|
16357
|
-
# catalog_hudi_source: {
|
|
16358
|
-
# name: "NodeName", # required
|
|
16359
|
-
# database: "EnclosedInStringProperty", # required
|
|
16360
|
-
# table: "EnclosedInStringProperty", # required
|
|
16361
|
-
# additional_hudi_options: {
|
|
16362
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16363
|
-
# },
|
|
16364
|
-
# output_schemas: [
|
|
16365
|
-
# {
|
|
16366
|
-
# columns: [
|
|
16367
|
-
# {
|
|
16368
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16369
|
-
# type: "ColumnTypeString",
|
|
16370
|
-
# },
|
|
16371
|
-
# ],
|
|
16372
|
-
# },
|
|
16373
|
-
# ],
|
|
16374
|
-
# },
|
|
16375
|
-
# s3_hudi_source: {
|
|
16376
|
-
# name: "NodeName", # required
|
|
16377
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
16378
|
-
# additional_hudi_options: {
|
|
16379
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16380
|
-
# },
|
|
16381
|
-
# additional_options: {
|
|
16382
|
-
# bounded_size: 1,
|
|
16383
|
-
# bounded_files: 1,
|
|
16384
|
-
# enable_sample_path: false,
|
|
16385
|
-
# sample_path: "EnclosedInStringProperty",
|
|
16386
|
-
# },
|
|
16387
|
-
# output_schemas: [
|
|
16388
|
-
# {
|
|
16389
|
-
# columns: [
|
|
16390
|
-
# {
|
|
16391
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16392
|
-
# type: "ColumnTypeString",
|
|
16393
|
-
# },
|
|
16394
|
-
# ],
|
|
16395
|
-
# },
|
|
16396
|
-
# ],
|
|
16397
|
-
# },
|
|
16398
|
-
# s3_hudi_catalog_target: {
|
|
16399
|
-
# name: "NodeName", # required
|
|
16400
|
-
# inputs: ["NodeId"], # required
|
|
16401
|
-
# partition_keys: [
|
|
16402
|
-
# ["EnclosedInStringProperty"],
|
|
16403
|
-
# ],
|
|
16404
|
-
# table: "EnclosedInStringProperty", # required
|
|
16405
|
-
# database: "EnclosedInStringProperty", # required
|
|
16406
|
-
# additional_options: { # required
|
|
16407
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16408
|
-
# },
|
|
16409
|
-
# schema_change_policy: {
|
|
16410
|
-
# enable_update_catalog: false,
|
|
16411
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
16412
|
-
# },
|
|
16413
|
-
# },
|
|
16414
|
-
# s3_hudi_direct_target: {
|
|
16415
|
-
# name: "NodeName", # required
|
|
16416
|
-
# inputs: ["NodeId"], # required
|
|
16417
|
-
# path: "EnclosedInStringProperty", # required
|
|
16418
|
-
# compression: "gzip", # required, accepts gzip, lzo, uncompressed, snappy
|
|
16419
|
-
# partition_keys: [
|
|
16420
|
-
# ["EnclosedInStringProperty"],
|
|
16421
|
-
# ],
|
|
16422
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi, delta
|
|
16423
|
-
# additional_options: { # required
|
|
16424
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16425
|
-
# },
|
|
16426
|
-
# schema_change_policy: {
|
|
16427
|
-
# enable_update_catalog: false,
|
|
16428
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
16429
|
-
# table: "EnclosedInStringProperty",
|
|
16430
|
-
# database: "EnclosedInStringProperty",
|
|
16431
|
-
# },
|
|
16432
|
-
# },
|
|
16433
|
-
# direct_jdbc_source: {
|
|
16434
|
-
# name: "NodeName", # required
|
|
16435
|
-
# database: "EnclosedInStringProperty", # required
|
|
16436
|
-
# table: "EnclosedInStringProperty", # required
|
|
16437
|
-
# connection_name: "EnclosedInStringProperty", # required
|
|
16438
|
-
# connection_type: "sqlserver", # required, accepts sqlserver, mysql, oracle, postgresql, redshift
|
|
16439
|
-
# redshift_tmp_dir: "EnclosedInStringProperty",
|
|
16440
|
-
# },
|
|
16441
|
-
# s3_catalog_delta_source: {
|
|
16442
|
-
# name: "NodeName", # required
|
|
16443
|
-
# database: "EnclosedInStringProperty", # required
|
|
16444
|
-
# table: "EnclosedInStringProperty", # required
|
|
16445
|
-
# additional_delta_options: {
|
|
16446
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16447
|
-
# },
|
|
16448
|
-
# output_schemas: [
|
|
16449
|
-
# {
|
|
16450
|
-
# columns: [
|
|
16451
|
-
# {
|
|
16452
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16453
|
-
# type: "ColumnTypeString",
|
|
16454
|
-
# },
|
|
16455
|
-
# ],
|
|
16456
|
-
# },
|
|
16457
|
-
# ],
|
|
16458
|
-
# },
|
|
16459
|
-
# catalog_delta_source: {
|
|
16460
|
-
# name: "NodeName", # required
|
|
16461
|
-
# database: "EnclosedInStringProperty", # required
|
|
16462
|
-
# table: "EnclosedInStringProperty", # required
|
|
16463
|
-
# additional_delta_options: {
|
|
16464
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16465
|
-
# },
|
|
16466
|
-
# output_schemas: [
|
|
16467
|
-
# {
|
|
16468
|
-
# columns: [
|
|
16469
|
-
# {
|
|
16470
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16471
|
-
# type: "ColumnTypeString",
|
|
16472
|
-
# },
|
|
16473
|
-
# ],
|
|
16474
|
-
# },
|
|
16475
|
-
# ],
|
|
16476
|
-
# },
|
|
16477
|
-
# s3_delta_source: {
|
|
16478
|
-
# name: "NodeName", # required
|
|
16479
|
-
# paths: ["EnclosedInStringProperty"], # required
|
|
16480
|
-
# additional_delta_options: {
|
|
16481
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16482
|
-
# },
|
|
16483
|
-
# additional_options: {
|
|
16484
|
-
# bounded_size: 1,
|
|
16485
|
-
# bounded_files: 1,
|
|
16486
|
-
# enable_sample_path: false,
|
|
16487
|
-
# sample_path: "EnclosedInStringProperty",
|
|
16488
|
-
# },
|
|
16489
|
-
# output_schemas: [
|
|
16490
|
-
# {
|
|
16491
|
-
# columns: [
|
|
16492
|
-
# {
|
|
16493
|
-
# name: "GlueStudioColumnNameString", # required
|
|
16494
|
-
# type: "ColumnTypeString",
|
|
16495
|
-
# },
|
|
16496
|
-
# ],
|
|
16497
|
-
# },
|
|
16498
|
-
# ],
|
|
16499
|
-
# },
|
|
16500
|
-
# s3_delta_catalog_target: {
|
|
16501
|
-
# name: "NodeName", # required
|
|
16502
|
-
# inputs: ["NodeId"], # required
|
|
16503
|
-
# partition_keys: [
|
|
16504
|
-
# ["EnclosedInStringProperty"],
|
|
16505
|
-
# ],
|
|
16506
|
-
# table: "EnclosedInStringProperty", # required
|
|
16507
|
-
# database: "EnclosedInStringProperty", # required
|
|
16508
|
-
# additional_options: {
|
|
16509
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16510
|
-
# },
|
|
16511
|
-
# schema_change_policy: {
|
|
16512
|
-
# enable_update_catalog: false,
|
|
16513
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
16514
|
-
# },
|
|
16515
|
-
# },
|
|
16516
|
-
# s3_delta_direct_target: {
|
|
16517
|
-
# name: "NodeName", # required
|
|
16518
|
-
# inputs: ["NodeId"], # required
|
|
16519
|
-
# partition_keys: [
|
|
16520
|
-
# ["EnclosedInStringProperty"],
|
|
16521
|
-
# ],
|
|
16522
|
-
# path: "EnclosedInStringProperty", # required
|
|
16523
|
-
# compression: "uncompressed", # required, accepts uncompressed, snappy
|
|
16524
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi, delta
|
|
16525
|
-
# additional_options: {
|
|
16526
|
-
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
16527
|
-
# },
|
|
16528
|
-
# schema_change_policy: {
|
|
16529
|
-
# enable_update_catalog: false,
|
|
16530
|
-
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
16531
|
-
# table: "EnclosedInStringProperty",
|
|
16532
|
-
# database: "EnclosedInStringProperty",
|
|
16533
|
-
# },
|
|
16534
|
-
# },
|
|
16535
|
-
# },
|
|
16536
|
-
# },
|
|
16537
|
-
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
|
16538
|
-
# source_control_details: {
|
|
16539
|
-
# provider: "GITHUB", # accepts GITHUB, AWS_CODE_COMMIT
|
|
16540
|
-
# repository: "Generic512CharString",
|
|
16541
|
-
# owner: "Generic512CharString",
|
|
16542
|
-
# branch: "Generic512CharString",
|
|
16543
|
-
# folder: "Generic512CharString",
|
|
16544
|
-
# last_commit_id: "Generic512CharString",
|
|
16545
|
-
# auth_strategy: "PERSONAL_ACCESS_TOKEN", # accepts PERSONAL_ACCESS_TOKEN, AWS_SECRETS_MANAGER
|
|
16546
|
-
# auth_token: "Generic512CharString",
|
|
16547
|
-
# },
|
|
16548
|
-
# },
|
|
16549
|
-
# })
|
|
16550
|
-
#
|
|
16551
14934
|
# @example Response structure
|
|
16552
14935
|
#
|
|
16553
14936
|
# resp.job_name #=> String
|
|
@@ -16728,7 +15111,7 @@ module Aws::Glue
|
|
|
16728
15111
|
# role: "RoleString",
|
|
16729
15112
|
# glue_version: "GlueVersionString",
|
|
16730
15113
|
# max_capacity: 1.0,
|
|
16731
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
|
15114
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
|
|
16732
15115
|
# number_of_workers: 1,
|
|
16733
15116
|
# timeout: 1,
|
|
16734
15117
|
# max_retries: 1,
|
|
@@ -17357,7 +15740,7 @@ module Aws::Glue
|
|
|
17357
15740
|
params: params,
|
|
17358
15741
|
config: config)
|
|
17359
15742
|
context[:gem_name] = 'aws-sdk-glue'
|
|
17360
|
-
context[:gem_version] = '1.
|
|
15743
|
+
context[:gem_version] = '1.136.0'
|
|
17361
15744
|
Seahorse::Client::Request.new(handlers, context)
|
|
17362
15745
|
end
|
|
17363
15746
|
|