aws-sdk-glue 1.128.0 → 1.130.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-glue/client.rb +418 -6
- data/lib/aws-sdk-glue/client_api.rb +56 -0
- data/lib/aws-sdk-glue/types.rb +280 -6
- data/lib/aws-sdk-glue.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fc20b3b495d3049435e01cdc159857979895302279086e6be82c53c04972f875
|
|
4
|
+
data.tar.gz: 3a0dc34cb167755056fbc5b8055a2511b565dbc7dfb7fdef4851270f2a25b2d5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b47595e4c1e206fcdac0846c3229400dfd4116392c25c0e5345de8fb7ee303f381680a0609964c2a0ff60108ddb55a66f8393a55fd72ded3f240c5bd21f804e4
|
|
7
|
+
data.tar.gz: c6ef1f44c9213abee66ad9dc3c22961c3336771ce1ed642e5b365b4912c2c4c4a5dfd17037edf4056df928ae81fcdbead33c9354def497ff4b99ae57d22b3eba
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
Unreleased Changes
|
|
2
2
|
------------------
|
|
3
3
|
|
|
4
|
+
1.130.0 (2023-02-08)
|
|
5
|
+
------------------
|
|
6
|
+
|
|
7
|
+
* Feature - DirectJDBCSource + Glue 4.0 streaming options
|
|
8
|
+
|
|
9
|
+
1.129.0 (2023-01-19)
|
|
10
|
+
------------------
|
|
11
|
+
|
|
12
|
+
* Feature - Release Glue Studio Hudi Data Lake Format for SDK/CLI
|
|
13
|
+
|
|
4
14
|
1.128.0 (2023-01-18)
|
|
5
15
|
------------------
|
|
6
16
|
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.
|
|
1
|
+
1.130.0
|
data/lib/aws-sdk-glue/client.rb
CHANGED
|
@@ -1236,7 +1236,7 @@ module Aws::Glue
|
|
|
1236
1236
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.partition_keys[0][0] #=> String
|
|
1237
1237
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.path #=> String
|
|
1238
1238
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.compression #=> String
|
|
1239
|
-
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet"
|
|
1239
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet", "hudi"
|
|
1240
1240
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
1241
1241
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
1242
1242
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.table #=> String
|
|
@@ -1353,6 +1353,8 @@ module Aws::Glue
|
|
|
1353
1353
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.stream_arn #=> String
|
|
1354
1354
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.role_arn #=> String
|
|
1355
1355
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.role_session_name #=> String
|
|
1356
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.add_record_timestamp #=> String
|
|
1357
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
1356
1358
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.data_preview_options.polling_time #=> Integer
|
|
1357
1359
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.data_preview_options.record_polling_limit #=> Integer
|
|
1358
1360
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.name #=> String
|
|
@@ -1371,6 +1373,9 @@ module Aws::Glue
|
|
|
1371
1373
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.retry_interval_ms #=> Integer
|
|
1372
1374
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.max_offsets_per_trigger #=> Integer
|
|
1373
1375
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.min_partitions #=> Integer
|
|
1376
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.include_headers #=> Boolean
|
|
1377
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.add_record_timestamp #=> String
|
|
1378
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
1374
1379
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.window_size #=> Integer
|
|
1375
1380
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.detect_schema #=> Boolean
|
|
1376
1381
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.data_preview_options.polling_time #=> Integer
|
|
@@ -1398,6 +1403,8 @@ module Aws::Glue
|
|
|
1398
1403
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.stream_arn #=> String
|
|
1399
1404
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.role_arn #=> String
|
|
1400
1405
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.role_session_name #=> String
|
|
1406
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.add_record_timestamp #=> String
|
|
1407
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
1401
1408
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.data_preview_options.polling_time #=> Integer
|
|
1402
1409
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.data_preview_options.record_polling_limit #=> Integer
|
|
1403
1410
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.name #=> String
|
|
@@ -1420,6 +1427,9 @@ module Aws::Glue
|
|
|
1420
1427
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.retry_interval_ms #=> Integer
|
|
1421
1428
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.max_offsets_per_trigger #=> Integer
|
|
1422
1429
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.min_partitions #=> Integer
|
|
1430
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.include_headers #=> Boolean
|
|
1431
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.add_record_timestamp #=> String
|
|
1432
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
1423
1433
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.data_preview_options.polling_time #=> Integer
|
|
1424
1434
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.data_preview_options.record_polling_limit #=> Integer
|
|
1425
1435
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].drop_null_fields.name #=> String
|
|
@@ -1543,6 +1553,64 @@ module Aws::Glue
|
|
|
1543
1553
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.publishing_options.cloud_watch_metrics_enabled #=> Boolean
|
|
1544
1554
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.publishing_options.results_publishing_enabled #=> Boolean
|
|
1545
1555
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.stop_job_on_failure_options.stop_job_on_failure_timing #=> String, one of "Immediate", "AfterDataLoad"
|
|
1556
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.name #=> String
|
|
1557
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.database #=> String
|
|
1558
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.table #=> String
|
|
1559
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.additional_hudi_options #=> Hash
|
|
1560
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
1561
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas #=> Array
|
|
1562
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns #=> Array
|
|
1563
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
1564
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
1565
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.name #=> String
|
|
1566
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.database #=> String
|
|
1567
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.table #=> String
|
|
1568
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.additional_hudi_options #=> Hash
|
|
1569
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
1570
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas #=> Array
|
|
1571
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns #=> Array
|
|
1572
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
1573
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
1574
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.name #=> String
|
|
1575
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.paths #=> Array
|
|
1576
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.paths[0] #=> String
|
|
1577
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_hudi_options #=> Hash
|
|
1578
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
1579
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.bounded_size #=> Integer
|
|
1580
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.bounded_files #=> Integer
|
|
1581
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.enable_sample_path #=> Boolean
|
|
1582
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.sample_path #=> String
|
|
1583
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas #=> Array
|
|
1584
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns #=> Array
|
|
1585
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
1586
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
1587
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.name #=> String
|
|
1588
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.inputs #=> Array
|
|
1589
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.inputs[0] #=> String
|
|
1590
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys #=> Array
|
|
1591
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys[0] #=> Array
|
|
1592
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys[0][0] #=> String
|
|
1593
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.table #=> String
|
|
1594
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.database #=> String
|
|
1595
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.additional_options #=> Hash
|
|
1596
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.additional_options["EnclosedInStringProperty"] #=> String
|
|
1597
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
1598
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
1599
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.name #=> String
|
|
1600
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.inputs #=> Array
|
|
1601
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.inputs[0] #=> String
|
|
1602
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.path #=> String
|
|
1603
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.compression #=> String, one of "gzip", "lzo", "uncompressed", "snappy"
|
|
1604
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys #=> Array
|
|
1605
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys[0] #=> Array
|
|
1606
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys[0][0] #=> String
|
|
1607
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet", "hudi"
|
|
1608
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.additional_options #=> Hash
|
|
1609
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.additional_options["EnclosedInStringProperty"] #=> String
|
|
1610
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
1611
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
1612
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.table #=> String
|
|
1613
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.database #=> String
|
|
1546
1614
|
# resp.jobs[0].execution_class #=> String, one of "FLEX", "STANDARD"
|
|
1547
1615
|
# resp.jobs[0].source_control_details.provider #=> String, one of "GITHUB", "AWS_CODE_COMMIT"
|
|
1548
1616
|
# resp.jobs[0].source_control_details.repository #=> String
|
|
@@ -3411,7 +3479,7 @@ module Aws::Glue
|
|
|
3411
3479
|
# ],
|
|
3412
3480
|
# path: "EnclosedInStringProperty", # required
|
|
3413
3481
|
# compression: "EnclosedInStringProperty",
|
|
3414
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet
|
|
3482
|
+
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi
|
|
3415
3483
|
# schema_change_policy: {
|
|
3416
3484
|
# enable_update_catalog: false,
|
|
3417
3485
|
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
@@ -3570,6 +3638,8 @@ module Aws::Glue
|
|
|
3570
3638
|
# stream_arn: "EnclosedInStringProperty",
|
|
3571
3639
|
# role_arn: "EnclosedInStringProperty",
|
|
3572
3640
|
# role_session_name: "EnclosedInStringProperty",
|
|
3641
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3642
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3573
3643
|
# },
|
|
3574
3644
|
# data_preview_options: {
|
|
3575
3645
|
# polling_time: 1,
|
|
@@ -3594,6 +3664,9 @@ module Aws::Glue
|
|
|
3594
3664
|
# retry_interval_ms: 1,
|
|
3595
3665
|
# max_offsets_per_trigger: 1,
|
|
3596
3666
|
# min_partitions: 1,
|
|
3667
|
+
# include_headers: false,
|
|
3668
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3669
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3597
3670
|
# },
|
|
3598
3671
|
# window_size: 1,
|
|
3599
3672
|
# detect_schema: false,
|
|
@@ -3627,6 +3700,8 @@ module Aws::Glue
|
|
|
3627
3700
|
# stream_arn: "EnclosedInStringProperty",
|
|
3628
3701
|
# role_arn: "EnclosedInStringProperty",
|
|
3629
3702
|
# role_session_name: "EnclosedInStringProperty",
|
|
3703
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3704
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3630
3705
|
# },
|
|
3631
3706
|
# data_preview_options: {
|
|
3632
3707
|
# polling_time: 1,
|
|
@@ -3655,6 +3730,9 @@ module Aws::Glue
|
|
|
3655
3730
|
# retry_interval_ms: 1,
|
|
3656
3731
|
# max_offsets_per_trigger: 1,
|
|
3657
3732
|
# min_partitions: 1,
|
|
3733
|
+
# include_headers: false,
|
|
3734
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
3735
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
3658
3736
|
# },
|
|
3659
3737
|
# data_preview_options: {
|
|
3660
3738
|
# polling_time: 1,
|
|
@@ -3823,6 +3901,100 @@ module Aws::Glue
|
|
|
3823
3901
|
# stop_job_on_failure_timing: "Immediate", # accepts Immediate, AfterDataLoad
|
|
3824
3902
|
# },
|
|
3825
3903
|
# },
|
|
3904
|
+
# s3_catalog_hudi_source: {
|
|
3905
|
+
# name: "NodeName", # required
|
|
3906
|
+
# database: "EnclosedInStringProperty", # required
|
|
3907
|
+
# table: "EnclosedInStringProperty", # required
|
|
3908
|
+
# additional_hudi_options: {
|
|
3909
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3910
|
+
# },
|
|
3911
|
+
# output_schemas: [
|
|
3912
|
+
# {
|
|
3913
|
+
# columns: [
|
|
3914
|
+
# {
|
|
3915
|
+
# name: "GlueStudioColumnNameString", # required
|
|
3916
|
+
# type: "ColumnTypeString",
|
|
3917
|
+
# },
|
|
3918
|
+
# ],
|
|
3919
|
+
# },
|
|
3920
|
+
# ],
|
|
3921
|
+
# },
|
|
3922
|
+
# catalog_hudi_source: {
|
|
3923
|
+
# name: "NodeName", # required
|
|
3924
|
+
# database: "EnclosedInStringProperty", # required
|
|
3925
|
+
# table: "EnclosedInStringProperty", # required
|
|
3926
|
+
# additional_hudi_options: {
|
|
3927
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3928
|
+
# },
|
|
3929
|
+
# output_schemas: [
|
|
3930
|
+
# {
|
|
3931
|
+
# columns: [
|
|
3932
|
+
# {
|
|
3933
|
+
# name: "GlueStudioColumnNameString", # required
|
|
3934
|
+
# type: "ColumnTypeString",
|
|
3935
|
+
# },
|
|
3936
|
+
# ],
|
|
3937
|
+
# },
|
|
3938
|
+
# ],
|
|
3939
|
+
# },
|
|
3940
|
+
# s3_hudi_source: {
|
|
3941
|
+
# name: "NodeName", # required
|
|
3942
|
+
# paths: ["EnclosedInStringProperty"], # required
|
|
3943
|
+
# additional_hudi_options: {
|
|
3944
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3945
|
+
# },
|
|
3946
|
+
# additional_options: {
|
|
3947
|
+
# bounded_size: 1,
|
|
3948
|
+
# bounded_files: 1,
|
|
3949
|
+
# enable_sample_path: false,
|
|
3950
|
+
# sample_path: "EnclosedInStringProperty",
|
|
3951
|
+
# },
|
|
3952
|
+
# output_schemas: [
|
|
3953
|
+
# {
|
|
3954
|
+
# columns: [
|
|
3955
|
+
# {
|
|
3956
|
+
# name: "GlueStudioColumnNameString", # required
|
|
3957
|
+
# type: "ColumnTypeString",
|
|
3958
|
+
# },
|
|
3959
|
+
# ],
|
|
3960
|
+
# },
|
|
3961
|
+
# ],
|
|
3962
|
+
# },
|
|
3963
|
+
# s3_hudi_catalog_target: {
|
|
3964
|
+
# name: "NodeName", # required
|
|
3965
|
+
# inputs: ["NodeId"], # required
|
|
3966
|
+
# partition_keys: [
|
|
3967
|
+
# ["EnclosedInStringProperty"],
|
|
3968
|
+
# ],
|
|
3969
|
+
# table: "EnclosedInStringProperty", # required
|
|
3970
|
+
# database: "EnclosedInStringProperty", # required
|
|
3971
|
+
# additional_options: { # required
|
|
3972
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3973
|
+
# },
|
|
3974
|
+
# schema_change_policy: {
|
|
3975
|
+
# enable_update_catalog: false,
|
|
3976
|
+
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
3977
|
+
# },
|
|
3978
|
+
# },
|
|
3979
|
+
# s3_hudi_direct_target: {
|
|
3980
|
+
# name: "NodeName", # required
|
|
3981
|
+
# inputs: ["NodeId"], # required
|
|
3982
|
+
# path: "EnclosedInStringProperty", # required
|
|
3983
|
+
# compression: "gzip", # required, accepts gzip, lzo, uncompressed, snappy
|
|
3984
|
+
# partition_keys: [
|
|
3985
|
+
# ["EnclosedInStringProperty"],
|
|
3986
|
+
# ],
|
|
3987
|
+
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi
|
|
3988
|
+
# additional_options: { # required
|
|
3989
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
3990
|
+
# },
|
|
3991
|
+
# schema_change_policy: {
|
|
3992
|
+
# enable_update_catalog: false,
|
|
3993
|
+
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
3994
|
+
# table: "EnclosedInStringProperty",
|
|
3995
|
+
# database: "EnclosedInStringProperty",
|
|
3996
|
+
# },
|
|
3997
|
+
# },
|
|
3826
3998
|
# },
|
|
3827
3999
|
# },
|
|
3828
4000
|
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
|
@@ -7452,7 +7624,7 @@ module Aws::Glue
|
|
|
7452
7624
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.partition_keys[0][0] #=> String
|
|
7453
7625
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.path #=> String
|
|
7454
7626
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.compression #=> String
|
|
7455
|
-
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet"
|
|
7627
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet", "hudi"
|
|
7456
7628
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
7457
7629
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
7458
7630
|
# resp.job.code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.table #=> String
|
|
@@ -7569,6 +7741,8 @@ module Aws::Glue
|
|
|
7569
7741
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.stream_arn #=> String
|
|
7570
7742
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.role_arn #=> String
|
|
7571
7743
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.role_session_name #=> String
|
|
7744
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.add_record_timestamp #=> String
|
|
7745
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
7572
7746
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kinesis_source.data_preview_options.polling_time #=> Integer
|
|
7573
7747
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kinesis_source.data_preview_options.record_polling_limit #=> Integer
|
|
7574
7748
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.name #=> String
|
|
@@ -7587,6 +7761,9 @@ module Aws::Glue
|
|
|
7587
7761
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.retry_interval_ms #=> Integer
|
|
7588
7762
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.max_offsets_per_trigger #=> Integer
|
|
7589
7763
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.min_partitions #=> Integer
|
|
7764
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.include_headers #=> Boolean
|
|
7765
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.add_record_timestamp #=> String
|
|
7766
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
7590
7767
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.window_size #=> Integer
|
|
7591
7768
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.detect_schema #=> Boolean
|
|
7592
7769
|
# resp.job.code_gen_configuration_nodes["NodeId"].direct_kafka_source.data_preview_options.polling_time #=> Integer
|
|
@@ -7614,6 +7791,8 @@ module Aws::Glue
|
|
|
7614
7791
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.stream_arn #=> String
|
|
7615
7792
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.role_arn #=> String
|
|
7616
7793
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.role_session_name #=> String
|
|
7794
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.add_record_timestamp #=> String
|
|
7795
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
7617
7796
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.data_preview_options.polling_time #=> Integer
|
|
7618
7797
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.data_preview_options.record_polling_limit #=> Integer
|
|
7619
7798
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.name #=> String
|
|
@@ -7636,6 +7815,9 @@ module Aws::Glue
|
|
|
7636
7815
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.retry_interval_ms #=> Integer
|
|
7637
7816
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.max_offsets_per_trigger #=> Integer
|
|
7638
7817
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.min_partitions #=> Integer
|
|
7818
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.include_headers #=> Boolean
|
|
7819
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.add_record_timestamp #=> String
|
|
7820
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
7639
7821
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.data_preview_options.polling_time #=> Integer
|
|
7640
7822
|
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_kafka_source.data_preview_options.record_polling_limit #=> Integer
|
|
7641
7823
|
# resp.job.code_gen_configuration_nodes["NodeId"].drop_null_fields.name #=> String
|
|
@@ -7759,6 +7941,64 @@ module Aws::Glue
|
|
|
7759
7941
|
# resp.job.code_gen_configuration_nodes["NodeId"].evaluate_data_quality.publishing_options.cloud_watch_metrics_enabled #=> Boolean
|
|
7760
7942
|
# resp.job.code_gen_configuration_nodes["NodeId"].evaluate_data_quality.publishing_options.results_publishing_enabled #=> Boolean
|
|
7761
7943
|
# resp.job.code_gen_configuration_nodes["NodeId"].evaluate_data_quality.stop_job_on_failure_options.stop_job_on_failure_timing #=> String, one of "Immediate", "AfterDataLoad"
|
|
7944
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.name #=> String
|
|
7945
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.database #=> String
|
|
7946
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.table #=> String
|
|
7947
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.additional_hudi_options #=> Hash
|
|
7948
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
7949
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas #=> Array
|
|
7950
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns #=> Array
|
|
7951
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
7952
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
7953
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.name #=> String
|
|
7954
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.database #=> String
|
|
7955
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.table #=> String
|
|
7956
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.additional_hudi_options #=> Hash
|
|
7957
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
7958
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas #=> Array
|
|
7959
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns #=> Array
|
|
7960
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
7961
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
7962
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.name #=> String
|
|
7963
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.paths #=> Array
|
|
7964
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.paths[0] #=> String
|
|
7965
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_hudi_options #=> Hash
|
|
7966
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
7967
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.bounded_size #=> Integer
|
|
7968
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.bounded_files #=> Integer
|
|
7969
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.enable_sample_path #=> Boolean
|
|
7970
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.sample_path #=> String
|
|
7971
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas #=> Array
|
|
7972
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns #=> Array
|
|
7973
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
7974
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
7975
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.name #=> String
|
|
7976
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.inputs #=> Array
|
|
7977
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.inputs[0] #=> String
|
|
7978
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys #=> Array
|
|
7979
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys[0] #=> Array
|
|
7980
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys[0][0] #=> String
|
|
7981
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.table #=> String
|
|
7982
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.database #=> String
|
|
7983
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.additional_options #=> Hash
|
|
7984
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.additional_options["EnclosedInStringProperty"] #=> String
|
|
7985
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
7986
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
7987
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.name #=> String
|
|
7988
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.inputs #=> Array
|
|
7989
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.inputs[0] #=> String
|
|
7990
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.path #=> String
|
|
7991
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.compression #=> String, one of "gzip", "lzo", "uncompressed", "snappy"
|
|
7992
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys #=> Array
|
|
7993
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys[0] #=> Array
|
|
7994
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys[0][0] #=> String
|
|
7995
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet", "hudi"
|
|
7996
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.additional_options #=> Hash
|
|
7997
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.additional_options["EnclosedInStringProperty"] #=> String
|
|
7998
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
7999
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
8000
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.table #=> String
|
|
8001
|
+
# resp.job.code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.database #=> String
|
|
7762
8002
|
# resp.job.execution_class #=> String, one of "FLEX", "STANDARD"
|
|
7763
8003
|
# resp.job.source_control_details.provider #=> String, one of "GITHUB", "AWS_CODE_COMMIT"
|
|
7764
8004
|
# resp.job.source_control_details.repository #=> String
|
|
@@ -8208,7 +8448,7 @@ module Aws::Glue
|
|
|
8208
8448
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.partition_keys[0][0] #=> String
|
|
8209
8449
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.path #=> String
|
|
8210
8450
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.compression #=> String
|
|
8211
|
-
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet"
|
|
8451
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet", "hudi"
|
|
8212
8452
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
8213
8453
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
8214
8454
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_direct_target.schema_change_policy.table #=> String
|
|
@@ -8325,6 +8565,8 @@ module Aws::Glue
|
|
|
8325
8565
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.stream_arn #=> String
|
|
8326
8566
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.role_arn #=> String
|
|
8327
8567
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.role_session_name #=> String
|
|
8568
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.add_record_timestamp #=> String
|
|
8569
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
8328
8570
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.data_preview_options.polling_time #=> Integer
|
|
8329
8571
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kinesis_source.data_preview_options.record_polling_limit #=> Integer
|
|
8330
8572
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.name #=> String
|
|
@@ -8343,6 +8585,9 @@ module Aws::Glue
|
|
|
8343
8585
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.retry_interval_ms #=> Integer
|
|
8344
8586
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.max_offsets_per_trigger #=> Integer
|
|
8345
8587
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.min_partitions #=> Integer
|
|
8588
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.include_headers #=> Boolean
|
|
8589
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.add_record_timestamp #=> String
|
|
8590
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
8346
8591
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.window_size #=> Integer
|
|
8347
8592
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.detect_schema #=> Boolean
|
|
8348
8593
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].direct_kafka_source.data_preview_options.polling_time #=> Integer
|
|
@@ -8370,6 +8615,8 @@ module Aws::Glue
|
|
|
8370
8615
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.stream_arn #=> String
|
|
8371
8616
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.role_arn #=> String
|
|
8372
8617
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.role_session_name #=> String
|
|
8618
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.add_record_timestamp #=> String
|
|
8619
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
8373
8620
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.data_preview_options.polling_time #=> Integer
|
|
8374
8621
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kinesis_source.data_preview_options.record_polling_limit #=> Integer
|
|
8375
8622
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.name #=> String
|
|
@@ -8392,6 +8639,9 @@ module Aws::Glue
|
|
|
8392
8639
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.retry_interval_ms #=> Integer
|
|
8393
8640
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.max_offsets_per_trigger #=> Integer
|
|
8394
8641
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.min_partitions #=> Integer
|
|
8642
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.include_headers #=> Boolean
|
|
8643
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.add_record_timestamp #=> String
|
|
8644
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.streaming_options.emit_consumer_lag_metrics #=> String
|
|
8395
8645
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.data_preview_options.polling_time #=> Integer
|
|
8396
8646
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_kafka_source.data_preview_options.record_polling_limit #=> Integer
|
|
8397
8647
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].drop_null_fields.name #=> String
|
|
@@ -8515,6 +8765,64 @@ module Aws::Glue
|
|
|
8515
8765
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.publishing_options.cloud_watch_metrics_enabled #=> Boolean
|
|
8516
8766
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.publishing_options.results_publishing_enabled #=> Boolean
|
|
8517
8767
|
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].evaluate_data_quality.stop_job_on_failure_options.stop_job_on_failure_timing #=> String, one of "Immediate", "AfterDataLoad"
|
|
8768
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.name #=> String
|
|
8769
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.database #=> String
|
|
8770
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.table #=> String
|
|
8771
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.additional_hudi_options #=> Hash
|
|
8772
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
8773
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas #=> Array
|
|
8774
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns #=> Array
|
|
8775
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
8776
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_catalog_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
8777
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.name #=> String
|
|
8778
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.database #=> String
|
|
8779
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.table #=> String
|
|
8780
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.additional_hudi_options #=> Hash
|
|
8781
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
8782
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas #=> Array
|
|
8783
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns #=> Array
|
|
8784
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
8785
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].catalog_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
8786
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.name #=> String
|
|
8787
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.paths #=> Array
|
|
8788
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.paths[0] #=> String
|
|
8789
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_hudi_options #=> Hash
|
|
8790
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_hudi_options["EnclosedInStringProperty"] #=> String
|
|
8791
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.bounded_size #=> Integer
|
|
8792
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.bounded_files #=> Integer
|
|
8793
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.enable_sample_path #=> Boolean
|
|
8794
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.additional_options.sample_path #=> String
|
|
8795
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas #=> Array
|
|
8796
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns #=> Array
|
|
8797
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns[0].name #=> String
|
|
8798
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_source.output_schemas[0].columns[0].type #=> String
|
|
8799
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.name #=> String
|
|
8800
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.inputs #=> Array
|
|
8801
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.inputs[0] #=> String
|
|
8802
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys #=> Array
|
|
8803
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys[0] #=> Array
|
|
8804
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.partition_keys[0][0] #=> String
|
|
8805
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.table #=> String
|
|
8806
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.database #=> String
|
|
8807
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.additional_options #=> Hash
|
|
8808
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.additional_options["EnclosedInStringProperty"] #=> String
|
|
8809
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
8810
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_catalog_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
8811
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.name #=> String
|
|
8812
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.inputs #=> Array
|
|
8813
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.inputs[0] #=> String
|
|
8814
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.path #=> String
|
|
8815
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.compression #=> String, one of "gzip", "lzo", "uncompressed", "snappy"
|
|
8816
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys #=> Array
|
|
8817
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys[0] #=> Array
|
|
8818
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.partition_keys[0][0] #=> String
|
|
8819
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.format #=> String, one of "json", "csv", "avro", "orc", "parquet", "hudi"
|
|
8820
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.additional_options #=> Hash
|
|
8821
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.additional_options["EnclosedInStringProperty"] #=> String
|
|
8822
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.enable_update_catalog #=> Boolean
|
|
8823
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.update_behavior #=> String, one of "UPDATE_IN_DATABASE", "LOG"
|
|
8824
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.table #=> String
|
|
8825
|
+
# resp.jobs[0].code_gen_configuration_nodes["NodeId"].s3_hudi_direct_target.schema_change_policy.database #=> String
|
|
8518
8826
|
# resp.jobs[0].execution_class #=> String, one of "FLEX", "STANDARD"
|
|
8519
8827
|
# resp.jobs[0].source_control_details.provider #=> String, one of "GITHUB", "AWS_CODE_COMMIT"
|
|
8520
8828
|
# resp.jobs[0].source_control_details.repository #=> String
|
|
@@ -15158,7 +15466,7 @@ module Aws::Glue
|
|
|
15158
15466
|
# ],
|
|
15159
15467
|
# path: "EnclosedInStringProperty", # required
|
|
15160
15468
|
# compression: "EnclosedInStringProperty",
|
|
15161
|
-
# format: "json", # required, accepts json, csv, avro, orc, parquet
|
|
15469
|
+
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi
|
|
15162
15470
|
# schema_change_policy: {
|
|
15163
15471
|
# enable_update_catalog: false,
|
|
15164
15472
|
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
@@ -15317,6 +15625,8 @@ module Aws::Glue
|
|
|
15317
15625
|
# stream_arn: "EnclosedInStringProperty",
|
|
15318
15626
|
# role_arn: "EnclosedInStringProperty",
|
|
15319
15627
|
# role_session_name: "EnclosedInStringProperty",
|
|
15628
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
15629
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
15320
15630
|
# },
|
|
15321
15631
|
# data_preview_options: {
|
|
15322
15632
|
# polling_time: 1,
|
|
@@ -15341,6 +15651,9 @@ module Aws::Glue
|
|
|
15341
15651
|
# retry_interval_ms: 1,
|
|
15342
15652
|
# max_offsets_per_trigger: 1,
|
|
15343
15653
|
# min_partitions: 1,
|
|
15654
|
+
# include_headers: false,
|
|
15655
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
15656
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
15344
15657
|
# },
|
|
15345
15658
|
# window_size: 1,
|
|
15346
15659
|
# detect_schema: false,
|
|
@@ -15374,6 +15687,8 @@ module Aws::Glue
|
|
|
15374
15687
|
# stream_arn: "EnclosedInStringProperty",
|
|
15375
15688
|
# role_arn: "EnclosedInStringProperty",
|
|
15376
15689
|
# role_session_name: "EnclosedInStringProperty",
|
|
15690
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
15691
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
15377
15692
|
# },
|
|
15378
15693
|
# data_preview_options: {
|
|
15379
15694
|
# polling_time: 1,
|
|
@@ -15402,6 +15717,9 @@ module Aws::Glue
|
|
|
15402
15717
|
# retry_interval_ms: 1,
|
|
15403
15718
|
# max_offsets_per_trigger: 1,
|
|
15404
15719
|
# min_partitions: 1,
|
|
15720
|
+
# include_headers: false,
|
|
15721
|
+
# add_record_timestamp: "EnclosedInStringProperty",
|
|
15722
|
+
# emit_consumer_lag_metrics: "EnclosedInStringProperty",
|
|
15405
15723
|
# },
|
|
15406
15724
|
# data_preview_options: {
|
|
15407
15725
|
# polling_time: 1,
|
|
@@ -15570,6 +15888,100 @@ module Aws::Glue
|
|
|
15570
15888
|
# stop_job_on_failure_timing: "Immediate", # accepts Immediate, AfterDataLoad
|
|
15571
15889
|
# },
|
|
15572
15890
|
# },
|
|
15891
|
+
# s3_catalog_hudi_source: {
|
|
15892
|
+
# name: "NodeName", # required
|
|
15893
|
+
# database: "EnclosedInStringProperty", # required
|
|
15894
|
+
# table: "EnclosedInStringProperty", # required
|
|
15895
|
+
# additional_hudi_options: {
|
|
15896
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15897
|
+
# },
|
|
15898
|
+
# output_schemas: [
|
|
15899
|
+
# {
|
|
15900
|
+
# columns: [
|
|
15901
|
+
# {
|
|
15902
|
+
# name: "GlueStudioColumnNameString", # required
|
|
15903
|
+
# type: "ColumnTypeString",
|
|
15904
|
+
# },
|
|
15905
|
+
# ],
|
|
15906
|
+
# },
|
|
15907
|
+
# ],
|
|
15908
|
+
# },
|
|
15909
|
+
# catalog_hudi_source: {
|
|
15910
|
+
# name: "NodeName", # required
|
|
15911
|
+
# database: "EnclosedInStringProperty", # required
|
|
15912
|
+
# table: "EnclosedInStringProperty", # required
|
|
15913
|
+
# additional_hudi_options: {
|
|
15914
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15915
|
+
# },
|
|
15916
|
+
# output_schemas: [
|
|
15917
|
+
# {
|
|
15918
|
+
# columns: [
|
|
15919
|
+
# {
|
|
15920
|
+
# name: "GlueStudioColumnNameString", # required
|
|
15921
|
+
# type: "ColumnTypeString",
|
|
15922
|
+
# },
|
|
15923
|
+
# ],
|
|
15924
|
+
# },
|
|
15925
|
+
# ],
|
|
15926
|
+
# },
|
|
15927
|
+
# s3_hudi_source: {
|
|
15928
|
+
# name: "NodeName", # required
|
|
15929
|
+
# paths: ["EnclosedInStringProperty"], # required
|
|
15930
|
+
# additional_hudi_options: {
|
|
15931
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15932
|
+
# },
|
|
15933
|
+
# additional_options: {
|
|
15934
|
+
# bounded_size: 1,
|
|
15935
|
+
# bounded_files: 1,
|
|
15936
|
+
# enable_sample_path: false,
|
|
15937
|
+
# sample_path: "EnclosedInStringProperty",
|
|
15938
|
+
# },
|
|
15939
|
+
# output_schemas: [
|
|
15940
|
+
# {
|
|
15941
|
+
# columns: [
|
|
15942
|
+
# {
|
|
15943
|
+
# name: "GlueStudioColumnNameString", # required
|
|
15944
|
+
# type: "ColumnTypeString",
|
|
15945
|
+
# },
|
|
15946
|
+
# ],
|
|
15947
|
+
# },
|
|
15948
|
+
# ],
|
|
15949
|
+
# },
|
|
15950
|
+
# s3_hudi_catalog_target: {
|
|
15951
|
+
# name: "NodeName", # required
|
|
15952
|
+
# inputs: ["NodeId"], # required
|
|
15953
|
+
# partition_keys: [
|
|
15954
|
+
# ["EnclosedInStringProperty"],
|
|
15955
|
+
# ],
|
|
15956
|
+
# table: "EnclosedInStringProperty", # required
|
|
15957
|
+
# database: "EnclosedInStringProperty", # required
|
|
15958
|
+
# additional_options: { # required
|
|
15959
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15960
|
+
# },
|
|
15961
|
+
# schema_change_policy: {
|
|
15962
|
+
# enable_update_catalog: false,
|
|
15963
|
+
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
15964
|
+
# },
|
|
15965
|
+
# },
|
|
15966
|
+
# s3_hudi_direct_target: {
|
|
15967
|
+
# name: "NodeName", # required
|
|
15968
|
+
# inputs: ["NodeId"], # required
|
|
15969
|
+
# path: "EnclosedInStringProperty", # required
|
|
15970
|
+
# compression: "gzip", # required, accepts gzip, lzo, uncompressed, snappy
|
|
15971
|
+
# partition_keys: [
|
|
15972
|
+
# ["EnclosedInStringProperty"],
|
|
15973
|
+
# ],
|
|
15974
|
+
# format: "json", # required, accepts json, csv, avro, orc, parquet, hudi
|
|
15975
|
+
# additional_options: { # required
|
|
15976
|
+
# "EnclosedInStringProperty" => "EnclosedInStringProperty",
|
|
15977
|
+
# },
|
|
15978
|
+
# schema_change_policy: {
|
|
15979
|
+
# enable_update_catalog: false,
|
|
15980
|
+
# update_behavior: "UPDATE_IN_DATABASE", # accepts UPDATE_IN_DATABASE, LOG
|
|
15981
|
+
# table: "EnclosedInStringProperty",
|
|
15982
|
+
# database: "EnclosedInStringProperty",
|
|
15983
|
+
# },
|
|
15984
|
+
# },
|
|
15573
15985
|
# },
|
|
15574
15986
|
# },
|
|
15575
15987
|
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
|
@@ -16395,7 +16807,7 @@ module Aws::Glue
|
|
|
16395
16807
|
params: params,
|
|
16396
16808
|
config: config)
|
|
16397
16809
|
context[:gem_name] = 'aws-sdk-glue'
|
|
16398
|
-
context[:gem_version] = '1.
|
|
16810
|
+
context[:gem_version] = '1.130.0'
|
|
16399
16811
|
Seahorse::Client::Request.new(handlers, context)
|
|
16400
16812
|
end
|
|
16401
16813
|
|
|
@@ -119,6 +119,7 @@ module Aws::Glue
|
|
|
119
119
|
CatalogEntries = Shapes::ListShape.new(name: 'CatalogEntries')
|
|
120
120
|
CatalogEntry = Shapes::StructureShape.new(name: 'CatalogEntry')
|
|
121
121
|
CatalogGetterPageSize = Shapes::IntegerShape.new(name: 'CatalogGetterPageSize')
|
|
122
|
+
CatalogHudiSource = Shapes::StructureShape.new(name: 'CatalogHudiSource')
|
|
122
123
|
CatalogIdString = Shapes::StringShape.new(name: 'CatalogIdString')
|
|
123
124
|
CatalogImportStatus = Shapes::StructureShape.new(name: 'CatalogImportStatus')
|
|
124
125
|
CatalogKafkaSource = Shapes::StructureShape.new(name: 'CatalogKafkaSource')
|
|
@@ -587,6 +588,7 @@ module Aws::Glue
|
|
|
587
588
|
GrokClassifier = Shapes::StructureShape.new(name: 'GrokClassifier')
|
|
588
589
|
GrokPattern = Shapes::StringShape.new(name: 'GrokPattern')
|
|
589
590
|
HashString = Shapes::StringShape.new(name: 'HashString')
|
|
591
|
+
HudiTargetCompressionType = Shapes::StringShape.new(name: 'HudiTargetCompressionType')
|
|
590
592
|
IdString = Shapes::StringShape.new(name: 'IdString')
|
|
591
593
|
IdempotentParameterMismatchException = Shapes::StructureShape.new(name: 'IdempotentParameterMismatchException')
|
|
592
594
|
IllegalBlueprintStateException = Shapes::StructureShape.new(name: 'IllegalBlueprintStateException')
|
|
@@ -855,6 +857,7 @@ module Aws::Glue
|
|
|
855
857
|
RunId = Shapes::StringShape.new(name: 'RunId')
|
|
856
858
|
RunStatementRequest = Shapes::StructureShape.new(name: 'RunStatementRequest')
|
|
857
859
|
RunStatementResponse = Shapes::StructureShape.new(name: 'RunStatementResponse')
|
|
860
|
+
S3CatalogHudiSource = Shapes::StructureShape.new(name: 'S3CatalogHudiSource')
|
|
858
861
|
S3CatalogSource = Shapes::StructureShape.new(name: 'S3CatalogSource')
|
|
859
862
|
S3CatalogTarget = Shapes::StructureShape.new(name: 'S3CatalogTarget')
|
|
860
863
|
S3CsvSource = Shapes::StructureShape.new(name: 'S3CsvSource')
|
|
@@ -864,6 +867,9 @@ module Aws::Glue
|
|
|
864
867
|
S3EncryptionList = Shapes::ListShape.new(name: 'S3EncryptionList')
|
|
865
868
|
S3EncryptionMode = Shapes::StringShape.new(name: 'S3EncryptionMode')
|
|
866
869
|
S3GlueParquetTarget = Shapes::StructureShape.new(name: 'S3GlueParquetTarget')
|
|
870
|
+
S3HudiCatalogTarget = Shapes::StructureShape.new(name: 'S3HudiCatalogTarget')
|
|
871
|
+
S3HudiDirectTarget = Shapes::StructureShape.new(name: 'S3HudiDirectTarget')
|
|
872
|
+
S3HudiSource = Shapes::StructureShape.new(name: 'S3HudiSource')
|
|
867
873
|
S3JsonSource = Shapes::StructureShape.new(name: 'S3JsonSource')
|
|
868
874
|
S3ParquetSource = Shapes::StructureShape.new(name: 'S3ParquetSource')
|
|
869
875
|
S3SourceAdditionalOptions = Shapes::StructureShape.new(name: 'S3SourceAdditionalOptions')
|
|
@@ -1427,6 +1433,13 @@ module Aws::Glue
|
|
|
1427
1433
|
CatalogEntry.add_member(:table_name, Shapes::ShapeRef.new(shape: NameString, required: true, location_name: "TableName"))
|
|
1428
1434
|
CatalogEntry.struct_class = Types::CatalogEntry
|
|
1429
1435
|
|
|
1436
|
+
CatalogHudiSource.add_member(:name, Shapes::ShapeRef.new(shape: NodeName, required: true, location_name: "Name"))
|
|
1437
|
+
CatalogHudiSource.add_member(:database, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Database"))
|
|
1438
|
+
CatalogHudiSource.add_member(:table, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Table"))
|
|
1439
|
+
CatalogHudiSource.add_member(:additional_hudi_options, Shapes::ShapeRef.new(shape: AdditionalOptions, location_name: "AdditionalHudiOptions"))
|
|
1440
|
+
CatalogHudiSource.add_member(:output_schemas, Shapes::ShapeRef.new(shape: GlueSchemas, location_name: "OutputSchemas"))
|
|
1441
|
+
CatalogHudiSource.struct_class = Types::CatalogHudiSource
|
|
1442
|
+
|
|
1430
1443
|
CatalogImportStatus.add_member(:import_completed, Shapes::ShapeRef.new(shape: Boolean, location_name: "ImportCompleted"))
|
|
1431
1444
|
CatalogImportStatus.add_member(:import_time, Shapes::ShapeRef.new(shape: Timestamp, location_name: "ImportTime"))
|
|
1432
1445
|
CatalogImportStatus.add_member(:imported_by, Shapes::ShapeRef.new(shape: NameString, location_name: "ImportedBy"))
|
|
@@ -1544,6 +1557,11 @@ module Aws::Glue
|
|
|
1544
1557
|
CodeGenConfigurationNode.add_member(:postgre_sql_catalog_target, Shapes::ShapeRef.new(shape: PostgreSQLCatalogTarget, location_name: "PostgreSQLCatalogTarget"))
|
|
1545
1558
|
CodeGenConfigurationNode.add_member(:dynamic_transform, Shapes::ShapeRef.new(shape: DynamicTransform, location_name: "DynamicTransform"))
|
|
1546
1559
|
CodeGenConfigurationNode.add_member(:evaluate_data_quality, Shapes::ShapeRef.new(shape: EvaluateDataQuality, location_name: "EvaluateDataQuality"))
|
|
1560
|
+
CodeGenConfigurationNode.add_member(:s3_catalog_hudi_source, Shapes::ShapeRef.new(shape: S3CatalogHudiSource, location_name: "S3CatalogHudiSource"))
|
|
1561
|
+
CodeGenConfigurationNode.add_member(:catalog_hudi_source, Shapes::ShapeRef.new(shape: CatalogHudiSource, location_name: "CatalogHudiSource"))
|
|
1562
|
+
CodeGenConfigurationNode.add_member(:s3_hudi_source, Shapes::ShapeRef.new(shape: S3HudiSource, location_name: "S3HudiSource"))
|
|
1563
|
+
CodeGenConfigurationNode.add_member(:s3_hudi_catalog_target, Shapes::ShapeRef.new(shape: S3HudiCatalogTarget, location_name: "S3HudiCatalogTarget"))
|
|
1564
|
+
CodeGenConfigurationNode.add_member(:s3_hudi_direct_target, Shapes::ShapeRef.new(shape: S3HudiDirectTarget, location_name: "S3HudiDirectTarget"))
|
|
1547
1565
|
CodeGenConfigurationNode.struct_class = Types::CodeGenConfigurationNode
|
|
1548
1566
|
|
|
1549
1567
|
CodeGenConfigurationNodes.key = Shapes::ShapeRef.new(shape: NodeId)
|
|
@@ -3569,6 +3587,9 @@ module Aws::Glue
|
|
|
3569
3587
|
KafkaStreamingSourceOptions.add_member(:retry_interval_ms, Shapes::ShapeRef.new(shape: BoxedNonNegativeLong, location_name: "RetryIntervalMs"))
|
|
3570
3588
|
KafkaStreamingSourceOptions.add_member(:max_offsets_per_trigger, Shapes::ShapeRef.new(shape: BoxedNonNegativeLong, location_name: "MaxOffsetsPerTrigger"))
|
|
3571
3589
|
KafkaStreamingSourceOptions.add_member(:min_partitions, Shapes::ShapeRef.new(shape: BoxedNonNegativeInt, location_name: "MinPartitions"))
|
|
3590
|
+
KafkaStreamingSourceOptions.add_member(:include_headers, Shapes::ShapeRef.new(shape: BoxedBoolean, location_name: "IncludeHeaders"))
|
|
3591
|
+
KafkaStreamingSourceOptions.add_member(:add_record_timestamp, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, location_name: "AddRecordTimestamp"))
|
|
3592
|
+
KafkaStreamingSourceOptions.add_member(:emit_consumer_lag_metrics, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, location_name: "EmitConsumerLagMetrics"))
|
|
3572
3593
|
KafkaStreamingSourceOptions.struct_class = Types::KafkaStreamingSourceOptions
|
|
3573
3594
|
|
|
3574
3595
|
KeyList.member = Shapes::ShapeRef.new(shape: NameString)
|
|
@@ -3597,6 +3618,8 @@ module Aws::Glue
|
|
|
3597
3618
|
KinesisStreamingSourceOptions.add_member(:stream_arn, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, location_name: "StreamArn"))
|
|
3598
3619
|
KinesisStreamingSourceOptions.add_member(:role_arn, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, location_name: "RoleArn"))
|
|
3599
3620
|
KinesisStreamingSourceOptions.add_member(:role_session_name, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, location_name: "RoleSessionName"))
|
|
3621
|
+
KinesisStreamingSourceOptions.add_member(:add_record_timestamp, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, location_name: "AddRecordTimestamp"))
|
|
3622
|
+
KinesisStreamingSourceOptions.add_member(:emit_consumer_lag_metrics, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, location_name: "EmitConsumerLagMetrics"))
|
|
3600
3623
|
KinesisStreamingSourceOptions.struct_class = Types::KinesisStreamingSourceOptions
|
|
3601
3624
|
|
|
3602
3625
|
LabelingSetGenerationTaskRunProperties.add_member(:output_s3_path, Shapes::ShapeRef.new(shape: UriString, location_name: "OutputS3Path"))
|
|
@@ -4243,6 +4266,13 @@ module Aws::Glue
|
|
|
4243
4266
|
RunStatementResponse.add_member(:id, Shapes::ShapeRef.new(shape: IntegerValue, location_name: "Id"))
|
|
4244
4267
|
RunStatementResponse.struct_class = Types::RunStatementResponse
|
|
4245
4268
|
|
|
4269
|
+
S3CatalogHudiSource.add_member(:name, Shapes::ShapeRef.new(shape: NodeName, required: true, location_name: "Name"))
|
|
4270
|
+
S3CatalogHudiSource.add_member(:database, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Database"))
|
|
4271
|
+
S3CatalogHudiSource.add_member(:table, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Table"))
|
|
4272
|
+
S3CatalogHudiSource.add_member(:additional_hudi_options, Shapes::ShapeRef.new(shape: AdditionalOptions, location_name: "AdditionalHudiOptions"))
|
|
4273
|
+
S3CatalogHudiSource.add_member(:output_schemas, Shapes::ShapeRef.new(shape: GlueSchemas, location_name: "OutputSchemas"))
|
|
4274
|
+
S3CatalogHudiSource.struct_class = Types::S3CatalogHudiSource
|
|
4275
|
+
|
|
4246
4276
|
S3CatalogSource.add_member(:name, Shapes::ShapeRef.new(shape: NodeName, required: true, location_name: "Name"))
|
|
4247
4277
|
S3CatalogSource.add_member(:database, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Database"))
|
|
4248
4278
|
S3CatalogSource.add_member(:table, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Table"))
|
|
@@ -4308,6 +4338,32 @@ module Aws::Glue
|
|
|
4308
4338
|
S3GlueParquetTarget.add_member(:schema_change_policy, Shapes::ShapeRef.new(shape: DirectSchemaChangePolicy, location_name: "SchemaChangePolicy"))
|
|
4309
4339
|
S3GlueParquetTarget.struct_class = Types::S3GlueParquetTarget
|
|
4310
4340
|
|
|
4341
|
+
S3HudiCatalogTarget.add_member(:name, Shapes::ShapeRef.new(shape: NodeName, required: true, location_name: "Name"))
|
|
4342
|
+
S3HudiCatalogTarget.add_member(:inputs, Shapes::ShapeRef.new(shape: OneInput, required: true, location_name: "Inputs"))
|
|
4343
|
+
S3HudiCatalogTarget.add_member(:partition_keys, Shapes::ShapeRef.new(shape: GlueStudioPathList, location_name: "PartitionKeys"))
|
|
4344
|
+
S3HudiCatalogTarget.add_member(:table, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Table"))
|
|
4345
|
+
S3HudiCatalogTarget.add_member(:database, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Database"))
|
|
4346
|
+
S3HudiCatalogTarget.add_member(:additional_options, Shapes::ShapeRef.new(shape: AdditionalOptions, required: true, location_name: "AdditionalOptions"))
|
|
4347
|
+
S3HudiCatalogTarget.add_member(:schema_change_policy, Shapes::ShapeRef.new(shape: CatalogSchemaChangePolicy, location_name: "SchemaChangePolicy"))
|
|
4348
|
+
S3HudiCatalogTarget.struct_class = Types::S3HudiCatalogTarget
|
|
4349
|
+
|
|
4350
|
+
S3HudiDirectTarget.add_member(:name, Shapes::ShapeRef.new(shape: NodeName, required: true, location_name: "Name"))
|
|
4351
|
+
S3HudiDirectTarget.add_member(:inputs, Shapes::ShapeRef.new(shape: OneInput, required: true, location_name: "Inputs"))
|
|
4352
|
+
S3HudiDirectTarget.add_member(:path, Shapes::ShapeRef.new(shape: EnclosedInStringProperty, required: true, location_name: "Path"))
|
|
4353
|
+
S3HudiDirectTarget.add_member(:compression, Shapes::ShapeRef.new(shape: HudiTargetCompressionType, required: true, location_name: "Compression"))
|
|
4354
|
+
S3HudiDirectTarget.add_member(:partition_keys, Shapes::ShapeRef.new(shape: GlueStudioPathList, location_name: "PartitionKeys"))
|
|
4355
|
+
S3HudiDirectTarget.add_member(:format, Shapes::ShapeRef.new(shape: TargetFormat, required: true, location_name: "Format"))
|
|
4356
|
+
S3HudiDirectTarget.add_member(:additional_options, Shapes::ShapeRef.new(shape: AdditionalOptions, required: true, location_name: "AdditionalOptions"))
|
|
4357
|
+
S3HudiDirectTarget.add_member(:schema_change_policy, Shapes::ShapeRef.new(shape: DirectSchemaChangePolicy, location_name: "SchemaChangePolicy"))
|
|
4358
|
+
S3HudiDirectTarget.struct_class = Types::S3HudiDirectTarget
|
|
4359
|
+
|
|
4360
|
+
S3HudiSource.add_member(:name, Shapes::ShapeRef.new(shape: NodeName, required: true, location_name: "Name"))
|
|
4361
|
+
S3HudiSource.add_member(:paths, Shapes::ShapeRef.new(shape: EnclosedInStringProperties, required: true, location_name: "Paths"))
|
|
4362
|
+
S3HudiSource.add_member(:additional_hudi_options, Shapes::ShapeRef.new(shape: AdditionalOptions, location_name: "AdditionalHudiOptions"))
|
|
4363
|
+
S3HudiSource.add_member(:additional_options, Shapes::ShapeRef.new(shape: S3DirectSourceAdditionalOptions, location_name: "AdditionalOptions"))
|
|
4364
|
+
S3HudiSource.add_member(:output_schemas, Shapes::ShapeRef.new(shape: GlueSchemas, location_name: "OutputSchemas"))
|
|
4365
|
+
S3HudiSource.struct_class = Types::S3HudiSource
|
|
4366
|
+
|
|
4311
4367
|
S3JsonSource.add_member(:name, Shapes::ShapeRef.new(shape: NodeName, required: true, location_name: "Name"))
|
|
4312
4368
|
S3JsonSource.add_member(:paths, Shapes::ShapeRef.new(shape: EnclosedInStringProperties, required: true, location_name: "Paths"))
|
|
4313
4369
|
S3JsonSource.add_member(:compression_type, Shapes::ShapeRef.new(shape: CompressionType, location_name: "CompressionType"))
|
data/lib/aws-sdk-glue/types.rb
CHANGED
|
@@ -1335,6 +1335,41 @@ module Aws::Glue
|
|
|
1335
1335
|
include Aws::Structure
|
|
1336
1336
|
end
|
|
1337
1337
|
|
|
1338
|
+
# Specifies a Hudi data source that is registered in the Glue Data
|
|
1339
|
+
# Catalog.
|
|
1340
|
+
#
|
|
1341
|
+
# @!attribute [rw] name
|
|
1342
|
+
# The name of the Hudi data source.
|
|
1343
|
+
# @return [String]
|
|
1344
|
+
#
|
|
1345
|
+
# @!attribute [rw] database
|
|
1346
|
+
# The name of the database to read from.
|
|
1347
|
+
# @return [String]
|
|
1348
|
+
#
|
|
1349
|
+
# @!attribute [rw] table
|
|
1350
|
+
# The name of the table in the database to read from.
|
|
1351
|
+
# @return [String]
|
|
1352
|
+
#
|
|
1353
|
+
# @!attribute [rw] additional_hudi_options
|
|
1354
|
+
# Specifies additional connection options.
|
|
1355
|
+
# @return [Hash<String,String>]
|
|
1356
|
+
#
|
|
1357
|
+
# @!attribute [rw] output_schemas
|
|
1358
|
+
# Specifies the data schema for the Hudi source.
|
|
1359
|
+
# @return [Array<Types::GlueSchema>]
|
|
1360
|
+
#
|
|
1361
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CatalogHudiSource AWS API Documentation
|
|
1362
|
+
#
|
|
1363
|
+
class CatalogHudiSource < Struct.new(
|
|
1364
|
+
:name,
|
|
1365
|
+
:database,
|
|
1366
|
+
:table,
|
|
1367
|
+
:additional_hudi_options,
|
|
1368
|
+
:output_schemas)
|
|
1369
|
+
SENSITIVE = []
|
|
1370
|
+
include Aws::Structure
|
|
1371
|
+
end
|
|
1372
|
+
|
|
1338
1373
|
# A structure containing migration status information.
|
|
1339
1374
|
#
|
|
1340
1375
|
# @!attribute [rw] import_completed
|
|
@@ -1667,12 +1702,11 @@ module Aws::Glue
|
|
|
1667
1702
|
# @return [Types::S3ParquetSource]
|
|
1668
1703
|
#
|
|
1669
1704
|
# @!attribute [rw] relational_catalog_source
|
|
1670
|
-
# Specifies a
|
|
1671
|
-
# Catalog.
|
|
1705
|
+
# Specifies a relational catalog data store in the Glue Data Catalog.
|
|
1672
1706
|
# @return [Types::RelationalCatalogSource]
|
|
1673
1707
|
#
|
|
1674
1708
|
# @!attribute [rw] dynamo_db_catalog_source
|
|
1675
|
-
# Specifies a
|
|
1709
|
+
# Specifies a DynamoDBC Catalog data store in the Glue Data Catalog.
|
|
1676
1710
|
# @return [Types::DynamoDBCatalogSource]
|
|
1677
1711
|
#
|
|
1678
1712
|
# @!attribute [rw] jdbc_connector_target
|
|
@@ -1873,6 +1907,29 @@ module Aws::Glue
|
|
|
1873
1907
|
# Specifies your data quality evaluation criteria.
|
|
1874
1908
|
# @return [Types::EvaluateDataQuality]
|
|
1875
1909
|
#
|
|
1910
|
+
# @!attribute [rw] s3_catalog_hudi_source
|
|
1911
|
+
# Specifies a Hudi data source that is registered in the Glue Data
|
|
1912
|
+
# Catalog. The Hudi data source must be stored in Amazon S3.
|
|
1913
|
+
# @return [Types::S3CatalogHudiSource]
|
|
1914
|
+
#
|
|
1915
|
+
# @!attribute [rw] catalog_hudi_source
|
|
1916
|
+
# Specifies a Hudi data source that is registered in the Glue Data
|
|
1917
|
+
# Catalog.
|
|
1918
|
+
# @return [Types::CatalogHudiSource]
|
|
1919
|
+
#
|
|
1920
|
+
# @!attribute [rw] s3_hudi_source
|
|
1921
|
+
# Specifies a Hudi data source stored in Amazon S3.
|
|
1922
|
+
# @return [Types::S3HudiSource]
|
|
1923
|
+
#
|
|
1924
|
+
# @!attribute [rw] s3_hudi_catalog_target
|
|
1925
|
+
# Specifies a target that writes to a Hudi data source in the Glue
|
|
1926
|
+
# Data Catalog.
|
|
1927
|
+
# @return [Types::S3HudiCatalogTarget]
|
|
1928
|
+
#
|
|
1929
|
+
# @!attribute [rw] s3_hudi_direct_target
|
|
1930
|
+
# Specifies a target that writes to a Hudi data source in Amazon S3.
|
|
1931
|
+
# @return [Types::S3HudiDirectTarget]
|
|
1932
|
+
#
|
|
1876
1933
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CodeGenConfigurationNode AWS API Documentation
|
|
1877
1934
|
#
|
|
1878
1935
|
class CodeGenConfigurationNode < Struct.new(
|
|
@@ -1927,7 +1984,12 @@ module Aws::Glue
|
|
|
1927
1984
|
:oracle_sql_catalog_target,
|
|
1928
1985
|
:postgre_sql_catalog_target,
|
|
1929
1986
|
:dynamic_transform,
|
|
1930
|
-
:evaluate_data_quality
|
|
1987
|
+
:evaluate_data_quality,
|
|
1988
|
+
:s3_catalog_hudi_source,
|
|
1989
|
+
:catalog_hudi_source,
|
|
1990
|
+
:s3_hudi_source,
|
|
1991
|
+
:s3_hudi_catalog_target,
|
|
1992
|
+
:s3_hudi_direct_target)
|
|
1931
1993
|
SENSITIVE = []
|
|
1932
1994
|
include Aws::Structure
|
|
1933
1995
|
end
|
|
@@ -12308,6 +12370,31 @@ module Aws::Glue
|
|
|
12308
12370
|
# partitions is equal to the number of Kafka partitions.
|
|
12309
12371
|
# @return [Integer]
|
|
12310
12372
|
#
|
|
12373
|
+
# @!attribute [rw] include_headers
|
|
12374
|
+
# Whether to include the Kafka headers. When the option is set to
|
|
12375
|
+
# "true", the data output will contain an additional column named
|
|
12376
|
+
# "glue\_streaming\_kafka\_headers" with type `Array[Struct(key:
|
|
12377
|
+
# String, value: String)]`. The default value is "false". This
|
|
12378
|
+
# option is available in Glue version 3.0 or later only.
|
|
12379
|
+
# @return [Boolean]
|
|
12380
|
+
#
|
|
12381
|
+
# @!attribute [rw] add_record_timestamp
|
|
12382
|
+
# When this option is set to 'true', the data output will contain an
|
|
12383
|
+
# additional column named "\_\_src\_timestamp" that indicates the
|
|
12384
|
+
# time when the corresponding record received by the topic. The
|
|
12385
|
+
# default value is 'false'. This option is supported in Glue version
|
|
12386
|
+
# 4.0 or later.
|
|
12387
|
+
# @return [String]
|
|
12388
|
+
#
|
|
12389
|
+
# @!attribute [rw] emit_consumer_lag_metrics
|
|
12390
|
+
# When this option is set to 'true', for each batch, it will emit
|
|
12391
|
+
# the metrics for the duration between the oldest record received by
|
|
12392
|
+
# the topic and the time it arrives in Glue to CloudWatch. The
|
|
12393
|
+
# metric's name is "glue.driver.streaming.maxConsumerLagInMs". The
|
|
12394
|
+
# default value is 'false'. This option is supported in Glue version
|
|
12395
|
+
# 4.0 or later.
|
|
12396
|
+
# @return [String]
|
|
12397
|
+
#
|
|
12311
12398
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/KafkaStreamingSourceOptions AWS API Documentation
|
|
12312
12399
|
#
|
|
12313
12400
|
class KafkaStreamingSourceOptions < Struct.new(
|
|
@@ -12325,7 +12412,10 @@ module Aws::Glue
|
|
|
12325
12412
|
:num_retries,
|
|
12326
12413
|
:retry_interval_ms,
|
|
12327
12414
|
:max_offsets_per_trigger,
|
|
12328
|
-
:min_partitions
|
|
12415
|
+
:min_partitions,
|
|
12416
|
+
:include_headers,
|
|
12417
|
+
:add_record_timestamp,
|
|
12418
|
+
:emit_consumer_lag_metrics)
|
|
12329
12419
|
SENSITIVE = []
|
|
12330
12420
|
include Aws::Structure
|
|
12331
12421
|
end
|
|
@@ -12446,6 +12536,23 @@ module Aws::Glue
|
|
|
12446
12536
|
# account. Used in conjunction with `"awsSTSRoleARN"`.
|
|
12447
12537
|
# @return [String]
|
|
12448
12538
|
#
|
|
12539
|
+
# @!attribute [rw] add_record_timestamp
|
|
12540
|
+
# When this option is set to 'true', the data output will contain an
|
|
12541
|
+
# additional column named "\_\_src\_timestamp" that indicates the
|
|
12542
|
+
# time when the corresponding record received by the stream. The
|
|
12543
|
+
# default value is 'false'. This option is supported in Glue version
|
|
12544
|
+
# 4.0 or later.
|
|
12545
|
+
# @return [String]
|
|
12546
|
+
#
|
|
12547
|
+
# @!attribute [rw] emit_consumer_lag_metrics
|
|
12548
|
+
# When this option is set to 'true', for each batch, it will emit
|
|
12549
|
+
# the metrics for the duration between the oldest record received by
|
|
12550
|
+
# the stream and the time it arrives in Glue to CloudWatch. The
|
|
12551
|
+
# metric's name is "glue.driver.streaming.maxConsumerLagInMs". The
|
|
12552
|
+
# default value is 'false'. This option is supported in Glue version
|
|
12553
|
+
# 4.0 or later.
|
|
12554
|
+
# @return [String]
|
|
12555
|
+
#
|
|
12449
12556
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/KinesisStreamingSourceOptions AWS API Documentation
|
|
12450
12557
|
#
|
|
12451
12558
|
class KinesisStreamingSourceOptions < Struct.new(
|
|
@@ -12466,7 +12573,9 @@ module Aws::Glue
|
|
|
12466
12573
|
:avoid_empty_batches,
|
|
12467
12574
|
:stream_arn,
|
|
12468
12575
|
:role_arn,
|
|
12469
|
-
:role_session_name
|
|
12576
|
+
:role_session_name,
|
|
12577
|
+
:add_record_timestamp,
|
|
12578
|
+
:emit_consumer_lag_metrics)
|
|
12470
12579
|
SENSITIVE = []
|
|
12471
12580
|
include Aws::Structure
|
|
12472
12581
|
end
|
|
@@ -15364,6 +15473,41 @@ module Aws::Glue
|
|
|
15364
15473
|
include Aws::Structure
|
|
15365
15474
|
end
|
|
15366
15475
|
|
|
15476
|
+
# Specifies a Hudi data source that is registered in the Glue Data
|
|
15477
|
+
# Catalog. The Hudi data source must be stored in Amazon S3.
|
|
15478
|
+
#
|
|
15479
|
+
# @!attribute [rw] name
|
|
15480
|
+
# The name of the Hudi data source.
|
|
15481
|
+
# @return [String]
|
|
15482
|
+
#
|
|
15483
|
+
# @!attribute [rw] database
|
|
15484
|
+
# The name of the database to read from.
|
|
15485
|
+
# @return [String]
|
|
15486
|
+
#
|
|
15487
|
+
# @!attribute [rw] table
|
|
15488
|
+
# The name of the table in the database to read from.
|
|
15489
|
+
# @return [String]
|
|
15490
|
+
#
|
|
15491
|
+
# @!attribute [rw] additional_hudi_options
|
|
15492
|
+
# Specifies additional connection options.
|
|
15493
|
+
# @return [Hash<String,String>]
|
|
15494
|
+
#
|
|
15495
|
+
# @!attribute [rw] output_schemas
|
|
15496
|
+
# Specifies the data schema for the Hudi source.
|
|
15497
|
+
# @return [Array<Types::GlueSchema>]
|
|
15498
|
+
#
|
|
15499
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/S3CatalogHudiSource AWS API Documentation
|
|
15500
|
+
#
|
|
15501
|
+
class S3CatalogHudiSource < Struct.new(
|
|
15502
|
+
:name,
|
|
15503
|
+
:database,
|
|
15504
|
+
:table,
|
|
15505
|
+
:additional_hudi_options,
|
|
15506
|
+
:output_schemas)
|
|
15507
|
+
SENSITIVE = []
|
|
15508
|
+
include Aws::Structure
|
|
15509
|
+
end
|
|
15510
|
+
|
|
15367
15511
|
# Specifies an Amazon S3 data store in the Glue Data Catalog.
|
|
15368
15512
|
#
|
|
15369
15513
|
# @!attribute [rw] name
|
|
@@ -15717,6 +15861,136 @@ module Aws::Glue
|
|
|
15717
15861
|
include Aws::Structure
|
|
15718
15862
|
end
|
|
15719
15863
|
|
|
15864
|
+
# Specifies a target that writes to a Hudi data source in the Glue Data
|
|
15865
|
+
# Catalog.
|
|
15866
|
+
#
|
|
15867
|
+
# @!attribute [rw] name
|
|
15868
|
+
# The name of the data target.
|
|
15869
|
+
# @return [String]
|
|
15870
|
+
#
|
|
15871
|
+
# @!attribute [rw] inputs
|
|
15872
|
+
# The nodes that are inputs to the data target.
|
|
15873
|
+
# @return [Array<String>]
|
|
15874
|
+
#
|
|
15875
|
+
# @!attribute [rw] partition_keys
|
|
15876
|
+
# Specifies native partitioning using a sequence of keys.
|
|
15877
|
+
# @return [Array<Array<String>>]
|
|
15878
|
+
#
|
|
15879
|
+
# @!attribute [rw] table
|
|
15880
|
+
# The name of the table in the database to write to.
|
|
15881
|
+
# @return [String]
|
|
15882
|
+
#
|
|
15883
|
+
# @!attribute [rw] database
|
|
15884
|
+
# The name of the database to write to.
|
|
15885
|
+
# @return [String]
|
|
15886
|
+
#
|
|
15887
|
+
# @!attribute [rw] additional_options
|
|
15888
|
+
# Specifies additional connection options for the connector.
|
|
15889
|
+
# @return [Hash<String,String>]
|
|
15890
|
+
#
|
|
15891
|
+
# @!attribute [rw] schema_change_policy
|
|
15892
|
+
# A policy that specifies update behavior for the crawler.
|
|
15893
|
+
# @return [Types::CatalogSchemaChangePolicy]
|
|
15894
|
+
#
|
|
15895
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/S3HudiCatalogTarget AWS API Documentation
|
|
15896
|
+
#
|
|
15897
|
+
class S3HudiCatalogTarget < Struct.new(
|
|
15898
|
+
:name,
|
|
15899
|
+
:inputs,
|
|
15900
|
+
:partition_keys,
|
|
15901
|
+
:table,
|
|
15902
|
+
:database,
|
|
15903
|
+
:additional_options,
|
|
15904
|
+
:schema_change_policy)
|
|
15905
|
+
SENSITIVE = []
|
|
15906
|
+
include Aws::Structure
|
|
15907
|
+
end
|
|
15908
|
+
|
|
15909
|
+
# Specifies a target that writes to a Hudi data source in Amazon S3.
|
|
15910
|
+
#
|
|
15911
|
+
# @!attribute [rw] name
|
|
15912
|
+
# The name of the data target.
|
|
15913
|
+
# @return [String]
|
|
15914
|
+
#
|
|
15915
|
+
# @!attribute [rw] inputs
|
|
15916
|
+
# The nodes that are inputs to the data target.
|
|
15917
|
+
# @return [Array<String>]
|
|
15918
|
+
#
|
|
15919
|
+
# @!attribute [rw] path
|
|
15920
|
+
# The Amazon S3 path of your Hudi data source to write to.
|
|
15921
|
+
# @return [String]
|
|
15922
|
+
#
|
|
15923
|
+
# @!attribute [rw] compression
|
|
15924
|
+
# Specifies how the data is compressed. This is generally not
|
|
15925
|
+
# necessary if the data has a standard file extension. Possible values
|
|
15926
|
+
# are `"gzip"` and `"bzip"`).
|
|
15927
|
+
# @return [String]
|
|
15928
|
+
#
|
|
15929
|
+
# @!attribute [rw] partition_keys
|
|
15930
|
+
# Specifies native partitioning using a sequence of keys.
|
|
15931
|
+
# @return [Array<Array<String>>]
|
|
15932
|
+
#
|
|
15933
|
+
# @!attribute [rw] format
|
|
15934
|
+
# Specifies the data output format for the target.
|
|
15935
|
+
# @return [String]
|
|
15936
|
+
#
|
|
15937
|
+
# @!attribute [rw] additional_options
|
|
15938
|
+
# Specifies additional connection options for the connector.
|
|
15939
|
+
# @return [Hash<String,String>]
|
|
15940
|
+
#
|
|
15941
|
+
# @!attribute [rw] schema_change_policy
|
|
15942
|
+
# A policy that specifies update behavior for the crawler.
|
|
15943
|
+
# @return [Types::DirectSchemaChangePolicy]
|
|
15944
|
+
#
|
|
15945
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/S3HudiDirectTarget AWS API Documentation
|
|
15946
|
+
#
|
|
15947
|
+
class S3HudiDirectTarget < Struct.new(
|
|
15948
|
+
:name,
|
|
15949
|
+
:inputs,
|
|
15950
|
+
:path,
|
|
15951
|
+
:compression,
|
|
15952
|
+
:partition_keys,
|
|
15953
|
+
:format,
|
|
15954
|
+
:additional_options,
|
|
15955
|
+
:schema_change_policy)
|
|
15956
|
+
SENSITIVE = []
|
|
15957
|
+
include Aws::Structure
|
|
15958
|
+
end
|
|
15959
|
+
|
|
15960
|
+
# Specifies a Hudi data source stored in Amazon S3.
|
|
15961
|
+
#
|
|
15962
|
+
# @!attribute [rw] name
|
|
15963
|
+
# The name of the Hudi source.
|
|
15964
|
+
# @return [String]
|
|
15965
|
+
#
|
|
15966
|
+
# @!attribute [rw] paths
|
|
15967
|
+
# A list of the Amazon S3 paths to read from.
|
|
15968
|
+
# @return [Array<String>]
|
|
15969
|
+
#
|
|
15970
|
+
# @!attribute [rw] additional_hudi_options
|
|
15971
|
+
# Specifies additional connection options.
|
|
15972
|
+
# @return [Hash<String,String>]
|
|
15973
|
+
#
|
|
15974
|
+
# @!attribute [rw] additional_options
|
|
15975
|
+
# Specifies additional options for the connector.
|
|
15976
|
+
# @return [Types::S3DirectSourceAdditionalOptions]
|
|
15977
|
+
#
|
|
15978
|
+
# @!attribute [rw] output_schemas
|
|
15979
|
+
# Specifies the data schema for the Hudi source.
|
|
15980
|
+
# @return [Array<Types::GlueSchema>]
|
|
15981
|
+
#
|
|
15982
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/S3HudiSource AWS API Documentation
|
|
15983
|
+
#
|
|
15984
|
+
class S3HudiSource < Struct.new(
|
|
15985
|
+
:name,
|
|
15986
|
+
:paths,
|
|
15987
|
+
:additional_hudi_options,
|
|
15988
|
+
:additional_options,
|
|
15989
|
+
:output_schemas)
|
|
15990
|
+
SENSITIVE = []
|
|
15991
|
+
include Aws::Structure
|
|
15992
|
+
end
|
|
15993
|
+
|
|
15720
15994
|
# Specifies a JSON data store stored in Amazon S3.
|
|
15721
15995
|
#
|
|
15722
15996
|
# @!attribute [rw] name
|
data/lib/aws-sdk-glue.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: aws-sdk-glue
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.130.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Amazon Web Services
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2023-
|
|
11
|
+
date: 2023-02-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: aws-sdk-core
|