@aws-sdk/client-glue 3.857.0 → 3.859.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-cjs/index.js +54 -18
- package/dist-es/models/models_0.js +8 -5
- package/dist-es/models/models_1.js +5 -0
- package/dist-es/models/models_2.js +0 -9
- package/dist-es/models/models_3.js +9 -0
- package/dist-es/protocols/Aws_json1_1.js +26 -0
- package/dist-types/commands/BatchGetJobsCommand.d.ts +181 -67
- package/dist-types/commands/CreateJobCommand.d.ts +181 -67
- package/dist-types/commands/GetClassifierCommand.d.ts +2 -1
- package/dist-types/commands/GetClassifiersCommand.d.ts +1 -1
- package/dist-types/commands/GetColumnStatisticsForPartitionCommand.d.ts +1 -2
- package/dist-types/commands/GetJobCommand.d.ts +181 -67
- package/dist-types/commands/GetJobsCommand.d.ts +181 -67
- package/dist-types/commands/ModifyIntegrationCommand.d.ts +2 -1
- package/dist-types/commands/PutDataCatalogEncryptionSettingsCommand.d.ts +1 -1
- package/dist-types/commands/PutDataQualityProfileAnnotationCommand.d.ts +1 -1
- package/dist-types/commands/PutResourcePolicyCommand.d.ts +1 -1
- package/dist-types/commands/UpdateJobCommand.d.ts +181 -67
- package/dist-types/models/models_0.d.ts +445 -448
- package/dist-types/models/models_1.d.ts +430 -231
- package/dist-types/models/models_2.d.ts +232 -232
- package/dist-types/models/models_3.d.ts +298 -27
- package/dist-types/ts3.4/commands/GetClassifierCommand.d.ts +2 -4
- package/dist-types/ts3.4/commands/GetClassifiersCommand.d.ts +1 -1
- package/dist-types/ts3.4/commands/GetColumnStatisticsForPartitionCommand.d.ts +4 -2
- package/dist-types/ts3.4/commands/ModifyIntegrationCommand.d.ts +2 -4
- package/dist-types/ts3.4/commands/PutDataCatalogEncryptionSettingsCommand.d.ts +1 -1
- package/dist-types/ts3.4/commands/PutDataQualityProfileAnnotationCommand.d.ts +1 -1
- package/dist-types/ts3.4/commands/PutResourcePolicyCommand.d.ts +1 -1
- package/dist-types/ts3.4/models/models_0.d.ts +94 -75
- package/dist-types/ts3.4/models/models_1.d.ts +82 -54
- package/dist-types/ts3.4/models/models_2.d.ts +56 -59
- package/dist-types/ts3.4/models/models_3.d.ts +69 -4
- package/package.json +5 -5
|
@@ -81,6 +81,7 @@ export interface Action {
|
|
|
81
81
|
*/
|
|
82
82
|
export declare const AdditionalOptionKeys: {
|
|
83
83
|
readonly CacheOption: "performanceTuning.caching";
|
|
84
|
+
readonly CompositeOption: "compositeRuleEvaluation.method";
|
|
84
85
|
readonly ObservationsOption: "observations.scope";
|
|
85
86
|
};
|
|
86
87
|
/**
|
|
@@ -501,6 +502,11 @@ export interface GlueStudioSchemaColumn {
|
|
|
501
502
|
* @public
|
|
502
503
|
*/
|
|
503
504
|
Type?: string | undefined;
|
|
505
|
+
/**
|
|
506
|
+
* <p>The data type of the column as defined in Glue Studio.</p>
|
|
507
|
+
* @public
|
|
508
|
+
*/
|
|
509
|
+
GlueStudioType?: string | undefined;
|
|
504
510
|
}
|
|
505
511
|
/**
|
|
506
512
|
* <p>Specifies a user-defined schema when a schema cannot be determined by Glue.</p>
|
|
@@ -894,6 +900,23 @@ export interface AuthenticationConfigurationInput {
|
|
|
894
900
|
*/
|
|
895
901
|
CustomAuthenticationCredentials?: Record<string, string> | undefined;
|
|
896
902
|
}
|
|
903
|
+
/**
|
|
904
|
+
* <p>Specifies configuration options for automatic data quality evaluation in Glue jobs. This structure enables automated data quality
|
|
905
|
+
* checks and monitoring during ETL operations, helping to ensure data integrity and reliability without manual intervention.</p>
|
|
906
|
+
* @public
|
|
907
|
+
*/
|
|
908
|
+
export interface AutoDataQuality {
|
|
909
|
+
/**
|
|
910
|
+
* <p>Specifies whether automatic data quality evaluation is enabled. When set to <code>true</code>, data quality checks are performed automatically.</p>
|
|
911
|
+
* @public
|
|
912
|
+
*/
|
|
913
|
+
IsEnabled?: boolean | undefined;
|
|
914
|
+
/**
|
|
915
|
+
* <p>The evaluation context for the automatic data quality checks. This defines the scope and parameters for the data quality evaluation.</p>
|
|
916
|
+
* @public
|
|
917
|
+
*/
|
|
918
|
+
EvaluationContext?: string | undefined;
|
|
919
|
+
}
|
|
897
920
|
/**
|
|
898
921
|
* <p>A column in a <code>Table</code>.</p>
|
|
899
922
|
* @public
|
|
@@ -3007,6 +3030,37 @@ export interface CatalogHudiSource {
|
|
|
3007
3030
|
*/
|
|
3008
3031
|
OutputSchemas?: GlueSchema[] | undefined;
|
|
3009
3032
|
}
|
|
3033
|
+
/**
|
|
3034
|
+
* <p>Specifies an Apache Iceberg data source that is registered in the Glue Data Catalog.</p>
|
|
3035
|
+
* @public
|
|
3036
|
+
*/
|
|
3037
|
+
export interface CatalogIcebergSource {
|
|
3038
|
+
/**
|
|
3039
|
+
* <p>The name of the Iceberg data source.</p>
|
|
3040
|
+
* @public
|
|
3041
|
+
*/
|
|
3042
|
+
Name: string | undefined;
|
|
3043
|
+
/**
|
|
3044
|
+
* <p>The name of the database to read from.</p>
|
|
3045
|
+
* @public
|
|
3046
|
+
*/
|
|
3047
|
+
Database: string | undefined;
|
|
3048
|
+
/**
|
|
3049
|
+
* <p>The name of the table in the database to read from.</p>
|
|
3050
|
+
* @public
|
|
3051
|
+
*/
|
|
3052
|
+
Table: string | undefined;
|
|
3053
|
+
/**
|
|
3054
|
+
* <p>Specifies additional connection options for the Iceberg data source.</p>
|
|
3055
|
+
* @public
|
|
3056
|
+
*/
|
|
3057
|
+
AdditionalIcebergOptions?: Record<string, string> | undefined;
|
|
3058
|
+
/**
|
|
3059
|
+
* <p>Specifies the data schema for the Iceberg source.</p>
|
|
3060
|
+
* @public
|
|
3061
|
+
*/
|
|
3062
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
3063
|
+
}
|
|
3010
3064
|
/**
|
|
3011
3065
|
* <p>Specifies options related to data preview for viewing a sample of your data.</p>
|
|
3012
3066
|
* @public
|
|
@@ -3293,6 +3347,12 @@ export interface KinesisStreamingSourceOptions {
|
|
|
3293
3347
|
* @public
|
|
3294
3348
|
*/
|
|
3295
3349
|
StartingTimestamp?: Date | undefined;
|
|
3350
|
+
/**
|
|
3351
|
+
* <p>The Amazon Resource Name (ARN) of the Kinesis Data Streams enhanced fan-out consumer. When specified, enables enhanced fan-out for
|
|
3352
|
+
* dedicated throughput and lower latency data consumption.</p>
|
|
3353
|
+
* @public
|
|
3354
|
+
*/
|
|
3355
|
+
FanoutConsumerARN?: string | undefined;
|
|
3296
3356
|
}
|
|
3297
3357
|
/**
|
|
3298
3358
|
* <p>Specifies a Kinesis data source in the Glue Data Catalog.</p>
|
|
@@ -3355,6 +3415,18 @@ export interface CatalogSource {
|
|
|
3355
3415
|
* @public
|
|
3356
3416
|
*/
|
|
3357
3417
|
Table: string | undefined;
|
|
3418
|
+
/**
|
|
3419
|
+
* <p>
|
|
3420
|
+
* Partitions satisfying this predicate are deleted. Files within the retention period in these partitions are not deleted.
|
|
3421
|
+
* </p>
|
|
3422
|
+
* @public
|
|
3423
|
+
*/
|
|
3424
|
+
PartitionPredicate?: string | undefined;
|
|
3425
|
+
/**
|
|
3426
|
+
* <p>Specifies the data schema for the catalog source.</p>
|
|
3427
|
+
* @public
|
|
3428
|
+
*/
|
|
3429
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
3358
3430
|
}
|
|
3359
3431
|
/**
|
|
3360
3432
|
* <p>Specifies a target that uses a Glue Data Catalog table.</p>
|
|
@@ -3602,6 +3674,11 @@ export interface DirectJDBCSource {
|
|
|
3602
3674
|
* @public
|
|
3603
3675
|
*/
|
|
3604
3676
|
RedshiftTmpDir?: string | undefined;
|
|
3677
|
+
/**
|
|
3678
|
+
* <p>Specifies the data schema for the direct JDBC source.</p>
|
|
3679
|
+
* @public
|
|
3680
|
+
*/
|
|
3681
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
3605
3682
|
}
|
|
3606
3683
|
/**
|
|
3607
3684
|
* <p>Specifies an Apache Kafka data store.</p>
|
|
@@ -3891,6 +3968,22 @@ export interface DynamicTransform {
|
|
|
3891
3968
|
*/
|
|
3892
3969
|
OutputSchemas?: GlueSchema[] | undefined;
|
|
3893
3970
|
}
|
|
3971
|
+
/**
|
|
3972
|
+
* <p>Specifies additional options for DynamoDB ELT catalog operations.</p>
|
|
3973
|
+
* @public
|
|
3974
|
+
*/
|
|
3975
|
+
export interface DDBELTCatalogAdditionalOptions {
|
|
3976
|
+
/**
|
|
3977
|
+
* <p>Specifies the DynamoDB export configuration for the ELT operation.</p>
|
|
3978
|
+
* @public
|
|
3979
|
+
*/
|
|
3980
|
+
DynamodbExport?: string | undefined;
|
|
3981
|
+
/**
|
|
3982
|
+
* <p>Specifies whether to unnest DynamoDB JSON format. When set to <code>true</code>, nested JSON structures in DynamoDB items are flattened.</p>
|
|
3983
|
+
* @public
|
|
3984
|
+
*/
|
|
3985
|
+
DynamodbUnnestDDBJson?: boolean | undefined;
|
|
3986
|
+
}
|
|
3894
3987
|
/**
|
|
3895
3988
|
* <p>Specifies a DynamoDB data source in the Glue Data Catalog.</p>
|
|
3896
3989
|
* @public
|
|
@@ -3911,6 +4004,99 @@ export interface DynamoDBCatalogSource {
|
|
|
3911
4004
|
* @public
|
|
3912
4005
|
*/
|
|
3913
4006
|
Table: string | undefined;
|
|
4007
|
+
/**
|
|
4008
|
+
* <p>Specifies whether Point-in-Time Recovery (PITR) is enabled for the DynamoDB table. When set to <code>true</code>,
|
|
4009
|
+
* allows reading from a specific point in time. The default value is <code>false</code>.</p>
|
|
4010
|
+
* @public
|
|
4011
|
+
*/
|
|
4012
|
+
PitrEnabled?: boolean | undefined;
|
|
4013
|
+
/**
|
|
4014
|
+
* <p>Specifies additional connection options for the DynamoDB data source.</p>
|
|
4015
|
+
* @public
|
|
4016
|
+
*/
|
|
4017
|
+
AdditionalOptions?: DDBELTCatalogAdditionalOptions | undefined;
|
|
4018
|
+
}
|
|
4019
|
+
/**
|
|
4020
|
+
* @public
|
|
4021
|
+
* @enum
|
|
4022
|
+
*/
|
|
4023
|
+
export declare const DdbExportType: {
|
|
4024
|
+
readonly ddb: "ddb";
|
|
4025
|
+
readonly s3: "s3";
|
|
4026
|
+
};
|
|
4027
|
+
/**
|
|
4028
|
+
* @public
|
|
4029
|
+
*/
|
|
4030
|
+
export type DdbExportType = (typeof DdbExportType)[keyof typeof DdbExportType];
|
|
4031
|
+
/**
|
|
4032
|
+
* <p>Specifies connection options for DynamoDB ELT (Extract, Load, Transform) operations. This structure contains configuration parameters for connecting
|
|
4033
|
+
* to and extracting data from DynamoDB tables using the ELT connector.</p>
|
|
4034
|
+
* @public
|
|
4035
|
+
*/
|
|
4036
|
+
export interface DDBELTConnectionOptions {
|
|
4037
|
+
/**
|
|
4038
|
+
* <p>Specifies the export type for DynamoDB data extraction. This parameter determines how data is exported from the DynamoDB table during the ELT process.</p>
|
|
4039
|
+
* @public
|
|
4040
|
+
*/
|
|
4041
|
+
DynamodbExport?: DdbExportType | undefined;
|
|
4042
|
+
/**
|
|
4043
|
+
* <p>A boolean value that specifies whether to unnest DynamoDB JSON format during data extraction. When set to <code>true</code>, the connector will
|
|
4044
|
+
* flatten nested JSON structures from DynamoDB items. When set to <code>false</code>, the original DynamoDB JSON structure is preserved.</p>
|
|
4045
|
+
* @public
|
|
4046
|
+
*/
|
|
4047
|
+
DynamodbUnnestDDBJson?: boolean | undefined;
|
|
4048
|
+
/**
|
|
4049
|
+
* <p>The Amazon Resource Name (ARN) of the DynamoDB table to extract data from. This parameter specifies the source table for the ELT operation.
|
|
4050
|
+
* </p>
|
|
4051
|
+
* @public
|
|
4052
|
+
*/
|
|
4053
|
+
DynamodbTableArn: string | undefined;
|
|
4054
|
+
/**
|
|
4055
|
+
* <p>The name of the Amazon S3 bucket used for intermediate storage during the DynamoDB ELT process. This bucket is used to temporarily store exported
|
|
4056
|
+
* DynamoDB data before it is processed by the ELT job.</p>
|
|
4057
|
+
* @public
|
|
4058
|
+
*/
|
|
4059
|
+
DynamodbS3Bucket?: string | undefined;
|
|
4060
|
+
/**
|
|
4061
|
+
* <p>The S3 object key prefix for files stored in the intermediate S3 bucket during the DynamoDB ELT process. This prefix helps organize and identify the
|
|
4062
|
+
* temporary files created during data extraction.</p>
|
|
4063
|
+
* @public
|
|
4064
|
+
*/
|
|
4065
|
+
DynamodbS3Prefix?: string | undefined;
|
|
4066
|
+
/**
|
|
4067
|
+
* <p>The Amazon Web Services account ID of the owner of the S3 bucket specified in <code>DynamodbS3Bucket</code>. This parameter is required when the S3 bucket is owned by
|
|
4068
|
+
* a different Amazon Web Services account than the one running the ELT job, enabling cross-account access to the intermediate storage bucket.</p>
|
|
4069
|
+
* @public
|
|
4070
|
+
*/
|
|
4071
|
+
DynamodbS3BucketOwner?: string | undefined;
|
|
4072
|
+
/**
|
|
4073
|
+
* <p>The Amazon Resource Name (ARN) of the Amazon Web Services Security Token Service (STS) role to assume for accessing DynamoDB and S3 resources during
|
|
4074
|
+
* the ELT operation.
|
|
4075
|
+
* This role must have the necessary permissions to read from the DynamoDB table and write to the intermediate S3 bucket. </p>
|
|
4076
|
+
* @public
|
|
4077
|
+
*/
|
|
4078
|
+
DynamodbStsRoleArn?: string | undefined;
|
|
4079
|
+
}
|
|
4080
|
+
/**
|
|
4081
|
+
* <p>Specifies a DynamoDB ELT connector source for extracting data from DynamoDB tables.</p>
|
|
4082
|
+
* @public
|
|
4083
|
+
*/
|
|
4084
|
+
export interface DynamoDBELTConnectorSource {
|
|
4085
|
+
/**
|
|
4086
|
+
* <p>The name of the DynamoDB ELT connector source.</p>
|
|
4087
|
+
* @public
|
|
4088
|
+
*/
|
|
4089
|
+
Name: string | undefined;
|
|
4090
|
+
/**
|
|
4091
|
+
* <p>The connection options for the DynamoDB ELT connector source.</p>
|
|
4092
|
+
* @public
|
|
4093
|
+
*/
|
|
4094
|
+
ConnectionOptions?: DDBELTConnectionOptions | undefined;
|
|
4095
|
+
/**
|
|
4096
|
+
* <p>Specifies the data schema for the DynamoDB ELT connector source.</p>
|
|
4097
|
+
* @public
|
|
4098
|
+
*/
|
|
4099
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
3914
4100
|
}
|
|
3915
4101
|
/**
|
|
3916
4102
|
* @public
|
|
@@ -4733,9 +4919,12 @@ export interface OracleSQLCatalogTarget {
|
|
|
4733
4919
|
*/
|
|
4734
4920
|
export declare const PiiType: {
|
|
4735
4921
|
readonly ColumnAudit: "ColumnAudit";
|
|
4922
|
+
readonly ColumnHashing: "ColumnHashing";
|
|
4736
4923
|
readonly ColumnMasking: "ColumnMasking";
|
|
4737
4924
|
readonly RowAudit: "RowAudit";
|
|
4925
|
+
readonly RowHashing: "RowHashing";
|
|
4738
4926
|
readonly RowMasking: "RowMasking";
|
|
4927
|
+
readonly RowPartialMasking: "RowPartialMasking";
|
|
4739
4928
|
};
|
|
4740
4929
|
/**
|
|
4741
4930
|
* @public
|
|
@@ -4791,6 +4980,41 @@ export interface PIIDetection {
|
|
|
4791
4980
|
* @public
|
|
4792
4981
|
*/
|
|
4793
4982
|
MaskValue?: string | undefined;
|
|
4983
|
+
/**
|
|
4984
|
+
* <p>Specifies whether to redact the detected PII text. When set to <code>true</code>, PII content is replaced with redaction characters.</p>
|
|
4985
|
+
* @public
|
|
4986
|
+
*/
|
|
4987
|
+
RedactText?: string | undefined;
|
|
4988
|
+
/**
|
|
4989
|
+
* <p>The character used to replace detected PII content when redaction is enabled. The default redaction character is <code>*</code>.</p>
|
|
4990
|
+
* @public
|
|
4991
|
+
*/
|
|
4992
|
+
RedactChar?: string | undefined;
|
|
4993
|
+
/**
|
|
4994
|
+
* <p>A regular expression pattern used to identify additional PII content beyond the standard detection algorithms.</p>
|
|
4995
|
+
* @public
|
|
4996
|
+
*/
|
|
4997
|
+
MatchPattern?: string | undefined;
|
|
4998
|
+
/**
|
|
4999
|
+
* <p>The number of characters to exclude from redaction on the left side of detected PII content. This allows preserving context around the sensitive data.</p>
|
|
5000
|
+
* @public
|
|
5001
|
+
*/
|
|
5002
|
+
NumLeftCharsToExclude?: number | undefined;
|
|
5003
|
+
/**
|
|
5004
|
+
* <p>The number of characters to exclude from redaction on the right side of detected PII content. This allows preserving context around the sensitive data.</p>
|
|
5005
|
+
* @public
|
|
5006
|
+
*/
|
|
5007
|
+
NumRightCharsToExclude?: number | undefined;
|
|
5008
|
+
/**
|
|
5009
|
+
* <p>Additional parameters for configuring PII detection behavior and sensitivity settings.</p>
|
|
5010
|
+
* @public
|
|
5011
|
+
*/
|
|
5012
|
+
DetectionParameters?: string | undefined;
|
|
5013
|
+
/**
|
|
5014
|
+
* <p>The sensitivity level for PII detection. Higher sensitivity levels detect more potential PII but may result in more false positives.</p>
|
|
5015
|
+
* @public
|
|
5016
|
+
*/
|
|
5017
|
+
DetectionSensitivity?: string | undefined;
|
|
4794
5018
|
}
|
|
4795
5019
|
/**
|
|
4796
5020
|
* <p>Specifies a PostgresSQL data source in the Glue Data Catalog.</p>
|
|
@@ -5074,6 +5298,48 @@ export interface RenameField {
|
|
|
5074
5298
|
*/
|
|
5075
5299
|
TargetPath: string[] | undefined;
|
|
5076
5300
|
}
|
|
5301
|
+
/**
|
|
5302
|
+
* <p>Specifies a group of filters with a logical operator that determines how the filters are combined to evaluate routing conditions.</p>
|
|
5303
|
+
* @public
|
|
5304
|
+
*/
|
|
5305
|
+
export interface GroupFilters {
|
|
5306
|
+
/**
|
|
5307
|
+
* <p>The name of the filter group.</p>
|
|
5308
|
+
* @public
|
|
5309
|
+
*/
|
|
5310
|
+
GroupName: string | undefined;
|
|
5311
|
+
/**
|
|
5312
|
+
* <p>A list of filter expressions that define the conditions for this group.</p>
|
|
5313
|
+
* @public
|
|
5314
|
+
*/
|
|
5315
|
+
Filters: FilterExpression[] | undefined;
|
|
5316
|
+
/**
|
|
5317
|
+
* <p>The logical operator used to combine the filters in this group. Determines whether all filters must match (AND) or any filter can match (OR).</p>
|
|
5318
|
+
* @public
|
|
5319
|
+
*/
|
|
5320
|
+
LogicalOperator: FilterLogicalOperator | undefined;
|
|
5321
|
+
}
|
|
5322
|
+
/**
|
|
5323
|
+
* <p>Specifies a route node that directs data to different output paths based on defined filtering conditions.</p>
|
|
5324
|
+
* @public
|
|
5325
|
+
*/
|
|
5326
|
+
export interface Route {
|
|
5327
|
+
/**
|
|
5328
|
+
* <p>The name of the route node.</p>
|
|
5329
|
+
* @public
|
|
5330
|
+
*/
|
|
5331
|
+
Name: string | undefined;
|
|
5332
|
+
/**
|
|
5333
|
+
* <p>The input connection for the route node.</p>
|
|
5334
|
+
* @public
|
|
5335
|
+
*/
|
|
5336
|
+
Inputs: string[] | undefined;
|
|
5337
|
+
/**
|
|
5338
|
+
* <p>A list of group filters that define the routing conditions and criteria for directing data to different output paths.</p>
|
|
5339
|
+
* @public
|
|
5340
|
+
*/
|
|
5341
|
+
GroupFiltersList: GroupFilters[] | undefined;
|
|
5342
|
+
}
|
|
5077
5343
|
/**
|
|
5078
5344
|
* <p>Specifies a Delta Lake data source that is registered in the Glue Data Catalog. The data source must be stored in Amazon S3.</p>
|
|
5079
5345
|
* @public
|
|
@@ -5136,6 +5402,37 @@ export interface S3CatalogHudiSource {
|
|
|
5136
5402
|
*/
|
|
5137
5403
|
OutputSchemas?: GlueSchema[] | undefined;
|
|
5138
5404
|
}
|
|
5405
|
+
/**
|
|
5406
|
+
* <p>Specifies an Apache Iceberg data source that is registered in the Glue Data Catalog. The Iceberg data source must be stored in Amazon S3.</p>
|
|
5407
|
+
* @public
|
|
5408
|
+
*/
|
|
5409
|
+
export interface S3CatalogIcebergSource {
|
|
5410
|
+
/**
|
|
5411
|
+
* <p>The name of the Iceberg data source.</p>
|
|
5412
|
+
* @public
|
|
5413
|
+
*/
|
|
5414
|
+
Name: string | undefined;
|
|
5415
|
+
/**
|
|
5416
|
+
* <p>The name of the database to read from.</p>
|
|
5417
|
+
* @public
|
|
5418
|
+
*/
|
|
5419
|
+
Database: string | undefined;
|
|
5420
|
+
/**
|
|
5421
|
+
* <p>The name of the table in the database to read from.</p>
|
|
5422
|
+
* @public
|
|
5423
|
+
*/
|
|
5424
|
+
Table: string | undefined;
|
|
5425
|
+
/**
|
|
5426
|
+
* <p>Specifies additional connection options for the Iceberg data source.</p>
|
|
5427
|
+
* @public
|
|
5428
|
+
*/
|
|
5429
|
+
AdditionalIcebergOptions?: Record<string, string> | undefined;
|
|
5430
|
+
/**
|
|
5431
|
+
* <p>Specifies the data schema for the Iceberg source.</p>
|
|
5432
|
+
* @public
|
|
5433
|
+
*/
|
|
5434
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
5435
|
+
}
|
|
5139
5436
|
/**
|
|
5140
5437
|
* <p>Specifies an Amazon S3 data store in the Glue Data Catalog.</p>
|
|
5141
5438
|
* @public
|
|
@@ -5202,6 +5499,12 @@ export interface S3CatalogTarget {
|
|
|
5202
5499
|
* @public
|
|
5203
5500
|
*/
|
|
5204
5501
|
SchemaChangePolicy?: CatalogSchemaChangePolicy | undefined;
|
|
5502
|
+
/**
|
|
5503
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 catalog target.
|
|
5504
|
+
* When set to <code>true</code>, data quality checks are performed automatically during the write operation.</p>
|
|
5505
|
+
* @public
|
|
5506
|
+
*/
|
|
5507
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
5205
5508
|
}
|
|
5206
5509
|
/**
|
|
5207
5510
|
* <p>Specifies additional connection options for the Amazon S3 data store.</p>
|
|
@@ -5411,6 +5714,17 @@ export interface S3DeltaCatalogTarget {
|
|
|
5411
5714
|
* @public
|
|
5412
5715
|
*/
|
|
5413
5716
|
SchemaChangePolicy?: CatalogSchemaChangePolicy | undefined;
|
|
5717
|
+
/**
|
|
5718
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 Delta catalog target. When set to <code>true</code>,
|
|
5719
|
+
* data quality checks are performed automatically during the write operation.</p>
|
|
5720
|
+
* @public
|
|
5721
|
+
*/
|
|
5722
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
5723
|
+
/**
|
|
5724
|
+
* <p>Specifies the data schema for the S3 Delta catalog target.</p>
|
|
5725
|
+
* @public
|
|
5726
|
+
*/
|
|
5727
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
5414
5728
|
}
|
|
5415
5729
|
/**
|
|
5416
5730
|
* @public
|
|
@@ -5520,6 +5834,12 @@ export interface S3DeltaDirectTarget {
|
|
|
5520
5834
|
* @public
|
|
5521
5835
|
*/
|
|
5522
5836
|
SchemaChangePolicy?: DirectSchemaChangePolicy | undefined;
|
|
5837
|
+
/**
|
|
5838
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 Delta direct target. When set to <code>true</code>,
|
|
5839
|
+
* data quality checks are performed automatically during the write operation.</p>
|
|
5840
|
+
* @public
|
|
5841
|
+
*/
|
|
5842
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
5523
5843
|
}
|
|
5524
5844
|
/**
|
|
5525
5845
|
* <p>Specifies a Delta Lake data source stored in Amazon S3.</p>
|
|
@@ -5597,6 +5917,17 @@ export interface S3DirectTarget {
|
|
|
5597
5917
|
* @public
|
|
5598
5918
|
*/
|
|
5599
5919
|
SchemaChangePolicy?: DirectSchemaChangePolicy | undefined;
|
|
5920
|
+
/**
|
|
5921
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 direct target. When set to <code>true</code>,
|
|
5922
|
+
* data quality checks are performed automatically during the write operation.</p>
|
|
5923
|
+
* @public
|
|
5924
|
+
*/
|
|
5925
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
5926
|
+
/**
|
|
5927
|
+
* <p>Specifies the data schema for the S3 direct target.</p>
|
|
5928
|
+
* @public
|
|
5929
|
+
*/
|
|
5930
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
5600
5931
|
}
|
|
5601
5932
|
/**
|
|
5602
5933
|
* @public
|
|
@@ -5681,7 +6012,7 @@ export interface S3ExcelSource {
|
|
|
5681
6012
|
*/
|
|
5682
6013
|
SkipFooter?: number | undefined;
|
|
5683
6014
|
/**
|
|
5684
|
-
* <p>The
|
|
6015
|
+
* <p>The Glue schemas to apply to the processed data.</p>
|
|
5685
6016
|
* @public
|
|
5686
6017
|
*/
|
|
5687
6018
|
OutputSchemas?: GlueSchema[] | undefined;
|
|
@@ -5717,7 +6048,7 @@ export interface S3GlueParquetTarget {
|
|
|
5717
6048
|
*/
|
|
5718
6049
|
Compression?: ParquetCompressionType | undefined;
|
|
5719
6050
|
/**
|
|
5720
|
-
* <p>Specifies the number of target partitions for Parquet files when writing to Amazon S3 using
|
|
6051
|
+
* <p>Specifies the number of target partitions for Parquet files when writing to Amazon S3 using Glue.</p>
|
|
5721
6052
|
* @public
|
|
5722
6053
|
*/
|
|
5723
6054
|
NumberTargetPartitions?: string | undefined;
|
|
@@ -5726,6 +6057,12 @@ export interface S3GlueParquetTarget {
|
|
|
5726
6057
|
* @public
|
|
5727
6058
|
*/
|
|
5728
6059
|
SchemaChangePolicy?: DirectSchemaChangePolicy | undefined;
|
|
6060
|
+
/**
|
|
6061
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 Glue Parquet target. When set to <code>true</code>,
|
|
6062
|
+
* data quality checks are performed automatically during the write operation.</p>
|
|
6063
|
+
* @public
|
|
6064
|
+
*/
|
|
6065
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
5729
6066
|
}
|
|
5730
6067
|
/**
|
|
5731
6068
|
* <p>Specifies a target that writes to a Hudi data source in the Glue Data Catalog.</p>
|
|
@@ -5767,6 +6104,17 @@ export interface S3HudiCatalogTarget {
|
|
|
5767
6104
|
* @public
|
|
5768
6105
|
*/
|
|
5769
6106
|
SchemaChangePolicy?: CatalogSchemaChangePolicy | undefined;
|
|
6107
|
+
/**
|
|
6108
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 Hudi catalog target. When set to <code>true</code>,
|
|
6109
|
+
* data quality checks are performed automatically during the write operation.</p>
|
|
6110
|
+
* @public
|
|
6111
|
+
*/
|
|
6112
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
6113
|
+
/**
|
|
6114
|
+
* <p>Specifies the data schema for the S3 Hudi catalog target.</p>
|
|
6115
|
+
* @public
|
|
6116
|
+
*/
|
|
6117
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
5770
6118
|
}
|
|
5771
6119
|
/**
|
|
5772
6120
|
* @public
|
|
@@ -5832,6 +6180,12 @@ export interface S3HudiDirectTarget {
|
|
|
5832
6180
|
* @public
|
|
5833
6181
|
*/
|
|
5834
6182
|
SchemaChangePolicy?: DirectSchemaChangePolicy | undefined;
|
|
6183
|
+
/**
|
|
6184
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 Hudi direct target. When set to <code>true</code>,
|
|
6185
|
+
* data quality checks are performed automatically during the write operation.</p>
|
|
6186
|
+
* @public
|
|
6187
|
+
*/
|
|
6188
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
5835
6189
|
}
|
|
5836
6190
|
/**
|
|
5837
6191
|
* <p>Specifies a Hudi data source stored in Amazon S3.</p>
|
|
@@ -5890,6 +6244,11 @@ export interface S3HyperDirectTarget {
|
|
|
5890
6244
|
* @public
|
|
5891
6245
|
*/
|
|
5892
6246
|
Inputs: string[] | undefined;
|
|
6247
|
+
/**
|
|
6248
|
+
* <p>Specifies the data output format for the HyperDirect target.</p>
|
|
6249
|
+
* @public
|
|
6250
|
+
*/
|
|
6251
|
+
Format?: TargetFormat | undefined;
|
|
5893
6252
|
/**
|
|
5894
6253
|
* <p>Defines the partitioning strategy for the output data.</p>
|
|
5895
6254
|
* @public
|
|
@@ -5910,28 +6269,84 @@ export interface S3HyperDirectTarget {
|
|
|
5910
6269
|
* @public
|
|
5911
6270
|
*/
|
|
5912
6271
|
SchemaChangePolicy?: DirectSchemaChangePolicy | undefined;
|
|
6272
|
+
/**
|
|
6273
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 Hyper direct target. When set to <code>true</code>, data quality checks are performed automatically during the write operation.</p>
|
|
6274
|
+
* @public
|
|
6275
|
+
*/
|
|
6276
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
6277
|
+
/**
|
|
6278
|
+
* <p>Specifies the data schema for the S3 Hyper direct target.</p>
|
|
6279
|
+
* @public
|
|
6280
|
+
*/
|
|
6281
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
5913
6282
|
}
|
|
5914
6283
|
/**
|
|
5915
|
-
*
|
|
5916
|
-
* @enum
|
|
5917
|
-
*/
|
|
5918
|
-
export declare const IcebergTargetCompressionType: {
|
|
5919
|
-
readonly GZIP: "gzip";
|
|
5920
|
-
readonly LZO: "lzo";
|
|
5921
|
-
readonly SNAPPY: "snappy";
|
|
5922
|
-
readonly UNCOMPRESSED: "uncompressed";
|
|
5923
|
-
};
|
|
5924
|
-
/**
|
|
5925
|
-
* @public
|
|
5926
|
-
*/
|
|
5927
|
-
export type IcebergTargetCompressionType = (typeof IcebergTargetCompressionType)[keyof typeof IcebergTargetCompressionType];
|
|
5928
|
-
/**
|
|
5929
|
-
* <p>Specifies a target that writes to an Iceberg data source in Amazon S3.</p>
|
|
6284
|
+
* <p>Specifies an Apache Iceberg catalog target that writes data to Amazon S3 and registers the table in the Glue Data Catalog.</p>
|
|
5930
6285
|
* @public
|
|
5931
6286
|
*/
|
|
5932
|
-
export interface
|
|
6287
|
+
export interface S3IcebergCatalogTarget {
|
|
5933
6288
|
/**
|
|
5934
|
-
* <p>
|
|
6289
|
+
* <p>The name of the Iceberg catalog target.</p>
|
|
6290
|
+
* @public
|
|
6291
|
+
*/
|
|
6292
|
+
Name: string | undefined;
|
|
6293
|
+
/**
|
|
6294
|
+
* <p>The input connection for the Iceberg catalog target.</p>
|
|
6295
|
+
* @public
|
|
6296
|
+
*/
|
|
6297
|
+
Inputs: string[] | undefined;
|
|
6298
|
+
/**
|
|
6299
|
+
* <p>A list of partition keys for the Iceberg table.</p>
|
|
6300
|
+
* @public
|
|
6301
|
+
*/
|
|
6302
|
+
PartitionKeys?: string[][] | undefined;
|
|
6303
|
+
/**
|
|
6304
|
+
* <p>The name of the table to write to in the catalog.</p>
|
|
6305
|
+
* @public
|
|
6306
|
+
*/
|
|
6307
|
+
Table: string | undefined;
|
|
6308
|
+
/**
|
|
6309
|
+
* <p>The name of the database to write to.</p>
|
|
6310
|
+
* @public
|
|
6311
|
+
*/
|
|
6312
|
+
Database: string | undefined;
|
|
6313
|
+
/**
|
|
6314
|
+
* <p>Specifies additional connection options for the Iceberg catalog target.</p>
|
|
6315
|
+
* @public
|
|
6316
|
+
*/
|
|
6317
|
+
AdditionalOptions?: Record<string, string> | undefined;
|
|
6318
|
+
/**
|
|
6319
|
+
* <p>The policy for handling schema changes in the catalog target.</p>
|
|
6320
|
+
* @public
|
|
6321
|
+
*/
|
|
6322
|
+
SchemaChangePolicy?: CatalogSchemaChangePolicy | undefined;
|
|
6323
|
+
/**
|
|
6324
|
+
* <p>Specifies whether to automatically enable data quality evaluation for the S3 Iceberg catalog target. When set to <code>true</code>, data quality checks are performed automatically during the write operation.</p>
|
|
6325
|
+
* @public
|
|
6326
|
+
*/
|
|
6327
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
6328
|
+
}
|
|
6329
|
+
/**
|
|
6330
|
+
* @public
|
|
6331
|
+
* @enum
|
|
6332
|
+
*/
|
|
6333
|
+
export declare const IcebergTargetCompressionType: {
|
|
6334
|
+
readonly GZIP: "gzip";
|
|
6335
|
+
readonly LZO: "lzo";
|
|
6336
|
+
readonly SNAPPY: "snappy";
|
|
6337
|
+
readonly UNCOMPRESSED: "uncompressed";
|
|
6338
|
+
};
|
|
6339
|
+
/**
|
|
6340
|
+
* @public
|
|
6341
|
+
*/
|
|
6342
|
+
export type IcebergTargetCompressionType = (typeof IcebergTargetCompressionType)[keyof typeof IcebergTargetCompressionType];
|
|
6343
|
+
/**
|
|
6344
|
+
* <p>Specifies a target that writes to an Iceberg data source in Amazon S3.</p>
|
|
6345
|
+
* @public
|
|
6346
|
+
*/
|
|
6347
|
+
export interface S3IcebergDirectTarget {
|
|
6348
|
+
/**
|
|
6349
|
+
* <p>Specifies the unique identifier for the Iceberg target node in your data pipeline.</p>
|
|
5935
6350
|
* @public
|
|
5936
6351
|
*/
|
|
5937
6352
|
Name: string | undefined;
|
|
@@ -5965,6 +6380,12 @@ export interface S3IcebergDirectTarget {
|
|
|
5965
6380
|
* @public
|
|
5966
6381
|
*/
|
|
5967
6382
|
SchemaChangePolicy?: DirectSchemaChangePolicy | undefined;
|
|
6383
|
+
/**
|
|
6384
|
+
* <p>Specifies configuration options for automatic data quality evaluation in Glue jobs. This structure enables automated data quality
|
|
6385
|
+
* checks and monitoring during ETL operations, helping to ensure data integrity and reliability without manual intervention.</p>
|
|
6386
|
+
* @public
|
|
6387
|
+
*/
|
|
6388
|
+
AutoDataQuality?: AutoDataQuality | undefined;
|
|
5968
6389
|
/**
|
|
5969
6390
|
* <p>Specifies the compression codec used for Iceberg table files in S3.</p>
|
|
5970
6391
|
* @public
|
|
@@ -5975,6 +6396,11 @@ export interface S3IcebergDirectTarget {
|
|
|
5975
6396
|
* @public
|
|
5976
6397
|
*/
|
|
5977
6398
|
NumberTargetPartitions?: string | undefined;
|
|
6399
|
+
/**
|
|
6400
|
+
* <p>Specifies the data schema for the S3 Iceberg direct target.</p>
|
|
6401
|
+
* @public
|
|
6402
|
+
*/
|
|
6403
|
+
OutputSchemas?: GlueSchema[] | undefined;
|
|
5978
6404
|
}
|
|
5979
6405
|
/**
|
|
5980
6406
|
* <p>Specifies a JSON data store stored in Amazon S3.</p>
|
|
@@ -7755,435 +8181,6 @@ export interface Edge {
|
|
|
7755
8181
|
*/
|
|
7756
8182
|
DestinationId?: string | undefined;
|
|
7757
8183
|
}
|
|
7758
|
-
/**
|
|
7759
|
-
* <p>The details of a crawl in the workflow.</p>
|
|
7760
|
-
* @public
|
|
7761
|
-
*/
|
|
7762
|
-
export interface Crawl {
|
|
7763
|
-
/**
|
|
7764
|
-
* <p>The state of the crawler.</p>
|
|
7765
|
-
* @public
|
|
7766
|
-
*/
|
|
7767
|
-
State?: CrawlState | undefined;
|
|
7768
|
-
/**
|
|
7769
|
-
* <p>The date and time on which the crawl started.</p>
|
|
7770
|
-
* @public
|
|
7771
|
-
*/
|
|
7772
|
-
StartedOn?: Date | undefined;
|
|
7773
|
-
/**
|
|
7774
|
-
* <p>The date and time on which the crawl completed.</p>
|
|
7775
|
-
* @public
|
|
7776
|
-
*/
|
|
7777
|
-
CompletedOn?: Date | undefined;
|
|
7778
|
-
/**
|
|
7779
|
-
* <p>The error message associated with the crawl.</p>
|
|
7780
|
-
* @public
|
|
7781
|
-
*/
|
|
7782
|
-
ErrorMessage?: string | undefined;
|
|
7783
|
-
/**
|
|
7784
|
-
* <p>The log group associated with the crawl.</p>
|
|
7785
|
-
* @public
|
|
7786
|
-
*/
|
|
7787
|
-
LogGroup?: string | undefined;
|
|
7788
|
-
/**
|
|
7789
|
-
* <p>The log stream associated with the crawl.</p>
|
|
7790
|
-
* @public
|
|
7791
|
-
*/
|
|
7792
|
-
LogStream?: string | undefined;
|
|
7793
|
-
}
|
|
7794
|
-
/**
|
|
7795
|
-
* <p>The details of a Crawler node present in the workflow.</p>
|
|
7796
|
-
* @public
|
|
7797
|
-
*/
|
|
7798
|
-
export interface CrawlerNodeDetails {
|
|
7799
|
-
/**
|
|
7800
|
-
* <p>A list of crawls represented by the crawl node.</p>
|
|
7801
|
-
* @public
|
|
7802
|
-
*/
|
|
7803
|
-
Crawls?: Crawl[] | undefined;
|
|
7804
|
-
}
|
|
7805
|
-
/**
|
|
7806
|
-
* <p>A job run that was used in the predicate of a conditional trigger
|
|
7807
|
-
* that triggered this job run.</p>
|
|
7808
|
-
* @public
|
|
7809
|
-
*/
|
|
7810
|
-
export interface Predecessor {
|
|
7811
|
-
/**
|
|
7812
|
-
* <p>The name of the job definition used by the predecessor job run.</p>
|
|
7813
|
-
* @public
|
|
7814
|
-
*/
|
|
7815
|
-
JobName?: string | undefined;
|
|
7816
|
-
/**
|
|
7817
|
-
* <p>The job-run ID of the predecessor job run.</p>
|
|
7818
|
-
* @public
|
|
7819
|
-
*/
|
|
7820
|
-
RunId?: string | undefined;
|
|
7821
|
-
}
|
|
7822
|
-
/**
|
|
7823
|
-
* <p>Contains information about a job run.</p>
|
|
7824
|
-
* @public
|
|
7825
|
-
*/
|
|
7826
|
-
export interface JobRun {
|
|
7827
|
-
/**
|
|
7828
|
-
* <p>The ID of this job run.</p>
|
|
7829
|
-
* @public
|
|
7830
|
-
*/
|
|
7831
|
-
Id?: string | undefined;
|
|
7832
|
-
/**
|
|
7833
|
-
* <p>The number of the attempt to run this job.</p>
|
|
7834
|
-
* @public
|
|
7835
|
-
*/
|
|
7836
|
-
Attempt?: number | undefined;
|
|
7837
|
-
/**
|
|
7838
|
-
* <p>The ID of the previous run of this job. For example, the <code>JobRunId</code> specified
|
|
7839
|
-
* in the <code>StartJobRun</code> action.</p>
|
|
7840
|
-
* @public
|
|
7841
|
-
*/
|
|
7842
|
-
PreviousRunId?: string | undefined;
|
|
7843
|
-
/**
|
|
7844
|
-
* <p>The name of the trigger that started this job run.</p>
|
|
7845
|
-
* @public
|
|
7846
|
-
*/
|
|
7847
|
-
TriggerName?: string | undefined;
|
|
7848
|
-
/**
|
|
7849
|
-
* <p>The name of the job definition being used in this run.</p>
|
|
7850
|
-
* @public
|
|
7851
|
-
*/
|
|
7852
|
-
JobName?: string | undefined;
|
|
7853
|
-
/**
|
|
7854
|
-
* <p>A mode that describes how a job was created. Valid values are:</p>
|
|
7855
|
-
* <ul>
|
|
7856
|
-
* <li>
|
|
7857
|
-
* <p>
|
|
7858
|
-
* <code>SCRIPT</code> - The job was created using the Glue Studio script editor.</p>
|
|
7859
|
-
* </li>
|
|
7860
|
-
* <li>
|
|
7861
|
-
* <p>
|
|
7862
|
-
* <code>VISUAL</code> - The job was created using the Glue Studio visual editor.</p>
|
|
7863
|
-
* </li>
|
|
7864
|
-
* <li>
|
|
7865
|
-
* <p>
|
|
7866
|
-
* <code>NOTEBOOK</code> - The job was created using an interactive sessions notebook.</p>
|
|
7867
|
-
* </li>
|
|
7868
|
-
* </ul>
|
|
7869
|
-
* <p>When the <code>JobMode</code> field is missing or null, <code>SCRIPT</code> is assigned as the default value.</p>
|
|
7870
|
-
* @public
|
|
7871
|
-
*/
|
|
7872
|
-
JobMode?: JobMode | undefined;
|
|
7873
|
-
/**
|
|
7874
|
-
* <p>Specifies whether job run queuing is enabled for the job run.</p>
|
|
7875
|
-
* <p>A value of true means job run queuing is enabled for the job run. If false or not populated, the job run will not be considered for queueing.</p>
|
|
7876
|
-
* @public
|
|
7877
|
-
*/
|
|
7878
|
-
JobRunQueuingEnabled?: boolean | undefined;
|
|
7879
|
-
/**
|
|
7880
|
-
* <p>The date and time at which this job run was started.</p>
|
|
7881
|
-
* @public
|
|
7882
|
-
*/
|
|
7883
|
-
StartedOn?: Date | undefined;
|
|
7884
|
-
/**
|
|
7885
|
-
* <p>The last time that this job run was modified.</p>
|
|
7886
|
-
* @public
|
|
7887
|
-
*/
|
|
7888
|
-
LastModifiedOn?: Date | undefined;
|
|
7889
|
-
/**
|
|
7890
|
-
* <p>The date and time that this job run completed.</p>
|
|
7891
|
-
* @public
|
|
7892
|
-
*/
|
|
7893
|
-
CompletedOn?: Date | undefined;
|
|
7894
|
-
/**
|
|
7895
|
-
* <p>The current state of the job run. For more information about the statuses of jobs that have terminated abnormally, see <a href="https://docs.aws.amazon.com/glue/latest/dg/job-run-statuses.html">Glue Job Run Statuses</a>.</p>
|
|
7896
|
-
* @public
|
|
7897
|
-
*/
|
|
7898
|
-
JobRunState?: JobRunState | undefined;
|
|
7899
|
-
/**
|
|
7900
|
-
* <p>The job arguments associated with this run. For this job run, they replace the default
|
|
7901
|
-
* arguments set in the job definition itself.</p>
|
|
7902
|
-
* <p>You can specify arguments here that your own job-execution script
|
|
7903
|
-
* consumes, as well as arguments that Glue itself consumes.</p>
|
|
7904
|
-
* <p>Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets
|
|
7905
|
-
* from a Glue Connection, Secrets Manager or other secret management
|
|
7906
|
-
* mechanism if you intend to keep them within the Job. </p>
|
|
7907
|
-
* <p>For information about how to specify and consume your own Job arguments, see the <a href="https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html">Calling Glue APIs in Python</a> topic in the developer guide.</p>
|
|
7908
|
-
* <p>For information about the arguments you can provide to this field when configuring Spark jobs,
|
|
7909
|
-
* see the <a href="https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html">Special Parameters Used by Glue</a> topic in the developer guide.</p>
|
|
7910
|
-
* <p>For information about the arguments you can provide to this field when configuring Ray
|
|
7911
|
-
* jobs, see <a href="https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html">Using
|
|
7912
|
-
* job parameters in Ray jobs</a> in the developer guide.</p>
|
|
7913
|
-
* @public
|
|
7914
|
-
*/
|
|
7915
|
-
Arguments?: Record<string, string> | undefined;
|
|
7916
|
-
/**
|
|
7917
|
-
* <p>An error message associated with this job run.</p>
|
|
7918
|
-
* @public
|
|
7919
|
-
*/
|
|
7920
|
-
ErrorMessage?: string | undefined;
|
|
7921
|
-
/**
|
|
7922
|
-
* <p>A list of predecessors to this job run.</p>
|
|
7923
|
-
* @public
|
|
7924
|
-
*/
|
|
7925
|
-
PredecessorRuns?: Predecessor[] | undefined;
|
|
7926
|
-
/**
|
|
7927
|
-
* <p>This field is deprecated. Use <code>MaxCapacity</code> instead.</p>
|
|
7928
|
-
* <p>The number of Glue data processing units (DPUs) allocated to this JobRun.
|
|
7929
|
-
* From 2 to 100 DPUs can be allocated; the default is 10. A DPU is a relative measure
|
|
7930
|
-
* of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory.
|
|
7931
|
-
* For more information, see the <a href="https://aws.amazon.com/glue/pricing/">Glue
|
|
7932
|
-
* pricing page</a>.</p>
|
|
7933
|
-
*
|
|
7934
|
-
* @deprecated
|
|
7935
|
-
* @public
|
|
7936
|
-
*/
|
|
7937
|
-
AllocatedCapacity?: number | undefined;
|
|
7938
|
-
/**
|
|
7939
|
-
* <p>The amount of time (in seconds) that the job run consumed resources.</p>
|
|
7940
|
-
* @public
|
|
7941
|
-
*/
|
|
7942
|
-
ExecutionTime?: number | undefined;
|
|
7943
|
-
/**
|
|
7944
|
-
* <p>The <code>JobRun</code> timeout in minutes. This is the maximum time that a job run can
|
|
7945
|
-
* consume resources before it is terminated and enters <code>TIMEOUT</code> status. This value overrides the timeout value set in the parent job.</p>
|
|
7946
|
-
* <p>Jobs must have timeout values less than 7 days or 10080 minutes. Otherwise, the jobs will throw an exception.</p>
|
|
7947
|
-
* <p>When the value is left blank, the timeout is defaulted to 2880 minutes.</p>
|
|
7948
|
-
* <p>Any existing Glue jobs that had a timeout value greater than 7 days will be defaulted to 7 days. For instance if you have specified a timeout of 20 days for a batch job, it will be stopped on the 7th day.</p>
|
|
7949
|
-
* <p>For streaming jobs, if you have set up a maintenance window, it will be restarted during the maintenance window after 7 days.</p>
|
|
7950
|
-
* @public
|
|
7951
|
-
*/
|
|
7952
|
-
Timeout?: number | undefined;
|
|
7953
|
-
/**
|
|
7954
|
-
* <p>For Glue version 1.0 or earlier jobs, using the standard worker type, the number of
|
|
7955
|
-
* Glue data processing units (DPUs) that can be allocated when this job runs. A DPU is
|
|
7956
|
-
* a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB
|
|
7957
|
-
* of memory. For more information, see the <a href="https://aws.amazon.com/glue/pricing/">
|
|
7958
|
-
* Glue pricing page</a>.</p>
|
|
7959
|
-
* <p>For Glue version 2.0+ jobs, you cannot specify a <code>Maximum capacity</code>.
|
|
7960
|
-
* Instead, you should specify a <code>Worker type</code> and the <code>Number of workers</code>.</p>
|
|
7961
|
-
* <p>Do not set <code>MaxCapacity</code> if using <code>WorkerType</code> and <code>NumberOfWorkers</code>.</p>
|
|
7962
|
-
* <p>The value that can be allocated for <code>MaxCapacity</code> depends on whether you are
|
|
7963
|
-
* running a Python shell job, an Apache Spark ETL job, or an Apache Spark streaming ETL
|
|
7964
|
-
* job:</p>
|
|
7965
|
-
* <ul>
|
|
7966
|
-
* <li>
|
|
7967
|
-
* <p>When you specify a Python shell job (<code>JobCommand.Name</code>="pythonshell"), you can
|
|
7968
|
-
* allocate either 0.0625 or 1 DPU. The default is 0.0625 DPU.</p>
|
|
7969
|
-
* </li>
|
|
7970
|
-
* <li>
|
|
7971
|
-
* <p>When you specify an Apache Spark ETL job (<code>JobCommand.Name</code>="glueetl") or Apache
|
|
7972
|
-
* Spark streaming ETL job (<code>JobCommand.Name</code>="gluestreaming"), you can allocate from 2 to 100 DPUs.
|
|
7973
|
-
* The default is 10 DPUs. This job type cannot have a fractional DPU allocation.</p>
|
|
7974
|
-
* </li>
|
|
7975
|
-
* </ul>
|
|
7976
|
-
* @public
|
|
7977
|
-
*/
|
|
7978
|
-
MaxCapacity?: number | undefined;
|
|
7979
|
-
/**
|
|
7980
|
-
* <p>The type of predefined worker that is allocated when a job runs. Accepts a value of
|
|
7981
|
-
* G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.</p>
|
|
7982
|
-
* <ul>
|
|
7983
|
-
* <li>
|
|
7984
|
-
* <p>For the <code>G.1X</code> worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.</p>
|
|
7985
|
-
* </li>
|
|
7986
|
-
* <li>
|
|
7987
|
-
* <p>For the <code>G.2X</code> worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.</p>
|
|
7988
|
-
* </li>
|
|
7989
|
-
* <li>
|
|
7990
|
-
* <p>For the <code>G.4X</code> worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).</p>
|
|
7991
|
-
* </li>
|
|
7992
|
-
* <li>
|
|
7993
|
-
* <p>For the <code>G.8X</code> worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the <code>G.4X</code> worker type.</p>
|
|
7994
|
-
* </li>
|
|
7995
|
-
* <li>
|
|
7996
|
-
* <p>For the <code>G.025X</code> worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.</p>
|
|
7997
|
-
* </li>
|
|
7998
|
-
* <li>
|
|
7999
|
-
* <p>For the <code>Z.2X</code> worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.</p>
|
|
8000
|
-
* </li>
|
|
8001
|
-
* </ul>
|
|
8002
|
-
* @public
|
|
8003
|
-
*/
|
|
8004
|
-
WorkerType?: WorkerType | undefined;
|
|
8005
|
-
/**
|
|
8006
|
-
* <p>The number of workers of a defined <code>workerType</code> that are allocated when a job runs.</p>
|
|
8007
|
-
* @public
|
|
8008
|
-
*/
|
|
8009
|
-
NumberOfWorkers?: number | undefined;
|
|
8010
|
-
/**
|
|
8011
|
-
* <p>The name of the <code>SecurityConfiguration</code> structure to be used with this job
|
|
8012
|
-
* run.</p>
|
|
8013
|
-
* @public
|
|
8014
|
-
*/
|
|
8015
|
-
SecurityConfiguration?: string | undefined;
|
|
8016
|
-
/**
|
|
8017
|
-
* <p>The name of the log group for secure logging that can be server-side encrypted in Amazon
|
|
8018
|
-
* CloudWatch using KMS. This name can be <code>/aws-glue/jobs/</code>, in which case the
|
|
8019
|
-
* default encryption is <code>NONE</code>. If you add a role name and
|
|
8020
|
-
* <code>SecurityConfiguration</code> name (in other words,
|
|
8021
|
-
* <code>/aws-glue/jobs-yourRoleName-yourSecurityConfigurationName/</code>), then that security
|
|
8022
|
-
* configuration is used to encrypt the log group.</p>
|
|
8023
|
-
* @public
|
|
8024
|
-
*/
|
|
8025
|
-
LogGroupName?: string | undefined;
|
|
8026
|
-
/**
|
|
8027
|
-
* <p>Specifies configuration properties of a job run notification.</p>
|
|
8028
|
-
* @public
|
|
8029
|
-
*/
|
|
8030
|
-
NotificationProperty?: NotificationProperty | undefined;
|
|
8031
|
-
/**
|
|
8032
|
-
* <p>In Spark jobs, <code>GlueVersion</code> determines the versions of Apache Spark and Python
|
|
8033
|
-
* that Glue available in a job. The Python version indicates the version
|
|
8034
|
-
* supported for jobs of type Spark. </p>
|
|
8035
|
-
* <p>Ray jobs should set <code>GlueVersion</code> to <code>4.0</code> or greater. However,
|
|
8036
|
-
* the versions of Ray, Python and additional libraries available in your Ray job are determined
|
|
8037
|
-
* by the <code>Runtime</code> parameter of the Job command.</p>
|
|
8038
|
-
* <p>For more information about the available Glue versions and corresponding
|
|
8039
|
-
* Spark and Python versions, see <a href="https://docs.aws.amazon.com/glue/latest/dg/add-job.html">Glue version</a> in the developer
|
|
8040
|
-
* guide.</p>
|
|
8041
|
-
* <p>Jobs that are created without specifying a Glue version default to Glue 0.9.</p>
|
|
8042
|
-
* @public
|
|
8043
|
-
*/
|
|
8044
|
-
GlueVersion?: string | undefined;
|
|
8045
|
-
/**
|
|
8046
|
-
* <p>This field can be set for either job runs with execution class <code>FLEX</code> or when Auto Scaling is enabled, and represents the total time each executor ran during the lifecycle of a job run in seconds, multiplied by a DPU factor (1 for <code>G.1X</code>, 2 for <code>G.2X</code>, or 0.25 for <code>G.025X</code> workers). This value may be different than the <code>executionEngineRuntime</code> * <code>MaxCapacity</code> as in the case of Auto Scaling jobs, as the number of executors running at a given time may be less than the <code>MaxCapacity</code>. Therefore, it is possible that the value of <code>DPUSeconds</code> is less than <code>executionEngineRuntime</code> * <code>MaxCapacity</code>.</p>
|
|
8047
|
-
* @public
|
|
8048
|
-
*/
|
|
8049
|
-
DPUSeconds?: number | undefined;
|
|
8050
|
-
/**
|
|
8051
|
-
* <p>Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.</p>
|
|
8052
|
-
* <p>The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary. </p>
|
|
8053
|
-
* <p>Only jobs with Glue version 3.0 and above and command type <code>glueetl</code> will be allowed to set <code>ExecutionClass</code> to <code>FLEX</code>. The flexible execution class is available for Spark jobs.</p>
|
|
8054
|
-
* @public
|
|
8055
|
-
*/
|
|
8056
|
-
ExecutionClass?: ExecutionClass | undefined;
|
|
8057
|
-
/**
|
|
8058
|
-
* <p>This field specifies a day of the week and hour for a maintenance window for streaming jobs. Glue periodically performs maintenance activities. During these maintenance windows, Glue will need to restart your streaming jobs.</p>
|
|
8059
|
-
* <p>Glue will restart the job within 3 hours of the specified maintenance window. For instance, if you set up the maintenance window for Monday at 10:00AM GMT, your jobs will be restarted between 10:00AM GMT to 1:00PM GMT.</p>
|
|
8060
|
-
* @public
|
|
8061
|
-
*/
|
|
8062
|
-
MaintenanceWindow?: string | undefined;
|
|
8063
|
-
/**
|
|
8064
|
-
* <p>The name of an Glue usage profile associated with the job run.</p>
|
|
8065
|
-
* @public
|
|
8066
|
-
*/
|
|
8067
|
-
ProfileName?: string | undefined;
|
|
8068
|
-
/**
|
|
8069
|
-
* <p>This field holds details that pertain to the state of a job run. The field is nullable.</p>
|
|
8070
|
-
* <p>For example, when a job run is in a WAITING state as a result of job run queuing, the field has the reason why the job run is in that state.</p>
|
|
8071
|
-
* @public
|
|
8072
|
-
*/
|
|
8073
|
-
StateDetail?: string | undefined;
|
|
8074
|
-
/**
|
|
8075
|
-
* <p>This inline session policy to the StartJobRun API allows you to dynamically restrict the permissions of the specified
|
|
8076
|
-
* execution role for the scope of the job, without requiring the creation of additional IAM roles.</p>
|
|
8077
|
-
* @public
|
|
8078
|
-
*/
|
|
8079
|
-
ExecutionRoleSessionPolicy?: string | undefined;
|
|
8080
|
-
}
|
|
8081
|
-
/**
|
|
8082
|
-
* <p>The details of a Job node present in the workflow.</p>
|
|
8083
|
-
* @public
|
|
8084
|
-
*/
|
|
8085
|
-
export interface JobNodeDetails {
|
|
8086
|
-
/**
|
|
8087
|
-
* <p>The information for the job runs represented by the job node.</p>
|
|
8088
|
-
* @public
|
|
8089
|
-
*/
|
|
8090
|
-
JobRuns?: JobRun[] | undefined;
|
|
8091
|
-
}
|
|
8092
|
-
/**
|
|
8093
|
-
* <p>The details of a Trigger node present in the workflow.</p>
|
|
8094
|
-
* @public
|
|
8095
|
-
*/
|
|
8096
|
-
export interface TriggerNodeDetails {
|
|
8097
|
-
/**
|
|
8098
|
-
* <p>The information of the trigger represented by the trigger node.</p>
|
|
8099
|
-
* @public
|
|
8100
|
-
*/
|
|
8101
|
-
Trigger?: Trigger | undefined;
|
|
8102
|
-
}
|
|
8103
|
-
/**
|
|
8104
|
-
* @public
|
|
8105
|
-
* @enum
|
|
8106
|
-
*/
|
|
8107
|
-
export declare const NodeType: {
|
|
8108
|
-
readonly CRAWLER: "CRAWLER";
|
|
8109
|
-
readonly JOB: "JOB";
|
|
8110
|
-
readonly TRIGGER: "TRIGGER";
|
|
8111
|
-
};
|
|
8112
|
-
/**
|
|
8113
|
-
* @public
|
|
8114
|
-
*/
|
|
8115
|
-
export type NodeType = (typeof NodeType)[keyof typeof NodeType];
|
|
8116
|
-
/**
|
|
8117
|
-
* <p>A node represents an Glue component (trigger, crawler, or job) on a workflow graph.</p>
|
|
8118
|
-
* @public
|
|
8119
|
-
*/
|
|
8120
|
-
export interface Node {
|
|
8121
|
-
/**
|
|
8122
|
-
* <p>The type of Glue component represented by the node.</p>
|
|
8123
|
-
* @public
|
|
8124
|
-
*/
|
|
8125
|
-
Type?: NodeType | undefined;
|
|
8126
|
-
/**
|
|
8127
|
-
* <p>The name of the Glue component represented by the node.</p>
|
|
8128
|
-
* @public
|
|
8129
|
-
*/
|
|
8130
|
-
Name?: string | undefined;
|
|
8131
|
-
/**
|
|
8132
|
-
* <p>The unique Id assigned to the node within the workflow.</p>
|
|
8133
|
-
* @public
|
|
8134
|
-
*/
|
|
8135
|
-
UniqueId?: string | undefined;
|
|
8136
|
-
/**
|
|
8137
|
-
* <p>Details of the Trigger when the node represents a Trigger.</p>
|
|
8138
|
-
* @public
|
|
8139
|
-
*/
|
|
8140
|
-
TriggerDetails?: TriggerNodeDetails | undefined;
|
|
8141
|
-
/**
|
|
8142
|
-
* <p>Details of the Job when the node represents a Job.</p>
|
|
8143
|
-
* @public
|
|
8144
|
-
*/
|
|
8145
|
-
JobDetails?: JobNodeDetails | undefined;
|
|
8146
|
-
/**
|
|
8147
|
-
* <p>Details of the crawler when the node represents a crawler.</p>
|
|
8148
|
-
* @public
|
|
8149
|
-
*/
|
|
8150
|
-
CrawlerDetails?: CrawlerNodeDetails | undefined;
|
|
8151
|
-
}
|
|
8152
|
-
/**
|
|
8153
|
-
* <p>A workflow graph represents the complete workflow containing all the Glue components present in the
|
|
8154
|
-
* workflow and all the directed connections between them.</p>
|
|
8155
|
-
* @public
|
|
8156
|
-
*/
|
|
8157
|
-
export interface WorkflowGraph {
|
|
8158
|
-
/**
|
|
8159
|
-
* <p>A list of the the Glue components belong to the workflow represented as nodes.</p>
|
|
8160
|
-
* @public
|
|
8161
|
-
*/
|
|
8162
|
-
Nodes?: Node[] | undefined;
|
|
8163
|
-
/**
|
|
8164
|
-
* <p>A list of all the directed connections between the nodes belonging to the workflow.</p>
|
|
8165
|
-
* @public
|
|
8166
|
-
*/
|
|
8167
|
-
Edges?: Edge[] | undefined;
|
|
8168
|
-
}
|
|
8169
|
-
/**
|
|
8170
|
-
* <p>The batch condition that started the workflow run. Either the number of events in the batch size arrived,
|
|
8171
|
-
* in which case the BatchSize member is non-zero, or the batch window expired, in which case the BatchWindow
|
|
8172
|
-
* member is non-zero.</p>
|
|
8173
|
-
* @public
|
|
8174
|
-
*/
|
|
8175
|
-
export interface StartingEventBatchCondition {
|
|
8176
|
-
/**
|
|
8177
|
-
* <p>Number of events in the batch.</p>
|
|
8178
|
-
* @public
|
|
8179
|
-
*/
|
|
8180
|
-
BatchSize?: number | undefined;
|
|
8181
|
-
/**
|
|
8182
|
-
* <p>Duration of the batch window in seconds.</p>
|
|
8183
|
-
* @public
|
|
8184
|
-
*/
|
|
8185
|
-
BatchWindow?: number | undefined;
|
|
8186
|
-
}
|
|
8187
8184
|
/**
|
|
8188
8185
|
* @internal
|
|
8189
8186
|
*/
|