@aws-cdk/aws-glue-alpha 2.265.0-alpha.0 → 2.266.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +504 -436
- package/.jsii.tabl.json.gz +0 -0
- package/.warnings.jsii.js +25 -33
- package/README.md +111 -43
- package/awslint.json +2 -1
- package/lib/catalog.js +3 -3
- package/lib/code.js +3 -3
- package/lib/connection.js +2 -2
- package/lib/data-format.d.ts +2 -2
- package/lib/data-format.js +8 -8
- package/lib/data-quality-ruleset.d.ts +28 -3
- package/lib/data-quality-ruleset.js +37 -5
- package/lib/database.d.ts +1 -1
- package/lib/database.js +2 -2
- package/lib/external-table.js +6 -4
- package/lib/jobs/job.d.ts +1 -24
- package/lib/jobs/job.js +3 -3
- package/lib/jobs/pyspark-etl-job.d.ts +2 -1
- package/lib/jobs/pyspark-etl-job.js +10 -5
- package/lib/jobs/pyspark-flex-etl-job.d.ts +3 -2
- package/lib/jobs/pyspark-flex-etl-job.js +7 -6
- package/lib/jobs/pyspark-streaming-job.d.ts +2 -1
- package/lib/jobs/pyspark-streaming-job.js +6 -5
- package/lib/jobs/python-shell-job.js +1 -1
- package/lib/jobs/ray-job.d.ts +8 -0
- package/lib/jobs/ray-job.js +3 -7
- package/lib/jobs/scala-spark-etl-job.js +4 -8
- package/lib/jobs/scala-spark-flex-etl-job.js +4 -4
- package/lib/jobs/scala-spark-streaming-job.js +4 -8
- package/lib/jobs/spark-job.d.ts +26 -0
- package/lib/jobs/spark-job.js +2 -2
- package/lib/partition-projection.js +1 -1
- package/lib/s3-table.d.ts +2 -1
- package/lib/s3-table.js +5 -3
- package/lib/schema.js +3 -8
- package/lib/security-configuration.d.ts +41 -47
- package/lib/security-configuration.js +83 -25
- package/lib/storage-parameter.js +1 -1
- package/lib/table-base.d.ts +33 -0
- package/lib/table-base.js +33 -2
- package/lib/table-deprecated.js +1 -1
- package/lib/triggers/trigger-options.js +1 -1
- package/lib/triggers/workflow.js +2 -2
- package/package.json +7 -7
package/.jsii.tabl.json.gz
CHANGED
|
Binary file
|
package/.warnings.jsii.js
CHANGED
|
@@ -75,21 +75,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
75
75
|
finally {
|
|
76
76
|
visitedObjects.delete(p);
|
|
77
77
|
}
|
|
78
|
-
}, _aws_cdk_aws_glue_alpha_SecurityConfigurationProps: function _aws_cdk_aws_glue_alpha_SecurityConfigurationProps(p) {
|
|
79
|
-
if (p == null)
|
|
80
|
-
return;
|
|
81
|
-
visitedObjects.add(p);
|
|
82
|
-
try {
|
|
83
|
-
if (!visitedObjects.has(p.cloudWatchEncryption))
|
|
84
|
-
module.exports._aws_cdk_aws_glue_alpha_CloudWatchEncryption(p.cloudWatchEncryption);
|
|
85
|
-
if (!visitedObjects.has(p.jobBookmarksEncryption))
|
|
86
|
-
module.exports._aws_cdk_aws_glue_alpha_JobBookmarksEncryption(p.jobBookmarksEncryption);
|
|
87
|
-
if (!visitedObjects.has(p.s3Encryption))
|
|
88
|
-
module.exports._aws_cdk_aws_glue_alpha_S3Encryption(p.s3Encryption);
|
|
89
|
-
}
|
|
90
|
-
finally {
|
|
91
|
-
visitedObjects.delete(p);
|
|
92
|
-
}
|
|
93
78
|
}, _aws_cdk_aws_glue_alpha_WorkerType: function _aws_cdk_aws_glue_alpha_WorkerType(p) {
|
|
94
79
|
if (p == null)
|
|
95
80
|
return;
|
|
@@ -135,6 +120,15 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
135
120
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
136
121
|
if (!visitedObjects.has(p.continuousLogging))
|
|
137
122
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
visitedObjects.delete(p);
|
|
126
|
+
}
|
|
127
|
+
}, _aws_cdk_aws_glue_alpha_WorkerConfiguration: function _aws_cdk_aws_glue_alpha_WorkerConfiguration(p) {
|
|
128
|
+
if (p == null)
|
|
129
|
+
return;
|
|
130
|
+
visitedObjects.add(p);
|
|
131
|
+
try {
|
|
138
132
|
if (!visitedObjects.has(p.workerType))
|
|
139
133
|
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
140
134
|
}
|
|
@@ -148,14 +142,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
148
142
|
try {
|
|
149
143
|
if (!visitedObjects.has(p.sparkUI))
|
|
150
144
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
145
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
146
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
151
147
|
if (p.connections != null)
|
|
152
148
|
for (const o of p.connections)
|
|
153
149
|
if (!visitedObjects.has(o))
|
|
154
150
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
155
151
|
if (!visitedObjects.has(p.continuousLogging))
|
|
156
152
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
157
|
-
if (!visitedObjects.has(p.workerType))
|
|
158
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
159
153
|
}
|
|
160
154
|
finally {
|
|
161
155
|
visitedObjects.delete(p);
|
|
@@ -179,14 +173,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
179
173
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
180
174
|
if (!visitedObjects.has(p.sparkUI))
|
|
181
175
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
176
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
177
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
182
178
|
if (p.connections != null)
|
|
183
179
|
for (const o of p.connections)
|
|
184
180
|
if (!visitedObjects.has(o))
|
|
185
181
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
186
182
|
if (!visitedObjects.has(p.continuousLogging))
|
|
187
183
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
188
|
-
if (!visitedObjects.has(p.workerType))
|
|
189
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
190
184
|
}
|
|
191
185
|
finally {
|
|
192
186
|
visitedObjects.delete(p);
|
|
@@ -210,14 +204,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
210
204
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
211
205
|
if (!visitedObjects.has(p.sparkUI))
|
|
212
206
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
207
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
208
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
213
209
|
if (p.connections != null)
|
|
214
210
|
for (const o of p.connections)
|
|
215
211
|
if (!visitedObjects.has(o))
|
|
216
212
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
217
213
|
if (!visitedObjects.has(p.continuousLogging))
|
|
218
214
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
219
|
-
if (!visitedObjects.has(p.workerType))
|
|
220
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
221
215
|
}
|
|
222
216
|
finally {
|
|
223
217
|
visitedObjects.delete(p);
|
|
@@ -241,14 +235,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
241
235
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
242
236
|
if (!visitedObjects.has(p.sparkUI))
|
|
243
237
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
238
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
239
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
244
240
|
if (p.connections != null)
|
|
245
241
|
for (const o of p.connections)
|
|
246
242
|
if (!visitedObjects.has(o))
|
|
247
243
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
248
244
|
if (!visitedObjects.has(p.continuousLogging))
|
|
249
245
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
250
|
-
if (!visitedObjects.has(p.workerType))
|
|
251
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
252
246
|
}
|
|
253
247
|
finally {
|
|
254
248
|
visitedObjects.delete(p);
|
|
@@ -268,8 +262,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
268
262
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
269
263
|
if (!visitedObjects.has(p.continuousLogging))
|
|
270
264
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
271
|
-
if (!visitedObjects.has(p.workerType))
|
|
272
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
273
265
|
}
|
|
274
266
|
finally {
|
|
275
267
|
visitedObjects.delete(p);
|
|
@@ -285,6 +277,8 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
285
277
|
print("@aws-cdk/aws-glue-alpha.RayJobProps#enableObservabilityMetrics", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
286
278
|
if ("jobRunQueuingEnabled" in p)
|
|
287
279
|
print("@aws-cdk/aws-glue-alpha.RayJobProps#jobRunQueuingEnabled", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
280
|
+
if ("numberOfWorkers" in p)
|
|
281
|
+
print("@aws-cdk/aws-glue-alpha.RayJobProps#numberOfWorkers", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
288
282
|
if ("runtime" in p)
|
|
289
283
|
print("@aws-cdk/aws-glue-alpha.RayJobProps#runtime", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
290
284
|
if (!visitedObjects.has(p.runtime))
|
|
@@ -295,8 +289,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
295
289
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
296
290
|
if (!visitedObjects.has(p.continuousLogging))
|
|
297
291
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
298
|
-
if (!visitedObjects.has(p.workerType))
|
|
299
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
300
292
|
}
|
|
301
293
|
finally {
|
|
302
294
|
visitedObjects.delete(p);
|
|
@@ -316,14 +308,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
316
308
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
317
309
|
if (!visitedObjects.has(p.sparkUI))
|
|
318
310
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
311
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
312
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
319
313
|
if (p.connections != null)
|
|
320
314
|
for (const o of p.connections)
|
|
321
315
|
if (!visitedObjects.has(o))
|
|
322
316
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
323
317
|
if (!visitedObjects.has(p.continuousLogging))
|
|
324
318
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
325
|
-
if (!visitedObjects.has(p.workerType))
|
|
326
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
327
319
|
}
|
|
328
320
|
finally {
|
|
329
321
|
visitedObjects.delete(p);
|
|
@@ -343,14 +335,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
343
335
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
344
336
|
if (!visitedObjects.has(p.sparkUI))
|
|
345
337
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
338
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
339
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
346
340
|
if (p.connections != null)
|
|
347
341
|
for (const o of p.connections)
|
|
348
342
|
if (!visitedObjects.has(o))
|
|
349
343
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
350
344
|
if (!visitedObjects.has(p.continuousLogging))
|
|
351
345
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
352
|
-
if (!visitedObjects.has(p.workerType))
|
|
353
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
354
346
|
}
|
|
355
347
|
finally {
|
|
356
348
|
visitedObjects.delete(p);
|
|
@@ -370,14 +362,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
370
362
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
371
363
|
if (!visitedObjects.has(p.sparkUI))
|
|
372
364
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
365
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
366
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
373
367
|
if (p.connections != null)
|
|
374
368
|
for (const o of p.connections)
|
|
375
369
|
if (!visitedObjects.has(o))
|
|
376
370
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
377
371
|
if (!visitedObjects.has(p.continuousLogging))
|
|
378
372
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
379
|
-
if (!visitedObjects.has(p.workerType))
|
|
380
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
381
373
|
}
|
|
382
374
|
finally {
|
|
383
375
|
visitedObjects.delete(p);
|
package/README.md
CHANGED
|
@@ -75,7 +75,9 @@ similar constructors. ETL jobs default to the G2 worker type, but you can
|
|
|
75
75
|
override this default with other supported worker type values (G1, G2, G4
|
|
76
76
|
and G8). ETL jobs defaults to Glue version 4.0, which you can override to 3.0.
|
|
77
77
|
The following ETL features are enabled by default:
|
|
78
|
-
`—enable-metrics, —enable-
|
|
78
|
+
`—enable-metrics, —enable-continuous-cloudwatch-log.`
|
|
79
|
+
The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
|
|
80
|
+
`sparkUI` prop.
|
|
79
81
|
You can find more details about version, worker type and other features in
|
|
80
82
|
[Glue's public documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html).
|
|
81
83
|
|
|
@@ -115,7 +117,10 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
115
117
|
script,
|
|
116
118
|
glueVersion: glue.GlueVersion.V5_1,
|
|
117
119
|
continuousLogging: { enabled: false },
|
|
118
|
-
|
|
120
|
+
workerConfiguration: {
|
|
121
|
+
workerType: glue.WorkerType.G_2X,
|
|
122
|
+
numberOfWorkers: 2,
|
|
123
|
+
},
|
|
119
124
|
maxConcurrentRuns: 100,
|
|
120
125
|
timeout: cdk.Duration.hours(2),
|
|
121
126
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -125,7 +130,6 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
125
130
|
SecondTagName: 'SecondTagValue',
|
|
126
131
|
XTagName: 'XTagValue',
|
|
127
132
|
},
|
|
128
|
-
numberOfWorkers: 2,
|
|
129
133
|
maxRetries: 2,
|
|
130
134
|
});
|
|
131
135
|
```
|
|
@@ -135,11 +139,12 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
135
139
|
Streaming jobs are similar to ETL jobs, except that they perform ETL on data
|
|
136
140
|
streams using the Apache Spark Structured Streaming framework. Some Spark
|
|
137
141
|
job features are not available to Streaming ETL jobs. They support Scala
|
|
138
|
-
and pySpark languages. PySpark streaming jobs
|
|
139
|
-
which you can override with any non-deprecated version of Python. It
|
|
142
|
+
and pySpark languages. PySpark streaming jobs run on Python 3. It
|
|
140
143
|
defaults to the G2 worker type and Glue 4.0, both of which you can override.
|
|
141
144
|
The following best practice features are enabled by default:
|
|
142
|
-
`—enable-metrics, —enable-
|
|
145
|
+
`—enable-metrics, —enable-continuous-cloudwatch-log`.
|
|
146
|
+
The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
|
|
147
|
+
`sparkUI` prop.
|
|
143
148
|
|
|
144
149
|
Reference the pyspark-streaming-jobs.test.ts and scalaspark-streaming-jobs.test.ts
|
|
145
150
|
unit tests for examples of required-only and optional job parameters when creating
|
|
@@ -171,7 +176,10 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
|
|
|
171
176
|
script,
|
|
172
177
|
glueVersion: glue.GlueVersion.V5_1,
|
|
173
178
|
continuousLogging: { enabled: false },
|
|
174
|
-
|
|
179
|
+
workerConfiguration: {
|
|
180
|
+
workerType: glue.WorkerType.G_2X,
|
|
181
|
+
numberOfWorkers: 2,
|
|
182
|
+
},
|
|
175
183
|
maxConcurrentRuns: 100,
|
|
176
184
|
timeout: cdk.Duration.hours(2),
|
|
177
185
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -181,7 +189,6 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
|
|
|
181
189
|
SecondTagName: 'SecondTagValue',
|
|
182
190
|
XTagName: 'XTagValue',
|
|
183
191
|
},
|
|
184
|
-
numberOfWorkers: 2,
|
|
185
192
|
maxRetries: 2,
|
|
186
193
|
});
|
|
187
194
|
```
|
|
@@ -192,7 +199,9 @@ The flexible execution class is appropriate for non-urgent jobs such as
|
|
|
192
199
|
pre-production jobs, testing, and one-time data loads. Flexible jobs default
|
|
193
200
|
to Glue version 5.0 and worker type `G_2X`. The following best practice
|
|
194
201
|
features are enabled by default:
|
|
195
|
-
`—enable-metrics, —enable-
|
|
202
|
+
`—enable-metrics, —enable-continuous-cloudwatch-log`
|
|
203
|
+
The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
|
|
204
|
+
`sparkUI` prop.
|
|
196
205
|
|
|
197
206
|
Reference the pyspark-flex-etl-jobs.test.ts and scalaspark-flex-etl-jobs.test.ts
|
|
198
207
|
unit tests for examples of required-only and optional job parameters when creating
|
|
@@ -217,14 +226,17 @@ import * as iam from 'aws-cdk-lib/aws-iam';
|
|
|
217
226
|
declare const stack: cdk.Stack;
|
|
218
227
|
declare const role: iam.IRole;
|
|
219
228
|
declare const script: glue.Code;
|
|
220
|
-
new glue.
|
|
221
|
-
jobName: '
|
|
229
|
+
new glue.PySparkFlexEtlJob(stack, 'pySparkFlexEtlJob', {
|
|
230
|
+
jobName: 'pySparkFlexEtlJob',
|
|
222
231
|
description: 'This is a description',
|
|
223
232
|
role,
|
|
224
233
|
script,
|
|
225
234
|
glueVersion: glue.GlueVersion.V5_1,
|
|
226
235
|
continuousLogging: { enabled: false },
|
|
227
|
-
|
|
236
|
+
workerConfiguration: {
|
|
237
|
+
workerType: glue.WorkerType.G_2X,
|
|
238
|
+
numberOfWorkers: 2,
|
|
239
|
+
},
|
|
228
240
|
maxConcurrentRuns: 100,
|
|
229
241
|
timeout: cdk.Duration.hours(2),
|
|
230
242
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -234,7 +246,6 @@ new glue.PySparkEtlJob(stack, 'pySparkEtlJob', {
|
|
|
234
246
|
SecondTagName: 'SecondTagValue',
|
|
235
247
|
XTagName: 'XTagValue',
|
|
236
248
|
},
|
|
237
|
-
numberOfWorkers: 2,
|
|
238
249
|
maxRetries: 2,
|
|
239
250
|
});
|
|
240
251
|
```
|
|
@@ -274,14 +285,13 @@ declare const extraPythonFile: glue.Code;
|
|
|
274
285
|
new glue.PythonShellJob(stack, 'PythonShellJob', {
|
|
275
286
|
jobName: 'PythonShellJobCustomName',
|
|
276
287
|
description: 'This is a description',
|
|
277
|
-
pythonVersion: glue.PythonVersion.
|
|
288
|
+
pythonVersion: glue.PythonVersion.THREE_NINE,
|
|
278
289
|
maxCapacity: glue.MaxCapacity.DPU_1,
|
|
279
290
|
role,
|
|
280
291
|
script,
|
|
281
292
|
extraPythonFiles: [extraPythonFile],
|
|
282
|
-
glueVersion: glue.GlueVersion.
|
|
293
|
+
glueVersion: glue.GlueVersion.V3_0,
|
|
283
294
|
continuousLogging: { enabled: false },
|
|
284
|
-
workerType: glue.WorkerType.G_2X,
|
|
285
295
|
maxConcurrentRuns: 100,
|
|
286
296
|
timeout: cdk.Duration.hours(2),
|
|
287
297
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -291,7 +301,6 @@ new glue.PythonShellJob(stack, 'PythonShellJob', {
|
|
|
291
301
|
SecondTagName: 'SecondTagValue',
|
|
292
302
|
XTagName: 'XTagValue',
|
|
293
303
|
},
|
|
294
|
-
numberOfWorkers: 2,
|
|
295
304
|
maxRetries: 2,
|
|
296
305
|
});
|
|
297
306
|
```
|
|
@@ -355,10 +364,12 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
355
364
|
|
|
356
365
|
### Uploading scripts from the CDK app repository to S3
|
|
357
366
|
|
|
358
|
-
Similar to other L2 constructs, the Glue L2 automates uploading
|
|
359
|
-
scripts to S3
|
|
360
|
-
|
|
361
|
-
|
|
367
|
+
Similar to other L2 constructs, the Glue L2 automates uploading local
|
|
368
|
+
scripts to S3. Use `glue.Code.fromAsset(path)` to point at a script in your
|
|
369
|
+
local file structure; it is uploaded to the CDK-managed asset bucket. To
|
|
370
|
+
reference a script that already exists in S3, use
|
|
371
|
+
`glue.Code.fromBucket(bucket, key)`, which performs no upload. A `script` is
|
|
372
|
+
required for every job.
|
|
362
373
|
|
|
363
374
|
Reference the unit tests for examples of repo and S3 code target examples.
|
|
364
375
|
|
|
@@ -370,15 +381,34 @@ jobs, and triggers. Standalone triggers are an anti-pattern, so you must
|
|
|
370
381
|
create triggers from within a workflow using the L2 construct.
|
|
371
382
|
|
|
372
383
|
Within a workflow object, there are functions to create different
|
|
373
|
-
types of triggers with actions and predicates. You
|
|
374
|
-
|
|
384
|
+
types of triggers with actions and predicates. You add triggers to the
|
|
385
|
+
workflow, and each trigger references the jobs or crawlers it runs as its
|
|
386
|
+
actions.
|
|
375
387
|
|
|
376
|
-
|
|
377
|
-
|
|
388
|
+
`startOnCreation` applies to scheduled triggers (and, via
|
|
389
|
+
`ConditionalTriggerOptions`, conditional triggers) only. It defaults to `false`,
|
|
390
|
+
but you can override it if you prefer for your trigger to start on creation.
|
|
378
391
|
|
|
379
392
|
Reference the workflow-triggers.test.ts unit tests for examples of creating
|
|
380
393
|
workflows and triggers.
|
|
381
394
|
|
|
395
|
+
```ts
|
|
396
|
+
import * as cdk from 'aws-cdk-lib';
|
|
397
|
+
import * as iam from 'aws-cdk-lib/aws-iam';
|
|
398
|
+
declare const stack: cdk.Stack;
|
|
399
|
+
declare const role: iam.IRole;
|
|
400
|
+
declare const script: glue.Code;
|
|
401
|
+
|
|
402
|
+
// Create a job to run from the workflow
|
|
403
|
+
const job = new glue.PySparkEtlJob(stack, 'Job', { role, script });
|
|
404
|
+
|
|
405
|
+
// Create a workflow and add a trigger that runs the job
|
|
406
|
+
const workflow = new glue.Workflow(stack, 'Workflow');
|
|
407
|
+
workflow.addOnDemandTrigger('OnDemandTrigger', {
|
|
408
|
+
actions: [{ job }],
|
|
409
|
+
});
|
|
410
|
+
```
|
|
411
|
+
|
|
382
412
|
#### **1. On-Demand Triggers**
|
|
383
413
|
|
|
384
414
|
On-demand triggers can start glue jobs or crawlers. This construct provides
|
|
@@ -389,7 +419,7 @@ actions list using the job or crawler objects using conditional types.
|
|
|
389
419
|
#### **2. Scheduled Triggers**
|
|
390
420
|
|
|
391
421
|
You can create scheduled triggers using cron expressions. This construct
|
|
392
|
-
provides daily
|
|
422
|
+
provides daily and weekly convenience functions,
|
|
393
423
|
as well as a custom function that allows you to create your own
|
|
394
424
|
custom timing using the [existing event Schedule class](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_events.Schedule.html)
|
|
395
425
|
without having to build your own cron expressions. The L2 extracts
|
|
@@ -473,32 +503,29 @@ See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/la
|
|
|
473
503
|
|
|
474
504
|
A `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.
|
|
475
505
|
|
|
506
|
+
Each encryption config is built with a factory that pairs the encryption mode
|
|
507
|
+
with its key, so illegal combinations (such as an S3-managed encryption carrying
|
|
508
|
+
a KMS key) cannot be expressed:
|
|
509
|
+
|
|
476
510
|
```ts
|
|
477
511
|
new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
|
|
478
|
-
cloudWatchEncryption:
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
jobBookmarksEncryption: {
|
|
482
|
-
mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,
|
|
483
|
-
},
|
|
484
|
-
s3Encryption: {
|
|
485
|
-
mode: glue.S3EncryptionMode.KMS,
|
|
486
|
-
},
|
|
512
|
+
cloudWatchEncryption: glue.CloudWatchEncryption.kms(),
|
|
513
|
+
jobBookmarksEncryption: glue.JobBookmarksEncryption.clientSideKms(),
|
|
514
|
+
s3Encryption: glue.S3Encryption.kms(),
|
|
487
515
|
});
|
|
488
516
|
```
|
|
489
517
|
|
|
490
|
-
By default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key
|
|
518
|
+
By default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key to any factory, for example, for CloudWatch encryption:
|
|
491
519
|
|
|
492
520
|
```ts
|
|
493
521
|
declare const key: kms.Key;
|
|
494
522
|
new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
|
|
495
|
-
cloudWatchEncryption:
|
|
496
|
-
mode: glue.CloudWatchEncryptionMode.KMS,
|
|
497
|
-
kmsKey: key,
|
|
498
|
-
},
|
|
523
|
+
cloudWatchEncryption: glue.CloudWatchEncryption.kms(key),
|
|
499
524
|
});
|
|
500
525
|
```
|
|
501
526
|
|
|
527
|
+
Use `glue.S3Encryption.s3Managed()` for S3-managed (SSE-S3) encryption, which takes no key.
|
|
528
|
+
|
|
502
529
|
See [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.
|
|
503
530
|
|
|
504
531
|
## Catalog
|
|
@@ -560,7 +587,7 @@ new glue.Catalog(this, 'MyCatalog', {
|
|
|
560
587
|
### Encryption at rest
|
|
561
588
|
|
|
562
589
|
Configure Data Catalog encryption at rest through the `encryptionAtRest` option
|
|
563
|
-
(on `Catalog.encryptAccount
|
|
590
|
+
(on `Catalog.encryptAccount` or the `Catalog` constructor).
|
|
564
591
|
It accepts a `DataCatalogEncryptionAtRest` describing the mode:
|
|
565
592
|
|
|
566
593
|
```ts
|
|
@@ -814,7 +841,7 @@ new glue.S3Table(this, 'MyTable', {
|
|
|
814
841
|
Alternatively, you can call the `addPartitionIndex()` function on a table:
|
|
815
842
|
|
|
816
843
|
```ts
|
|
817
|
-
declare const myTable: glue.
|
|
844
|
+
declare const myTable: glue.S3Table;
|
|
818
845
|
myTable.addPartitionIndex({
|
|
819
846
|
indexName: 'my-index',
|
|
820
847
|
keyNames: ['year'],
|
|
@@ -1047,6 +1074,25 @@ new glue.ExternalTable(this, 'MyTable', {
|
|
|
1047
1074
|
});
|
|
1048
1075
|
```
|
|
1049
1076
|
|
|
1077
|
+
## Data Quality Ruleset
|
|
1078
|
+
|
|
1079
|
+
A `DataQualityRuleset` defines a set of data quality rules — authored in Glue's
|
|
1080
|
+
Data Quality Definition Language (DQDL) — that are evaluated against a table in
|
|
1081
|
+
the Data Catalog.
|
|
1082
|
+
|
|
1083
|
+
```ts
|
|
1084
|
+
new glue.DataQualityRuleset(this, 'MyRuleset', {
|
|
1085
|
+
rulesetName: 'my_ruleset',
|
|
1086
|
+
dqdl: glue.Dqdl.fromString('Rules = [ RowCount > 100, IsComplete "order_id" ]'),
|
|
1087
|
+
targetTable: new glue.DataQualityTargetTable('my_database', 'my_table'),
|
|
1088
|
+
});
|
|
1089
|
+
```
|
|
1090
|
+
|
|
1091
|
+
Build the DQDL document with `Dqdl.fromString(...)`. Glue parses and validates the
|
|
1092
|
+
DQDL when the ruleset is deployed; see the
|
|
1093
|
+
[DQDL reference](https://docs.aws.amazon.com/glue/latest/dg/dqdl.html) for the
|
|
1094
|
+
full rule syntax.
|
|
1095
|
+
|
|
1050
1096
|
## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)
|
|
1051
1097
|
|
|
1052
1098
|
When the table creates its own S3 bucket (i.e. you do not pass an explicit `bucket`), that bucket enforces SSL: a bucket policy denies any request made over plain HTTP. If you provide your own bucket, enabling `enforceSSL` on it is your responsibility.
|
|
@@ -1147,6 +1193,28 @@ new glue.S3Table(this, 'MyTable', {
|
|
|
1147
1193
|
|
|
1148
1194
|
*Note: you cannot provide a `Bucket` when creating the `S3Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.
|
|
1149
1195
|
|
|
1196
|
+
### Marking table data as encrypted
|
|
1197
|
+
|
|
1198
|
+
Both `S3Table` and `ExternalTable` set the `has_encrypted_data` table parameter, which
|
|
1199
|
+
Athena reads when querying client-side (`CSE-KMS`) encrypted datasets. It defaults to `true`.
|
|
1200
|
+
Set `hasEncryptedData` to `false` when the underlying data is not encrypted:
|
|
1201
|
+
|
|
1202
|
+
```ts
|
|
1203
|
+
declare const myDatabase: glue.Database;
|
|
1204
|
+
new glue.S3Table(this, 'MyTable', {
|
|
1205
|
+
hasEncryptedData: false,
|
|
1206
|
+
database: myDatabase,
|
|
1207
|
+
columns: [{
|
|
1208
|
+
name: 'col1',
|
|
1209
|
+
type: glue.Schema.STRING,
|
|
1210
|
+
}],
|
|
1211
|
+
dataFormat: glue.DataFormat.JSON,
|
|
1212
|
+
});
|
|
1213
|
+
```
|
|
1214
|
+
|
|
1215
|
+
Do not set `has_encrypted_data` through the free-form `parameters` map as well - a value
|
|
1216
|
+
there that conflicts with `hasEncryptedData` is rejected at synthesis time.
|
|
1217
|
+
|
|
1150
1218
|
## Types
|
|
1151
1219
|
|
|
1152
1220
|
A table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:
|
|
@@ -1166,7 +1234,7 @@ new glue.S3Table(this, 'MyTable', {
|
|
|
1166
1234
|
type: glue.Schema.map(
|
|
1167
1235
|
glue.Schema.STRING,
|
|
1168
1236
|
glue.Schema.TIMESTAMP),
|
|
1169
|
-
comment: 'map<string,
|
|
1237
|
+
comment: 'map<string,timestamp>',
|
|
1170
1238
|
}, {
|
|
1171
1239
|
name: 'struct_column',
|
|
1172
1240
|
type: glue.Schema.struct([{
|
package/awslint.json
CHANGED
|
@@ -109,6 +109,7 @@
|
|
|
109
109
|
"interface-extends-ref:@aws-cdk/aws-glue-alpha.ISecurityConfiguration",
|
|
110
110
|
"interface-extends-ref:@aws-cdk/aws-glue-alpha.ITable",
|
|
111
111
|
"interface-extends-ref:@aws-cdk/aws-glue-alpha.IWorkflow",
|
|
112
|
-
"prefer-ref-interface:@aws-cdk/aws-glue-alpha.DatabaseProps.catalog"
|
|
112
|
+
"prefer-ref-interface:@aws-cdk/aws-glue-alpha.DatabaseProps.catalog",
|
|
113
|
+
"no-unused-type:@aws-cdk/aws-glue-alpha.S3EncryptionMode"
|
|
113
114
|
]
|
|
114
115
|
}
|
package/lib/catalog.js
CHANGED
|
@@ -74,7 +74,7 @@ var CatalogEncryptionMode;
|
|
|
74
74
|
* @see https://docs.aws.amazon.com/glue/latest/webapi/API_EncryptionAtRest.html
|
|
75
75
|
*/
|
|
76
76
|
class DataCatalogEncryptionAtRest {
|
|
77
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.DataCatalogEncryptionAtRest", version: "2.
|
|
77
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.DataCatalogEncryptionAtRest", version: "2.266.0-alpha.0" };
|
|
78
78
|
/**
|
|
79
79
|
* Disable encryption at rest for the Data Catalog.
|
|
80
80
|
*/
|
|
@@ -130,7 +130,7 @@ exports.DataCatalogEncryptionAtRest = DataCatalogEncryptionAtRest;
|
|
|
130
130
|
* construction, so a catalog either carries settings or it does not.
|
|
131
131
|
*/
|
|
132
132
|
class CatalogBase extends core_1.Resource {
|
|
133
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.CatalogBase", version: "2.
|
|
133
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.CatalogBase", version: "2.266.0-alpha.0" };
|
|
134
134
|
_encryptionKey;
|
|
135
135
|
_connectionPasswordKey;
|
|
136
136
|
get encryptionKey() {
|
|
@@ -224,7 +224,7 @@ let Catalog = (() => {
|
|
|
224
224
|
Catalog = _classThis = _classDescriptor.value;
|
|
225
225
|
if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
|
|
226
226
|
}
|
|
227
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Catalog", version: "2.
|
|
227
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Catalog", version: "2.266.0-alpha.0" };
|
|
228
228
|
/** Uniquely identifies this class. */
|
|
229
229
|
static PROPERTY_INJECTION_ID = '@aws-cdk.aws-glue-alpha.Catalog';
|
|
230
230
|
/**
|
package/lib/code.js
CHANGED
|
@@ -43,7 +43,7 @@ const helpers_internal_1 = require("aws-cdk-lib/core/lib/helpers-internal");
|
|
|
43
43
|
* Represents a Glue Job's Code assets (an asset can be a scripts, a jar, a python file or any other file).
|
|
44
44
|
*/
|
|
45
45
|
class Code {
|
|
46
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Code", version: "2.
|
|
46
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Code", version: "2.266.0-alpha.0" };
|
|
47
47
|
/**
|
|
48
48
|
* Job code as an S3 object.
|
|
49
49
|
* @param bucket The S3 bucket
|
|
@@ -68,7 +68,7 @@ exports.Code = Code;
|
|
|
68
68
|
class S3Code extends Code {
|
|
69
69
|
bucket;
|
|
70
70
|
key;
|
|
71
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.S3Code", version: "2.
|
|
71
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.S3Code", version: "2.266.0-alpha.0" };
|
|
72
72
|
constructor(bucket, key) {
|
|
73
73
|
super();
|
|
74
74
|
this.bucket = bucket;
|
|
@@ -91,7 +91,7 @@ exports.S3Code = S3Code;
|
|
|
91
91
|
class AssetCode extends Code {
|
|
92
92
|
path;
|
|
93
93
|
options;
|
|
94
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.AssetCode", version: "2.
|
|
94
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.AssetCode", version: "2.266.0-alpha.0" };
|
|
95
95
|
asset;
|
|
96
96
|
/**
|
|
97
97
|
* @param path The path to the Code file.
|
package/lib/connection.js
CHANGED
|
@@ -85,7 +85,7 @@ const secret_detection_1 = require("./private/secret-detection");
|
|
|
85
85
|
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-connection-connectioninput.html#cfn-glue-connection-connectioninput-connectiontype
|
|
86
86
|
*/
|
|
87
87
|
class ConnectionType {
|
|
88
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.ConnectionType", version: "2.
|
|
88
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.ConnectionType", version: "2.266.0-alpha.0" };
|
|
89
89
|
/**
|
|
90
90
|
* Designates a connection to a database through Java Database Connectivity (JDBC).
|
|
91
91
|
*/
|
|
@@ -290,7 +290,7 @@ let Connection = (() => {
|
|
|
290
290
|
Connection = _classThis = _classDescriptor.value;
|
|
291
291
|
if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
|
|
292
292
|
}
|
|
293
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Connection", version: "2.
|
|
293
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Connection", version: "2.266.0-alpha.0" };
|
|
294
294
|
/** Uniquely identifies this class. */
|
|
295
295
|
static PROPERTY_INJECTION_ID = '@aws-cdk.aws-glue-alpha.Connection';
|
|
296
296
|
/**
|
package/lib/data-format.d.ts
CHANGED
|
@@ -54,13 +54,13 @@ export declare class OutputFormat {
|
|
|
54
54
|
*
|
|
55
55
|
* @see https://svn.apache.org/repos/infra/websites/production/hive/content/javadocs/r3.1.3/api/org/apache/hadoop/hive/ql/io/avro/AvroContainerOutputFormat.html
|
|
56
56
|
*/
|
|
57
|
-
static readonly AVRO:
|
|
57
|
+
static readonly AVRO: OutputFormat;
|
|
58
58
|
/**
|
|
59
59
|
* OutputFormat for Orc files.
|
|
60
60
|
*
|
|
61
61
|
* @see https://svn.apache.org/repos/infra/websites/production/hive/content/javadocs/r3.1.3/api/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.html
|
|
62
62
|
*/
|
|
63
|
-
static readonly ORC:
|
|
63
|
+
static readonly ORC: OutputFormat;
|
|
64
64
|
/**
|
|
65
65
|
* OutputFormat for Parquet files.
|
|
66
66
|
*
|