@aws-cdk/aws-glue-alpha 2.264.0-alpha.0 → 2.266.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +689 -552
- package/.jsii.tabl.json.gz +0 -0
- package/.warnings.jsii.js +25 -33
- package/README.md +132 -44
- package/awslint.json +2 -1
- package/lib/catalog.js +3 -3
- package/lib/code.js +3 -3
- package/lib/connection.js +12 -4
- package/lib/data-format.d.ts +2 -2
- package/lib/data-format.js +8 -8
- package/lib/data-quality-ruleset.d.ts +35 -4
- package/lib/data-quality-ruleset.js +38 -5
- package/lib/database.d.ts +11 -2
- package/lib/database.js +5 -2
- package/lib/external-table.js +6 -4
- package/lib/jobs/job.d.ts +6 -24
- package/lib/jobs/job.js +5 -3
- package/lib/jobs/pyspark-etl-job.d.ts +2 -1
- package/lib/jobs/pyspark-etl-job.js +10 -5
- package/lib/jobs/pyspark-flex-etl-job.d.ts +3 -2
- package/lib/jobs/pyspark-flex-etl-job.js +8 -7
- package/lib/jobs/pyspark-streaming-job.d.ts +2 -1
- package/lib/jobs/pyspark-streaming-job.js +6 -5
- package/lib/jobs/python-shell-job.js +1 -1
- package/lib/jobs/ray-job.d.ts +8 -0
- package/lib/jobs/ray-job.js +3 -7
- package/lib/jobs/scala-spark-etl-job.js +4 -8
- package/lib/jobs/scala-spark-flex-etl-job.js +5 -5
- package/lib/jobs/scala-spark-streaming-job.js +4 -8
- package/lib/jobs/spark-job.d.ts +26 -0
- package/lib/jobs/spark-job.js +2 -2
- package/lib/partition-projection.js +1 -1
- package/lib/private/secret-detection.d.ts +22 -0
- package/lib/private/secret-detection.js +42 -0
- package/lib/s3-table.d.ts +12 -1
- package/lib/s3-table.js +20 -3
- package/lib/schema.js +3 -8
- package/lib/security-configuration.d.ts +47 -47
- package/lib/security-configuration.js +84 -25
- package/lib/storage-parameter.js +1 -1
- package/lib/table-base.d.ts +56 -3
- package/lib/table-base.js +56 -5
- package/lib/table-deprecated.js +1 -1
- package/lib/triggers/trigger-options.js +1 -1
- package/lib/triggers/workflow.js +2 -2
- package/package.json +7 -7
package/.jsii.tabl.json.gz
CHANGED
|
Binary file
|
package/.warnings.jsii.js
CHANGED
|
@@ -75,21 +75,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
75
75
|
finally {
|
|
76
76
|
visitedObjects.delete(p);
|
|
77
77
|
}
|
|
78
|
-
}, _aws_cdk_aws_glue_alpha_SecurityConfigurationProps: function _aws_cdk_aws_glue_alpha_SecurityConfigurationProps(p) {
|
|
79
|
-
if (p == null)
|
|
80
|
-
return;
|
|
81
|
-
visitedObjects.add(p);
|
|
82
|
-
try {
|
|
83
|
-
if (!visitedObjects.has(p.cloudWatchEncryption))
|
|
84
|
-
module.exports._aws_cdk_aws_glue_alpha_CloudWatchEncryption(p.cloudWatchEncryption);
|
|
85
|
-
if (!visitedObjects.has(p.jobBookmarksEncryption))
|
|
86
|
-
module.exports._aws_cdk_aws_glue_alpha_JobBookmarksEncryption(p.jobBookmarksEncryption);
|
|
87
|
-
if (!visitedObjects.has(p.s3Encryption))
|
|
88
|
-
module.exports._aws_cdk_aws_glue_alpha_S3Encryption(p.s3Encryption);
|
|
89
|
-
}
|
|
90
|
-
finally {
|
|
91
|
-
visitedObjects.delete(p);
|
|
92
|
-
}
|
|
93
78
|
}, _aws_cdk_aws_glue_alpha_WorkerType: function _aws_cdk_aws_glue_alpha_WorkerType(p) {
|
|
94
79
|
if (p == null)
|
|
95
80
|
return;
|
|
@@ -135,6 +120,15 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
135
120
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
136
121
|
if (!visitedObjects.has(p.continuousLogging))
|
|
137
122
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
visitedObjects.delete(p);
|
|
126
|
+
}
|
|
127
|
+
}, _aws_cdk_aws_glue_alpha_WorkerConfiguration: function _aws_cdk_aws_glue_alpha_WorkerConfiguration(p) {
|
|
128
|
+
if (p == null)
|
|
129
|
+
return;
|
|
130
|
+
visitedObjects.add(p);
|
|
131
|
+
try {
|
|
138
132
|
if (!visitedObjects.has(p.workerType))
|
|
139
133
|
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
140
134
|
}
|
|
@@ -148,14 +142,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
148
142
|
try {
|
|
149
143
|
if (!visitedObjects.has(p.sparkUI))
|
|
150
144
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
145
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
146
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
151
147
|
if (p.connections != null)
|
|
152
148
|
for (const o of p.connections)
|
|
153
149
|
if (!visitedObjects.has(o))
|
|
154
150
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
155
151
|
if (!visitedObjects.has(p.continuousLogging))
|
|
156
152
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
157
|
-
if (!visitedObjects.has(p.workerType))
|
|
158
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
159
153
|
}
|
|
160
154
|
finally {
|
|
161
155
|
visitedObjects.delete(p);
|
|
@@ -179,14 +173,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
179
173
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
180
174
|
if (!visitedObjects.has(p.sparkUI))
|
|
181
175
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
176
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
177
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
182
178
|
if (p.connections != null)
|
|
183
179
|
for (const o of p.connections)
|
|
184
180
|
if (!visitedObjects.has(o))
|
|
185
181
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
186
182
|
if (!visitedObjects.has(p.continuousLogging))
|
|
187
183
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
188
|
-
if (!visitedObjects.has(p.workerType))
|
|
189
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
190
184
|
}
|
|
191
185
|
finally {
|
|
192
186
|
visitedObjects.delete(p);
|
|
@@ -210,14 +204,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
210
204
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
211
205
|
if (!visitedObjects.has(p.sparkUI))
|
|
212
206
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
207
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
208
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
213
209
|
if (p.connections != null)
|
|
214
210
|
for (const o of p.connections)
|
|
215
211
|
if (!visitedObjects.has(o))
|
|
216
212
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
217
213
|
if (!visitedObjects.has(p.continuousLogging))
|
|
218
214
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
219
|
-
if (!visitedObjects.has(p.workerType))
|
|
220
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
221
215
|
}
|
|
222
216
|
finally {
|
|
223
217
|
visitedObjects.delete(p);
|
|
@@ -241,14 +235,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
241
235
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
242
236
|
if (!visitedObjects.has(p.sparkUI))
|
|
243
237
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
238
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
239
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
244
240
|
if (p.connections != null)
|
|
245
241
|
for (const o of p.connections)
|
|
246
242
|
if (!visitedObjects.has(o))
|
|
247
243
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
248
244
|
if (!visitedObjects.has(p.continuousLogging))
|
|
249
245
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
250
|
-
if (!visitedObjects.has(p.workerType))
|
|
251
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
252
246
|
}
|
|
253
247
|
finally {
|
|
254
248
|
visitedObjects.delete(p);
|
|
@@ -268,8 +262,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
268
262
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
269
263
|
if (!visitedObjects.has(p.continuousLogging))
|
|
270
264
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
271
|
-
if (!visitedObjects.has(p.workerType))
|
|
272
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
273
265
|
}
|
|
274
266
|
finally {
|
|
275
267
|
visitedObjects.delete(p);
|
|
@@ -285,6 +277,8 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
285
277
|
print("@aws-cdk/aws-glue-alpha.RayJobProps#enableObservabilityMetrics", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
286
278
|
if ("jobRunQueuingEnabled" in p)
|
|
287
279
|
print("@aws-cdk/aws-glue-alpha.RayJobProps#jobRunQueuingEnabled", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
280
|
+
if ("numberOfWorkers" in p)
|
|
281
|
+
print("@aws-cdk/aws-glue-alpha.RayJobProps#numberOfWorkers", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
288
282
|
if ("runtime" in p)
|
|
289
283
|
print("@aws-cdk/aws-glue-alpha.RayJobProps#runtime", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
|
|
290
284
|
if (!visitedObjects.has(p.runtime))
|
|
@@ -295,8 +289,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
295
289
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
296
290
|
if (!visitedObjects.has(p.continuousLogging))
|
|
297
291
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
298
|
-
if (!visitedObjects.has(p.workerType))
|
|
299
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
300
292
|
}
|
|
301
293
|
finally {
|
|
302
294
|
visitedObjects.delete(p);
|
|
@@ -316,14 +308,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
316
308
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
317
309
|
if (!visitedObjects.has(p.sparkUI))
|
|
318
310
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
311
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
312
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
319
313
|
if (p.connections != null)
|
|
320
314
|
for (const o of p.connections)
|
|
321
315
|
if (!visitedObjects.has(o))
|
|
322
316
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
323
317
|
if (!visitedObjects.has(p.continuousLogging))
|
|
324
318
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
325
|
-
if (!visitedObjects.has(p.workerType))
|
|
326
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
327
319
|
}
|
|
328
320
|
finally {
|
|
329
321
|
visitedObjects.delete(p);
|
|
@@ -343,14 +335,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
343
335
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
344
336
|
if (!visitedObjects.has(p.sparkUI))
|
|
345
337
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
338
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
339
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
346
340
|
if (p.connections != null)
|
|
347
341
|
for (const o of p.connections)
|
|
348
342
|
if (!visitedObjects.has(o))
|
|
349
343
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
350
344
|
if (!visitedObjects.has(p.continuousLogging))
|
|
351
345
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
352
|
-
if (!visitedObjects.has(p.workerType))
|
|
353
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
354
346
|
}
|
|
355
347
|
finally {
|
|
356
348
|
visitedObjects.delete(p);
|
|
@@ -370,14 +362,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
|
|
|
370
362
|
module.exports._aws_cdk_aws_glue_alpha_Code(o);
|
|
371
363
|
if (!visitedObjects.has(p.sparkUI))
|
|
372
364
|
module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
365
|
+
if (!visitedObjects.has(p.workerConfiguration))
|
|
366
|
+
module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
|
|
373
367
|
if (p.connections != null)
|
|
374
368
|
for (const o of p.connections)
|
|
375
369
|
if (!visitedObjects.has(o))
|
|
376
370
|
module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
|
|
377
371
|
if (!visitedObjects.has(p.continuousLogging))
|
|
378
372
|
module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
379
|
-
if (!visitedObjects.has(p.workerType))
|
|
380
|
-
module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
381
373
|
}
|
|
382
374
|
finally {
|
|
383
375
|
visitedObjects.delete(p);
|
package/README.md
CHANGED
|
@@ -75,7 +75,9 @@ similar constructors. ETL jobs default to the G2 worker type, but you can
|
|
|
75
75
|
override this default with other supported worker type values (G1, G2, G4
|
|
76
76
|
and G8). ETL jobs defaults to Glue version 4.0, which you can override to 3.0.
|
|
77
77
|
The following ETL features are enabled by default:
|
|
78
|
-
`—enable-metrics, —enable-
|
|
78
|
+
`—enable-metrics, —enable-continuous-cloudwatch-log.`
|
|
79
|
+
The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
|
|
80
|
+
`sparkUI` prop.
|
|
79
81
|
You can find more details about version, worker type and other features in
|
|
80
82
|
[Glue's public documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html).
|
|
81
83
|
|
|
@@ -115,7 +117,10 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
115
117
|
script,
|
|
116
118
|
glueVersion: glue.GlueVersion.V5_1,
|
|
117
119
|
continuousLogging: { enabled: false },
|
|
118
|
-
|
|
120
|
+
workerConfiguration: {
|
|
121
|
+
workerType: glue.WorkerType.G_2X,
|
|
122
|
+
numberOfWorkers: 2,
|
|
123
|
+
},
|
|
119
124
|
maxConcurrentRuns: 100,
|
|
120
125
|
timeout: cdk.Duration.hours(2),
|
|
121
126
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -125,7 +130,6 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
125
130
|
SecondTagName: 'SecondTagValue',
|
|
126
131
|
XTagName: 'XTagValue',
|
|
127
132
|
},
|
|
128
|
-
numberOfWorkers: 2,
|
|
129
133
|
maxRetries: 2,
|
|
130
134
|
});
|
|
131
135
|
```
|
|
@@ -135,11 +139,12 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
135
139
|
Streaming jobs are similar to ETL jobs, except that they perform ETL on data
|
|
136
140
|
streams using the Apache Spark Structured Streaming framework. Some Spark
|
|
137
141
|
job features are not available to Streaming ETL jobs. They support Scala
|
|
138
|
-
and pySpark languages. PySpark streaming jobs
|
|
139
|
-
which you can override with any non-deprecated version of Python. It
|
|
142
|
+
and pySpark languages. PySpark streaming jobs run on Python 3. It
|
|
140
143
|
defaults to the G2 worker type and Glue 4.0, both of which you can override.
|
|
141
144
|
The following best practice features are enabled by default:
|
|
142
|
-
`—enable-metrics, —enable-
|
|
145
|
+
`—enable-metrics, —enable-continuous-cloudwatch-log`.
|
|
146
|
+
The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
|
|
147
|
+
`sparkUI` prop.
|
|
143
148
|
|
|
144
149
|
Reference the pyspark-streaming-jobs.test.ts and scalaspark-streaming-jobs.test.ts
|
|
145
150
|
unit tests for examples of required-only and optional job parameters when creating
|
|
@@ -171,7 +176,10 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
|
|
|
171
176
|
script,
|
|
172
177
|
glueVersion: glue.GlueVersion.V5_1,
|
|
173
178
|
continuousLogging: { enabled: false },
|
|
174
|
-
|
|
179
|
+
workerConfiguration: {
|
|
180
|
+
workerType: glue.WorkerType.G_2X,
|
|
181
|
+
numberOfWorkers: 2,
|
|
182
|
+
},
|
|
175
183
|
maxConcurrentRuns: 100,
|
|
176
184
|
timeout: cdk.Duration.hours(2),
|
|
177
185
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -181,7 +189,6 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
|
|
|
181
189
|
SecondTagName: 'SecondTagValue',
|
|
182
190
|
XTagName: 'XTagValue',
|
|
183
191
|
},
|
|
184
|
-
numberOfWorkers: 2,
|
|
185
192
|
maxRetries: 2,
|
|
186
193
|
});
|
|
187
194
|
```
|
|
@@ -190,9 +197,11 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
|
|
|
190
197
|
|
|
191
198
|
The flexible execution class is appropriate for non-urgent jobs such as
|
|
192
199
|
pre-production jobs, testing, and one-time data loads. Flexible jobs default
|
|
193
|
-
to Glue version
|
|
200
|
+
to Glue version 5.0 and worker type `G_2X`. The following best practice
|
|
194
201
|
features are enabled by default:
|
|
195
|
-
`—enable-metrics, —enable-
|
|
202
|
+
`—enable-metrics, —enable-continuous-cloudwatch-log`
|
|
203
|
+
The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
|
|
204
|
+
`sparkUI` prop.
|
|
196
205
|
|
|
197
206
|
Reference the pyspark-flex-etl-jobs.test.ts and scalaspark-flex-etl-jobs.test.ts
|
|
198
207
|
unit tests for examples of required-only and optional job parameters when creating
|
|
@@ -217,14 +226,17 @@ import * as iam from 'aws-cdk-lib/aws-iam';
|
|
|
217
226
|
declare const stack: cdk.Stack;
|
|
218
227
|
declare const role: iam.IRole;
|
|
219
228
|
declare const script: glue.Code;
|
|
220
|
-
new glue.
|
|
221
|
-
jobName: '
|
|
229
|
+
new glue.PySparkFlexEtlJob(stack, 'pySparkFlexEtlJob', {
|
|
230
|
+
jobName: 'pySparkFlexEtlJob',
|
|
222
231
|
description: 'This is a description',
|
|
223
232
|
role,
|
|
224
233
|
script,
|
|
225
234
|
glueVersion: glue.GlueVersion.V5_1,
|
|
226
235
|
continuousLogging: { enabled: false },
|
|
227
|
-
|
|
236
|
+
workerConfiguration: {
|
|
237
|
+
workerType: glue.WorkerType.G_2X,
|
|
238
|
+
numberOfWorkers: 2,
|
|
239
|
+
},
|
|
228
240
|
maxConcurrentRuns: 100,
|
|
229
241
|
timeout: cdk.Duration.hours(2),
|
|
230
242
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -234,7 +246,6 @@ new glue.PySparkEtlJob(stack, 'pySparkEtlJob', {
|
|
|
234
246
|
SecondTagName: 'SecondTagValue',
|
|
235
247
|
XTagName: 'XTagValue',
|
|
236
248
|
},
|
|
237
|
-
numberOfWorkers: 2,
|
|
238
249
|
maxRetries: 2,
|
|
239
250
|
});
|
|
240
251
|
```
|
|
@@ -274,14 +285,13 @@ declare const extraPythonFile: glue.Code;
|
|
|
274
285
|
new glue.PythonShellJob(stack, 'PythonShellJob', {
|
|
275
286
|
jobName: 'PythonShellJobCustomName',
|
|
276
287
|
description: 'This is a description',
|
|
277
|
-
pythonVersion: glue.PythonVersion.
|
|
288
|
+
pythonVersion: glue.PythonVersion.THREE_NINE,
|
|
278
289
|
maxCapacity: glue.MaxCapacity.DPU_1,
|
|
279
290
|
role,
|
|
280
291
|
script,
|
|
281
292
|
extraPythonFiles: [extraPythonFile],
|
|
282
|
-
glueVersion: glue.GlueVersion.
|
|
293
|
+
glueVersion: glue.GlueVersion.V3_0,
|
|
283
294
|
continuousLogging: { enabled: false },
|
|
284
|
-
workerType: glue.WorkerType.G_2X,
|
|
285
295
|
maxConcurrentRuns: 100,
|
|
286
296
|
timeout: cdk.Duration.hours(2),
|
|
287
297
|
connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
|
|
@@ -291,7 +301,6 @@ new glue.PythonShellJob(stack, 'PythonShellJob', {
|
|
|
291
301
|
SecondTagName: 'SecondTagValue',
|
|
292
302
|
XTagName: 'XTagValue',
|
|
293
303
|
},
|
|
294
|
-
numberOfWorkers: 2,
|
|
295
304
|
maxRetries: 2,
|
|
296
305
|
});
|
|
297
306
|
```
|
|
@@ -355,10 +364,12 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
|
|
|
355
364
|
|
|
356
365
|
### Uploading scripts from the CDK app repository to S3
|
|
357
366
|
|
|
358
|
-
Similar to other L2 constructs, the Glue L2 automates uploading
|
|
359
|
-
scripts to S3
|
|
360
|
-
|
|
361
|
-
|
|
367
|
+
Similar to other L2 constructs, the Glue L2 automates uploading local
|
|
368
|
+
scripts to S3. Use `glue.Code.fromAsset(path)` to point at a script in your
|
|
369
|
+
local file structure; it is uploaded to the CDK-managed asset bucket. To
|
|
370
|
+
reference a script that already exists in S3, use
|
|
371
|
+
`glue.Code.fromBucket(bucket, key)`, which performs no upload. A `script` is
|
|
372
|
+
required for every job.
|
|
362
373
|
|
|
363
374
|
Reference the unit tests for examples of repo and S3 code target examples.
|
|
364
375
|
|
|
@@ -370,15 +381,34 @@ jobs, and triggers. Standalone triggers are an anti-pattern, so you must
|
|
|
370
381
|
create triggers from within a workflow using the L2 construct.
|
|
371
382
|
|
|
372
383
|
Within a workflow object, there are functions to create different
|
|
373
|
-
types of triggers with actions and predicates. You
|
|
374
|
-
|
|
384
|
+
types of triggers with actions and predicates. You add triggers to the
|
|
385
|
+
workflow, and each trigger references the jobs or crawlers it runs as its
|
|
386
|
+
actions.
|
|
375
387
|
|
|
376
|
-
|
|
377
|
-
|
|
388
|
+
`startOnCreation` applies to scheduled triggers (and, via
|
|
389
|
+
`ConditionalTriggerOptions`, conditional triggers) only. It defaults to `false`,
|
|
390
|
+
but you can override it if you prefer for your trigger to start on creation.
|
|
378
391
|
|
|
379
392
|
Reference the workflow-triggers.test.ts unit tests for examples of creating
|
|
380
393
|
workflows and triggers.
|
|
381
394
|
|
|
395
|
+
```ts
|
|
396
|
+
import * as cdk from 'aws-cdk-lib';
|
|
397
|
+
import * as iam from 'aws-cdk-lib/aws-iam';
|
|
398
|
+
declare const stack: cdk.Stack;
|
|
399
|
+
declare const role: iam.IRole;
|
|
400
|
+
declare const script: glue.Code;
|
|
401
|
+
|
|
402
|
+
// Create a job to run from the workflow
|
|
403
|
+
const job = new glue.PySparkEtlJob(stack, 'Job', { role, script });
|
|
404
|
+
|
|
405
|
+
// Create a workflow and add a trigger that runs the job
|
|
406
|
+
const workflow = new glue.Workflow(stack, 'Workflow');
|
|
407
|
+
workflow.addOnDemandTrigger('OnDemandTrigger', {
|
|
408
|
+
actions: [{ job }],
|
|
409
|
+
});
|
|
410
|
+
```
|
|
411
|
+
|
|
382
412
|
#### **1. On-Demand Triggers**
|
|
383
413
|
|
|
384
414
|
On-demand triggers can start glue jobs or crawlers. This construct provides
|
|
@@ -389,7 +419,7 @@ actions list using the job or crawler objects using conditional types.
|
|
|
389
419
|
#### **2. Scheduled Triggers**
|
|
390
420
|
|
|
391
421
|
You can create scheduled triggers using cron expressions. This construct
|
|
392
|
-
provides daily
|
|
422
|
+
provides daily and weekly convenience functions,
|
|
393
423
|
as well as a custom function that allows you to create your own
|
|
394
424
|
custom timing using the [existing event Schedule class](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_events.Schedule.html)
|
|
395
425
|
without having to build your own cron expressions. The L2 extracts
|
|
@@ -458,6 +488,13 @@ new glue.Connection(this, "RdsConnection", {
|
|
|
458
488
|
});
|
|
459
489
|
```
|
|
460
490
|
|
|
491
|
+
Connection `properties` are emitted verbatim into the CloudFormation template, so
|
|
492
|
+
any credential placed there in plaintext is stored in plaintext in the template,
|
|
493
|
+
`cdk.out`, and source control. Reference a Secrets Manager secret through
|
|
494
|
+
`SECRET_ID` (as above) instead. If a property key looks like a credential (for
|
|
495
|
+
example `PASSWORD`, `SECRET`, or `TOKEN`) and holds a plaintext literal, the
|
|
496
|
+
construct emits a synthesis-time warning.
|
|
497
|
+
|
|
461
498
|
If you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.
|
|
462
499
|
|
|
463
500
|
See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.
|
|
@@ -466,32 +503,29 @@ See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/la
|
|
|
466
503
|
|
|
467
504
|
A `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.
|
|
468
505
|
|
|
506
|
+
Each encryption config is built with a factory that pairs the encryption mode
|
|
507
|
+
with its key, so illegal combinations (such as an S3-managed encryption carrying
|
|
508
|
+
a KMS key) cannot be expressed:
|
|
509
|
+
|
|
469
510
|
```ts
|
|
470
511
|
new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
|
|
471
|
-
cloudWatchEncryption:
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
jobBookmarksEncryption: {
|
|
475
|
-
mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,
|
|
476
|
-
},
|
|
477
|
-
s3Encryption: {
|
|
478
|
-
mode: glue.S3EncryptionMode.KMS,
|
|
479
|
-
},
|
|
512
|
+
cloudWatchEncryption: glue.CloudWatchEncryption.kms(),
|
|
513
|
+
jobBookmarksEncryption: glue.JobBookmarksEncryption.clientSideKms(),
|
|
514
|
+
s3Encryption: glue.S3Encryption.kms(),
|
|
480
515
|
});
|
|
481
516
|
```
|
|
482
517
|
|
|
483
|
-
By default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key
|
|
518
|
+
By default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key to any factory, for example, for CloudWatch encryption:
|
|
484
519
|
|
|
485
520
|
```ts
|
|
486
521
|
declare const key: kms.Key;
|
|
487
522
|
new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
|
|
488
|
-
cloudWatchEncryption:
|
|
489
|
-
mode: glue.CloudWatchEncryptionMode.KMS,
|
|
490
|
-
kmsKey: key,
|
|
491
|
-
},
|
|
523
|
+
cloudWatchEncryption: glue.CloudWatchEncryption.kms(key),
|
|
492
524
|
});
|
|
493
525
|
```
|
|
494
526
|
|
|
527
|
+
Use `glue.S3Encryption.s3Managed()` for S3-managed (SSE-S3) encryption, which takes no key.
|
|
528
|
+
|
|
495
529
|
See [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.
|
|
496
530
|
|
|
497
531
|
## Catalog
|
|
@@ -553,7 +587,7 @@ new glue.Catalog(this, 'MyCatalog', {
|
|
|
553
587
|
### Encryption at rest
|
|
554
588
|
|
|
555
589
|
Configure Data Catalog encryption at rest through the `encryptionAtRest` option
|
|
556
|
-
(on `Catalog.encryptAccount
|
|
590
|
+
(on `Catalog.encryptAccount` or the `Catalog` constructor).
|
|
557
591
|
It accepts a `DataCatalogEncryptionAtRest` describing the mode:
|
|
558
592
|
|
|
559
593
|
```ts
|
|
@@ -657,6 +691,19 @@ new glue.Database(this, 'MyDatabase', {
|
|
|
657
691
|
});
|
|
658
692
|
```
|
|
659
693
|
|
|
694
|
+
Because a database is a container for tables and their metadata, it is retained
|
|
695
|
+
by default when removed from the stack, to avoid accidental data loss. Set
|
|
696
|
+
`removalPolicy` to `RemovalPolicy.DESTROY` to have it deleted instead:
|
|
697
|
+
|
|
698
|
+
```ts
|
|
699
|
+
import { RemovalPolicy } from 'aws-cdk-lib';
|
|
700
|
+
|
|
701
|
+
new glue.Database(this, 'MyDatabase', {
|
|
702
|
+
databaseName: 'my_database',
|
|
703
|
+
removalPolicy: RemovalPolicy.DESTROY,
|
|
704
|
+
});
|
|
705
|
+
```
|
|
706
|
+
|
|
660
707
|
## Table
|
|
661
708
|
|
|
662
709
|
A Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):
|
|
@@ -794,7 +841,7 @@ new glue.S3Table(this, 'MyTable', {
|
|
|
794
841
|
Alternatively, you can call the `addPartitionIndex()` function on a table:
|
|
795
842
|
|
|
796
843
|
```ts
|
|
797
|
-
declare const myTable: glue.
|
|
844
|
+
declare const myTable: glue.S3Table;
|
|
798
845
|
myTable.addPartitionIndex({
|
|
799
846
|
indexName: 'my-index',
|
|
800
847
|
keyNames: ['year'],
|
|
@@ -1027,6 +1074,25 @@ new glue.ExternalTable(this, 'MyTable', {
|
|
|
1027
1074
|
});
|
|
1028
1075
|
```
|
|
1029
1076
|
|
|
1077
|
+
## Data Quality Ruleset
|
|
1078
|
+
|
|
1079
|
+
A `DataQualityRuleset` defines a set of data quality rules — authored in Glue's
|
|
1080
|
+
Data Quality Definition Language (DQDL) — that are evaluated against a table in
|
|
1081
|
+
the Data Catalog.
|
|
1082
|
+
|
|
1083
|
+
```ts
|
|
1084
|
+
new glue.DataQualityRuleset(this, 'MyRuleset', {
|
|
1085
|
+
rulesetName: 'my_ruleset',
|
|
1086
|
+
dqdl: glue.Dqdl.fromString('Rules = [ RowCount > 100, IsComplete "order_id" ]'),
|
|
1087
|
+
targetTable: new glue.DataQualityTargetTable('my_database', 'my_table'),
|
|
1088
|
+
});
|
|
1089
|
+
```
|
|
1090
|
+
|
|
1091
|
+
Build the DQDL document with `Dqdl.fromString(...)`. Glue parses and validates the
|
|
1092
|
+
DQDL when the ruleset is deployed; see the
|
|
1093
|
+
[DQDL reference](https://docs.aws.amazon.com/glue/latest/dg/dqdl.html) for the
|
|
1094
|
+
full rule syntax.
|
|
1095
|
+
|
|
1030
1096
|
## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)
|
|
1031
1097
|
|
|
1032
1098
|
When the table creates its own S3 bucket (i.e. you do not pass an explicit `bucket`), that bucket enforces SSL: a bucket policy denies any request made over plain HTTP. If you provide your own bucket, enabling `enforceSSL` on it is your responsibility.
|
|
@@ -1127,6 +1193,28 @@ new glue.S3Table(this, 'MyTable', {
|
|
|
1127
1193
|
|
|
1128
1194
|
*Note: you cannot provide a `Bucket` when creating the `S3Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.
|
|
1129
1195
|
|
|
1196
|
+
### Marking table data as encrypted
|
|
1197
|
+
|
|
1198
|
+
Both `S3Table` and `ExternalTable` set the `has_encrypted_data` table parameter, which
|
|
1199
|
+
Athena reads when querying client-side (`CSE-KMS`) encrypted datasets. It defaults to `true`.
|
|
1200
|
+
Set `hasEncryptedData` to `false` when the underlying data is not encrypted:
|
|
1201
|
+
|
|
1202
|
+
```ts
|
|
1203
|
+
declare const myDatabase: glue.Database;
|
|
1204
|
+
new glue.S3Table(this, 'MyTable', {
|
|
1205
|
+
hasEncryptedData: false,
|
|
1206
|
+
database: myDatabase,
|
|
1207
|
+
columns: [{
|
|
1208
|
+
name: 'col1',
|
|
1209
|
+
type: glue.Schema.STRING,
|
|
1210
|
+
}],
|
|
1211
|
+
dataFormat: glue.DataFormat.JSON,
|
|
1212
|
+
});
|
|
1213
|
+
```
|
|
1214
|
+
|
|
1215
|
+
Do not set `has_encrypted_data` through the free-form `parameters` map as well - a value
|
|
1216
|
+
there that conflicts with `hasEncryptedData` is rejected at synthesis time.
|
|
1217
|
+
|
|
1130
1218
|
## Types
|
|
1131
1219
|
|
|
1132
1220
|
A table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:
|
|
@@ -1146,7 +1234,7 @@ new glue.S3Table(this, 'MyTable', {
|
|
|
1146
1234
|
type: glue.Schema.map(
|
|
1147
1235
|
glue.Schema.STRING,
|
|
1148
1236
|
glue.Schema.TIMESTAMP),
|
|
1149
|
-
comment: 'map<string,
|
|
1237
|
+
comment: 'map<string,timestamp>',
|
|
1150
1238
|
}, {
|
|
1151
1239
|
name: 'struct_column',
|
|
1152
1240
|
type: glue.Schema.struct([{
|
package/awslint.json
CHANGED
|
@@ -109,6 +109,7 @@
|
|
|
109
109
|
"interface-extends-ref:@aws-cdk/aws-glue-alpha.ISecurityConfiguration",
|
|
110
110
|
"interface-extends-ref:@aws-cdk/aws-glue-alpha.ITable",
|
|
111
111
|
"interface-extends-ref:@aws-cdk/aws-glue-alpha.IWorkflow",
|
|
112
|
-
"prefer-ref-interface:@aws-cdk/aws-glue-alpha.DatabaseProps.catalog"
|
|
112
|
+
"prefer-ref-interface:@aws-cdk/aws-glue-alpha.DatabaseProps.catalog",
|
|
113
|
+
"no-unused-type:@aws-cdk/aws-glue-alpha.S3EncryptionMode"
|
|
113
114
|
]
|
|
114
115
|
}
|
package/lib/catalog.js
CHANGED
|
@@ -74,7 +74,7 @@ var CatalogEncryptionMode;
|
|
|
74
74
|
* @see https://docs.aws.amazon.com/glue/latest/webapi/API_EncryptionAtRest.html
|
|
75
75
|
*/
|
|
76
76
|
class DataCatalogEncryptionAtRest {
|
|
77
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.DataCatalogEncryptionAtRest", version: "2.
|
|
77
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.DataCatalogEncryptionAtRest", version: "2.266.0-alpha.0" };
|
|
78
78
|
/**
|
|
79
79
|
* Disable encryption at rest for the Data Catalog.
|
|
80
80
|
*/
|
|
@@ -130,7 +130,7 @@ exports.DataCatalogEncryptionAtRest = DataCatalogEncryptionAtRest;
|
|
|
130
130
|
* construction, so a catalog either carries settings or it does not.
|
|
131
131
|
*/
|
|
132
132
|
class CatalogBase extends core_1.Resource {
|
|
133
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.CatalogBase", version: "2.
|
|
133
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.CatalogBase", version: "2.266.0-alpha.0" };
|
|
134
134
|
_encryptionKey;
|
|
135
135
|
_connectionPasswordKey;
|
|
136
136
|
get encryptionKey() {
|
|
@@ -224,7 +224,7 @@ let Catalog = (() => {
|
|
|
224
224
|
Catalog = _classThis = _classDescriptor.value;
|
|
225
225
|
if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
|
|
226
226
|
}
|
|
227
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Catalog", version: "2.
|
|
227
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Catalog", version: "2.266.0-alpha.0" };
|
|
228
228
|
/** Uniquely identifies this class. */
|
|
229
229
|
static PROPERTY_INJECTION_ID = '@aws-cdk.aws-glue-alpha.Catalog';
|
|
230
230
|
/**
|
package/lib/code.js
CHANGED
|
@@ -43,7 +43,7 @@ const helpers_internal_1 = require("aws-cdk-lib/core/lib/helpers-internal");
|
|
|
43
43
|
* Represents a Glue Job's Code assets (an asset can be a scripts, a jar, a python file or any other file).
|
|
44
44
|
*/
|
|
45
45
|
class Code {
|
|
46
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Code", version: "2.
|
|
46
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Code", version: "2.266.0-alpha.0" };
|
|
47
47
|
/**
|
|
48
48
|
* Job code as an S3 object.
|
|
49
49
|
* @param bucket The S3 bucket
|
|
@@ -68,7 +68,7 @@ exports.Code = Code;
|
|
|
68
68
|
class S3Code extends Code {
|
|
69
69
|
bucket;
|
|
70
70
|
key;
|
|
71
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.S3Code", version: "2.
|
|
71
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.S3Code", version: "2.266.0-alpha.0" };
|
|
72
72
|
constructor(bucket, key) {
|
|
73
73
|
super();
|
|
74
74
|
this.bucket = bucket;
|
|
@@ -91,7 +91,7 @@ exports.S3Code = S3Code;
|
|
|
91
91
|
class AssetCode extends Code {
|
|
92
92
|
path;
|
|
93
93
|
options;
|
|
94
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.AssetCode", version: "2.
|
|
94
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.AssetCode", version: "2.266.0-alpha.0" };
|
|
95
95
|
asset;
|
|
96
96
|
/**
|
|
97
97
|
* @param path The path to the Code file.
|