@aws-cdk/aws-glue-alpha 2.264.0-alpha.0 → 2.266.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/.jsii +689 -552
  2. package/.jsii.tabl.json.gz +0 -0
  3. package/.warnings.jsii.js +25 -33
  4. package/README.md +132 -44
  5. package/awslint.json +2 -1
  6. package/lib/catalog.js +3 -3
  7. package/lib/code.js +3 -3
  8. package/lib/connection.js +12 -4
  9. package/lib/data-format.d.ts +2 -2
  10. package/lib/data-format.js +8 -8
  11. package/lib/data-quality-ruleset.d.ts +35 -4
  12. package/lib/data-quality-ruleset.js +38 -5
  13. package/lib/database.d.ts +11 -2
  14. package/lib/database.js +5 -2
  15. package/lib/external-table.js +6 -4
  16. package/lib/jobs/job.d.ts +6 -24
  17. package/lib/jobs/job.js +5 -3
  18. package/lib/jobs/pyspark-etl-job.d.ts +2 -1
  19. package/lib/jobs/pyspark-etl-job.js +10 -5
  20. package/lib/jobs/pyspark-flex-etl-job.d.ts +3 -2
  21. package/lib/jobs/pyspark-flex-etl-job.js +8 -7
  22. package/lib/jobs/pyspark-streaming-job.d.ts +2 -1
  23. package/lib/jobs/pyspark-streaming-job.js +6 -5
  24. package/lib/jobs/python-shell-job.js +1 -1
  25. package/lib/jobs/ray-job.d.ts +8 -0
  26. package/lib/jobs/ray-job.js +3 -7
  27. package/lib/jobs/scala-spark-etl-job.js +4 -8
  28. package/lib/jobs/scala-spark-flex-etl-job.js +5 -5
  29. package/lib/jobs/scala-spark-streaming-job.js +4 -8
  30. package/lib/jobs/spark-job.d.ts +26 -0
  31. package/lib/jobs/spark-job.js +2 -2
  32. package/lib/partition-projection.js +1 -1
  33. package/lib/private/secret-detection.d.ts +22 -0
  34. package/lib/private/secret-detection.js +42 -0
  35. package/lib/s3-table.d.ts +12 -1
  36. package/lib/s3-table.js +20 -3
  37. package/lib/schema.js +3 -8
  38. package/lib/security-configuration.d.ts +47 -47
  39. package/lib/security-configuration.js +84 -25
  40. package/lib/storage-parameter.js +1 -1
  41. package/lib/table-base.d.ts +56 -3
  42. package/lib/table-base.js +56 -5
  43. package/lib/table-deprecated.js +1 -1
  44. package/lib/triggers/trigger-options.js +1 -1
  45. package/lib/triggers/workflow.js +2 -2
  46. package/package.json +7 -7
Binary file
package/.warnings.jsii.js CHANGED
@@ -75,21 +75,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
75
75
  finally {
76
76
  visitedObjects.delete(p);
77
77
  }
78
- }, _aws_cdk_aws_glue_alpha_SecurityConfigurationProps: function _aws_cdk_aws_glue_alpha_SecurityConfigurationProps(p) {
79
- if (p == null)
80
- return;
81
- visitedObjects.add(p);
82
- try {
83
- if (!visitedObjects.has(p.cloudWatchEncryption))
84
- module.exports._aws_cdk_aws_glue_alpha_CloudWatchEncryption(p.cloudWatchEncryption);
85
- if (!visitedObjects.has(p.jobBookmarksEncryption))
86
- module.exports._aws_cdk_aws_glue_alpha_JobBookmarksEncryption(p.jobBookmarksEncryption);
87
- if (!visitedObjects.has(p.s3Encryption))
88
- module.exports._aws_cdk_aws_glue_alpha_S3Encryption(p.s3Encryption);
89
- }
90
- finally {
91
- visitedObjects.delete(p);
92
- }
93
78
  }, _aws_cdk_aws_glue_alpha_WorkerType: function _aws_cdk_aws_glue_alpha_WorkerType(p) {
94
79
  if (p == null)
95
80
  return;
@@ -135,6 +120,15 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
135
120
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
136
121
  if (!visitedObjects.has(p.continuousLogging))
137
122
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
123
+ }
124
+ finally {
125
+ visitedObjects.delete(p);
126
+ }
127
+ }, _aws_cdk_aws_glue_alpha_WorkerConfiguration: function _aws_cdk_aws_glue_alpha_WorkerConfiguration(p) {
128
+ if (p == null)
129
+ return;
130
+ visitedObjects.add(p);
131
+ try {
138
132
  if (!visitedObjects.has(p.workerType))
139
133
  module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
140
134
  }
@@ -148,14 +142,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
148
142
  try {
149
143
  if (!visitedObjects.has(p.sparkUI))
150
144
  module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
145
+ if (!visitedObjects.has(p.workerConfiguration))
146
+ module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
151
147
  if (p.connections != null)
152
148
  for (const o of p.connections)
153
149
  if (!visitedObjects.has(o))
154
150
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
155
151
  if (!visitedObjects.has(p.continuousLogging))
156
152
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
157
- if (!visitedObjects.has(p.workerType))
158
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
159
153
  }
160
154
  finally {
161
155
  visitedObjects.delete(p);
@@ -179,14 +173,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
179
173
  module.exports._aws_cdk_aws_glue_alpha_Code(o);
180
174
  if (!visitedObjects.has(p.sparkUI))
181
175
  module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
176
+ if (!visitedObjects.has(p.workerConfiguration))
177
+ module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
182
178
  if (p.connections != null)
183
179
  for (const o of p.connections)
184
180
  if (!visitedObjects.has(o))
185
181
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
186
182
  if (!visitedObjects.has(p.continuousLogging))
187
183
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
188
- if (!visitedObjects.has(p.workerType))
189
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
190
184
  }
191
185
  finally {
192
186
  visitedObjects.delete(p);
@@ -210,14 +204,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
210
204
  module.exports._aws_cdk_aws_glue_alpha_Code(o);
211
205
  if (!visitedObjects.has(p.sparkUI))
212
206
  module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
207
+ if (!visitedObjects.has(p.workerConfiguration))
208
+ module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
213
209
  if (p.connections != null)
214
210
  for (const o of p.connections)
215
211
  if (!visitedObjects.has(o))
216
212
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
217
213
  if (!visitedObjects.has(p.continuousLogging))
218
214
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
219
- if (!visitedObjects.has(p.workerType))
220
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
221
215
  }
222
216
  finally {
223
217
  visitedObjects.delete(p);
@@ -241,14 +235,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
241
235
  module.exports._aws_cdk_aws_glue_alpha_Code(o);
242
236
  if (!visitedObjects.has(p.sparkUI))
243
237
  module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
238
+ if (!visitedObjects.has(p.workerConfiguration))
239
+ module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
244
240
  if (p.connections != null)
245
241
  for (const o of p.connections)
246
242
  if (!visitedObjects.has(o))
247
243
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
248
244
  if (!visitedObjects.has(p.continuousLogging))
249
245
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
250
- if (!visitedObjects.has(p.workerType))
251
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
252
246
  }
253
247
  finally {
254
248
  visitedObjects.delete(p);
@@ -268,8 +262,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
268
262
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
269
263
  if (!visitedObjects.has(p.continuousLogging))
270
264
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
271
- if (!visitedObjects.has(p.workerType))
272
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
273
265
  }
274
266
  finally {
275
267
  visitedObjects.delete(p);
@@ -285,6 +277,8 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
285
277
  print("@aws-cdk/aws-glue-alpha.RayJobProps#enableObservabilityMetrics", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
286
278
  if ("jobRunQueuingEnabled" in p)
287
279
  print("@aws-cdk/aws-glue-alpha.RayJobProps#jobRunQueuingEnabled", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
280
+ if ("numberOfWorkers" in p)
281
+ print("@aws-cdk/aws-glue-alpha.RayJobProps#numberOfWorkers", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
288
282
  if ("runtime" in p)
289
283
  print("@aws-cdk/aws-glue-alpha.RayJobProps#runtime", "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html");
290
284
  if (!visitedObjects.has(p.runtime))
@@ -295,8 +289,6 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
295
289
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
296
290
  if (!visitedObjects.has(p.continuousLogging))
297
291
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
298
- if (!visitedObjects.has(p.workerType))
299
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
300
292
  }
301
293
  finally {
302
294
  visitedObjects.delete(p);
@@ -316,14 +308,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
316
308
  module.exports._aws_cdk_aws_glue_alpha_Code(o);
317
309
  if (!visitedObjects.has(p.sparkUI))
318
310
  module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
311
+ if (!visitedObjects.has(p.workerConfiguration))
312
+ module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
319
313
  if (p.connections != null)
320
314
  for (const o of p.connections)
321
315
  if (!visitedObjects.has(o))
322
316
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
323
317
  if (!visitedObjects.has(p.continuousLogging))
324
318
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
325
- if (!visitedObjects.has(p.workerType))
326
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
327
319
  }
328
320
  finally {
329
321
  visitedObjects.delete(p);
@@ -343,14 +335,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
343
335
  module.exports._aws_cdk_aws_glue_alpha_Code(o);
344
336
  if (!visitedObjects.has(p.sparkUI))
345
337
  module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
338
+ if (!visitedObjects.has(p.workerConfiguration))
339
+ module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
346
340
  if (p.connections != null)
347
341
  for (const o of p.connections)
348
342
  if (!visitedObjects.has(o))
349
343
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
350
344
  if (!visitedObjects.has(p.continuousLogging))
351
345
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
352
- if (!visitedObjects.has(p.workerType))
353
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
354
346
  }
355
347
  finally {
356
348
  visitedObjects.delete(p);
@@ -370,14 +362,14 @@ const VALIDATORS = { _aws_cdk_aws_glue_alpha_CatalogEncryptionOptions: function
370
362
  module.exports._aws_cdk_aws_glue_alpha_Code(o);
371
363
  if (!visitedObjects.has(p.sparkUI))
372
364
  module.exports._aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
365
+ if (!visitedObjects.has(p.workerConfiguration))
366
+ module.exports._aws_cdk_aws_glue_alpha_WorkerConfiguration(p.workerConfiguration);
373
367
  if (p.connections != null)
374
368
  for (const o of p.connections)
375
369
  if (!visitedObjects.has(o))
376
370
  module.exports._aws_cdk_aws_glue_alpha_IConnection(o);
377
371
  if (!visitedObjects.has(p.continuousLogging))
378
372
  module.exports._aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
379
- if (!visitedObjects.has(p.workerType))
380
- module.exports._aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
381
373
  }
382
374
  finally {
383
375
  visitedObjects.delete(p);
package/README.md CHANGED
@@ -75,7 +75,9 @@ similar constructors. ETL jobs default to the G2 worker type, but you can
75
75
  override this default with other supported worker type values (G1, G2, G4
76
76
  and G8). ETL jobs defaults to Glue version 4.0, which you can override to 3.0.
77
77
  The following ETL features are enabled by default:
78
- `—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.`
78
+ `—enable-metrics, —enable-continuous-cloudwatch-log.`
79
+ The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
80
+ `sparkUI` prop.
79
81
  You can find more details about version, worker type and other features in
80
82
  [Glue's public documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html).
81
83
 
@@ -115,7 +117,10 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
115
117
  script,
116
118
  glueVersion: glue.GlueVersion.V5_1,
117
119
  continuousLogging: { enabled: false },
118
- workerType: glue.WorkerType.G_2X,
120
+ workerConfiguration: {
121
+ workerType: glue.WorkerType.G_2X,
122
+ numberOfWorkers: 2,
123
+ },
119
124
  maxConcurrentRuns: 100,
120
125
  timeout: cdk.Duration.hours(2),
121
126
  connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
@@ -125,7 +130,6 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
125
130
  SecondTagName: 'SecondTagValue',
126
131
  XTagName: 'XTagValue',
127
132
  },
128
- numberOfWorkers: 2,
129
133
  maxRetries: 2,
130
134
  });
131
135
  ```
@@ -135,11 +139,12 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
135
139
  Streaming jobs are similar to ETL jobs, except that they perform ETL on data
136
140
  streams using the Apache Spark Structured Streaming framework. Some Spark
137
141
  job features are not available to Streaming ETL jobs. They support Scala
138
- and pySpark languages. PySpark streaming jobs default Python 3.9,
139
- which you can override with any non-deprecated version of Python. It
142
+ and pySpark languages. PySpark streaming jobs run on Python 3. It
140
143
  defaults to the G2 worker type and Glue 4.0, both of which you can override.
141
144
  The following best practice features are enabled by default:
142
- `—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log`.
145
+ `—enable-metrics, —enable-continuous-cloudwatch-log`.
146
+ The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
147
+ `sparkUI` prop.
143
148
 
144
149
  Reference the pyspark-streaming-jobs.test.ts and scalaspark-streaming-jobs.test.ts
145
150
  unit tests for examples of required-only and optional job parameters when creating
@@ -171,7 +176,10 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
171
176
  script,
172
177
  glueVersion: glue.GlueVersion.V5_1,
173
178
  continuousLogging: { enabled: false },
174
- workerType: glue.WorkerType.G_2X,
179
+ workerConfiguration: {
180
+ workerType: glue.WorkerType.G_2X,
181
+ numberOfWorkers: 2,
182
+ },
175
183
  maxConcurrentRuns: 100,
176
184
  timeout: cdk.Duration.hours(2),
177
185
  connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
@@ -181,7 +189,6 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
181
189
  SecondTagName: 'SecondTagValue',
182
190
  XTagName: 'XTagValue',
183
191
  },
184
- numberOfWorkers: 2,
185
192
  maxRetries: 2,
186
193
  });
187
194
  ```
@@ -190,9 +197,11 @@ new glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {
190
197
 
191
198
  The flexible execution class is appropriate for non-urgent jobs such as
192
199
  pre-production jobs, testing, and one-time data loads. Flexible jobs default
193
- to Glue version 3.0 and worker type `G_2X`. The following best practice
200
+ to Glue version 5.0 and worker type `G_2X`. The following best practice
194
201
  features are enabled by default:
195
- `—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log`
202
+ `—enable-metrics, —enable-continuous-cloudwatch-log`
203
+ The Spark UI (`—enable-spark-ui`) is off by default; enable it by setting the
204
+ `sparkUI` prop.
196
205
 
197
206
  Reference the pyspark-flex-etl-jobs.test.ts and scalaspark-flex-etl-jobs.test.ts
198
207
  unit tests for examples of required-only and optional job parameters when creating
@@ -217,14 +226,17 @@ import * as iam from 'aws-cdk-lib/aws-iam';
217
226
  declare const stack: cdk.Stack;
218
227
  declare const role: iam.IRole;
219
228
  declare const script: glue.Code;
220
- new glue.PySparkEtlJob(stack, 'pySparkEtlJob', {
221
- jobName: 'pySparkEtlJob',
229
+ new glue.PySparkFlexEtlJob(stack, 'pySparkFlexEtlJob', {
230
+ jobName: 'pySparkFlexEtlJob',
222
231
  description: 'This is a description',
223
232
  role,
224
233
  script,
225
234
  glueVersion: glue.GlueVersion.V5_1,
226
235
  continuousLogging: { enabled: false },
227
- workerType: glue.WorkerType.G_2X,
236
+ workerConfiguration: {
237
+ workerType: glue.WorkerType.G_2X,
238
+ numberOfWorkers: 2,
239
+ },
228
240
  maxConcurrentRuns: 100,
229
241
  timeout: cdk.Duration.hours(2),
230
242
  connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
@@ -234,7 +246,6 @@ new glue.PySparkEtlJob(stack, 'pySparkEtlJob', {
234
246
  SecondTagName: 'SecondTagValue',
235
247
  XTagName: 'XTagValue',
236
248
  },
237
- numberOfWorkers: 2,
238
249
  maxRetries: 2,
239
250
  });
240
251
  ```
@@ -274,14 +285,13 @@ declare const extraPythonFile: glue.Code;
274
285
  new glue.PythonShellJob(stack, 'PythonShellJob', {
275
286
  jobName: 'PythonShellJobCustomName',
276
287
  description: 'This is a description',
277
- pythonVersion: glue.PythonVersion.TWO,
288
+ pythonVersion: glue.PythonVersion.THREE_NINE,
278
289
  maxCapacity: glue.MaxCapacity.DPU_1,
279
290
  role,
280
291
  script,
281
292
  extraPythonFiles: [extraPythonFile],
282
- glueVersion: glue.GlueVersion.V2_0,
293
+ glueVersion: glue.GlueVersion.V3_0,
283
294
  continuousLogging: { enabled: false },
284
- workerType: glue.WorkerType.G_2X,
285
295
  maxConcurrentRuns: 100,
286
296
  timeout: cdk.Duration.hours(2),
287
297
  connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],
@@ -291,7 +301,6 @@ new glue.PythonShellJob(stack, 'PythonShellJob', {
291
301
  SecondTagName: 'SecondTagValue',
292
302
  XTagName: 'XTagValue',
293
303
  },
294
- numberOfWorkers: 2,
295
304
  maxRetries: 2,
296
305
  });
297
306
  ```
@@ -355,10 +364,12 @@ new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
355
364
 
356
365
  ### Uploading scripts from the CDK app repository to S3
357
366
 
358
- Similar to other L2 constructs, the Glue L2 automates uploading / updating
359
- scripts to S3 via an optional fromAsset parameter pointing to a script
360
- in the local file structure. You provide the existing S3 bucket and
361
- path to which you'd like the script to be uploaded.
367
+ Similar to other L2 constructs, the Glue L2 automates uploading local
368
+ scripts to S3. Use `glue.Code.fromAsset(path)` to point at a script in your
369
+ local file structure; it is uploaded to the CDK-managed asset bucket. To
370
+ reference a script that already exists in S3, use
371
+ `glue.Code.fromBucket(bucket, key)`, which performs no upload. A `script` is
372
+ required for every job.
362
373
 
363
374
  Reference the unit tests for examples of repo and S3 code target examples.
364
375
 
@@ -370,15 +381,34 @@ jobs, and triggers. Standalone triggers are an anti-pattern, so you must
370
381
  create triggers from within a workflow using the L2 construct.
371
382
 
372
383
  Within a workflow object, there are functions to create different
373
- types of triggers with actions and predicates. You then add those triggers
374
- to jobs.
384
+ types of triggers with actions and predicates. You add triggers to the
385
+ workflow, and each trigger references the jobs or crawlers it runs as its
386
+ actions.
375
387
 
376
- StartOnCreation defaults to true for all trigger types, but you can
377
- override it if you prefer for your trigger not to start on creation.
388
+ `startOnCreation` applies to scheduled triggers (and, via
389
+ `ConditionalTriggerOptions`, conditional triggers) only. It defaults to `false`,
390
+ but you can override it if you prefer for your trigger to start on creation.
378
391
 
379
392
  Reference the workflow-triggers.test.ts unit tests for examples of creating
380
393
  workflows and triggers.
381
394
 
395
+ ```ts
396
+ import * as cdk from 'aws-cdk-lib';
397
+ import * as iam from 'aws-cdk-lib/aws-iam';
398
+ declare const stack: cdk.Stack;
399
+ declare const role: iam.IRole;
400
+ declare const script: glue.Code;
401
+
402
+ // Create a job to run from the workflow
403
+ const job = new glue.PySparkEtlJob(stack, 'Job', { role, script });
404
+
405
+ // Create a workflow and add a trigger that runs the job
406
+ const workflow = new glue.Workflow(stack, 'Workflow');
407
+ workflow.addOnDemandTrigger('OnDemandTrigger', {
408
+ actions: [{ job }],
409
+ });
410
+ ```
411
+
382
412
  #### **1. On-Demand Triggers**
383
413
 
384
414
  On-demand triggers can start glue jobs or crawlers. This construct provides
@@ -389,7 +419,7 @@ actions list using the job or crawler objects using conditional types.
389
419
  #### **2. Scheduled Triggers**
390
420
 
391
421
  You can create scheduled triggers using cron expressions. This construct
392
- provides daily, weekly, and monthly convenience functions,
422
+ provides daily and weekly convenience functions,
393
423
  as well as a custom function that allows you to create your own
394
424
  custom timing using the [existing event Schedule class](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_events.Schedule.html)
395
425
  without having to build your own cron expressions. The L2 extracts
@@ -458,6 +488,13 @@ new glue.Connection(this, "RdsConnection", {
458
488
  });
459
489
  ```
460
490
 
491
+ Connection `properties` are emitted verbatim into the CloudFormation template, so
492
+ any credential placed there in plaintext is stored in plaintext in the template,
493
+ `cdk.out`, and source control. Reference a Secrets Manager secret through
494
+ `SECRET_ID` (as above) instead. If a property key looks like a credential (for
495
+ example `PASSWORD`, `SECRET`, or `TOKEN`) and holds a plaintext literal, the
496
+ construct emits a synthesis-time warning.
497
+
461
498
  If you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.
462
499
 
463
500
  See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.
@@ -466,32 +503,29 @@ See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/la
466
503
 
467
504
  A `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.
468
505
 
506
+ Each encryption config is built with a factory that pairs the encryption mode
507
+ with its key, so illegal combinations (such as an S3-managed encryption carrying
508
+ a KMS key) cannot be expressed:
509
+
469
510
  ```ts
470
511
  new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
471
- cloudWatchEncryption: {
472
- mode: glue.CloudWatchEncryptionMode.KMS,
473
- },
474
- jobBookmarksEncryption: {
475
- mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,
476
- },
477
- s3Encryption: {
478
- mode: glue.S3EncryptionMode.KMS,
479
- },
512
+ cloudWatchEncryption: glue.CloudWatchEncryption.kms(),
513
+ jobBookmarksEncryption: glue.JobBookmarksEncryption.clientSideKms(),
514
+ s3Encryption: glue.S3Encryption.kms(),
480
515
  });
481
516
  ```
482
517
 
483
- By default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:
518
+ By default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key to any factory, for example, for CloudWatch encryption:
484
519
 
485
520
  ```ts
486
521
  declare const key: kms.Key;
487
522
  new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
488
- cloudWatchEncryption: {
489
- mode: glue.CloudWatchEncryptionMode.KMS,
490
- kmsKey: key,
491
- },
523
+ cloudWatchEncryption: glue.CloudWatchEncryption.kms(key),
492
524
  });
493
525
  ```
494
526
 
527
+ Use `glue.S3Encryption.s3Managed()` for S3-managed (SSE-S3) encryption, which takes no key.
528
+
495
529
  See [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.
496
530
 
497
531
  ## Catalog
@@ -553,7 +587,7 @@ new glue.Catalog(this, 'MyCatalog', {
553
587
  ### Encryption at rest
554
588
 
555
589
  Configure Data Catalog encryption at rest through the `encryptionAtRest` option
556
- (on `Catalog.encryptAccount`, the `Catalog` constructor, or the import factories).
590
+ (on `Catalog.encryptAccount` or the `Catalog` constructor).
557
591
  It accepts a `DataCatalogEncryptionAtRest` describing the mode:
558
592
 
559
593
  ```ts
@@ -657,6 +691,19 @@ new glue.Database(this, 'MyDatabase', {
657
691
  });
658
692
  ```
659
693
 
694
+ Because a database is a container for tables and their metadata, it is retained
695
+ by default when removed from the stack, to avoid accidental data loss. Set
696
+ `removalPolicy` to `RemovalPolicy.DESTROY` to have it deleted instead:
697
+
698
+ ```ts
699
+ import { RemovalPolicy } from 'aws-cdk-lib';
700
+
701
+ new glue.Database(this, 'MyDatabase', {
702
+ databaseName: 'my_database',
703
+ removalPolicy: RemovalPolicy.DESTROY,
704
+ });
705
+ ```
706
+
660
707
  ## Table
661
708
 
662
709
  A Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):
@@ -794,7 +841,7 @@ new glue.S3Table(this, 'MyTable', {
794
841
  Alternatively, you can call the `addPartitionIndex()` function on a table:
795
842
 
796
843
  ```ts
797
- declare const myTable: glue.Table;
844
+ declare const myTable: glue.S3Table;
798
845
  myTable.addPartitionIndex({
799
846
  indexName: 'my-index',
800
847
  keyNames: ['year'],
@@ -1027,6 +1074,25 @@ new glue.ExternalTable(this, 'MyTable', {
1027
1074
  });
1028
1075
  ```
1029
1076
 
1077
+ ## Data Quality Ruleset
1078
+
1079
+ A `DataQualityRuleset` defines a set of data quality rules — authored in Glue's
1080
+ Data Quality Definition Language (DQDL) — that are evaluated against a table in
1081
+ the Data Catalog.
1082
+
1083
+ ```ts
1084
+ new glue.DataQualityRuleset(this, 'MyRuleset', {
1085
+ rulesetName: 'my_ruleset',
1086
+ dqdl: glue.Dqdl.fromString('Rules = [ RowCount > 100, IsComplete "order_id" ]'),
1087
+ targetTable: new glue.DataQualityTargetTable('my_database', 'my_table'),
1088
+ });
1089
+ ```
1090
+
1091
+ Build the DQDL document with `Dqdl.fromString(...)`. Glue parses and validates the
1092
+ DQDL when the ruleset is deployed; see the
1093
+ [DQDL reference](https://docs.aws.amazon.com/glue/latest/dg/dqdl.html) for the
1094
+ full rule syntax.
1095
+
1030
1096
  ## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)
1031
1097
 
1032
1098
  When the table creates its own S3 bucket (i.e. you do not pass an explicit `bucket`), that bucket enforces SSL: a bucket policy denies any request made over plain HTTP. If you provide your own bucket, enabling `enforceSSL` on it is your responsibility.
@@ -1127,6 +1193,28 @@ new glue.S3Table(this, 'MyTable', {
1127
1193
 
1128
1194
  *Note: you cannot provide a `Bucket` when creating the `S3Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.
1129
1195
 
1196
+ ### Marking table data as encrypted
1197
+
1198
+ Both `S3Table` and `ExternalTable` set the `has_encrypted_data` table parameter, which
1199
+ Athena reads when querying client-side (`CSE-KMS`) encrypted datasets. It defaults to `true`.
1200
+ Set `hasEncryptedData` to `false` when the underlying data is not encrypted:
1201
+
1202
+ ```ts
1203
+ declare const myDatabase: glue.Database;
1204
+ new glue.S3Table(this, 'MyTable', {
1205
+ hasEncryptedData: false,
1206
+ database: myDatabase,
1207
+ columns: [{
1208
+ name: 'col1',
1209
+ type: glue.Schema.STRING,
1210
+ }],
1211
+ dataFormat: glue.DataFormat.JSON,
1212
+ });
1213
+ ```
1214
+
1215
+ Do not set `has_encrypted_data` through the free-form `parameters` map as well - a value
1216
+ there that conflicts with `hasEncryptedData` is rejected at synthesis time.
1217
+
1130
1218
  ## Types
1131
1219
 
1132
1220
  A table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:
@@ -1146,7 +1234,7 @@ new glue.S3Table(this, 'MyTable', {
1146
1234
  type: glue.Schema.map(
1147
1235
  glue.Schema.STRING,
1148
1236
  glue.Schema.TIMESTAMP),
1149
- comment: 'map<string,string>',
1237
+ comment: 'map<string,timestamp>',
1150
1238
  }, {
1151
1239
  name: 'struct_column',
1152
1240
  type: glue.Schema.struct([{
package/awslint.json CHANGED
@@ -109,6 +109,7 @@
109
109
  "interface-extends-ref:@aws-cdk/aws-glue-alpha.ISecurityConfiguration",
110
110
  "interface-extends-ref:@aws-cdk/aws-glue-alpha.ITable",
111
111
  "interface-extends-ref:@aws-cdk/aws-glue-alpha.IWorkflow",
112
- "prefer-ref-interface:@aws-cdk/aws-glue-alpha.DatabaseProps.catalog"
112
+ "prefer-ref-interface:@aws-cdk/aws-glue-alpha.DatabaseProps.catalog",
113
+ "no-unused-type:@aws-cdk/aws-glue-alpha.S3EncryptionMode"
113
114
  ]
114
115
  }
package/lib/catalog.js CHANGED
@@ -74,7 +74,7 @@ var CatalogEncryptionMode;
74
74
  * @see https://docs.aws.amazon.com/glue/latest/webapi/API_EncryptionAtRest.html
75
75
  */
76
76
  class DataCatalogEncryptionAtRest {
77
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.DataCatalogEncryptionAtRest", version: "2.264.0-alpha.0" };
77
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.DataCatalogEncryptionAtRest", version: "2.266.0-alpha.0" };
78
78
  /**
79
79
  * Disable encryption at rest for the Data Catalog.
80
80
  */
@@ -130,7 +130,7 @@ exports.DataCatalogEncryptionAtRest = DataCatalogEncryptionAtRest;
130
130
  * construction, so a catalog either carries settings or it does not.
131
131
  */
132
132
  class CatalogBase extends core_1.Resource {
133
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.CatalogBase", version: "2.264.0-alpha.0" };
133
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.CatalogBase", version: "2.266.0-alpha.0" };
134
134
  _encryptionKey;
135
135
  _connectionPasswordKey;
136
136
  get encryptionKey() {
@@ -224,7 +224,7 @@ let Catalog = (() => {
224
224
  Catalog = _classThis = _classDescriptor.value;
225
225
  if (_metadata) Object.defineProperty(_classThis, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
226
226
  }
227
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Catalog", version: "2.264.0-alpha.0" };
227
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Catalog", version: "2.266.0-alpha.0" };
228
228
  /** Uniquely identifies this class. */
229
229
  static PROPERTY_INJECTION_ID = '@aws-cdk.aws-glue-alpha.Catalog';
230
230
  /**
package/lib/code.js CHANGED
@@ -43,7 +43,7 @@ const helpers_internal_1 = require("aws-cdk-lib/core/lib/helpers-internal");
43
43
  * Represents a Glue Job's Code assets (an asset can be a scripts, a jar, a python file or any other file).
44
44
  */
45
45
  class Code {
46
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Code", version: "2.264.0-alpha.0" };
46
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.Code", version: "2.266.0-alpha.0" };
47
47
  /**
48
48
  * Job code as an S3 object.
49
49
  * @param bucket The S3 bucket
@@ -68,7 +68,7 @@ exports.Code = Code;
68
68
  class S3Code extends Code {
69
69
  bucket;
70
70
  key;
71
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.S3Code", version: "2.264.0-alpha.0" };
71
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.S3Code", version: "2.266.0-alpha.0" };
72
72
  constructor(bucket, key) {
73
73
  super();
74
74
  this.bucket = bucket;
@@ -91,7 +91,7 @@ exports.S3Code = S3Code;
91
91
  class AssetCode extends Code {
92
92
  path;
93
93
  options;
94
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.AssetCode", version: "2.264.0-alpha.0" };
94
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@aws-cdk/aws-glue-alpha.AssetCode", version: "2.266.0-alpha.0" };
95
95
  asset;
96
96
  /**
97
97
  * @param path The path to the Code file.