@aws-cdk/aws-glue-alpha 2.0.0-rc.24 → 2.4.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +586 -1501
- package/.jsii.tabl.json +3678 -0
- package/.warnings.jsii.js +281 -0
- package/LICENSE +1 -1
- package/NOTICE +1 -1
- package/README.md +160 -53
- package/lib/code.js +4 -4
- package/lib/connection.js +5 -3
- package/lib/data-format.js +8 -6
- package/lib/database.js +5 -3
- package/lib/job-executable.js +11 -4
- package/lib/job.js +8 -5
- package/lib/schema.js +6 -2
- package/lib/security-configuration.js +4 -2
- package/lib/table.d.ts +73 -1
- package/lib/table.js +104 -4
- package/package.json +21 -16
- package/rosetta/_generated.ts-fixture +6 -0
- package/rosetta/default.ts-fixture +16 -0
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
function _aws_cdk_aws_glue_alpha_JobState(p) {
|
|
2
|
+
}
|
|
3
|
+
function _aws_cdk_aws_glue_alpha_MetricType(p) {
|
|
4
|
+
}
|
|
5
|
+
function _aws_cdk_aws_glue_alpha_JobLanguage(p) {
|
|
6
|
+
}
|
|
7
|
+
function _aws_cdk_aws_glue_alpha_PythonVersion(p) {
|
|
8
|
+
}
|
|
9
|
+
function _aws_cdk_aws_glue_alpha_S3EncryptionMode(p) {
|
|
10
|
+
}
|
|
11
|
+
function _aws_cdk_aws_glue_alpha_CloudWatchEncryptionMode(p) {
|
|
12
|
+
}
|
|
13
|
+
function _aws_cdk_aws_glue_alpha_JobBookmarksEncryptionMode(p) {
|
|
14
|
+
}
|
|
15
|
+
function _aws_cdk_aws_glue_alpha_TableEncryption(p) {
|
|
16
|
+
}
|
|
17
|
+
function _aws_cdk_aws_glue_alpha_Type(p) {
|
|
18
|
+
}
|
|
19
|
+
function _aws_cdk_aws_glue_alpha_TableAttributes(p) {
|
|
20
|
+
}
|
|
21
|
+
function _aws_cdk_aws_glue_alpha_DatabaseProps(p) {
|
|
22
|
+
}
|
|
23
|
+
function _aws_cdk_aws_glue_alpha_PartitionIndex(p) {
|
|
24
|
+
}
|
|
25
|
+
function _aws_cdk_aws_glue_alpha_Column(p) {
|
|
26
|
+
if (p == null)
|
|
27
|
+
return;
|
|
28
|
+
visitedObjects.add(p);
|
|
29
|
+
if (!visitedObjects.has(p.type))
|
|
30
|
+
_aws_cdk_aws_glue_alpha_Type(p.type);
|
|
31
|
+
visitedObjects.delete(p);
|
|
32
|
+
}
|
|
33
|
+
function _aws_cdk_aws_glue_alpha_IConnection(p) {
|
|
34
|
+
}
|
|
35
|
+
function _aws_cdk_aws_glue_alpha_IDatabase(p) {
|
|
36
|
+
}
|
|
37
|
+
function _aws_cdk_aws_glue_alpha_ISecurityConfiguration(p) {
|
|
38
|
+
}
|
|
39
|
+
function _aws_cdk_aws_glue_alpha_ITable(p) {
|
|
40
|
+
}
|
|
41
|
+
function _aws_cdk_aws_glue_alpha_SparkUILoggingLocation(p) {
|
|
42
|
+
}
|
|
43
|
+
function _aws_cdk_aws_glue_alpha_CodeConfig(p) {
|
|
44
|
+
}
|
|
45
|
+
function _aws_cdk_aws_glue_alpha_SparkUIProps(p) {
|
|
46
|
+
}
|
|
47
|
+
function _aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p) {
|
|
48
|
+
}
|
|
49
|
+
function _aws_cdk_aws_glue_alpha_JobAttributes(p) {
|
|
50
|
+
}
|
|
51
|
+
function _aws_cdk_aws_glue_alpha_ConnectionProps(p) {
|
|
52
|
+
if (p == null)
|
|
53
|
+
return;
|
|
54
|
+
visitedObjects.add(p);
|
|
55
|
+
if (!visitedObjects.has(p.type))
|
|
56
|
+
_aws_cdk_aws_glue_alpha_ConnectionType(p.type);
|
|
57
|
+
if (!visitedObjects.has(p.securityGroups))
|
|
58
|
+
require("aws-cdk-lib/.warnings.jsii.js").aws_cdk_lib_aws_ec2_ISecurityGroup(p.securityGroups);
|
|
59
|
+
visitedObjects.delete(p);
|
|
60
|
+
}
|
|
61
|
+
function _aws_cdk_aws_glue_alpha_S3Encryption(p) {
|
|
62
|
+
if (p == null)
|
|
63
|
+
return;
|
|
64
|
+
visitedObjects.add(p);
|
|
65
|
+
if (!visitedObjects.has(p.mode))
|
|
66
|
+
_aws_cdk_aws_glue_alpha_S3EncryptionMode(p.mode);
|
|
67
|
+
visitedObjects.delete(p);
|
|
68
|
+
}
|
|
69
|
+
function _aws_cdk_aws_glue_alpha_CloudWatchEncryption(p) {
|
|
70
|
+
if (p == null)
|
|
71
|
+
return;
|
|
72
|
+
visitedObjects.add(p);
|
|
73
|
+
if (!visitedObjects.has(p.mode))
|
|
74
|
+
_aws_cdk_aws_glue_alpha_CloudWatchEncryptionMode(p.mode);
|
|
75
|
+
visitedObjects.delete(p);
|
|
76
|
+
}
|
|
77
|
+
function _aws_cdk_aws_glue_alpha_JobBookmarksEncryption(p) {
|
|
78
|
+
if (p == null)
|
|
79
|
+
return;
|
|
80
|
+
visitedObjects.add(p);
|
|
81
|
+
if (!visitedObjects.has(p.mode))
|
|
82
|
+
_aws_cdk_aws_glue_alpha_JobBookmarksEncryptionMode(p.mode);
|
|
83
|
+
visitedObjects.delete(p);
|
|
84
|
+
}
|
|
85
|
+
function _aws_cdk_aws_glue_alpha_ConnectionOptions(p) {
|
|
86
|
+
if (p == null)
|
|
87
|
+
return;
|
|
88
|
+
visitedObjects.add(p);
|
|
89
|
+
if (!visitedObjects.has(p.securityGroups))
|
|
90
|
+
require("aws-cdk-lib/.warnings.jsii.js").aws_cdk_lib_aws_ec2_ISecurityGroup(p.securityGroups);
|
|
91
|
+
visitedObjects.delete(p);
|
|
92
|
+
}
|
|
93
|
+
function _aws_cdk_aws_glue_alpha_SecurityConfigurationProps(p) {
|
|
94
|
+
if (p == null)
|
|
95
|
+
return;
|
|
96
|
+
visitedObjects.add(p);
|
|
97
|
+
if (!visitedObjects.has(p.cloudWatchEncryption))
|
|
98
|
+
_aws_cdk_aws_glue_alpha_CloudWatchEncryption(p.cloudWatchEncryption);
|
|
99
|
+
if (!visitedObjects.has(p.jobBookmarksEncryption))
|
|
100
|
+
_aws_cdk_aws_glue_alpha_JobBookmarksEncryption(p.jobBookmarksEncryption);
|
|
101
|
+
if (!visitedObjects.has(p.s3Encryption))
|
|
102
|
+
_aws_cdk_aws_glue_alpha_S3Encryption(p.s3Encryption);
|
|
103
|
+
visitedObjects.delete(p);
|
|
104
|
+
}
|
|
105
|
+
function _aws_cdk_aws_glue_alpha_ConnectionType(p) {
|
|
106
|
+
}
|
|
107
|
+
function _aws_cdk_aws_glue_alpha_OutputFormat(p) {
|
|
108
|
+
}
|
|
109
|
+
function _aws_cdk_aws_glue_alpha_WorkerType(p) {
|
|
110
|
+
}
|
|
111
|
+
function _aws_cdk_aws_glue_alpha_JobType(p) {
|
|
112
|
+
}
|
|
113
|
+
function _aws_cdk_aws_glue_alpha_DataFormatProps(p) {
|
|
114
|
+
if (p == null)
|
|
115
|
+
return;
|
|
116
|
+
visitedObjects.add(p);
|
|
117
|
+
if (!visitedObjects.has(p.inputFormat))
|
|
118
|
+
_aws_cdk_aws_glue_alpha_InputFormat(p.inputFormat);
|
|
119
|
+
if (!visitedObjects.has(p.outputFormat))
|
|
120
|
+
_aws_cdk_aws_glue_alpha_OutputFormat(p.outputFormat);
|
|
121
|
+
if (!visitedObjects.has(p.serializationLibrary))
|
|
122
|
+
_aws_cdk_aws_glue_alpha_SerializationLibrary(p.serializationLibrary);
|
|
123
|
+
if (!visitedObjects.has(p.classificationString))
|
|
124
|
+
_aws_cdk_aws_glue_alpha_ClassificationString(p.classificationString);
|
|
125
|
+
visitedObjects.delete(p);
|
|
126
|
+
}
|
|
127
|
+
function _aws_cdk_aws_glue_alpha_ScalaJobExecutableProps(p) {
|
|
128
|
+
if (p == null)
|
|
129
|
+
return;
|
|
130
|
+
visitedObjects.add(p);
|
|
131
|
+
if (!visitedObjects.has(p.glueVersion))
|
|
132
|
+
_aws_cdk_aws_glue_alpha_GlueVersion(p.glueVersion);
|
|
133
|
+
if (!visitedObjects.has(p.script))
|
|
134
|
+
_aws_cdk_aws_glue_alpha_Code(p.script);
|
|
135
|
+
if (!visitedObjects.has(p.extraFiles))
|
|
136
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraFiles);
|
|
137
|
+
if (!visitedObjects.has(p.extraJars))
|
|
138
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraJars);
|
|
139
|
+
visitedObjects.delete(p);
|
|
140
|
+
}
|
|
141
|
+
function _aws_cdk_aws_glue_alpha_S3Code(p) {
|
|
142
|
+
}
|
|
143
|
+
function _aws_cdk_aws_glue_alpha_AssetCode(p) {
|
|
144
|
+
}
|
|
145
|
+
function _aws_cdk_aws_glue_alpha_InputFormat(p) {
|
|
146
|
+
}
|
|
147
|
+
function _aws_cdk_aws_glue_alpha_GlueVersion(p) {
|
|
148
|
+
}
|
|
149
|
+
function _aws_cdk_aws_glue_alpha_PythonShellExecutableProps(p) {
|
|
150
|
+
if (p == null)
|
|
151
|
+
return;
|
|
152
|
+
visitedObjects.add(p);
|
|
153
|
+
if (!visitedObjects.has(p.glueVersion))
|
|
154
|
+
_aws_cdk_aws_glue_alpha_GlueVersion(p.glueVersion);
|
|
155
|
+
if (!visitedObjects.has(p.pythonVersion))
|
|
156
|
+
_aws_cdk_aws_glue_alpha_PythonVersion(p.pythonVersion);
|
|
157
|
+
if (!visitedObjects.has(p.script))
|
|
158
|
+
_aws_cdk_aws_glue_alpha_Code(p.script);
|
|
159
|
+
if (!visitedObjects.has(p.extraFiles))
|
|
160
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraFiles);
|
|
161
|
+
if (!visitedObjects.has(p.extraPythonFiles))
|
|
162
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraPythonFiles);
|
|
163
|
+
visitedObjects.delete(p);
|
|
164
|
+
}
|
|
165
|
+
function _aws_cdk_aws_glue_alpha_ClassificationString(p) {
|
|
166
|
+
}
|
|
167
|
+
function _aws_cdk_aws_glue_alpha_PythonSparkJobExecutableProps(p) {
|
|
168
|
+
if (p == null)
|
|
169
|
+
return;
|
|
170
|
+
visitedObjects.add(p);
|
|
171
|
+
if (!visitedObjects.has(p.glueVersion))
|
|
172
|
+
_aws_cdk_aws_glue_alpha_GlueVersion(p.glueVersion);
|
|
173
|
+
if (!visitedObjects.has(p.pythonVersion))
|
|
174
|
+
_aws_cdk_aws_glue_alpha_PythonVersion(p.pythonVersion);
|
|
175
|
+
if (!visitedObjects.has(p.script))
|
|
176
|
+
_aws_cdk_aws_glue_alpha_Code(p.script);
|
|
177
|
+
if (!visitedObjects.has(p.extraFiles))
|
|
178
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraFiles);
|
|
179
|
+
if (!visitedObjects.has(p.extraJars))
|
|
180
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraJars);
|
|
181
|
+
if (!visitedObjects.has(p.extraPythonFiles))
|
|
182
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraPythonFiles);
|
|
183
|
+
visitedObjects.delete(p);
|
|
184
|
+
}
|
|
185
|
+
function _aws_cdk_aws_glue_alpha_Database(p) {
|
|
186
|
+
}
|
|
187
|
+
function _aws_cdk_aws_glue_alpha_Code(p) {
|
|
188
|
+
}
|
|
189
|
+
function _aws_cdk_aws_glue_alpha_Connection(p) {
|
|
190
|
+
}
|
|
191
|
+
function _aws_cdk_aws_glue_alpha_JobExecutableConfig(p) {
|
|
192
|
+
if (p == null)
|
|
193
|
+
return;
|
|
194
|
+
visitedObjects.add(p);
|
|
195
|
+
if (!visitedObjects.has(p.glueVersion))
|
|
196
|
+
_aws_cdk_aws_glue_alpha_GlueVersion(p.glueVersion);
|
|
197
|
+
if (!visitedObjects.has(p.language))
|
|
198
|
+
_aws_cdk_aws_glue_alpha_JobLanguage(p.language);
|
|
199
|
+
if (!visitedObjects.has(p.script))
|
|
200
|
+
_aws_cdk_aws_glue_alpha_Code(p.script);
|
|
201
|
+
if (!visitedObjects.has(p.type))
|
|
202
|
+
_aws_cdk_aws_glue_alpha_JobType(p.type);
|
|
203
|
+
if (!visitedObjects.has(p.extraFiles))
|
|
204
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraFiles);
|
|
205
|
+
if (!visitedObjects.has(p.extraJars))
|
|
206
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraJars);
|
|
207
|
+
if (!visitedObjects.has(p.extraPythonFiles))
|
|
208
|
+
_aws_cdk_aws_glue_alpha_Code(p.extraPythonFiles);
|
|
209
|
+
if (!visitedObjects.has(p.pythonVersion))
|
|
210
|
+
_aws_cdk_aws_glue_alpha_PythonVersion(p.pythonVersion);
|
|
211
|
+
visitedObjects.delete(p);
|
|
212
|
+
}
|
|
213
|
+
function _aws_cdk_aws_glue_alpha_JobProps(p) {
|
|
214
|
+
if (p == null)
|
|
215
|
+
return;
|
|
216
|
+
visitedObjects.add(p);
|
|
217
|
+
if (!visitedObjects.has(p.executable))
|
|
218
|
+
_aws_cdk_aws_glue_alpha_JobExecutable(p.executable);
|
|
219
|
+
if (!visitedObjects.has(p.connections))
|
|
220
|
+
_aws_cdk_aws_glue_alpha_IConnection(p.connections);
|
|
221
|
+
if (!visitedObjects.has(p.continuousLogging))
|
|
222
|
+
_aws_cdk_aws_glue_alpha_ContinuousLoggingProps(p.continuousLogging);
|
|
223
|
+
if (!visitedObjects.has(p.securityConfiguration))
|
|
224
|
+
_aws_cdk_aws_glue_alpha_ISecurityConfiguration(p.securityConfiguration);
|
|
225
|
+
if (!visitedObjects.has(p.sparkUI))
|
|
226
|
+
_aws_cdk_aws_glue_alpha_SparkUIProps(p.sparkUI);
|
|
227
|
+
if (!visitedObjects.has(p.workerType))
|
|
228
|
+
_aws_cdk_aws_glue_alpha_WorkerType(p.workerType);
|
|
229
|
+
visitedObjects.delete(p);
|
|
230
|
+
}
|
|
231
|
+
function _aws_cdk_aws_glue_alpha_SecurityConfiguration(p) {
|
|
232
|
+
}
|
|
233
|
+
function _aws_cdk_aws_glue_alpha_SerializationLibrary(p) {
|
|
234
|
+
}
|
|
235
|
+
function _aws_cdk_aws_glue_alpha_TableProps(p) {
|
|
236
|
+
if (p == null)
|
|
237
|
+
return;
|
|
238
|
+
visitedObjects.add(p);
|
|
239
|
+
if (!visitedObjects.has(p.columns))
|
|
240
|
+
_aws_cdk_aws_glue_alpha_Column(p.columns);
|
|
241
|
+
if (!visitedObjects.has(p.database))
|
|
242
|
+
_aws_cdk_aws_glue_alpha_IDatabase(p.database);
|
|
243
|
+
if (!visitedObjects.has(p.dataFormat))
|
|
244
|
+
_aws_cdk_aws_glue_alpha_DataFormat(p.dataFormat);
|
|
245
|
+
if (!visitedObjects.has(p.encryption))
|
|
246
|
+
_aws_cdk_aws_glue_alpha_TableEncryption(p.encryption);
|
|
247
|
+
if (!visitedObjects.has(p.partitionIndexes))
|
|
248
|
+
_aws_cdk_aws_glue_alpha_PartitionIndex(p.partitionIndexes);
|
|
249
|
+
if (!visitedObjects.has(p.partitionKeys))
|
|
250
|
+
_aws_cdk_aws_glue_alpha_Column(p.partitionKeys);
|
|
251
|
+
visitedObjects.delete(p);
|
|
252
|
+
}
|
|
253
|
+
function _aws_cdk_aws_glue_alpha_JobExecutable(p) {
|
|
254
|
+
}
|
|
255
|
+
function _aws_cdk_aws_glue_alpha_DataFormat(p) {
|
|
256
|
+
}
|
|
257
|
+
function _aws_cdk_aws_glue_alpha_Schema(p) {
|
|
258
|
+
}
|
|
259
|
+
function _aws_cdk_aws_glue_alpha_IJob(p) {
|
|
260
|
+
}
|
|
261
|
+
function _aws_cdk_aws_glue_alpha_Table(p) {
|
|
262
|
+
}
|
|
263
|
+
function _aws_cdk_aws_glue_alpha_Job(p) {
|
|
264
|
+
}
|
|
265
|
+
function print(name, deprecationMessage) {
|
|
266
|
+
const deprecated = process.env.JSII_DEPRECATED;
|
|
267
|
+
const deprecationMode = ["warn", "fail", "quiet"].includes(deprecated) ? deprecated : "warn";
|
|
268
|
+
const message = `${name} is deprecated.\n ${deprecationMessage}\n This API will be removed in the next major release.`;
|
|
269
|
+
switch (deprecationMode) {
|
|
270
|
+
case "fail":
|
|
271
|
+
throw new DeprecationError(message);
|
|
272
|
+
case "warn":
|
|
273
|
+
console.warn("[WARNING]", message);
|
|
274
|
+
break;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const visitedObjects = new Set();
|
|
278
|
+
class DeprecationError extends Error {
|
|
279
|
+
}
|
|
280
|
+
module.exports = { print, _aws_cdk_aws_glue_alpha_JobState, _aws_cdk_aws_glue_alpha_MetricType, _aws_cdk_aws_glue_alpha_JobLanguage, _aws_cdk_aws_glue_alpha_PythonVersion, _aws_cdk_aws_glue_alpha_S3EncryptionMode, _aws_cdk_aws_glue_alpha_CloudWatchEncryptionMode, _aws_cdk_aws_glue_alpha_JobBookmarksEncryptionMode, _aws_cdk_aws_glue_alpha_TableEncryption, _aws_cdk_aws_glue_alpha_Type, _aws_cdk_aws_glue_alpha_TableAttributes, _aws_cdk_aws_glue_alpha_DatabaseProps, _aws_cdk_aws_glue_alpha_PartitionIndex, _aws_cdk_aws_glue_alpha_Column, _aws_cdk_aws_glue_alpha_IConnection, _aws_cdk_aws_glue_alpha_IDatabase, _aws_cdk_aws_glue_alpha_ISecurityConfiguration, _aws_cdk_aws_glue_alpha_ITable, _aws_cdk_aws_glue_alpha_SparkUILoggingLocation, _aws_cdk_aws_glue_alpha_CodeConfig, _aws_cdk_aws_glue_alpha_SparkUIProps, _aws_cdk_aws_glue_alpha_ContinuousLoggingProps, _aws_cdk_aws_glue_alpha_JobAttributes, _aws_cdk_aws_glue_alpha_ConnectionProps, _aws_cdk_aws_glue_alpha_S3Encryption, _aws_cdk_aws_glue_alpha_CloudWatchEncryption, _aws_cdk_aws_glue_alpha_JobBookmarksEncryption, _aws_cdk_aws_glue_alpha_ConnectionOptions, _aws_cdk_aws_glue_alpha_SecurityConfigurationProps, _aws_cdk_aws_glue_alpha_ConnectionType, _aws_cdk_aws_glue_alpha_OutputFormat, _aws_cdk_aws_glue_alpha_WorkerType, _aws_cdk_aws_glue_alpha_JobType, _aws_cdk_aws_glue_alpha_DataFormatProps, _aws_cdk_aws_glue_alpha_ScalaJobExecutableProps, _aws_cdk_aws_glue_alpha_S3Code, _aws_cdk_aws_glue_alpha_AssetCode, _aws_cdk_aws_glue_alpha_InputFormat, _aws_cdk_aws_glue_alpha_GlueVersion, _aws_cdk_aws_glue_alpha_PythonShellExecutableProps, _aws_cdk_aws_glue_alpha_ClassificationString, _aws_cdk_aws_glue_alpha_PythonSparkJobExecutableProps, _aws_cdk_aws_glue_alpha_Database, _aws_cdk_aws_glue_alpha_Code, _aws_cdk_aws_glue_alpha_Connection, _aws_cdk_aws_glue_alpha_JobExecutableConfig, _aws_cdk_aws_glue_alpha_JobProps, _aws_cdk_aws_glue_alpha_SecurityConfiguration, _aws_cdk_aws_glue_alpha_SerializationLibrary, _aws_cdk_aws_glue_alpha_TableProps, _aws_cdk_aws_glue_alpha_JobExecutable, _aws_cdk_aws_glue_alpha_DataFormat, _aws_cdk_aws_glue_alpha_Schema, _aws_cdk_aws_glue_alpha_IJob, _aws_cdk_aws_glue_alpha_Table, _aws_cdk_aws_glue_alpha_Job };
|
|
281
|
+
module.exports.DeprecationError = DeprecationError;
|
package/LICENSE
CHANGED
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2018-
|
|
189
|
+
Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
package/NOTICE
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
AWS Cloud Development Kit (AWS CDK)
|
|
2
|
-
Copyright 2018-
|
|
2
|
+
Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
package/README.md
CHANGED
|
@@ -3,12 +3,6 @@
|
|
|
3
3
|
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-

|
|
7
|
-
|
|
8
|
-
> All classes with the `Cfn` prefix in this module ([CFN Resources]) are always stable and safe to use.
|
|
9
|
-
>
|
|
10
|
-
> [CFN Resources]: https://docs.aws.amazon.com/cdk/latest/guide/constructs.html#constructs_lib
|
|
11
|
-
|
|
12
6
|

|
|
13
7
|
|
|
14
8
|
> The APIs of higher level constructs in this module are experimental and under active development.
|
|
@@ -42,7 +36,8 @@ These jobs run in an Apache Spark environment managed by AWS Glue.
|
|
|
42
36
|
An ETL job processes data in batches using Apache Spark.
|
|
43
37
|
|
|
44
38
|
```ts
|
|
45
|
-
|
|
39
|
+
declare const bucket: s3.Bucket;
|
|
40
|
+
new glue.Job(this, 'ScalaSparkEtlJob', {
|
|
46
41
|
executable: glue.JobExecutable.scalaEtl({
|
|
47
42
|
glueVersion: glue.GlueVersion.V2_0,
|
|
48
43
|
script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),
|
|
@@ -58,7 +53,7 @@ new glue.Job(stack, 'ScalaSparkEtlJob', {
|
|
|
58
53
|
A Streaming job is similar to an ETL job, except that it performs ETL on data streams. It uses the Apache Spark Structured Streaming framework. Some Spark job features are not available to streaming ETL jobs.
|
|
59
54
|
|
|
60
55
|
```ts
|
|
61
|
-
new glue.Job(
|
|
56
|
+
new glue.Job(this, 'PythonSparkStreamingJob', {
|
|
62
57
|
executable: glue.JobExecutable.pythonStreaming({
|
|
63
58
|
glueVersion: glue.GlueVersion.V2_0,
|
|
64
59
|
pythonVersion: glue.PythonVersion.THREE,
|
|
@@ -74,10 +69,11 @@ A Python shell job runs Python scripts as a shell and supports a Python version
|
|
|
74
69
|
This can be used to schedule and run tasks that don't require an Apache Spark environment.
|
|
75
70
|
|
|
76
71
|
```ts
|
|
77
|
-
|
|
72
|
+
declare const bucket: s3.Bucket;
|
|
73
|
+
new glue.Job(this, 'PythonShellJob', {
|
|
78
74
|
executable: glue.JobExecutable.pythonShell({
|
|
79
75
|
glueVersion: glue.GlueVersion.V1_0,
|
|
80
|
-
pythonVersion: PythonVersion.THREE,
|
|
76
|
+
pythonVersion: glue.PythonVersion.THREE,
|
|
81
77
|
script: glue.Code.fromBucket(bucket, 'script.py'),
|
|
82
78
|
}),
|
|
83
79
|
description: 'an example Python Shell job',
|
|
@@ -91,8 +87,10 @@ See [documentation](https://docs.aws.amazon.com/glue/latest/dg/add-job.html) for
|
|
|
91
87
|
A `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. For example, to create a network connection to connect to a data source within a VPC:
|
|
92
88
|
|
|
93
89
|
```ts
|
|
94
|
-
|
|
95
|
-
|
|
90
|
+
declare const securityGroup: ec2.SecurityGroup;
|
|
91
|
+
declare const subnet: ec2.Subnet;
|
|
92
|
+
new glue.Connection(this, 'MyConnection', {
|
|
93
|
+
type: glue.ConnectionType.NETWORK,
|
|
96
94
|
// The security groups granting AWS Glue inbound access to the data source within the VPC
|
|
97
95
|
securityGroups: [securityGroup],
|
|
98
96
|
// The VPC subnet which contains the data source
|
|
@@ -109,7 +107,7 @@ See [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/la
|
|
|
109
107
|
A `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.
|
|
110
108
|
|
|
111
109
|
```ts
|
|
112
|
-
new glue.SecurityConfiguration(
|
|
110
|
+
new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
|
|
113
111
|
securityConfigurationName: 'name',
|
|
114
112
|
cloudWatchEncryption: {
|
|
115
113
|
mode: glue.CloudWatchEncryptionMode.KMS,
|
|
@@ -126,7 +124,8 @@ new glue.SecurityConfiguration(stack, 'MySecurityConfiguration', {
|
|
|
126
124
|
By default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:
|
|
127
125
|
|
|
128
126
|
```ts
|
|
129
|
-
|
|
127
|
+
declare const key: kms.Key;
|
|
128
|
+
new glue.SecurityConfiguration(this, 'MySecurityConfiguration', {
|
|
130
129
|
securityConfigurationName: 'name',
|
|
131
130
|
cloudWatchEncryption: {
|
|
132
131
|
mode: glue.CloudWatchEncryptionMode.KMS,
|
|
@@ -142,8 +141,8 @@ See [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-securi
|
|
|
142
141
|
A `Database` is a logical grouping of `Tables` in the Glue Catalog.
|
|
143
142
|
|
|
144
143
|
```ts
|
|
145
|
-
new glue.Database(
|
|
146
|
-
databaseName: 'my_database'
|
|
144
|
+
new glue.Database(this, 'MyDatabase', {
|
|
145
|
+
databaseName: 'my_database',
|
|
147
146
|
});
|
|
148
147
|
```
|
|
149
148
|
|
|
@@ -152,7 +151,8 @@ new glue.Database(stack, 'MyDatabase', {
|
|
|
152
151
|
A Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):
|
|
153
152
|
|
|
154
153
|
```ts
|
|
155
|
-
|
|
154
|
+
declare const myDatabase: glue.Database;
|
|
155
|
+
new glue.Table(this, 'MyTable', {
|
|
156
156
|
database: myDatabase,
|
|
157
157
|
tableName: 'my_table',
|
|
158
158
|
columns: [{
|
|
@@ -160,45 +160,101 @@ new glue.Table(stack, 'MyTable', {
|
|
|
160
160
|
type: glue.Schema.STRING,
|
|
161
161
|
}, {
|
|
162
162
|
name: 'col2',
|
|
163
|
-
type: glue.Schema.array(Schema.STRING),
|
|
163
|
+
type: glue.Schema.array(glue.Schema.STRING),
|
|
164
164
|
comment: 'col2 is an array of strings' // comment is optional
|
|
165
165
|
}],
|
|
166
|
-
dataFormat: glue.DataFormat.JSON
|
|
166
|
+
dataFormat: glue.DataFormat.JSON,
|
|
167
167
|
});
|
|
168
168
|
```
|
|
169
169
|
|
|
170
170
|
By default, a S3 bucket will be created to store the table's data but you can manually pass the `bucket` and `s3Prefix`:
|
|
171
171
|
|
|
172
172
|
```ts
|
|
173
|
-
|
|
173
|
+
declare const myBucket: s3.Bucket;
|
|
174
|
+
declare const myDatabase: glue.Database;
|
|
175
|
+
new glue.Table(this, 'MyTable', {
|
|
174
176
|
bucket: myBucket,
|
|
175
|
-
s3Prefix: 'my-table/'
|
|
176
|
-
...
|
|
177
|
+
s3Prefix: 'my-table/',
|
|
178
|
+
// ...
|
|
179
|
+
database: myDatabase,
|
|
180
|
+
tableName: 'my_table',
|
|
181
|
+
columns: [{
|
|
182
|
+
name: 'col1',
|
|
183
|
+
type: glue.Schema.STRING,
|
|
184
|
+
}],
|
|
185
|
+
dataFormat: glue.DataFormat.JSON,
|
|
177
186
|
});
|
|
178
187
|
```
|
|
179
188
|
|
|
180
189
|
By default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`:
|
|
181
190
|
|
|
182
|
-
###
|
|
191
|
+
### Partition Keys
|
|
183
192
|
|
|
184
193
|
To improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window:
|
|
185
194
|
|
|
186
195
|
```ts
|
|
187
|
-
|
|
196
|
+
declare const myDatabase: glue.Database;
|
|
197
|
+
new glue.Table(this, 'MyTable', {
|
|
188
198
|
database: myDatabase,
|
|
189
199
|
tableName: 'my_table',
|
|
190
200
|
columns: [{
|
|
191
201
|
name: 'col1',
|
|
192
|
-
type: glue.Schema.STRING
|
|
202
|
+
type: glue.Schema.STRING,
|
|
193
203
|
}],
|
|
194
204
|
partitionKeys: [{
|
|
195
205
|
name: 'year',
|
|
196
|
-
type: glue.Schema.SMALL_INT
|
|
206
|
+
type: glue.Schema.SMALL_INT,
|
|
197
207
|
}, {
|
|
198
208
|
name: 'month',
|
|
199
|
-
type: glue.Schema.SMALL_INT
|
|
209
|
+
type: glue.Schema.SMALL_INT,
|
|
200
210
|
}],
|
|
201
|
-
dataFormat: glue.DataFormat.JSON
|
|
211
|
+
dataFormat: glue.DataFormat.JSON,
|
|
212
|
+
});
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### Partition Indexes
|
|
216
|
+
|
|
217
|
+
Another way to improve query performance is to specify partition indexes. If no partition indexes are
|
|
218
|
+
present on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using
|
|
219
|
+
the query expression. The query takes more time to run as the number of partitions increase. With an
|
|
220
|
+
index, the query will try to fetch a subset of the partitions instead of loading all partitions of the
|
|
221
|
+
table.
|
|
222
|
+
|
|
223
|
+
The keys of a partition index must be a subset of the partition keys of the table. You can have a
|
|
224
|
+
maximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes`
|
|
225
|
+
property:
|
|
226
|
+
|
|
227
|
+
```ts
|
|
228
|
+
declare const myDatabase: glue.Database;
|
|
229
|
+
new glue.Table(this, 'MyTable', {
|
|
230
|
+
database: myDatabase,
|
|
231
|
+
tableName: 'my_table',
|
|
232
|
+
columns: [{
|
|
233
|
+
name: 'col1',
|
|
234
|
+
type: glue.Schema.STRING,
|
|
235
|
+
}],
|
|
236
|
+
partitionKeys: [{
|
|
237
|
+
name: 'year',
|
|
238
|
+
type: glue.Schema.SMALL_INT,
|
|
239
|
+
}, {
|
|
240
|
+
name: 'month',
|
|
241
|
+
type: glue.Schema.SMALL_INT,
|
|
242
|
+
}],
|
|
243
|
+
partitionIndexes: [{
|
|
244
|
+
indexName: 'my-index', // optional
|
|
245
|
+
keyNames: ['year'],
|
|
246
|
+
}], // supply up to 3 indexes
|
|
247
|
+
dataFormat: glue.DataFormat.JSON,
|
|
248
|
+
});
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Alternatively, you can call the `addPartitionIndex()` function on a table:
|
|
252
|
+
|
|
253
|
+
```ts
|
|
254
|
+
declare const myTable: glue.Table;
|
|
255
|
+
myTable.addPartitionIndex({
|
|
256
|
+
indexName: 'my-index',
|
|
257
|
+
keyNames: ['year'],
|
|
202
258
|
});
|
|
203
259
|
```
|
|
204
260
|
|
|
@@ -210,52 +266,98 @@ You can enable encryption on a Table's data:
|
|
|
210
266
|
* [S3Managed](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) - Server side encryption (`SSE-S3`) with an Amazon S3-managed key.
|
|
211
267
|
|
|
212
268
|
```ts
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
269
|
+
declare const myDatabase: glue.Database;
|
|
270
|
+
new glue.Table(this, 'MyTable', {
|
|
271
|
+
encryption: glue.TableEncryption.S3_MANAGED,
|
|
272
|
+
// ...
|
|
273
|
+
database: myDatabase,
|
|
274
|
+
tableName: 'my_table',
|
|
275
|
+
columns: [{
|
|
276
|
+
name: 'col1',
|
|
277
|
+
type: glue.Schema.STRING,
|
|
278
|
+
}],
|
|
279
|
+
dataFormat: glue.DataFormat.JSON,
|
|
216
280
|
});
|
|
217
281
|
```
|
|
218
282
|
|
|
219
283
|
* [Kms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`) with an AWS KMS Key managed by the account owner.
|
|
220
284
|
|
|
221
285
|
```ts
|
|
286
|
+
declare const myDatabase: glue.Database;
|
|
222
287
|
// KMS key is created automatically
|
|
223
|
-
new glue.Table(
|
|
224
|
-
encryption: glue.TableEncryption.KMS
|
|
225
|
-
...
|
|
288
|
+
new glue.Table(this, 'MyTable', {
|
|
289
|
+
encryption: glue.TableEncryption.KMS,
|
|
290
|
+
// ...
|
|
291
|
+
database: myDatabase,
|
|
292
|
+
tableName: 'my_table',
|
|
293
|
+
columns: [{
|
|
294
|
+
name: 'col1',
|
|
295
|
+
type: glue.Schema.STRING,
|
|
296
|
+
}],
|
|
297
|
+
dataFormat: glue.DataFormat.JSON,
|
|
226
298
|
});
|
|
227
299
|
|
|
228
300
|
// with an explicit KMS key
|
|
229
|
-
new glue.Table(
|
|
301
|
+
new glue.Table(this, 'MyTable', {
|
|
230
302
|
encryption: glue.TableEncryption.KMS,
|
|
231
|
-
encryptionKey: new kms.Key(
|
|
232
|
-
...
|
|
303
|
+
encryptionKey: new kms.Key(this, 'MyKey'),
|
|
304
|
+
// ...
|
|
305
|
+
database: myDatabase,
|
|
306
|
+
tableName: 'my_table',
|
|
307
|
+
columns: [{
|
|
308
|
+
name: 'col1',
|
|
309
|
+
type: glue.Schema.STRING,
|
|
310
|
+
}],
|
|
311
|
+
dataFormat: glue.DataFormat.JSON,
|
|
233
312
|
});
|
|
234
313
|
```
|
|
235
314
|
|
|
236
315
|
* [KmsManaged](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`), like `Kms`, except with an AWS KMS Key managed by the AWS Key Management Service.
|
|
237
316
|
|
|
238
317
|
```ts
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
318
|
+
declare const myDatabase: glue.Database;
|
|
319
|
+
new glue.Table(this, 'MyTable', {
|
|
320
|
+
encryption: glue.TableEncryption.KMS_MANAGED,
|
|
321
|
+
// ...
|
|
322
|
+
database: myDatabase,
|
|
323
|
+
tableName: 'my_table',
|
|
324
|
+
columns: [{
|
|
325
|
+
name: 'col1',
|
|
326
|
+
type: glue.Schema.STRING,
|
|
327
|
+
}],
|
|
328
|
+
dataFormat: glue.DataFormat.JSON,
|
|
242
329
|
});
|
|
243
330
|
```
|
|
244
331
|
|
|
245
332
|
* [ClientSideKms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro) - Client-side encryption (`CSE-KMS`) with an AWS KMS Key managed by the account owner.
|
|
246
333
|
|
|
247
334
|
```ts
|
|
335
|
+
declare const myDatabase: glue.Database;
|
|
248
336
|
// KMS key is created automatically
|
|
249
|
-
new glue.Table(
|
|
250
|
-
encryption: glue.TableEncryption.CLIENT_SIDE_KMS
|
|
251
|
-
...
|
|
337
|
+
new glue.Table(this, 'MyTable', {
|
|
338
|
+
encryption: glue.TableEncryption.CLIENT_SIDE_KMS,
|
|
339
|
+
// ...
|
|
340
|
+
database: myDatabase,
|
|
341
|
+
tableName: 'my_table',
|
|
342
|
+
columns: [{
|
|
343
|
+
name: 'col1',
|
|
344
|
+
type: glue.Schema.STRING,
|
|
345
|
+
}],
|
|
346
|
+
dataFormat: glue.DataFormat.JSON,
|
|
252
347
|
});
|
|
253
348
|
|
|
254
349
|
// with an explicit KMS key
|
|
255
|
-
new glue.Table(
|
|
350
|
+
new glue.Table(this, 'MyTable', {
|
|
256
351
|
encryption: glue.TableEncryption.CLIENT_SIDE_KMS,
|
|
257
|
-
encryptionKey: new kms.Key(
|
|
258
|
-
...
|
|
352
|
+
encryptionKey: new kms.Key(this, 'MyKey'),
|
|
353
|
+
// ...
|
|
354
|
+
database: myDatabase,
|
|
355
|
+
tableName: 'my_table',
|
|
356
|
+
columns: [{
|
|
357
|
+
name: 'col1',
|
|
358
|
+
type: glue.Schema.STRING,
|
|
359
|
+
}],
|
|
360
|
+
dataFormat: glue.DataFormat.JSON,
|
|
259
361
|
});
|
|
260
362
|
```
|
|
261
363
|
|
|
@@ -266,30 +368,35 @@ new glue.Table(stack, 'MyTable', {
|
|
|
266
368
|
A table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:
|
|
267
369
|
|
|
268
370
|
```ts
|
|
269
|
-
|
|
371
|
+
declare const myDatabase: glue.Database;
|
|
372
|
+
new glue.Table(this, 'MyTable', {
|
|
270
373
|
columns: [{
|
|
271
374
|
name: 'primitive_column',
|
|
272
|
-
type: glue.Schema.STRING
|
|
375
|
+
type: glue.Schema.STRING,
|
|
273
376
|
}, {
|
|
274
377
|
name: 'array_column',
|
|
275
378
|
type: glue.Schema.array(glue.Schema.INTEGER),
|
|
276
|
-
comment: 'array<integer>'
|
|
379
|
+
comment: 'array<integer>',
|
|
277
380
|
}, {
|
|
278
381
|
name: 'map_column',
|
|
279
382
|
type: glue.Schema.map(
|
|
280
383
|
glue.Schema.STRING,
|
|
281
384
|
glue.Schema.TIMESTAMP),
|
|
282
|
-
comment: 'map<string,string>'
|
|
385
|
+
comment: 'map<string,string>',
|
|
283
386
|
}, {
|
|
284
387
|
name: 'struct_column',
|
|
285
388
|
type: glue.Schema.struct([{
|
|
286
389
|
name: 'nested_column',
|
|
287
390
|
type: glue.Schema.DATE,
|
|
288
|
-
comment: 'nested comment'
|
|
391
|
+
comment: 'nested comment',
|
|
289
392
|
}]),
|
|
290
|
-
comment: "struct<nested_column:date COMMENT 'nested comment'>"
|
|
393
|
+
comment: "struct<nested_column:date COMMENT 'nested comment'>",
|
|
291
394
|
}],
|
|
292
|
-
...
|
|
395
|
+
// ...
|
|
396
|
+
database: myDatabase,
|
|
397
|
+
tableName: 'my_table',
|
|
398
|
+
dataFormat: glue.DataFormat.JSON,
|
|
399
|
+
});
|
|
293
400
|
```
|
|
294
401
|
|
|
295
402
|
### Primitives
|