@aws-cdk/aws-glue-alpha 2.12.0-alpha.0 → 2.13.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +3 -3
- package/.jsii.tabl.json +52 -52
- package/.warnings.jsii.js +3 -3
- package/lib/code.d.ts +10 -39
- package/lib/code.js +11 -34
- package/lib/connection.d.ts +31 -88
- package/lib/connection.js +15 -39
- package/lib/data-format.d.ts +32 -111
- package/lib/data-format.js +29 -91
- package/lib/database.d.ts +14 -47
- package/lib/database.js +3 -11
- package/lib/job-executable.d.ts +65 -136
- package/lib/job-executable.js +43 -66
- package/lib/job.d.ts +107 -215
- package/lib/job.js +60 -62
- package/lib/schema.d.ts +23 -69
- package/lib/schema.js +18 -51
- package/lib/security-configuration.d.ts +31 -83
- package/lib/security-configuration.js +25 -13
- package/lib/table.d.ts +48 -145
- package/lib/table.js +33 -36
- package/package.json +7 -7
package/lib/job-executable.d.ts
CHANGED
|
@@ -1,358 +1,287 @@
|
|
|
1
1
|
import { Code } from './code';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* AWS Glue version determines the versions of Apache Spark and Python that are available to the job.
|
|
4
4
|
*
|
|
5
5
|
* @see https://docs.aws.amazon.com/glue/latest/dg/add-job.html.
|
|
6
6
|
*
|
|
7
7
|
* If you need to use a GlueVersion that doesn't exist as a static member, you
|
|
8
8
|
* can instantiate a `GlueVersion` object, e.g: `GlueVersion.of('1.5')`.
|
|
9
|
-
* @experimental
|
|
10
9
|
*/
|
|
11
10
|
export declare class GlueVersion {
|
|
12
11
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* @experimental
|
|
12
|
+
* Glue version using Spark 2.2.1 and Python 2.7
|
|
16
13
|
*/
|
|
17
14
|
static readonly V0_9: GlueVersion;
|
|
18
15
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* @experimental
|
|
16
|
+
* Glue version using Spark 2.4.3, Python 2.7 and Python 3.6
|
|
22
17
|
*/
|
|
23
18
|
static readonly V1_0: GlueVersion;
|
|
24
19
|
/**
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* @experimental
|
|
20
|
+
* Glue version using Spark 2.4.3 and Python 3.7
|
|
28
21
|
*/
|
|
29
22
|
static readonly V2_0: GlueVersion;
|
|
30
23
|
/**
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
* @experimental
|
|
24
|
+
* Glue version using Spark 3.1.1 and Python 3.7
|
|
34
25
|
*/
|
|
35
26
|
static readonly V3_0: GlueVersion;
|
|
36
27
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* @param version custom version.
|
|
40
|
-
* @experimental
|
|
28
|
+
* Custom Glue version
|
|
29
|
+
* @param version custom version
|
|
41
30
|
*/
|
|
42
31
|
static of(version: string): GlueVersion;
|
|
43
32
|
/**
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* @experimental
|
|
33
|
+
* The name of this GlueVersion, as expected by Job resource.
|
|
47
34
|
*/
|
|
48
35
|
readonly name: string;
|
|
49
36
|
private constructor();
|
|
50
37
|
}
|
|
51
38
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* @experimental
|
|
39
|
+
* Runtime language of the Glue job
|
|
55
40
|
*/
|
|
56
41
|
export declare enum JobLanguage {
|
|
57
42
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
* @experimental
|
|
43
|
+
* Scala
|
|
61
44
|
*/
|
|
62
45
|
SCALA = "scala",
|
|
63
46
|
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
* @experimental
|
|
47
|
+
* Python
|
|
67
48
|
*/
|
|
68
49
|
PYTHON = "python"
|
|
69
50
|
}
|
|
70
51
|
/**
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
* @experimental
|
|
52
|
+
* Python version
|
|
74
53
|
*/
|
|
75
54
|
export declare enum PythonVersion {
|
|
76
55
|
/**
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
* @experimental
|
|
56
|
+
* Python 2 (the exact version depends on GlueVersion and JobCommand used)
|
|
80
57
|
*/
|
|
81
58
|
TWO = "2",
|
|
82
59
|
/**
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
* @experimental
|
|
60
|
+
* Python 3 (the exact version depends on GlueVersion and JobCommand used)
|
|
86
61
|
*/
|
|
87
62
|
THREE = "3"
|
|
88
63
|
}
|
|
89
64
|
/**
|
|
90
|
-
*
|
|
65
|
+
* The job type.
|
|
91
66
|
*
|
|
92
67
|
* If you need to use a JobType that doesn't exist as a static member, you
|
|
93
68
|
* can instantiate a `JobType` object, e.g: `JobType.of('other name')`.
|
|
94
|
-
*
|
|
95
|
-
* @experimental
|
|
96
69
|
*/
|
|
97
70
|
export declare class JobType {
|
|
98
71
|
/**
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* @experimental
|
|
72
|
+
* Command for running a Glue ETL job.
|
|
102
73
|
*/
|
|
103
74
|
static readonly ETL: JobType;
|
|
104
75
|
/**
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
* @experimental
|
|
76
|
+
* Command for running a Glue streaming job.
|
|
108
77
|
*/
|
|
109
78
|
static readonly STREAMING: JobType;
|
|
110
79
|
/**
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* @experimental
|
|
80
|
+
* Command for running a Glue python shell job.
|
|
114
81
|
*/
|
|
115
82
|
static readonly PYTHON_SHELL: JobType;
|
|
116
83
|
/**
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
* @param name type name.
|
|
120
|
-
* @experimental
|
|
84
|
+
* Custom type name
|
|
85
|
+
* @param name type name
|
|
121
86
|
*/
|
|
122
87
|
static of(name: string): JobType;
|
|
123
88
|
/**
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
* @experimental
|
|
89
|
+
* The name of this JobType, as expected by Job resource.
|
|
127
90
|
*/
|
|
128
91
|
readonly name: string;
|
|
129
92
|
private constructor();
|
|
130
93
|
}
|
|
131
94
|
interface PythonExecutableProps {
|
|
132
95
|
/**
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
* @experimental
|
|
96
|
+
* The Python version to use.
|
|
136
97
|
*/
|
|
137
98
|
readonly pythonVersion: PythonVersion;
|
|
138
99
|
/**
|
|
139
|
-
*
|
|
140
|
-
*
|
|
100
|
+
* Additional Python files that AWS Glue adds to the Python path before executing your script.
|
|
141
101
|
* Only individual files are supported, directories are not supported.
|
|
142
102
|
*
|
|
143
103
|
* @default - no extra python files and argument is not set
|
|
104
|
+
*
|
|
144
105
|
* @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
145
|
-
* @experimental
|
|
146
106
|
*/
|
|
147
107
|
readonly extraPythonFiles?: Code[];
|
|
148
108
|
}
|
|
149
109
|
interface SharedJobExecutableProps {
|
|
150
110
|
/**
|
|
151
|
-
*
|
|
111
|
+
* Glue version.
|
|
152
112
|
*
|
|
153
113
|
* @see https://docs.aws.amazon.com/glue/latest/dg/release-notes.html
|
|
154
|
-
* @experimental
|
|
155
114
|
*/
|
|
156
115
|
readonly glueVersion: GlueVersion;
|
|
157
116
|
/**
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
* @experimental
|
|
117
|
+
* The script that executes a job.
|
|
161
118
|
*/
|
|
162
119
|
readonly script: Code;
|
|
163
120
|
/**
|
|
164
|
-
*
|
|
165
|
-
*
|
|
121
|
+
* Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it.
|
|
166
122
|
* Only individual files are supported, directories are not supported.
|
|
167
123
|
*
|
|
168
124
|
* @default [] - no extra files are copied to the working directory
|
|
125
|
+
*
|
|
169
126
|
* @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
170
|
-
* @experimental
|
|
171
127
|
*/
|
|
172
128
|
readonly extraFiles?: Code[];
|
|
173
129
|
}
|
|
174
130
|
interface SharedSparkJobExecutableProps extends SharedJobExecutableProps {
|
|
175
131
|
/**
|
|
176
|
-
*
|
|
132
|
+
* Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script.
|
|
133
|
+
* Only individual files are supported, directories are not supported.
|
|
177
134
|
*
|
|
178
135
|
* @default [] - no extra jars are added to the classpath
|
|
136
|
+
*
|
|
179
137
|
* @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
180
|
-
* @experimental
|
|
181
138
|
*/
|
|
182
139
|
readonly extraJars?: Code[];
|
|
183
140
|
/**
|
|
184
|
-
*
|
|
141
|
+
* Setting this value to true prioritizes the customer's extra JAR files in the classpath.
|
|
185
142
|
*
|
|
186
143
|
* @default false - priority is not given to user-provided jars
|
|
144
|
+
*
|
|
187
145
|
* @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
188
|
-
* @experimental
|
|
189
146
|
*/
|
|
190
147
|
readonly extraJarsFirst?: boolean;
|
|
191
148
|
}
|
|
192
149
|
/**
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
* @experimental
|
|
150
|
+
* Props for creating a Scala Spark (ETL or Streaming) job executable
|
|
196
151
|
*/
|
|
197
152
|
export interface ScalaJobExecutableProps extends SharedSparkJobExecutableProps {
|
|
198
153
|
/**
|
|
199
|
-
*
|
|
154
|
+
* The fully qualified Scala class name that serves as the entry point for the job.
|
|
200
155
|
*
|
|
201
156
|
* @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
202
|
-
* @experimental
|
|
203
157
|
*/
|
|
204
158
|
readonly className: string;
|
|
205
159
|
}
|
|
206
160
|
/**
|
|
207
|
-
*
|
|
208
|
-
*
|
|
209
|
-
* @experimental
|
|
161
|
+
* Props for creating a Python Spark (ETL or Streaming) job executable
|
|
210
162
|
*/
|
|
211
163
|
export interface PythonSparkJobExecutableProps extends SharedSparkJobExecutableProps, PythonExecutableProps {
|
|
212
164
|
}
|
|
213
165
|
/**
|
|
214
|
-
*
|
|
215
|
-
*
|
|
216
|
-
* @experimental
|
|
166
|
+
* Props for creating a Python shell job executable
|
|
217
167
|
*/
|
|
218
168
|
export interface PythonShellExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {
|
|
219
169
|
}
|
|
220
170
|
/**
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
* @experimental
|
|
171
|
+
* The executable properties related to the Glue job's GlueVersion, JobType and code
|
|
224
172
|
*/
|
|
225
173
|
export declare class JobExecutable {
|
|
226
174
|
/**
|
|
227
|
-
*
|
|
175
|
+
* Create Scala executable props for Apache Spark ETL job.
|
|
228
176
|
*
|
|
229
|
-
* @param props Scala Apache Spark Job props
|
|
230
|
-
* @experimental
|
|
177
|
+
* @param props Scala Apache Spark Job props
|
|
231
178
|
*/
|
|
232
179
|
static scalaEtl(props: ScalaJobExecutableProps): JobExecutable;
|
|
233
180
|
/**
|
|
234
|
-
*
|
|
181
|
+
* Create Scala executable props for Apache Spark Streaming job.
|
|
235
182
|
*
|
|
236
|
-
* @param props Scala Apache Spark Job props
|
|
237
|
-
* @experimental
|
|
183
|
+
* @param props Scala Apache Spark Job props
|
|
238
184
|
*/
|
|
239
185
|
static scalaStreaming(props: ScalaJobExecutableProps): JobExecutable;
|
|
240
186
|
/**
|
|
241
|
-
*
|
|
187
|
+
* Create Python executable props for Apache Spark ETL job.
|
|
242
188
|
*
|
|
243
|
-
* @param props Python Apache Spark Job props
|
|
244
|
-
* @experimental
|
|
189
|
+
* @param props Python Apache Spark Job props
|
|
245
190
|
*/
|
|
246
191
|
static pythonEtl(props: PythonSparkJobExecutableProps): JobExecutable;
|
|
247
192
|
/**
|
|
248
|
-
*
|
|
193
|
+
* Create Python executable props for Apache Spark Streaming job.
|
|
249
194
|
*
|
|
250
|
-
* @param props Python Apache Spark Job props
|
|
251
|
-
* @experimental
|
|
195
|
+
* @param props Python Apache Spark Job props
|
|
252
196
|
*/
|
|
253
197
|
static pythonStreaming(props: PythonSparkJobExecutableProps): JobExecutable;
|
|
254
198
|
/**
|
|
255
|
-
*
|
|
199
|
+
* Create Python executable props for python shell jobs.
|
|
256
200
|
*
|
|
257
201
|
* @param props Python Shell Job props.
|
|
258
|
-
* @experimental
|
|
259
202
|
*/
|
|
260
203
|
static pythonShell(props: PythonShellExecutableProps): JobExecutable;
|
|
261
204
|
/**
|
|
262
|
-
*
|
|
205
|
+
* Create a custom JobExecutable.
|
|
263
206
|
*
|
|
264
207
|
* @param config custom job executable configuration.
|
|
265
|
-
* @experimental
|
|
266
208
|
*/
|
|
267
209
|
static of(config: JobExecutableConfig): JobExecutable;
|
|
268
210
|
private config;
|
|
269
211
|
private constructor();
|
|
270
212
|
/**
|
|
271
|
-
*
|
|
272
|
-
*
|
|
273
|
-
* @experimental
|
|
213
|
+
* Called during Job initialization to get JobExecutableConfig.
|
|
274
214
|
*/
|
|
275
215
|
bind(): JobExecutableConfig;
|
|
276
216
|
}
|
|
277
217
|
/**
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
* @experimental
|
|
218
|
+
* Result of binding a `JobExecutable` into a `Job`.
|
|
281
219
|
*/
|
|
282
220
|
export interface JobExecutableConfig {
|
|
283
221
|
/**
|
|
284
|
-
*
|
|
222
|
+
* Glue version.
|
|
285
223
|
*
|
|
286
224
|
* @see https://docs.aws.amazon.com/glue/latest/dg/release-notes.html
|
|
287
|
-
* @experimental
|
|
288
225
|
*/
|
|
289
226
|
readonly glueVersion: GlueVersion;
|
|
290
227
|
/**
|
|
291
|
-
*
|
|
228
|
+
* The language of the job (Scala or Python).
|
|
292
229
|
*
|
|
293
230
|
* @see `--job-language` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
294
|
-
* @experimental
|
|
295
231
|
*/
|
|
296
232
|
readonly language: JobLanguage;
|
|
297
233
|
/**
|
|
298
|
-
*
|
|
299
|
-
*
|
|
300
|
-
* @experimental
|
|
234
|
+
* Specify the type of the job whether it's an Apache Spark ETL or streaming one or if it's a Python shell job.
|
|
301
235
|
*/
|
|
302
236
|
readonly type: JobType;
|
|
303
237
|
/**
|
|
304
|
-
*
|
|
238
|
+
* The Python version to use.
|
|
305
239
|
*
|
|
306
240
|
* @default - no python version specified
|
|
307
|
-
* @experimental
|
|
308
241
|
*/
|
|
309
242
|
readonly pythonVersion?: PythonVersion;
|
|
310
243
|
/**
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
* @experimental
|
|
244
|
+
* The script that is executed by a job.
|
|
314
245
|
*/
|
|
315
246
|
readonly script: Code;
|
|
316
247
|
/**
|
|
317
|
-
*
|
|
318
|
-
*
|
|
319
|
-
* This applies only if your the job langauage is Scala.
|
|
248
|
+
* The Scala class that serves as the entry point for the job. This applies only if your the job langauage is Scala.
|
|
320
249
|
*
|
|
321
250
|
* @default - no scala className specified
|
|
251
|
+
*
|
|
322
252
|
* @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
323
|
-
* @experimental
|
|
324
253
|
*/
|
|
325
254
|
readonly className?: string;
|
|
326
255
|
/**
|
|
327
|
-
*
|
|
256
|
+
* Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script.
|
|
328
257
|
*
|
|
329
258
|
* @default - no extra jars specified.
|
|
259
|
+
*
|
|
330
260
|
* @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
331
|
-
* @experimental
|
|
332
261
|
*/
|
|
333
262
|
readonly extraJars?: Code[];
|
|
334
263
|
/**
|
|
335
|
-
*
|
|
264
|
+
* Additional Python files that AWS Glue adds to the Python path before executing your script.
|
|
336
265
|
*
|
|
337
266
|
* @default - no extra python files specified.
|
|
267
|
+
*
|
|
338
268
|
* @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
339
|
-
* @experimental
|
|
340
269
|
*/
|
|
341
270
|
readonly extraPythonFiles?: Code[];
|
|
342
271
|
/**
|
|
343
|
-
*
|
|
272
|
+
* Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it.
|
|
344
273
|
*
|
|
345
274
|
* @default - no extra files specified.
|
|
275
|
+
*
|
|
346
276
|
* @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
347
|
-
* @experimental
|
|
348
277
|
*/
|
|
349
278
|
readonly extraFiles?: Code[];
|
|
350
279
|
/**
|
|
351
|
-
*
|
|
280
|
+
* Setting this value to true prioritizes the customer's extra JAR files in the classpath.
|
|
352
281
|
*
|
|
353
282
|
* @default - extra jars are not prioritized.
|
|
283
|
+
*
|
|
354
284
|
* @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
|
|
355
|
-
* @experimental
|
|
356
285
|
*/
|
|
357
286
|
readonly extraJarsFirst?: boolean;
|
|
358
287
|
}
|