@aws-cdk/aws-glue-alpha 2.182.0-alpha.0 → 2.183.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +637 -592
- package/.jsii.tabl.json.gz +0 -0
- package/.warnings.jsii.js +69 -7
- package/lib/code.js +3 -3
- package/lib/connection.js +2 -2
- package/lib/data-format.js +5 -5
- package/lib/data-quality-ruleset.js +2 -2
- package/lib/database.js +1 -1
- package/lib/external-table.js +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.js +2 -2
- package/lib/jobs/job.d.ts +11 -11
- package/lib/jobs/job.js +8 -8
- package/lib/jobs/pyspark-etl-job.d.ts +11 -24
- package/lib/jobs/pyspark-etl-job.js +9 -60
- package/lib/jobs/pyspark-flex-etl-job.d.ts +14 -33
- package/lib/jobs/pyspark-flex-etl-job.js +9 -65
- package/lib/jobs/pyspark-streaming-job.d.ts +11 -24
- package/lib/jobs/pyspark-streaming-job.js +9 -60
- package/lib/jobs/python-shell-job.d.ts +2 -2
- package/lib/jobs/python-shell-job.js +3 -7
- package/lib/jobs/ray-job.d.ts +2 -2
- package/lib/jobs/ray-job.js +3 -7
- package/lib/jobs/scala-spark-etl-job.d.ts +19 -24
- package/lib/jobs/scala-spark-etl-job.js +9 -58
- package/lib/jobs/scala-spark-flex-etl-job.d.ts +12 -42
- package/lib/jobs/scala-spark-flex-etl-job.js +9 -68
- package/lib/jobs/scala-spark-streaming-job.d.ts +21 -25
- package/lib/jobs/scala-spark-streaming-job.js +9 -55
- package/lib/jobs/spark-job.d.ts +117 -0
- package/lib/jobs/spark-job.js +120 -0
- package/lib/s3-table.js +1 -1
- package/lib/schema.js +1 -1
- package/lib/security-configuration.js +1 -1
- package/lib/storage-parameter.js +1 -1
- package/lib/table-base.js +1 -1
- package/lib/table-deprecated.js +1 -1
- package/lib/triggers/trigger-options.js +1 -1
- package/lib/triggers/workflow.js +2 -2
- package/package.json +7 -7
- package/lib/jobs/spark-ui-utils.d.ts +0 -53
- package/lib/jobs/spark-ui-utils.js +0 -26
package/.jsii
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"url": "https://aws.amazon.com"
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"aws-cdk-lib": "^2.
|
|
11
|
+
"aws-cdk-lib": "^2.183.0",
|
|
12
12
|
"constructs": "^10.0.0"
|
|
13
13
|
},
|
|
14
14
|
"dependencyClosure": {
|
|
@@ -1618,6 +1618,19 @@
|
|
|
1618
1618
|
}
|
|
1619
1619
|
}
|
|
1620
1620
|
},
|
|
1621
|
+
"aws-cdk-lib.aws_gameliftstreams": {
|
|
1622
|
+
"targets": {
|
|
1623
|
+
"dotnet": {
|
|
1624
|
+
"package": "Amazon.CDK.AWS.GameLiftStreams"
|
|
1625
|
+
},
|
|
1626
|
+
"java": {
|
|
1627
|
+
"package": "software.amazon.awscdk.services.gameliftstreams"
|
|
1628
|
+
},
|
|
1629
|
+
"python": {
|
|
1630
|
+
"module": "aws_cdk.aws_gameliftstreams"
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
},
|
|
1621
1634
|
"aws-cdk-lib.aws_globalaccelerator": {
|
|
1622
1635
|
"targets": {
|
|
1623
1636
|
"dotnet": {
|
|
@@ -3900,7 +3913,7 @@
|
|
|
3900
3913
|
"stability": "experimental"
|
|
3901
3914
|
},
|
|
3902
3915
|
"homepage": "https://github.com/aws/aws-cdk",
|
|
3903
|
-
"jsiiVersion": "5.7.
|
|
3916
|
+
"jsiiVersion": "5.7.8 (build 2bc6834)",
|
|
3904
3917
|
"keywords": [
|
|
3905
3918
|
"aws",
|
|
3906
3919
|
"cdk",
|
|
@@ -5885,7 +5898,7 @@
|
|
|
5885
5898
|
"abstract": true,
|
|
5886
5899
|
"docs": {
|
|
5887
5900
|
"stability": "experimental",
|
|
5888
|
-
"summary": "Enable
|
|
5901
|
+
"summary": "Enable continuous logging."
|
|
5889
5902
|
},
|
|
5890
5903
|
"immutable": true,
|
|
5891
5904
|
"locationInModule": {
|
|
@@ -8608,7 +8621,7 @@
|
|
|
8608
8621
|
},
|
|
8609
8622
|
"name": "attrs",
|
|
8610
8623
|
"type": {
|
|
8611
|
-
"fqn": "@aws-cdk/aws-glue-alpha.
|
|
8624
|
+
"fqn": "@aws-cdk/aws-glue-alpha.JobAttributes"
|
|
8612
8625
|
}
|
|
8613
8626
|
}
|
|
8614
8627
|
],
|
|
@@ -8685,7 +8698,7 @@
|
|
|
8685
8698
|
"docs": {
|
|
8686
8699
|
"returns": "String containing the args for the continuous logging command",
|
|
8687
8700
|
"stability": "experimental",
|
|
8688
|
-
"summary": "Setup Continuous
|
|
8701
|
+
"summary": "Setup Continuous Logging Properties."
|
|
8689
8702
|
},
|
|
8690
8703
|
"locationInModule": {
|
|
8691
8704
|
"filename": "lib/jobs/job.ts",
|
|
@@ -8713,6 +8726,7 @@
|
|
|
8713
8726
|
}
|
|
8714
8727
|
}
|
|
8715
8728
|
],
|
|
8729
|
+
"protected": true,
|
|
8716
8730
|
"returns": {
|
|
8717
8731
|
"type": {
|
|
8718
8732
|
"primitive": "any"
|
|
@@ -8741,12 +8755,69 @@
|
|
|
8741
8755
|
],
|
|
8742
8756
|
"symbolId": "lib/jobs/job:Job"
|
|
8743
8757
|
},
|
|
8758
|
+
"@aws-cdk/aws-glue-alpha.JobAttributes": {
|
|
8759
|
+
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
8760
|
+
"datatype": true,
|
|
8761
|
+
"docs": {
|
|
8762
|
+
"remarks": "This is only used when using fromJobAttributes\nto identify and reference the existing job.",
|
|
8763
|
+
"stability": "experimental",
|
|
8764
|
+
"summary": "A subset of Job attributes are required for importing an existing job into a CDK project.",
|
|
8765
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport { aws_iam as iam } from 'aws-cdk-lib';\n\ndeclare const role: iam.Role;\nconst jobAttributes: glue_alpha.JobAttributes = {\n jobName: 'jobName',\n\n // the properties below are optional\n role: role,\n};",
|
|
8766
|
+
"custom": {
|
|
8767
|
+
"exampleMetadata": "fixture=_generated"
|
|
8768
|
+
}
|
|
8769
|
+
},
|
|
8770
|
+
"fqn": "@aws-cdk/aws-glue-alpha.JobAttributes",
|
|
8771
|
+
"kind": "interface",
|
|
8772
|
+
"locationInModule": {
|
|
8773
|
+
"filename": "lib/jobs/job.ts",
|
|
8774
|
+
"line": 288
|
|
8775
|
+
},
|
|
8776
|
+
"name": "JobAttributes",
|
|
8777
|
+
"properties": [
|
|
8778
|
+
{
|
|
8779
|
+
"abstract": true,
|
|
8780
|
+
"docs": {
|
|
8781
|
+
"stability": "experimental",
|
|
8782
|
+
"summary": "The name of the job."
|
|
8783
|
+
},
|
|
8784
|
+
"immutable": true,
|
|
8785
|
+
"locationInModule": {
|
|
8786
|
+
"filename": "lib/jobs/job.ts",
|
|
8787
|
+
"line": 292
|
|
8788
|
+
},
|
|
8789
|
+
"name": "jobName",
|
|
8790
|
+
"type": {
|
|
8791
|
+
"primitive": "string"
|
|
8792
|
+
}
|
|
8793
|
+
},
|
|
8794
|
+
{
|
|
8795
|
+
"abstract": true,
|
|
8796
|
+
"docs": {
|
|
8797
|
+
"default": "- undefined",
|
|
8798
|
+
"stability": "experimental",
|
|
8799
|
+
"summary": "The IAM role assumed by Glue to run this job."
|
|
8800
|
+
},
|
|
8801
|
+
"immutable": true,
|
|
8802
|
+
"locationInModule": {
|
|
8803
|
+
"filename": "lib/jobs/job.ts",
|
|
8804
|
+
"line": 299
|
|
8805
|
+
},
|
|
8806
|
+
"name": "role",
|
|
8807
|
+
"optional": true,
|
|
8808
|
+
"type": {
|
|
8809
|
+
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
8810
|
+
}
|
|
8811
|
+
}
|
|
8812
|
+
],
|
|
8813
|
+
"symbolId": "lib/jobs/job:JobAttributes"
|
|
8814
|
+
},
|
|
8744
8815
|
"@aws-cdk/aws-glue-alpha.JobBase": {
|
|
8745
8816
|
"abstract": true,
|
|
8746
8817
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
8747
8818
|
"base": "aws-cdk-lib.Resource",
|
|
8748
8819
|
"docs": {
|
|
8749
|
-
"remarks": "JobBase has the subset\nof
|
|
8820
|
+
"remarks": "JobBase has the subset\nof attributes required to identify and reference an existing Glue Job,\nas well as some CloudWatch metric convenience functions to configure an\nevent-driven flow using the job.",
|
|
8750
8821
|
"stability": "experimental",
|
|
8751
8822
|
"summary": "A base class is needed to be able to import existing Jobs into a CDK app to reference as part of a larger stack or construct."
|
|
8752
8823
|
},
|
|
@@ -9297,63 +9368,6 @@
|
|
|
9297
9368
|
"name": "JobBookmarksEncryptionMode",
|
|
9298
9369
|
"symbolId": "lib/security-configuration:JobBookmarksEncryptionMode"
|
|
9299
9370
|
},
|
|
9300
|
-
"@aws-cdk/aws-glue-alpha.JobImportAttributes": {
|
|
9301
|
-
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
9302
|
-
"datatype": true,
|
|
9303
|
-
"docs": {
|
|
9304
|
-
"remarks": "This is ionly used when using fromJobAttributes\nto identify and reference the existing job.",
|
|
9305
|
-
"stability": "experimental",
|
|
9306
|
-
"summary": "A subset of Job attributes are required for importing an existing job into a CDK project.",
|
|
9307
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport { aws_iam as iam } from 'aws-cdk-lib';\n\ndeclare const role: iam.Role;\nconst jobImportAttributes: glue_alpha.JobImportAttributes = {\n jobName: 'jobName',\n\n // the properties below are optional\n role: role,\n};",
|
|
9308
|
-
"custom": {
|
|
9309
|
-
"exampleMetadata": "fixture=_generated"
|
|
9310
|
-
}
|
|
9311
|
-
},
|
|
9312
|
-
"fqn": "@aws-cdk/aws-glue-alpha.JobImportAttributes",
|
|
9313
|
-
"kind": "interface",
|
|
9314
|
-
"locationInModule": {
|
|
9315
|
-
"filename": "lib/jobs/job.ts",
|
|
9316
|
-
"line": 288
|
|
9317
|
-
},
|
|
9318
|
-
"name": "JobImportAttributes",
|
|
9319
|
-
"properties": [
|
|
9320
|
-
{
|
|
9321
|
-
"abstract": true,
|
|
9322
|
-
"docs": {
|
|
9323
|
-
"stability": "experimental",
|
|
9324
|
-
"summary": "The name of the job."
|
|
9325
|
-
},
|
|
9326
|
-
"immutable": true,
|
|
9327
|
-
"locationInModule": {
|
|
9328
|
-
"filename": "lib/jobs/job.ts",
|
|
9329
|
-
"line": 292
|
|
9330
|
-
},
|
|
9331
|
-
"name": "jobName",
|
|
9332
|
-
"type": {
|
|
9333
|
-
"primitive": "string"
|
|
9334
|
-
}
|
|
9335
|
-
},
|
|
9336
|
-
{
|
|
9337
|
-
"abstract": true,
|
|
9338
|
-
"docs": {
|
|
9339
|
-
"default": "- undefined",
|
|
9340
|
-
"stability": "experimental",
|
|
9341
|
-
"summary": "The IAM role assumed by Glue to run this job."
|
|
9342
|
-
},
|
|
9343
|
-
"immutable": true,
|
|
9344
|
-
"locationInModule": {
|
|
9345
|
-
"filename": "lib/jobs/job.ts",
|
|
9346
|
-
"line": 299
|
|
9347
|
-
},
|
|
9348
|
-
"name": "role",
|
|
9349
|
-
"optional": true,
|
|
9350
|
-
"type": {
|
|
9351
|
-
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
9352
|
-
}
|
|
9353
|
-
}
|
|
9354
|
-
],
|
|
9355
|
-
"symbolId": "lib/jobs/job:JobImportAttributes"
|
|
9356
|
-
},
|
|
9357
9371
|
"@aws-cdk/aws-glue-alpha.JobLanguage": {
|
|
9358
9372
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
9359
9373
|
"docs": {
|
|
@@ -9385,24 +9399,24 @@
|
|
|
9385
9399
|
"name": "JobLanguage",
|
|
9386
9400
|
"symbolId": "lib/constants:JobLanguage"
|
|
9387
9401
|
},
|
|
9388
|
-
"@aws-cdk/aws-glue-alpha.
|
|
9402
|
+
"@aws-cdk/aws-glue-alpha.JobProps": {
|
|
9389
9403
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
9390
9404
|
"datatype": true,
|
|
9391
9405
|
"docs": {
|
|
9392
9406
|
"stability": "experimental",
|
|
9393
|
-
"summary": "
|
|
9394
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\n\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst
|
|
9407
|
+
"summary": "JobProps will be used to create new Glue Jobs using this L2 Construct.",
|
|
9408
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\n\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst jobProps: glue_alpha.JobProps = {\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
9395
9409
|
"custom": {
|
|
9396
9410
|
"exampleMetadata": "fixture=_generated"
|
|
9397
9411
|
}
|
|
9398
9412
|
},
|
|
9399
|
-
"fqn": "@aws-cdk/aws-glue-alpha.
|
|
9413
|
+
"fqn": "@aws-cdk/aws-glue-alpha.JobProps",
|
|
9400
9414
|
"kind": "interface",
|
|
9401
9415
|
"locationInModule": {
|
|
9402
9416
|
"filename": "lib/jobs/job.ts",
|
|
9403
9417
|
"line": 306
|
|
9404
9418
|
},
|
|
9405
|
-
"name": "
|
|
9419
|
+
"name": "JobProps",
|
|
9406
9420
|
"properties": [
|
|
9407
9421
|
{
|
|
9408
9422
|
"abstract": true,
|
|
@@ -9712,7 +9726,7 @@
|
|
|
9712
9726
|
}
|
|
9713
9727
|
}
|
|
9714
9728
|
],
|
|
9715
|
-
"symbolId": "lib/jobs/job:
|
|
9729
|
+
"symbolId": "lib/jobs/job:JobProps"
|
|
9716
9730
|
},
|
|
9717
9731
|
"@aws-cdk/aws-glue-alpha.JobState": {
|
|
9718
9732
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
@@ -10320,7 +10334,7 @@
|
|
|
10320
10334
|
},
|
|
10321
10335
|
"@aws-cdk/aws-glue-alpha.PySparkEtlJob": {
|
|
10322
10336
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
10323
|
-
"base": "@aws-cdk/aws-glue-alpha.
|
|
10337
|
+
"base": "@aws-cdk/aws-glue-alpha.SparkJob",
|
|
10324
10338
|
"docs": {
|
|
10325
10339
|
"remarks": "ETL jobs support pySpark and Scala languages, for which there are separate\nbut similar constructors. ETL jobs default to the G2 worker type, but you\ncan override this default with other supported worker type values\n(G1, G2, G4 and G8). ETL jobs defaults to Glue version 4.0, which you can\noverride to 3.0. The following ETL features are enabled by default:\n—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.\nYou can find more details about version, worker type and other features\nin Glue's public documentation.",
|
|
10326
10340
|
"stability": "experimental",
|
|
@@ -10338,7 +10352,7 @@
|
|
|
10338
10352
|
},
|
|
10339
10353
|
"locationInModule": {
|
|
10340
10354
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10341
|
-
"line":
|
|
10355
|
+
"line": 76
|
|
10342
10356
|
},
|
|
10343
10357
|
"parameters": [
|
|
10344
10358
|
{
|
|
@@ -10364,26 +10378,10 @@
|
|
|
10364
10378
|
"kind": "class",
|
|
10365
10379
|
"locationInModule": {
|
|
10366
10380
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10367
|
-
"line":
|
|
10381
|
+
"line": 69
|
|
10368
10382
|
},
|
|
10369
10383
|
"name": "PySparkEtlJob",
|
|
10370
10384
|
"properties": [
|
|
10371
|
-
{
|
|
10372
|
-
"docs": {
|
|
10373
|
-
"stability": "experimental",
|
|
10374
|
-
"summary": "The principal to grant permissions to."
|
|
10375
|
-
},
|
|
10376
|
-
"immutable": true,
|
|
10377
|
-
"locationInModule": {
|
|
10378
|
-
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10379
|
-
"line": 78
|
|
10380
|
-
},
|
|
10381
|
-
"name": "grantPrincipal",
|
|
10382
|
-
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
10383
|
-
"type": {
|
|
10384
|
-
"fqn": "aws-cdk-lib.aws_iam.IPrincipal"
|
|
10385
|
-
}
|
|
10386
|
-
},
|
|
10387
10385
|
{
|
|
10388
10386
|
"docs": {
|
|
10389
10387
|
"stability": "experimental",
|
|
@@ -10392,7 +10390,7 @@
|
|
|
10392
10390
|
"immutable": true,
|
|
10393
10391
|
"locationInModule": {
|
|
10394
10392
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10395
|
-
"line":
|
|
10393
|
+
"line": 70
|
|
10396
10394
|
},
|
|
10397
10395
|
"name": "jobArn",
|
|
10398
10396
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -10408,46 +10406,13 @@
|
|
|
10408
10406
|
"immutable": true,
|
|
10409
10407
|
"locationInModule": {
|
|
10410
10408
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10411
|
-
"line":
|
|
10409
|
+
"line": 71
|
|
10412
10410
|
},
|
|
10413
10411
|
"name": "jobName",
|
|
10414
10412
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
10415
10413
|
"type": {
|
|
10416
10414
|
"primitive": "string"
|
|
10417
10415
|
}
|
|
10418
|
-
},
|
|
10419
|
-
{
|
|
10420
|
-
"docs": {
|
|
10421
|
-
"stability": "experimental",
|
|
10422
|
-
"summary": "The IAM role Glue assumes to run this job."
|
|
10423
|
-
},
|
|
10424
|
-
"immutable": true,
|
|
10425
|
-
"locationInModule": {
|
|
10426
|
-
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10427
|
-
"line": 77
|
|
10428
|
-
},
|
|
10429
|
-
"name": "role",
|
|
10430
|
-
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
10431
|
-
"type": {
|
|
10432
|
-
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
10433
|
-
}
|
|
10434
|
-
},
|
|
10435
|
-
{
|
|
10436
|
-
"docs": {
|
|
10437
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10438
|
-
"stability": "experimental",
|
|
10439
|
-
"summary": "The Spark UI logs location if Spark UI monitoring and debugging is enabled."
|
|
10440
|
-
},
|
|
10441
|
-
"immutable": true,
|
|
10442
|
-
"locationInModule": {
|
|
10443
|
-
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10444
|
-
"line": 86
|
|
10445
|
-
},
|
|
10446
|
-
"name": "sparkUILoggingLocation",
|
|
10447
|
-
"optional": true,
|
|
10448
|
-
"type": {
|
|
10449
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation"
|
|
10450
|
-
}
|
|
10451
10416
|
}
|
|
10452
10417
|
],
|
|
10453
10418
|
"symbolId": "lib/jobs/pyspark-etl-job:PySparkEtlJob"
|
|
@@ -10465,12 +10430,12 @@
|
|
|
10465
10430
|
},
|
|
10466
10431
|
"fqn": "@aws-cdk/aws-glue-alpha.PySparkEtlJobProps",
|
|
10467
10432
|
"interfaces": [
|
|
10468
|
-
"@aws-cdk/aws-glue-alpha.
|
|
10433
|
+
"@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
10469
10434
|
],
|
|
10470
10435
|
"kind": "interface",
|
|
10471
10436
|
"locationInModule": {
|
|
10472
10437
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10473
|
-
"line":
|
|
10438
|
+
"line": 10
|
|
10474
10439
|
},
|
|
10475
10440
|
"name": "PySparkEtlJobProps",
|
|
10476
10441
|
"properties": [
|
|
@@ -10485,7 +10450,7 @@
|
|
|
10485
10450
|
"immutable": true,
|
|
10486
10451
|
"locationInModule": {
|
|
10487
10452
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10488
|
-
"line":
|
|
10453
|
+
"line": 26
|
|
10489
10454
|
},
|
|
10490
10455
|
"name": "extraFiles",
|
|
10491
10456
|
"optional": true,
|
|
@@ -10508,7 +10473,7 @@
|
|
|
10508
10473
|
"immutable": true,
|
|
10509
10474
|
"locationInModule": {
|
|
10510
10475
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10511
|
-
"line":
|
|
10476
|
+
"line": 33
|
|
10512
10477
|
},
|
|
10513
10478
|
"name": "extraJars",
|
|
10514
10479
|
"optional": true,
|
|
@@ -10521,6 +10486,25 @@
|
|
|
10521
10486
|
}
|
|
10522
10487
|
}
|
|
10523
10488
|
},
|
|
10489
|
+
{
|
|
10490
|
+
"abstract": true,
|
|
10491
|
+
"docs": {
|
|
10492
|
+
"default": "false - priority is not given to user-provided jars",
|
|
10493
|
+
"see": " `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10494
|
+
"stability": "experimental",
|
|
10495
|
+
"summary": "Setting this value to true prioritizes the customer's extra JAR files in the classpath."
|
|
10496
|
+
},
|
|
10497
|
+
"immutable": true,
|
|
10498
|
+
"locationInModule": {
|
|
10499
|
+
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10500
|
+
"line": 42
|
|
10501
|
+
},
|
|
10502
|
+
"name": "extraJarsFirst",
|
|
10503
|
+
"optional": true,
|
|
10504
|
+
"type": {
|
|
10505
|
+
"primitive": "boolean"
|
|
10506
|
+
}
|
|
10507
|
+
},
|
|
10524
10508
|
{
|
|
10525
10509
|
"abstract": true,
|
|
10526
10510
|
"docs": {
|
|
@@ -10531,7 +10515,7 @@
|
|
|
10531
10515
|
"immutable": true,
|
|
10532
10516
|
"locationInModule": {
|
|
10533
10517
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10534
|
-
"line":
|
|
10518
|
+
"line": 17
|
|
10535
10519
|
},
|
|
10536
10520
|
"name": "extraPythonFiles",
|
|
10537
10521
|
"optional": true,
|
|
@@ -10555,39 +10539,20 @@
|
|
|
10555
10539
|
"immutable": true,
|
|
10556
10540
|
"locationInModule": {
|
|
10557
10541
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10558
|
-
"line":
|
|
10542
|
+
"line": 54
|
|
10559
10543
|
},
|
|
10560
10544
|
"name": "jobRunQueuingEnabled",
|
|
10561
10545
|
"optional": true,
|
|
10562
10546
|
"type": {
|
|
10563
10547
|
"primitive": "boolean"
|
|
10564
10548
|
}
|
|
10565
|
-
},
|
|
10566
|
-
{
|
|
10567
|
-
"abstract": true,
|
|
10568
|
-
"docs": {
|
|
10569
|
-
"default": "- Spark UI debugging and monitoring is disabled.",
|
|
10570
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10571
|
-
"stability": "experimental",
|
|
10572
|
-
"summary": "Enables the Spark UI debugging and monitoring with the specified props."
|
|
10573
|
-
},
|
|
10574
|
-
"immutable": true,
|
|
10575
|
-
"locationInModule": {
|
|
10576
|
-
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
10577
|
-
"line": 23
|
|
10578
|
-
},
|
|
10579
|
-
"name": "sparkUI",
|
|
10580
|
-
"optional": true,
|
|
10581
|
-
"type": {
|
|
10582
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUIProps"
|
|
10583
|
-
}
|
|
10584
10549
|
}
|
|
10585
10550
|
],
|
|
10586
10551
|
"symbolId": "lib/jobs/pyspark-etl-job:PySparkEtlJobProps"
|
|
10587
10552
|
},
|
|
10588
10553
|
"@aws-cdk/aws-glue-alpha.PySparkFlexEtlJob": {
|
|
10589
10554
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
10590
|
-
"base": "@aws-cdk/aws-glue-alpha.
|
|
10555
|
+
"base": "@aws-cdk/aws-glue-alpha.SparkJob",
|
|
10591
10556
|
"docs": {
|
|
10592
10557
|
"remarks": "Flex jobs supports Python and Scala language.\nThe flexible execution class is appropriate for non-urgent jobs such as\npre-production jobs, testing, and one-time data loads.\nFlexible job runs are supported for jobs using AWS Glue version 3.0 or later and G.1X or\nG.2X worker types but will default to the latest version of Glue (currently Glue 3.0.)\n\nSimilar to ETL, we’ll enable these features: —enable-metrics, —enable-spark-ui,\n—enable-continuous-cloudwatch-log",
|
|
10593
10558
|
"stability": "experimental",
|
|
@@ -10605,7 +10570,7 @@
|
|
|
10605
10570
|
},
|
|
10606
10571
|
"locationInModule": {
|
|
10607
10572
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10608
|
-
"line":
|
|
10573
|
+
"line": 72
|
|
10609
10574
|
},
|
|
10610
10575
|
"parameters": [
|
|
10611
10576
|
{
|
|
@@ -10631,26 +10596,10 @@
|
|
|
10631
10596
|
"kind": "class",
|
|
10632
10597
|
"locationInModule": {
|
|
10633
10598
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10634
|
-
"line":
|
|
10599
|
+
"line": 65
|
|
10635
10600
|
},
|
|
10636
10601
|
"name": "PySparkFlexEtlJob",
|
|
10637
10602
|
"properties": [
|
|
10638
|
-
{
|
|
10639
|
-
"docs": {
|
|
10640
|
-
"stability": "experimental",
|
|
10641
|
-
"summary": "The principal to grant permissions to."
|
|
10642
|
-
},
|
|
10643
|
-
"immutable": true,
|
|
10644
|
-
"locationInModule": {
|
|
10645
|
-
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10646
|
-
"line": 76
|
|
10647
|
-
},
|
|
10648
|
-
"name": "grantPrincipal",
|
|
10649
|
-
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
10650
|
-
"type": {
|
|
10651
|
-
"fqn": "aws-cdk-lib.aws_iam.IPrincipal"
|
|
10652
|
-
}
|
|
10653
|
-
},
|
|
10654
10603
|
{
|
|
10655
10604
|
"docs": {
|
|
10656
10605
|
"stability": "experimental",
|
|
@@ -10659,7 +10608,7 @@
|
|
|
10659
10608
|
"immutable": true,
|
|
10660
10609
|
"locationInModule": {
|
|
10661
10610
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10662
|
-
"line":
|
|
10611
|
+
"line": 66
|
|
10663
10612
|
},
|
|
10664
10613
|
"name": "jobArn",
|
|
10665
10614
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -10675,46 +10624,13 @@
|
|
|
10675
10624
|
"immutable": true,
|
|
10676
10625
|
"locationInModule": {
|
|
10677
10626
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10678
|
-
"line":
|
|
10627
|
+
"line": 67
|
|
10679
10628
|
},
|
|
10680
10629
|
"name": "jobName",
|
|
10681
10630
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
10682
10631
|
"type": {
|
|
10683
10632
|
"primitive": "string"
|
|
10684
10633
|
}
|
|
10685
|
-
},
|
|
10686
|
-
{
|
|
10687
|
-
"docs": {
|
|
10688
|
-
"stability": "experimental",
|
|
10689
|
-
"summary": "The IAM role Glue assumes to run this job."
|
|
10690
|
-
},
|
|
10691
|
-
"immutable": true,
|
|
10692
|
-
"locationInModule": {
|
|
10693
|
-
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10694
|
-
"line": 75
|
|
10695
|
-
},
|
|
10696
|
-
"name": "role",
|
|
10697
|
-
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
10698
|
-
"type": {
|
|
10699
|
-
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
10700
|
-
}
|
|
10701
|
-
},
|
|
10702
|
-
{
|
|
10703
|
-
"docs": {
|
|
10704
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10705
|
-
"stability": "experimental",
|
|
10706
|
-
"summary": "The Spark UI logs location if Spark UI monitoring and debugging is enabled."
|
|
10707
|
-
},
|
|
10708
|
-
"immutable": true,
|
|
10709
|
-
"locationInModule": {
|
|
10710
|
-
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10711
|
-
"line": 84
|
|
10712
|
-
},
|
|
10713
|
-
"name": "sparkUILoggingLocation",
|
|
10714
|
-
"optional": true,
|
|
10715
|
-
"type": {
|
|
10716
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation"
|
|
10717
|
-
}
|
|
10718
10634
|
}
|
|
10719
10635
|
],
|
|
10720
10636
|
"symbolId": "lib/jobs/pyspark-flex-etl-job:PySparkFlexEtlJob"
|
|
@@ -10732,12 +10648,12 @@
|
|
|
10732
10648
|
},
|
|
10733
10649
|
"fqn": "@aws-cdk/aws-glue-alpha.PySparkFlexEtlJobProps",
|
|
10734
10650
|
"interfaces": [
|
|
10735
|
-
"@aws-cdk/aws-glue-alpha.
|
|
10651
|
+
"@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
10736
10652
|
],
|
|
10737
10653
|
"kind": "interface",
|
|
10738
10654
|
"locationInModule": {
|
|
10739
10655
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10740
|
-
"line":
|
|
10656
|
+
"line": 11
|
|
10741
10657
|
},
|
|
10742
10658
|
"name": "PySparkFlexEtlJobProps",
|
|
10743
10659
|
"properties": [
|
|
@@ -10752,7 +10668,7 @@
|
|
|
10752
10668
|
"immutable": true,
|
|
10753
10669
|
"locationInModule": {
|
|
10754
10670
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10755
|
-
"line":
|
|
10671
|
+
"line": 34
|
|
10756
10672
|
},
|
|
10757
10673
|
"name": "extraFiles",
|
|
10758
10674
|
"optional": true,
|
|
@@ -10775,7 +10691,7 @@
|
|
|
10775
10691
|
"immutable": true,
|
|
10776
10692
|
"locationInModule": {
|
|
10777
10693
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10778
|
-
"line":
|
|
10694
|
+
"line": 41
|
|
10779
10695
|
},
|
|
10780
10696
|
"name": "extraJars",
|
|
10781
10697
|
"optional": true,
|
|
@@ -10791,63 +10707,62 @@
|
|
|
10791
10707
|
{
|
|
10792
10708
|
"abstract": true,
|
|
10793
10709
|
"docs": {
|
|
10794
|
-
"default": "-
|
|
10795
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10710
|
+
"default": "false - priority is not given to user-provided jars",
|
|
10711
|
+
"see": " `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10796
10712
|
"stability": "experimental",
|
|
10797
|
-
"summary": "
|
|
10713
|
+
"summary": "Setting this value to true prioritizes the customer's extra JAR files in the classpath."
|
|
10798
10714
|
},
|
|
10799
10715
|
"immutable": true,
|
|
10800
10716
|
"locationInModule": {
|
|
10801
10717
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10802
|
-
"line":
|
|
10718
|
+
"line": 50
|
|
10803
10719
|
},
|
|
10804
|
-
"name": "
|
|
10720
|
+
"name": "extraJarsFirst",
|
|
10805
10721
|
"optional": true,
|
|
10806
10722
|
"type": {
|
|
10807
|
-
"
|
|
10808
|
-
"elementtype": {
|
|
10809
|
-
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
10810
|
-
},
|
|
10811
|
-
"kind": "array"
|
|
10812
|
-
}
|
|
10723
|
+
"primitive": "boolean"
|
|
10813
10724
|
}
|
|
10814
10725
|
},
|
|
10815
10726
|
{
|
|
10816
10727
|
"abstract": true,
|
|
10817
10728
|
"docs": {
|
|
10818
|
-
"default": "-
|
|
10819
|
-
"remarks": "After a job run starts, the number of minutes to wait before sending a job run delay notification.",
|
|
10729
|
+
"default": "- no extra files",
|
|
10820
10730
|
"stability": "experimental",
|
|
10821
|
-
"summary": "
|
|
10731
|
+
"summary": "Extra Python Files S3 URL (optional) S3 URL where additional python dependencies are located."
|
|
10822
10732
|
},
|
|
10823
10733
|
"immutable": true,
|
|
10824
10734
|
"locationInModule": {
|
|
10825
10735
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10826
|
-
"line":
|
|
10736
|
+
"line": 25
|
|
10827
10737
|
},
|
|
10828
|
-
"name": "
|
|
10738
|
+
"name": "extraPythonFiles",
|
|
10829
10739
|
"optional": true,
|
|
10830
10740
|
"type": {
|
|
10831
|
-
"
|
|
10741
|
+
"collection": {
|
|
10742
|
+
"elementtype": {
|
|
10743
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
10744
|
+
},
|
|
10745
|
+
"kind": "array"
|
|
10746
|
+
}
|
|
10832
10747
|
}
|
|
10833
10748
|
},
|
|
10834
10749
|
{
|
|
10835
10750
|
"abstract": true,
|
|
10836
10751
|
"docs": {
|
|
10837
|
-
"default": "-
|
|
10838
|
-
"
|
|
10752
|
+
"default": "- undefined",
|
|
10753
|
+
"remarks": "After a job run starts, the number of minutes to wait before sending a job run delay notification.",
|
|
10839
10754
|
"stability": "experimental",
|
|
10840
|
-
"summary": "
|
|
10755
|
+
"summary": "Specifies configuration properties of a notification (optional)."
|
|
10841
10756
|
},
|
|
10842
10757
|
"immutable": true,
|
|
10843
10758
|
"locationInModule": {
|
|
10844
10759
|
"filename": "lib/jobs/pyspark-flex-etl-job.ts",
|
|
10845
|
-
"line":
|
|
10760
|
+
"line": 17
|
|
10846
10761
|
},
|
|
10847
|
-
"name": "
|
|
10762
|
+
"name": "notifyDelayAfter",
|
|
10848
10763
|
"optional": true,
|
|
10849
10764
|
"type": {
|
|
10850
|
-
"fqn": "
|
|
10765
|
+
"fqn": "aws-cdk-lib.Duration"
|
|
10851
10766
|
}
|
|
10852
10767
|
}
|
|
10853
10768
|
],
|
|
@@ -10855,7 +10770,7 @@
|
|
|
10855
10770
|
},
|
|
10856
10771
|
"@aws-cdk/aws-glue-alpha.PySparkStreamingJob": {
|
|
10857
10772
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
10858
|
-
"base": "@aws-cdk/aws-glue-alpha.
|
|
10773
|
+
"base": "@aws-cdk/aws-glue-alpha.SparkJob",
|
|
10859
10774
|
"docs": {
|
|
10860
10775
|
"remarks": "A Streaming job is similar to an ETL job, except that it performs ETL on data streams\nusing the Apache Spark Structured Streaming framework.\nThese jobs will default to use Python 3.9.\n\nSimilar to ETL jobs, streaming job supports Scala and Python languages. Similar to ETL,\nit supports G1 and G2 worker type and 2.0, 3.0 and 4.0 version. We’ll default to G2 worker\nand 4.0 version for streaming jobs which developers can override.\nWe will enable —enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.",
|
|
10861
10776
|
"stability": "experimental",
|
|
@@ -10873,7 +10788,7 @@
|
|
|
10873
10788
|
},
|
|
10874
10789
|
"locationInModule": {
|
|
10875
10790
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10876
|
-
"line":
|
|
10791
|
+
"line": 76
|
|
10877
10792
|
},
|
|
10878
10793
|
"parameters": [
|
|
10879
10794
|
{
|
|
@@ -10899,26 +10814,10 @@
|
|
|
10899
10814
|
"kind": "class",
|
|
10900
10815
|
"locationInModule": {
|
|
10901
10816
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10902
|
-
"line":
|
|
10817
|
+
"line": 69
|
|
10903
10818
|
},
|
|
10904
10819
|
"name": "PySparkStreamingJob",
|
|
10905
10820
|
"properties": [
|
|
10906
|
-
{
|
|
10907
|
-
"docs": {
|
|
10908
|
-
"stability": "experimental",
|
|
10909
|
-
"summary": "The principal to grant permissions to."
|
|
10910
|
-
},
|
|
10911
|
-
"immutable": true,
|
|
10912
|
-
"locationInModule": {
|
|
10913
|
-
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10914
|
-
"line": 78
|
|
10915
|
-
},
|
|
10916
|
-
"name": "grantPrincipal",
|
|
10917
|
-
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
10918
|
-
"type": {
|
|
10919
|
-
"fqn": "aws-cdk-lib.aws_iam.IPrincipal"
|
|
10920
|
-
}
|
|
10921
|
-
},
|
|
10922
10821
|
{
|
|
10923
10822
|
"docs": {
|
|
10924
10823
|
"stability": "experimental",
|
|
@@ -10927,7 +10826,7 @@
|
|
|
10927
10826
|
"immutable": true,
|
|
10928
10827
|
"locationInModule": {
|
|
10929
10828
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10930
|
-
"line":
|
|
10829
|
+
"line": 70
|
|
10931
10830
|
},
|
|
10932
10831
|
"name": "jobArn",
|
|
10933
10832
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -10943,46 +10842,13 @@
|
|
|
10943
10842
|
"immutable": true,
|
|
10944
10843
|
"locationInModule": {
|
|
10945
10844
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10946
|
-
"line":
|
|
10845
|
+
"line": 71
|
|
10947
10846
|
},
|
|
10948
10847
|
"name": "jobName",
|
|
10949
10848
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
10950
10849
|
"type": {
|
|
10951
10850
|
"primitive": "string"
|
|
10952
10851
|
}
|
|
10953
|
-
},
|
|
10954
|
-
{
|
|
10955
|
-
"docs": {
|
|
10956
|
-
"stability": "experimental",
|
|
10957
|
-
"summary": "The IAM role Glue assumes to run this job."
|
|
10958
|
-
},
|
|
10959
|
-
"immutable": true,
|
|
10960
|
-
"locationInModule": {
|
|
10961
|
-
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10962
|
-
"line": 77
|
|
10963
|
-
},
|
|
10964
|
-
"name": "role",
|
|
10965
|
-
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
10966
|
-
"type": {
|
|
10967
|
-
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
10968
|
-
}
|
|
10969
|
-
},
|
|
10970
|
-
{
|
|
10971
|
-
"docs": {
|
|
10972
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10973
|
-
"stability": "experimental",
|
|
10974
|
-
"summary": "The Spark UI logs location if Spark UI monitoring and debugging is enabled."
|
|
10975
|
-
},
|
|
10976
|
-
"immutable": true,
|
|
10977
|
-
"locationInModule": {
|
|
10978
|
-
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10979
|
-
"line": 86
|
|
10980
|
-
},
|
|
10981
|
-
"name": "sparkUILoggingLocation",
|
|
10982
|
-
"optional": true,
|
|
10983
|
-
"type": {
|
|
10984
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation"
|
|
10985
|
-
}
|
|
10986
10852
|
}
|
|
10987
10853
|
],
|
|
10988
10854
|
"symbolId": "lib/jobs/pyspark-streaming-job:PySparkStreamingJob"
|
|
@@ -11000,12 +10866,12 @@
|
|
|
11000
10866
|
},
|
|
11001
10867
|
"fqn": "@aws-cdk/aws-glue-alpha.PySparkStreamingJobProps",
|
|
11002
10868
|
"interfaces": [
|
|
11003
|
-
"@aws-cdk/aws-glue-alpha.
|
|
10869
|
+
"@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
11004
10870
|
],
|
|
11005
10871
|
"kind": "interface",
|
|
11006
10872
|
"locationInModule": {
|
|
11007
10873
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
11008
|
-
"line":
|
|
10874
|
+
"line": 10
|
|
11009
10875
|
},
|
|
11010
10876
|
"name": "PySparkStreamingJobProps",
|
|
11011
10877
|
"properties": [
|
|
@@ -11020,7 +10886,7 @@
|
|
|
11020
10886
|
"immutable": true,
|
|
11021
10887
|
"locationInModule": {
|
|
11022
10888
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
11023
|
-
"line":
|
|
10889
|
+
"line": 26
|
|
11024
10890
|
},
|
|
11025
10891
|
"name": "extraFiles",
|
|
11026
10892
|
"optional": true,
|
|
@@ -11043,7 +10909,7 @@
|
|
|
11043
10909
|
"immutable": true,
|
|
11044
10910
|
"locationInModule": {
|
|
11045
10911
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
11046
|
-
"line":
|
|
10912
|
+
"line": 33
|
|
11047
10913
|
},
|
|
11048
10914
|
"name": "extraJars",
|
|
11049
10915
|
"optional": true,
|
|
@@ -11056,6 +10922,25 @@
|
|
|
11056
10922
|
}
|
|
11057
10923
|
}
|
|
11058
10924
|
},
|
|
10925
|
+
{
|
|
10926
|
+
"abstract": true,
|
|
10927
|
+
"docs": {
|
|
10928
|
+
"default": "false - priority is not given to user-provided jars",
|
|
10929
|
+
"see": " `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
10930
|
+
"stability": "experimental",
|
|
10931
|
+
"summary": "Setting this value to true prioritizes the customer's extra JAR files in the classpath."
|
|
10932
|
+
},
|
|
10933
|
+
"immutable": true,
|
|
10934
|
+
"locationInModule": {
|
|
10935
|
+
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
10936
|
+
"line": 42
|
|
10937
|
+
},
|
|
10938
|
+
"name": "extraJarsFirst",
|
|
10939
|
+
"optional": true,
|
|
10940
|
+
"type": {
|
|
10941
|
+
"primitive": "boolean"
|
|
10942
|
+
}
|
|
10943
|
+
},
|
|
11059
10944
|
{
|
|
11060
10945
|
"abstract": true,
|
|
11061
10946
|
"docs": {
|
|
@@ -11066,7 +10951,7 @@
|
|
|
11066
10951
|
"immutable": true,
|
|
11067
10952
|
"locationInModule": {
|
|
11068
10953
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
11069
|
-
"line":
|
|
10954
|
+
"line": 17
|
|
11070
10955
|
},
|
|
11071
10956
|
"name": "extraPythonFiles",
|
|
11072
10957
|
"optional": true,
|
|
@@ -11090,32 +10975,13 @@
|
|
|
11090
10975
|
"immutable": true,
|
|
11091
10976
|
"locationInModule": {
|
|
11092
10977
|
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
11093
|
-
"line":
|
|
10978
|
+
"line": 54
|
|
11094
10979
|
},
|
|
11095
10980
|
"name": "jobRunQueuingEnabled",
|
|
11096
10981
|
"optional": true,
|
|
11097
10982
|
"type": {
|
|
11098
10983
|
"primitive": "boolean"
|
|
11099
10984
|
}
|
|
11100
|
-
},
|
|
11101
|
-
{
|
|
11102
|
-
"abstract": true,
|
|
11103
|
-
"docs": {
|
|
11104
|
-
"default": "- Spark UI debugging and monitoring is disabled.",
|
|
11105
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
11106
|
-
"stability": "experimental",
|
|
11107
|
-
"summary": "Enables the Spark UI debugging and monitoring with the specified props."
|
|
11108
|
-
},
|
|
11109
|
-
"immutable": true,
|
|
11110
|
-
"locationInModule": {
|
|
11111
|
-
"filename": "lib/jobs/pyspark-streaming-job.ts",
|
|
11112
|
-
"line": 23
|
|
11113
|
-
},
|
|
11114
|
-
"name": "sparkUI",
|
|
11115
|
-
"optional": true,
|
|
11116
|
-
"type": {
|
|
11117
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUIProps"
|
|
11118
|
-
}
|
|
11119
10985
|
}
|
|
11120
10986
|
],
|
|
11121
10987
|
"symbolId": "lib/jobs/pyspark-streaming-job:PySparkStreamingJobProps"
|
|
@@ -11250,7 +11116,7 @@
|
|
|
11250
11116
|
},
|
|
11251
11117
|
"fqn": "@aws-cdk/aws-glue-alpha.PythonShellJobProps",
|
|
11252
11118
|
"interfaces": [
|
|
11253
|
-
"@aws-cdk/aws-glue-alpha.
|
|
11119
|
+
"@aws-cdk/aws-glue-alpha.JobProps"
|
|
11254
11120
|
],
|
|
11255
11121
|
"kind": "interface",
|
|
11256
11122
|
"locationInModule": {
|
|
@@ -11489,7 +11355,7 @@
|
|
|
11489
11355
|
},
|
|
11490
11356
|
"fqn": "@aws-cdk/aws-glue-alpha.RayJobProps",
|
|
11491
11357
|
"interfaces": [
|
|
11492
|
-
"@aws-cdk/aws-glue-alpha.
|
|
11358
|
+
"@aws-cdk/aws-glue-alpha.JobProps"
|
|
11493
11359
|
],
|
|
11494
11360
|
"kind": "interface",
|
|
11495
11361
|
"locationInModule": {
|
|
@@ -12122,12 +11988,12 @@
|
|
|
12122
11988
|
},
|
|
12123
11989
|
"@aws-cdk/aws-glue-alpha.ScalaSparkEtlJob": {
|
|
12124
11990
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
12125
|
-
"base": "@aws-cdk/aws-glue-alpha.
|
|
11991
|
+
"base": "@aws-cdk/aws-glue-alpha.SparkJob",
|
|
12126
11992
|
"docs": {
|
|
12127
11993
|
"remarks": "ETL jobs support pySpark and Scala languages, for which there are separate\nbut similar constructors. ETL jobs default to the G2 worker type, but you\ncan override this default with other supported worker type values\n(G1, G2, G4 and G8). ETL jobs defaults to Glue version 4.0, which you can\noverride to 3.0. The following ETL features are enabled by default:\n—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.\nYou can find more details about version, worker type and other features\nin Glue's public documentation.",
|
|
12128
11994
|
"stability": "experimental",
|
|
12129
11995
|
"summary": "Spark ETL Jobs class.",
|
|
12130
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJob = new glue_alpha.ScalaSparkEtlJob(this, 'MyScalaSparkEtlJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraJars: [code],\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n
|
|
11996
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJob = new glue_alpha.ScalaSparkEtlJob(this, 'MyScalaSparkEtlJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n});",
|
|
12131
11997
|
"custom": {
|
|
12132
11998
|
"exampleMetadata": "fixture=_generated"
|
|
12133
11999
|
}
|
|
@@ -12140,7 +12006,7 @@
|
|
|
12140
12006
|
},
|
|
12141
12007
|
"locationInModule": {
|
|
12142
12008
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12143
|
-
"line":
|
|
12009
|
+
"line": 75
|
|
12144
12010
|
},
|
|
12145
12011
|
"parameters": [
|
|
12146
12012
|
{
|
|
@@ -12166,26 +12032,10 @@
|
|
|
12166
12032
|
"kind": "class",
|
|
12167
12033
|
"locationInModule": {
|
|
12168
12034
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12169
|
-
"line":
|
|
12035
|
+
"line": 68
|
|
12170
12036
|
},
|
|
12171
12037
|
"name": "ScalaSparkEtlJob",
|
|
12172
12038
|
"properties": [
|
|
12173
|
-
{
|
|
12174
|
-
"docs": {
|
|
12175
|
-
"stability": "experimental",
|
|
12176
|
-
"summary": "The principal to grant permissions to."
|
|
12177
|
-
},
|
|
12178
|
-
"immutable": true,
|
|
12179
|
-
"locationInModule": {
|
|
12180
|
-
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12181
|
-
"line": 68
|
|
12182
|
-
},
|
|
12183
|
-
"name": "grantPrincipal",
|
|
12184
|
-
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
12185
|
-
"type": {
|
|
12186
|
-
"fqn": "aws-cdk-lib.aws_iam.IPrincipal"
|
|
12187
|
-
}
|
|
12188
|
-
},
|
|
12189
12039
|
{
|
|
12190
12040
|
"docs": {
|
|
12191
12041
|
"stability": "experimental",
|
|
@@ -12194,7 +12044,7 @@
|
|
|
12194
12044
|
"immutable": true,
|
|
12195
12045
|
"locationInModule": {
|
|
12196
12046
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12197
|
-
"line":
|
|
12047
|
+
"line": 69
|
|
12198
12048
|
},
|
|
12199
12049
|
"name": "jobArn",
|
|
12200
12050
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -12210,46 +12060,13 @@
|
|
|
12210
12060
|
"immutable": true,
|
|
12211
12061
|
"locationInModule": {
|
|
12212
12062
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12213
|
-
"line":
|
|
12063
|
+
"line": 70
|
|
12214
12064
|
},
|
|
12215
12065
|
"name": "jobName",
|
|
12216
12066
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
12217
12067
|
"type": {
|
|
12218
12068
|
"primitive": "string"
|
|
12219
12069
|
}
|
|
12220
|
-
},
|
|
12221
|
-
{
|
|
12222
|
-
"docs": {
|
|
12223
|
-
"stability": "experimental",
|
|
12224
|
-
"summary": "The IAM role Glue assumes to run this job."
|
|
12225
|
-
},
|
|
12226
|
-
"immutable": true,
|
|
12227
|
-
"locationInModule": {
|
|
12228
|
-
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12229
|
-
"line": 67
|
|
12230
|
-
},
|
|
12231
|
-
"name": "role",
|
|
12232
|
-
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
12233
|
-
"type": {
|
|
12234
|
-
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
12235
|
-
}
|
|
12236
|
-
},
|
|
12237
|
-
{
|
|
12238
|
-
"docs": {
|
|
12239
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12240
|
-
"stability": "experimental",
|
|
12241
|
-
"summary": "The Spark UI logs location if Spark UI monitoring and debugging is enabled."
|
|
12242
|
-
},
|
|
12243
|
-
"immutable": true,
|
|
12244
|
-
"locationInModule": {
|
|
12245
|
-
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12246
|
-
"line": 76
|
|
12247
|
-
},
|
|
12248
|
-
"name": "sparkUILoggingLocation",
|
|
12249
|
-
"optional": true,
|
|
12250
|
-
"type": {
|
|
12251
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation"
|
|
12252
|
-
}
|
|
12253
12070
|
}
|
|
12254
12071
|
],
|
|
12255
12072
|
"symbolId": "lib/jobs/scala-spark-etl-job:ScalaSparkEtlJob"
|
|
@@ -12260,19 +12077,19 @@
|
|
|
12260
12077
|
"docs": {
|
|
12261
12078
|
"stability": "experimental",
|
|
12262
12079
|
"summary": "Properties for creating a Scala Spark ETL job.",
|
|
12263
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJobProps: glue_alpha.ScalaSparkEtlJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraJars: [code],\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n
|
|
12080
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJobProps: glue_alpha.ScalaSparkEtlJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
12264
12081
|
"custom": {
|
|
12265
12082
|
"exampleMetadata": "fixture=_generated"
|
|
12266
12083
|
}
|
|
12267
12084
|
},
|
|
12268
12085
|
"fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkEtlJobProps",
|
|
12269
12086
|
"interfaces": [
|
|
12270
|
-
"@aws-cdk/aws-glue-alpha.
|
|
12087
|
+
"@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
12271
12088
|
],
|
|
12272
12089
|
"kind": "interface",
|
|
12273
12090
|
"locationInModule": {
|
|
12274
12091
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12275
|
-
"line":
|
|
12092
|
+
"line": 10
|
|
12276
12093
|
},
|
|
12277
12094
|
"name": "ScalaSparkEtlJobProps",
|
|
12278
12095
|
"properties": [
|
|
@@ -12285,7 +12102,7 @@
|
|
|
12285
12102
|
"immutable": true,
|
|
12286
12103
|
"locationInModule": {
|
|
12287
12104
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12288
|
-
"line":
|
|
12105
|
+
"line": 16
|
|
12289
12106
|
},
|
|
12290
12107
|
"name": "className",
|
|
12291
12108
|
"type": {
|
|
@@ -12295,16 +12112,17 @@
|
|
|
12295
12112
|
{
|
|
12296
12113
|
"abstract": true,
|
|
12297
12114
|
"docs": {
|
|
12298
|
-
"default": "- no extra
|
|
12115
|
+
"default": "- no extra files specified.",
|
|
12116
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12299
12117
|
"stability": "experimental",
|
|
12300
|
-
"summary": "
|
|
12118
|
+
"summary": "Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it."
|
|
12301
12119
|
},
|
|
12302
12120
|
"immutable": true,
|
|
12303
12121
|
"locationInModule": {
|
|
12304
12122
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12305
|
-
"line":
|
|
12123
|
+
"line": 25
|
|
12306
12124
|
},
|
|
12307
|
-
"name": "
|
|
12125
|
+
"name": "extraFiles",
|
|
12308
12126
|
"optional": true,
|
|
12309
12127
|
"type": {
|
|
12310
12128
|
"collection": {
|
|
@@ -12318,52 +12136,75 @@
|
|
|
12318
12136
|
{
|
|
12319
12137
|
"abstract": true,
|
|
12320
12138
|
"docs": {
|
|
12321
|
-
"default": "- no
|
|
12322
|
-
"remarks": "A value of true means job run queuing is enabled for the job runs.\nIf false or not populated, the job runs will not be considered for queueing.\nIf this field does not match the value set in the job run, then the value from\nthe job run field will be used. This property must be set to false for flex jobs.\nIf this property is enabled, maxRetries must be set to zero.",
|
|
12139
|
+
"default": "- no extra jar files",
|
|
12323
12140
|
"stability": "experimental",
|
|
12324
|
-
"summary": "
|
|
12141
|
+
"summary": "Extra Jars S3 URL (optional) S3 URL where additional jar dependencies are located."
|
|
12325
12142
|
},
|
|
12326
12143
|
"immutable": true,
|
|
12327
12144
|
"locationInModule": {
|
|
12328
12145
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12329
|
-
"line":
|
|
12146
|
+
"line": 32
|
|
12330
12147
|
},
|
|
12331
|
-
"name": "
|
|
12148
|
+
"name": "extraJars",
|
|
12332
12149
|
"optional": true,
|
|
12333
12150
|
"type": {
|
|
12334
|
-
"
|
|
12151
|
+
"collection": {
|
|
12152
|
+
"elementtype": {
|
|
12153
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
12154
|
+
},
|
|
12155
|
+
"kind": "array"
|
|
12156
|
+
}
|
|
12335
12157
|
}
|
|
12336
12158
|
},
|
|
12337
12159
|
{
|
|
12338
12160
|
"abstract": true,
|
|
12339
12161
|
"docs": {
|
|
12340
|
-
"default": "-
|
|
12341
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12162
|
+
"default": "false - priority is not given to user-provided jars",
|
|
12163
|
+
"see": " `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12342
12164
|
"stability": "experimental",
|
|
12343
|
-
"summary": "
|
|
12165
|
+
"summary": "Setting this value to true prioritizes the customer's extra JAR files in the classpath."
|
|
12344
12166
|
},
|
|
12345
12167
|
"immutable": true,
|
|
12346
12168
|
"locationInModule": {
|
|
12347
12169
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12348
|
-
"line":
|
|
12170
|
+
"line": 41
|
|
12349
12171
|
},
|
|
12350
|
-
"name": "
|
|
12172
|
+
"name": "extraJarsFirst",
|
|
12351
12173
|
"optional": true,
|
|
12352
12174
|
"type": {
|
|
12353
|
-
"
|
|
12175
|
+
"primitive": "boolean"
|
|
12354
12176
|
}
|
|
12355
|
-
}
|
|
12177
|
+
},
|
|
12178
|
+
{
|
|
12179
|
+
"abstract": true,
|
|
12180
|
+
"docs": {
|
|
12181
|
+
"default": "- no job run queuing",
|
|
12182
|
+
"remarks": "A value of true means job run queuing is enabled for the job runs.\nIf false or not populated, the job runs will not be considered for queueing.\nIf this field does not match the value set in the job run, then the value from\nthe job run field will be used. This property must be set to false for flex jobs.\nIf this property is enabled, maxRetries must be set to zero.",
|
|
12183
|
+
"stability": "experimental",
|
|
12184
|
+
"summary": "Specifies whether job run queuing is enabled for the job runs for this job."
|
|
12185
|
+
},
|
|
12186
|
+
"immutable": true,
|
|
12187
|
+
"locationInModule": {
|
|
12188
|
+
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
12189
|
+
"line": 53
|
|
12190
|
+
},
|
|
12191
|
+
"name": "jobRunQueuingEnabled",
|
|
12192
|
+
"optional": true,
|
|
12193
|
+
"type": {
|
|
12194
|
+
"primitive": "boolean"
|
|
12195
|
+
}
|
|
12196
|
+
}
|
|
12356
12197
|
],
|
|
12357
12198
|
"symbolId": "lib/jobs/scala-spark-etl-job:ScalaSparkEtlJobProps"
|
|
12358
12199
|
},
|
|
12359
12200
|
"@aws-cdk/aws-glue-alpha.ScalaSparkFlexEtlJob": {
|
|
12360
12201
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
12361
|
-
"base": "@aws-cdk/aws-glue-alpha.
|
|
12202
|
+
"base": "@aws-cdk/aws-glue-alpha.SparkJob",
|
|
12362
12203
|
"docs": {
|
|
12363
12204
|
"remarks": "ETL jobs support pySpark and Scala languages, for which there are separate\nbut similar constructors. ETL jobs default to the G2 worker type, but you\ncan override this default with other supported worker type values\n(G1, G2, G4 and G8). ETL jobs defaults to Glue version 4.0, which you can\noverride to 3.0. The following ETL features are enabled by default:\n—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.\nYou can find more details about version, worker type and other features\nin Glue's public documentation.",
|
|
12364
12205
|
"stability": "experimental",
|
|
12365
12206
|
"summary": "Spark ETL Jobs class.",
|
|
12366
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkFlexEtlJob = new glue_alpha.ScalaSparkFlexEtlJob(this, 'MyScalaSparkFlexEtlJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n maxConcurrentRuns: 123,\n maxRetries: 123,\n notifyDelayAfter: cdk.Duration.minutes(30),\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n
|
|
12207
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkFlexEtlJob = new glue_alpha.ScalaSparkFlexEtlJob(this, 'MyScalaSparkFlexEtlJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n maxConcurrentRuns: 123,\n maxRetries: 123,\n notifyDelayAfter: cdk.Duration.minutes(30),\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n});",
|
|
12367
12208
|
"custom": {
|
|
12368
12209
|
"exampleMetadata": "fixture=_generated"
|
|
12369
12210
|
}
|
|
@@ -12376,7 +12217,7 @@
|
|
|
12376
12217
|
},
|
|
12377
12218
|
"locationInModule": {
|
|
12378
12219
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12379
|
-
"line":
|
|
12220
|
+
"line": 81
|
|
12380
12221
|
},
|
|
12381
12222
|
"parameters": [
|
|
12382
12223
|
{
|
|
@@ -12402,26 +12243,10 @@
|
|
|
12402
12243
|
"kind": "class",
|
|
12403
12244
|
"locationInModule": {
|
|
12404
12245
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12405
|
-
"line":
|
|
12246
|
+
"line": 74
|
|
12406
12247
|
},
|
|
12407
12248
|
"name": "ScalaSparkFlexEtlJob",
|
|
12408
12249
|
"properties": [
|
|
12409
|
-
{
|
|
12410
|
-
"docs": {
|
|
12411
|
-
"stability": "experimental",
|
|
12412
|
-
"summary": "The principal to grant permissions to."
|
|
12413
|
-
},
|
|
12414
|
-
"immutable": true,
|
|
12415
|
-
"locationInModule": {
|
|
12416
|
-
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12417
|
-
"line": 96
|
|
12418
|
-
},
|
|
12419
|
-
"name": "grantPrincipal",
|
|
12420
|
-
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
12421
|
-
"type": {
|
|
12422
|
-
"fqn": "aws-cdk-lib.aws_iam.IPrincipal"
|
|
12423
|
-
}
|
|
12424
|
-
},
|
|
12425
12250
|
{
|
|
12426
12251
|
"docs": {
|
|
12427
12252
|
"stability": "experimental",
|
|
@@ -12430,7 +12255,7 @@
|
|
|
12430
12255
|
"immutable": true,
|
|
12431
12256
|
"locationInModule": {
|
|
12432
12257
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12433
|
-
"line":
|
|
12258
|
+
"line": 75
|
|
12434
12259
|
},
|
|
12435
12260
|
"name": "jobArn",
|
|
12436
12261
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -12446,46 +12271,13 @@
|
|
|
12446
12271
|
"immutable": true,
|
|
12447
12272
|
"locationInModule": {
|
|
12448
12273
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12449
|
-
"line":
|
|
12274
|
+
"line": 76
|
|
12450
12275
|
},
|
|
12451
12276
|
"name": "jobName",
|
|
12452
12277
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
12453
12278
|
"type": {
|
|
12454
12279
|
"primitive": "string"
|
|
12455
12280
|
}
|
|
12456
|
-
},
|
|
12457
|
-
{
|
|
12458
|
-
"docs": {
|
|
12459
|
-
"stability": "experimental",
|
|
12460
|
-
"summary": "The IAM role Glue assumes to run this job."
|
|
12461
|
-
},
|
|
12462
|
-
"immutable": true,
|
|
12463
|
-
"locationInModule": {
|
|
12464
|
-
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12465
|
-
"line": 95
|
|
12466
|
-
},
|
|
12467
|
-
"name": "role",
|
|
12468
|
-
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
12469
|
-
"type": {
|
|
12470
|
-
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
12471
|
-
}
|
|
12472
|
-
},
|
|
12473
|
-
{
|
|
12474
|
-
"docs": {
|
|
12475
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12476
|
-
"stability": "experimental",
|
|
12477
|
-
"summary": "The Spark UI logs location if Spark UI monitoring and debugging is enabled."
|
|
12478
|
-
},
|
|
12479
|
-
"immutable": true,
|
|
12480
|
-
"locationInModule": {
|
|
12481
|
-
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12482
|
-
"line": 104
|
|
12483
|
-
},
|
|
12484
|
-
"name": "sparkUILoggingLocation",
|
|
12485
|
-
"optional": true,
|
|
12486
|
-
"type": {
|
|
12487
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation"
|
|
12488
|
-
}
|
|
12489
12281
|
}
|
|
12490
12282
|
],
|
|
12491
12283
|
"symbolId": "lib/jobs/scala-spark-flex-etl-job:ScalaSparkFlexEtlJob"
|
|
@@ -12497,19 +12289,19 @@
|
|
|
12497
12289
|
"remarks": "Flex jobs supports Python and Scala language.\nThe flexible execution class is appropriate for non-urgent jobs such as\npre-production jobs, testing, and one-time data loads.\nFlexible job runs are supported for jobs using AWS Glue version 3.0 or later and G.1X or\nG.2X worker types but will default to the latest version of Glue (currently Glue 3.0.)\n\nSimilar to ETL, we’ll enable these features: —enable-metrics, —enable-spark-ui,\n—enable-continuous-cloudwatch-log",
|
|
12498
12290
|
"stability": "experimental",
|
|
12499
12291
|
"summary": "Flex Jobs class.",
|
|
12500
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkFlexEtlJobProps: glue_alpha.ScalaSparkFlexEtlJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n maxConcurrentRuns: 123,\n maxRetries: 123,\n notifyDelayAfter: cdk.Duration.minutes(30),\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n
|
|
12292
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkFlexEtlJobProps: glue_alpha.ScalaSparkFlexEtlJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n maxConcurrentRuns: 123,\n maxRetries: 123,\n notifyDelayAfter: cdk.Duration.minutes(30),\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
12501
12293
|
"custom": {
|
|
12502
12294
|
"exampleMetadata": "fixture=_generated"
|
|
12503
12295
|
}
|
|
12504
12296
|
},
|
|
12505
12297
|
"fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkFlexEtlJobProps",
|
|
12506
12298
|
"interfaces": [
|
|
12507
|
-
"@aws-cdk/aws-glue-alpha.
|
|
12299
|
+
"@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
12508
12300
|
],
|
|
12509
12301
|
"kind": "interface",
|
|
12510
12302
|
"locationInModule": {
|
|
12511
12303
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12512
|
-
"line":
|
|
12304
|
+
"line": 21
|
|
12513
12305
|
},
|
|
12514
12306
|
"name": "ScalaSparkFlexEtlJobProps",
|
|
12515
12307
|
"properties": [
|
|
@@ -12523,7 +12315,7 @@
|
|
|
12523
12315
|
"immutable": true,
|
|
12524
12316
|
"locationInModule": {
|
|
12525
12317
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12526
|
-
"line":
|
|
12318
|
+
"line": 34
|
|
12527
12319
|
},
|
|
12528
12320
|
"name": "className",
|
|
12529
12321
|
"type": {
|
|
@@ -12533,16 +12325,15 @@
|
|
|
12533
12325
|
{
|
|
12534
12326
|
"abstract": true,
|
|
12535
12327
|
"docs": {
|
|
12536
|
-
"default": "
|
|
12537
|
-
"
|
|
12538
|
-
"see": " `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12328
|
+
"default": "- no extra files specified.",
|
|
12329
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12539
12330
|
"stability": "experimental",
|
|
12540
12331
|
"summary": "Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it."
|
|
12541
12332
|
},
|
|
12542
12333
|
"immutable": true,
|
|
12543
12334
|
"locationInModule": {
|
|
12544
12335
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12545
|
-
"line":
|
|
12336
|
+
"line": 43
|
|
12546
12337
|
},
|
|
12547
12338
|
"name": "extraFiles",
|
|
12548
12339
|
"optional": true,
|
|
@@ -12558,15 +12349,14 @@
|
|
|
12558
12349
|
{
|
|
12559
12350
|
"abstract": true,
|
|
12560
12351
|
"docs": {
|
|
12561
|
-
"default": "
|
|
12562
|
-
"see": " `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12352
|
+
"default": "- no extra jar files",
|
|
12563
12353
|
"stability": "experimental",
|
|
12564
|
-
"summary": "
|
|
12354
|
+
"summary": "Extra Jars S3 URL (optional) S3 URL where additional jar dependencies are located."
|
|
12565
12355
|
},
|
|
12566
12356
|
"immutable": true,
|
|
12567
12357
|
"locationInModule": {
|
|
12568
12358
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12569
|
-
"line":
|
|
12359
|
+
"line": 50
|
|
12570
12360
|
},
|
|
12571
12361
|
"name": "extraJars",
|
|
12572
12362
|
"optional": true,
|
|
@@ -12590,7 +12380,7 @@
|
|
|
12590
12380
|
"immutable": true,
|
|
12591
12381
|
"locationInModule": {
|
|
12592
12382
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12593
|
-
"line":
|
|
12383
|
+
"line": 59
|
|
12594
12384
|
},
|
|
12595
12385
|
"name": "extraJarsFirst",
|
|
12596
12386
|
"optional": true,
|
|
@@ -12609,44 +12399,25 @@
|
|
|
12609
12399
|
"immutable": true,
|
|
12610
12400
|
"locationInModule": {
|
|
12611
12401
|
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12612
|
-
"line":
|
|
12402
|
+
"line": 27
|
|
12613
12403
|
},
|
|
12614
12404
|
"name": "notifyDelayAfter",
|
|
12615
12405
|
"optional": true,
|
|
12616
12406
|
"type": {
|
|
12617
12407
|
"fqn": "aws-cdk-lib.Duration"
|
|
12618
12408
|
}
|
|
12619
|
-
},
|
|
12620
|
-
{
|
|
12621
|
-
"abstract": true,
|
|
12622
|
-
"docs": {
|
|
12623
|
-
"default": "- Spark UI debugging and monitoring is disabled.",
|
|
12624
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12625
|
-
"stability": "experimental",
|
|
12626
|
-
"summary": "Enables the Spark UI debugging and monitoring with the specified props."
|
|
12627
|
-
},
|
|
12628
|
-
"immutable": true,
|
|
12629
|
-
"locationInModule": {
|
|
12630
|
-
"filename": "lib/jobs/scala-spark-flex-etl-job.ts",
|
|
12631
|
-
"line": 34
|
|
12632
|
-
},
|
|
12633
|
-
"name": "sparkUI",
|
|
12634
|
-
"optional": true,
|
|
12635
|
-
"type": {
|
|
12636
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUIProps"
|
|
12637
|
-
}
|
|
12638
12409
|
}
|
|
12639
12410
|
],
|
|
12640
12411
|
"symbolId": "lib/jobs/scala-spark-flex-etl-job:ScalaSparkFlexEtlJobProps"
|
|
12641
12412
|
},
|
|
12642
12413
|
"@aws-cdk/aws-glue-alpha.ScalaSparkStreamingJob": {
|
|
12643
12414
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
12644
|
-
"base": "@aws-cdk/aws-glue-alpha.
|
|
12415
|
+
"base": "@aws-cdk/aws-glue-alpha.SparkJob",
|
|
12645
12416
|
"docs": {
|
|
12646
12417
|
"remarks": "A Streaming job is similar to an ETL job, except that it performs ETL on data streams\nusing the Apache Spark Structured Streaming framework.\nThese jobs will default to use Python 3.9.\n\nSimilar to ETL jobs, streaming job supports Scala and Python languages. Similar to ETL,\nit supports G1 and G2 worker type and 2.0, 3.0 and 4.0 version. We’ll default to G2 worker\nand 4.0 version for streaming jobs which developers can override.\nWe will enable —enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.",
|
|
12647
12418
|
"stability": "experimental",
|
|
12648
12419
|
"summary": "Scala Streaming Jobs class.",
|
|
12649
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkStreamingJob = new glue_alpha.ScalaSparkStreamingJob(this, 'MyScalaSparkStreamingJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraJars: [
|
|
12420
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkStreamingJob = new glue_alpha.ScalaSparkStreamingJob(this, 'MyScalaSparkStreamingJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n});",
|
|
12650
12421
|
"custom": {
|
|
12651
12422
|
"exampleMetadata": "fixture=_generated"
|
|
12652
12423
|
}
|
|
@@ -12659,7 +12430,7 @@
|
|
|
12659
12430
|
},
|
|
12660
12431
|
"locationInModule": {
|
|
12661
12432
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12662
|
-
"line":
|
|
12433
|
+
"line": 75
|
|
12663
12434
|
},
|
|
12664
12435
|
"parameters": [
|
|
12665
12436
|
{
|
|
@@ -12685,26 +12456,10 @@
|
|
|
12685
12456
|
"kind": "class",
|
|
12686
12457
|
"locationInModule": {
|
|
12687
12458
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12688
|
-
"line":
|
|
12459
|
+
"line": 68
|
|
12689
12460
|
},
|
|
12690
12461
|
"name": "ScalaSparkStreamingJob",
|
|
12691
12462
|
"properties": [
|
|
12692
|
-
{
|
|
12693
|
-
"docs": {
|
|
12694
|
-
"stability": "experimental",
|
|
12695
|
-
"summary": "The principal to grant permissions to."
|
|
12696
|
-
},
|
|
12697
|
-
"immutable": true,
|
|
12698
|
-
"locationInModule": {
|
|
12699
|
-
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12700
|
-
"line": 67
|
|
12701
|
-
},
|
|
12702
|
-
"name": "grantPrincipal",
|
|
12703
|
-
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
12704
|
-
"type": {
|
|
12705
|
-
"fqn": "aws-cdk-lib.aws_iam.IPrincipal"
|
|
12706
|
-
}
|
|
12707
|
-
},
|
|
12708
12463
|
{
|
|
12709
12464
|
"docs": {
|
|
12710
12465
|
"stability": "experimental",
|
|
@@ -12713,7 +12468,7 @@
|
|
|
12713
12468
|
"immutable": true,
|
|
12714
12469
|
"locationInModule": {
|
|
12715
12470
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12716
|
-
"line":
|
|
12471
|
+
"line": 69
|
|
12717
12472
|
},
|
|
12718
12473
|
"name": "jobArn",
|
|
12719
12474
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -12729,46 +12484,13 @@
|
|
|
12729
12484
|
"immutable": true,
|
|
12730
12485
|
"locationInModule": {
|
|
12731
12486
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12732
|
-
"line":
|
|
12487
|
+
"line": 70
|
|
12733
12488
|
},
|
|
12734
12489
|
"name": "jobName",
|
|
12735
12490
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
12736
12491
|
"type": {
|
|
12737
12492
|
"primitive": "string"
|
|
12738
12493
|
}
|
|
12739
|
-
},
|
|
12740
|
-
{
|
|
12741
|
-
"docs": {
|
|
12742
|
-
"stability": "experimental",
|
|
12743
|
-
"summary": "The IAM role Glue assumes to run this job."
|
|
12744
|
-
},
|
|
12745
|
-
"immutable": true,
|
|
12746
|
-
"locationInModule": {
|
|
12747
|
-
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12748
|
-
"line": 66
|
|
12749
|
-
},
|
|
12750
|
-
"name": "role",
|
|
12751
|
-
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
12752
|
-
"type": {
|
|
12753
|
-
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
12754
|
-
}
|
|
12755
|
-
},
|
|
12756
|
-
{
|
|
12757
|
-
"docs": {
|
|
12758
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12759
|
-
"stability": "experimental",
|
|
12760
|
-
"summary": "The Spark UI logs location if Spark UI monitoring and debugging is enabled."
|
|
12761
|
-
},
|
|
12762
|
-
"immutable": true,
|
|
12763
|
-
"locationInModule": {
|
|
12764
|
-
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12765
|
-
"line": 75
|
|
12766
|
-
},
|
|
12767
|
-
"name": "sparkUILoggingLocation",
|
|
12768
|
-
"optional": true,
|
|
12769
|
-
"type": {
|
|
12770
|
-
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation"
|
|
12771
|
-
}
|
|
12772
12494
|
}
|
|
12773
12495
|
],
|
|
12774
12496
|
"symbolId": "lib/jobs/scala-spark-streaming-job:ScalaSparkStreamingJob"
|
|
@@ -12779,19 +12501,19 @@
|
|
|
12779
12501
|
"docs": {
|
|
12780
12502
|
"stability": "experimental",
|
|
12781
12503
|
"summary": "Properties for creating a Scala Spark ETL job.",
|
|
12782
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkStreamingJobProps: glue_alpha.ScalaSparkStreamingJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraJars: [
|
|
12504
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkStreamingJobProps: glue_alpha.ScalaSparkStreamingJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
12783
12505
|
"custom": {
|
|
12784
12506
|
"exampleMetadata": "fixture=_generated"
|
|
12785
12507
|
}
|
|
12786
12508
|
},
|
|
12787
12509
|
"fqn": "@aws-cdk/aws-glue-alpha.ScalaSparkStreamingJobProps",
|
|
12788
12510
|
"interfaces": [
|
|
12789
|
-
"@aws-cdk/aws-glue-alpha.
|
|
12511
|
+
"@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
12790
12512
|
],
|
|
12791
12513
|
"kind": "interface",
|
|
12792
12514
|
"locationInModule": {
|
|
12793
12515
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12794
|
-
"line":
|
|
12516
|
+
"line": 10
|
|
12795
12517
|
},
|
|
12796
12518
|
"name": "ScalaSparkStreamingJobProps",
|
|
12797
12519
|
"properties": [
|
|
@@ -12804,13 +12526,37 @@
|
|
|
12804
12526
|
"immutable": true,
|
|
12805
12527
|
"locationInModule": {
|
|
12806
12528
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12807
|
-
"line":
|
|
12529
|
+
"line": 16
|
|
12808
12530
|
},
|
|
12809
12531
|
"name": "className",
|
|
12810
12532
|
"type": {
|
|
12811
12533
|
"primitive": "string"
|
|
12812
12534
|
}
|
|
12813
12535
|
},
|
|
12536
|
+
{
|
|
12537
|
+
"abstract": true,
|
|
12538
|
+
"docs": {
|
|
12539
|
+
"default": "- no extra files specified.",
|
|
12540
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12541
|
+
"stability": "experimental",
|
|
12542
|
+
"summary": "Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it."
|
|
12543
|
+
},
|
|
12544
|
+
"immutable": true,
|
|
12545
|
+
"locationInModule": {
|
|
12546
|
+
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12547
|
+
"line": 25
|
|
12548
|
+
},
|
|
12549
|
+
"name": "extraFiles",
|
|
12550
|
+
"optional": true,
|
|
12551
|
+
"type": {
|
|
12552
|
+
"collection": {
|
|
12553
|
+
"elementtype": {
|
|
12554
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
12555
|
+
},
|
|
12556
|
+
"kind": "array"
|
|
12557
|
+
}
|
|
12558
|
+
}
|
|
12559
|
+
},
|
|
12814
12560
|
{
|
|
12815
12561
|
"abstract": true,
|
|
12816
12562
|
"docs": {
|
|
@@ -12821,14 +12567,14 @@
|
|
|
12821
12567
|
"immutable": true,
|
|
12822
12568
|
"locationInModule": {
|
|
12823
12569
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12824
|
-
"line":
|
|
12570
|
+
"line": 32
|
|
12825
12571
|
},
|
|
12826
12572
|
"name": "extraJars",
|
|
12827
12573
|
"optional": true,
|
|
12828
12574
|
"type": {
|
|
12829
12575
|
"collection": {
|
|
12830
12576
|
"elementtype": {
|
|
12831
|
-
"
|
|
12577
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
12832
12578
|
},
|
|
12833
12579
|
"kind": "array"
|
|
12834
12580
|
}
|
|
@@ -12837,17 +12583,17 @@
|
|
|
12837
12583
|
{
|
|
12838
12584
|
"abstract": true,
|
|
12839
12585
|
"docs": {
|
|
12840
|
-
"default": "-
|
|
12841
|
-
"
|
|
12586
|
+
"default": "false - priority is not given to user-provided jars",
|
|
12587
|
+
"see": " `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
12842
12588
|
"stability": "experimental",
|
|
12843
|
-
"summary": "
|
|
12589
|
+
"summary": "Setting this value to true prioritizes the customer's extra JAR files in the classpath."
|
|
12844
12590
|
},
|
|
12845
12591
|
"immutable": true,
|
|
12846
12592
|
"locationInModule": {
|
|
12847
12593
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12848
|
-
"line":
|
|
12594
|
+
"line": 41
|
|
12849
12595
|
},
|
|
12850
|
-
"name": "
|
|
12596
|
+
"name": "extraJarsFirst",
|
|
12851
12597
|
"optional": true,
|
|
12852
12598
|
"type": {
|
|
12853
12599
|
"primitive": "boolean"
|
|
@@ -12856,20 +12602,20 @@
|
|
|
12856
12602
|
{
|
|
12857
12603
|
"abstract": true,
|
|
12858
12604
|
"docs": {
|
|
12859
|
-
"default": "-
|
|
12860
|
-
"
|
|
12605
|
+
"default": "- no job run queuing",
|
|
12606
|
+
"remarks": "A value of true means job run queuing is enabled for the job runs.\nIf false or not populated, the job runs will not be considered for queueing.\nIf this field does not match the value set in the job run, then the value from\nthe job run field will be used. This property must be set to false for flex jobs.\nIf this property is enabled, maxRetries must be set to zero.",
|
|
12861
12607
|
"stability": "experimental",
|
|
12862
|
-
"summary": "
|
|
12608
|
+
"summary": "Specifies whether job run queuing is enabled for the job runs for this job."
|
|
12863
12609
|
},
|
|
12864
12610
|
"immutable": true,
|
|
12865
12611
|
"locationInModule": {
|
|
12866
12612
|
"filename": "lib/jobs/scala-spark-streaming-job.ts",
|
|
12867
|
-
"line":
|
|
12613
|
+
"line": 53
|
|
12868
12614
|
},
|
|
12869
|
-
"name": "
|
|
12615
|
+
"name": "jobRunQueuingEnabled",
|
|
12870
12616
|
"optional": true,
|
|
12871
12617
|
"type": {
|
|
12872
|
-
"
|
|
12618
|
+
"primitive": "boolean"
|
|
12873
12619
|
}
|
|
12874
12620
|
}
|
|
12875
12621
|
],
|
|
@@ -13767,97 +13513,356 @@
|
|
|
13767
13513
|
],
|
|
13768
13514
|
"symbolId": "lib/data-format:SerializationLibrary"
|
|
13769
13515
|
},
|
|
13770
|
-
"@aws-cdk/aws-glue-alpha.
|
|
13516
|
+
"@aws-cdk/aws-glue-alpha.SparkExtraCodeProps": {
|
|
13771
13517
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
13772
13518
|
"datatype": true,
|
|
13773
13519
|
"docs": {
|
|
13774
|
-
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
13775
13520
|
"stability": "experimental",
|
|
13776
|
-
"summary": "
|
|
13777
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\
|
|
13521
|
+
"summary": "Code props for different {@link Code} assets used by different types of Spark jobs.",
|
|
13522
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\n\ndeclare const code: glue_alpha.Code;\nconst sparkExtraCodeProps: glue_alpha.SparkExtraCodeProps = {\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n extraPythonFiles: [code],\n};",
|
|
13778
13523
|
"custom": {
|
|
13779
13524
|
"exampleMetadata": "fixture=_generated"
|
|
13780
13525
|
}
|
|
13781
13526
|
},
|
|
13782
|
-
"fqn": "@aws-cdk/aws-glue-alpha.
|
|
13527
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkExtraCodeProps",
|
|
13783
13528
|
"kind": "interface",
|
|
13784
13529
|
"locationInModule": {
|
|
13785
|
-
"filename": "lib/jobs/spark-
|
|
13786
|
-
"line":
|
|
13530
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13531
|
+
"line": 13
|
|
13787
13532
|
},
|
|
13788
|
-
"name": "
|
|
13533
|
+
"name": "SparkExtraCodeProps",
|
|
13789
13534
|
"properties": [
|
|
13790
13535
|
{
|
|
13791
13536
|
"abstract": true,
|
|
13792
13537
|
"docs": {
|
|
13538
|
+
"default": "- no extra files specified.",
|
|
13539
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
13793
13540
|
"stability": "experimental",
|
|
13794
|
-
"summary": "
|
|
13541
|
+
"summary": "Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it."
|
|
13795
13542
|
},
|
|
13796
13543
|
"immutable": true,
|
|
13797
13544
|
"locationInModule": {
|
|
13798
|
-
"filename": "lib/jobs/spark-
|
|
13799
|
-
"line":
|
|
13545
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13546
|
+
"line": 29
|
|
13800
13547
|
},
|
|
13801
|
-
"name": "
|
|
13548
|
+
"name": "extraFiles",
|
|
13549
|
+
"optional": true,
|
|
13802
13550
|
"type": {
|
|
13803
|
-
"
|
|
13551
|
+
"collection": {
|
|
13552
|
+
"elementtype": {
|
|
13553
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
13554
|
+
},
|
|
13555
|
+
"kind": "array"
|
|
13556
|
+
}
|
|
13804
13557
|
}
|
|
13805
13558
|
},
|
|
13806
13559
|
{
|
|
13807
13560
|
"abstract": true,
|
|
13808
13561
|
"docs": {
|
|
13809
|
-
"default": "
|
|
13562
|
+
"default": "- no extra jar files",
|
|
13810
13563
|
"stability": "experimental",
|
|
13811
|
-
"summary": "
|
|
13564
|
+
"summary": "Extra Jars S3 URL (optional) S3 URL where additional jar dependencies are located."
|
|
13812
13565
|
},
|
|
13813
13566
|
"immutable": true,
|
|
13814
13567
|
"locationInModule": {
|
|
13815
|
-
"filename": "lib/jobs/spark-
|
|
13816
|
-
"line":
|
|
13568
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13569
|
+
"line": 36
|
|
13817
13570
|
},
|
|
13818
|
-
"name": "
|
|
13571
|
+
"name": "extraJars",
|
|
13819
13572
|
"optional": true,
|
|
13820
13573
|
"type": {
|
|
13821
|
-
"
|
|
13822
|
-
|
|
13823
|
-
|
|
13824
|
-
|
|
13825
|
-
|
|
13826
|
-
|
|
13827
|
-
|
|
13574
|
+
"collection": {
|
|
13575
|
+
"elementtype": {
|
|
13576
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
13577
|
+
},
|
|
13578
|
+
"kind": "array"
|
|
13579
|
+
}
|
|
13580
|
+
}
|
|
13581
|
+
},
|
|
13582
|
+
{
|
|
13583
|
+
"abstract": true,
|
|
13584
|
+
"docs": {
|
|
13585
|
+
"default": "false - priority is not given to user-provided jars",
|
|
13586
|
+
"see": " `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
13587
|
+
"stability": "experimental",
|
|
13588
|
+
"summary": "Setting this value to true prioritizes the customer's extra JAR files in the classpath."
|
|
13589
|
+
},
|
|
13590
|
+
"immutable": true,
|
|
13591
|
+
"locationInModule": {
|
|
13592
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13593
|
+
"line": 45
|
|
13594
|
+
},
|
|
13595
|
+
"name": "extraJarsFirst",
|
|
13596
|
+
"optional": true,
|
|
13597
|
+
"type": {
|
|
13598
|
+
"primitive": "boolean"
|
|
13599
|
+
}
|
|
13600
|
+
},
|
|
13601
|
+
{
|
|
13602
|
+
"abstract": true,
|
|
13603
|
+
"docs": {
|
|
13604
|
+
"default": "- no extra files",
|
|
13605
|
+
"stability": "experimental",
|
|
13606
|
+
"summary": "Extra Python Files S3 URL (optional) S3 URL where additional python dependencies are located."
|
|
13607
|
+
},
|
|
13608
|
+
"immutable": true,
|
|
13609
|
+
"locationInModule": {
|
|
13610
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13611
|
+
"line": 20
|
|
13612
|
+
},
|
|
13613
|
+
"name": "extraPythonFiles",
|
|
13614
|
+
"optional": true,
|
|
13615
|
+
"type": {
|
|
13616
|
+
"collection": {
|
|
13617
|
+
"elementtype": {
|
|
13618
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
13619
|
+
},
|
|
13620
|
+
"kind": "array"
|
|
13621
|
+
}
|
|
13622
|
+
}
|
|
13623
|
+
}
|
|
13624
|
+
],
|
|
13625
|
+
"symbolId": "lib/jobs/spark-job:SparkExtraCodeProps"
|
|
13626
|
+
},
|
|
13627
|
+
"@aws-cdk/aws-glue-alpha.SparkJob": {
|
|
13628
|
+
"abstract": true,
|
|
13629
|
+
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
13630
|
+
"base": "@aws-cdk/aws-glue-alpha.Job",
|
|
13631
|
+
"docs": {
|
|
13632
|
+
"stability": "experimental",
|
|
13633
|
+
"summary": "Base class for different types of Spark Jobs.",
|
|
13634
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport { aws_iam as iam } from 'aws-cdk-lib';\n\ndeclare const role: iam.Role;\nconst sparkJob = glue_alpha.SparkJob.fromJobAttributes(this, 'MySparkJob', {\n jobName: 'jobName',\n\n // the properties below are optional\n role: role,\n});",
|
|
13635
|
+
"custom": {
|
|
13636
|
+
"exampleMetadata": "fixture=_generated"
|
|
13637
|
+
}
|
|
13638
|
+
},
|
|
13639
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkJob",
|
|
13640
|
+
"initializer": {
|
|
13641
|
+
"docs": {
|
|
13642
|
+
"stability": "experimental"
|
|
13643
|
+
},
|
|
13644
|
+
"locationInModule": {
|
|
13645
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13646
|
+
"line": 121
|
|
13647
|
+
},
|
|
13648
|
+
"parameters": [
|
|
13649
|
+
{
|
|
13650
|
+
"name": "scope",
|
|
13651
|
+
"type": {
|
|
13652
|
+
"fqn": "constructs.Construct"
|
|
13653
|
+
}
|
|
13654
|
+
},
|
|
13655
|
+
{
|
|
13656
|
+
"name": "id",
|
|
13657
|
+
"type": {
|
|
13658
|
+
"primitive": "string"
|
|
13659
|
+
}
|
|
13660
|
+
},
|
|
13661
|
+
{
|
|
13662
|
+
"name": "props",
|
|
13663
|
+
"type": {
|
|
13664
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
13665
|
+
}
|
|
13666
|
+
}
|
|
13667
|
+
]
|
|
13668
|
+
},
|
|
13669
|
+
"kind": "class",
|
|
13670
|
+
"locationInModule": {
|
|
13671
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13672
|
+
"line": 109
|
|
13673
|
+
},
|
|
13674
|
+
"methods": [
|
|
13675
|
+
{
|
|
13676
|
+
"docs": {
|
|
13677
|
+
"stability": "experimental"
|
|
13678
|
+
},
|
|
13679
|
+
"locationInModule": {
|
|
13680
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13681
|
+
"line": 134
|
|
13682
|
+
},
|
|
13683
|
+
"name": "nonExecutableCommonArguments",
|
|
13684
|
+
"parameters": [
|
|
13685
|
+
{
|
|
13686
|
+
"name": "props",
|
|
13687
|
+
"type": {
|
|
13688
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkJobProps"
|
|
13689
|
+
}
|
|
13690
|
+
}
|
|
13691
|
+
],
|
|
13692
|
+
"protected": true,
|
|
13693
|
+
"returns": {
|
|
13694
|
+
"type": {
|
|
13695
|
+
"collection": {
|
|
13696
|
+
"elementtype": {
|
|
13697
|
+
"primitive": "string"
|
|
13698
|
+
},
|
|
13699
|
+
"kind": "map"
|
|
13700
|
+
}
|
|
13701
|
+
}
|
|
13702
|
+
}
|
|
13703
|
+
},
|
|
13704
|
+
{
|
|
13705
|
+
"docs": {
|
|
13706
|
+
"stability": "experimental",
|
|
13707
|
+
"summary": "Set the arguments for extra {@link Code}-related properties."
|
|
13708
|
+
},
|
|
13709
|
+
"locationInModule": {
|
|
13710
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13711
|
+
"line": 158
|
|
13712
|
+
},
|
|
13713
|
+
"name": "setupExtraCodeArguments",
|
|
13714
|
+
"parameters": [
|
|
13715
|
+
{
|
|
13716
|
+
"name": "args",
|
|
13717
|
+
"type": {
|
|
13718
|
+
"collection": {
|
|
13719
|
+
"elementtype": {
|
|
13720
|
+
"primitive": "string"
|
|
13721
|
+
},
|
|
13722
|
+
"kind": "map"
|
|
13723
|
+
}
|
|
13724
|
+
}
|
|
13725
|
+
},
|
|
13726
|
+
{
|
|
13727
|
+
"name": "props",
|
|
13728
|
+
"type": {
|
|
13729
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkExtraCodeProps"
|
|
13730
|
+
}
|
|
13731
|
+
}
|
|
13732
|
+
],
|
|
13733
|
+
"protected": true
|
|
13734
|
+
}
|
|
13735
|
+
],
|
|
13736
|
+
"name": "SparkJob",
|
|
13737
|
+
"properties": [
|
|
13738
|
+
{
|
|
13739
|
+
"docs": {
|
|
13740
|
+
"stability": "experimental",
|
|
13741
|
+
"summary": "The principal to grant permissions to."
|
|
13742
|
+
},
|
|
13743
|
+
"immutable": true,
|
|
13744
|
+
"locationInModule": {
|
|
13745
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13746
|
+
"line": 111
|
|
13747
|
+
},
|
|
13748
|
+
"name": "grantPrincipal",
|
|
13749
|
+
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
13750
|
+
"type": {
|
|
13751
|
+
"fqn": "aws-cdk-lib.aws_iam.IPrincipal"
|
|
13752
|
+
}
|
|
13753
|
+
},
|
|
13754
|
+
{
|
|
13755
|
+
"docs": {
|
|
13756
|
+
"stability": "experimental",
|
|
13757
|
+
"summary": "The IAM role Glue assumes to run this job."
|
|
13758
|
+
},
|
|
13759
|
+
"immutable": true,
|
|
13760
|
+
"locationInModule": {
|
|
13761
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13762
|
+
"line": 110
|
|
13763
|
+
},
|
|
13764
|
+
"name": "role",
|
|
13765
|
+
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
13766
|
+
"type": {
|
|
13767
|
+
"fqn": "aws-cdk-lib.aws_iam.IRole"
|
|
13768
|
+
}
|
|
13769
|
+
},
|
|
13770
|
+
{
|
|
13771
|
+
"docs": {
|
|
13772
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
13773
|
+
"stability": "experimental",
|
|
13774
|
+
"summary": "The Spark UI logs location if Spark UI monitoring and debugging is enabled."
|
|
13775
|
+
},
|
|
13776
|
+
"immutable": true,
|
|
13777
|
+
"locationInModule": {
|
|
13778
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13779
|
+
"line": 119
|
|
13780
|
+
},
|
|
13781
|
+
"name": "sparkUILoggingLocation",
|
|
13782
|
+
"optional": true,
|
|
13783
|
+
"type": {
|
|
13784
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation"
|
|
13785
|
+
}
|
|
13786
|
+
}
|
|
13787
|
+
],
|
|
13788
|
+
"symbolId": "lib/jobs/spark-job:SparkJob"
|
|
13789
|
+
},
|
|
13790
|
+
"@aws-cdk/aws-glue-alpha.SparkJobProps": {
|
|
13791
|
+
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
13792
|
+
"datatype": true,
|
|
13793
|
+
"docs": {
|
|
13794
|
+
"stability": "experimental",
|
|
13795
|
+
"summary": "Common properties for different types of Spark jobs.",
|
|
13796
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst sparkJobProps: glue_alpha.SparkJobProps = {\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableProfilingMetrics: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
13797
|
+
"custom": {
|
|
13798
|
+
"exampleMetadata": "fixture=_generated"
|
|
13799
|
+
}
|
|
13800
|
+
},
|
|
13801
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkJobProps",
|
|
13802
|
+
"interfaces": [
|
|
13803
|
+
"@aws-cdk/aws-glue-alpha.JobProps"
|
|
13804
|
+
],
|
|
13805
|
+
"kind": "interface",
|
|
13806
|
+
"locationInModule": {
|
|
13807
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13808
|
+
"line": 94
|
|
13809
|
+
},
|
|
13810
|
+
"name": "SparkJobProps",
|
|
13811
|
+
"properties": [
|
|
13812
|
+
{
|
|
13813
|
+
"abstract": true,
|
|
13814
|
+
"docs": {
|
|
13815
|
+
"default": "- Spark UI debugging and monitoring is disabled.",
|
|
13816
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
13817
|
+
"stability": "experimental",
|
|
13818
|
+
"summary": "Enables the Spark UI debugging and monitoring with the specified props."
|
|
13819
|
+
},
|
|
13820
|
+
"immutable": true,
|
|
13821
|
+
"locationInModule": {
|
|
13822
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13823
|
+
"line": 103
|
|
13824
|
+
},
|
|
13825
|
+
"name": "sparkUI",
|
|
13826
|
+
"optional": true,
|
|
13827
|
+
"type": {
|
|
13828
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkUIProps"
|
|
13829
|
+
}
|
|
13830
|
+
}
|
|
13831
|
+
],
|
|
13832
|
+
"symbolId": "lib/jobs/spark-job:SparkJobProps"
|
|
13833
|
+
},
|
|
13834
|
+
"@aws-cdk/aws-glue-alpha.SparkUILoggingLocation": {
|
|
13828
13835
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
13829
13836
|
"datatype": true,
|
|
13830
13837
|
"docs": {
|
|
13831
13838
|
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
13832
13839
|
"stability": "experimental",
|
|
13833
|
-
"summary": "
|
|
13834
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\nconst
|
|
13840
|
+
"summary": "The Spark UI logging location.",
|
|
13841
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\nconst sparkUILoggingLocation: glue_alpha.SparkUILoggingLocation = {\n bucket: bucket,\n\n // the properties below are optional\n prefix: 'prefix',\n};",
|
|
13835
13842
|
"custom": {
|
|
13836
13843
|
"exampleMetadata": "fixture=_generated"
|
|
13837
13844
|
}
|
|
13838
13845
|
},
|
|
13839
|
-
"fqn": "@aws-cdk/aws-glue-alpha.
|
|
13846
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkUILoggingLocation",
|
|
13840
13847
|
"kind": "interface",
|
|
13841
13848
|
"locationInModule": {
|
|
13842
|
-
"filename": "lib/jobs/spark-
|
|
13843
|
-
"line":
|
|
13849
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13850
|
+
"line": 77
|
|
13844
13851
|
},
|
|
13845
|
-
"name": "
|
|
13852
|
+
"name": "SparkUILoggingLocation",
|
|
13846
13853
|
"properties": [
|
|
13847
13854
|
{
|
|
13848
13855
|
"abstract": true,
|
|
13849
13856
|
"docs": {
|
|
13850
|
-
"default": "a new bucket will be created.",
|
|
13851
13857
|
"stability": "experimental",
|
|
13852
13858
|
"summary": "The bucket where the Glue job stores the logs."
|
|
13853
13859
|
},
|
|
13854
13860
|
"immutable": true,
|
|
13855
13861
|
"locationInModule": {
|
|
13856
|
-
"filename": "lib/jobs/spark-
|
|
13857
|
-
"line":
|
|
13862
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13863
|
+
"line": 81
|
|
13858
13864
|
},
|
|
13859
13865
|
"name": "bucket",
|
|
13860
|
-
"optional": true,
|
|
13861
13866
|
"type": {
|
|
13862
13867
|
"fqn": "aws-cdk-lib.aws_s3.IBucket"
|
|
13863
13868
|
}
|
|
@@ -13865,20 +13870,60 @@
|
|
|
13865
13870
|
{
|
|
13866
13871
|
"abstract": true,
|
|
13867
13872
|
"docs": {
|
|
13868
|
-
"default": "-
|
|
13869
|
-
"remarks": "A value of true means job run queuing is enabled for the job runs.\nIf false or not populated, the job runs will not be considered for queueing.\nIf this field does not match the value set in the job run, then the value from\nthe job run field will be used. This property must be set to false for flex jobs.\nIf this property is enabled, maxRetries must be set to zero.",
|
|
13873
|
+
"default": "'/' - the logs will be written at the root of the bucket",
|
|
13870
13874
|
"stability": "experimental",
|
|
13871
|
-
"summary": "
|
|
13875
|
+
"summary": "The path inside the bucket (objects prefix) where the Glue job stores the logs."
|
|
13872
13876
|
},
|
|
13873
13877
|
"immutable": true,
|
|
13874
13878
|
"locationInModule": {
|
|
13875
|
-
"filename": "lib/jobs/spark-
|
|
13876
|
-
"line":
|
|
13879
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13880
|
+
"line": 88
|
|
13877
13881
|
},
|
|
13878
|
-
"name": "
|
|
13882
|
+
"name": "prefix",
|
|
13879
13883
|
"optional": true,
|
|
13880
13884
|
"type": {
|
|
13881
|
-
"primitive": "
|
|
13885
|
+
"primitive": "string"
|
|
13886
|
+
}
|
|
13887
|
+
}
|
|
13888
|
+
],
|
|
13889
|
+
"symbolId": "lib/jobs/spark-job:SparkUILoggingLocation"
|
|
13890
|
+
},
|
|
13891
|
+
"@aws-cdk/aws-glue-alpha.SparkUIProps": {
|
|
13892
|
+
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
13893
|
+
"datatype": true,
|
|
13894
|
+
"docs": {
|
|
13895
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
13896
|
+
"stability": "experimental",
|
|
13897
|
+
"summary": "Properties for enabling Spark UI monitoring feature for Spark-based Glue jobs.",
|
|
13898
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\nconst sparkUIProps: glue_alpha.SparkUIProps = {\n bucket: bucket,\n prefix: 'prefix',\n};",
|
|
13899
|
+
"custom": {
|
|
13900
|
+
"exampleMetadata": "fixture=_generated"
|
|
13901
|
+
}
|
|
13902
|
+
},
|
|
13903
|
+
"fqn": "@aws-cdk/aws-glue-alpha.SparkUIProps",
|
|
13904
|
+
"kind": "interface",
|
|
13905
|
+
"locationInModule": {
|
|
13906
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13907
|
+
"line": 54
|
|
13908
|
+
},
|
|
13909
|
+
"name": "SparkUIProps",
|
|
13910
|
+
"properties": [
|
|
13911
|
+
{
|
|
13912
|
+
"abstract": true,
|
|
13913
|
+
"docs": {
|
|
13914
|
+
"default": "a new bucket will be created.",
|
|
13915
|
+
"stability": "experimental",
|
|
13916
|
+
"summary": "The bucket where the Glue job stores the logs."
|
|
13917
|
+
},
|
|
13918
|
+
"immutable": true,
|
|
13919
|
+
"locationInModule": {
|
|
13920
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13921
|
+
"line": 60
|
|
13922
|
+
},
|
|
13923
|
+
"name": "bucket",
|
|
13924
|
+
"optional": true,
|
|
13925
|
+
"type": {
|
|
13926
|
+
"fqn": "aws-cdk-lib.aws_s3.IBucket"
|
|
13882
13927
|
}
|
|
13883
13928
|
},
|
|
13884
13929
|
{
|
|
@@ -13891,8 +13936,8 @@
|
|
|
13891
13936
|
},
|
|
13892
13937
|
"immutable": true,
|
|
13893
13938
|
"locationInModule": {
|
|
13894
|
-
"filename": "lib/jobs/spark-
|
|
13895
|
-
"line":
|
|
13939
|
+
"filename": "lib/jobs/spark-job.ts",
|
|
13940
|
+
"line": 68
|
|
13896
13941
|
},
|
|
13897
13942
|
"name": "prefix",
|
|
13898
13943
|
"optional": true,
|
|
@@ -13901,7 +13946,7 @@
|
|
|
13901
13946
|
}
|
|
13902
13947
|
}
|
|
13903
13948
|
],
|
|
13904
|
-
"symbolId": "lib/jobs/spark-
|
|
13949
|
+
"symbolId": "lib/jobs/spark-job:SparkUIProps"
|
|
13905
13950
|
},
|
|
13906
13951
|
"@aws-cdk/aws-glue-alpha.StorageParameter": {
|
|
13907
13952
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
@@ -16690,6 +16735,6 @@
|
|
|
16690
16735
|
"symbolId": "lib/storage-parameter:WriteParallel"
|
|
16691
16736
|
}
|
|
16692
16737
|
},
|
|
16693
|
-
"version": "2.
|
|
16738
|
+
"version": "2.183.0-alpha.0",
|
|
16694
16739
|
"fingerprint": "**********"
|
|
16695
16740
|
}
|