@aws-mdaa/dataops-job-l3-construct 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/.jsii +38 -28
  2. package/README.md +5 -0
  3. package/lib/dataops-job-l3-construct.d.ts +8 -1
  4. package/lib/dataops-job-l3-construct.js +40 -13
  5. package/node_modules/@aws-mdaa/config/README.md +3 -0
  6. package/node_modules/@aws-mdaa/config/lib/blueprint-value-transformer.d.ts +20 -0
  7. package/node_modules/@aws-mdaa/config/lib/blueprint-value-transformer.js +70 -0
  8. package/node_modules/@aws-mdaa/config/lib/blueprint-value-transformer.ts +88 -0
  9. package/node_modules/@aws-mdaa/config/lib/config.d.ts +87 -0
  10. package/node_modules/@aws-mdaa/config/lib/config.js +7 -0
  11. package/node_modules/@aws-mdaa/config/lib/config.ts +92 -0
  12. package/node_modules/@aws-mdaa/config/lib/index.d.ts +11 -0
  13. package/node_modules/@aws-mdaa/config/lib/index.js +28 -0
  14. package/node_modules/@aws-mdaa/config/lib/index.ts +12 -0
  15. package/node_modules/@aws-mdaa/config/lib/param-transformer.d.ts +49 -0
  16. package/node_modules/@aws-mdaa/config/lib/param-transformer.js +160 -0
  17. package/node_modules/@aws-mdaa/config/lib/param-transformer.ts +159 -0
  18. package/node_modules/@aws-mdaa/config/lib/path-value-transformer.d.ts +10 -0
  19. package/node_modules/@aws-mdaa/config/lib/path-value-transformer.js +30 -0
  20. package/node_modules/@aws-mdaa/config/lib/path-value-transformer.ts +27 -0
  21. package/node_modules/@aws-mdaa/config/lib/ref-value-transformer.d.ts +44 -0
  22. package/node_modules/@aws-mdaa/config/lib/ref-value-transformer.js +243 -0
  23. package/node_modules/@aws-mdaa/config/lib/ref-value-transformer.ts +302 -0
  24. package/node_modules/@aws-mdaa/config/lib/ssm-ref-transformer.d.ts +8 -0
  25. package/node_modules/@aws-mdaa/config/lib/ssm-ref-transformer.js +22 -0
  26. package/node_modules/@aws-mdaa/config/lib/ssm-ref-transformer.ts +21 -0
  27. package/node_modules/@aws-mdaa/config/lib/transformer.d.ts +35 -0
  28. package/node_modules/@aws-mdaa/config/lib/transformer.js +66 -0
  29. package/node_modules/@aws-mdaa/config/lib/transformer.ts +74 -0
  30. package/node_modules/@aws-mdaa/config/package.json +42 -0
  31. package/node_modules/@aws-mdaa/config/test/blueprint-value-transformer.test.js +224 -0
  32. package/node_modules/@aws-mdaa/config/test/blueprint-value-transformer.test.ts +259 -0
  33. package/node_modules/@aws-mdaa/config/test/config-nt.test.d.ts +5 -0
  34. package/node_modules/@aws-mdaa/config/test/config-nt.test.js +129 -0
  35. package/node_modules/@aws-mdaa/config/test/config-nt.test.ts +163 -0
  36. package/node_modules/@aws-mdaa/config/test/config.test.d.ts +5 -0
  37. package/node_modules/@aws-mdaa/config/test/config.test.js +409 -0
  38. package/node_modules/@aws-mdaa/config/test/config.test.ts +517 -0
  39. package/node_modules/@aws-mdaa/config/test/param-transformer.test.d.ts +5 -0
  40. package/node_modules/@aws-mdaa/config/test/param-transformer.test.js +216 -0
  41. package/node_modules/@aws-mdaa/config/test/param-transformer.test.ts +234 -0
  42. package/node_modules/@aws-mdaa/config/test/path-value-transformer.test.d.ts +5 -0
  43. package/node_modules/@aws-mdaa/config/test/path-value-transformer.test.js +59 -0
  44. package/node_modules/@aws-mdaa/config/test/path-value-transformer.test.ts +68 -0
  45. package/node_modules/@aws-mdaa/config/test/ref-value-transformer.test.d.ts +5 -0
  46. package/node_modules/@aws-mdaa/config/test/ref-value-transformer.test.js +254 -0
  47. package/node_modules/@aws-mdaa/config/test/ref-value-transformer.test.ts +304 -0
  48. package/node_modules/@aws-mdaa/config/test/ssm-ref-transformer.test.d.ts +5 -0
  49. package/node_modules/@aws-mdaa/config/test/ssm-ref-transformer.test.js +66 -0
  50. package/node_modules/@aws-mdaa/config/test/ssm-ref-transformer.test.ts +79 -0
  51. package/node_modules/@aws-mdaa/config/tsconfig.tsbuildinfo +1 -0
  52. package/node_modules/lodash/README.md +2 -2
  53. package/node_modules/lodash/_baseOrderBy.js +1 -1
  54. package/node_modules/lodash/_baseUnset.js +7 -20
  55. package/node_modules/lodash/_setCacheHas.js +1 -1
  56. package/node_modules/lodash/compact.js +1 -1
  57. package/node_modules/lodash/core.js +3 -3
  58. package/node_modules/lodash/core.min.js +26 -25
  59. package/node_modules/lodash/fromPairs.js +3 -1
  60. package/node_modules/lodash/lodash.js +38 -27
  61. package/node_modules/lodash/lodash.min.js +125 -129
  62. package/node_modules/lodash/package.json +4 -2
  63. package/node_modules/lodash/random.js +9 -0
  64. package/node_modules/lodash/template.js +16 -4
  65. package/node_modules/lodash/templateSettings.js +4 -0
  66. package/package.json +27 -32
  67. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/README.md +0 -185
  68. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/lib/index.d.ts +0 -57
  69. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/lib/index.js +0 -198
  70. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/lib/index.ts +0 -241
  71. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/package.json +0 -44
  72. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/test/bucketpolicy-helper.test.js +0 -200
  73. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/test/bucketpolicy-helper.test.ts +0 -215
  74. package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/tsconfig.tsbuildinfo +0 -1
  75. package/node_modules/@aws-mdaa/s3-inventory-helper/.npmignore +0 -34
  76. package/node_modules/@aws-mdaa/s3-inventory-helper/README.md +0 -3
  77. package/node_modules/@aws-mdaa/s3-inventory-helper/jest.config.js +0 -5
  78. package/node_modules/@aws-mdaa/s3-inventory-helper/lib/index.d.ts +0 -48
  79. package/node_modules/@aws-mdaa/s3-inventory-helper/lib/index.js +0 -213
  80. package/node_modules/@aws-mdaa/s3-inventory-helper/lib/index.ts +0 -241
  81. package/node_modules/@aws-mdaa/s3-inventory-helper/package.json +0 -44
  82. package/node_modules/@aws-mdaa/s3-inventory-helper/test/TODO +0 -0
  83. package/node_modules/@aws-mdaa/s3-inventory-helper/tsconfig.json +0 -40
  84. package/node_modules/@aws-mdaa/s3-inventory-helper/tsconfig.tsbuildinfo +0 -1
  85. package/node_modules/@aws-mdaa/s3-inventory-helper/typedoc.json +0 -7
  86. /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/.npmignore +0 -0
  87. /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/jest.config.js +0 -0
  88. /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper/test/bucketpolicy-helper.test.d.ts → config/test/blueprint-value-transformer.test.d.ts} +0 -0
  89. /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/tsconfig.json +0 -0
  90. /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/typedoc.json +0 -0
@@ -1,34 +0,0 @@
1
- # Don't include original .ts files when doing `npm pack`
2
- *.ts
3
- *.tsbuildinfo
4
- .eslintrc.js
5
- tsconfig.json
6
- typedoc.json
7
-
8
- # dev-related files
9
- coverage
10
- test/
11
-
12
- # pack output
13
- *.tgz
14
-
15
- dist
16
-
17
-
18
- #Exclude coverage reports for bundled deps
19
- node_modules/**/coverage/*
20
- # Exclude typescript source and config
21
- #*.ts
22
- #tsconfig.json
23
-
24
- # Include javascript files and typescript declarations
25
- !*.js
26
- !*.d.ts
27
-
28
- # Exclude jsii outdir
29
- jsii-dist
30
-
31
- # Include .jsii and .jsii.gz
32
- !.jsii
33
- !.jsii.gz
34
- jest.config.js
@@ -1,3 +0,0 @@
1
- # S3 Inventory Helper
2
-
3
- This is a helper class that helps construct working S3 inventory configurations that can be added to a bucket construct.
@@ -1,5 +0,0 @@
1
- const baseConfig = require("../../../jest.config");
2
-
3
- module.exports = {
4
- ...baseConfig,
5
- };
@@ -1,48 +0,0 @@
1
- /*!
2
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
- * SPDX-License-Identifier: Apache-2.0
4
- */
5
- import { Database } from '@aws-cdk/aws-glue-alpha';
6
- import { CfnTable } from 'aws-cdk-lib/aws-glue';
7
- import { PolicyStatement } from 'aws-cdk-lib/aws-iam';
8
- import { IBucket, Inventory } from 'aws-cdk-lib/aws-s3';
9
- import { Construct } from 'constructs';
10
- export interface BucketInventory {
11
- /** Name of the S3 bucket for which inventory data has been generated */
12
- readonly bucketName: string;
13
- /** Name of the S3 inventory configuration that generated the inventory data for the specified bucket */
14
- readonly inventoryName: string;
15
- }
16
- /** Helper class for creating S3 Inventories and associated Glue/Athena resources for querying these inventories. */
17
- export declare class InventoryHelper {
18
- /**
19
- * Helper method for creating an S3 inventory.
20
- * @param destinationBucket The bucket where S3 inventory data will be written
21
- * @param inventoryId The S3 inventory ID
22
- * @param objectsPrefix The object prefix from which inventory will be produced (on the bucket to which the inventory config is applied)
23
- * @param destinationPrefix The object prefix under which inventory will be written on the destination bucket.
24
- * @returns An S3 Inventory Config which can be applied to a bucket in order to produce periodic inventory data.
25
- */
26
- static createInvConfig(destinationBucket: IBucket, inventoryId: string, objectsPrefix?: string, destinationPrefix?: string, destinationAccount?: string): Inventory;
27
- /**
28
- * Helper method for creating a Glue table which can be used to query inventory data generated by an S3 Inventory config
29
- * @param scope Stack in which the table will be created
30
- * @param tableNamePrefix Will be used to prefix the table name.
31
- * @param locationBucketName The bucket where the inventory data is stored.
32
- * @param inventoryBucket The bucket for which inventory data has been generated
33
- * @param database The Glue database in which the table will be created
34
- * @param inventoryName The name of the S3 Inventory config
35
- * @param locationPrefix The S3 Prefix on the location bucket where the inventory is stored.
36
- * @returns A Glue table which can be used to query S3 inventory data in an S3 bucket.
37
- */
38
- static createGlueInvTable(scope: Construct, catalogId: string, tableNamePrefix: string, database: Database, locationBucketName: string, bucketInventories: BucketInventory[], locationPrefix?: string): CfnTable;
39
- /**
40
- * Helper method to generate an S3 Bucket Policy Statement allowing S3 service to write inventories to a bucket.
41
- * @param sourceBucketArn The bucket arn for which inventory data is being produced
42
- * @param bucketArn The bucket arn where inventory will be stored
43
- * @param srcAccount The account which owns the source bucket from which inventory is being produced.
44
- * @param inventoryPrefix The prefix on the destination bucket where inventory data will be stored.
45
- * @returns A policy statement which permits the S3 service principal to write inventory for the source bucket/account to the destination bucket.
46
- */
47
- static createInventoryBucketPolicyStatement(bucketArn: string, srcAccount: string, sourceBucketArn?: string, inventoryPrefix?: string): PolicyStatement;
48
- }
@@ -1,213 +0,0 @@
1
- "use strict";
2
- /*!
3
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4
- * SPDX-License-Identifier: Apache-2.0
5
- */
6
- Object.defineProperty(exports, "__esModule", { value: true });
7
- exports.InventoryHelper = void 0;
8
- const s3_constructs_1 = require("@aws-mdaa/s3-constructs");
9
- const aws_glue_1 = require("aws-cdk-lib/aws-glue");
10
- const aws_iam_1 = require("aws-cdk-lib/aws-iam");
11
- const aws_s3_1 = require("aws-cdk-lib/aws-s3");
12
- /** Helper class for creating S3 Inventories and associated Glue/Athena resources for querying these inventories. */
13
- class InventoryHelper {
14
- /**
15
- * Helper method for creating an S3 inventory.
16
- * @param destinationBucket The bucket where S3 inventory data will be written
17
- * @param inventoryId The S3 inventory ID
18
- * @param objectsPrefix The object prefix from which inventory will be produced (on the bucket to which the inventory config is applied)
19
- * @param destinationPrefix The object prefix under which inventory will be written on the destination bucket.
20
- * @returns An S3 Inventory Config which can be applied to a bucket in order to produce periodic inventory data.
21
- */
22
- static createInvConfig(destinationBucket, inventoryId, objectsPrefix, destinationPrefix, destinationAccount) {
23
- const bucketOwner = destinationAccount ? `${destinationAccount}` : destinationAccount;
24
- return {
25
- destination: {
26
- bucket: destinationBucket,
27
- /** S3 prefix for inventory report destination organization enabling systematic inventory file */
28
- prefix: s3_constructs_1.MdaaBucket.formatS3Prefix(destinationPrefix),
29
- bucketOwner: bucketOwner,
30
- },
31
- format: aws_s3_1.InventoryFormat.PARQUET,
32
- frequency: aws_s3_1.InventoryFrequency.DAILY,
33
- includeObjectVersions: aws_s3_1.InventoryObjectVersion.ALL,
34
- inventoryId: inventoryId,
35
- objectsPrefix: s3_constructs_1.MdaaBucket.formatS3Prefix(objectsPrefix, false, true),
36
- optionalFields: [
37
- 'Size',
38
- 'LastModifiedDate',
39
- 'StorageClass',
40
- 'ETag',
41
- 'IsMultipartUploaded',
42
- 'ReplicationStatus',
43
- 'EncryptionStatus',
44
- 'ObjectLockRetainUntilDate',
45
- 'ObjectLockMode',
46
- 'ObjectLockLegalHoldStatus',
47
- 'IntelligentTieringAccessTier',
48
- ],
49
- };
50
- }
51
- /**
52
- * Helper method for creating a Glue table which can be used to query inventory data generated by an S3 Inventory config
53
- * @param scope Stack in which the table will be created
54
- * @param tableNamePrefix Will be used to prefix the table name.
55
- * @param locationBucketName The bucket where the inventory data is stored.
56
- * @param inventoryBucket The bucket for which inventory data has been generated
57
- * @param database The Glue database in which the table will be created
58
- * @param inventoryName The name of the S3 Inventory config
59
- * @param locationPrefix The S3 Prefix on the location bucket where the inventory is stored.
60
- * @returns A Glue table which can be used to query S3 inventory data in an S3 bucket.
61
- */
62
- static createGlueInvTable(scope, catalogId, tableNamePrefix, database, locationBucketName, bucketInventories, locationPrefix) {
63
- const location = locationPrefix
64
- ? `s3://${locationBucketName}/${s3_constructs_1.MdaaBucket.formatS3Prefix(locationPrefix)}/`
65
- : `s3://${locationBucketName}/`;
66
- const tableInput = {
67
- name: `${tableNamePrefix}_inv`.replace(/-/gi, '_'),
68
- parameters: {
69
- EXTERNAL: 'TRUE',
70
- 'projection.enabled': 'true',
71
- 'projection.dt.type': 'date',
72
- 'projection.dt.range': '2022-01-01-00-00,NOW',
73
- 'projection.dt.format': 'yyyy-MM-dd-HH-mm',
74
- 'projection.dt.interval': '1',
75
- 'projection.dt.interval.unit': 'HOURS',
76
- 'projection.bucket_inventory.type': 'enum',
77
- 'projection.bucket_inventory.values': bucketInventories
78
- .map(x => `${x.bucketName}/${x.inventoryName}`)
79
- .join(','),
80
- 'storage.location.template': location + '${bucket_inventory}/hive/dt=${dt}',
81
- },
82
- tableType: 'EXTERNAL_TABLE',
83
- partitionKeys: [
84
- {
85
- name: 'bucket_inventory',
86
- type: 'string',
87
- },
88
- {
89
- name: 'dt',
90
- type: 'string',
91
- },
92
- ],
93
- storageDescriptor: {
94
- columns: [
95
- {
96
- name: 'bucket',
97
- type: 'string',
98
- },
99
- {
100
- name: 'key',
101
- type: 'string',
102
- },
103
- {
104
- name: 'version_id',
105
- type: 'string',
106
- },
107
- {
108
- name: 'is_latest',
109
- type: 'boolean',
110
- },
111
- {
112
- name: 'is_delete_marker',
113
- type: 'boolean',
114
- },
115
- {
116
- name: 'size',
117
- type: 'bigint',
118
- },
119
- {
120
- name: 'last_modified_date',
121
- type: 'timestamp',
122
- },
123
- {
124
- name: 'e_tag',
125
- type: 'string',
126
- },
127
- {
128
- name: 'storage_class',
129
- type: 'string',
130
- },
131
- {
132
- name: 'is_multipart_uploaded',
133
- type: 'boolean',
134
- },
135
- {
136
- name: 'replication_status',
137
- type: 'string',
138
- },
139
- {
140
- name: 'encryption_status',
141
- type: 'string',
142
- },
143
- {
144
- name: 'intelligent_tiering_tier',
145
- type: 'string',
146
- },
147
- // ,
148
- // {
149
- // name: "object_lock_retain_until_date",
150
- // type: "timestamp"
151
- // },
152
- // {
153
- // name: "object_lock_mode",
154
- // type: "string"
155
- // },
156
- // {
157
- // name: "object_lock_legal_hold_status",
158
- // type: "string"
159
- // }
160
- ],
161
- location: location,
162
- inputFormat: 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat',
163
- outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
164
- compressed: false,
165
- numberOfBuckets: -1,
166
- serdeInfo: {
167
- serializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
168
- parameters: {
169
- 'serialization.format': '1',
170
- },
171
- },
172
- },
173
- };
174
- const tableProps = {
175
- catalogId: catalogId,
176
- databaseName: database.databaseName,
177
- tableInput: tableInput,
178
- };
179
- return new aws_glue_1.CfnTable(scope, `inv-table-${tableNamePrefix}`, tableProps);
180
- }
181
- /**
182
- * Helper method to generate an S3 Bucket Policy Statement allowing S3 service to write inventories to a bucket.
183
- * @param sourceBucketArn The bucket arn for which inventory data is being produced
184
- * @param bucketArn The bucket arn where inventory will be stored
185
- * @param srcAccount The account which owns the source bucket from which inventory is being produced.
186
- * @param inventoryPrefix The prefix on the destination bucket where inventory data will be stored.
187
- * @returns A policy statement which permits the S3 service principal to write inventory for the source bucket/account to the destination bucket.
188
- */
189
- static createInventoryBucketPolicyStatement(bucketArn, srcAccount, sourceBucketArn, inventoryPrefix) {
190
- const resource = inventoryPrefix
191
- ? `${bucketArn}/${s3_constructs_1.MdaaBucket.formatS3Prefix(inventoryPrefix)}/*`
192
- : `${bucketArn}/*`;
193
- const allowS3Inventory = new aws_iam_1.PolicyStatement({
194
- sid: 'AllowS3Inventory',
195
- effect: aws_iam_1.Effect.ALLOW,
196
- resources: [resource],
197
- actions: ['s3:PutObject'],
198
- });
199
- allowS3Inventory.addCondition('StringEquals', {
200
- 'aws:SourceAccount': srcAccount,
201
- 's3:x-amz-acl': 'bucket-owner-full-control',
202
- });
203
- if (sourceBucketArn) {
204
- allowS3Inventory.addCondition('ArnLike', {
205
- 'aws:SourceArn': sourceBucketArn,
206
- });
207
- }
208
- allowS3Inventory.addServicePrincipal('s3.amazonaws.com');
209
- return allowS3Inventory;
210
- }
211
- }
212
- exports.InventoryHelper = InventoryHelper;
213
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJpbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUE7OztHQUdHOzs7QUFFSCwyREFBcUQ7QUFFckQsbURBQStEO0FBQy9ELGlEQUE4RDtBQUM5RCwrQ0FBcUg7QUFVckgsb0hBQW9IO0FBQ3BILE1BQWEsZUFBZTtJQUMxQjs7Ozs7OztPQU9HO0lBQ0ksTUFBTSxDQUFDLGVBQWUsQ0FDM0IsaUJBQTBCLEVBQzFCLFdBQW1CLEVBQ25CLGFBQXNCLEVBQ3RCLGlCQUEwQixFQUMxQixrQkFBMkI7UUFFM0IsTUFBTSxXQUFXLEdBQUcsa0JBQWtCLENBQUMsQ0FBQyxDQUFDLEdBQUcsa0JBQWtCLEVBQUUsQ0FBQyxDQUFDLENBQUMsa0JBQWtCLENBQUM7UUFDdEYsT0FBTztZQUNMLFdBQVcsRUFBRTtnQkFDWCxNQUFNLEVBQUUsaUJBQWlCO2dCQUN6QixpR0FBaUc7Z0JBQ2pHLE1BQU0sRUFBRSwwQkFBVSxDQUFDLGNBQWMsQ0FBQyxpQkFBaUIsQ0FBQztnQkFDcEQsV0FBVyxFQUFFLFdBQVc7YUFDekI7WUFDRCxNQUFNLEVBQUUsd0JBQWUsQ0FBQyxPQUFPO1lBQy9CLFNBQVMsRUFBRSwyQkFBa0IsQ0FBQyxLQUFLO1lBQ25DLHFCQUFxQixFQUFFLCtCQUFzQixDQUFDLEdBQUc7WUFDakQsV0FBVyxFQUFFLFdBQVc7WUFDeEIsYUFBYSxFQUFFLDBCQUFVLENBQUMsY0FBYyxDQUFDLGFBQWEsRUFBRSxLQUFLLEVBQUUsSUFBSSxDQUFDO1lBQ3BFLGNBQWMsRUFBRTtnQkFDZCxNQUFNO2dCQUNOLGtCQUFrQjtnQkFDbEIsY0FBYztnQkFDZCxNQUFNO2dCQUNOLHFCQUFxQjtnQkFDckIsbUJBQW1CO2dCQUNuQixrQkFBa0I7Z0JBQ2xCLDJCQUEyQjtnQkFDM0IsZ0JBQWdCO2dCQUNoQiwyQkFBMkI7Z0JBQzNCLDhCQUE4QjthQUMvQjtTQUNGLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7Ozs7T0FVRztJQUNJLE1BQU0sQ0FBQyxrQkFBa0IsQ0FDOUIsS0FBZ0IsRUFDaEIsU0FBaUIsRUFDakIsZUFBdUIsRUFDdkIsUUFBa0IsRUFDbEIsa0JBQTBCLEVBQzFCLGlCQUFvQyxFQUNwQyxjQUF1QjtRQUV2QixNQUFNLFFBQVEsR0FBRyxjQUFjO1lBQzdCLENBQUMsQ0FBQyxRQUFRLGtCQUFrQixJQUFJLDBCQUFVLENBQUMsY0FBYyxDQUFDLGNBQWMsQ0FBQyxHQUFHO1lBQzVFLENBQUMsQ0FBQyxRQUFRLGtCQUFrQixHQUFHLENBQUM7UUFDbEMsTUFBTSxVQUFVLEdBQUc7WUFDakIsSUFBSSxFQUFFLEdBQUcsZUFBZSxNQUFNLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxHQUFHLENBQUM7WUFDbEQsVUFBVSxFQUFFO2dCQUNWLFFBQVEsRUFBRSxNQUFNO2dCQUNoQixvQkFBb0IsRUFBRSxNQUFNO2dCQUM1QixvQkFBb0IsRUFBRSxNQUFNO2dCQUM1QixxQkFBcUIsRUFBRSxzQkFBc0I7Z0JBQzdDLHNCQUFzQixFQUFFLGtCQUFrQjtnQkFDMUMsd0JBQXdCLEVBQUUsR0FBRztnQkFDN0IsNkJBQTZCLEVBQUUsT0FBTztnQkFDdEMsa0NBQWtDLEVBQUUsTUFBTTtnQkFDMUMsb0NBQW9DLEVBQUUsaUJBQWlCO3FCQUNwRCxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxHQUFHLENBQUMsQ0FBQyxVQUFVLElBQUksQ0FBQyxDQUFDLGFBQWEsRUFBRSxDQUFDO3FCQUM5QyxJQUFJLENBQUMsR0FBRyxDQUFDO2dCQUNaLDJCQUEyQixFQUFFLFFBQVEsR0FBRyxtQ0FBbUM7YUFDNUU7WUFDRCxTQUFTLEVBQUUsZ0JBQWdCO1lBQzNCLGFBQWEsRUFBRTtnQkFDYjtvQkFDRSxJQUFJLEVBQUUsa0JBQWtCO29CQUN4QixJQUFJLEVBQUUsUUFBUTtpQkFDZjtnQkFDRDtvQkFDRSxJQUFJLEVBQUUsSUFBSTtvQkFDVixJQUFJLEVBQUUsUUFBUTtpQkFDZjthQUNGO1lBQ0QsaUJBQWlCLEVBQUU7Z0JBQ2pCLE9BQU8sRUFBRTtvQkFDUDt3QkFDRSxJQUFJLEVBQUUsUUFBUTt3QkFDZCxJQUFJLEVBQUUsUUFBUTtxQkFDZjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsS0FBSzt3QkFDWCxJQUFJLEVBQUUsUUFBUTtxQkFDZjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsWUFBWTt3QkFDbEIsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLFdBQVc7d0JBQ2pCLElBQUksRUFBRSxTQUFTO3FCQUNoQjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsa0JBQWtCO3dCQUN4QixJQUFJLEVBQUUsU0FBUztxQkFDaEI7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLE1BQU07d0JBQ1osSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLG9CQUFvQjt3QkFDMUIsSUFBSSxFQUFFLFdBQVc7cUJBQ2xCO29CQUNEO3dCQUNFLElBQUksRUFBRSxPQUFPO3dCQUNiLElBQUksRUFBRSxRQUFRO3FCQUNmO29CQUNEO3dCQUNFLElBQUksRUFBRSxlQUFlO3dCQUNyQixJQUFJLEVBQUUsUUFBUTtxQkFDZjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsdUJBQXVCO3dCQUM3QixJQUFJLEVBQUUsU0FBUztxQkFDaEI7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLG9CQUFvQjt3QkFDMUIsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLG1CQUFtQjt3QkFDekIsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLDBCQUEwQjt3QkFDaEMsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0QsSUFBSTtvQkFDSixJQUFJO29CQUNKLDZDQUE2QztvQkFDN0Msd0JBQXdCO29CQUN4QixLQUFLO29CQUNMLElBQUk7b0JBQ0osZ0NBQWdDO29CQUNoQyxxQkFBcUI7b0JBQ3JCLEtBQUs7b0JBQ0wsSUFBSTtvQkFDSiw2Q0FBNkM7b0JBQzdDLHFCQUFxQjtvQkFDckIsSUFBSTtpQkFDTDtnQkFDRCxRQUFRLEVBQUUsUUFBUTtnQkFDbEIsV0FBVyxFQUFFLHFEQUFxRDtnQkFDbEUsWUFBWSxFQUFFLDREQUE0RDtnQkFDMUUsVUFBVSxFQUFFLEtBQUs7Z0JBQ2pCLGVBQWUsRUFBRSxDQUFDLENBQUM7Z0JBQ25CLFNBQVMsRUFBRTtvQkFDVCxvQkFBb0IsRUFBRSw2REFBNkQ7b0JBQ25GLFVBQVUsRUFBRTt3QkFDVixzQkFBc0IsRUFBRSxHQUFHO3FCQUM1QjtpQkFDRjthQUNGO1NBQ0YsQ0FBQztRQUVGLE1BQU0sVUFBVSxHQUFrQjtZQUNoQyxTQUFTLEVBQUUsU0FBUztZQUNwQixZQUFZLEVBQUUsUUFBUSxDQUFDLFlBQVk7WUFDbkMsVUFBVSxFQUFFLFVBQVU7U0FDdkIsQ0FBQztRQUNGLE9BQU8sSUFBSSxtQkFBUSxDQUFDLEtBQUssRUFBRSxhQUFhLGVBQWUsRUFBRSxFQUFFLFVBQVUsQ0FBQyxDQUFDO0lBQ3pFLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0ksTUFBTSxDQUFDLG9DQUFvQyxDQUNoRCxTQUFpQixFQUNqQixVQUFrQixFQUNsQixlQUF3QixFQUN4QixlQUF3QjtRQUV4QixNQUFNLFFBQVEsR0FBRyxlQUFlO1lBQzlCLENBQUMsQ0FBQyxHQUFHLFNBQVMsSUFBSSwwQkFBVSxDQUFDLGNBQWMsQ0FBQyxlQUFlLENBQUMsSUFBSTtZQUNoRSxDQUFDLENBQUMsR0FBRyxTQUFTLElBQUksQ0FBQztRQUNyQixNQUFNLGdCQUFnQixHQUFHLElBQUkseUJBQWUsQ0FBQztZQUMzQyxHQUFHLEVBQUUsa0JBQWtCO1lBQ3ZCLE1BQU0sRUFBRSxnQkFBTSxDQUFDLEtBQUs7WUFDcEIsU0FBUyxFQUFFLENBQUMsUUFBUSxDQUFDO1lBQ3JCLE9BQU8sRUFBRSxDQUFDLGNBQWMsQ0FBQztTQUMxQixDQUFDLENBQUM7UUFDSCxnQkFBZ0IsQ0FBQyxZQUFZLENBQUMsY0FBYyxFQUFFO1lBQzVDLG1CQUFtQixFQUFFLFVBQVU7WUFDL0IsY0FBYyxFQUFFLDJCQUEyQjtTQUM1QyxDQUFDLENBQUM7UUFDSCxJQUFJLGVBQWUsRUFBRSxDQUFDO1lBQ3BCLGdCQUFnQixDQUFDLFlBQVksQ0FBQyxTQUFTLEVBQUU7Z0JBQ3ZDLGVBQWUsRUFBRSxlQUFlO2FBQ2pDLENBQUMsQ0FBQztRQUNMLENBQUM7UUFDRCxnQkFBZ0IsQ0FBQyxtQkFBbUIsQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDO1FBQ3pELE9BQU8sZ0JBQWdCLENBQUM7SUFDMUIsQ0FBQztDQUNGO0FBNU5ELDBDQTROQyIsInNvdXJjZXNDb250ZW50IjpbIi8qIVxuICogQ29weXJpZ2h0IEFtYXpvbi5jb20sIEluYy4gb3IgaXRzIGFmZmlsaWF0ZXMuIEFsbCBSaWdodHMgUmVzZXJ2ZWQuXG4gKiBTUERYLUxpY2Vuc2UtSWRlbnRpZmllcjogQXBhY2hlLTIuMFxuICovXG5cbmltcG9ydCB7IE1kYWFCdWNrZXQgfSBmcm9tICdAYXdzLW1kYWEvczMtY29uc3RydWN0cyc7XG5pbXBvcnQgeyBEYXRhYmFzZSB9IGZyb20gJ0Bhd3MtY2RrL2F3cy1nbHVlLWFscGhhJztcbmltcG9ydCB7IENmblRhYmxlLCBDZm5UYWJsZVByb3BzIH0gZnJvbSAnYXdzLWNkay1saWIvYXdzLWdsdWUnO1xuaW1wb3J0IHsgRWZmZWN0LCBQb2xpY3lTdGF0ZW1lbnQgfSBmcm9tICdhd3MtY2RrLWxpYi9hd3MtaWFtJztcbmltcG9ydCB7IElCdWNrZXQsIEludmVudG9yeSwgSW52ZW50b3J5Rm9ybWF0LCBJbnZlbnRvcnlGcmVxdWVuY3ksIEludmVudG9yeU9iamVjdFZlcnNpb24gfSBmcm9tICdhd3MtY2RrLWxpYi9hd3MtczMnO1xuaW1wb3J0IHsgQ29uc3RydWN0IH0gZnJvbSAnY29uc3RydWN0cyc7XG5cbmV4cG9ydCBpbnRlcmZhY2UgQnVja2V0SW52ZW50b3J5IHtcbiAgLyoqIE5hbWUgb2YgdGhlIFMzIGJ1Y2tldCBmb3Igd2hpY2ggaW52ZW50b3J5IGRhdGEgaGFzIGJlZW4gZ2VuZXJhdGVkICovXG4gIHJlYWRvbmx5IGJ1Y2tldE5hbWU6IHN0cmluZztcbiAgLyoqIE5hbWUgb2YgdGhlIFMzIGludmVudG9yeSBjb25maWd1cmF0aW9uIHRoYXQgZ2VuZXJhdGVkIHRoZSBpbnZlbnRvcnkgZGF0YSBmb3IgdGhlIHNwZWNpZmllZCBidWNrZXQgKi9cbiAgcmVhZG9ubHkgaW52ZW50b3J5TmFtZTogc3RyaW5nO1xufVxuXG4vKiogSGVscGVyIGNsYXNzIGZvciBjcmVhdGluZyBTMyBJbnZlbnRvcmllcyBhbmQgYXNzb2NpYXRlZCBHbHVlL0F0aGVuYSByZXNvdXJjZXMgZm9yIHF1ZXJ5aW5nIHRoZXNlIGludmVudG9yaWVzLiAqL1xuZXhwb3J0IGNsYXNzIEludmVudG9yeUhlbHBlciB7XG4gIC8qKlxuICAgKiBIZWxwZXIgbWV0aG9kIGZvciBjcmVhdGluZyBhbiBTMyBpbnZlbnRvcnkuXG4gICAqIEBwYXJhbSBkZXN0aW5hdGlvbkJ1Y2tldCBUaGUgYnVja2V0IHdoZXJlIFMzIGludmVudG9yeSBkYXRhIHdpbGwgYmUgd3JpdHRlblxuICAgKiBAcGFyYW0gaW52ZW50b3J5SWQgVGhlIFMzIGludmVudG9yeSBJRFxuICAgKiBAcGFyYW0gb2JqZWN0c1ByZWZpeCBUaGUgb2JqZWN0IHByZWZpeCBmcm9tIHdoaWNoIGludmVudG9yeSB3aWxsIGJlIHByb2R1Y2VkIChvbiB0aGUgYnVja2V0IHRvIHdoaWNoIHRoZSBpbnZlbnRvcnkgY29uZmlnIGlzIGFwcGxpZWQpXG4gICAqIEBwYXJhbSBkZXN0aW5hdGlvblByZWZpeCBUaGUgb2JqZWN0IHByZWZpeCB1bmRlciB3aGljaCBpbnZlbnRvcnkgd2lsbCBiZSB3cml0dGVuIG9uIHRoZSBkZXN0aW5hdGlvbiBidWNrZXQuXG4gICAqIEByZXR1cm5zIEFuIFMzIEludmVudG9yeSBDb25maWcgd2hpY2ggY2FuIGJlIGFwcGxpZWQgdG8gYSBidWNrZXQgaW4gb3JkZXIgdG8gcHJvZHVjZSBwZXJpb2RpYyBpbnZlbnRvcnkgZGF0YS5cbiAgICovXG4gIHB1YmxpYyBzdGF0aWMgY3JlYXRlSW52Q29uZmlnKFxuICAgIGRlc3RpbmF0aW9uQnVja2V0OiBJQnVja2V0LFxuICAgIGludmVudG9yeUlkOiBzdHJpbmcsXG4gICAgb2JqZWN0c1ByZWZpeD86IHN0cmluZyxcbiAgICBkZXN0aW5hdGlvblByZWZpeD86IHN0cmluZyxcbiAgICBkZXN0aW5hdGlvbkFjY291bnQ/OiBzdHJpbmcsXG4gICk6IEludmVudG9yeSB7XG4gICAgY29uc3QgYnVja2V0T3duZXIgPSBkZXN0aW5hdGlvbkFjY291bnQgPyBgJHtkZXN0aW5hdGlvbkFjY291bnR9YCA6IGRlc3RpbmF0aW9uQWNjb3VudDtcbiAgICByZXR1cm4ge1xuICAgICAgZGVzdGluYXRpb246IHtcbiAgICAgICAgYnVja2V0OiBkZXN0aW5hdGlvbkJ1Y2tldCxcbiAgICAgICAgLyoqIFMzIHByZWZpeCBmb3IgaW52ZW50b3J5IHJlcG9ydCBkZXN0aW5hdGlvbiBvcmdhbml6YXRpb24gZW5hYmxpbmcgc3lzdGVtYXRpYyBpbnZlbnRvcnkgZmlsZSAqL1xuICAgICAgICBwcmVmaXg6IE1kYWFCdWNrZXQuZm9ybWF0UzNQcmVmaXgoZGVzdGluYXRpb25QcmVmaXgpLFxuICAgICAgICBidWNrZXRPd25lcjogYnVja2V0T3duZXIsXG4gICAgICB9LFxuICAgICAgZm9ybWF0OiBJbnZlbnRvcnlGb3JtYXQuUEFSUVVFVCxcbiAgICAgIGZyZXF1ZW5jeTogSW52ZW50b3J5RnJlcXVlbmN5LkRBSUxZLFxuICAgICAgaW5jbHVkZU9iamVjdFZlcnNpb25zOiBJbnZlbnRvcnlPYmplY3RWZXJzaW9uLkFMTCxcbiAgICAgIGludmVudG9yeUlkOiBpbnZlbnRvcnlJZCxcbiAgICAgIG9iamVjdHNQcmVmaXg6IE1kYWFCdWNrZXQuZm9ybWF0UzNQcmVmaXgob2JqZWN0c1ByZWZpeCwgZmFsc2UsIHRydWUpLFxuICAgICAgb3B0aW9uYWxGaWVsZHM6IFtcbiAgICAgICAgJ1NpemUnLFxuICAgICAgICAnTGFzdE1vZGlmaWVkRGF0ZScsXG4gICAgICAgICdTdG9yYWdlQ2xhc3MnLFxuICAgICAgICAnRVRhZycsXG4gICAgICAgICdJc011bHRpcGFydFVwbG9hZGVkJyxcbiAgICAgICAgJ1JlcGxpY2F0aW9uU3RhdHVzJyxcbiAgICAgICAgJ0VuY3J5cHRpb25TdGF0dXMnLFxuICAgICAgICAnT2JqZWN0TG9ja1JldGFpblVudGlsRGF0ZScsXG4gICAgICAgICdPYmplY3RMb2NrTW9kZScsXG4gICAgICAgICdPYmplY3RMb2NrTGVnYWxIb2xkU3RhdHVzJyxcbiAgICAgICAgJ0ludGVsbGlnZW50VGllcmluZ0FjY2Vzc1RpZXInLFxuICAgICAgXSxcbiAgICB9O1xuICB9XG5cbiAgLyoqXG4gICAqIEhlbHBlciBtZXRob2QgZm9yIGNyZWF0aW5nIGEgR2x1ZSB0YWJsZSB3aGljaCBjYW4gYmUgdXNlZCB0byBxdWVyeSBpbnZlbnRvcnkgZGF0YSBnZW5lcmF0ZWQgYnkgYW4gUzMgSW52ZW50b3J5IGNvbmZpZ1xuICAgKiBAcGFyYW0gc2NvcGUgU3RhY2sgaW4gd2hpY2ggdGhlIHRhYmxlIHdpbGwgYmUgY3JlYXRlZFxuICAgKiBAcGFyYW0gdGFibGVOYW1lUHJlZml4IFdpbGwgYmUgdXNlZCB0byBwcmVmaXggdGhlIHRhYmxlIG5hbWUuXG4gICAqIEBwYXJhbSBsb2NhdGlvbkJ1Y2tldE5hbWUgVGhlIGJ1Y2tldCB3aGVyZSB0aGUgaW52ZW50b3J5IGRhdGEgaXMgc3RvcmVkLlxuICAgKiBAcGFyYW0gaW52ZW50b3J5QnVja2V0IFRoZSBidWNrZXQgZm9yIHdoaWNoIGludmVudG9yeSBkYXRhIGhhcyBiZWVuIGdlbmVyYXRlZFxuICAgKiBAcGFyYW0gZGF0YWJhc2UgVGhlIEdsdWUgZGF0YWJhc2UgaW4gd2hpY2ggdGhlIHRhYmxlIHdpbGwgYmUgY3JlYXRlZFxuICAgKiBAcGFyYW0gaW52ZW50b3J5TmFtZSBUaGUgbmFtZSBvZiB0aGUgUzMgSW52ZW50b3J5IGNvbmZpZ1xuICAgKiBAcGFyYW0gbG9jYXRpb25QcmVmaXggVGhlIFMzIFByZWZpeCBvbiB0aGUgbG9jYXRpb24gYnVja2V0IHdoZXJlIHRoZSBpbnZlbnRvcnkgaXMgc3RvcmVkLlxuICAgKiBAcmV0dXJucyBBIEdsdWUgdGFibGUgd2hpY2ggY2FuIGJlIHVzZWQgdG8gcXVlcnkgUzMgaW52ZW50b3J5IGRhdGEgaW4gYW4gUzMgYnVja2V0LlxuICAgKi9cbiAgcHVibGljIHN0YXRpYyBjcmVhdGVHbHVlSW52VGFibGUoXG4gICAgc2NvcGU6IENvbnN0cnVjdCxcbiAgICBjYXRhbG9nSWQ6IHN0cmluZyxcbiAgICB0YWJsZU5hbWVQcmVmaXg6IHN0cmluZyxcbiAgICBkYXRhYmFzZTogRGF0YWJhc2UsXG4gICAgbG9jYXRpb25CdWNrZXROYW1lOiBzdHJpbmcsXG4gICAgYnVja2V0SW52ZW50b3JpZXM6IEJ1Y2tldEludmVudG9yeVtdLFxuICAgIGxvY2F0aW9uUHJlZml4Pzogc3RyaW5nLFxuICApOiBDZm5UYWJsZSB7XG4gICAgY29uc3QgbG9jYXRpb24gPSBsb2NhdGlvblByZWZpeFxuICAgICAgPyBgczM6Ly8ke2xvY2F0aW9uQnVja2V0TmFtZX0vJHtNZGFhQnVja2V0LmZvcm1hdFMzUHJlZml4KGxvY2F0aW9uUHJlZml4KX0vYFxuICAgICAgOiBgczM6Ly8ke2xvY2F0aW9uQnVja2V0TmFtZX0vYDtcbiAgICBjb25zdCB0YWJsZUlucHV0ID0ge1xuICAgICAgbmFtZTogYCR7dGFibGVOYW1lUHJlZml4fV9pbnZgLnJlcGxhY2UoLy0vZ2ksICdfJyksXG4gICAgICBwYXJhbWV0ZXJzOiB7XG4gICAgICAgIEVYVEVSTkFMOiAnVFJVRScsXG4gICAgICAgICdwcm9qZWN0aW9uLmVuYWJsZWQnOiAndHJ1ZScsXG4gICAgICAgICdwcm9qZWN0aW9uLmR0LnR5cGUnOiAnZGF0ZScsXG4gICAgICAgICdwcm9qZWN0aW9uLmR0LnJhbmdlJzogJzIwMjItMDEtMDEtMDAtMDAsTk9XJyxcbiAgICAgICAgJ3Byb2plY3Rpb24uZHQuZm9ybWF0JzogJ3l5eXktTU0tZGQtSEgtbW0nLFxuICAgICAgICAncHJvamVjdGlvbi5kdC5pbnRlcnZhbCc6ICcxJyxcbiAgICAgICAgJ3Byb2plY3Rpb24uZHQuaW50ZXJ2YWwudW5pdCc6ICdIT1VSUycsXG4gICAgICAgICdwcm9qZWN0aW9uLmJ1Y2tldF9pbnZlbnRvcnkudHlwZSc6ICdlbnVtJyxcbiAgICAgICAgJ3Byb2plY3Rpb24uYnVja2V0X2ludmVudG9yeS52YWx1ZXMnOiBidWNrZXRJbnZlbnRvcmllc1xuICAgICAgICAgIC5tYXAoeCA9PiBgJHt4LmJ1Y2tldE5hbWV9LyR7eC5pbnZlbnRvcnlOYW1lfWApXG4gICAgICAgICAgLmpvaW4oJywnKSxcbiAgICAgICAgJ3N0b3JhZ2UubG9jYXRpb24udGVtcGxhdGUnOiBsb2NhdGlvbiArICcke2J1Y2tldF9pbnZlbnRvcnl9L2hpdmUvZHQ9JHtkdH0nLFxuICAgICAgfSxcbiAgICAgIHRhYmxlVHlwZTogJ0VYVEVSTkFMX1RBQkxFJyxcbiAgICAgIHBhcnRpdGlvbktleXM6IFtcbiAgICAgICAge1xuICAgICAgICAgIG5hbWU6ICdidWNrZXRfaW52ZW50b3J5JyxcbiAgICAgICAgICB0eXBlOiAnc3RyaW5nJyxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIG5hbWU6ICdkdCcsXG4gICAgICAgICAgdHlwZTogJ3N0cmluZycsXG4gICAgICAgIH0sXG4gICAgICBdLFxuICAgICAgc3RvcmFnZURlc2NyaXB0b3I6IHtcbiAgICAgICAgY29sdW1uczogW1xuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdidWNrZXQnLFxuICAgICAgICAgICAgdHlwZTogJ3N0cmluZycsXG4gICAgICAgICAgfSxcbiAgICAgICAgICB7XG4gICAgICAgICAgICBuYW1lOiAna2V5JyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ3ZlcnNpb25faWQnLFxuICAgICAgICAgICAgdHlwZTogJ3N0cmluZycsXG4gICAgICAgICAgfSxcbiAgICAgICAgICB7XG4gICAgICAgICAgICBuYW1lOiAnaXNfbGF0ZXN0JyxcbiAgICAgICAgICAgIHR5cGU6ICdib29sZWFuJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdpc19kZWxldGVfbWFya2VyJyxcbiAgICAgICAgICAgIHR5cGU6ICdib29sZWFuJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdzaXplJyxcbiAgICAgICAgICAgIHR5cGU6ICdiaWdpbnQnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2xhc3RfbW9kaWZpZWRfZGF0ZScsXG4gICAgICAgICAgICB0eXBlOiAndGltZXN0YW1wJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdlX3RhZycsXG4gICAgICAgICAgICB0eXBlOiAnc3RyaW5nJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdzdG9yYWdlX2NsYXNzJyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2lzX211bHRpcGFydF91cGxvYWRlZCcsXG4gICAgICAgICAgICB0eXBlOiAnYm9vbGVhbicsXG4gICAgICAgICAgfSxcbiAgICAgICAgICB7XG4gICAgICAgICAgICBuYW1lOiAncmVwbGljYXRpb25fc3RhdHVzJyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2VuY3J5cHRpb25fc3RhdHVzJyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2ludGVsbGlnZW50X3RpZXJpbmdfdGllcicsXG4gICAgICAgICAgICB0eXBlOiAnc3RyaW5nJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIC8vICxcbiAgICAgICAgICAvLyB7XG4gICAgICAgICAgLy8gICAgIG5hbWU6IFwib2JqZWN0X2xvY2tfcmV0YWluX3VudGlsX2RhdGVcIixcbiAgICAgICAgICAvLyAgICAgdHlwZTogXCJ0aW1lc3RhbXBcIlxuICAgICAgICAgIC8vIH0sXG4gICAgICAgICAgLy8ge1xuICAgICAgICAgIC8vICAgICBuYW1lOiBcIm9iamVjdF9sb2NrX21vZGVcIixcbiAgICAgICAgICAvLyAgICAgdHlwZTogXCJzdHJpbmdcIlxuICAgICAgICAgIC8vIH0sXG4gICAgICAgICAgLy8ge1xuICAgICAgICAgIC8vICAgICBuYW1lOiBcIm9iamVjdF9sb2NrX2xlZ2FsX2hvbGRfc3RhdHVzXCIsXG4gICAgICAgICAgLy8gICAgIHR5cGU6IFwic3RyaW5nXCJcbiAgICAgICAgICAvLyB9XG4gICAgICAgIF0sXG4gICAgICAgIGxvY2F0aW9uOiBsb2NhdGlvbixcbiAgICAgICAgaW5wdXRGb3JtYXQ6ICdvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLmlvLlN5bWxpbmtUZXh0SW5wdXRGb3JtYXQnLFxuICAgICAgICBvdXRwdXRGb3JtYXQ6ICdvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLmlvLkhpdmVJZ25vcmVLZXlUZXh0T3V0cHV0Rm9ybWF0JyxcbiAgICAgICAgY29tcHJlc3NlZDogZmFsc2UsXG4gICAgICAgIG51bWJlck9mQnVja2V0czogLTEsXG4gICAgICAgIHNlcmRlSW5mbzoge1xuICAgICAgICAgIHNlcmlhbGl6YXRpb25MaWJyYXJ5OiAnb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5xbC5pby5wYXJxdWV0LnNlcmRlLlBhcnF1ZXRIaXZlU2VyRGUnLFxuICAgICAgICAgIHBhcmFtZXRlcnM6IHtcbiAgICAgICAgICAgICdzZXJpYWxpemF0aW9uLmZvcm1hdCc6ICcxJyxcbiAgICAgICAgICB9LFxuICAgICAgICB9LFxuICAgICAgfSxcbiAgICB9O1xuXG4gICAgY29uc3QgdGFibGVQcm9wczogQ2ZuVGFibGVQcm9wcyA9IHtcbiAgICAgIGNhdGFsb2dJZDogY2F0YWxvZ0lkLFxuICAgICAgZGF0YWJhc2VOYW1lOiBkYXRhYmFzZS5kYXRhYmFzZU5hbWUsXG4gICAgICB0YWJsZUlucHV0OiB0YWJsZUlucHV0LFxuICAgIH07XG4gICAgcmV0dXJuIG5ldyBDZm5UYWJsZShzY29wZSwgYGludi10YWJsZS0ke3RhYmxlTmFtZVByZWZpeH1gLCB0YWJsZVByb3BzKTtcbiAgfVxuXG4gIC8qKlxuICAgKiBIZWxwZXIgbWV0aG9kIHRvIGdlbmVyYXRlIGFuIFMzIEJ1Y2tldCBQb2xpY3kgU3RhdGVtZW50IGFsbG93aW5nIFMzIHNlcnZpY2UgdG8gd3JpdGUgaW52ZW50b3JpZXMgdG8gYSBidWNrZXQuXG4gICAqIEBwYXJhbSBzb3VyY2VCdWNrZXRBcm4gVGhlIGJ1Y2tldCBhcm4gZm9yIHdoaWNoIGludmVudG9yeSBkYXRhIGlzIGJlaW5nIHByb2R1Y2VkXG4gICAqIEBwYXJhbSBidWNrZXRBcm4gIFRoZSBidWNrZXQgYXJuIHdoZXJlIGludmVudG9yeSB3aWxsIGJlIHN0b3JlZFxuICAgKiBAcGFyYW0gc3JjQWNjb3VudCBUaGUgYWNjb3VudCB3aGljaCBvd25zIHRoZSBzb3VyY2UgYnVja2V0IGZyb20gd2hpY2ggaW52ZW50b3J5IGlzIGJlaW5nIHByb2R1Y2VkLlxuICAgKiBAcGFyYW0gaW52ZW50b3J5UHJlZml4IFRoZSBwcmVmaXggb24gdGhlIGRlc3RpbmF0aW9uIGJ1Y2tldCB3aGVyZSBpbnZlbnRvcnkgZGF0YSB3aWxsIGJlIHN0b3JlZC5cbiAgICogQHJldHVybnMgQSBwb2xpY3kgc3RhdGVtZW50IHdoaWNoIHBlcm1pdHMgdGhlIFMzIHNlcnZpY2UgcHJpbmNpcGFsIHRvIHdyaXRlIGludmVudG9yeSBmb3IgdGhlIHNvdXJjZSBidWNrZXQvYWNjb3VudCB0byB0aGUgZGVzdGluYXRpb24gYnVja2V0LlxuICAgKi9cbiAgcHVibGljIHN0YXRpYyBjcmVhdGVJbnZlbnRvcnlCdWNrZXRQb2xpY3lTdGF0ZW1lbnQoXG4gICAgYnVja2V0QXJuOiBzdHJpbmcsXG4gICAgc3JjQWNjb3VudDogc3RyaW5nLFxuICAgIHNvdXJjZUJ1Y2tldEFybj86IHN0cmluZyxcbiAgICBpbnZlbnRvcnlQcmVmaXg/OiBzdHJpbmcsXG4gICk6IFBvbGljeVN0YXRlbWVudCB7XG4gICAgY29uc3QgcmVzb3VyY2UgPSBpbnZlbnRvcnlQcmVmaXhcbiAgICAgID8gYCR7YnVja2V0QXJufS8ke01kYWFCdWNrZXQuZm9ybWF0UzNQcmVmaXgoaW52ZW50b3J5UHJlZml4KX0vKmBcbiAgICAgIDogYCR7YnVja2V0QXJufS8qYDtcbiAgICBjb25zdCBhbGxvd1MzSW52ZW50b3J5ID0gbmV3IFBvbGljeVN0YXRlbWVudCh7XG4gICAgICBzaWQ6ICdBbGxvd1MzSW52ZW50b3J5JyxcbiAgICAgIGVmZmVjdDogRWZmZWN0LkFMTE9XLFxuICAgICAgcmVzb3VyY2VzOiBbcmVzb3VyY2VdLFxuICAgICAgYWN0aW9uczogWydzMzpQdXRPYmplY3QnXSxcbiAgICB9KTtcbiAgICBhbGxvd1MzSW52ZW50b3J5LmFkZENvbmRpdGlvbignU3RyaW5nRXF1YWxzJywge1xuICAgICAgJ2F3czpTb3VyY2VBY2NvdW50Jzogc3JjQWNjb3VudCxcbiAgICAgICdzMzp4LWFtei1hY2wnOiAnYnVja2V0LW93bmVyLWZ1bGwtY29udHJvbCcsXG4gICAgfSk7XG4gICAgaWYgKHNvdXJjZUJ1Y2tldEFybikge1xuICAgICAgYWxsb3dTM0ludmVudG9yeS5hZGRDb25kaXRpb24oJ0Fybkxpa2UnLCB7XG4gICAgICAgICdhd3M6U291cmNlQXJuJzogc291cmNlQnVja2V0QXJuLFxuICAgICAgfSk7XG4gICAgfVxuICAgIGFsbG93UzNJbnZlbnRvcnkuYWRkU2VydmljZVByaW5jaXBhbCgnczMuYW1hem9uYXdzLmNvbScpO1xuICAgIHJldHVybiBhbGxvd1MzSW52ZW50b3J5O1xuICB9XG59XG4iXX0=
@@ -1,241 +0,0 @@
1
- /*!
2
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
- * SPDX-License-Identifier: Apache-2.0
4
- */
5
-
6
- import { MdaaBucket } from '@aws-mdaa/s3-constructs';
7
- import { Database } from '@aws-cdk/aws-glue-alpha';
8
- import { CfnTable, CfnTableProps } from 'aws-cdk-lib/aws-glue';
9
- import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam';
10
- import { IBucket, Inventory, InventoryFormat, InventoryFrequency, InventoryObjectVersion } from 'aws-cdk-lib/aws-s3';
11
- import { Construct } from 'constructs';
12
-
13
- export interface BucketInventory {
14
- /** Name of the S3 bucket for which inventory data has been generated */
15
- readonly bucketName: string;
16
- /** Name of the S3 inventory configuration that generated the inventory data for the specified bucket */
17
- readonly inventoryName: string;
18
- }
19
-
20
- /** Helper class for creating S3 Inventories and associated Glue/Athena resources for querying these inventories. */
21
- export class InventoryHelper {
22
- /**
23
- * Helper method for creating an S3 inventory.
24
- * @param destinationBucket The bucket where S3 inventory data will be written
25
- * @param inventoryId The S3 inventory ID
26
- * @param objectsPrefix The object prefix from which inventory will be produced (on the bucket to which the inventory config is applied)
27
- * @param destinationPrefix The object prefix under which inventory will be written on the destination bucket.
28
- * @returns An S3 Inventory Config which can be applied to a bucket in order to produce periodic inventory data.
29
- */
30
- public static createInvConfig(
31
- destinationBucket: IBucket,
32
- inventoryId: string,
33
- objectsPrefix?: string,
34
- destinationPrefix?: string,
35
- destinationAccount?: string,
36
- ): Inventory {
37
- const bucketOwner = destinationAccount ? `${destinationAccount}` : destinationAccount;
38
- return {
39
- destination: {
40
- bucket: destinationBucket,
41
- /** S3 prefix for inventory report destination organization enabling systematic inventory file */
42
- prefix: MdaaBucket.formatS3Prefix(destinationPrefix),
43
- bucketOwner: bucketOwner,
44
- },
45
- format: InventoryFormat.PARQUET,
46
- frequency: InventoryFrequency.DAILY,
47
- includeObjectVersions: InventoryObjectVersion.ALL,
48
- inventoryId: inventoryId,
49
- objectsPrefix: MdaaBucket.formatS3Prefix(objectsPrefix, false, true),
50
- optionalFields: [
51
- 'Size',
52
- 'LastModifiedDate',
53
- 'StorageClass',
54
- 'ETag',
55
- 'IsMultipartUploaded',
56
- 'ReplicationStatus',
57
- 'EncryptionStatus',
58
- 'ObjectLockRetainUntilDate',
59
- 'ObjectLockMode',
60
- 'ObjectLockLegalHoldStatus',
61
- 'IntelligentTieringAccessTier',
62
- ],
63
- };
64
- }
65
-
66
- /**
67
- * Helper method for creating a Glue table which can be used to query inventory data generated by an S3 Inventory config
68
- * @param scope Stack in which the table will be created
69
- * @param tableNamePrefix Will be used to prefix the table name.
70
- * @param locationBucketName The bucket where the inventory data is stored.
71
- * @param inventoryBucket The bucket for which inventory data has been generated
72
- * @param database The Glue database in which the table will be created
73
- * @param inventoryName The name of the S3 Inventory config
74
- * @param locationPrefix The S3 Prefix on the location bucket where the inventory is stored.
75
- * @returns A Glue table which can be used to query S3 inventory data in an S3 bucket.
76
- */
77
- public static createGlueInvTable(
78
- scope: Construct,
79
- catalogId: string,
80
- tableNamePrefix: string,
81
- database: Database,
82
- locationBucketName: string,
83
- bucketInventories: BucketInventory[],
84
- locationPrefix?: string,
85
- ): CfnTable {
86
- const location = locationPrefix
87
- ? `s3://${locationBucketName}/${MdaaBucket.formatS3Prefix(locationPrefix)}/`
88
- : `s3://${locationBucketName}/`;
89
- const tableInput = {
90
- name: `${tableNamePrefix}_inv`.replace(/-/gi, '_'),
91
- parameters: {
92
- EXTERNAL: 'TRUE',
93
- 'projection.enabled': 'true',
94
- 'projection.dt.type': 'date',
95
- 'projection.dt.range': '2022-01-01-00-00,NOW',
96
- 'projection.dt.format': 'yyyy-MM-dd-HH-mm',
97
- 'projection.dt.interval': '1',
98
- 'projection.dt.interval.unit': 'HOURS',
99
- 'projection.bucket_inventory.type': 'enum',
100
- 'projection.bucket_inventory.values': bucketInventories
101
- .map(x => `${x.bucketName}/${x.inventoryName}`)
102
- .join(','),
103
- 'storage.location.template': location + '${bucket_inventory}/hive/dt=${dt}',
104
- },
105
- tableType: 'EXTERNAL_TABLE',
106
- partitionKeys: [
107
- {
108
- name: 'bucket_inventory',
109
- type: 'string',
110
- },
111
- {
112
- name: 'dt',
113
- type: 'string',
114
- },
115
- ],
116
- storageDescriptor: {
117
- columns: [
118
- {
119
- name: 'bucket',
120
- type: 'string',
121
- },
122
- {
123
- name: 'key',
124
- type: 'string',
125
- },
126
- {
127
- name: 'version_id',
128
- type: 'string',
129
- },
130
- {
131
- name: 'is_latest',
132
- type: 'boolean',
133
- },
134
- {
135
- name: 'is_delete_marker',
136
- type: 'boolean',
137
- },
138
- {
139
- name: 'size',
140
- type: 'bigint',
141
- },
142
- {
143
- name: 'last_modified_date',
144
- type: 'timestamp',
145
- },
146
- {
147
- name: 'e_tag',
148
- type: 'string',
149
- },
150
- {
151
- name: 'storage_class',
152
- type: 'string',
153
- },
154
- {
155
- name: 'is_multipart_uploaded',
156
- type: 'boolean',
157
- },
158
- {
159
- name: 'replication_status',
160
- type: 'string',
161
- },
162
- {
163
- name: 'encryption_status',
164
- type: 'string',
165
- },
166
- {
167
- name: 'intelligent_tiering_tier',
168
- type: 'string',
169
- },
170
- // ,
171
- // {
172
- // name: "object_lock_retain_until_date",
173
- // type: "timestamp"
174
- // },
175
- // {
176
- // name: "object_lock_mode",
177
- // type: "string"
178
- // },
179
- // {
180
- // name: "object_lock_legal_hold_status",
181
- // type: "string"
182
- // }
183
- ],
184
- location: location,
185
- inputFormat: 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat',
186
- outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
187
- compressed: false,
188
- numberOfBuckets: -1,
189
- serdeInfo: {
190
- serializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
191
- parameters: {
192
- 'serialization.format': '1',
193
- },
194
- },
195
- },
196
- };
197
-
198
- const tableProps: CfnTableProps = {
199
- catalogId: catalogId,
200
- databaseName: database.databaseName,
201
- tableInput: tableInput,
202
- };
203
- return new CfnTable(scope, `inv-table-${tableNamePrefix}`, tableProps);
204
- }
205
-
206
- /**
207
- * Helper method to generate an S3 Bucket Policy Statement allowing S3 service to write inventories to a bucket.
208
- * @param sourceBucketArn The bucket arn for which inventory data is being produced
209
- * @param bucketArn The bucket arn where inventory will be stored
210
- * @param srcAccount The account which owns the source bucket from which inventory is being produced.
211
- * @param inventoryPrefix The prefix on the destination bucket where inventory data will be stored.
212
- * @returns A policy statement which permits the S3 service principal to write inventory for the source bucket/account to the destination bucket.
213
- */
214
- public static createInventoryBucketPolicyStatement(
215
- bucketArn: string,
216
- srcAccount: string,
217
- sourceBucketArn?: string,
218
- inventoryPrefix?: string,
219
- ): PolicyStatement {
220
- const resource = inventoryPrefix
221
- ? `${bucketArn}/${MdaaBucket.formatS3Prefix(inventoryPrefix)}/*`
222
- : `${bucketArn}/*`;
223
- const allowS3Inventory = new PolicyStatement({
224
- sid: 'AllowS3Inventory',
225
- effect: Effect.ALLOW,
226
- resources: [resource],
227
- actions: ['s3:PutObject'],
228
- });
229
- allowS3Inventory.addCondition('StringEquals', {
230
- 'aws:SourceAccount': srcAccount,
231
- 's3:x-amz-acl': 'bucket-owner-full-control',
232
- });
233
- if (sourceBucketArn) {
234
- allowS3Inventory.addCondition('ArnLike', {
235
- 'aws:SourceArn': sourceBucketArn,
236
- });
237
- }
238
- allowS3Inventory.addServicePrincipal('s3.amazonaws.com');
239
- return allowS3Inventory;
240
- }
241
- }
@@ -1,44 +0,0 @@
1
- {
2
- "name": "@aws-mdaa/s3-inventory-helper",
3
- "description": "MDAA s3-inventory-helper utility",
4
- "author": {
5
- "name": "Amazon Web Services",
6
- "url": "https://aws.amazon.com/solutions"
7
- },
8
- "version": "1.5.0",
9
- "main": "lib/index.js",
10
- "types": "lib/index.d.ts",
11
- "license": "Apache-2.0",
12
- "scripts": {
13
- "build": "tsc",
14
- "watch": "tsc -w",
15
- "test": "jest --passWithNoTests --testPathIgnorePatterns='.*\\.snapshot\\.test\\.ts'",
16
- "lint": "eslint --max-warnings 0 -c ../../../eslint.config.mjs",
17
- "test:coverage": "jest --passWithNoTests --coverage --testPathIgnorePatterns='.*\\.snapshot\\.test\\.ts'",
18
- "test:snapshots": "jest --passWithNoTests --testPathPattern='.*\\.snapshot\\.test\\.ts'",
19
- "test:snapshots:update": "jest --passWithNoTests --testPathPattern='.*\\.snapshot\\.test\\.ts' --updateSnapshot"
20
- },
21
- "devDependencies": {
22
- "@types/jest": "29.5.14",
23
- "@types/node": "22.9.0",
24
- "@types/prettier": "2.6.0",
25
- "jest": "29.7.0",
26
- "ts-jest": "29.4.6",
27
- "ts-node": "10.9.2",
28
- "typescript": "5.9.3",
29
- "typescript-json-schema": "0.67.1"
30
- },
31
- "dependencies": {
32
- "@aws-cdk/aws-glue-alpha": "2.220.0-alpha.0",
33
- "@aws-mdaa/naming": "1.5.0",
34
- "@aws-mdaa/s3-constructs": "1.5.0",
35
- "aws-cdk-lib": "2.220.0",
36
- "cdk-nag": "2.37.55",
37
- "constructs": "10.0.96"
38
- },
39
- "gitHead": "8b49a2b371014baec046605ffdbfe38951099c31",
40
- "repository": {
41
- "type": "git",
42
- "url": "https://github.com/aws/modern-data-architecture-accelerator"
43
- }
44
- }
@@ -1,40 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "declarationMap": false,
4
- "inlineSourceMap": true,
5
- "inlineSources": true,
6
- "alwaysStrict": true,
7
- "removeComments": false,
8
- "declaration": true,
9
- "experimentalDecorators": true,
10
- "incremental": true,
11
- "lib": [
12
- "es2020"
13
- ],
14
- "module": "CommonJS",
15
- "newLine": "lf",
16
- "noEmitOnError": true,
17
- "noFallthroughCasesInSwitch": true,
18
- "noImplicitAny": true,
19
- "noImplicitReturns": true,
20
- "noImplicitThis": true,
21
- "noUnusedLocals": true,
22
- "noUnusedParameters": true,
23
- "resolveJsonModule": true,
24
- "skipLibCheck": true,
25
- "strict": true,
26
- "strictNullChecks": true,
27
- "strictPropertyInitialization": true,
28
- "stripInternal": false,
29
- "target": "ES2020",
30
- "composite": true,
31
- "tsBuildInfoFile": "tsconfig.tsbuildinfo"
32
- },
33
- "include": [
34
- "**/*.ts"
35
- ],
36
- "exclude": [
37
- "node_modules"
38
- ],
39
- "_generated_by_jsii_": "Generated by jsii - safe to delete, and ideally should be in .gitignore"
40
- }