@aws-mdaa/dataops-job-l3-construct 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +38 -28
- package/README.md +5 -0
- package/lib/dataops-job-l3-construct.d.ts +8 -1
- package/lib/dataops-job-l3-construct.js +40 -13
- package/node_modules/@aws-mdaa/config/README.md +3 -0
- package/node_modules/@aws-mdaa/config/lib/blueprint-value-transformer.d.ts +20 -0
- package/node_modules/@aws-mdaa/config/lib/blueprint-value-transformer.js +70 -0
- package/node_modules/@aws-mdaa/config/lib/blueprint-value-transformer.ts +88 -0
- package/node_modules/@aws-mdaa/config/lib/config.d.ts +87 -0
- package/node_modules/@aws-mdaa/config/lib/config.js +7 -0
- package/node_modules/@aws-mdaa/config/lib/config.ts +92 -0
- package/node_modules/@aws-mdaa/config/lib/index.d.ts +11 -0
- package/node_modules/@aws-mdaa/config/lib/index.js +28 -0
- package/node_modules/@aws-mdaa/config/lib/index.ts +12 -0
- package/node_modules/@aws-mdaa/config/lib/param-transformer.d.ts +49 -0
- package/node_modules/@aws-mdaa/config/lib/param-transformer.js +160 -0
- package/node_modules/@aws-mdaa/config/lib/param-transformer.ts +159 -0
- package/node_modules/@aws-mdaa/config/lib/path-value-transformer.d.ts +10 -0
- package/node_modules/@aws-mdaa/config/lib/path-value-transformer.js +30 -0
- package/node_modules/@aws-mdaa/config/lib/path-value-transformer.ts +27 -0
- package/node_modules/@aws-mdaa/config/lib/ref-value-transformer.d.ts +44 -0
- package/node_modules/@aws-mdaa/config/lib/ref-value-transformer.js +243 -0
- package/node_modules/@aws-mdaa/config/lib/ref-value-transformer.ts +302 -0
- package/node_modules/@aws-mdaa/config/lib/ssm-ref-transformer.d.ts +8 -0
- package/node_modules/@aws-mdaa/config/lib/ssm-ref-transformer.js +22 -0
- package/node_modules/@aws-mdaa/config/lib/ssm-ref-transformer.ts +21 -0
- package/node_modules/@aws-mdaa/config/lib/transformer.d.ts +35 -0
- package/node_modules/@aws-mdaa/config/lib/transformer.js +66 -0
- package/node_modules/@aws-mdaa/config/lib/transformer.ts +74 -0
- package/node_modules/@aws-mdaa/config/package.json +42 -0
- package/node_modules/@aws-mdaa/config/test/blueprint-value-transformer.test.js +224 -0
- package/node_modules/@aws-mdaa/config/test/blueprint-value-transformer.test.ts +259 -0
- package/node_modules/@aws-mdaa/config/test/config-nt.test.d.ts +5 -0
- package/node_modules/@aws-mdaa/config/test/config-nt.test.js +129 -0
- package/node_modules/@aws-mdaa/config/test/config-nt.test.ts +163 -0
- package/node_modules/@aws-mdaa/config/test/config.test.d.ts +5 -0
- package/node_modules/@aws-mdaa/config/test/config.test.js +409 -0
- package/node_modules/@aws-mdaa/config/test/config.test.ts +517 -0
- package/node_modules/@aws-mdaa/config/test/param-transformer.test.d.ts +5 -0
- package/node_modules/@aws-mdaa/config/test/param-transformer.test.js +216 -0
- package/node_modules/@aws-mdaa/config/test/param-transformer.test.ts +234 -0
- package/node_modules/@aws-mdaa/config/test/path-value-transformer.test.d.ts +5 -0
- package/node_modules/@aws-mdaa/config/test/path-value-transformer.test.js +59 -0
- package/node_modules/@aws-mdaa/config/test/path-value-transformer.test.ts +68 -0
- package/node_modules/@aws-mdaa/config/test/ref-value-transformer.test.d.ts +5 -0
- package/node_modules/@aws-mdaa/config/test/ref-value-transformer.test.js +254 -0
- package/node_modules/@aws-mdaa/config/test/ref-value-transformer.test.ts +304 -0
- package/node_modules/@aws-mdaa/config/test/ssm-ref-transformer.test.d.ts +5 -0
- package/node_modules/@aws-mdaa/config/test/ssm-ref-transformer.test.js +66 -0
- package/node_modules/@aws-mdaa/config/test/ssm-ref-transformer.test.ts +79 -0
- package/node_modules/@aws-mdaa/config/tsconfig.tsbuildinfo +1 -0
- package/node_modules/lodash/README.md +2 -2
- package/node_modules/lodash/_baseOrderBy.js +1 -1
- package/node_modules/lodash/_baseUnset.js +7 -20
- package/node_modules/lodash/_setCacheHas.js +1 -1
- package/node_modules/lodash/compact.js +1 -1
- package/node_modules/lodash/core.js +3 -3
- package/node_modules/lodash/core.min.js +26 -25
- package/node_modules/lodash/fromPairs.js +3 -1
- package/node_modules/lodash/lodash.js +38 -27
- package/node_modules/lodash/lodash.min.js +125 -129
- package/node_modules/lodash/package.json +4 -2
- package/node_modules/lodash/random.js +9 -0
- package/node_modules/lodash/template.js +16 -4
- package/node_modules/lodash/templateSettings.js +4 -0
- package/package.json +27 -32
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/README.md +0 -185
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/lib/index.d.ts +0 -57
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/lib/index.js +0 -198
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/lib/index.ts +0 -241
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/package.json +0 -44
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/test/bucketpolicy-helper.test.js +0 -200
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/test/bucketpolicy-helper.test.ts +0 -215
- package/node_modules/@aws-mdaa/s3-bucketpolicy-helper/tsconfig.tsbuildinfo +0 -1
- package/node_modules/@aws-mdaa/s3-inventory-helper/.npmignore +0 -34
- package/node_modules/@aws-mdaa/s3-inventory-helper/README.md +0 -3
- package/node_modules/@aws-mdaa/s3-inventory-helper/jest.config.js +0 -5
- package/node_modules/@aws-mdaa/s3-inventory-helper/lib/index.d.ts +0 -48
- package/node_modules/@aws-mdaa/s3-inventory-helper/lib/index.js +0 -213
- package/node_modules/@aws-mdaa/s3-inventory-helper/lib/index.ts +0 -241
- package/node_modules/@aws-mdaa/s3-inventory-helper/package.json +0 -44
- package/node_modules/@aws-mdaa/s3-inventory-helper/test/TODO +0 -0
- package/node_modules/@aws-mdaa/s3-inventory-helper/tsconfig.json +0 -40
- package/node_modules/@aws-mdaa/s3-inventory-helper/tsconfig.tsbuildinfo +0 -1
- package/node_modules/@aws-mdaa/s3-inventory-helper/typedoc.json +0 -7
- /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/.npmignore +0 -0
- /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/jest.config.js +0 -0
- /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper/test/bucketpolicy-helper.test.d.ts → config/test/blueprint-value-transformer.test.d.ts} +0 -0
- /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/tsconfig.json +0 -0
- /package/node_modules/@aws-mdaa/{s3-bucketpolicy-helper → config}/typedoc.json +0 -0
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
# Don't include original .ts files when doing `npm pack`
|
|
2
|
-
*.ts
|
|
3
|
-
*.tsbuildinfo
|
|
4
|
-
.eslintrc.js
|
|
5
|
-
tsconfig.json
|
|
6
|
-
typedoc.json
|
|
7
|
-
|
|
8
|
-
# dev-related files
|
|
9
|
-
coverage
|
|
10
|
-
test/
|
|
11
|
-
|
|
12
|
-
# pack output
|
|
13
|
-
*.tgz
|
|
14
|
-
|
|
15
|
-
dist
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
#Exclude coverage reports for bundled deps
|
|
19
|
-
node_modules/**/coverage/*
|
|
20
|
-
# Exclude typescript source and config
|
|
21
|
-
#*.ts
|
|
22
|
-
#tsconfig.json
|
|
23
|
-
|
|
24
|
-
# Include javascript files and typescript declarations
|
|
25
|
-
!*.js
|
|
26
|
-
!*.d.ts
|
|
27
|
-
|
|
28
|
-
# Exclude jsii outdir
|
|
29
|
-
jsii-dist
|
|
30
|
-
|
|
31
|
-
# Include .jsii and .jsii.gz
|
|
32
|
-
!.jsii
|
|
33
|
-
!.jsii.gz
|
|
34
|
-
jest.config.js
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
/*!
|
|
2
|
-
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
3
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
*/
|
|
5
|
-
import { Database } from '@aws-cdk/aws-glue-alpha';
|
|
6
|
-
import { CfnTable } from 'aws-cdk-lib/aws-glue';
|
|
7
|
-
import { PolicyStatement } from 'aws-cdk-lib/aws-iam';
|
|
8
|
-
import { IBucket, Inventory } from 'aws-cdk-lib/aws-s3';
|
|
9
|
-
import { Construct } from 'constructs';
|
|
10
|
-
export interface BucketInventory {
|
|
11
|
-
/** Name of the S3 bucket for which inventory data has been generated */
|
|
12
|
-
readonly bucketName: string;
|
|
13
|
-
/** Name of the S3 inventory configuration that generated the inventory data for the specified bucket */
|
|
14
|
-
readonly inventoryName: string;
|
|
15
|
-
}
|
|
16
|
-
/** Helper class for creating S3 Inventories and associated Glue/Athena resources for querying these inventories. */
|
|
17
|
-
export declare class InventoryHelper {
|
|
18
|
-
/**
|
|
19
|
-
* Helper method for creating an S3 inventory.
|
|
20
|
-
* @param destinationBucket The bucket where S3 inventory data will be written
|
|
21
|
-
* @param inventoryId The S3 inventory ID
|
|
22
|
-
* @param objectsPrefix The object prefix from which inventory will be produced (on the bucket to which the inventory config is applied)
|
|
23
|
-
* @param destinationPrefix The object prefix under which inventory will be written on the destination bucket.
|
|
24
|
-
* @returns An S3 Inventory Config which can be applied to a bucket in order to produce periodic inventory data.
|
|
25
|
-
*/
|
|
26
|
-
static createInvConfig(destinationBucket: IBucket, inventoryId: string, objectsPrefix?: string, destinationPrefix?: string, destinationAccount?: string): Inventory;
|
|
27
|
-
/**
|
|
28
|
-
* Helper method for creating a Glue table which can be used to query inventory data generated by an S3 Inventory config
|
|
29
|
-
* @param scope Stack in which the table will be created
|
|
30
|
-
* @param tableNamePrefix Will be used to prefix the table name.
|
|
31
|
-
* @param locationBucketName The bucket where the inventory data is stored.
|
|
32
|
-
* @param inventoryBucket The bucket for which inventory data has been generated
|
|
33
|
-
* @param database The Glue database in which the table will be created
|
|
34
|
-
* @param inventoryName The name of the S3 Inventory config
|
|
35
|
-
* @param locationPrefix The S3 Prefix on the location bucket where the inventory is stored.
|
|
36
|
-
* @returns A Glue table which can be used to query S3 inventory data in an S3 bucket.
|
|
37
|
-
*/
|
|
38
|
-
static createGlueInvTable(scope: Construct, catalogId: string, tableNamePrefix: string, database: Database, locationBucketName: string, bucketInventories: BucketInventory[], locationPrefix?: string): CfnTable;
|
|
39
|
-
/**
|
|
40
|
-
* Helper method to generate an S3 Bucket Policy Statement allowing S3 service to write inventories to a bucket.
|
|
41
|
-
* @param sourceBucketArn The bucket arn for which inventory data is being produced
|
|
42
|
-
* @param bucketArn The bucket arn where inventory will be stored
|
|
43
|
-
* @param srcAccount The account which owns the source bucket from which inventory is being produced.
|
|
44
|
-
* @param inventoryPrefix The prefix on the destination bucket where inventory data will be stored.
|
|
45
|
-
* @returns A policy statement which permits the S3 service principal to write inventory for the source bucket/account to the destination bucket.
|
|
46
|
-
*/
|
|
47
|
-
static createInventoryBucketPolicyStatement(bucketArn: string, srcAccount: string, sourceBucketArn?: string, inventoryPrefix?: string): PolicyStatement;
|
|
48
|
-
}
|
|
@@ -1,213 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/*!
|
|
3
|
-
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
4
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
*/
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
exports.InventoryHelper = void 0;
|
|
8
|
-
const s3_constructs_1 = require("@aws-mdaa/s3-constructs");
|
|
9
|
-
const aws_glue_1 = require("aws-cdk-lib/aws-glue");
|
|
10
|
-
const aws_iam_1 = require("aws-cdk-lib/aws-iam");
|
|
11
|
-
const aws_s3_1 = require("aws-cdk-lib/aws-s3");
|
|
12
|
-
/** Helper class for creating S3 Inventories and associated Glue/Athena resources for querying these inventories. */
|
|
13
|
-
class InventoryHelper {
|
|
14
|
-
/**
|
|
15
|
-
* Helper method for creating an S3 inventory.
|
|
16
|
-
* @param destinationBucket The bucket where S3 inventory data will be written
|
|
17
|
-
* @param inventoryId The S3 inventory ID
|
|
18
|
-
* @param objectsPrefix The object prefix from which inventory will be produced (on the bucket to which the inventory config is applied)
|
|
19
|
-
* @param destinationPrefix The object prefix under which inventory will be written on the destination bucket.
|
|
20
|
-
* @returns An S3 Inventory Config which can be applied to a bucket in order to produce periodic inventory data.
|
|
21
|
-
*/
|
|
22
|
-
static createInvConfig(destinationBucket, inventoryId, objectsPrefix, destinationPrefix, destinationAccount) {
|
|
23
|
-
const bucketOwner = destinationAccount ? `${destinationAccount}` : destinationAccount;
|
|
24
|
-
return {
|
|
25
|
-
destination: {
|
|
26
|
-
bucket: destinationBucket,
|
|
27
|
-
/** S3 prefix for inventory report destination organization enabling systematic inventory file */
|
|
28
|
-
prefix: s3_constructs_1.MdaaBucket.formatS3Prefix(destinationPrefix),
|
|
29
|
-
bucketOwner: bucketOwner,
|
|
30
|
-
},
|
|
31
|
-
format: aws_s3_1.InventoryFormat.PARQUET,
|
|
32
|
-
frequency: aws_s3_1.InventoryFrequency.DAILY,
|
|
33
|
-
includeObjectVersions: aws_s3_1.InventoryObjectVersion.ALL,
|
|
34
|
-
inventoryId: inventoryId,
|
|
35
|
-
objectsPrefix: s3_constructs_1.MdaaBucket.formatS3Prefix(objectsPrefix, false, true),
|
|
36
|
-
optionalFields: [
|
|
37
|
-
'Size',
|
|
38
|
-
'LastModifiedDate',
|
|
39
|
-
'StorageClass',
|
|
40
|
-
'ETag',
|
|
41
|
-
'IsMultipartUploaded',
|
|
42
|
-
'ReplicationStatus',
|
|
43
|
-
'EncryptionStatus',
|
|
44
|
-
'ObjectLockRetainUntilDate',
|
|
45
|
-
'ObjectLockMode',
|
|
46
|
-
'ObjectLockLegalHoldStatus',
|
|
47
|
-
'IntelligentTieringAccessTier',
|
|
48
|
-
],
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Helper method for creating a Glue table which can be used to query inventory data generated by an S3 Inventory config
|
|
53
|
-
* @param scope Stack in which the table will be created
|
|
54
|
-
* @param tableNamePrefix Will be used to prefix the table name.
|
|
55
|
-
* @param locationBucketName The bucket where the inventory data is stored.
|
|
56
|
-
* @param inventoryBucket The bucket for which inventory data has been generated
|
|
57
|
-
* @param database The Glue database in which the table will be created
|
|
58
|
-
* @param inventoryName The name of the S3 Inventory config
|
|
59
|
-
* @param locationPrefix The S3 Prefix on the location bucket where the inventory is stored.
|
|
60
|
-
* @returns A Glue table which can be used to query S3 inventory data in an S3 bucket.
|
|
61
|
-
*/
|
|
62
|
-
static createGlueInvTable(scope, catalogId, tableNamePrefix, database, locationBucketName, bucketInventories, locationPrefix) {
|
|
63
|
-
const location = locationPrefix
|
|
64
|
-
? `s3://${locationBucketName}/${s3_constructs_1.MdaaBucket.formatS3Prefix(locationPrefix)}/`
|
|
65
|
-
: `s3://${locationBucketName}/`;
|
|
66
|
-
const tableInput = {
|
|
67
|
-
name: `${tableNamePrefix}_inv`.replace(/-/gi, '_'),
|
|
68
|
-
parameters: {
|
|
69
|
-
EXTERNAL: 'TRUE',
|
|
70
|
-
'projection.enabled': 'true',
|
|
71
|
-
'projection.dt.type': 'date',
|
|
72
|
-
'projection.dt.range': '2022-01-01-00-00,NOW',
|
|
73
|
-
'projection.dt.format': 'yyyy-MM-dd-HH-mm',
|
|
74
|
-
'projection.dt.interval': '1',
|
|
75
|
-
'projection.dt.interval.unit': 'HOURS',
|
|
76
|
-
'projection.bucket_inventory.type': 'enum',
|
|
77
|
-
'projection.bucket_inventory.values': bucketInventories
|
|
78
|
-
.map(x => `${x.bucketName}/${x.inventoryName}`)
|
|
79
|
-
.join(','),
|
|
80
|
-
'storage.location.template': location + '${bucket_inventory}/hive/dt=${dt}',
|
|
81
|
-
},
|
|
82
|
-
tableType: 'EXTERNAL_TABLE',
|
|
83
|
-
partitionKeys: [
|
|
84
|
-
{
|
|
85
|
-
name: 'bucket_inventory',
|
|
86
|
-
type: 'string',
|
|
87
|
-
},
|
|
88
|
-
{
|
|
89
|
-
name: 'dt',
|
|
90
|
-
type: 'string',
|
|
91
|
-
},
|
|
92
|
-
],
|
|
93
|
-
storageDescriptor: {
|
|
94
|
-
columns: [
|
|
95
|
-
{
|
|
96
|
-
name: 'bucket',
|
|
97
|
-
type: 'string',
|
|
98
|
-
},
|
|
99
|
-
{
|
|
100
|
-
name: 'key',
|
|
101
|
-
type: 'string',
|
|
102
|
-
},
|
|
103
|
-
{
|
|
104
|
-
name: 'version_id',
|
|
105
|
-
type: 'string',
|
|
106
|
-
},
|
|
107
|
-
{
|
|
108
|
-
name: 'is_latest',
|
|
109
|
-
type: 'boolean',
|
|
110
|
-
},
|
|
111
|
-
{
|
|
112
|
-
name: 'is_delete_marker',
|
|
113
|
-
type: 'boolean',
|
|
114
|
-
},
|
|
115
|
-
{
|
|
116
|
-
name: 'size',
|
|
117
|
-
type: 'bigint',
|
|
118
|
-
},
|
|
119
|
-
{
|
|
120
|
-
name: 'last_modified_date',
|
|
121
|
-
type: 'timestamp',
|
|
122
|
-
},
|
|
123
|
-
{
|
|
124
|
-
name: 'e_tag',
|
|
125
|
-
type: 'string',
|
|
126
|
-
},
|
|
127
|
-
{
|
|
128
|
-
name: 'storage_class',
|
|
129
|
-
type: 'string',
|
|
130
|
-
},
|
|
131
|
-
{
|
|
132
|
-
name: 'is_multipart_uploaded',
|
|
133
|
-
type: 'boolean',
|
|
134
|
-
},
|
|
135
|
-
{
|
|
136
|
-
name: 'replication_status',
|
|
137
|
-
type: 'string',
|
|
138
|
-
},
|
|
139
|
-
{
|
|
140
|
-
name: 'encryption_status',
|
|
141
|
-
type: 'string',
|
|
142
|
-
},
|
|
143
|
-
{
|
|
144
|
-
name: 'intelligent_tiering_tier',
|
|
145
|
-
type: 'string',
|
|
146
|
-
},
|
|
147
|
-
// ,
|
|
148
|
-
// {
|
|
149
|
-
// name: "object_lock_retain_until_date",
|
|
150
|
-
// type: "timestamp"
|
|
151
|
-
// },
|
|
152
|
-
// {
|
|
153
|
-
// name: "object_lock_mode",
|
|
154
|
-
// type: "string"
|
|
155
|
-
// },
|
|
156
|
-
// {
|
|
157
|
-
// name: "object_lock_legal_hold_status",
|
|
158
|
-
// type: "string"
|
|
159
|
-
// }
|
|
160
|
-
],
|
|
161
|
-
location: location,
|
|
162
|
-
inputFormat: 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat',
|
|
163
|
-
outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
|
|
164
|
-
compressed: false,
|
|
165
|
-
numberOfBuckets: -1,
|
|
166
|
-
serdeInfo: {
|
|
167
|
-
serializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
168
|
-
parameters: {
|
|
169
|
-
'serialization.format': '1',
|
|
170
|
-
},
|
|
171
|
-
},
|
|
172
|
-
},
|
|
173
|
-
};
|
|
174
|
-
const tableProps = {
|
|
175
|
-
catalogId: catalogId,
|
|
176
|
-
databaseName: database.databaseName,
|
|
177
|
-
tableInput: tableInput,
|
|
178
|
-
};
|
|
179
|
-
return new aws_glue_1.CfnTable(scope, `inv-table-${tableNamePrefix}`, tableProps);
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Helper method to generate an S3 Bucket Policy Statement allowing S3 service to write inventories to a bucket.
|
|
183
|
-
* @param sourceBucketArn The bucket arn for which inventory data is being produced
|
|
184
|
-
* @param bucketArn The bucket arn where inventory will be stored
|
|
185
|
-
* @param srcAccount The account which owns the source bucket from which inventory is being produced.
|
|
186
|
-
* @param inventoryPrefix The prefix on the destination bucket where inventory data will be stored.
|
|
187
|
-
* @returns A policy statement which permits the S3 service principal to write inventory for the source bucket/account to the destination bucket.
|
|
188
|
-
*/
|
|
189
|
-
static createInventoryBucketPolicyStatement(bucketArn, srcAccount, sourceBucketArn, inventoryPrefix) {
|
|
190
|
-
const resource = inventoryPrefix
|
|
191
|
-
? `${bucketArn}/${s3_constructs_1.MdaaBucket.formatS3Prefix(inventoryPrefix)}/*`
|
|
192
|
-
: `${bucketArn}/*`;
|
|
193
|
-
const allowS3Inventory = new aws_iam_1.PolicyStatement({
|
|
194
|
-
sid: 'AllowS3Inventory',
|
|
195
|
-
effect: aws_iam_1.Effect.ALLOW,
|
|
196
|
-
resources: [resource],
|
|
197
|
-
actions: ['s3:PutObject'],
|
|
198
|
-
});
|
|
199
|
-
allowS3Inventory.addCondition('StringEquals', {
|
|
200
|
-
'aws:SourceAccount': srcAccount,
|
|
201
|
-
's3:x-amz-acl': 'bucket-owner-full-control',
|
|
202
|
-
});
|
|
203
|
-
if (sourceBucketArn) {
|
|
204
|
-
allowS3Inventory.addCondition('ArnLike', {
|
|
205
|
-
'aws:SourceArn': sourceBucketArn,
|
|
206
|
-
});
|
|
207
|
-
}
|
|
208
|
-
allowS3Inventory.addServicePrincipal('s3.amazonaws.com');
|
|
209
|
-
return allowS3Inventory;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
exports.InventoryHelper = InventoryHelper;
|
|
213
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJpbmRleC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUE7OztHQUdHOzs7QUFFSCwyREFBcUQ7QUFFckQsbURBQStEO0FBQy9ELGlEQUE4RDtBQUM5RCwrQ0FBcUg7QUFVckgsb0hBQW9IO0FBQ3BILE1BQWEsZUFBZTtJQUMxQjs7Ozs7OztPQU9HO0lBQ0ksTUFBTSxDQUFDLGVBQWUsQ0FDM0IsaUJBQTBCLEVBQzFCLFdBQW1CLEVBQ25CLGFBQXNCLEVBQ3RCLGlCQUEwQixFQUMxQixrQkFBMkI7UUFFM0IsTUFBTSxXQUFXLEdBQUcsa0JBQWtCLENBQUMsQ0FBQyxDQUFDLEdBQUcsa0JBQWtCLEVBQUUsQ0FBQyxDQUFDLENBQUMsa0JBQWtCLENBQUM7UUFDdEYsT0FBTztZQUNMLFdBQVcsRUFBRTtnQkFDWCxNQUFNLEVBQUUsaUJBQWlCO2dCQUN6QixpR0FBaUc7Z0JBQ2pHLE1BQU0sRUFBRSwwQkFBVSxDQUFDLGNBQWMsQ0FBQyxpQkFBaUIsQ0FBQztnQkFDcEQsV0FBVyxFQUFFLFdBQVc7YUFDekI7WUFDRCxNQUFNLEVBQUUsd0JBQWUsQ0FBQyxPQUFPO1lBQy9CLFNBQVMsRUFBRSwyQkFBa0IsQ0FBQyxLQUFLO1lBQ25DLHFCQUFxQixFQUFFLCtCQUFzQixDQUFDLEdBQUc7WUFDakQsV0FBVyxFQUFFLFdBQVc7WUFDeEIsYUFBYSxFQUFFLDBCQUFVLENBQUMsY0FBYyxDQUFDLGFBQWEsRUFBRSxLQUFLLEVBQUUsSUFBSSxDQUFDO1lBQ3BFLGNBQWMsRUFBRTtnQkFDZCxNQUFNO2dCQUNOLGtCQUFrQjtnQkFDbEIsY0FBYztnQkFDZCxNQUFNO2dCQUNOLHFCQUFxQjtnQkFDckIsbUJBQW1CO2dCQUNuQixrQkFBa0I7Z0JBQ2xCLDJCQUEyQjtnQkFDM0IsZ0JBQWdCO2dCQUNoQiwyQkFBMkI7Z0JBQzNCLDhCQUE4QjthQUMvQjtTQUNGLENBQUM7SUFDSixDQUFDO0lBRUQ7Ozs7Ozs7Ozs7T0FVRztJQUNJLE1BQU0sQ0FBQyxrQkFBa0IsQ0FDOUIsS0FBZ0IsRUFDaEIsU0FBaUIsRUFDakIsZUFBdUIsRUFDdkIsUUFBa0IsRUFDbEIsa0JBQTBCLEVBQzFCLGlCQUFvQyxFQUNwQyxjQUF1QjtRQUV2QixNQUFNLFFBQVEsR0FBRyxjQUFjO1lBQzdCLENBQUMsQ0FBQyxRQUFRLGtCQUFrQixJQUFJLDBCQUFVLENBQUMsY0FBYyxDQUFDLGNBQWMsQ0FBQyxHQUFHO1lBQzVFLENBQUMsQ0FBQyxRQUFRLGtCQUFrQixHQUFHLENBQUM7UUFDbEMsTUFBTSxVQUFVLEdBQUc7WUFDakIsSUFBSSxFQUFFLEdBQUcsZUFBZSxNQUFNLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxHQUFHLENBQUM7WUFDbEQsVUFBVSxFQUFFO2dCQUNWLFFBQVEsRUFBRSxNQUFNO2dCQUNoQixvQkFBb0IsRUFBRSxNQUFNO2dCQUM1QixvQkFBb0IsRUFBRSxNQUFNO2dCQUM1QixxQkFBcUIsRUFBRSxzQkFBc0I7Z0JBQzdDLHNCQUFzQixFQUFFLGtCQUFrQjtnQkFDMUMsd0JBQXdCLEVBQUUsR0FBRztnQkFDN0IsNkJBQTZCLEVBQUUsT0FBTztnQkFDdEMsa0NBQWtDLEVBQUUsTUFBTTtnQkFDMUMsb0NBQW9DLEVBQUUsaUJBQWlCO3FCQUNwRCxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxHQUFHLENBQUMsQ0FBQyxVQUFVLElBQUksQ0FBQyxDQUFDLGFBQWEsRUFBRSxDQUFDO3FCQUM5QyxJQUFJLENBQUMsR0FBRyxDQUFDO2dCQUNaLDJCQUEyQixFQUFFLFFBQVEsR0FBRyxtQ0FBbUM7YUFDNUU7WUFDRCxTQUFTLEVBQUUsZ0JBQWdCO1lBQzNCLGFBQWEsRUFBRTtnQkFDYjtvQkFDRSxJQUFJLEVBQUUsa0JBQWtCO29CQUN4QixJQUFJLEVBQUUsUUFBUTtpQkFDZjtnQkFDRDtvQkFDRSxJQUFJLEVBQUUsSUFBSTtvQkFDVixJQUFJLEVBQUUsUUFBUTtpQkFDZjthQUNGO1lBQ0QsaUJBQWlCLEVBQUU7Z0JBQ2pCLE9BQU8sRUFBRTtvQkFDUDt3QkFDRSxJQUFJLEVBQUUsUUFBUTt3QkFDZCxJQUFJLEVBQUUsUUFBUTtxQkFDZjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsS0FBSzt3QkFDWCxJQUFJLEVBQUUsUUFBUTtxQkFDZjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsWUFBWTt3QkFDbEIsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLFdBQVc7d0JBQ2pCLElBQUksRUFBRSxTQUFTO3FCQUNoQjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsa0JBQWtCO3dCQUN4QixJQUFJLEVBQUUsU0FBUztxQkFDaEI7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLE1BQU07d0JBQ1osSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLG9CQUFvQjt3QkFDMUIsSUFBSSxFQUFFLFdBQVc7cUJBQ2xCO29CQUNEO3dCQUNFLElBQUksRUFBRSxPQUFPO3dCQUNiLElBQUksRUFBRSxRQUFRO3FCQUNmO29CQUNEO3dCQUNFLElBQUksRUFBRSxlQUFlO3dCQUNyQixJQUFJLEVBQUUsUUFBUTtxQkFDZjtvQkFDRDt3QkFDRSxJQUFJLEVBQUUsdUJBQXVCO3dCQUM3QixJQUFJLEVBQUUsU0FBUztxQkFDaEI7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLG9CQUFvQjt3QkFDMUIsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLG1CQUFtQjt3QkFDekIsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0Q7d0JBQ0UsSUFBSSxFQUFFLDBCQUEwQjt3QkFDaEMsSUFBSSxFQUFFLFFBQVE7cUJBQ2Y7b0JBQ0QsSUFBSTtvQkFDSixJQUFJO29CQUNKLDZDQUE2QztvQkFDN0Msd0JBQXdCO29CQUN4QixLQUFLO29CQUNMLElBQUk7b0JBQ0osZ0NBQWdDO29CQUNoQyxxQkFBcUI7b0JBQ3JCLEtBQUs7b0JBQ0wsSUFBSTtvQkFDSiw2Q0FBNkM7b0JBQzdDLHFCQUFxQjtvQkFDckIsSUFBSTtpQkFDTDtnQkFDRCxRQUFRLEVBQUUsUUFBUTtnQkFDbEIsV0FBVyxFQUFFLHFEQUFxRDtnQkFDbEUsWUFBWSxFQUFFLDREQUE0RDtnQkFDMUUsVUFBVSxFQUFFLEtBQUs7Z0JBQ2pCLGVBQWUsRUFBRSxDQUFDLENBQUM7Z0JBQ25CLFNBQVMsRUFBRTtvQkFDVCxvQkFBb0IsRUFBRSw2REFBNkQ7b0JBQ25GLFVBQVUsRUFBRTt3QkFDVixzQkFBc0IsRUFBRSxHQUFHO3FCQUM1QjtpQkFDRjthQUNGO1NBQ0YsQ0FBQztRQUVGLE1BQU0sVUFBVSxHQUFrQjtZQUNoQyxTQUFTLEVBQUUsU0FBUztZQUNwQixZQUFZLEVBQUUsUUFBUSxDQUFDLFlBQVk7WUFDbkMsVUFBVSxFQUFFLFVBQVU7U0FDdkIsQ0FBQztRQUNGLE9BQU8sSUFBSSxtQkFBUSxDQUFDLEtBQUssRUFBRSxhQUFhLGVBQWUsRUFBRSxFQUFFLFVBQVUsQ0FBQyxDQUFDO0lBQ3pFLENBQUM7SUFFRDs7Ozs7OztPQU9HO0lBQ0ksTUFBTSxDQUFDLG9DQUFvQyxDQUNoRCxTQUFpQixFQUNqQixVQUFrQixFQUNsQixlQUF3QixFQUN4QixlQUF3QjtRQUV4QixNQUFNLFFBQVEsR0FBRyxlQUFlO1lBQzlCLENBQUMsQ0FBQyxHQUFHLFNBQVMsSUFBSSwwQkFBVSxDQUFDLGNBQWMsQ0FBQyxlQUFlLENBQUMsSUFBSTtZQUNoRSxDQUFDLENBQUMsR0FBRyxTQUFTLElBQUksQ0FBQztRQUNyQixNQUFNLGdCQUFnQixHQUFHLElBQUkseUJBQWUsQ0FBQztZQUMzQyxHQUFHLEVBQUUsa0JBQWtCO1lBQ3ZCLE1BQU0sRUFBRSxnQkFBTSxDQUFDLEtBQUs7WUFDcEIsU0FBUyxFQUFFLENBQUMsUUFBUSxDQUFDO1lBQ3JCLE9BQU8sRUFBRSxDQUFDLGNBQWMsQ0FBQztTQUMxQixDQUFDLENBQUM7UUFDSCxnQkFBZ0IsQ0FBQyxZQUFZLENBQUMsY0FBYyxFQUFFO1lBQzVDLG1CQUFtQixFQUFFLFVBQVU7WUFDL0IsY0FBYyxFQUFFLDJCQUEyQjtTQUM1QyxDQUFDLENBQUM7UUFDSCxJQUFJLGVBQWUsRUFBRSxDQUFDO1lBQ3BCLGdCQUFnQixDQUFDLFlBQVksQ0FBQyxTQUFTLEVBQUU7Z0JBQ3ZDLGVBQWUsRUFBRSxlQUFlO2FBQ2pDLENBQUMsQ0FBQztRQUNMLENBQUM7UUFDRCxnQkFBZ0IsQ0FBQyxtQkFBbUIsQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDO1FBQ3pELE9BQU8sZ0JBQWdCLENBQUM7SUFDMUIsQ0FBQztDQUNGO0FBNU5ELDBDQTROQyIsInNvdXJjZXNDb250ZW50IjpbIi8qIVxuICogQ29weXJpZ2h0IEFtYXpvbi5jb20sIEluYy4gb3IgaXRzIGFmZmlsaWF0ZXMuIEFsbCBSaWdodHMgUmVzZXJ2ZWQuXG4gKiBTUERYLUxpY2Vuc2UtSWRlbnRpZmllcjogQXBhY2hlLTIuMFxuICovXG5cbmltcG9ydCB7IE1kYWFCdWNrZXQgfSBmcm9tICdAYXdzLW1kYWEvczMtY29uc3RydWN0cyc7XG5pbXBvcnQgeyBEYXRhYmFzZSB9IGZyb20gJ0Bhd3MtY2RrL2F3cy1nbHVlLWFscGhhJztcbmltcG9ydCB7IENmblRhYmxlLCBDZm5UYWJsZVByb3BzIH0gZnJvbSAnYXdzLWNkay1saWIvYXdzLWdsdWUnO1xuaW1wb3J0IHsgRWZmZWN0LCBQb2xpY3lTdGF0ZW1lbnQgfSBmcm9tICdhd3MtY2RrLWxpYi9hd3MtaWFtJztcbmltcG9ydCB7IElCdWNrZXQsIEludmVudG9yeSwgSW52ZW50b3J5Rm9ybWF0LCBJbnZlbnRvcnlGcmVxdWVuY3ksIEludmVudG9yeU9iamVjdFZlcnNpb24gfSBmcm9tICdhd3MtY2RrLWxpYi9hd3MtczMnO1xuaW1wb3J0IHsgQ29uc3RydWN0IH0gZnJvbSAnY29uc3RydWN0cyc7XG5cbmV4cG9ydCBpbnRlcmZhY2UgQnVja2V0SW52ZW50b3J5IHtcbiAgLyoqIE5hbWUgb2YgdGhlIFMzIGJ1Y2tldCBmb3Igd2hpY2ggaW52ZW50b3J5IGRhdGEgaGFzIGJlZW4gZ2VuZXJhdGVkICovXG4gIHJlYWRvbmx5IGJ1Y2tldE5hbWU6IHN0cmluZztcbiAgLyoqIE5hbWUgb2YgdGhlIFMzIGludmVudG9yeSBjb25maWd1cmF0aW9uIHRoYXQgZ2VuZXJhdGVkIHRoZSBpbnZlbnRvcnkgZGF0YSBmb3IgdGhlIHNwZWNpZmllZCBidWNrZXQgKi9cbiAgcmVhZG9ubHkgaW52ZW50b3J5TmFtZTogc3RyaW5nO1xufVxuXG4vKiogSGVscGVyIGNsYXNzIGZvciBjcmVhdGluZyBTMyBJbnZlbnRvcmllcyBhbmQgYXNzb2NpYXRlZCBHbHVlL0F0aGVuYSByZXNvdXJjZXMgZm9yIHF1ZXJ5aW5nIHRoZXNlIGludmVudG9yaWVzLiAqL1xuZXhwb3J0IGNsYXNzIEludmVudG9yeUhlbHBlciB7XG4gIC8qKlxuICAgKiBIZWxwZXIgbWV0aG9kIGZvciBjcmVhdGluZyBhbiBTMyBpbnZlbnRvcnkuXG4gICAqIEBwYXJhbSBkZXN0aW5hdGlvbkJ1Y2tldCBUaGUgYnVja2V0IHdoZXJlIFMzIGludmVudG9yeSBkYXRhIHdpbGwgYmUgd3JpdHRlblxuICAgKiBAcGFyYW0gaW52ZW50b3J5SWQgVGhlIFMzIGludmVudG9yeSBJRFxuICAgKiBAcGFyYW0gb2JqZWN0c1ByZWZpeCBUaGUgb2JqZWN0IHByZWZpeCBmcm9tIHdoaWNoIGludmVudG9yeSB3aWxsIGJlIHByb2R1Y2VkIChvbiB0aGUgYnVja2V0IHRvIHdoaWNoIHRoZSBpbnZlbnRvcnkgY29uZmlnIGlzIGFwcGxpZWQpXG4gICAqIEBwYXJhbSBkZXN0aW5hdGlvblByZWZpeCBUaGUgb2JqZWN0IHByZWZpeCB1bmRlciB3aGljaCBpbnZlbnRvcnkgd2lsbCBiZSB3cml0dGVuIG9uIHRoZSBkZXN0aW5hdGlvbiBidWNrZXQuXG4gICAqIEByZXR1cm5zIEFuIFMzIEludmVudG9yeSBDb25maWcgd2hpY2ggY2FuIGJlIGFwcGxpZWQgdG8gYSBidWNrZXQgaW4gb3JkZXIgdG8gcHJvZHVjZSBwZXJpb2RpYyBpbnZlbnRvcnkgZGF0YS5cbiAgICovXG4gIHB1YmxpYyBzdGF0aWMgY3JlYXRlSW52Q29uZmlnKFxuICAgIGRlc3RpbmF0aW9uQnVja2V0OiBJQnVja2V0LFxuICAgIGludmVudG9yeUlkOiBzdHJpbmcsXG4gICAgb2JqZWN0c1ByZWZpeD86IHN0cmluZyxcbiAgICBkZXN0aW5hdGlvblByZWZpeD86IHN0cmluZyxcbiAgICBkZXN0aW5hdGlvbkFjY291bnQ/OiBzdHJpbmcsXG4gICk6IEludmVudG9yeSB7XG4gICAgY29uc3QgYnVja2V0T3duZXIgPSBkZXN0aW5hdGlvbkFjY291bnQgPyBgJHtkZXN0aW5hdGlvbkFjY291bnR9YCA6IGRlc3RpbmF0aW9uQWNjb3VudDtcbiAgICByZXR1cm4ge1xuICAgICAgZGVzdGluYXRpb246IHtcbiAgICAgICAgYnVja2V0OiBkZXN0aW5hdGlvbkJ1Y2tldCxcbiAgICAgICAgLyoqIFMzIHByZWZpeCBmb3IgaW52ZW50b3J5IHJlcG9ydCBkZXN0aW5hdGlvbiBvcmdhbml6YXRpb24gZW5hYmxpbmcgc3lzdGVtYXRpYyBpbnZlbnRvcnkgZmlsZSAqL1xuICAgICAgICBwcmVmaXg6IE1kYWFCdWNrZXQuZm9ybWF0UzNQcmVmaXgoZGVzdGluYXRpb25QcmVmaXgpLFxuICAgICAgICBidWNrZXRPd25lcjogYnVja2V0T3duZXIsXG4gICAgICB9LFxuICAgICAgZm9ybWF0OiBJbnZlbnRvcnlGb3JtYXQuUEFSUVVFVCxcbiAgICAgIGZyZXF1ZW5jeTogSW52ZW50b3J5RnJlcXVlbmN5LkRBSUxZLFxuICAgICAgaW5jbHVkZU9iamVjdFZlcnNpb25zOiBJbnZlbnRvcnlPYmplY3RWZXJzaW9uLkFMTCxcbiAgICAgIGludmVudG9yeUlkOiBpbnZlbnRvcnlJZCxcbiAgICAgIG9iamVjdHNQcmVmaXg6IE1kYWFCdWNrZXQuZm9ybWF0UzNQcmVmaXgob2JqZWN0c1ByZWZpeCwgZmFsc2UsIHRydWUpLFxuICAgICAgb3B0aW9uYWxGaWVsZHM6IFtcbiAgICAgICAgJ1NpemUnLFxuICAgICAgICAnTGFzdE1vZGlmaWVkRGF0ZScsXG4gICAgICAgICdTdG9yYWdlQ2xhc3MnLFxuICAgICAgICAnRVRhZycsXG4gICAgICAgICdJc011bHRpcGFydFVwbG9hZGVkJyxcbiAgICAgICAgJ1JlcGxpY2F0aW9uU3RhdHVzJyxcbiAgICAgICAgJ0VuY3J5cHRpb25TdGF0dXMnLFxuICAgICAgICAnT2JqZWN0TG9ja1JldGFpblVudGlsRGF0ZScsXG4gICAgICAgICdPYmplY3RMb2NrTW9kZScsXG4gICAgICAgICdPYmplY3RMb2NrTGVnYWxIb2xkU3RhdHVzJyxcbiAgICAgICAgJ0ludGVsbGlnZW50VGllcmluZ0FjY2Vzc1RpZXInLFxuICAgICAgXSxcbiAgICB9O1xuICB9XG5cbiAgLyoqXG4gICAqIEhlbHBlciBtZXRob2QgZm9yIGNyZWF0aW5nIGEgR2x1ZSB0YWJsZSB3aGljaCBjYW4gYmUgdXNlZCB0byBxdWVyeSBpbnZlbnRvcnkgZGF0YSBnZW5lcmF0ZWQgYnkgYW4gUzMgSW52ZW50b3J5IGNvbmZpZ1xuICAgKiBAcGFyYW0gc2NvcGUgU3RhY2sgaW4gd2hpY2ggdGhlIHRhYmxlIHdpbGwgYmUgY3JlYXRlZFxuICAgKiBAcGFyYW0gdGFibGVOYW1lUHJlZml4IFdpbGwgYmUgdXNlZCB0byBwcmVmaXggdGhlIHRhYmxlIG5hbWUuXG4gICAqIEBwYXJhbSBsb2NhdGlvbkJ1Y2tldE5hbWUgVGhlIGJ1Y2tldCB3aGVyZSB0aGUgaW52ZW50b3J5IGRhdGEgaXMgc3RvcmVkLlxuICAgKiBAcGFyYW0gaW52ZW50b3J5QnVja2V0IFRoZSBidWNrZXQgZm9yIHdoaWNoIGludmVudG9yeSBkYXRhIGhhcyBiZWVuIGdlbmVyYXRlZFxuICAgKiBAcGFyYW0gZGF0YWJhc2UgVGhlIEdsdWUgZGF0YWJhc2UgaW4gd2hpY2ggdGhlIHRhYmxlIHdpbGwgYmUgY3JlYXRlZFxuICAgKiBAcGFyYW0gaW52ZW50b3J5TmFtZSBUaGUgbmFtZSBvZiB0aGUgUzMgSW52ZW50b3J5IGNvbmZpZ1xuICAgKiBAcGFyYW0gbG9jYXRpb25QcmVmaXggVGhlIFMzIFByZWZpeCBvbiB0aGUgbG9jYXRpb24gYnVja2V0IHdoZXJlIHRoZSBpbnZlbnRvcnkgaXMgc3RvcmVkLlxuICAgKiBAcmV0dXJucyBBIEdsdWUgdGFibGUgd2hpY2ggY2FuIGJlIHVzZWQgdG8gcXVlcnkgUzMgaW52ZW50b3J5IGRhdGEgaW4gYW4gUzMgYnVja2V0LlxuICAgKi9cbiAgcHVibGljIHN0YXRpYyBjcmVhdGVHbHVlSW52VGFibGUoXG4gICAgc2NvcGU6IENvbnN0cnVjdCxcbiAgICBjYXRhbG9nSWQ6IHN0cmluZyxcbiAgICB0YWJsZU5hbWVQcmVmaXg6IHN0cmluZyxcbiAgICBkYXRhYmFzZTogRGF0YWJhc2UsXG4gICAgbG9jYXRpb25CdWNrZXROYW1lOiBzdHJpbmcsXG4gICAgYnVja2V0SW52ZW50b3JpZXM6IEJ1Y2tldEludmVudG9yeVtdLFxuICAgIGxvY2F0aW9uUHJlZml4Pzogc3RyaW5nLFxuICApOiBDZm5UYWJsZSB7XG4gICAgY29uc3QgbG9jYXRpb24gPSBsb2NhdGlvblByZWZpeFxuICAgICAgPyBgczM6Ly8ke2xvY2F0aW9uQnVja2V0TmFtZX0vJHtNZGFhQnVja2V0LmZvcm1hdFMzUHJlZml4KGxvY2F0aW9uUHJlZml4KX0vYFxuICAgICAgOiBgczM6Ly8ke2xvY2F0aW9uQnVja2V0TmFtZX0vYDtcbiAgICBjb25zdCB0YWJsZUlucHV0ID0ge1xuICAgICAgbmFtZTogYCR7dGFibGVOYW1lUHJlZml4fV9pbnZgLnJlcGxhY2UoLy0vZ2ksICdfJyksXG4gICAgICBwYXJhbWV0ZXJzOiB7XG4gICAgICAgIEVYVEVSTkFMOiAnVFJVRScsXG4gICAgICAgICdwcm9qZWN0aW9uLmVuYWJsZWQnOiAndHJ1ZScsXG4gICAgICAgICdwcm9qZWN0aW9uLmR0LnR5cGUnOiAnZGF0ZScsXG4gICAgICAgICdwcm9qZWN0aW9uLmR0LnJhbmdlJzogJzIwMjItMDEtMDEtMDAtMDAsTk9XJyxcbiAgICAgICAgJ3Byb2plY3Rpb24uZHQuZm9ybWF0JzogJ3l5eXktTU0tZGQtSEgtbW0nLFxuICAgICAgICAncHJvamVjdGlvbi5kdC5pbnRlcnZhbCc6ICcxJyxcbiAgICAgICAgJ3Byb2plY3Rpb24uZHQuaW50ZXJ2YWwudW5pdCc6ICdIT1VSUycsXG4gICAgICAgICdwcm9qZWN0aW9uLmJ1Y2tldF9pbnZlbnRvcnkudHlwZSc6ICdlbnVtJyxcbiAgICAgICAgJ3Byb2plY3Rpb24uYnVja2V0X2ludmVudG9yeS52YWx1ZXMnOiBidWNrZXRJbnZlbnRvcmllc1xuICAgICAgICAgIC5tYXAoeCA9PiBgJHt4LmJ1Y2tldE5hbWV9LyR7eC5pbnZlbnRvcnlOYW1lfWApXG4gICAgICAgICAgLmpvaW4oJywnKSxcbiAgICAgICAgJ3N0b3JhZ2UubG9jYXRpb24udGVtcGxhdGUnOiBsb2NhdGlvbiArICcke2J1Y2tldF9pbnZlbnRvcnl9L2hpdmUvZHQ9JHtkdH0nLFxuICAgICAgfSxcbiAgICAgIHRhYmxlVHlwZTogJ0VYVEVSTkFMX1RBQkxFJyxcbiAgICAgIHBhcnRpdGlvbktleXM6IFtcbiAgICAgICAge1xuICAgICAgICAgIG5hbWU6ICdidWNrZXRfaW52ZW50b3J5JyxcbiAgICAgICAgICB0eXBlOiAnc3RyaW5nJyxcbiAgICAgICAgfSxcbiAgICAgICAge1xuICAgICAgICAgIG5hbWU6ICdkdCcsXG4gICAgICAgICAgdHlwZTogJ3N0cmluZycsXG4gICAgICAgIH0sXG4gICAgICBdLFxuICAgICAgc3RvcmFnZURlc2NyaXB0b3I6IHtcbiAgICAgICAgY29sdW1uczogW1xuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdidWNrZXQnLFxuICAgICAgICAgICAgdHlwZTogJ3N0cmluZycsXG4gICAgICAgICAgfSxcbiAgICAgICAgICB7XG4gICAgICAgICAgICBuYW1lOiAna2V5JyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ3ZlcnNpb25faWQnLFxuICAgICAgICAgICAgdHlwZTogJ3N0cmluZycsXG4gICAgICAgICAgfSxcbiAgICAgICAgICB7XG4gICAgICAgICAgICBuYW1lOiAnaXNfbGF0ZXN0JyxcbiAgICAgICAgICAgIHR5cGU6ICdib29sZWFuJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdpc19kZWxldGVfbWFya2VyJyxcbiAgICAgICAgICAgIHR5cGU6ICdib29sZWFuJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdzaXplJyxcbiAgICAgICAgICAgIHR5cGU6ICdiaWdpbnQnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2xhc3RfbW9kaWZpZWRfZGF0ZScsXG4gICAgICAgICAgICB0eXBlOiAndGltZXN0YW1wJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdlX3RhZycsXG4gICAgICAgICAgICB0eXBlOiAnc3RyaW5nJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIHtcbiAgICAgICAgICAgIG5hbWU6ICdzdG9yYWdlX2NsYXNzJyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2lzX211bHRpcGFydF91cGxvYWRlZCcsXG4gICAgICAgICAgICB0eXBlOiAnYm9vbGVhbicsXG4gICAgICAgICAgfSxcbiAgICAgICAgICB7XG4gICAgICAgICAgICBuYW1lOiAncmVwbGljYXRpb25fc3RhdHVzJyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2VuY3J5cHRpb25fc3RhdHVzJyxcbiAgICAgICAgICAgIHR5cGU6ICdzdHJpbmcnLFxuICAgICAgICAgIH0sXG4gICAgICAgICAge1xuICAgICAgICAgICAgbmFtZTogJ2ludGVsbGlnZW50X3RpZXJpbmdfdGllcicsXG4gICAgICAgICAgICB0eXBlOiAnc3RyaW5nJyxcbiAgICAgICAgICB9LFxuICAgICAgICAgIC8vICxcbiAgICAgICAgICAvLyB7XG4gICAgICAgICAgLy8gICAgIG5hbWU6IFwib2JqZWN0X2xvY2tfcmV0YWluX3VudGlsX2RhdGVcIixcbiAgICAgICAgICAvLyAgICAgdHlwZTogXCJ0aW1lc3RhbXBcIlxuICAgICAgICAgIC8vIH0sXG4gICAgICAgICAgLy8ge1xuICAgICAgICAgIC8vICAgICBuYW1lOiBcIm9iamVjdF9sb2NrX21vZGVcIixcbiAgICAgICAgICAvLyAgICAgdHlwZTogXCJzdHJpbmdcIlxuICAgICAgICAgIC8vIH0sXG4gICAgICAgICAgLy8ge1xuICAgICAgICAgIC8vICAgICBuYW1lOiBcIm9iamVjdF9sb2NrX2xlZ2FsX2hvbGRfc3RhdHVzXCIsXG4gICAgICAgICAgLy8gICAgIHR5cGU6IFwic3RyaW5nXCJcbiAgICAgICAgICAvLyB9XG4gICAgICAgIF0sXG4gICAgICAgIGxvY2F0aW9uOiBsb2NhdGlvbixcbiAgICAgICAgaW5wdXRGb3JtYXQ6ICdvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLmlvLlN5bWxpbmtUZXh0SW5wdXRGb3JtYXQnLFxuICAgICAgICBvdXRwdXRGb3JtYXQ6ICdvcmcuYXBhY2hlLmhhZG9vcC5oaXZlLnFsLmlvLkhpdmVJZ25vcmVLZXlUZXh0T3V0cHV0Rm9ybWF0JyxcbiAgICAgICAgY29tcHJlc3NlZDogZmFsc2UsXG4gICAgICAgIG51bWJlck9mQnVja2V0czogLTEsXG4gICAgICAgIHNlcmRlSW5mbzoge1xuICAgICAgICAgIHNlcmlhbGl6YXRpb25MaWJyYXJ5OiAnb3JnLmFwYWNoZS5oYWRvb3AuaGl2ZS5xbC5pby5wYXJxdWV0LnNlcmRlLlBhcnF1ZXRIaXZlU2VyRGUnLFxuICAgICAgICAgIHBhcmFtZXRlcnM6IHtcbiAgICAgICAgICAgICdzZXJpYWxpemF0aW9uLmZvcm1hdCc6ICcxJyxcbiAgICAgICAgICB9LFxuICAgICAgICB9LFxuICAgICAgfSxcbiAgICB9O1xuXG4gICAgY29uc3QgdGFibGVQcm9wczogQ2ZuVGFibGVQcm9wcyA9IHtcbiAgICAgIGNhdGFsb2dJZDogY2F0YWxvZ0lkLFxuICAgICAgZGF0YWJhc2VOYW1lOiBkYXRhYmFzZS5kYXRhYmFzZU5hbWUsXG4gICAgICB0YWJsZUlucHV0OiB0YWJsZUlucHV0LFxuICAgIH07XG4gICAgcmV0dXJuIG5ldyBDZm5UYWJsZShzY29wZSwgYGludi10YWJsZS0ke3RhYmxlTmFtZVByZWZpeH1gLCB0YWJsZVByb3BzKTtcbiAgfVxuXG4gIC8qKlxuICAgKiBIZWxwZXIgbWV0aG9kIHRvIGdlbmVyYXRlIGFuIFMzIEJ1Y2tldCBQb2xpY3kgU3RhdGVtZW50IGFsbG93aW5nIFMzIHNlcnZpY2UgdG8gd3JpdGUgaW52ZW50b3JpZXMgdG8gYSBidWNrZXQuXG4gICAqIEBwYXJhbSBzb3VyY2VCdWNrZXRBcm4gVGhlIGJ1Y2tldCBhcm4gZm9yIHdoaWNoIGludmVudG9yeSBkYXRhIGlzIGJlaW5nIHByb2R1Y2VkXG4gICAqIEBwYXJhbSBidWNrZXRBcm4gIFRoZSBidWNrZXQgYXJuIHdoZXJlIGludmVudG9yeSB3aWxsIGJlIHN0b3JlZFxuICAgKiBAcGFyYW0gc3JjQWNjb3VudCBUaGUgYWNjb3VudCB3aGljaCBvd25zIHRoZSBzb3VyY2UgYnVja2V0IGZyb20gd2hpY2ggaW52ZW50b3J5IGlzIGJlaW5nIHByb2R1Y2VkLlxuICAgKiBAcGFyYW0gaW52ZW50b3J5UHJlZml4IFRoZSBwcmVmaXggb24gdGhlIGRlc3RpbmF0aW9uIGJ1Y2tldCB3aGVyZSBpbnZlbnRvcnkgZGF0YSB3aWxsIGJlIHN0b3JlZC5cbiAgICogQHJldHVybnMgQSBwb2xpY3kgc3RhdGVtZW50IHdoaWNoIHBlcm1pdHMgdGhlIFMzIHNlcnZpY2UgcHJpbmNpcGFsIHRvIHdyaXRlIGludmVudG9yeSBmb3IgdGhlIHNvdXJjZSBidWNrZXQvYWNjb3VudCB0byB0aGUgZGVzdGluYXRpb24gYnVja2V0LlxuICAgKi9cbiAgcHVibGljIHN0YXRpYyBjcmVhdGVJbnZlbnRvcnlCdWNrZXRQb2xpY3lTdGF0ZW1lbnQoXG4gICAgYnVja2V0QXJuOiBzdHJpbmcsXG4gICAgc3JjQWNjb3VudDogc3RyaW5nLFxuICAgIHNvdXJjZUJ1Y2tldEFybj86IHN0cmluZyxcbiAgICBpbnZlbnRvcnlQcmVmaXg/OiBzdHJpbmcsXG4gICk6IFBvbGljeVN0YXRlbWVudCB7XG4gICAgY29uc3QgcmVzb3VyY2UgPSBpbnZlbnRvcnlQcmVmaXhcbiAgICAgID8gYCR7YnVja2V0QXJufS8ke01kYWFCdWNrZXQuZm9ybWF0UzNQcmVmaXgoaW52ZW50b3J5UHJlZml4KX0vKmBcbiAgICAgIDogYCR7YnVja2V0QXJufS8qYDtcbiAgICBjb25zdCBhbGxvd1MzSW52ZW50b3J5ID0gbmV3IFBvbGljeVN0YXRlbWVudCh7XG4gICAgICBzaWQ6ICdBbGxvd1MzSW52ZW50b3J5JyxcbiAgICAgIGVmZmVjdDogRWZmZWN0LkFMTE9XLFxuICAgICAgcmVzb3VyY2VzOiBbcmVzb3VyY2VdLFxuICAgICAgYWN0aW9uczogWydzMzpQdXRPYmplY3QnXSxcbiAgICB9KTtcbiAgICBhbGxvd1MzSW52ZW50b3J5LmFkZENvbmRpdGlvbignU3RyaW5nRXF1YWxzJywge1xuICAgICAgJ2F3czpTb3VyY2VBY2NvdW50Jzogc3JjQWNjb3VudCxcbiAgICAgICdzMzp4LWFtei1hY2wnOiAnYnVja2V0LW93bmVyLWZ1bGwtY29udHJvbCcsXG4gICAgfSk7XG4gICAgaWYgKHNvdXJjZUJ1Y2tldEFybikge1xuICAgICAgYWxsb3dTM0ludmVudG9yeS5hZGRDb25kaXRpb24oJ0Fybkxpa2UnLCB7XG4gICAgICAgICdhd3M6U291cmNlQXJuJzogc291cmNlQnVja2V0QXJuLFxuICAgICAgfSk7XG4gICAgfVxuICAgIGFsbG93UzNJbnZlbnRvcnkuYWRkU2VydmljZVByaW5jaXBhbCgnczMuYW1hem9uYXdzLmNvbScpO1xuICAgIHJldHVybiBhbGxvd1MzSW52ZW50b3J5O1xuICB9XG59XG4iXX0=
|
|
@@ -1,241 +0,0 @@
|
|
|
1
|
-
/*!
|
|
2
|
-
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
3
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import { MdaaBucket } from '@aws-mdaa/s3-constructs';
|
|
7
|
-
import { Database } from '@aws-cdk/aws-glue-alpha';
|
|
8
|
-
import { CfnTable, CfnTableProps } from 'aws-cdk-lib/aws-glue';
|
|
9
|
-
import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam';
|
|
10
|
-
import { IBucket, Inventory, InventoryFormat, InventoryFrequency, InventoryObjectVersion } from 'aws-cdk-lib/aws-s3';
|
|
11
|
-
import { Construct } from 'constructs';
|
|
12
|
-
|
|
13
|
-
export interface BucketInventory {
|
|
14
|
-
/** Name of the S3 bucket for which inventory data has been generated */
|
|
15
|
-
readonly bucketName: string;
|
|
16
|
-
/** Name of the S3 inventory configuration that generated the inventory data for the specified bucket */
|
|
17
|
-
readonly inventoryName: string;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
/** Helper class for creating S3 Inventories and associated Glue/Athena resources for querying these inventories. */
|
|
21
|
-
export class InventoryHelper {
|
|
22
|
-
/**
|
|
23
|
-
* Helper method for creating an S3 inventory.
|
|
24
|
-
* @param destinationBucket The bucket where S3 inventory data will be written
|
|
25
|
-
* @param inventoryId The S3 inventory ID
|
|
26
|
-
* @param objectsPrefix The object prefix from which inventory will be produced (on the bucket to which the inventory config is applied)
|
|
27
|
-
* @param destinationPrefix The object prefix under which inventory will be written on the destination bucket.
|
|
28
|
-
* @returns An S3 Inventory Config which can be applied to a bucket in order to produce periodic inventory data.
|
|
29
|
-
*/
|
|
30
|
-
public static createInvConfig(
|
|
31
|
-
destinationBucket: IBucket,
|
|
32
|
-
inventoryId: string,
|
|
33
|
-
objectsPrefix?: string,
|
|
34
|
-
destinationPrefix?: string,
|
|
35
|
-
destinationAccount?: string,
|
|
36
|
-
): Inventory {
|
|
37
|
-
const bucketOwner = destinationAccount ? `${destinationAccount}` : destinationAccount;
|
|
38
|
-
return {
|
|
39
|
-
destination: {
|
|
40
|
-
bucket: destinationBucket,
|
|
41
|
-
/** S3 prefix for inventory report destination organization enabling systematic inventory file */
|
|
42
|
-
prefix: MdaaBucket.formatS3Prefix(destinationPrefix),
|
|
43
|
-
bucketOwner: bucketOwner,
|
|
44
|
-
},
|
|
45
|
-
format: InventoryFormat.PARQUET,
|
|
46
|
-
frequency: InventoryFrequency.DAILY,
|
|
47
|
-
includeObjectVersions: InventoryObjectVersion.ALL,
|
|
48
|
-
inventoryId: inventoryId,
|
|
49
|
-
objectsPrefix: MdaaBucket.formatS3Prefix(objectsPrefix, false, true),
|
|
50
|
-
optionalFields: [
|
|
51
|
-
'Size',
|
|
52
|
-
'LastModifiedDate',
|
|
53
|
-
'StorageClass',
|
|
54
|
-
'ETag',
|
|
55
|
-
'IsMultipartUploaded',
|
|
56
|
-
'ReplicationStatus',
|
|
57
|
-
'EncryptionStatus',
|
|
58
|
-
'ObjectLockRetainUntilDate',
|
|
59
|
-
'ObjectLockMode',
|
|
60
|
-
'ObjectLockLegalHoldStatus',
|
|
61
|
-
'IntelligentTieringAccessTier',
|
|
62
|
-
],
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
/**
|
|
67
|
-
* Helper method for creating a Glue table which can be used to query inventory data generated by an S3 Inventory config
|
|
68
|
-
* @param scope Stack in which the table will be created
|
|
69
|
-
* @param tableNamePrefix Will be used to prefix the table name.
|
|
70
|
-
* @param locationBucketName The bucket where the inventory data is stored.
|
|
71
|
-
* @param inventoryBucket The bucket for which inventory data has been generated
|
|
72
|
-
* @param database The Glue database in which the table will be created
|
|
73
|
-
* @param inventoryName The name of the S3 Inventory config
|
|
74
|
-
* @param locationPrefix The S3 Prefix on the location bucket where the inventory is stored.
|
|
75
|
-
* @returns A Glue table which can be used to query S3 inventory data in an S3 bucket.
|
|
76
|
-
*/
|
|
77
|
-
public static createGlueInvTable(
|
|
78
|
-
scope: Construct,
|
|
79
|
-
catalogId: string,
|
|
80
|
-
tableNamePrefix: string,
|
|
81
|
-
database: Database,
|
|
82
|
-
locationBucketName: string,
|
|
83
|
-
bucketInventories: BucketInventory[],
|
|
84
|
-
locationPrefix?: string,
|
|
85
|
-
): CfnTable {
|
|
86
|
-
const location = locationPrefix
|
|
87
|
-
? `s3://${locationBucketName}/${MdaaBucket.formatS3Prefix(locationPrefix)}/`
|
|
88
|
-
: `s3://${locationBucketName}/`;
|
|
89
|
-
const tableInput = {
|
|
90
|
-
name: `${tableNamePrefix}_inv`.replace(/-/gi, '_'),
|
|
91
|
-
parameters: {
|
|
92
|
-
EXTERNAL: 'TRUE',
|
|
93
|
-
'projection.enabled': 'true',
|
|
94
|
-
'projection.dt.type': 'date',
|
|
95
|
-
'projection.dt.range': '2022-01-01-00-00,NOW',
|
|
96
|
-
'projection.dt.format': 'yyyy-MM-dd-HH-mm',
|
|
97
|
-
'projection.dt.interval': '1',
|
|
98
|
-
'projection.dt.interval.unit': 'HOURS',
|
|
99
|
-
'projection.bucket_inventory.type': 'enum',
|
|
100
|
-
'projection.bucket_inventory.values': bucketInventories
|
|
101
|
-
.map(x => `${x.bucketName}/${x.inventoryName}`)
|
|
102
|
-
.join(','),
|
|
103
|
-
'storage.location.template': location + '${bucket_inventory}/hive/dt=${dt}',
|
|
104
|
-
},
|
|
105
|
-
tableType: 'EXTERNAL_TABLE',
|
|
106
|
-
partitionKeys: [
|
|
107
|
-
{
|
|
108
|
-
name: 'bucket_inventory',
|
|
109
|
-
type: 'string',
|
|
110
|
-
},
|
|
111
|
-
{
|
|
112
|
-
name: 'dt',
|
|
113
|
-
type: 'string',
|
|
114
|
-
},
|
|
115
|
-
],
|
|
116
|
-
storageDescriptor: {
|
|
117
|
-
columns: [
|
|
118
|
-
{
|
|
119
|
-
name: 'bucket',
|
|
120
|
-
type: 'string',
|
|
121
|
-
},
|
|
122
|
-
{
|
|
123
|
-
name: 'key',
|
|
124
|
-
type: 'string',
|
|
125
|
-
},
|
|
126
|
-
{
|
|
127
|
-
name: 'version_id',
|
|
128
|
-
type: 'string',
|
|
129
|
-
},
|
|
130
|
-
{
|
|
131
|
-
name: 'is_latest',
|
|
132
|
-
type: 'boolean',
|
|
133
|
-
},
|
|
134
|
-
{
|
|
135
|
-
name: 'is_delete_marker',
|
|
136
|
-
type: 'boolean',
|
|
137
|
-
},
|
|
138
|
-
{
|
|
139
|
-
name: 'size',
|
|
140
|
-
type: 'bigint',
|
|
141
|
-
},
|
|
142
|
-
{
|
|
143
|
-
name: 'last_modified_date',
|
|
144
|
-
type: 'timestamp',
|
|
145
|
-
},
|
|
146
|
-
{
|
|
147
|
-
name: 'e_tag',
|
|
148
|
-
type: 'string',
|
|
149
|
-
},
|
|
150
|
-
{
|
|
151
|
-
name: 'storage_class',
|
|
152
|
-
type: 'string',
|
|
153
|
-
},
|
|
154
|
-
{
|
|
155
|
-
name: 'is_multipart_uploaded',
|
|
156
|
-
type: 'boolean',
|
|
157
|
-
},
|
|
158
|
-
{
|
|
159
|
-
name: 'replication_status',
|
|
160
|
-
type: 'string',
|
|
161
|
-
},
|
|
162
|
-
{
|
|
163
|
-
name: 'encryption_status',
|
|
164
|
-
type: 'string',
|
|
165
|
-
},
|
|
166
|
-
{
|
|
167
|
-
name: 'intelligent_tiering_tier',
|
|
168
|
-
type: 'string',
|
|
169
|
-
},
|
|
170
|
-
// ,
|
|
171
|
-
// {
|
|
172
|
-
// name: "object_lock_retain_until_date",
|
|
173
|
-
// type: "timestamp"
|
|
174
|
-
// },
|
|
175
|
-
// {
|
|
176
|
-
// name: "object_lock_mode",
|
|
177
|
-
// type: "string"
|
|
178
|
-
// },
|
|
179
|
-
// {
|
|
180
|
-
// name: "object_lock_legal_hold_status",
|
|
181
|
-
// type: "string"
|
|
182
|
-
// }
|
|
183
|
-
],
|
|
184
|
-
location: location,
|
|
185
|
-
inputFormat: 'org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat',
|
|
186
|
-
outputFormat: 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat',
|
|
187
|
-
compressed: false,
|
|
188
|
-
numberOfBuckets: -1,
|
|
189
|
-
serdeInfo: {
|
|
190
|
-
serializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe',
|
|
191
|
-
parameters: {
|
|
192
|
-
'serialization.format': '1',
|
|
193
|
-
},
|
|
194
|
-
},
|
|
195
|
-
},
|
|
196
|
-
};
|
|
197
|
-
|
|
198
|
-
const tableProps: CfnTableProps = {
|
|
199
|
-
catalogId: catalogId,
|
|
200
|
-
databaseName: database.databaseName,
|
|
201
|
-
tableInput: tableInput,
|
|
202
|
-
};
|
|
203
|
-
return new CfnTable(scope, `inv-table-${tableNamePrefix}`, tableProps);
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
/**
|
|
207
|
-
* Helper method to generate an S3 Bucket Policy Statement allowing S3 service to write inventories to a bucket.
|
|
208
|
-
* @param sourceBucketArn The bucket arn for which inventory data is being produced
|
|
209
|
-
* @param bucketArn The bucket arn where inventory will be stored
|
|
210
|
-
* @param srcAccount The account which owns the source bucket from which inventory is being produced.
|
|
211
|
-
* @param inventoryPrefix The prefix on the destination bucket where inventory data will be stored.
|
|
212
|
-
* @returns A policy statement which permits the S3 service principal to write inventory for the source bucket/account to the destination bucket.
|
|
213
|
-
*/
|
|
214
|
-
public static createInventoryBucketPolicyStatement(
|
|
215
|
-
bucketArn: string,
|
|
216
|
-
srcAccount: string,
|
|
217
|
-
sourceBucketArn?: string,
|
|
218
|
-
inventoryPrefix?: string,
|
|
219
|
-
): PolicyStatement {
|
|
220
|
-
const resource = inventoryPrefix
|
|
221
|
-
? `${bucketArn}/${MdaaBucket.formatS3Prefix(inventoryPrefix)}/*`
|
|
222
|
-
: `${bucketArn}/*`;
|
|
223
|
-
const allowS3Inventory = new PolicyStatement({
|
|
224
|
-
sid: 'AllowS3Inventory',
|
|
225
|
-
effect: Effect.ALLOW,
|
|
226
|
-
resources: [resource],
|
|
227
|
-
actions: ['s3:PutObject'],
|
|
228
|
-
});
|
|
229
|
-
allowS3Inventory.addCondition('StringEquals', {
|
|
230
|
-
'aws:SourceAccount': srcAccount,
|
|
231
|
-
's3:x-amz-acl': 'bucket-owner-full-control',
|
|
232
|
-
});
|
|
233
|
-
if (sourceBucketArn) {
|
|
234
|
-
allowS3Inventory.addCondition('ArnLike', {
|
|
235
|
-
'aws:SourceArn': sourceBucketArn,
|
|
236
|
-
});
|
|
237
|
-
}
|
|
238
|
-
allowS3Inventory.addServicePrincipal('s3.amazonaws.com');
|
|
239
|
-
return allowS3Inventory;
|
|
240
|
-
}
|
|
241
|
-
}
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@aws-mdaa/s3-inventory-helper",
|
|
3
|
-
"description": "MDAA s3-inventory-helper utility",
|
|
4
|
-
"author": {
|
|
5
|
-
"name": "Amazon Web Services",
|
|
6
|
-
"url": "https://aws.amazon.com/solutions"
|
|
7
|
-
},
|
|
8
|
-
"version": "1.5.0",
|
|
9
|
-
"main": "lib/index.js",
|
|
10
|
-
"types": "lib/index.d.ts",
|
|
11
|
-
"license": "Apache-2.0",
|
|
12
|
-
"scripts": {
|
|
13
|
-
"build": "tsc",
|
|
14
|
-
"watch": "tsc -w",
|
|
15
|
-
"test": "jest --passWithNoTests --testPathIgnorePatterns='.*\\.snapshot\\.test\\.ts'",
|
|
16
|
-
"lint": "eslint --max-warnings 0 -c ../../../eslint.config.mjs",
|
|
17
|
-
"test:coverage": "jest --passWithNoTests --coverage --testPathIgnorePatterns='.*\\.snapshot\\.test\\.ts'",
|
|
18
|
-
"test:snapshots": "jest --passWithNoTests --testPathPattern='.*\\.snapshot\\.test\\.ts'",
|
|
19
|
-
"test:snapshots:update": "jest --passWithNoTests --testPathPattern='.*\\.snapshot\\.test\\.ts' --updateSnapshot"
|
|
20
|
-
},
|
|
21
|
-
"devDependencies": {
|
|
22
|
-
"@types/jest": "29.5.14",
|
|
23
|
-
"@types/node": "22.9.0",
|
|
24
|
-
"@types/prettier": "2.6.0",
|
|
25
|
-
"jest": "29.7.0",
|
|
26
|
-
"ts-jest": "29.4.6",
|
|
27
|
-
"ts-node": "10.9.2",
|
|
28
|
-
"typescript": "5.9.3",
|
|
29
|
-
"typescript-json-schema": "0.67.1"
|
|
30
|
-
},
|
|
31
|
-
"dependencies": {
|
|
32
|
-
"@aws-cdk/aws-glue-alpha": "2.220.0-alpha.0",
|
|
33
|
-
"@aws-mdaa/naming": "1.5.0",
|
|
34
|
-
"@aws-mdaa/s3-constructs": "1.5.0",
|
|
35
|
-
"aws-cdk-lib": "2.220.0",
|
|
36
|
-
"cdk-nag": "2.37.55",
|
|
37
|
-
"constructs": "10.0.96"
|
|
38
|
-
},
|
|
39
|
-
"gitHead": "8b49a2b371014baec046605ffdbfe38951099c31",
|
|
40
|
-
"repository": {
|
|
41
|
-
"type": "git",
|
|
42
|
-
"url": "https://github.com/aws/modern-data-architecture-accelerator"
|
|
43
|
-
}
|
|
44
|
-
}
|
|
File without changes
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"declarationMap": false,
|
|
4
|
-
"inlineSourceMap": true,
|
|
5
|
-
"inlineSources": true,
|
|
6
|
-
"alwaysStrict": true,
|
|
7
|
-
"removeComments": false,
|
|
8
|
-
"declaration": true,
|
|
9
|
-
"experimentalDecorators": true,
|
|
10
|
-
"incremental": true,
|
|
11
|
-
"lib": [
|
|
12
|
-
"es2020"
|
|
13
|
-
],
|
|
14
|
-
"module": "CommonJS",
|
|
15
|
-
"newLine": "lf",
|
|
16
|
-
"noEmitOnError": true,
|
|
17
|
-
"noFallthroughCasesInSwitch": true,
|
|
18
|
-
"noImplicitAny": true,
|
|
19
|
-
"noImplicitReturns": true,
|
|
20
|
-
"noImplicitThis": true,
|
|
21
|
-
"noUnusedLocals": true,
|
|
22
|
-
"noUnusedParameters": true,
|
|
23
|
-
"resolveJsonModule": true,
|
|
24
|
-
"skipLibCheck": true,
|
|
25
|
-
"strict": true,
|
|
26
|
-
"strictNullChecks": true,
|
|
27
|
-
"strictPropertyInitialization": true,
|
|
28
|
-
"stripInternal": false,
|
|
29
|
-
"target": "ES2020",
|
|
30
|
-
"composite": true,
|
|
31
|
-
"tsBuildInfoFile": "tsconfig.tsbuildinfo"
|
|
32
|
-
},
|
|
33
|
-
"include": [
|
|
34
|
-
"**/*.ts"
|
|
35
|
-
],
|
|
36
|
-
"exclude": [
|
|
37
|
-
"node_modules"
|
|
38
|
-
],
|
|
39
|
-
"_generated_by_jsii_": "Generated by jsii - safe to delete, and ideally should be in .gitignore"
|
|
40
|
-
}
|