embulk-output-s3_parquet 0.0.3 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/release.yml +40 -0
- data/.github/workflows/test.yml +26 -0
- data/CHANGELOG.md +9 -0
- data/README.md +44 -7
- data/build.gradle +7 -8
- data/example/with_catalog.yml +36 -0
- data/example/with_logicaltypes.yml +31 -0
- data/src/main/scala/org/embulk/output/s3_parquet/CatalogRegistrator.scala +29 -5
- data/src/main/scala/org/embulk/output/s3_parquet/S3ParquetOutputPlugin.scala +44 -6
- data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsCredentials.scala +11 -1
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/EmbulkMessageType.scala +39 -11
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandler.scala +145 -0
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandlerStore.scala +107 -0
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriteSupport.scala +4 -2
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriter.scala +51 -34
- data/src/test/scala/org/embulk/output/s3_parquet/TestS3ParquetOutputPlugin.scala +34 -29
- data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandler.scala +78 -0
- data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandlerStore.scala +162 -0
- metadata +23 -14
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9a5fcc051188467ff067c7542e3c12d32b9ce57e
|
|
4
|
+
data.tar.gz: c93d01c345e6e3a8b43f335f0467ee47532cc32d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 510bf2837f6c57e225b53084790dc0e79feef60247b73d8aee7cd268725676e0783e9ecee0cb2db4a3235969634802b2b8005208f36a4f1a86f1d15777ea9bb9
|
|
7
|
+
data.tar.gz: a1c23cbf8e5bc1c5414e4e906b0060cb5a7da9085767f319df57763724cfc07cda3925ba085015951dd3a5e40a9dba6a211d777474f00383b9d3f08d9f1d706a
|
data/.github/FUNDING.yml
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
github: civitaspo
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: Release CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
types:
|
|
8
|
+
- closed
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
release:
|
|
12
|
+
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
services:
|
|
15
|
+
localstack:
|
|
16
|
+
image: localstack/localstack
|
|
17
|
+
ports:
|
|
18
|
+
- 4572:4572
|
|
19
|
+
env:
|
|
20
|
+
SERVICES: s3
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v1
|
|
24
|
+
- name: Set up JDK 1.8
|
|
25
|
+
uses: actions/setup-java@v1
|
|
26
|
+
with:
|
|
27
|
+
java-version: 1.8
|
|
28
|
+
- name: Test with Gradle
|
|
29
|
+
if: github.event.pull_request.merged == true
|
|
30
|
+
run: ./gradlew test
|
|
31
|
+
- name: Release the new gem
|
|
32
|
+
if: github.event.pull_request.merged == true
|
|
33
|
+
run: |
|
|
34
|
+
mkdir -p $HOME/.gem
|
|
35
|
+
touch $HOME/.gem/credentials
|
|
36
|
+
chmod 0600 $HOME/.gem/credentials
|
|
37
|
+
printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
|
|
38
|
+
./gradlew gemPush
|
|
39
|
+
env:
|
|
40
|
+
RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Test CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
- push
|
|
5
|
+
|
|
6
|
+
jobs:
|
|
7
|
+
test:
|
|
8
|
+
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
services:
|
|
11
|
+
localstack:
|
|
12
|
+
image: localstack/localstack
|
|
13
|
+
ports:
|
|
14
|
+
- 4572:4572
|
|
15
|
+
env:
|
|
16
|
+
SERVICES: s3
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v1
|
|
20
|
+
- name: Set up JDK 1.8
|
|
21
|
+
uses: actions/setup-java@v1
|
|
22
|
+
with:
|
|
23
|
+
java-version: 1.8
|
|
24
|
+
- name: Test with Gradle
|
|
25
|
+
run: ./gradlew test
|
|
26
|
+
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
0.1.0 (2019-11-17)
|
|
2
|
+
==================
|
|
3
|
+
|
|
4
|
+
* [New Feature] Support Logical Types older representations(OriginalTypes) #12
|
|
5
|
+
* [Enhancement] Add Github Actions CI settings #13
|
|
6
|
+
* [Enhancement] Support LogicalTypes for Glue Data Catalog #14
|
|
7
|
+
* [Enhancement] Update dependencies #15
|
|
8
|
+
* [New Feature] Support `auth_method: web_identity_token` #15
|
|
9
|
+
|
|
1
10
|
0.0.3 (2019-07-17)
|
|
2
11
|
==================
|
|
3
12
|
|
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# S3 Parquet output plugin for Embulk
|
|
2
2
|
|
|
3
|
+
 
|
|
4
|
+
|
|
3
5
|
[Embulk](https://github.com/embulk/embulk/) output plugin to dump records as [Apache Parquet](https://parquet.apache.org/) files on S3.
|
|
4
6
|
|
|
5
7
|
## Overview
|
|
@@ -22,12 +24,13 @@
|
|
|
22
24
|
- **column_options**: a map whose keys are name of columns, and values are configuration with following parameters (optional)
|
|
23
25
|
- **timezone**: timezone if type of this column is timestamp. If not set, **default_timezone** is used. (string, optional)
|
|
24
26
|
- **format**: timestamp format if type of this column is timestamp. If not set, **default_timestamp_format**: is used. (string, optional)
|
|
27
|
+
- **logical_type**: a Parquet logical type name (`timestamp-millis`, `timestamp-micros`, `json`, `int8`, `int16`, `int32`, `int64`, `uint8`, `uint16`, `uint32`, `uint64`) (string, optional)
|
|
25
28
|
- **canned_acl**: grants one of [canned ACLs](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL) for created objects (string, default: `private`)
|
|
26
29
|
- **block_size**: The block size is the size of a row group being buffered in memory. This limits the memory usage when writing. Larger values will improve the I/O when reading but consume more memory when writing. (int, default: `134217728` (128MB))
|
|
27
30
|
- **page_size**: The page size is for compression. When reading, each page can be decompressed independently. A block is composed of pages. The page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. (int, default: `1048576` (1MB))
|
|
28
31
|
- **max_padding_size**: The max size (bytes) to write as padding and the min size of a row group (int, default: `8388608` (8MB))
|
|
29
32
|
- **enable_dictionary_encoding**: The boolean value is to enable/disable dictionary encoding. (boolean, default: `true`)
|
|
30
|
-
- **auth_method**: name of mechanism to authenticate requests (`"basic"`, `"env"`, `"instance"`, `"profile"`, `"properties"`, `"anonymous"`,
|
|
33
|
+
- **auth_method**: name of mechanism to authenticate requests (`"basic"`, `"env"`, `"instance"`, `"profile"`, `"properties"`, `"anonymous"`, `"session"`, `"web_identity_token"`, default: `"default"`)
|
|
31
34
|
- `"basic"`: uses **access_key_id** and **secret_access_key** to authenticate.
|
|
32
35
|
- `"env"`: uses `AWS_ACCESS_KEY_ID` (or `AWS_ACCESS_KEY`) and `AWS_SECRET_KEY` (or `AWS_SECRET_ACCESS_KEY`) environment variables.
|
|
33
36
|
- `"instance"`: uses EC2 instance profile or attached ECS task role.
|
|
@@ -44,6 +47,7 @@
|
|
|
44
47
|
- `"anonymous"`: uses anonymous access. This auth method can access only public files.
|
|
45
48
|
- `"session"`: uses temporary-generated **access_key_id**, **secret_access_key** and **session_token**.
|
|
46
49
|
- `"assume_role"`: uses temporary-generated credentials by assuming **role_arn** role.
|
|
50
|
+
- `"web_identity_token"`: uses temporary-generated credentials by assuming **role_arn** role with web identity.
|
|
47
51
|
- `"default"`: uses AWS SDK's default strategy to look up available credentials from runtime environment. This method behaves like the combination of the following methods.
|
|
48
52
|
1. `"env"`
|
|
49
53
|
1. `"properties"`
|
|
@@ -54,17 +58,42 @@
|
|
|
54
58
|
- **access_key_id**: aws access key id. this is required when **auth_method** is `"basic"` or `"session"`. (string, optional)
|
|
55
59
|
- **secret_access_key**: aws secret access key. this is required when **auth_method** is `"basic"` or `"session"`. (string, optional)
|
|
56
60
|
- **session_token**: aws session token. this is required when **auth_method** is `"session"`. (string, optional)
|
|
57
|
-
- **role_arn**: arn of the role to assume. this is required for **auth_method** is `"assume_role"`. (string, optional)
|
|
58
|
-
- **role_session_name**: an identifier for the assumed role session. this is required when **auth_method** is `"assume_role"`. (string, optional)
|
|
61
|
+
- **role_arn**: arn of the role to assume. this is required for **auth_method** is `"assume_role"` or `"web_identity_token"`. (string, optional)
|
|
62
|
+
- **role_session_name**: an identifier for the assumed role session. this is required when **auth_method** is `"assume_role"` or `"web_identity_token"`. (string, optional)
|
|
59
63
|
- **role_external_id**: a unique identifier that is used by third parties when assuming roles in their customers' accounts. this is optionally used for **auth_method**: `"assume_role"`. (string, optional)
|
|
60
|
-
- **role_session_duration_seconds**: duration, in seconds, of the role session. this is optionally used for **auth_method**: `"assume_role"`. (int, optional)
|
|
64
|
+
- **role_session_duration_seconds**: duration, in seconds, of the role session. this is optionally used for **auth_method**: `"assume_role"`. (int, optional)
|
|
65
|
+
- **web_identity_token_file**: the absolute path to the web identity token file. this is required when **auth_method** is `"web_identity_token"`. (string, optional)
|
|
61
66
|
- **scope_down_policy**: an iam policy in json format. this is optionally used for **auth_method**: `"assume_role"`. (string, optional)
|
|
62
67
|
- **catalog**: Register a table if this option is specified (optional)
|
|
63
68
|
- **catalog_id**: glue data catalog id if you use a catalog different from account/region default catalog. (string, optional)
|
|
64
69
|
- **database**: The name of the database (string, required)
|
|
65
70
|
- **table**: The name of the table (string, required)
|
|
66
71
|
- **column_options**: a key-value pairs where key is a column name and value is options for the column. (string to options map, default: `{}`)
|
|
67
|
-
- **type**: type of
|
|
72
|
+
- **type**: type of column when this plugin creates new tables (e.g. `string`, `bigint`) (string, default: depends on the input embulk column type, or the parquet logical type. See the below table)
|
|
73
|
+
|
|
74
|
+
|embulk column type|glue data type|
|
|
75
|
+
|:---|:---|
|
|
76
|
+
|long|bigint|
|
|
77
|
+
|boolean|boolean|
|
|
78
|
+
|double|double|
|
|
79
|
+
|string|string|
|
|
80
|
+
|timestamp|string|
|
|
81
|
+
|json|string|
|
|
82
|
+
|
|
83
|
+
|parquet logical type|glue data type|note|
|
|
84
|
+
|:---|:---|:---|
|
|
85
|
+
|timestamp-millis|timestamp||
|
|
86
|
+
|timestamp-micros|long|Glue cannot recognize timestamp-micros.|
|
|
87
|
+
|int8|tinyint||
|
|
88
|
+
|int16|smallint||
|
|
89
|
+
|int32|int||
|
|
90
|
+
|int64|bigint||
|
|
91
|
+
|uint8|smallint|Glue tinyint is a minimum value of -2^7 and a maximum value of 2^7-1|
|
|
92
|
+
|uint16|int|Glue smallint is a minimum value of -2^15 and a maximum value of 2^15-1.|
|
|
93
|
+
|uint32|bigint|Glue int is a minimum value of-2^31 and a maximum value of 2^31-1.|
|
|
94
|
+
|uint64|ConfigException|Glue bigint supports only a 64-bit signed integer.|
|
|
95
|
+
|json|string||
|
|
96
|
+
|
|
68
97
|
- **operation_if_exists**: operation if the table already exist. Available operations are `"delete"` and `"skip"` (string, default: `"delete"`)
|
|
69
98
|
- **endpoint**: The AWS Service endpoint (string, optional)
|
|
70
99
|
- **region**: The AWS region (string, optional)
|
|
@@ -75,6 +104,8 @@
|
|
|
75
104
|
- **user** proxy user (string, optional)
|
|
76
105
|
- **password** proxy password (string, optional)
|
|
77
106
|
- **buffer_dir**: buffer directory for parquet files to be uploaded on S3 (string, default: Create a Temporary Directory)
|
|
107
|
+
- **type_options**: a map whose keys are name of embulk type(`boolean`, `long`, `double`, `string`, `timestamp`, `json`), and values are configuration with following parameters (optional)
|
|
108
|
+
- **logical_type**: a Parquet logical type name (`timestamp-millis`, `timestamp-micros`, `json`, `int8`, `int16`, `int32`, `int64`, `uint8`, `uint16`, `uint32`, `uint64`) (string, optional)
|
|
78
109
|
|
|
79
110
|
|
|
80
111
|
## Example
|
|
@@ -92,7 +123,8 @@ out:
|
|
|
92
123
|
|
|
93
124
|
## Note
|
|
94
125
|
|
|
95
|
-
* The current
|
|
126
|
+
* The current Parquet [LogicalTypes](https://github.com/apache/parquet-format/blob/2b38663/LogicalTypes.md) implementation does only old representation.
|
|
127
|
+
* Some kind of LogicalTypes are sometimes not supported on your middleware. Be careful to giving logical type name.
|
|
96
128
|
|
|
97
129
|
## Development
|
|
98
130
|
|
|
@@ -106,6 +138,8 @@ $ embulk run example/config.yml -Ilib
|
|
|
106
138
|
### Run test:
|
|
107
139
|
|
|
108
140
|
```shell
|
|
141
|
+
## Run fake S3 with localstack
|
|
142
|
+
$ docker run -it --rm -p 4572:4572 -e SERVICES=s3 localstack/localstack
|
|
109
143
|
$ ./gradlew test
|
|
110
144
|
```
|
|
111
145
|
|
|
@@ -121,9 +155,12 @@ Fix [build.gradle](./build.gradle), then
|
|
|
121
155
|
|
|
122
156
|
```shell
|
|
123
157
|
$ ./gradlew gemPush
|
|
124
|
-
|
|
125
158
|
```
|
|
126
159
|
|
|
127
160
|
## ChangeLog
|
|
128
161
|
|
|
129
162
|
[CHANGELOG.md](./CHANGELOG.md)
|
|
163
|
+
|
|
164
|
+
## Contributors
|
|
165
|
+
|
|
166
|
+
- @syucream
|
data/build.gradle
CHANGED
|
@@ -13,18 +13,18 @@ configurations {
|
|
|
13
13
|
provided
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
version = "0.0
|
|
16
|
+
version = "0.1.0"
|
|
17
17
|
|
|
18
18
|
sourceCompatibility = 1.8
|
|
19
19
|
targetCompatibility = 1.8
|
|
20
20
|
|
|
21
21
|
dependencies {
|
|
22
|
-
compile "org.embulk:embulk-core:0.9.
|
|
23
|
-
provided "org.embulk:embulk-core:0.9.
|
|
22
|
+
compile "org.embulk:embulk-core:0.9.20"
|
|
23
|
+
provided "org.embulk:embulk-core:0.9.20"
|
|
24
24
|
|
|
25
|
-
compile 'org.scala-lang:scala-library:2.13.
|
|
25
|
+
compile 'org.scala-lang:scala-library:2.13.1'
|
|
26
26
|
['glue', 's3', 'sts'].each { v ->
|
|
27
|
-
compile "com.amazonaws:aws-java-sdk-${v}:1.11.
|
|
27
|
+
compile "com.amazonaws:aws-java-sdk-${v}:1.11.676"
|
|
28
28
|
}
|
|
29
29
|
['column', 'common', 'encoding', 'format', 'hadoop', 'jackson'].each { v ->
|
|
30
30
|
compile "org.apache.parquet:parquet-${v}:1.10.1"
|
|
@@ -33,9 +33,8 @@ dependencies {
|
|
|
33
33
|
compile 'org.xerial.snappy:snappy-java:1.1.7.3'
|
|
34
34
|
|
|
35
35
|
testCompile 'org.scalatest:scalatest_2.13:3.0.8'
|
|
36
|
-
testCompile 'org.embulk:embulk-test:0.9.
|
|
37
|
-
testCompile 'org.embulk:embulk-standards:0.9.
|
|
38
|
-
testCompile 'cloud.localstack:localstack-utils:0.1.15'
|
|
36
|
+
testCompile 'org.embulk:embulk-test:0.9.20'
|
|
37
|
+
testCompile 'org.embulk:embulk-standards:0.9.20'
|
|
39
38
|
testCompile 'org.apache.parquet:parquet-tools:1.10.1'
|
|
40
39
|
testCompile 'org.apache.hadoop:hadoop-client:2.9.2'
|
|
41
40
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
|
|
2
|
+
in:
|
|
3
|
+
type: file
|
|
4
|
+
path_prefix: ./example/data.tsv
|
|
5
|
+
parser:
|
|
6
|
+
type: csv
|
|
7
|
+
delimiter: "\t"
|
|
8
|
+
skip_header_lines: 0
|
|
9
|
+
null_string: ""
|
|
10
|
+
columns:
|
|
11
|
+
- { name: id, type: long }
|
|
12
|
+
- { name: description, type: string }
|
|
13
|
+
- { name: name, type: string }
|
|
14
|
+
- { name: t, type: timestamp, format: "%Y-%m-%d %H:%M:%S %z"}
|
|
15
|
+
- { name: payload, type: json}
|
|
16
|
+
stop_on_invalid_record: true
|
|
17
|
+
|
|
18
|
+
out:
|
|
19
|
+
type: s3_parquet
|
|
20
|
+
bucket: dev-baikal-workspace
|
|
21
|
+
path_prefix: path/to/my-obj-2.
|
|
22
|
+
file_ext: snappy.parquet
|
|
23
|
+
compression_codec: snappy
|
|
24
|
+
default_timezone: Asia/Tokyo
|
|
25
|
+
canned_acl: bucket-owner-full-control
|
|
26
|
+
column_options:
|
|
27
|
+
id:
|
|
28
|
+
logical_type: "int64"
|
|
29
|
+
payload:
|
|
30
|
+
logical_type: "json"
|
|
31
|
+
type_options:
|
|
32
|
+
timestamp:
|
|
33
|
+
logical_type: "timestamp-millis"
|
|
34
|
+
catalog:
|
|
35
|
+
database: example_db
|
|
36
|
+
table: example_tbl
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
|
|
2
|
+
in:
|
|
3
|
+
type: file
|
|
4
|
+
path_prefix: ./example/data.tsv
|
|
5
|
+
parser:
|
|
6
|
+
type: csv
|
|
7
|
+
delimiter: "\t"
|
|
8
|
+
skip_header_lines: 0
|
|
9
|
+
null_string: ""
|
|
10
|
+
columns:
|
|
11
|
+
- { name: id, type: long }
|
|
12
|
+
- { name: description, type: string }
|
|
13
|
+
- { name: name, type: string }
|
|
14
|
+
- { name: t, type: timestamp, format: "%Y-%m-%d %H:%M:%S %z"}
|
|
15
|
+
- { name: payload, type: json}
|
|
16
|
+
stop_on_invalid_record: true
|
|
17
|
+
|
|
18
|
+
out:
|
|
19
|
+
type: s3_parquet
|
|
20
|
+
bucket: my-bucket
|
|
21
|
+
path_prefix: path/to/my-obj-2.
|
|
22
|
+
file_ext: snappy.parquet
|
|
23
|
+
compression_codec: snappy
|
|
24
|
+
default_timezone: Asia/Tokyo
|
|
25
|
+
canned_acl: bucket-owner-full-control
|
|
26
|
+
column_options:
|
|
27
|
+
id:
|
|
28
|
+
logical_type: "uint64"
|
|
29
|
+
type_options:
|
|
30
|
+
timestamp:
|
|
31
|
+
logical_type: "timestamp-millis"
|
|
@@ -51,9 +51,10 @@ object CatalogRegistrator
|
|
|
51
51
|
schema: Schema,
|
|
52
52
|
location: String,
|
|
53
53
|
compressionCodec: CompressionCodecName,
|
|
54
|
-
loggerOption: Option[Logger] = None
|
|
54
|
+
loggerOption: Option[Logger] = None,
|
|
55
|
+
parquetColumnLogicalTypes: Map[String, String] = Map.empty): CatalogRegistrator =
|
|
55
56
|
{
|
|
56
|
-
new CatalogRegistrator(aws, task, schema, location, compressionCodec, loggerOption)
|
|
57
|
+
new CatalogRegistrator(aws, task, schema, location, compressionCodec, loggerOption, parquetColumnLogicalTypes)
|
|
57
58
|
}
|
|
58
59
|
}
|
|
59
60
|
|
|
@@ -62,7 +63,8 @@ class CatalogRegistrator(aws: Aws,
|
|
|
62
63
|
schema: Schema,
|
|
63
64
|
location: String,
|
|
64
65
|
compressionCodec: CompressionCodecName,
|
|
65
|
-
loggerOption: Option[Logger] = None
|
|
66
|
+
loggerOption: Option[Logger] = None,
|
|
67
|
+
parquetColumnLogicalTypes: Map[String, String] = Map.empty)
|
|
66
68
|
{
|
|
67
69
|
val logger: Logger = loggerOption.getOrElse(LoggerFactory.getLogger(classOf[CatalogRegistrator]))
|
|
68
70
|
|
|
@@ -150,14 +152,36 @@ class CatalogRegistrator(aws: Aws,
|
|
|
150
152
|
schema.getColumns.asScala.toSeq.map { c =>
|
|
151
153
|
val cType: String =
|
|
152
154
|
if (columnOptions.contains(c.getName)) columnOptions(c.getName).getType
|
|
153
|
-
else
|
|
155
|
+
else if (parquetColumnLogicalTypes.contains(c.getName)) convertParquetLogicalTypeToGlueType(parquetColumnLogicalTypes(c.getName))
|
|
156
|
+
else convertEmbulkTypeToGlueType(c.getType)
|
|
154
157
|
new Column()
|
|
155
158
|
.withName(c.getName)
|
|
156
159
|
.withType(cType)
|
|
157
160
|
}
|
|
158
161
|
}
|
|
159
162
|
|
|
160
|
-
private def
|
|
163
|
+
private def convertParquetLogicalTypeToGlueType(t: String): String =
|
|
164
|
+
{
|
|
165
|
+
t match {
|
|
166
|
+
case "timestamp-millis" => "timestamp"
|
|
167
|
+
case "timestamp-micros" => "bigint" // Glue cannot recognize timestamp-micros.
|
|
168
|
+
case "int8" => "tinyint"
|
|
169
|
+
case "int16" => "smallint"
|
|
170
|
+
case "int32" => "int"
|
|
171
|
+
case "int64" => "bigint"
|
|
172
|
+
case "uint8" => "smallint" // Glue tinyint is a minimum value of -2^7 and a maximum value of 2^7-1
|
|
173
|
+
case "uint16" => "int" // Glue smallint is a minimum value of -2^15 and a maximum value of 2^15-1.
|
|
174
|
+
case "uint32" => "bigint" // Glue int is a minimum value of-2^31 and a maximum value of 2^31-1.
|
|
175
|
+
case "uint64" => throw new ConfigException("Cannot convert uint64 to Glue data types automatically" +
|
|
176
|
+
" because the Glue bigint supports a 64-bit signed integer." +
|
|
177
|
+
" Please use `catalog.column_options` to define the type.")
|
|
178
|
+
case "json" => "string"
|
|
179
|
+
case _ => throw new ConfigException(s"Unsupported a parquet logical type: $t. Please use `catalog.column_options` to define the type.")
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
private def convertEmbulkTypeToGlueType(t: Type): String =
|
|
161
185
|
{
|
|
162
186
|
t match {
|
|
163
187
|
case _: BooleanType => "boolean"
|
|
@@ -9,15 +9,18 @@ import org.apache.parquet.column.ParquetProperties
|
|
|
9
9
|
import org.apache.parquet.hadoop.ParquetWriter
|
|
10
10
|
import org.apache.parquet.hadoop.metadata.CompressionCodecName
|
|
11
11
|
import org.embulk.config.{Config, ConfigDefault, ConfigDiff, ConfigException, ConfigSource, Task, TaskReport, TaskSource}
|
|
12
|
-
import org.embulk.output.s3_parquet.S3ParquetOutputPlugin.PluginTask
|
|
12
|
+
import org.embulk.output.s3_parquet.S3ParquetOutputPlugin.{ColumnOptionTask, PluginTask}
|
|
13
13
|
import org.embulk.output.s3_parquet.aws.Aws
|
|
14
|
-
import org.embulk.output.s3_parquet.parquet.ParquetFileWriter
|
|
14
|
+
import org.embulk.output.s3_parquet.parquet.{LogicalTypeHandlerStore, ParquetFileWriter}
|
|
15
15
|
import org.embulk.spi.{Exec, OutputPlugin, PageReader, Schema, TransactionalPageOutput}
|
|
16
16
|
import org.embulk.spi.time.TimestampFormatter
|
|
17
17
|
import org.embulk.spi.time.TimestampFormatter.TimestampColumnOption
|
|
18
18
|
import org.embulk.spi.util.Timestamps
|
|
19
19
|
import org.slf4j.{Logger, LoggerFactory}
|
|
20
20
|
|
|
21
|
+
import scala.jdk.CollectionConverters._
|
|
22
|
+
import scala.util.chaining._
|
|
23
|
+
|
|
21
24
|
|
|
22
25
|
object S3ParquetOutputPlugin
|
|
23
26
|
{
|
|
@@ -53,7 +56,7 @@ object S3ParquetOutputPlugin
|
|
|
53
56
|
|
|
54
57
|
@Config("column_options")
|
|
55
58
|
@ConfigDefault("{}")
|
|
56
|
-
def getColumnOptions: JMap[String,
|
|
59
|
+
def getColumnOptions: JMap[String, ColumnOptionTask]
|
|
57
60
|
|
|
58
61
|
@Config("canned_acl")
|
|
59
62
|
@ConfigDefault("\"private\"")
|
|
@@ -86,8 +89,23 @@ object S3ParquetOutputPlugin
|
|
|
86
89
|
@Config("catalog")
|
|
87
90
|
@ConfigDefault("null")
|
|
88
91
|
def getCatalog: Optional[CatalogRegistrator.Task]
|
|
92
|
+
|
|
93
|
+
@Config("type_options")
|
|
94
|
+
@ConfigDefault("{}")
|
|
95
|
+
def getTypeOptions: JMap[String, TypeOptionTask]
|
|
89
96
|
}
|
|
90
97
|
|
|
98
|
+
trait ColumnOptionTask
|
|
99
|
+
extends Task with TimestampColumnOption with LogicalTypeOption
|
|
100
|
+
|
|
101
|
+
trait TypeOptionTask
|
|
102
|
+
extends Task with LogicalTypeOption
|
|
103
|
+
|
|
104
|
+
trait LogicalTypeOption
|
|
105
|
+
{
|
|
106
|
+
@Config("logical_type")
|
|
107
|
+
def getLogicalType: Optional[String]
|
|
108
|
+
}
|
|
91
109
|
}
|
|
92
110
|
|
|
93
111
|
class S3ParquetOutputPlugin
|
|
@@ -117,11 +135,26 @@ class S3ParquetOutputPlugin
|
|
|
117
135
|
}
|
|
118
136
|
task.getCatalog.ifPresent { catalog =>
|
|
119
137
|
val location = s"s3://${task.getBucket}/${task.getPathPrefix.replaceFirst("(.*/)[^/]+$", "$1")}"
|
|
138
|
+
val parquetColumnLogicalTypes: Map[String, String] = Map.newBuilder[String, String].pipe {builder =>
|
|
139
|
+
val cOptions = task.getColumnOptions.asScala
|
|
140
|
+
val tOptions = task.getTypeOptions.asScala
|
|
141
|
+
schema.getColumns.asScala.foreach {c =>
|
|
142
|
+
cOptions.get(c.getName)
|
|
143
|
+
if (cOptions.contains(c.getName) && cOptions(c.getName).getLogicalType.isPresent) {
|
|
144
|
+
builder.addOne(c.getName -> cOptions(c.getName).getLogicalType.get())
|
|
145
|
+
}
|
|
146
|
+
else if (tOptions.contains(c.getType.getName) && tOptions(c.getType.getName).getLogicalType.isPresent) {
|
|
147
|
+
builder.addOne(c.getName -> tOptions(c.getType.getName).getLogicalType.get())
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
builder.result()
|
|
151
|
+
}
|
|
120
152
|
val cr = CatalogRegistrator(aws = Aws(task),
|
|
121
153
|
task = catalog,
|
|
122
154
|
schema = schema,
|
|
123
155
|
location = location,
|
|
124
|
-
compressionCodec = task.getCompressionCodec
|
|
156
|
+
compressionCodec = task.getCompressionCodec,
|
|
157
|
+
parquetColumnLogicalTypes = parquetColumnLogicalTypes)
|
|
125
158
|
cr.run()
|
|
126
159
|
}
|
|
127
160
|
|
|
@@ -148,9 +181,12 @@ class S3ParquetOutputPlugin
|
|
|
148
181
|
|
|
149
182
|
// column_options
|
|
150
183
|
task.getColumnOptions.forEach { (k: String,
|
|
151
|
-
|
|
184
|
+
opt: ColumnOptionTask) =>
|
|
152
185
|
val c = schema.lookupColumn(k)
|
|
153
|
-
|
|
186
|
+
val useTimestampOption = opt.getFormat.isPresent || opt.getTimeZoneId.isPresent
|
|
187
|
+
if (!c.getType.getName.equals("timestamp") && useTimestampOption) {
|
|
188
|
+
throw new ConfigException(s"column:$k is not 'timestamp' type.")
|
|
189
|
+
}
|
|
154
190
|
}
|
|
155
191
|
|
|
156
192
|
// canned_acl
|
|
@@ -198,9 +234,11 @@ class S3ParquetOutputPlugin
|
|
|
198
234
|
val pageReader: PageReader = new PageReader(schema)
|
|
199
235
|
val aws: Aws = Aws(task)
|
|
200
236
|
val timestampFormatters: Seq[TimestampFormatter] = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions).toSeq
|
|
237
|
+
val logicalTypeHandlers = LogicalTypeHandlerStore.fromEmbulkOptions(task.getTypeOptions, task.getColumnOptions)
|
|
201
238
|
val parquetWriter: ParquetWriter[PageReader] = ParquetFileWriter.builder()
|
|
202
239
|
.withPath(bufferFile)
|
|
203
240
|
.withSchema(schema)
|
|
241
|
+
.withLogicalTypeHandlers(logicalTypeHandlers)
|
|
204
242
|
.withTimestampFormatters(timestampFormatters)
|
|
205
243
|
.withCompressionCodec(task.getCompressionCodec)
|
|
206
244
|
.withDictionaryEncoding(task.getEnableDictionaryEncoding.orElse(ParquetProperties.DEFAULT_IS_DICTIONARY_ENABLED))
|