embulk-output-s3_parquet 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/release.yml +3 -0
- data/.github/workflows/test.yml +2 -0
- data/.scalafmt.conf +5 -0
- data/CHANGELOG.md +15 -0
- data/README.md +3 -2
- data/build.gradle +19 -9
- data/example/config.yml +3 -1
- data/example/prepare_s3_bucket.sh +6 -0
- data/example/with_catalog.yml +3 -1
- data/example/with_logicaltypes.yml +3 -1
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -1
- data/gradlew +31 -20
- data/gradlew.bat +17 -1
- data/run_s3_local.sh +7 -0
- data/src/main/scala/org/embulk/output/s3_parquet/CatalogRegistrator.scala +226 -178
- data/src/main/scala/org/embulk/output/s3_parquet/ContextClassLoaderSwapper.scala +18 -0
- data/src/main/scala/org/embulk/output/s3_parquet/S3ParquetOutputPlugin.scala +293 -204
- data/src/main/scala/org/embulk/output/s3_parquet/S3ParquetPageOutput.scala +46 -49
- data/src/main/scala/org/embulk/output/s3_parquet/aws/Aws.scala +46 -50
- data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsClientConfiguration.scala +18 -23
- data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsCredentials.scala +146 -119
- data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsEndpointConfiguration.scala +32 -35
- data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsS3Configuration.scala +45 -41
- data/src/main/scala/org/embulk/output/s3_parquet/aws/HttpProxy.scala +40 -43
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/EmbulkMessageType.scala +138 -92
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandler.scala +117 -102
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandlerStore.scala +91 -84
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriteSupport.scala +30 -29
- data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriter.scala +143 -152
- data/src/test/scala/org/embulk/output/s3_parquet/TestS3ParquetOutputPlugin.scala +144 -117
- data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandler.scala +72 -66
- data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandlerStore.scala +149 -132
- metadata +22 -15
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-s3_parquet
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-03-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -49,6 +49,7 @@ files:
|
|
49
49
|
- ".github/workflows/release.yml"
|
50
50
|
- ".github/workflows/test.yml"
|
51
51
|
- ".gitignore"
|
52
|
+
- ".scalafmt.conf"
|
52
53
|
- CHANGELOG.md
|
53
54
|
- LICENSE.txt
|
54
55
|
- README.md
|
@@ -61,12 +62,13 @@ files:
|
|
61
62
|
- classpath/api-util-1.0.0-M20.jar
|
62
63
|
- classpath/asm-3.1.jar
|
63
64
|
- classpath/asm-5.0.4.jar
|
65
|
+
- classpath/audience-annotations-0.11.0.jar
|
64
66
|
- classpath/avro-1.7.7.jar
|
65
|
-
- classpath/aws-java-sdk-core-1.11.
|
66
|
-
- classpath/aws-java-sdk-glue-1.11.
|
67
|
-
- classpath/aws-java-sdk-kms-1.11.
|
68
|
-
- classpath/aws-java-sdk-s3-1.11.
|
69
|
-
- classpath/aws-java-sdk-sts-1.11.
|
67
|
+
- classpath/aws-java-sdk-core-1.11.739.jar
|
68
|
+
- classpath/aws-java-sdk-glue-1.11.739.jar
|
69
|
+
- classpath/aws-java-sdk-kms-1.11.739.jar
|
70
|
+
- classpath/aws-java-sdk-s3-1.11.739.jar
|
71
|
+
- classpath/aws-java-sdk-sts-1.11.739.jar
|
70
72
|
- classpath/commons-beanutils-1.7.0.jar
|
71
73
|
- classpath/commons-cli-1.2.jar
|
72
74
|
- classpath/commons-codec-1.11.jar
|
@@ -83,7 +85,7 @@ files:
|
|
83
85
|
- classpath/curator-client-2.7.1.jar
|
84
86
|
- classpath/curator-framework-2.7.1.jar
|
85
87
|
- classpath/curator-recipes-2.7.1.jar
|
86
|
-
- classpath/embulk-output-s3_parquet-0.
|
88
|
+
- classpath/embulk-output-s3_parquet-0.2.0.jar
|
87
89
|
- classpath/gson-2.2.4.jar
|
88
90
|
- classpath/hadoop-annotations-2.9.2.jar
|
89
91
|
- classpath/hadoop-auth-2.9.2.jar
|
@@ -99,6 +101,7 @@ files:
|
|
99
101
|
- classpath/jackson-mapper-asl-1.9.13.jar
|
100
102
|
- classpath/jackson-xc-1.8.3.jar
|
101
103
|
- classpath/java-xmlbuilder-0.4.jar
|
104
|
+
- classpath/javax.annotation-api-1.3.2.jar
|
102
105
|
- classpath/jaxb-api-2.2.2.jar
|
103
106
|
- classpath/jaxb-impl-2.2.3-1.jar
|
104
107
|
- classpath/jcip-annotations-1.0-1.jar
|
@@ -111,7 +114,7 @@ files:
|
|
111
114
|
- classpath/jetty-sslengine-6.1.26.jar
|
112
115
|
- classpath/jetty-util-6.1.26.jar
|
113
116
|
- classpath/jline-0.9.94.jar
|
114
|
-
- classpath/jmespath-java-1.11.
|
117
|
+
- classpath/jmespath-java-1.11.739.jar
|
115
118
|
- classpath/jsch-0.1.54.jar
|
116
119
|
- classpath/json-smart-2.3.jar
|
117
120
|
- classpath/jsp-api-2.1.jar
|
@@ -120,12 +123,13 @@ files:
|
|
120
123
|
- classpath/netty-3.7.0.Final.jar
|
121
124
|
- classpath/nimbus-jose-jwt-4.41.1.jar
|
122
125
|
- classpath/paranamer-2.3.jar
|
123
|
-
- classpath/parquet-column-1.
|
124
|
-
- classpath/parquet-common-1.
|
125
|
-
- classpath/parquet-encoding-1.
|
126
|
-
- classpath/parquet-format-2.
|
127
|
-
- classpath/parquet-
|
128
|
-
- classpath/parquet-
|
126
|
+
- classpath/parquet-column-1.11.0.jar
|
127
|
+
- classpath/parquet-common-1.11.0.jar
|
128
|
+
- classpath/parquet-encoding-1.11.0.jar
|
129
|
+
- classpath/parquet-format-2.7.0.jar
|
130
|
+
- classpath/parquet-format-structures-1.11.0.jar
|
131
|
+
- classpath/parquet-hadoop-1.11.0.jar
|
132
|
+
- classpath/parquet-jackson-1.11.0.jar
|
129
133
|
- classpath/protobuf-java-2.5.0.jar
|
130
134
|
- classpath/scala-library-2.13.1.jar
|
131
135
|
- classpath/servlet-api-2.5-20081211.jar
|
@@ -141,6 +145,7 @@ files:
|
|
141
145
|
- classpath/zookeeper-3.4.6.jar
|
142
146
|
- example/config.yml
|
143
147
|
- example/data.tsv
|
148
|
+
- example/prepare_s3_bucket.sh
|
144
149
|
- example/with_catalog.yml
|
145
150
|
- example/with_logicaltypes.yml
|
146
151
|
- gradle/wrapper/gradle-wrapper.jar
|
@@ -148,8 +153,10 @@ files:
|
|
148
153
|
- gradlew
|
149
154
|
- gradlew.bat
|
150
155
|
- lib/embulk/output/s3_parquet.rb
|
156
|
+
- run_s3_local.sh
|
151
157
|
- settings.gradle
|
152
158
|
- src/main/scala/org/embulk/output/s3_parquet/CatalogRegistrator.scala
|
159
|
+
- src/main/scala/org/embulk/output/s3_parquet/ContextClassLoaderSwapper.scala
|
153
160
|
- src/main/scala/org/embulk/output/s3_parquet/S3ParquetOutputPlugin.scala
|
154
161
|
- src/main/scala/org/embulk/output/s3_parquet/S3ParquetPageOutput.scala
|
155
162
|
- src/main/scala/org/embulk/output/s3_parquet/aws/Aws.scala
|