embulk-output-s3_parquet 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/release.yml +3 -0
  3. data/.github/workflows/test.yml +2 -0
  4. data/.scalafmt.conf +5 -0
  5. data/CHANGELOG.md +15 -0
  6. data/README.md +3 -2
  7. data/build.gradle +19 -9
  8. data/example/config.yml +3 -1
  9. data/example/prepare_s3_bucket.sh +6 -0
  10. data/example/with_catalog.yml +3 -1
  11. data/example/with_logicaltypes.yml +3 -1
  12. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  13. data/gradle/wrapper/gradle-wrapper.properties +1 -1
  14. data/gradlew +31 -20
  15. data/gradlew.bat +17 -1
  16. data/run_s3_local.sh +7 -0
  17. data/src/main/scala/org/embulk/output/s3_parquet/CatalogRegistrator.scala +226 -178
  18. data/src/main/scala/org/embulk/output/s3_parquet/ContextClassLoaderSwapper.scala +18 -0
  19. data/src/main/scala/org/embulk/output/s3_parquet/S3ParquetOutputPlugin.scala +293 -204
  20. data/src/main/scala/org/embulk/output/s3_parquet/S3ParquetPageOutput.scala +46 -49
  21. data/src/main/scala/org/embulk/output/s3_parquet/aws/Aws.scala +46 -50
  22. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsClientConfiguration.scala +18 -23
  23. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsCredentials.scala +146 -119
  24. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsEndpointConfiguration.scala +32 -35
  25. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsS3Configuration.scala +45 -41
  26. data/src/main/scala/org/embulk/output/s3_parquet/aws/HttpProxy.scala +40 -43
  27. data/src/main/scala/org/embulk/output/s3_parquet/parquet/EmbulkMessageType.scala +138 -92
  28. data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandler.scala +117 -102
  29. data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandlerStore.scala +91 -84
  30. data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriteSupport.scala +30 -29
  31. data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriter.scala +143 -152
  32. data/src/test/scala/org/embulk/output/s3_parquet/TestS3ParquetOutputPlugin.scala +144 -117
  33. data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandler.scala +72 -66
  34. data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandlerStore.scala +149 -132
  35. metadata +22 -15
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-s3_parquet
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-11-17 00:00:00.000000000 Z
11
+ date: 2020-03-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,6 +49,7 @@ files:
49
49
  - ".github/workflows/release.yml"
50
50
  - ".github/workflows/test.yml"
51
51
  - ".gitignore"
52
+ - ".scalafmt.conf"
52
53
  - CHANGELOG.md
53
54
  - LICENSE.txt
54
55
  - README.md
@@ -61,12 +62,13 @@ files:
61
62
  - classpath/api-util-1.0.0-M20.jar
62
63
  - classpath/asm-3.1.jar
63
64
  - classpath/asm-5.0.4.jar
65
+ - classpath/audience-annotations-0.11.0.jar
64
66
  - classpath/avro-1.7.7.jar
65
- - classpath/aws-java-sdk-core-1.11.676.jar
66
- - classpath/aws-java-sdk-glue-1.11.676.jar
67
- - classpath/aws-java-sdk-kms-1.11.676.jar
68
- - classpath/aws-java-sdk-s3-1.11.676.jar
69
- - classpath/aws-java-sdk-sts-1.11.676.jar
67
+ - classpath/aws-java-sdk-core-1.11.739.jar
68
+ - classpath/aws-java-sdk-glue-1.11.739.jar
69
+ - classpath/aws-java-sdk-kms-1.11.739.jar
70
+ - classpath/aws-java-sdk-s3-1.11.739.jar
71
+ - classpath/aws-java-sdk-sts-1.11.739.jar
70
72
  - classpath/commons-beanutils-1.7.0.jar
71
73
  - classpath/commons-cli-1.2.jar
72
74
  - classpath/commons-codec-1.11.jar
@@ -83,7 +85,7 @@ files:
83
85
  - classpath/curator-client-2.7.1.jar
84
86
  - classpath/curator-framework-2.7.1.jar
85
87
  - classpath/curator-recipes-2.7.1.jar
86
- - classpath/embulk-output-s3_parquet-0.1.0.jar
88
+ - classpath/embulk-output-s3_parquet-0.2.0.jar
87
89
  - classpath/gson-2.2.4.jar
88
90
  - classpath/hadoop-annotations-2.9.2.jar
89
91
  - classpath/hadoop-auth-2.9.2.jar
@@ -99,6 +101,7 @@ files:
99
101
  - classpath/jackson-mapper-asl-1.9.13.jar
100
102
  - classpath/jackson-xc-1.8.3.jar
101
103
  - classpath/java-xmlbuilder-0.4.jar
104
+ - classpath/javax.annotation-api-1.3.2.jar
102
105
  - classpath/jaxb-api-2.2.2.jar
103
106
  - classpath/jaxb-impl-2.2.3-1.jar
104
107
  - classpath/jcip-annotations-1.0-1.jar
@@ -111,7 +114,7 @@ files:
111
114
  - classpath/jetty-sslengine-6.1.26.jar
112
115
  - classpath/jetty-util-6.1.26.jar
113
116
  - classpath/jline-0.9.94.jar
114
- - classpath/jmespath-java-1.11.676.jar
117
+ - classpath/jmespath-java-1.11.739.jar
115
118
  - classpath/jsch-0.1.54.jar
116
119
  - classpath/json-smart-2.3.jar
117
120
  - classpath/jsp-api-2.1.jar
@@ -120,12 +123,13 @@ files:
120
123
  - classpath/netty-3.7.0.Final.jar
121
124
  - classpath/nimbus-jose-jwt-4.41.1.jar
122
125
  - classpath/paranamer-2.3.jar
123
- - classpath/parquet-column-1.10.1.jar
124
- - classpath/parquet-common-1.10.1.jar
125
- - classpath/parquet-encoding-1.10.1.jar
126
- - classpath/parquet-format-2.4.0.jar
127
- - classpath/parquet-hadoop-1.10.1.jar
128
- - classpath/parquet-jackson-1.10.1.jar
126
+ - classpath/parquet-column-1.11.0.jar
127
+ - classpath/parquet-common-1.11.0.jar
128
+ - classpath/parquet-encoding-1.11.0.jar
129
+ - classpath/parquet-format-2.7.0.jar
130
+ - classpath/parquet-format-structures-1.11.0.jar
131
+ - classpath/parquet-hadoop-1.11.0.jar
132
+ - classpath/parquet-jackson-1.11.0.jar
129
133
  - classpath/protobuf-java-2.5.0.jar
130
134
  - classpath/scala-library-2.13.1.jar
131
135
  - classpath/servlet-api-2.5-20081211.jar
@@ -141,6 +145,7 @@ files:
141
145
  - classpath/zookeeper-3.4.6.jar
142
146
  - example/config.yml
143
147
  - example/data.tsv
148
+ - example/prepare_s3_bucket.sh
144
149
  - example/with_catalog.yml
145
150
  - example/with_logicaltypes.yml
146
151
  - gradle/wrapper/gradle-wrapper.jar
@@ -148,8 +153,10 @@ files:
148
153
  - gradlew
149
154
  - gradlew.bat
150
155
  - lib/embulk/output/s3_parquet.rb
156
+ - run_s3_local.sh
151
157
  - settings.gradle
152
158
  - src/main/scala/org/embulk/output/s3_parquet/CatalogRegistrator.scala
159
+ - src/main/scala/org/embulk/output/s3_parquet/ContextClassLoaderSwapper.scala
153
160
  - src/main/scala/org/embulk/output/s3_parquet/S3ParquetOutputPlugin.scala
154
161
  - src/main/scala/org/embulk/output/s3_parquet/S3ParquetPageOutput.scala
155
162
  - src/main/scala/org/embulk/output/s3_parquet/aws/Aws.scala