embulk-output-s3_parquet 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/release.yml +3 -0
  3. data/.github/workflows/test.yml +2 -0
  4. data/.scalafmt.conf +5 -0
  5. data/CHANGELOG.md +15 -0
  6. data/README.md +3 -2
  7. data/build.gradle +19 -9
  8. data/example/config.yml +3 -1
  9. data/example/prepare_s3_bucket.sh +6 -0
  10. data/example/with_catalog.yml +3 -1
  11. data/example/with_logicaltypes.yml +3 -1
  12. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  13. data/gradle/wrapper/gradle-wrapper.properties +1 -1
  14. data/gradlew +31 -20
  15. data/gradlew.bat +17 -1
  16. data/run_s3_local.sh +7 -0
  17. data/src/main/scala/org/embulk/output/s3_parquet/CatalogRegistrator.scala +226 -178
  18. data/src/main/scala/org/embulk/output/s3_parquet/ContextClassLoaderSwapper.scala +18 -0
  19. data/src/main/scala/org/embulk/output/s3_parquet/S3ParquetOutputPlugin.scala +293 -204
  20. data/src/main/scala/org/embulk/output/s3_parquet/S3ParquetPageOutput.scala +46 -49
  21. data/src/main/scala/org/embulk/output/s3_parquet/aws/Aws.scala +46 -50
  22. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsClientConfiguration.scala +18 -23
  23. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsCredentials.scala +146 -119
  24. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsEndpointConfiguration.scala +32 -35
  25. data/src/main/scala/org/embulk/output/s3_parquet/aws/AwsS3Configuration.scala +45 -41
  26. data/src/main/scala/org/embulk/output/s3_parquet/aws/HttpProxy.scala +40 -43
  27. data/src/main/scala/org/embulk/output/s3_parquet/parquet/EmbulkMessageType.scala +138 -92
  28. data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandler.scala +117 -102
  29. data/src/main/scala/org/embulk/output/s3_parquet/parquet/LogicalTypeHandlerStore.scala +91 -84
  30. data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriteSupport.scala +30 -29
  31. data/src/main/scala/org/embulk/output/s3_parquet/parquet/ParquetFileWriter.scala +143 -152
  32. data/src/test/scala/org/embulk/output/s3_parquet/TestS3ParquetOutputPlugin.scala +144 -117
  33. data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandler.scala +72 -66
  34. data/src/test/scala/org/embulk/output/s3_parquet/parquet/TestLogicalTypeHandlerStore.scala +149 -132
  35. metadata +22 -15
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-s3_parquet
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-11-17 00:00:00.000000000 Z
11
+ date: 2020-03-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,6 +49,7 @@ files:
49
49
  - ".github/workflows/release.yml"
50
50
  - ".github/workflows/test.yml"
51
51
  - ".gitignore"
52
+ - ".scalafmt.conf"
52
53
  - CHANGELOG.md
53
54
  - LICENSE.txt
54
55
  - README.md
@@ -61,12 +62,13 @@ files:
61
62
  - classpath/api-util-1.0.0-M20.jar
62
63
  - classpath/asm-3.1.jar
63
64
  - classpath/asm-5.0.4.jar
65
+ - classpath/audience-annotations-0.11.0.jar
64
66
  - classpath/avro-1.7.7.jar
65
- - classpath/aws-java-sdk-core-1.11.676.jar
66
- - classpath/aws-java-sdk-glue-1.11.676.jar
67
- - classpath/aws-java-sdk-kms-1.11.676.jar
68
- - classpath/aws-java-sdk-s3-1.11.676.jar
69
- - classpath/aws-java-sdk-sts-1.11.676.jar
67
+ - classpath/aws-java-sdk-core-1.11.739.jar
68
+ - classpath/aws-java-sdk-glue-1.11.739.jar
69
+ - classpath/aws-java-sdk-kms-1.11.739.jar
70
+ - classpath/aws-java-sdk-s3-1.11.739.jar
71
+ - classpath/aws-java-sdk-sts-1.11.739.jar
70
72
  - classpath/commons-beanutils-1.7.0.jar
71
73
  - classpath/commons-cli-1.2.jar
72
74
  - classpath/commons-codec-1.11.jar
@@ -83,7 +85,7 @@ files:
83
85
  - classpath/curator-client-2.7.1.jar
84
86
  - classpath/curator-framework-2.7.1.jar
85
87
  - classpath/curator-recipes-2.7.1.jar
86
- - classpath/embulk-output-s3_parquet-0.1.0.jar
88
+ - classpath/embulk-output-s3_parquet-0.2.0.jar
87
89
  - classpath/gson-2.2.4.jar
88
90
  - classpath/hadoop-annotations-2.9.2.jar
89
91
  - classpath/hadoop-auth-2.9.2.jar
@@ -99,6 +101,7 @@ files:
99
101
  - classpath/jackson-mapper-asl-1.9.13.jar
100
102
  - classpath/jackson-xc-1.8.3.jar
101
103
  - classpath/java-xmlbuilder-0.4.jar
104
+ - classpath/javax.annotation-api-1.3.2.jar
102
105
  - classpath/jaxb-api-2.2.2.jar
103
106
  - classpath/jaxb-impl-2.2.3-1.jar
104
107
  - classpath/jcip-annotations-1.0-1.jar
@@ -111,7 +114,7 @@ files:
111
114
  - classpath/jetty-sslengine-6.1.26.jar
112
115
  - classpath/jetty-util-6.1.26.jar
113
116
  - classpath/jline-0.9.94.jar
114
- - classpath/jmespath-java-1.11.676.jar
117
+ - classpath/jmespath-java-1.11.739.jar
115
118
  - classpath/jsch-0.1.54.jar
116
119
  - classpath/json-smart-2.3.jar
117
120
  - classpath/jsp-api-2.1.jar
@@ -120,12 +123,13 @@ files:
120
123
  - classpath/netty-3.7.0.Final.jar
121
124
  - classpath/nimbus-jose-jwt-4.41.1.jar
122
125
  - classpath/paranamer-2.3.jar
123
- - classpath/parquet-column-1.10.1.jar
124
- - classpath/parquet-common-1.10.1.jar
125
- - classpath/parquet-encoding-1.10.1.jar
126
- - classpath/parquet-format-2.4.0.jar
127
- - classpath/parquet-hadoop-1.10.1.jar
128
- - classpath/parquet-jackson-1.10.1.jar
126
+ - classpath/parquet-column-1.11.0.jar
127
+ - classpath/parquet-common-1.11.0.jar
128
+ - classpath/parquet-encoding-1.11.0.jar
129
+ - classpath/parquet-format-2.7.0.jar
130
+ - classpath/parquet-format-structures-1.11.0.jar
131
+ - classpath/parquet-hadoop-1.11.0.jar
132
+ - classpath/parquet-jackson-1.11.0.jar
129
133
  - classpath/protobuf-java-2.5.0.jar
130
134
  - classpath/scala-library-2.13.1.jar
131
135
  - classpath/servlet-api-2.5-20081211.jar
@@ -141,6 +145,7 @@ files:
141
145
  - classpath/zookeeper-3.4.6.jar
142
146
  - example/config.yml
143
147
  - example/data.tsv
148
+ - example/prepare_s3_bucket.sh
144
149
  - example/with_catalog.yml
145
150
  - example/with_logicaltypes.yml
146
151
  - gradle/wrapper/gradle-wrapper.jar
@@ -148,8 +153,10 @@ files:
148
153
  - gradlew
149
154
  - gradlew.bat
150
155
  - lib/embulk/output/s3_parquet.rb
156
+ - run_s3_local.sh
151
157
  - settings.gradle
152
158
  - src/main/scala/org/embulk/output/s3_parquet/CatalogRegistrator.scala
159
+ - src/main/scala/org/embulk/output/s3_parquet/ContextClassLoaderSwapper.scala
153
160
  - src/main/scala/org/embulk/output/s3_parquet/S3ParquetOutputPlugin.scala
154
161
  - src/main/scala/org/embulk/output/s3_parquet/S3ParquetPageOutput.scala
155
162
  - src/main/scala/org/embulk/output/s3_parquet/aws/Aws.scala