embulk-output-orc 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 018f6f65a5d6949886d5d0e3b5758befee6d40bf
-  data.tar.gz: f5cf2c9745105300c5f15031f4eaa787ebf3b7b3
+  metadata.gz: 5bf0784f61bbc808d36ebce5e46aaab889b891a3
+  data.tar.gz: 8937c475721a4f2c347575580982ce2a772f8d63
 SHA512:
-  metadata.gz: 7da18070bcf26399ea3835c0336f922518f15e9f1e4fe27f6da32e0a7254234e40f70fbce934b7f3f4dfcef06af08687e20a7ca58b82b1ed16edf1d762878c7d
-  data.tar.gz: 93224c587d49b5d758d5d311c8ba88efd45340f0d4d5a0e7f9cdd895b39160358c339ae761e83f1fb7e5aa941b5369cc69e77c45688d92a52dc24a0d1dc9ffe1
+  metadata.gz: 23a1a87ca07df8ebc6d17575a3abcf58cf9c7eb5cd6569ba62cfd7fa3cb52c42cc27a00e65f99d821c396e897ebaac78c3dc5dfe9ee6a750049c2017f08d9fa5
+  data.tar.gz: 0c08613e8c5182987a4bbb03ae3a0ce9eddb474a1b8672aa5fbc25e69da4ea0784a9982c6bc3888263eddb68213ec885eb3bb2000aaf187cb94148fa593a780d

data/README.md

@@ -16,8 +16,9 @@
   - support: `file`, `s3n` and `s3a`.
 - **file_ext**: An extension of output file. (string, default: `.orc`)
 - **sequence_format**: (string, default: `.%03d`)
-- **buffer_size**: Set the ORC buffer size (integer, default: `10000`)
-- **strip_size**: Set the ORC strip size (integer,  default: `100000`)
+- **buffer_size**: Set the ORC buffer size (integer, default: `262144`)
+- **strip_size**: Set the ORC strip size (integer,  default: `67108864`)
+- **block_size**: Set the ORC block size (integer, default: `268435456`)
 - **compression_kind**: description (string, default: `'ZLIB'`)
     - `NONE`, `ZLIB`, `SNAPPY`
 - **overwrite**: (LocalFileSystem only) Overwrite if output files already exist. (boolean, default: `false`)

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.2.4"
+version = "0.3.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -33,7 +33,6 @@ import java.util.List;
 public class OrcOutputPlugin
         implements OutputPlugin
 {
-
     @Override
     public ConfigDiff transaction(ConfigSource config,
             Schema schema, int taskCount,
@@ -173,10 +172,12 @@ public class OrcOutputPlugin
     {
         final Integer bufferSize = task.getBufferSize();
         final Integer stripSize = task.getStripSize();
+        final Integer blockSize = task.getBlockSize();
         final String kindString = task.getCompressionKind();
         CompressionKind kind = CompressionKind.valueOf(kindString);
-        return OrcFile.writerOptions(conf).
-                bufferSize(bufferSize)
+        return OrcFile.writerOptions(conf)
+                .bufferSize(bufferSize)
+                .blockSize(blockSize)
                 .stripeSize(stripSize)
                 .compress(kind);
     }

data/src/main/java/org/embulk/output/orc/PluginTask.java

@@ -28,15 +28,20 @@ public interface PluginTask
     @ConfigDefault("\".%03d\"")
     String getSequenceFormat();
 
+    // see: https://orc.apache.org/docs/hive-config.html
     // ORC File options
     @Config("strip_size")
-    @ConfigDefault("100000")
+    @ConfigDefault("67108864") // 64MB
     Integer getStripSize();
 
     @Config("buffer_size")
-    @ConfigDefault("10000")
+    @ConfigDefault("262144") // 256KB
     Integer getBufferSize();
 
+    @Config("block_size")
+    @ConfigDefault("268435456") // 256MB
+    Integer getBlockSize();
+
     @Config("compression_kind")
     @ConfigDefault("ZLIB")
     public String getCompressionKind();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-02 00:00:00.000000000 Z
+date: 2017-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +94,7 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-orc-0.2.4.jar
+- classpath/embulk-output-orc-0.3.0.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.7.3.jar