embulk-output-orc 0.2.2 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 041e79d159d0ffe346b0c28f17c7009438bc65e5
-  data.tar.gz: 13d96f705101ce32a4d389c1539ea23c50db3872
+  metadata.gz: 018f6f65a5d6949886d5d0e3b5758befee6d40bf
+  data.tar.gz: f5cf2c9745105300c5f15031f4eaa787ebf3b7b3
 SHA512:
-  metadata.gz: fdd305bfb25dfd998f0c49e55ef4ffb916f268fa5189678102b3f6e7228532da4d3ea141afe03491a2cf11bbe394975f73868a7620ab0d410306ab998d1f0d95
-  data.tar.gz: 88574a4b9e2982b80c93307c361ec1de26c0878d8f08acfc0333eb770df74a7e76ef38747b94a57c351d4cd42e6fce015d141ef8d94e5f47bc6ecab1eeca085e
+  metadata.gz: 7da18070bcf26399ea3835c0336f922518f15e9f1e4fe27f6da32e0a7254234e40f70fbce934b7f3f4dfcef06af08687e20a7ca58b82b1ed16edf1d762878c7d
+  data.tar.gz: 93224c587d49b5d758d5d311c8ba88efd45340f0d4d5a0e7f9cdd895b39160358c339ae761e83f1fb7e5aa941b5369cc69e77c45688d92a52dc24a0d1dc9ffe1

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.2.2"
+version = "0.2.4"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.output.orc;
 
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalFileSystem;
@@ -12,11 +11,8 @@ import org.apache.orc.CompressionKind;
 import org.apache.orc.OrcFile;
 import org.apache.orc.TypeDescription;
 import org.apache.orc.Writer;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
@@ -30,72 +26,13 @@ import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.util.Timestamps;
 import org.embulk.util.aws.credentials.AwsCredentials;
-import org.embulk.util.aws.credentials.AwsCredentialsTask;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 public class OrcOutputPlugin
         implements OutputPlugin
 {
-    public interface PluginTask
-            extends Task, TimestampFormatter.Task, AwsCredentialsTask
-    {
-        @Config("path_prefix")
-        String getPathPrefix();
-
-        @Config("file_ext")
-        @ConfigDefault("\".orc\"")
-        String getFileNameExtension();
-
-        @Config("column_options")
-        @ConfigDefault("{}")
-        Map<String, TimestampColumnOption> getColumnOptions();
-
-        @Config("sequence_format")
-        @ConfigDefault("\".%03d\"")
-        String getSequenceFormat();
-
-        // ORC File options
-        @Config("strip_size")
-        @ConfigDefault("100000")
-        Integer getStripSize();
-
-        @Config("buffer_size")
-        @ConfigDefault("10000")
-        Integer getBufferSize();
-
-        @Config("compression_kind")
-        @ConfigDefault("ZLIB")
-        public String getCompressionKind();
-
-        @Config("overwrite")
-        @ConfigDefault("false")
-        boolean getOverwrite();
-
-        @Config("default_from_timezone")
-        @ConfigDefault("\"UTC\"")
-        DateTimeZone getDefaultFromTimeZone();
-
-        @Config("endpoint")
-        @ConfigDefault("null")
-        Optional<String> getEndpoint();
-    }
-
-    public interface TimestampColumnOption
-            extends Task, TimestampFormatter.TimestampColumnOption
-    {
-        @Config("from_timezone")
-        @ConfigDefault("null")
-        Optional<DateTimeZone> getFromTimeZone();
-
-        @Config("from_format")
-        @ConfigDefault("null")
-        Optional<List<String>> getFromFormat();
-    }
 
     @Override
     public ConfigDiff transaction(ConfigSource config,
@@ -237,24 +174,7 @@ public class OrcOutputPlugin
         final Integer bufferSize = task.getBufferSize();
         final Integer stripSize = task.getStripSize();
         final String kindString = task.getCompressionKind();
-        CompressionKind kind;
-        switch (kindString) {
-            case "ZLIB":
-                kind = CompressionKind.ZLIB;
-                break;
-            case "SNAPPY":
-                kind = CompressionKind.SNAPPY;
-                break;
-            case "LZO":
-                kind = CompressionKind.LZO;
-                break;
-            case "LZ4":
-                kind = CompressionKind.LZ4;
-                break;
-            default:
-                kind = CompressionKind.NONE;
-                break;
-        }
+        CompressionKind kind = CompressionKind.valueOf(kindString);
         return OrcFile.writerOptions(conf).
                 bufferSize(bufferSize)
                 .stripeSize(stripSize)
@@ -266,7 +186,6 @@ public class OrcOutputPlugin
     {
         private final PageReader reader;
         private final Writer writer;
-        private final ArrayList<VectorizedRowBatch> rowBatches = new ArrayList<>();
 
         public OrcTransactionalPageOutput(PageReader reader, Writer writer, PluginTask task)
         {
@@ -290,8 +209,12 @@ public class OrcOutputPlugin
                 );
                 i++;
             }
-            synchronized (this) {
-                rowBatches.add(batch);
+            try {
+                writer.addRowBatch(batch);
+                batch.reset();
+            }
+            catch (IOException e) {
+                e.printStackTrace();
             }
         }
 
@@ -299,9 +222,6 @@ public class OrcOutputPlugin
         public void finish()
         {
             try {
-                for (VectorizedRowBatch batch : rowBatches) {
-                    writer.addRowBatch(batch);
-                }
                 writer.close();
             }
             catch (IOException e) {

data/src/main/java/org/embulk/output/orc/PluginTask.java

@@ -0,0 +1,55 @@
+package org.embulk.output.orc;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.util.aws.credentials.AwsCredentialsTask;
+import org.joda.time.DateTimeZone;
+
+import java.util.Map;
+
+public interface PluginTask
+        extends Task, TimestampFormatter.Task, AwsCredentialsTask
+{
+    @Config("path_prefix")
+    String getPathPrefix();
+
+    @Config("file_ext")
+    @ConfigDefault("\".orc\"")
+    String getFileNameExtension();
+
+    @Config("column_options")
+    @ConfigDefault("{}")
+    Map<String, TimestampColumnOption> getColumnOptions();
+
+    @Config("sequence_format")
+    @ConfigDefault("\".%03d\"")
+    String getSequenceFormat();
+
+    // ORC File options
+    @Config("strip_size")
+    @ConfigDefault("100000")
+    Integer getStripSize();
+
+    @Config("buffer_size")
+    @ConfigDefault("10000")
+    Integer getBufferSize();
+
+    @Config("compression_kind")
+    @ConfigDefault("ZLIB")
+    public String getCompressionKind();
+
+    @Config("overwrite")
+    @ConfigDefault("false")
+    boolean getOverwrite();
+
+    @Config("default_from_timezone")
+    @ConfigDefault("\"UTC\"")
+    DateTimeZone getDefaultFromTimeZone();
+
+    @Config("endpoint")
+    @ConfigDefault("null")
+    Optional<String> getEndpoint();
+}

data/src/main/java/org/embulk/output/orc/TimestampColumnOption.java

@@ -0,0 +1,22 @@
+package org.embulk.output.orc;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.spi.time.TimestampFormatter;
+import org.joda.time.DateTimeZone;
+
+import java.util.List;
+
+public interface TimestampColumnOption
+        extends Task, TimestampFormatter.TimestampColumnOption
+{
+    @Config("from_timezone")
+    @ConfigDefault("null")
+    Optional<DateTimeZone> getFromTimeZone();
+
+    @Config("from_format")
+    @ConfigDefault("null")
+    Optional<List<String>> getFromFormat();
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-17 00:00:00.000000000 Z
+date: 2017-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,10 +58,11 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/output/orc.rb
-- src/main/java/org/embulk/output/orc/OrcCodec.java
 - src/main/java/org/embulk/output/orc/OrcColumnVisitor.java
 - src/main/java/org/embulk/output/orc/OrcOutputPlugin.java
 - src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java
+- src/main/java/org/embulk/output/orc/PluginTask.java
+- src/main/java/org/embulk/output/orc/TimestampColumnOption.java
 - src/test/java/org/embulk/output/orc/TestOrcOutputPlugin.java
 - classpath/activation-1.1.jar
 - classpath/aircompressor-0.3.jar
@@ -93,7 +94,7 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-orc-0.2.2.jar
+- classpath/embulk-output-orc-0.2.4.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.7.3.jar

data/src/main/java/org/embulk/output/orc/OrcCodec.java

@@ -1,21 +0,0 @@
-package org.embulk.output.orc;
-
-public enum OrcCodec
-{
-    ZLIB("zlib"),
-    SNAPPY("snappy"),
-    LZO("lzo"),
-    LZ4("lz4"),
-    NONE("none"),;
-    String kind;
-
-    OrcCodec(String kind)
-    {
-        this.kind = kind;
-    }
-
-    public String getKind()
-    {
-        return kind;
-    }
-}