embulk-output-orc 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c6d1115e79d4012717df2f43f447bfbdd1518a88
-  data.tar.gz: 80e8e7f5e470724a33125b4ecce74112ef098d6b
+  metadata.gz: 3c842edfe45c7e992faae16afd3331c7f8ecf256
+  data.tar.gz: a6b3d098e7b012a07f4870e2fc92c898b09e1560
 SHA512:
-  metadata.gz: 290aec04feda06d83ddd9cd2995d62a186fbfc3399b90ede431e54eec8e34aac606077919610f446e703ee820e7625ed1d79f9d08378fccd6fcb052f15e8ab31
-  data.tar.gz: 899435e450c217c4f8b08fa9ea617efb64b5628ba0cb42b37849a11e5363b96889b67e595b51f2f3b2b396386166d5a5ddd96fd34cfa761f735572262b59f716
+  metadata.gz: 3bf5bc9e310496191419ee1e9a76cf9912321e9df2d178081c324ed46f83e7600e04912df59a7ebf38e0e3761be4b8ea1931cae549d681a8ba5b6d35b1e19990
+  data.tar.gz: bbec4349adf56b4c684084a39f61eee8dd41152a11de02c2b5573e5e933cf8d76f280e7110f030cb31b157fdfc9e7fa457f575cd8a6b5b4d8df8ea484a51f358

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.0.1"
+version = "0.0.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/example/example.yml

@@ -54,3 +54,6 @@ exec:
 out:
   type: orc
   path_prefix: "/tmp/output"
+  buffer_size: 8000
+  strip_size:  90000
+  compression_kind: ZLIB

data/src/main/java/org/embulk/output/orc/OrcColumnVisitor.java

@@ -3,66 +3,82 @@ package org.embulk.output.orc;
 import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
+import org.embulk.spi.time.Timestamp;
 
-public class OrcColumnVisitor implements ColumnVisitor
+public class OrcColumnVisitor
+        implements ColumnVisitor
 {
     private PageReader reader;
-    VectorizedRowBatch batch;
-    Integer finalI;
+    private VectorizedRowBatch batch;
+    private Integer i;
 
-    public OrcColumnVisitor(PageReader pageReader, VectorizedRowBatch rowBatch, Page page, Integer i)
+    public OrcColumnVisitor(PageReader pageReader, VectorizedRowBatch rowBatch, Integer i)
     {
-        int size = page.getStringReferences().size();
-
         this.reader = pageReader;
         this.batch = rowBatch;
-        this.finalI = i;
+        this.i = i;
     }
 
     @Override
     public void booleanColumn(Column column)
     {
         if (reader.isNull(column)) {
-            ((LongColumnVector) batch.cols[column.getIndex()]).vector[finalI] = 0;
+            ((LongColumnVector) batch.cols[column.getIndex()]).vector[i] = 0;
         }
         else {
-            ((LongColumnVector) batch.cols[column.getIndex()]).vector[finalI] = reader.getLong(column);
+            // TODO; Fix all true bug
+            if (reader.getBoolean(column)) {
+                ((LongColumnVector) batch.cols[column.getIndex()]).vector[i] = 1;
+            }
+            else {
+                ((LongColumnVector) batch.cols[column.getIndex()]).vector[i] = 0;
+            }
         }
     }
 
     @Override
     public void longColumn(Column column)
     {
-        ((LongColumnVector) batch.cols[column.getIndex()]).vector[finalI] = reader.getLong(column);
+        ((LongColumnVector) batch.cols[column.getIndex()]).vector[i] = reader.getLong(column);
     }
 
     @Override
     public void doubleColumn(Column column)
     {
-        ((DoubleColumnVector) batch.cols[column.getIndex()]).vector[finalI] = reader.getDouble(column);
+        ((DoubleColumnVector) batch.cols[column.getIndex()]).vector[i] = reader.getDouble(column);
     }
 
     @Override
     public void stringColumn(Column column)
     {
-        ((BytesColumnVector) batch.cols[column.getIndex()]).setVal(finalI,
+        ((BytesColumnVector) batch.cols[column.getIndex()]).setVal(i,
                 reader.getString(column).getBytes());
     }
 
     @Override
     public void timestampColumn(Column column)
     {
-
+        if (reader.isNull(column)) {
+            ((TimestampColumnVector) batch.cols[column.getIndex()]).setNullValue(i);
+        }
+        else {
+            Timestamp timestamp = reader.getTimestamp(column);
+            if (!timestamp.equals("")) {
+                java.sql.Timestamp ts = new java.sql.Timestamp(timestamp.getEpochSecond() * 1000);
+                ((TimestampColumnVector) batch.cols[column.getIndex()]).set(i, ts);
+            }
+            // throw new UnsupportedOperationException("orc output plugin does not support timestamp yet");
+        }
     }
 
     @Override
     public void jsonColumn(Column column)
     {
-        // throw unsupported
+        throw new UnsupportedOperationException("orc output plugin does not support json type");
     }
 }

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -6,10 +6,6 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.orc.CompressionKind;
@@ -24,14 +20,12 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.util.Timestamps;
@@ -64,6 +58,19 @@ public class OrcOutputPlugin
         @ConfigDefault("\".%03d\"")
         String getSequenceFormat();
 
+        // ORC File options
+        @Config("strip_size")
+        @ConfigDefault("100000")
+        Integer getStripSize();
+
+        @Config("buffer_size")
+        @ConfigDefault("10000")
+        Integer getBufferSize();
+
+        @Config("compression_kind")
+        @ConfigDefault("ZLIB")
+        public String getCompressionKind();
+
         @Config("overwrite")
         @ConfigDefault("false")
         boolean getOverwrite();
@@ -191,12 +198,12 @@ public class OrcOutputPlugin
 
         Writer writer = null;
         try {
+            // Make writerOptions
+            OrcFile.WriterOptions writerOptions = createWriterOptions(task, conf);
             // see: https://stackoverflow.com/questions/9256733/how-to-connect-hive-in-ireport
             // see: https://community.hortonworks.com/content/kbentry/73458/connecting-dbvisualizer-and-datagrip-to-hive-with.html
             writer = OrcFile.createWriter(new Path(buildPath(task, processorIndex)),
-                    OrcFile.writerOptions(conf)
-                            .setSchema(oschema)
-                            .compress(CompressionKind.ZLIB)
+                    writerOptions.setSchema(oschema)
                             .version(OrcFile.Version.V_0_12));
         }
         catch (IOException e) {
@@ -205,6 +212,35 @@ public class OrcOutputPlugin
         return writer;
     }
 
+    private OrcFile.WriterOptions createWriterOptions(PluginTask task, Configuration conf)
+    {
+        final Integer bufferSize = task.getBufferSize();
+        final Integer stripSize = task.getStripSize();
+        final String kindString = task.getCompressionKind();
+        CompressionKind kind;
+        switch (kindString) {
+            case "ZLIB":
+                kind = CompressionKind.ZLIB;
+                break;
+            case "SNAPPY":
+                kind = CompressionKind.SNAPPY;
+                break;
+            case "LZO":
+                kind = CompressionKind.LZO;
+                break;
+            case "LZ4":
+                kind = CompressionKind.LZ4;
+                break;
+            default:
+                kind = CompressionKind.NONE;
+                break;
+        }
+        return OrcFile.writerOptions(conf).
+                bufferSize(bufferSize)
+                .stripeSize(stripSize)
+                .compress(kind);
+    }
+
     class OrcTransactionalPageOutput
             implements TransactionalPageOutput
     {
@@ -225,78 +261,18 @@ public class OrcOutputPlugin
         @Override
         public void add(Page page)
         {
-            List<String> strings = page.getStringReferences();
+            int size = page.getStringReferences().size();
             TypeDescription schema = getSchema(reader.getSchema());
             VectorizedRowBatch batch = schema.createRowBatch();
-            batch.size = strings.size();
+            batch.size = size;
 
             reader.setPage(page);
             int i = 0;
             while (reader.nextRecord()) {
                 // batch.size = page.getStringReferences().size();
-                final int finalI = i;
-
-                reader.getSchema().visitColumns(new ColumnVisitor()
-                {
-
-                    @Override
-                    public void booleanColumn(Column column)
-                    {
-                        if (reader.isNull(column)) {
-                            ((LongColumnVector) batch.cols[column.getIndex()]).vector[finalI] = 0;
-                        }
-                        else {
-                            // TODO; Fix all true bug
-                            if (reader.getBoolean(column)) {
-                                ((LongColumnVector) batch.cols[column.getIndex()]).vector[finalI] = 1;
-                            }
-                            else {
-                                ((LongColumnVector) batch.cols[column.getIndex()]).vector[finalI] = 0;
-                            }
-                        }
-                    }
-
-                    @Override
-                    public void longColumn(Column column)
-                    {
-                        ((LongColumnVector) batch.cols[column.getIndex()]).vector[finalI] = reader.getLong(column);
-                    }
-
-                    @Override
-                    public void doubleColumn(Column column)
-                    {
-                        ((DoubleColumnVector) batch.cols[column.getIndex()]).vector[finalI] = reader.getDouble(column);
-                    }
-
-                    @Override
-                    public void stringColumn(Column column)
-                    {
-                        ((BytesColumnVector) batch.cols[column.getIndex()]).setVal(finalI,
-                                reader.getString(column).getBytes());
-                    }
-
-                    @Override
-                    public void timestampColumn(Column column)
-                    {
-                        if (reader.isNull(column)) {
-                            ((TimestampColumnVector) batch.cols[column.getIndex()]).setNullValue(finalI);
-                        }
-                        else {
-                            Timestamp timestamp = reader.getTimestamp(column);
-                            if (!timestamp.equals("")) {
-                                java.sql.Timestamp ts = new java.sql.Timestamp(timestamp.getEpochSecond() * 1000);
-                                ((TimestampColumnVector) batch.cols[column.getIndex()]).set(finalI, ts);
-                            }
-                            // throw new UnsupportedOperationException("orc output plugin does not support timestamp yet");
-                        }
-                    }
-
-                    @Override
-                    public void jsonColumn(Column column)
-                    {
-                        throw new UnsupportedOperationException("orc output plugin does not support json type");
-                    }
-                });
+                reader.getSchema().visitColumns(
+                        new OrcColumnVisitor(reader, batch, i)
+                );
                 i++;
             }
             try {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - yuokada
@@ -85,7 +85,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-output-orc-0.0.1.jar
+- classpath/embulk-output-orc-0.0.2.jar
 - classpath/gson-2.2.4.jar
 - classpath/guice-servlet-3.0.jar
 - classpath/hadoop-annotations-2.6.4.jar