embulk-output-orc 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bf0784f61bbc808d36ebce5e46aaab889b891a3
-  data.tar.gz: 8937c475721a4f2c347575580982ce2a772f8d63
+  metadata.gz: 8b6323f654a352230f551c23017a85e9ceee97ef
+  data.tar.gz: 400429eadfbb55ddaaaa2e954bddb54306a31e78
 SHA512:
-  metadata.gz: 23a1a87ca07df8ebc6d17575a3abcf58cf9c7eb5cd6569ba62cfd7fa3cb52c42cc27a00e65f99d821c396e897ebaac78c3dc5dfe9ee6a750049c2017f08d9fa5
-  data.tar.gz: 0c08613e8c5182987a4bbb03ae3a0ce9eddb474a1b8672aa5fbc25e69da4ea0784a9982c6bc3888263eddb68213ec885eb3bb2000aaf187cb94148fa593a780d
+  metadata.gz: 8359917c8f9b429189faec83b51aa3fee3abfbfc74130eef9b83489b46e8be4ff40a2ca273c7c17a6dd770bb329cb25195c1e3c85bf103b46acee5b952c7cc66
+  data.tar.gz: 3910abc4803eec8a36fc275229a4ff3e5b0f14c4fbf1bdea7071cd49c62b2782025acef0d485987509ea15723713a822abc3a306e570e76c57df2c951be22925

data/README.md

@@ -36,14 +36,19 @@
 out:
   type: orc
   path_prefix: "/tmp/output"
-  buffer_size: 8000
-  strip_size:  90000
   compression_kind: ZLIB
   overwrite:   true
 ```
 
 ## ChangeLog
 
+### ver 0.3.0
+
+- Change default value : (block_size, buffer_size, strip_size)
+
+    - default value is Hive's default value.  
+      (see: https://orc.apache.org/docs/hive-config.html)
+
 ### ver 0.2.0
 
 - support: output to s3

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.3.0"
+version = "0.3.1"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/example/example.yml

@@ -1,7 +1,7 @@
 ---
 in:
   type: randomj
-  rows: 1024
+  rows: 1024000
   threads: 1
   # default_timezone: Asia/Tokyo
   primary_key: myid
@@ -14,14 +14,12 @@ in:
     - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
     - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
 
-#exec:
-#  max_threads: 6         # run at most 8 tasks concurrently
-#  min_output_tasks: 2    # disable page scattering
+exec:
+  max_threads: 2         # run at most 8 tasks concurrently
+  min_output_tasks: 1    # disable page scattering
 
 out:
   type: orc
   overwrite:   true
   path_prefix: "/tmp/output"
-  buffer_size: 8000
-  strip_size:  90000
   compression_kind: ZLIB

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -197,25 +197,32 @@ public class OrcOutputPlugin
         @Override
         public void add(Page page)
         {
-            int size = page.getStringReferences().size();
-            final TypeDescription schema = getSchema(reader.getSchema());
-            final VectorizedRowBatch batch = schema.createRowBatch();
-            batch.size = size;
-
-            reader.setPage(page);
-            int i = 0;
-            while (reader.nextRecord()) {
-                reader.getSchema().visitColumns(
-                        new OrcColumnVisitor(reader, batch, i)
-                );
-                i++;
-            }
-            try {
-                writer.addRowBatch(batch);
-                batch.reset();
-            }
-            catch (IOException e) {
-                e.printStackTrace();
+            synchronized (this) {
+                try {
+                    // int size = page.getStringReferences().size();
+                    final TypeDescription schema = getSchema(reader.getSchema());
+                    final VectorizedRowBatch batch = schema.createRowBatch();
+                    // batch.size = size;
+
+                    reader.setPage(page);
+                    while (reader.nextRecord()) {
+                        final int row = batch.size++;
+                        reader.getSchema().visitColumns(
+                                new OrcColumnVisitor(reader, batch, row)
+                        );
+                        if (batch.size >= batch.getMaxSize()) {
+                            writer.addRowBatch(batch);
+                            batch.reset();
+                        }
+                    }
+                    if (batch.size != 0) {
+                        writer.addRowBatch(batch);
+                        batch.reset();
+                    }
+                }
+                catch (IOException e) {
+                    e.printStackTrace();
+                }
             }
         }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-28 00:00:00.000000000 Z
+date: 2017-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +94,7 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-orc-0.3.0.jar
+- classpath/embulk-output-orc-0.3.1.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.7.3.jar