embulk-output-orc 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f90685be3f76457be27d9ee129b56276a66dd42b
-  data.tar.gz: ac33ac212a7bb6352aef8eb0caf310f5afd25bbe
+  metadata.gz: 725b869f19175110dd6e542619f5bfd9c8c0168b
+  data.tar.gz: 9c8016b015f380815cb2ad7127c192243ebfa45a
 SHA512:
-  metadata.gz: aecde246d738967d2a91906560d9735ea11981e8f7349da57d7418a3ab08a7796fb5573f1e9b72b54ff6eee1a08f4ade83194024bfcf67ff6113ff6f570fb7b6
-  data.tar.gz: 34b3eb0dc0c01388edf4310f5f55d00b0eab65b95b2def0a888867df8f79e0087c66d70e099d2af6327e57be03a80a75a6135cf281a58f5c4a0b8e22ed7caa1a
+  metadata.gz: 197da7bd52b19ccc2c3f30ad0d2c2c36446fa00e48426de9bb333b0ef2a8eddc213f0db947e50215297333a933e5d6d3b63ad0c0388f78e19d984fae6810bb99
+  data.tar.gz: ccf9bc5b8f73c8d0f48e7d17e16dd4b113b3f9b7a1b9178045109ef3476f4cb47f58e0699e4a9516ab81830d59a4bae3be2fba8f936c4cef0f705f6acc384f2d

data/README.md

@@ -43,6 +43,12 @@ out:
 
 ## ChangeLog
 
+### ver 0.3.4
+
+- Bump `orc` library to `1.5.4`
+- bugfix
+  - https://github.com/yuokada/embulk-output-orc/pull/17
+
 ### ver 0.3.3
 
 - bugfix

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.3.3"
+version = "0.3.4"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -27,8 +27,8 @@ dependencies {
     compile  "org.embulk:embulk-core:0.8.34"
     provided "org.embulk:embulk-core:0.8.34"
 
-    compile "org.apache.orc:orc:1.4.4"
-    compile "org.apache.orc:orc-core:1.4.4"
+    compile "org.apache.orc:orc:1.5.4"
+    compile "org.apache.orc:orc-core:1.5.4"
     compile "org.apache.hadoop:hadoop-hdfs:2.7.5"
 
     compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8'

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -9,6 +9,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.orc.CompressionKind;
+import org.apache.orc.MemoryManager;
 import org.apache.orc.OrcFile;
 import org.apache.orc.TypeDescription;
 import org.apache.orc.Writer;
@@ -163,6 +164,7 @@ public class OrcOutputPlugin
             writer = OrcFile.createWriter(
                     new Path(buildPath(task, processorIndex)),
                     writerOptions.setSchema(oschema)
+                            .memory(new WriterLocalMemoryManager())
                             .version(OrcFile.Version.V_0_12)
             );
         }
@@ -201,33 +203,31 @@ public class OrcOutputPlugin
         @Override
         public void add(Page page)
         {
-            synchronized (this) {
-                try {
-                    // int size = page.getStringReferences().size();
-                    final TypeDescription schema = getSchema(reader.getSchema());
-                    final VectorizedRowBatch batch = schema.createRowBatch();
-                    // batch.size = size;
-
-                    reader.setPage(page);
-                    while (reader.nextRecord()) {
-                        final int row = batch.size++;
-                        reader.getSchema().visitColumns(
-                                new OrcColumnVisitor(reader, batch, row)
-                        );
-                        if (batch.size >= batch.getMaxSize()) {
-                            writer.addRowBatch(batch);
-                            batch.reset();
-                        }
-                    }
-                    if (batch.size != 0) {
+            try {
+                // int size = page.getStringReferences().size();
+                final TypeDescription schema = getSchema(reader.getSchema());
+                final VectorizedRowBatch batch = schema.createRowBatch();
+                // batch.size = size;
+
+                reader.setPage(page);
+                while (reader.nextRecord()) {
+                    final int row = batch.size++;
+                    reader.getSchema().visitColumns(
+                            new OrcColumnVisitor(reader, batch, row)
+                    );
+                    if (batch.size >= batch.getMaxSize()) {
                         writer.addRowBatch(batch);
                         batch.reset();
                     }
                 }
-                catch (IOException e) {
-                    e.printStackTrace();
+                if (batch.size != 0) {
+                    writer.addRowBatch(batch);
+                    batch.reset();
                 }
             }
+            catch (IOException e) {
+                e.printStackTrace();
+            }
         }
 
         @Override
@@ -257,4 +257,42 @@ public class OrcOutputPlugin
             return Exec.newTaskReport();
         }
     }
+
+    // We avoid using orc.MemoryManagerImpl since it is not threadsafe, but embulk is multi-threaded.
+    // Embulk creates and uses multiple instances of TransactionalPageOutput in worker threads.
+    // As a workaround, WriterLocalMemoryManager is bound to a single orc.Writer instance, and
+    // notifies checkMemory() only to that instance.
+    private static class WriterLocalMemoryManager implements MemoryManager
+    {
+        final long rowsBetweenChecks = 10000;
+
+        private int rowsAddedSinceCheck = 0;
+        Callback boundCallback = null;
+
+        @Override
+        public void addWriter(Path path, long requestedAllocation, Callback callback) throws IOException
+        {
+            if (boundCallback != null) {
+                throw new IllegalStateException("WriterLocalMemoryManager should be bound to a single orc.Writer instance.");
+            }
+
+            boundCallback = callback;
+        }
+
+        @Override
+        public void removeWriter(Path path) throws IOException
+        {
+            boundCallback = null;
+        }
+
+        @Override
+        public void addedRow(int rows) throws IOException
+        {
+            rowsAddedSinceCheck += rows;
+            if (rowsAddedSinceCheck > rowsBetweenChecks) {
+                boundCallback.checkMemory(1);
+                rowsAddedSinceCheck = 0;
+            }
+        }
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-10 00:00:00.000000000 Z
+date: 2019-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,8 +50,7 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
-- classpath/activation-1.1.jar
-- classpath/aircompressor-0.8.jar
+- classpath/aircompressor-0.10.jar
 - classpath/animal-sniffer-annotations-1.14.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
@@ -81,7 +80,7 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-orc-0.3.3.jar
+- classpath/embulk-output-orc-0.3.4.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/error_prone_annotations-2.1.3.jar
 - classpath/gson-2.2.4.jar
@@ -91,7 +90,7 @@ files:
 - classpath/hadoop-aws-2.7.5.jar
 - classpath/hadoop-common-2.7.5.jar
 - classpath/hadoop-hdfs-2.7.5.jar
-- classpath/hive-storage-api-2.2.1.jar
+- classpath/hive-storage-api-2.6.0.jar
 - classpath/htrace-core-3.1.0-incubating.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
@@ -101,7 +100,7 @@ files:
 - classpath/jackson-mapper-asl-1.9.13.jar
 - classpath/jackson-xc-1.8.3.jar
 - classpath/java-xmlbuilder-0.4.jar
-- classpath/jaxb-api-2.2.2.jar
+- classpath/jaxb-api-2.2.11.jar
 - classpath/jaxb-impl-2.2.3-1.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
 - classpath/jersey-core-1.9.jar
@@ -120,13 +119,13 @@ files:
 - classpath/log4j-1.2.17.jar
 - classpath/netty-3.7.0.Final.jar
 - classpath/netty-all-4.0.23.Final.jar
-- classpath/orc-core-1.4.4.jar
+- classpath/orc-core-1.5.4.jar
+- classpath/orc-shims-1.5.4.jar
 - classpath/paranamer-2.3.jar
 - classpath/protobuf-java-2.5.0.jar
 - classpath/servlet-api-2.5-20081211.jar
 - classpath/servlet-api-2.5.jar
 - classpath/snappy-java-1.0.4.1.jar
-- classpath/stax-api-1.0-2.jar
 - classpath/xercesImpl-2.9.1.jar
 - classpath/xml-apis-1.3.04.jar
 - classpath/xmlenc-0.52.jar