embulk-output-orc 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a4d872e06384fd47cf8c1707398234974e0ecb3
-  data.tar.gz: bb3855c44c55f16bc5e1707f09e3a1b6c683aee6
+  metadata.gz: 041e79d159d0ffe346b0c28f17c7009438bc65e5
+  data.tar.gz: 13d96f705101ce32a4d389c1539ea23c50db3872
 SHA512:
-  metadata.gz: a16e0c62d57089d5ff431021b97586a8203a423ab3f8797a4e2274f26cb168a91a59ae72d6d42b4f2540209565d05d9e3f1ae66af242e52c90ab1f239afe90cd
-  data.tar.gz: 21f77bd871793f06b014ce9df5c08ab7e0f0d610b7cc69b749e415a1f181891288ba06211dbe3d769bfd756bfaaf344d1eeecd42db9f085a7feba28560921c96
+  metadata.gz: fdd305bfb25dfd998f0c49e55ef4ffb916f268fa5189678102b3f6e7228532da4d3ea141afe03491a2cf11bbe394975f73868a7620ab0d410306ab998d1f0d95
+  data.tar.gz: 88574a4b9e2982b80c93307c361ec1de26c0878d8f08acfc0333eb770df74a7e76ef38747b94a57c351d4cd42e6fce015d141ef8d94e5f47bc6ecab1eeca085e

data/.travis.yml

@@ -10,3 +10,5 @@ sudo: false
 script:
   - ./gradlew --info checkstyle
   - ./gradlew --info check
+
+after_success: ./gradlew sonarqube -Dsonar.organization=yuokada-github -Dsonar.host.url=https://sonarcloud.io -Dsonar.login=${SONAR_TOKEN}

data/README.md

@@ -19,6 +19,7 @@
 - **buffer_size**: Set the ORC buffer size (integer, default: `10000`)
 - **strip_size**: Set the ORC strip size (integer,  default: `100000`)
 - **compression_kind**: description (string, default: `'ZLIB'`)
+    - `NONE`, `ZLIB`, `SNAPPY`
 - **overwrite**: (LocalFileSystem only) Overwrite if output files already exist. (boolean, default: `false`)
 - **default_from_timezone** Time zone of timestamp columns. This can be overwritten for each column using column_options (DateTimeZone, default: `UTC`)
 
@@ -57,3 +58,7 @@ out:
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
+
+## SonarQube
+
+[embulk-output-orc](https://sonarcloud.io/dashboard?id=embulk-output-orc "embulk-output-orc - Yukihiro Okada")

data/build.gradle

@@ -3,6 +3,7 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "checkstyle"
+    id "org.sonarqube" version "2.5"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -17,14 +18,14 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.2.0"
+version = "0.2.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.29"
-    provided "org.embulk:embulk-core:0.8.29"
+    compile  "org.embulk:embulk-core:0.8.34"
+    provided "org.embulk:embulk-core:0.8.34"
 
     compile "org.apache.orc:orc:1.4.0"
     compile "org.apache.orc:orc-core:1.4.0"
@@ -35,8 +36,8 @@ dependencies {
     compile "org.apache.hadoop:hadoop-aws:2.7.3"
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.29:tests"
-    testCompile "org.embulk:embulk-standards:0.8.29"
+    testCompile "org.embulk:embulk-core:0.8.34:tests"
+    testCompile "org.embulk:embulk-standards:0.8.34"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/example/example.yml

@@ -1,41 +1,11 @@
-# in:
-#   type: file
-#   path_prefix: example/example.csv
-#   parser:
-#     type: csv
-#     charset: UTF-8
-#     newline: CRLF
-#     null_string: 'NULL'
-#     skip_header_lines: 1
-#     comment_line_marker: '#'
-#     columns:
-#       #- {name: time,  type: timestamp, format: "%Y-%m-%d"}
-#       - {name: id, type: long}
-#       - {name: name, type: string}
-#       - {name: score, type: double}
-#      - {name: json, type: json}
-#filters:
-#  - type: column
-#    columns:
-#      - {name: time, default: "2015-07-13", format: "%Y-%m-%d"}
-#      - {name: name, default: "foo"}
-#      - {name: foo,  default: 1, type: long}
-#      - {name: id}
-#      - {name: copy_score, src: score}
-#      - {name: json, default: "{\"foo\":\"FOO\"}"}
-#      - {name: $.json.foo}
-#      - {name: $.json.copy_foo, src: $.json.foo}
+---
 in:
- type: file
- path_prefix: example/sample.csv
- parser:
-   type: csv
-   charset: UTF-8
-   newline: CRLF
-   null_string: 'NULL'
-   skip_header_lines: 0
-   comment_line_marker: '#'
-   columns:
+  type: randomj
+  rows: 1024
+  threads: 1
+  # default_timezone: Asia/Tokyo
+  primary_key: myid
+  schema:
     - {name: myid,     type: long}
     - {name: named,    type: string}
     - {name: x_flag,   type: boolean}
@@ -44,12 +14,9 @@ in:
     - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
     - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
 
-exec:
-  max_threads: 2         # run at most 8 tasks concurrently
-  min_output_tasks: 1    # disable page scattering
-
-#out:
-#  type: stdout
+#exec:
+#  max_threads: 6         # run at most 8 tasks concurrently
+#  min_output_tasks: 2    # disable page scattering
 
 out:
   type: orc

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Mon Aug 14 21:51:29 JST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -154,16 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
   cd "$(dirname "$0")"
 fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/src/main/java/org/embulk/output/orc/OrcCodec.java

@@ -0,0 +1,21 @@
+package org.embulk.output.orc;
+
+public enum OrcCodec
+{
+    ZLIB("zlib"),
+    SNAPPY("snappy"),
+    LZO("lzo"),
+    LZ4("lz4"),
+    NONE("none"),;
+    String kind;
+
+    OrcCodec(String kind)
+    {
+        this.kind = kind;
+    }
+
+    public String getKind()
+    {
+        return kind;
+    }
+}

data/src/main/java/org/embulk/output/orc/OrcColumnVisitor.java

@@ -10,12 +10,14 @@ import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.time.Timestamp;
 
+import java.nio.charset.StandardCharsets;
+
 public class OrcColumnVisitor
         implements ColumnVisitor
 {
-    private PageReader reader;
-    private VectorizedRowBatch batch;
-    private Integer i;
+    private final PageReader reader;
+    private final VectorizedRowBatch batch;
+    private final Integer i;
 
     public OrcColumnVisitor(PageReader pageReader, VectorizedRowBatch rowBatch, Integer i)
     {
@@ -31,7 +33,6 @@ public class OrcColumnVisitor
             ((LongColumnVector) batch.cols[column.getIndex()]).vector[i] = 0;
         }
         else {
-            // TODO; Fix all true bug
             if (reader.getBoolean(column)) {
                 ((LongColumnVector) batch.cols[column.getIndex()]).vector[i] = 1;
             }
@@ -57,7 +58,7 @@ public class OrcColumnVisitor
     public void stringColumn(Column column)
     {
         ((BytesColumnVector) batch.cols[column.getIndex()]).setVal(i,
-                reader.getString(column).getBytes());
+                reader.getString(column).getBytes(StandardCharsets.UTF_8));
     }
 
     @Override
@@ -68,11 +69,8 @@ public class OrcColumnVisitor
         }
         else {
             Timestamp timestamp = reader.getTimestamp(column);
-            if (!timestamp.equals("")) {
-                java.sql.Timestamp ts = new java.sql.Timestamp(timestamp.getEpochSecond() * 1000);
-                ((TimestampColumnVector) batch.cols[column.getIndex()]).set(i, ts);
-            }
-            // throw new UnsupportedOperationException("orc output plugin does not support timestamp yet");
+            java.sql.Timestamp ts = new java.sql.Timestamp(timestamp.getEpochSecond() * 1000);
+            ((TimestampColumnVector) batch.cols[column.getIndex()]).set(i, ts);
         }
     }
 

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -32,10 +32,9 @@ import org.embulk.spi.util.Timestamps;
 import org.embulk.util.aws.credentials.AwsCredentials;
 import org.embulk.util.aws.credentials.AwsCredentialsTask;
 import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -80,6 +79,10 @@ public class OrcOutputPlugin
         @Config("default_from_timezone")
         @ConfigDefault("\"UTC\"")
         DateTimeZone getDefaultFromTimeZone();
+
+        @Config("endpoint")
+        @ConfigDefault("null")
+        Optional<String> getEndpoint();
     }
 
     public interface TimestampColumnOption
@@ -196,7 +199,9 @@ public class OrcOutputPlugin
             conf.set("fs.s3a.secret.key", task.getSecretAccessKey().get());
             conf.set("fs.s3n.awsSecretAccessKey", task.getSecretAccessKey().get());
         }
-
+        if (task.getEndpoint().isPresent()) {
+            conf.set("fs.s3a.endpoint", task.getEndpoint().get());
+        }
         return conf;
     }
 
@@ -222,7 +227,7 @@ public class OrcOutputPlugin
                             .version(OrcFile.Version.V_0_12));
         }
         catch (IOException e) {
-            e.printStackTrace();
+            Throwables.propagate(e);
         }
         return writer;
     }
@@ -259,43 +264,34 @@ public class OrcOutputPlugin
     class OrcTransactionalPageOutput
             implements TransactionalPageOutput
     {
-        private PageReader reader;
-        private Writer writer;
-        private DateTimeFormatter formatter;
+        private final PageReader reader;
+        private final Writer writer;
+        private final ArrayList<VectorizedRowBatch> rowBatches = new ArrayList<>();
 
         public OrcTransactionalPageOutput(PageReader reader, Writer writer, PluginTask task)
         {
             this.reader = reader;
             this.writer = writer;
-
-            // formatter
-            DateTimeZone defaultTimeZone = DateTimeZone
-                    .forTimeZone(task.getDefaultFromTimeZone().toTimeZone());
-            formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(defaultTimeZone);
         }
 
         @Override
         public void add(Page page)
         {
             int size = page.getStringReferences().size();
-            TypeDescription schema = getSchema(reader.getSchema());
-            VectorizedRowBatch batch = schema.createRowBatch();
+            final TypeDescription schema = getSchema(reader.getSchema());
+            final VectorizedRowBatch batch = schema.createRowBatch();
             batch.size = size;
 
             reader.setPage(page);
             int i = 0;
             while (reader.nextRecord()) {
-                // batch.size = page.getStringReferences().size();
                 reader.getSchema().visitColumns(
                         new OrcColumnVisitor(reader, batch, i)
                 );
                 i++;
             }
-            try {
-                writer.addRowBatch(batch);
-            }
-            catch (IOException e) {
-                e.printStackTrace();
+            synchronized (this) {
+                rowBatches.add(batch);
             }
         }
 
@@ -303,8 +299,10 @@ public class OrcOutputPlugin
         public void finish()
         {
             try {
+                for (VectorizedRowBatch batch : rowBatches) {
+                    writer.addRowBatch(batch);
+                }
                 writer.close();
-                writer = null;
             }
             catch (IOException e) {
                 Throwables.propagate(e);
@@ -314,19 +312,16 @@ public class OrcOutputPlugin
         @Override
         public void close()
         {
-            // TODO: something
         }
 
         @Override
         public void abort()
         {
-            // TODO: something
         }
 
         @Override
         public TaskReport commit()
         {
-            // TODO: something
             return Exec.newTaskReport();
         }
     }

data/src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java

@@ -1,5 +1,7 @@
 package org.embulk.output.orc;
 
+import com.google.common.base.Throwables;
+
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -20,7 +22,7 @@ class OrcOutputPluginHelper
             Files.deleteIfExists(path);
         }
         catch (IOException e) {
-            e.printStackTrace();
+            Throwables.propagate(e);
         }
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-31 00:00:00.000000000 Z
+date: 2017-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,6 +58,7 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/output/orc.rb
+- src/main/java/org/embulk/output/orc/OrcCodec.java
 - src/main/java/org/embulk/output/orc/OrcColumnVisitor.java
 - src/main/java/org/embulk/output/orc/OrcOutputPlugin.java
 - src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java
@@ -92,7 +93,7 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-orc-0.2.0.jar
+- classpath/embulk-output-orc-0.2.2.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.7.3.jar