embulk-output-orc 0.3.0 → 0.3.5

data/src/main/scala/org/embulk/output/orc/OrcOutputPluginHelper.scala

@@ -0,0 +1,57 @@
+package org.embulk.output.orc
+
+import java.io.IOException
+import java.nio.file.{Files, Paths}
+
+import com.amazonaws.auth.profile.ProfileCredentialsProvider
+import com.amazonaws.services.s3.AmazonS3Client
+import com.amazonaws.services.s3.model.DeleteObjectRequest
+
+import scala.beans.BeanProperty
+
+object OrcOutputPluginHelper {
+  def removeOldFile(fpath: String, task: PluginTask): Unit = {
+    // NOTE: Delete a file if local-filesystem, not HDFS or S3.
+    val schema = getSchema(fpath)
+    if (isDeleteTarget(schema)) schema match {
+      case "file" =>
+        try Files.deleteIfExists(Paths.get(fpath))
+        catch {
+          case e: IOException => throw e
+        }
+      case "s3" | "s3n" | "s3a" =>
+        val s3Url = parseS3Url(fpath)
+        val s3client = new AmazonS3Client(new ProfileCredentialsProvider)
+        if (task.getEndpoint.isPresent) s3client.setEndpoint(task.getEndpoint.get)
+        s3client.deleteObject(new DeleteObjectRequest(s3Url.bucket, s3Url.key))
+      case _ =>
+    }
+  }
+
+  def isDeleteTarget(schema: String): Boolean = schema match {
+    case "file" => true
+    case "s3" | "s3n" | "s3a" => true
+    case _ => false
+  }
+
+  def getSchema(fpath: String): String = {
+    val schema = fpath.split("://").toList.head
+    schema match {
+      case "s3" | "s3a" | "s3n" => schema
+      case _ => {
+        val path = Paths.get(fpath)
+        path.getFileSystem.provider.getScheme
+      }
+    }
+  }
+
+  def parseS3Url(s3url: String): AmazonS3URILikeObject = {
+    val parts = s3url.split("(://|/)").toList
+    val bucket = parts.apply(1)
+    val key = parts.slice(2, parts.size).mkString("/")
+    OrcOutputPluginHelper.AmazonS3URILikeObject(bucket, key)
+  }
+
+  case class AmazonS3URILikeObject(@BeanProperty bucket: String, @BeanProperty key: String)
+
+}

data/src/main/scala/org/embulk/output/orc/OrcTransactionalPageOutput.scala

@@ -0,0 +1,52 @@
+package org.embulk.output.orc
+
+import java.io.IOException
+
+import org.apache.orc.Writer
+import org.embulk.config.TaskReport
+import org.embulk.spi.{Exec, Page, PageReader, TransactionalPageOutput}
+
+class OrcTransactionalPageOutput(val reader: PageReader, val writer: Writer, val task: PluginTask) extends TransactionalPageOutput {
+  override def add(page: Page): Unit = synchronized {
+    try {
+      // int size = page.getStringReferences().size();
+      val schema = OrcOutputPlugin.getSchema(reader.getSchema)
+      val batch = schema.createRowBatch
+      // batch.size = size;
+      reader.setPage(page)
+      while ( {
+        reader.nextRecord
+      }) {
+        val row = {
+          batch.size += 1;
+          batch.size - 1
+        }
+        reader.getSchema.visitColumns(new OrcColumnVisitor(reader, batch, row))
+        if (batch.size >= batch.getMaxSize) {
+          writer.addRowBatch(batch)
+          batch.reset()
+        }
+      }
+      if (batch.size != 0) {
+        writer.addRowBatch(batch)
+        batch.reset()
+      }
+    } catch {
+      case e: IOException =>
+        e.printStackTrace()
+    }
+  }
+
+  override def finish(): Unit = {
+    try writer.close()
+    catch {
+      case e: IOException => throw e
+    }
+  }
+
+  override def close(): Unit = {}
+
+  override def abort(): Unit = {}
+
+  override def commit: TaskReport = Exec.newTaskReport
+}

data/src/main/scala/org/embulk/output/orc/PluginTask.scala

@@ -0,0 +1,56 @@
+package org.embulk.output.orc
+
+import java.util
+
+import com.google.common.base.Optional
+import org.embulk.config.{Config, ConfigDefault, Task}
+import org.embulk.spi.time.TimestampFormatter
+import org.embulk.util.aws.credentials.AwsCredentialsTask
+import org.joda.time.DateTimeZone
+
+trait PluginTask extends Task with TimestampFormatter.Task with AwsCredentialsTask {
+  @Config("path_prefix")
+  def getPathPrefix: String
+
+  @Config("file_ext")
+  @ConfigDefault("\".orc\"")
+  def getFileNameExtension: String
+
+  @Config("column_options")
+  @ConfigDefault("{}")
+  def getColumnOptions: util.Map[String, TimestampColumnOption]
+
+  @Config("sequence_format")
+  @ConfigDefault("\".%03d\"")
+  def getSequenceFormat: String
+
+  // see: https://orc.apache.org/docs/hive-config.html
+  // ORC File options
+  @Config("strip_size")
+  @ConfigDefault("67108864") // 64MB
+  def getStripSize: Integer
+
+  @Config("buffer_size")
+  @ConfigDefault("262144") // 256KB
+  def getBufferSize: Integer
+
+  @Config("block_size")
+  @ConfigDefault("268435456") // 256MB
+  def getBlockSize: Integer
+
+  @Config("compression_kind")
+  @ConfigDefault("ZLIB")
+  def getCompressionKind: String
+
+  @Config("overwrite")
+  @ConfigDefault("false")
+  def getOverwrite: Boolean
+
+  @Config("default_from_timezone")
+  @ConfigDefault("\"UTC\"")
+  def getDefaultFromTimeZone: DateTimeZone
+
+  @Config("endpoint")
+  @ConfigDefault("null")
+  def getEndpoint: Optional[String]
+}

data/src/main/scala/org/embulk/output/orc/TimestampColumnOption.scala

@@ -0,0 +1,32 @@
+package org.embulk.output.orc
+
+import java.util
+
+import com.google.common.base.Optional
+import org.embulk.config.{Config, ConfigDefault, Task}
+import org.embulk.spi.time.TimestampFormatter
+import org.joda.time.DateTimeZone
+
+/*
+public interface TimestampColumnOption
+  extends Task, TimestampFormatter.TimestampColumnOption
+{
+    @Config("from_timezone")
+    @ConfigDefault("null")
+    Optional<DateTimeZone> getFromTimeZone();
+
+    @Config("from_format")
+    @ConfigDefault("null")
+    Optional<List<String>> getFromFormat();
+}
+ */
+
+trait TimestampColumnOption extends Task with TimestampFormatter.TimestampColumnOption {
+  @Config("from_timezone")
+  @ConfigDefault("null")
+  def getFromTimeZone: Optional[DateTimeZone]
+
+  @Config("from_format")
+  @ConfigDefault("null")
+  def getFromFormat: Optional[util.List[String]]
+}

data/src/test/java/org/embulk/output/orc/OrcOutputPluginHelperTest.java

@@ -0,0 +1,71 @@
+package org.embulk.output.orc;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+public class OrcOutputPluginHelperTest
+{
+    @DataProvider(name = "url-provider")
+    public Object[][] dataProvider()
+    {
+        return new Object[][] {
+                {"file://tmp/output.orc", "file"},
+                {"/tmp/output.000.orc", "file"},
+                {"s3n://embulk-test/output.0001.orc", "s3n"},
+                {"s3a://embulk-test/output.0001.orc", "s3a"},
+                {"s3://embulk-test/output.0001.orc", "s3"},
+                };
+    }
+
+    @Test(dataProvider = "url-provider")
+    public void getFPathTest(String file, String expect)
+    {
+        String schema = OrcOutputPluginHelper.getSchema(file);
+        assertThat(schema, is(expect));
+    }
+
+    @DataProvider(name = "schema-provider")
+    public Object[][] schemaProvider()
+    {
+        return new Object[][] {
+                {"file", true},
+                {"s3", true},
+                {"s3n", true},
+                {"s3a", true},
+                {"hdfs", false},
+                };
+    }
+
+    @Test(dataProvider = "schema-provider")
+    public void isDeleteTargetTest(String schema, boolean expect)
+    {
+        boolean result = OrcOutputPluginHelper.isDeleteTarget(schema);
+        assertThat(result, is(expect));
+    }
+
+    @DataProvider(name = "parserTest-provider")
+    public Object[][] parserTestProvider()
+    {
+        String baseurl = "demo-bucket/test/output.000.orc";
+        String bucket = "demo-bucket";
+        String keyname = "test/output.000.orc";
+
+        return new Object[][] {
+                {"s3://" + baseurl, bucket, keyname},
+                {"s3a://" + baseurl, bucket, keyname},
+                {"s3n://" + baseurl, bucket, keyname},
+                };
+    }
+
+    @Test(dataProvider = "parserTest-provider")
+    public void parseS3UrlTest(String url, String bucket, String key)
+    {
+        OrcOutputPluginHelper.AmazonS3URILikeObject parts =
+                OrcOutputPluginHelper.parseS3Url(url);
+        assertThat(parts.getBucket(), is(bucket));
+        assertThat(parts.getKey(), is(key));
+    }
+}

data/src/test/resources/example-null.yml

@@ -0,0 +1,25 @@
+---
+in:
+  type: randomj
+  rows: 8
+  threads: 1
+  # default_timezone: Asia/Tokyo
+  primary_key: myid
+  schema:
+    - {name: myid,     type: long}
+    - {name: named,    type: string, null_rate: 10000}
+    - {name: x_flag,   type: boolean, null_rate: 10000}
+    - {name: pit_rate, type: double, null_rate: 10000}
+    - {name: score,    type: long, null_rate: 10000}
+    - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S', null_rate: 10000}
+    - {name: purchase, type: timestamp, format: '%Y/%m/%d', null_rate: 10000}
+
+exec:
+  max_threads: 2         # run at most 8 tasks concurrently
+  min_output_tasks: 1    # disable page scattering
+
+out:
+  type: orc
+  overwrite:   true
+  path_prefix: "/tmp/output"
+  compression_kind: ZLIB

data/src/test/resources/example.yml

@@ -0,0 +1,25 @@
+---
+in:
+  type: randomj
+  rows: 1024
+  threads: 1
+  # default_timezone: Asia/Tokyo
+  primary_key: myid
+  schema:
+    - {name: myid,     type: long}
+    - {name: named,    type: string, null_rate: 1000}
+    - {name: x_flag,   type: boolean, null_rate: 1000}
+    - {name: pit_rate, type: double, null_rate: 1000}
+    - {name: score,    type: long, null_rate: 1000}
+    - {name: time,     type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y/%m/%d'}
+
+exec:
+  max_threads: 2         # run at most 8 tasks concurrently
+  min_output_tasks: 1    # disable page scattering
+
+out:
+  type: orc
+  overwrite:   true
+  path_prefix: "/tmp/output"
+  compression_kind: ZLIB

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.5
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-28 00:00:00.000000000 Z
+date: 2020-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -21,13 +21,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -35,7 +35,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Dumps records to Orc format file.
@@ -45,27 +45,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".github/workflows/gradle.yml"
+- ".gitignore"
 - LICENSE.txt
 - README.md
 - build.gradle
-- config/checkstyle/checkstyle.xml
-- config/checkstyle/default.xml
-- example/example.yml
-- gradle/wrapper/gradle-wrapper.jar
-- gradle/wrapper/gradle-wrapper.properties
-- gradlew
-- gradlew.bat
-- lib/embulk/output/orc.rb
-- src/main/java/org/embulk/output/orc/OrcColumnVisitor.java
-- src/main/java/org/embulk/output/orc/OrcOutputPlugin.java
-- src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java
-- src/main/java/org/embulk/output/orc/PluginTask.java
-- src/main/java/org/embulk/output/orc/TimestampColumnOption.java
-- src/test/java/org/embulk/output/orc/TestOrcOutputPlugin.java
-- classpath/activation-1.1.jar
-- classpath/aircompressor-0.3.jar
+- classpath/aircompressor-0.10.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
 - classpath/api-asn1-api-1.0.0-M20.jar
@@ -84,7 +69,6 @@ files:
 - classpath/commons-configuration-1.6.jar
 - classpath/commons-daemon-1.0.13.jar
 - classpath/commons-digester-1.8.jar
-- classpath/commons-el-1.0.jar
 - classpath/commons-httpclient-3.1.jar
 - classpath/commons-io-2.4.jar
 - classpath/commons-lang-2.6.jar
@@ -94,16 +78,15 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-orc-0.3.0.jar
+- classpath/embulk-output-orc-0.3.5.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/gson-2.2.4.jar
-- classpath/hadoop-annotations-2.7.3.jar
-- classpath/hadoop-auth-2.7.3.jar
-- classpath/hadoop-aws-2.7.3.jar
-- classpath/hadoop-common-2.7.3.jar
-- classpath/hadoop-hdfs-2.6.4.jar
-- classpath/hive-storage-api-2.2.1.jar
-- classpath/htrace-core-3.0.4.jar
+- classpath/hadoop-annotations-2.7.5.jar
+- classpath/hadoop-auth-2.7.5.jar
+- classpath/hadoop-aws-2.7.5.jar
+- classpath/hadoop-common-2.7.5.jar
+- classpath/hadoop-hdfs-2.7.5.jar
+- classpath/hive-storage-api-2.6.0.jar
 - classpath/htrace-core-3.1.0-incubating.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
@@ -111,9 +94,8 @@ files:
 - classpath/jackson-jaxrs-1.8.3.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
 - classpath/jackson-xc-1.8.3.jar
-- classpath/jasper-runtime-5.5.23.jar
 - classpath/java-xmlbuilder-0.4.jar
-- classpath/jaxb-api-2.2.2.jar
+- classpath/jaxb-api-2.2.11.jar
 - classpath/jaxb-impl-2.2.3-1.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
 - classpath/jersey-core-1.9.jar
@@ -122,25 +104,46 @@ files:
 - classpath/jets3t-0.9.0.jar
 - classpath/jettison-1.1.jar
 - classpath/jetty-6.1.26.jar
+- classpath/jetty-sslengine-6.1.26.jar
 - classpath/jetty-util-6.1.26.jar
 - classpath/jline-0.9.94.jar
-- classpath/joda-time-2.9.9.jar
-- classpath/jsch-0.1.42.jar
+- classpath/jsch-0.1.54.jar
 - classpath/jsp-api-2.1.jar
 - classpath/jsr305-3.0.0.jar
+- classpath/leveldbjni-all-1.8.jar
 - classpath/log4j-1.2.17.jar
 - classpath/netty-3.7.0.Final.jar
-- classpath/orc-core-1.4.0.jar
+- classpath/netty-all-4.0.23.Final.jar
+- classpath/orc-core-1.5.4.jar
+- classpath/orc-shims-1.5.4.jar
 - classpath/paranamer-2.3.jar
 - classpath/protobuf-java-2.5.0.jar
+- classpath/scala-library-2.12.12.jar
+- classpath/servlet-api-2.5-20081211.jar
 - classpath/servlet-api-2.5.jar
 - classpath/snappy-java-1.0.4.1.jar
-- classpath/stax-api-1.0-2.jar
 - classpath/xercesImpl-2.9.1.jar
 - classpath/xml-apis-1.3.04.jar
 - classpath/xmlenc-0.52.jar
 - classpath/xz-1.0.jar
 - classpath/zookeeper-3.4.6.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- example/example.yml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/output/orc.rb
+- src/main/scala/org/embulk/output/orc/OrcColumnVisitor.scala
+- src/main/scala/org/embulk/output/orc/OrcOutputPlugin.scala
+- src/main/scala/org/embulk/output/orc/OrcOutputPluginHelper.scala
+- src/main/scala/org/embulk/output/orc/OrcTransactionalPageOutput.scala
+- src/main/scala/org/embulk/output/orc/PluginTask.scala
+- src/main/scala/org/embulk/output/orc/TimestampColumnOption.scala
+- src/test/java/org/embulk/output/orc/OrcOutputPluginHelperTest.java
+- src/test/resources/example-null.yml
+- src/test/resources/example.yml
 homepage: https://github.com/yuokada/embulk-output-orc
 licenses:
 - MIT
@@ -151,17 +154,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: Orc output plugin for Embulk