embulk-output-orc 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c842edfe45c7e992faae16afd3331c7f8ecf256
-  data.tar.gz: a6b3d098e7b012a07f4870e2fc92c898b09e1560
+  metadata.gz: 376187534c3d236c20656a656ed04d5c858e9a83
+  data.tar.gz: 4d4b74153e1e2fe0248b4ba5d097723dbc42022e
 SHA512:
-  metadata.gz: 3bf5bc9e310496191419ee1e9a76cf9912321e9df2d178081c324ed46f83e7600e04912df59a7ebf38e0e3761be4b8ea1931cae549d681a8ba5b6d35b1e19990
-  data.tar.gz: bbec4349adf56b4c684084a39f61eee8dd41152a11de02c2b5573e5e933cf8d76f280e7110f030cb31b157fdfc9e7fa457f575cd8a6b5b4d8df8ea484a51f358
+  metadata.gz: 888dab123d9cea81defbd51c0d5b812bd6c90e84c578927a72a491a8fffb4fac61fcced52ba7d759fdd1c36d9dba08a424c2d704b6f3f1ae51bbfdb3fac50728
+  data.tar.gz: bb7c0de447e7bf98b2ad7e7ba2b309854b0beaf37b6c10fd2a82be3736f6e39c8a576c985be0c62e9223ab24cb1a9bd4662e64d12974be490cd042002c933699

data/.travis.yml

@@ -0,0 +1,12 @@
+language: java
+jdk:
+  - oraclejdk8
+
+cache:
+  directories:  # run "travis cache --delete" to delete caches
+    - $HOME/.gradle
+
+sudo: false
+script:
+  - ./gradlew --info checkstyle
+  - ./gradlew --info check

data/README.md

@@ -1,6 +1,6 @@
 # Orc output plugin for Embulk
 
-TODO: Write short description here and build.gradle file.
+[![Build Status](https://travis-ci.org/yuokada/embulk-output-orc.svg?branch=master)](https://travis-ci.org/yuokada/embulk-output-orc)
 
 ## Overview
 
@@ -11,17 +11,25 @@ TODO: Write short description here and build.gradle file.
 
 ## Configuration
 
-- **option1**: description (integer, required)
-- **option2**: description (string, default: `"myvalue"`)
-- **option3**: description (string, default: `null`)
+- **path_prefix**: A prefix of output path. (string, required)
+- **file_ext**: An extension of output file. (string, default: `.orc`)
+- **sequence_format**: (string, default: `.%03d`)
+- **buffer_size**: Set the ORC buffer size (integer, default: `10000`)
+- **strip_size**: Set the ORC strip size (integer,  default: `100000`)
+- **compression_kind**: description (string, default: `'ZLIB'`)
+- **overwrite**: (LocalFileSystem only) Overwrite if output files already exist. (boolean, default: `false`)
+- **default_from_timezone** Time zone of timestamp columns. This can be overwritten for each column using column_options (DateTimeZone, default: `UTC`)
 
 ## Example
 
 ```yaml
 out:
   type: orc
-  option1: example1
-  option2: example2
+  path_prefix: "/tmp/output"
+  buffer_size: 8000
+  strip_size:  90000
+  compression_kind: ZLIB
+  overwrite:   true
 ```
 
 

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.0.2"
+version = "0.1.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -22,15 +22,10 @@ targetCompatibility = 1.8
 dependencies {
     compile  "org.embulk:embulk-core:0.8.29"
     provided "org.embulk:embulk-core:0.8.29"
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
 
     compile "org.apache.orc:orc:1.4.0"
     compile "org.apache.orc:orc-core:1.4.0"
-    compile "org.apache.orc:orc-mapreduce:1.4.0"
-
-    compile "org.apache.hadoop:hadoop-common:2.6.4"
-    compile "org.apache.hadoop:hadoop-client:2.6.4"
-//    compile "org.apache.hadoop:hadoop-hdfs:2.7.4"
+    compile "org.apache.hadoop:hadoop-hdfs:2.6.4"
 
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.29:tests"
@@ -89,10 +84,10 @@ Gem::Specification.new do |spec|
   spec.version       = "${project.version}"
   spec.authors       = ["yuokada"]
   spec.summary       = %[Orc output plugin for Embulk]
-  spec.description   = %[Dumps records to Orc.]
+  spec.description   = %[Dumps records to Orc format file.]
   spec.email         = ["callistoiv+git@gmail.com"]
   spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/callistoiv+git/embulk-output-orc"
+  spec.homepage      = "https://github.com/yuokada/embulk-output-orc"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/example/example.yml

@@ -53,6 +53,7 @@ exec:
 
 out:
   type: orc
+  overwrite:   true
   path_prefix: "/tmp/output"
   buffer_size: 8000
   strip_size:  90000

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -128,12 +128,14 @@ public class OrcOutputPlugin
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
+        if (task.getOverwrite()) {
+            OrcOutputPluginHelper.removeOldFile(buildPath(task, taskIndex));
+        }
+
         final PageReader reader = new PageReader(schema);
         Writer writer = createWriter(task, schema, taskIndex);
 
         return new OrcTransactionalPageOutput(reader, writer, task);
-        // Write your code here :)
-//        throw new UnsupportedOperationException("OrcOutputPlugin.run method is not implemented yet");
     }
 
     private String buildPath(PluginTask task, int processorIndex)

data/src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java

@@ -0,0 +1,26 @@
+package org.embulk.output.orc;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+class OrcOutputPluginHelper
+{
+    protected OrcOutputPluginHelper()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    static void removeOldFile(String fpath)
+    {
+        Path path = Paths.get(fpath);
+        // TODO: Check local file. not HDFS or S3.
+        try {
+            Files.deleteIfExists(path);
+        }
+        catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-21 00:00:00.000000000 Z
+date: 2017-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Dumps records to Orc.
+description: Dumps records to Orc format file.
 email:
 - callistoiv+git@gmail.com
 executables: []
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -59,22 +60,20 @@ files:
 - lib/embulk/output/orc.rb
 - src/main/java/org/embulk/output/orc/OrcColumnVisitor.java
 - src/main/java/org/embulk/output/orc/OrcOutputPlugin.java
+- src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java
 - src/test/java/org/embulk/output/orc/TestOrcOutputPlugin.java
-- classpath/activation-1.1.jar
 - classpath/aircompressor-0.3.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
 - classpath/api-asn1-api-1.0.0-M20.jar
 - classpath/api-util-1.0.0-M20.jar
 - classpath/asm-3.1.jar
-- classpath/avro-1.7.4.jar
-- classpath/commons-beanutils-1.7.0.jar
 - classpath/commons-cli-1.2.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-collections-3.2.2.jar
 - classpath/commons-compress-1.4.1.jar
 - classpath/commons-configuration-1.6.jar
-- classpath/commons-digester-1.8.jar
+- classpath/commons-daemon-1.0.13.jar
 - classpath/commons-el-1.0.jar
 - classpath/commons-httpclient-3.1.jar
 - classpath/commons-io-2.4.jar
@@ -84,70 +83,37 @@ files:
 - classpath/commons-net-3.1.jar
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
-- classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-output-orc-0.0.2.jar
+- classpath/embulk-output-orc-0.1.0.jar
 - classpath/gson-2.2.4.jar
-- classpath/guice-servlet-3.0.jar
 - classpath/hadoop-annotations-2.6.4.jar
 - classpath/hadoop-auth-2.6.4.jar
-- classpath/hadoop-client-2.6.4.jar
 - classpath/hadoop-common-2.6.4.jar
 - classpath/hadoop-hdfs-2.6.4.jar
-- classpath/hadoop-mapreduce-client-app-2.6.4.jar
-- classpath/hadoop-mapreduce-client-common-2.6.4.jar
-- classpath/hadoop-mapreduce-client-core-2.6.4.jar
-- classpath/hadoop-mapreduce-client-jobclient-2.6.4.jar
-- classpath/hadoop-mapreduce-client-shuffle-2.6.4.jar
-- classpath/hadoop-yarn-api-2.6.4.jar
-- classpath/hadoop-yarn-client-2.6.4.jar
-- classpath/hadoop-yarn-common-2.6.4.jar
-- classpath/hadoop-yarn-server-common-2.6.4.jar
-- classpath/hadoop-yarn-server-nodemanager-2.6.4.jar
 - classpath/hive-storage-api-2.2.1.jar
 - classpath/htrace-core-3.0.4.jar
 - classpath/httpclient-4.2.5.jar
 - classpath/httpcore-4.2.4.jar
 - classpath/jackson-core-asl-1.9.13.jar
-- classpath/jackson-jaxrs-1.9.13.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
-- classpath/jackson-xc-1.9.13.jar
-- classpath/jasper-compiler-5.5.23.jar
 - classpath/jasper-runtime-5.5.23.jar
-- classpath/java-xmlbuilder-0.4.jar
-- classpath/jaxb-api-2.2.2.jar
-- classpath/jaxb-impl-2.2.3-1.jar
-- classpath/jersey-client-1.9.jar
 - classpath/jersey-core-1.9.jar
-- classpath/jersey-guice-1.9.jar
-- classpath/jersey-json-1.9.jar
 - classpath/jersey-server-1.9.jar
-- classpath/jets3t-0.9.0.jar
-- classpath/jettison-1.1.jar
 - classpath/jetty-6.1.26.jar
 - classpath/jetty-util-6.1.26.jar
 - classpath/jline-0.9.94.jar
 - classpath/jsch-0.1.42.jar
 - classpath/jsp-api-2.1.jar
-- classpath/jsr305-1.3.9.jar
-- classpath/kryo-shaded-3.0.3.jar
-- classpath/leveldbjni-all-1.8.jar
 - classpath/log4j-1.2.17.jar
-- classpath/minlog-1.3.0.jar
 - classpath/netty-3.7.0.Final.jar
-- classpath/objenesis-2.1.jar
 - classpath/orc-core-1.4.0.jar
-- classpath/orc-mapreduce-1.4.0.jar
-- classpath/paranamer-2.3.jar
 - classpath/protobuf-java-2.5.0.jar
 - classpath/servlet-api-2.5.jar
-- classpath/snappy-java-1.0.4.1.jar
-- classpath/stax-api-1.0-2.jar
 - classpath/xercesImpl-2.9.1.jar
 - classpath/xml-apis-1.3.04.jar
 - classpath/xmlenc-0.52.jar
 - classpath/xz-1.0.jar
 - classpath/zookeeper-3.4.6.jar
-homepage:
+homepage: https://github.com/yuokada/embulk-output-orc
 licenses:
 - MIT
 metadata: {}