embulk-output-orc 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 376187534c3d236c20656a656ed04d5c858e9a83
-  data.tar.gz: 4d4b74153e1e2fe0248b4ba5d097723dbc42022e
+  metadata.gz: 9a4d872e06384fd47cf8c1707398234974e0ecb3
+  data.tar.gz: bb3855c44c55f16bc5e1707f09e3a1b6c683aee6
 SHA512:
-  metadata.gz: 888dab123d9cea81defbd51c0d5b812bd6c90e84c578927a72a491a8fffb4fac61fcced52ba7d759fdd1c36d9dba08a424c2d704b6f3f1ae51bbfdb3fac50728
-  data.tar.gz: bb7c0de447e7bf98b2ad7e7ba2b309854b0beaf37b6c10fd2a82be3736f6e39c8a576c985be0c62e9223ab24cb1a9bd4662e64d12974be490cd042002c933699
+  metadata.gz: a16e0c62d57089d5ff431021b97586a8203a423ab3f8797a4e2274f26cb168a91a59ae72d6d42b4f2540209565d05d9e3f1ae66af242e52c90ab1f239afe90cd
+  data.tar.gz: 21f77bd871793f06b014ce9df5c08ab7e0f0d610b7cc69b749e415a1f181891288ba06211dbe3d769bfd756bfaaf344d1eeecd42db9f085a7feba28560921c96

data/README.md

@@ -1,6 +1,7 @@
 # Orc output plugin for Embulk
 
 [![Build Status](https://travis-ci.org/yuokada/embulk-output-orc.svg?branch=master)](https://travis-ci.org/yuokada/embulk-output-orc)
+[![Gem Version](https://badge.fury.io/rb/embulk-output-orc.svg)](https://badge.fury.io/rb/embulk-output-orc)
 
 ## Overview
 
@@ -12,6 +13,7 @@
 ## Configuration
 
 - **path_prefix**: A prefix of output path. (string, required)
+  - support: `file`, `s3n` and `s3a`.
 - **file_ext**: An extension of output file. (string, default: `.orc`)
 - **sequence_format**: (string, default: `.%03d`)
 - **buffer_size**: Set the ORC buffer size (integer, default: `10000`)
@@ -20,6 +22,12 @@
 - **overwrite**: (LocalFileSystem only) Overwrite if output files already exist. (boolean, default: `false`)
 - **default_from_timezone** Time zone of timestamp columns. This can be overwritten for each column using column_options (DateTimeZone, default: `UTC`)
 
+- **auth_method**: name of mechanism to authenticate requests (basic, env, instance, profile, properties, anonymous, or session. default: basic)  
+  see: https://github.com/embulk/embulk-input-s3#configuration
+
+    - `env`, `basic`, `profile`, `default`, `session`, `anonymous`, `properties`
+    
+
 ## Example
 
 ```yaml
@@ -32,6 +40,17 @@ out:
   overwrite:   true
 ```
 
+## ChangeLog
+
+### ver 0.2.0
+
+- support: output to s3
+
+    - `s3n`, `s3a` protocol
+
+### ver 0.1.0
+
+- initial release
 
 ## Build
 

data/build.gradle

@@ -8,13 +8,16 @@ import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
+    maven {
+        url "http://dl.bintray.com/embulk-input-s3/maven"
+    }
 }
 configurations {
     provided
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.1.0"
+version = "0.2.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -27,6 +30,10 @@ dependencies {
     compile "org.apache.orc:orc-core:1.4.0"
     compile "org.apache.hadoop:hadoop-hdfs:2.6.4"
 
+    compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8'
+    compile "com.amazonaws:aws-java-sdk-s3:1.10.33"
+    compile "org.apache.hadoop:hadoop-aws:2.7.3"
+
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.29:tests"
     testCompile "org.embulk:embulk-standards:0.8.29"

data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java

@@ -29,6 +29,8 @@ import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.util.Timestamps;
+import org.embulk.util.aws.credentials.AwsCredentials;
+import org.embulk.util.aws.credentials.AwsCredentialsTask;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
@@ -41,7 +43,7 @@ public class OrcOutputPlugin
         implements OutputPlugin
 {
     public interface PluginTask
-            extends Task, TimestampFormatter.Task
+            extends Task, TimestampFormatter.Task, AwsCredentialsTask
     {
         @Config("path_prefix")
         String getPathPrefix();
@@ -176,7 +178,7 @@ public class OrcOutputPlugin
         return oschema;
     }
 
-    private Configuration getHadoopConfiguration()
+    private Configuration getHadoopConfiguration(PluginTask task)
     {
         Configuration conf = new Configuration();
 
@@ -185,14 +187,25 @@ public class OrcOutputPlugin
         conf.set("fs.file.impl", LocalFileSystem.class.getName());
         // see: https://stackoverflow.com/questions/20833444/how-to-set-objects-in-hadoop-configuration
 
+        AwsCredentials.getAWSCredentialsProvider(task);
+        if (task.getAccessKeyId().isPresent()) {
+            conf.set("fs.s3a.access.key", task.getAccessKeyId().get());
+            conf.set("fs.s3n.awsAccessKeyId", task.getAccessKeyId().get());
+        }
+        if (task.getSecretAccessKey().isPresent()) {
+            conf.set("fs.s3a.secret.key", task.getSecretAccessKey().get());
+            conf.set("fs.s3n.awsSecretAccessKey", task.getSecretAccessKey().get());
+        }
+
         return conf;
     }
 
     private Writer createWriter(PluginTask task, Schema schema, int processorIndex)
     {
-        final TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
+        final TimestampFormatter[] timestampFormatters = Timestamps
+                .newTimestampColumnFormatters(task, schema, task.getColumnOptions());
 
-        Configuration conf = getHadoopConfiguration();
+        Configuration conf = getHadoopConfiguration(task);
         TypeDescription oschema = getSchema(schema);
 
         // see: https://groups.google.com/forum/#!topic/vertx/lLb-slzpWVg
@@ -256,7 +269,8 @@ public class OrcOutputPlugin
             this.writer = writer;
 
             // formatter
-            DateTimeZone defaultTimeZone = DateTimeZone.forTimeZone(task.getDefaultFromTimeZone().toTimeZone());
+            DateTimeZone defaultTimeZone = DateTimeZone
+                    .forTimeZone(task.getDefaultFromTimeZone().toTimeZone());
             formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(defaultTimeZone);
         }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-orc
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - yuokada
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-23 00:00:00.000000000 Z
+date: 2017-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,18 +62,26 @@ files:
 - src/main/java/org/embulk/output/orc/OrcOutputPlugin.java
 - src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java
 - src/test/java/org/embulk/output/orc/TestOrcOutputPlugin.java
+- classpath/activation-1.1.jar
 - classpath/aircompressor-0.3.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
 - classpath/api-asn1-api-1.0.0-M20.jar
 - classpath/api-util-1.0.0-M20.jar
 - classpath/asm-3.1.jar
+- classpath/avro-1.7.4.jar
+- classpath/aws-java-sdk-1.7.4.jar
+- classpath/aws-java-sdk-core-1.10.33.jar
+- classpath/aws-java-sdk-kms-1.10.33.jar
+- classpath/aws-java-sdk-s3-1.10.33.jar
+- classpath/commons-beanutils-1.7.0.jar
 - classpath/commons-cli-1.2.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-collections-3.2.2.jar
 - classpath/commons-compress-1.4.1.jar
 - classpath/commons-configuration-1.6.jar
 - classpath/commons-daemon-1.0.13.jar
+- classpath/commons-digester-1.8.jar
 - classpath/commons-el-1.0.jar
 - classpath/commons-httpclient-3.1.jar
 - classpath/commons-io-2.4.jar
@@ -81,33 +89,51 @@ files:
 - classpath/commons-logging-1.1.3.jar
 - classpath/commons-math3-3.1.1.jar
 - classpath/commons-net-3.1.jar
-- classpath/curator-client-2.6.0.jar
-- classpath/curator-framework-2.6.0.jar
-- classpath/embulk-output-orc-0.1.0.jar
+- classpath/curator-client-2.7.1.jar
+- classpath/curator-framework-2.7.1.jar
+- classpath/curator-recipes-2.7.1.jar
+- classpath/embulk-output-orc-0.2.0.jar
+- classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/gson-2.2.4.jar
-- classpath/hadoop-annotations-2.6.4.jar
-- classpath/hadoop-auth-2.6.4.jar
-- classpath/hadoop-common-2.6.4.jar
+- classpath/hadoop-annotations-2.7.3.jar
+- classpath/hadoop-auth-2.7.3.jar
+- classpath/hadoop-aws-2.7.3.jar
+- classpath/hadoop-common-2.7.3.jar
 - classpath/hadoop-hdfs-2.6.4.jar
 - classpath/hive-storage-api-2.2.1.jar
 - classpath/htrace-core-3.0.4.jar
-- classpath/httpclient-4.2.5.jar
-- classpath/httpcore-4.2.4.jar
+- classpath/htrace-core-3.1.0-incubating.jar
+- classpath/httpclient-4.3.6.jar
+- classpath/httpcore-4.3.3.jar
 - classpath/jackson-core-asl-1.9.13.jar
+- classpath/jackson-jaxrs-1.8.3.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
+- classpath/jackson-xc-1.8.3.jar
 - classpath/jasper-runtime-5.5.23.jar
+- classpath/java-xmlbuilder-0.4.jar
+- classpath/jaxb-api-2.2.2.jar
+- classpath/jaxb-impl-2.2.3-1.jar
+- classpath/jcl-over-slf4j-1.7.12.jar
 - classpath/jersey-core-1.9.jar
+- classpath/jersey-json-1.9.jar
 - classpath/jersey-server-1.9.jar
+- classpath/jets3t-0.9.0.jar
+- classpath/jettison-1.1.jar
 - classpath/jetty-6.1.26.jar
 - classpath/jetty-util-6.1.26.jar
 - classpath/jline-0.9.94.jar
+- classpath/joda-time-2.9.9.jar
 - classpath/jsch-0.1.42.jar
 - classpath/jsp-api-2.1.jar
+- classpath/jsr305-3.0.0.jar
 - classpath/log4j-1.2.17.jar
 - classpath/netty-3.7.0.Final.jar
 - classpath/orc-core-1.4.0.jar
+- classpath/paranamer-2.3.jar
 - classpath/protobuf-java-2.5.0.jar
 - classpath/servlet-api-2.5.jar
+- classpath/snappy-java-1.0.4.1.jar
+- classpath/stax-api-1.0-2.jar
 - classpath/xercesImpl-2.9.1.jar
 - classpath/xml-apis-1.3.04.jar
 - classpath/xmlenc-0.52.jar