embulk-output-parquet 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c834b87845a6520887275fe90bf447820eee265b
-  data.tar.gz: 6515640ef55fd9d02eeda515d8cbe765431a1dab
+  metadata.gz: e731ad9c445bd5adef66ce8c994fa73dca87252e
+  data.tar.gz: 392706f907d2dd684f2d2778863ae7dd28e4dfca
 SHA512:
-  metadata.gz: 3af4575ecf73aa13157e988db3219e3a79774ba4119966107847091e0ecddd6e611d97413a088c1151f5749decd8fcf757bff4951527099926a234e46efd205d
-  data.tar.gz: 66348d724ad2bb2835a23c7850a831a09f176baa89fb8cfd02fa0fb7a601874664f749d4e848cd89700f0baa2940ef4ac5f2600d0c47a705d24f21f066715955
+  metadata.gz: ed522bf62b23bd0d236c945f4bbffcd3991d86ed93a835bae4739a3be96717e83ed0f584bdd968330ee426a8fd81e3d2cde11090605050bf2d9a3d70d700c603
+  data.tar.gz: a3e6429370cfaefc7e777f8ba27912049d9649e51b8574330b957e2495a29640466e68f8173f052f6c881c33564efbc49fb4b301d59ae9f4b21ce7b58039b684

data/.travis.yml

@@ -0,0 +1 @@
+language: java

data/README.md

@@ -1,6 +1,5 @@
 # Parquet output plugin for Embulk
 
-
 ## Overview
 
 * **Plugin type**: output
@@ -16,7 +15,10 @@
 - **block_size**: A block size of parquet file. (int, default: 134217728(128M))
 - **page_size**: A page size of parquet file. (int, default: 1048576(1M))
 - **compression_codec**: A compression codec. available: UNCOMPRESSED, SNAPPY, GZIP (string, default: UNCOMPRESSED)
-- **timezone**: A timezone for timestamp format. (string, default: UTC)
+- **default_timezone**: Time zone of timestamp columns. This can be overwritten for each column using column_options
+- **default_timestamp_format**: Format of timestamp columns. This can be overwritten for each column using column_options
+- **column_options**: Specify timezone and timestamp format for each column. Format of this option is the same as the official csv formatter. See [document](
+http://www.embulk.org/docs/built-in.html#csv-formatter-plugin).
 
 ## Example
 

data/build.gradle

@@ -13,19 +13,23 @@ repositories {
 }
 configurations {
     provided
+    runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.2.0"
+version = "0.3.0"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.4"
-    provided "org.embulk:embulk-core:0.7.4"
+    compile  "org.embulk:embulk-core:0.7.10"
+    provided "org.embulk:embulk-core:0.7.10"
 
     compile "com.twitter:parquet-hadoop:1.5.0"
     compile "org.apache.hadoop:hadoop-client:2.6.0"
     compile "org.xerial.snappy:snappy-java:1.1.1.6"
+    compile "org.apache.hadoop:hadoop-aws:2.6.0"
 
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.7.7:tests"
+    testCompile "org.embulk:embulk-standards:0.7.7"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/output/EmbulkWriteSupport.java

@@ -24,9 +24,9 @@ public class EmbulkWriteSupport extends WriteSupport<PageReader> {
     final Schema schema;
     RecordConsumer consumer;
     WriteContext writeContext;
-    Map<Integer, TimestampFormatter> timestampFormatters;
+    TimestampFormatter[] timestampFormatters;
 
-    public EmbulkWriteSupport(Schema schema, Map<Integer, TimestampFormatter> timestampFormatters) {
+    public EmbulkWriteSupport(Schema schema, TimestampFormatter[] timestampFormatters) {
         this.schema = schema;
         this.timestampFormatters = timestampFormatters;
     }
@@ -112,7 +112,7 @@ public class EmbulkWriteSupport extends WriteSupport<PageReader> {
         public void timestampColumn(Column column) {
             if (!record.isNull(column)) {
                 Timestamp t = record.getTimestamp(column);
-                String formatted = timestampFormatters.get(column.getIndex()).format(t);
+                String formatted = timestampFormatters[column.getIndex()].format(t);
                 consumer.addBinary(Binary.fromString(formatted));
             }
         }

data/src/main/java/org/embulk/output/ParquetOutputPlugin.java

@@ -1,24 +1,17 @@
 package org.embulk.output;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
 import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
@@ -26,42 +19,54 @@ import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.util.Timestamps;
 import parquet.hadoop.ParquetWriter;
 import parquet.hadoop.api.WriteSupport;
 import parquet.hadoop.metadata.CompressionCodecName;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
 @SuppressWarnings("unused")
 public class ParquetOutputPlugin
         implements OutputPlugin
 {
     public interface PluginTask
-            extends Task, TimestampFormatter.FormatterTask
+            extends Task, TimestampFormatter.Task
     {
         @Config("path_prefix")
-        public String getPathPrefix();
+        String getPathPrefix();
 
         @Config("file_ext")
         @ConfigDefault("\".parquet\"")
-        public String getFileNameExtension();
+        String getFileNameExtension();
 
         @Config("sequence_format")
         @ConfigDefault("\".%03d\"")
-        public String getSequenceFormat();
+        String getSequenceFormat();
 
         @Config("block_size")
         @ConfigDefault("134217728") // 128M
-        public int getBlockSize();
+        int getBlockSize();
 
         @Config("page_size")
         @ConfigDefault("1048576") // 1M
-        public int getPageSize();
+        int getPageSize();
 
         @Config("compression_codec")
         @ConfigDefault("\"UNCOMPRESSED\"")
-        public String getCompressionCodec();
+        String getCompressionCodec();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        Map<String, TimestampColumnOption> getColumnOptions();
     }
 
+    public interface TimestampColumnOption
+            extends Task, TimestampFormatter.TimestampColumnOption
+    { }
+
     public ConfigDiff transaction(ConfigSource config,
             Schema schema, int processorCount,
             OutputPlugin.Control control)
@@ -103,26 +108,13 @@ public class ParquetOutputPlugin
 
         final PageReader reader = new PageReader(schema);
 
-        final Map<Integer, TimestampFormatter> timestampFormatters = newTimestampFormatters(task, schema);
+        final TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
         final EmbulkWriteSupport writeSupport = new EmbulkWriteSupport(schema, timestampFormatters);
         ParquetWriter<PageReader> writer = createParquetWriter(new Path(path), writeSupport, codec, blockSize, pageSize);
 
         return new ParquetTransactionalPageOutput(reader, writer);
     }
 
-    private Map<Integer, TimestampFormatter> newTimestampFormatters(
-            TimestampFormatter.FormatterTask task, Schema schema)
-    {
-        ImmutableMap.Builder<Integer, TimestampFormatter> builder = new ImmutableBiMap.Builder<>();
-        for (Column column : schema.getColumns()) {
-            if (column.getType() instanceof TimestampType) {
-                TimestampType tt = (TimestampType) column.getType();
-                builder.put(column.getIndex(), new TimestampFormatter(tt.getFormat(), task));
-            }
-        }
-        return builder.build();
-    }
-
     private <T> ParquetWriter<T> createParquetWriter(Path path, WriteSupport<T> writeSupport, CompressionCodecName codec, int blockSize, int pageSize) {
         ParquetWriter<T> writer = null;
 

data/src/test/java/org/embulk/output/ParquetOutputPluginTest.java

@@ -0,0 +1,37 @@
+package org.embulk.output;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class ParquetOutputPluginTest {
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void checkDefaultValues() {
+        ConfigSource config = Exec.newConfigSource()
+                .set("path_prefix", "test");
+
+        ParquetOutputPlugin.PluginTask task = config.loadConfig(ParquetOutputPlugin.PluginTask.class);
+        assertEquals(".parquet", task.getFileNameExtension());
+        assertEquals(".%03d", task.getSequenceFormat());
+        assertEquals(134217728, task.getBlockSize());
+        assertEquals(1048576, task.getPageSize());
+        assertEquals("UNCOMPRESSED", task.getCompressionCodec());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void checkColumnsRequired() {
+        ConfigSource config = Exec.newConfigSource();
+
+        config.loadConfig(ParquetOutputPlugin.PluginTask.class);
+    }
+
+
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-parquet
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - OKUNO Akihiro
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-09 00:00:00.000000000 Z
+date: 2015-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -56,7 +57,7 @@ files:
 - lib/embulk/output/parquet.rb
 - src/main/java/org/embulk/output/EmbulkWriteSupport.java
 - src/main/java/org/embulk/output/ParquetOutputPlugin.java
-- src/test/java/org/embulk/output/TestParquetOutputPlugin.java
+- src/test/java/org/embulk/output/ParquetOutputPluginTest.java
 - classpath/activation-1.1.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
@@ -64,6 +65,7 @@ files:
 - classpath/api-util-1.0.0-M20.jar
 - classpath/asm-3.1.jar
 - classpath/avro-1.7.4.jar
+- classpath/aws-java-sdk-1.7.4.jar
 - classpath/commons-beanutils-1.7.0.jar
 - classpath/commons-cli-1.2.jar
 - classpath/commons-codec-1.6.jar
@@ -71,6 +73,7 @@ files:
 - classpath/commons-compress-1.4.1.jar
 - classpath/commons-configuration-1.6.jar
 - classpath/commons-digester-1.8.jar
+- classpath/commons-el-1.0.jar
 - classpath/commons-httpclient-3.1.jar
 - classpath/commons-io-2.4.jar
 - classpath/commons-lang-2.6.jar
@@ -80,10 +83,11 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-output-parquet-0.2.0.jar
+- classpath/embulk-output-parquet-0.3.0.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar
+- classpath/hadoop-aws-2.6.0.jar
 - classpath/hadoop-client-2.6.0.jar
 - classpath/hadoop-common-2.6.0.jar
 - classpath/hadoop-hdfs-2.6.0.jar
@@ -104,6 +108,9 @@ files:
 - classpath/jackson-jaxrs-1.9.13.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
 - classpath/jackson-xc-1.9.13.jar
+- classpath/jasper-compiler-5.5.23.jar
+- classpath/jasper-runtime-5.5.23.jar
+- classpath/java-xmlbuilder-0.4.jar
 - classpath/jaxb-api-2.2.2.jar
 - classpath/jaxb-impl-2.2.3-1.jar
 - classpath/jersey-client-1.9.jar
@@ -111,9 +118,14 @@ files:
 - classpath/jersey-guice-1.9.jar
 - classpath/jersey-json-1.9.jar
 - classpath/jersey-server-1.9.jar
+- classpath/jets3t-0.9.0.jar
 - classpath/jettison-1.1.jar
+- classpath/jetty-6.1.26.jar
 - classpath/jetty-util-6.1.26.jar
 - classpath/jline-0.9.94.jar
+- classpath/joda-time-2.9.1.jar
+- classpath/jsch-0.1.42.jar
+- classpath/jsp-api-2.1.jar
 - classpath/jsr305-1.3.9.jar
 - classpath/leveldbjni-all-1.8.jar
 - classpath/log4j-1.2.17.jar
@@ -128,7 +140,6 @@ files:
 - classpath/parquet-jackson-1.5.0.jar
 - classpath/protobuf-java-2.5.0.jar
 - classpath/servlet-api-2.5.jar
-- classpath/slf4j-log4j12-1.7.5.jar
 - classpath/snappy-java-1.1.1.6.jar
 - classpath/stax-api-1.0-2.jar
 - classpath/xercesImpl-2.9.1.jar

data/src/test/java/org/embulk/output/TestParquetOutputPlugin.java

@@ -1,5 +0,0 @@
-package org.embulk.output;
-
-public class TestParquetOutputPlugin
-{
-}