embulk-output-parquet 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e731ad9c445bd5adef66ce8c994fa73dca87252e
-  data.tar.gz: 392706f907d2dd684f2d2778863ae7dd28e4dfca
+  metadata.gz: 887eeba43ad66ae6159048504e253542162a8988
+  data.tar.gz: a04dba91a7abeecc957d7265f809c9bf1276ae0e
 SHA512:
-  metadata.gz: ed522bf62b23bd0d236c945f4bbffcd3991d86ed93a835bae4739a3be96717e83ed0f584bdd968330ee426a8fd81e3d2cde11090605050bf2d9a3d70d700c603
-  data.tar.gz: a3e6429370cfaefc7e777f8ba27912049d9649e51b8574330b957e2495a29640466e68f8173f052f6c881c33564efbc49fb4b301d59ae9f4b21ce7b58039b684
+  metadata.gz: 56ec1d8ad587e73a97ad769bbaad04793bb28ccf9119c638cde85e6dd38e06626fe8343ecfa9cc2e497cec78724366c2f3d8fdd7656b892c44ad32d9c4d65718
+  data.tar.gz: a99aa07e1b1507c6d375c4d2a21fa6d99341bf72f4022059eda90c88900f7597f687be5baed3c26728187212f59028e8679906fc8f6bba02795e1927f5e1ddcd

data/README.md

@@ -19,13 +19,19 @@
 - **default_timestamp_format**: Format of timestamp columns. This can be overwritten for each column using column_options
 - **column_options**: Specify timezone and timestamp format for each column. Format of this option is the same as the official csv formatter. See [document](
 http://www.embulk.org/docs/built-in.html#csv-formatter-plugin).
+- **extra_configurations**: Add extra entries to Configuration which will be passed to ParquetWriter
+- **overwrite**: Overwrite if output files already exist. (default: fail if files exist)
 
 ## Example
 
 ```yaml
 out:
   type: parquet
-  path_prefix: file:///data/output
+  path_prefix: s3a://bucket/keys
+  extra_configuration:
+    fs.s3a.access.key: 'your_access_key'
+    fs.s3a.secret.key: 'your_secret_access_key'
+
 ```
 
 ## Build

data/build.gradle

@@ -7,25 +7,22 @@ import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
-    maven {
-        url "http://maven.twttr.com/"
-    }
 }
 configurations {
     provided
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.3.0"
+version = "0.4.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.10"
     provided "org.embulk:embulk-core:0.7.10"
 
-    compile "com.twitter:parquet-hadoop:1.5.0"
-    compile "org.apache.hadoop:hadoop-client:2.6.0"
+    compile "org.apache.parquet:parquet-hadoop:1.8.1"
+    compile "org.apache.hadoop:hadoop-client:2.7.1"
+    compile "org.apache.hadoop:hadoop-aws:2.7.1"
     compile "org.xerial.snappy:snappy-java:1.1.1.6"
-    compile "org.apache.hadoop:hadoop-aws:2.6.0"
 
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.7.7:tests"

data/src/main/java/org/embulk/output/EmbulkWriteSupport.java

@@ -1,19 +1,19 @@
 package org.embulk.output;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Type;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
-import parquet.hadoop.api.WriteSupport;
-import parquet.io.api.Binary;
-import parquet.io.api.RecordConsumer;
-import parquet.schema.MessageType;
-import parquet.schema.PrimitiveType;
-import parquet.schema.PrimitiveType.PrimitiveTypeName;
-import parquet.schema.Type;
 
 import java.util.ArrayList;
 import java.util.HashMap;

data/src/main/java/org/embulk/output/EmbulkWriterBuilder.java

@@ -0,0 +1,30 @@
+package org.embulk.output;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.TimestampFormatter;
+
+public class EmbulkWriterBuilder extends ParquetWriter.Builder<PageReader, EmbulkWriterBuilder> {
+    final Schema schema;
+    final TimestampFormatter[] timestampFormatters;
+
+    public EmbulkWriterBuilder(Path file, Schema schema, TimestampFormatter[] timestampFormatters) {
+        super(file);
+        this.schema = schema;
+        this.timestampFormatters = timestampFormatters;
+    }
+
+    @Override
+    protected EmbulkWriterBuilder self() {
+        return this;
+    }
+
+    @Override
+    protected WriteSupport<PageReader> getWriteSupport(Configuration conf) {
+        return new EmbulkWriteSupport(schema, timestampFormatters);
+    }
+}

data/src/main/java/org/embulk/output/ParquetOutputPlugin.java

@@ -5,6 +5,9 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.parquet.hadoop.ParquetFileWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -20,9 +23,6 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.util.Timestamps;
-import parquet.hadoop.ParquetWriter;
-import parquet.hadoop.api.WriteSupport;
-import parquet.hadoop.metadata.CompressionCodecName;
 
 import java.io.IOException;
 import java.util.List;
@@ -30,11 +30,9 @@ import java.util.Map;
 
 @SuppressWarnings("unused")
 public class ParquetOutputPlugin
-        implements OutputPlugin
-{
+        implements OutputPlugin {
     public interface PluginTask
-            extends Task, TimestampFormatter.Task
-    {
+            extends Task, TimestampFormatter.Task {
         @Config("path_prefix")
         String getPathPrefix();
 
@@ -47,11 +45,13 @@ public class ParquetOutputPlugin
         String getSequenceFormat();
 
         @Config("block_size")
-        @ConfigDefault("134217728") // 128M
+        @ConfigDefault("134217728")
+            // 128M
         int getBlockSize();
 
         @Config("page_size")
-        @ConfigDefault("1048576") // 1M
+        @ConfigDefault("1048576")
+            // 1M
         int getPageSize();
 
         @Config("compression_codec")
@@ -61,16 +61,23 @@ public class ParquetOutputPlugin
         @Config("column_options")
         @ConfigDefault("{}")
         Map<String, TimestampColumnOption> getColumnOptions();
+
+        @Config("extra_configurations")
+        @ConfigDefault("{}")
+        Map<String, String> getExtraConfigurations();
+
+        @Config("overwrite")
+        @ConfigDefault("false")
+        boolean getOverwrite();
     }
 
     public interface TimestampColumnOption
-            extends Task, TimestampFormatter.TimestampColumnOption
-    { }
+            extends Task, TimestampFormatter.TimestampColumnOption {
+    }
 
     public ConfigDiff transaction(ConfigSource config,
-            Schema schema, int processorCount,
-            OutputPlugin.Control control)
-    {
+                                  Schema schema, int processorCount,
+                                  OutputPlugin.Control control) {
         PluginTask task = config.loadConfig(PluginTask.class);
 
         //TODO
@@ -80,65 +87,78 @@ public class ParquetOutputPlugin
     }
 
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
-            OutputPlugin.Control control)
-    {
+                             Schema schema, int processorCount,
+                             OutputPlugin.Control control) {
         throw new UnsupportedOperationException("parquet output plugin does not support resuming");
     }
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
-            List<TaskReport> successTaskReports)
-    {
+                        Schema schema, int processorCount,
+                        List<TaskReport> successTaskReports) {
         //TODO
     }
 
-    public TransactionalPageOutput open(TaskSource taskSource, final Schema schema, int processorIndex)
-    {
+    public TransactionalPageOutput open(TaskSource taskSource, final Schema schema, int processorIndex) {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
+        final PageReader reader = new PageReader(schema);
+        final ParquetWriter<PageReader> writer = createWriter(task, schema, processorIndex);
+
+        return new ParquetTransactionalPageOutput(reader, writer);
+    }
+
+    private String buildPath(PluginTask task, int processorIndex) {
         final String pathPrefix = task.getPathPrefix();
         final String pathSuffix = task.getFileNameExtension();
         final String sequenceFormat = task.getSequenceFormat();
+        return pathPrefix + String.format(sequenceFormat, processorIndex) + pathSuffix;
+    }
+
+    private ParquetWriter<PageReader> createWriter(PluginTask task, Schema schema, int processorIndex) {
+        final TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
+
+        final Path path = new Path(buildPath(task, processorIndex));
         final CompressionCodecName codec = CompressionCodecName.valueOf(task.getCompressionCodec());
         final int blockSize = task.getBlockSize();
         final int pageSize = task.getPageSize();
+        final Configuration conf = createConfiguration(task.getExtraConfigurations());
+        final boolean overwrite = task.getOverwrite();
 
-        final String path = pathPrefix + String.format(sequenceFormat, processorIndex) + pathSuffix;
-
-        final PageReader reader = new PageReader(schema);
-
-        final TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
-        final EmbulkWriteSupport writeSupport = new EmbulkWriteSupport(schema, timestampFormatters);
-        ParquetWriter<PageReader> writer = createParquetWriter(new Path(path), writeSupport, codec, blockSize, pageSize);
+        ParquetWriter<PageReader> writer = null;
+        try {
+            EmbulkWriterBuilder builder = new EmbulkWriterBuilder(path, schema, timestampFormatters)
+                    .withCompressionCodec(codec)
+                    .withRowGroupSize(blockSize)
+                    .withPageSize(pageSize)
+                    .withDictionaryPageSize(pageSize)
+                    .withConf(conf);
+
+            if (overwrite) {
+                builder.withWriteMode(ParquetFileWriter.Mode.OVERWRITE);
+            }
 
-        return new ParquetTransactionalPageOutput(reader, writer);
+            writer = builder.build();
+        } catch (IOException e) {
+            Throwables.propagate(e);
+        }
+        return writer;
     }
 
-    private <T> ParquetWriter<T> createParquetWriter(Path path, WriteSupport<T> writeSupport, CompressionCodecName codec, int blockSize, int pageSize) {
-        ParquetWriter<T> writer = null;
-
+    private Configuration createConfiguration(Map<String, String> extra) {
         Configuration conf = new Configuration();
+
+        // Default values
         conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName());
         conf.set("fs.file.impl", LocalFileSystem.class.getName());
-        conf.setClassLoader(this.getClass().getClassLoader());
 
-        try {
-            writer = new ParquetWriter<>(
-                    path,
-                    writeSupport,
-                    codec,
-                    blockSize,
-                    pageSize,
-                    pageSize,
-                    ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED,
-                    ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED,
-                    ParquetWriter.DEFAULT_WRITER_VERSION,
-                    conf);
-        } catch (IOException e) {
-            Throwables.propagate(e);
+        // Optional values
+        for (Map.Entry<String, String> entry : extra.entrySet()) {
+            conf.set(entry.getKey(), entry.getValue());
         }
-        return writer;
+
+        conf.setClassLoader(this.getClass().getClassLoader());
+
+        return conf;
     }
 
     class ParquetTransactionalPageOutput implements TransactionalPageOutput {
@@ -157,7 +177,7 @@ public class ParquetOutputPlugin
                 while (reader.nextRecord()) {
                     writer.write(reader);
                 }
-            } catch(IOException e) {
+            } catch (IOException e) {
                 Throwables.propagate(e);
             }
         }

data/src/test/java/org/embulk/output/ParquetOutputPluginTest.java

@@ -1,5 +1,6 @@
 package org.embulk.output;
 
+import org.apache.hadoop.conf.Configuration;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
@@ -7,6 +8,11 @@ import org.embulk.spi.Exec;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+
 import static org.junit.Assert.*;
 
 public class ParquetOutputPluginTest {
@@ -24,6 +30,7 @@ public class ParquetOutputPluginTest {
         assertEquals(134217728, task.getBlockSize());
         assertEquals(1048576, task.getPageSize());
         assertEquals("UNCOMPRESSED", task.getCompressionCodec());
+        assertFalse(task.getOverwrite());
     }
 
     @Test(expected = ConfigException.class)
@@ -33,5 +40,25 @@ public class ParquetOutputPluginTest {
         config.loadConfig(ParquetOutputPlugin.PluginTask.class);
     }
 
+    @Test
+    public void checkExtraConfigurations() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+        ConfigSource map = Exec.newConfigSource()
+                .set("foo", "bar");
+
+        ConfigSource config = Exec.newConfigSource()
+                .set("path_prefix", "test")
+                .setNested("extra_configurations", map);
+
+        ParquetOutputPlugin.PluginTask task = config.loadConfig(ParquetOutputPlugin.PluginTask.class);
+
+        Map<String, String> extra = task.getExtraConfigurations();
+        assertTrue(extra.containsKey("foo"));
+        assertEquals("bar", extra.get("foo"));
 
+        ParquetOutputPlugin plugin = new ParquetOutputPlugin();
+        Method method = ParquetOutputPlugin.class.getDeclaredMethod("createConfiguration", Map.class);
+        method.setAccessible(true);
+        Configuration conf = (Configuration) method.invoke(plugin, extra);
+        assertEquals("bar", conf.get("foo"));
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-parquet
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - OKUNO Akihiro
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-18 00:00:00.000000000 Z
+date: 2016-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -56,6 +56,7 @@ files:
 - gradlew.bat
 - lib/embulk/output/parquet.rb
 - src/main/java/org/embulk/output/EmbulkWriteSupport.java
+- src/main/java/org/embulk/output/EmbulkWriterBuilder.java
 - src/main/java/org/embulk/output/ParquetOutputPlugin.java
 - src/test/java/org/embulk/output/ParquetOutputPluginTest.java
 - classpath/activation-1.1.jar
@@ -73,43 +74,40 @@ files:
 - classpath/commons-compress-1.4.1.jar
 - classpath/commons-configuration-1.6.jar
 - classpath/commons-digester-1.8.jar
-- classpath/commons-el-1.0.jar
 - classpath/commons-httpclient-3.1.jar
 - classpath/commons-io-2.4.jar
 - classpath/commons-lang-2.6.jar
 - classpath/commons-logging-1.1.3.jar
 - classpath/commons-math3-3.1.1.jar
 - classpath/commons-net-3.1.jar
-- classpath/curator-client-2.6.0.jar
-- classpath/curator-framework-2.6.0.jar
-- classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-output-parquet-0.3.0.jar
+- classpath/curator-client-2.7.1.jar
+- classpath/curator-framework-2.7.1.jar
+- classpath/curator-recipes-2.7.1.jar
+- classpath/embulk-output-parquet-0.4.0.jar
 - classpath/gson-2.2.4.jar
-- classpath/hadoop-annotations-2.6.0.jar
-- classpath/hadoop-auth-2.6.0.jar
-- classpath/hadoop-aws-2.6.0.jar
-- classpath/hadoop-client-2.6.0.jar
-- classpath/hadoop-common-2.6.0.jar
-- classpath/hadoop-hdfs-2.6.0.jar
-- classpath/hadoop-mapreduce-client-app-2.6.0.jar
-- classpath/hadoop-mapreduce-client-common-2.6.0.jar
-- classpath/hadoop-mapreduce-client-core-2.6.0.jar
-- classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
-- classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
-- classpath/hadoop-yarn-api-2.6.0.jar
-- classpath/hadoop-yarn-client-2.6.0.jar
-- classpath/hadoop-yarn-common-2.6.0.jar
-- classpath/hadoop-yarn-server-common-2.6.0.jar
-- classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
-- classpath/htrace-core-3.0.4.jar
+- classpath/hadoop-annotations-2.7.1.jar
+- classpath/hadoop-auth-2.7.1.jar
+- classpath/hadoop-aws-2.7.1.jar
+- classpath/hadoop-client-2.7.1.jar
+- classpath/hadoop-common-2.7.1.jar
+- classpath/hadoop-hdfs-2.7.1.jar
+- classpath/hadoop-mapreduce-client-app-2.7.1.jar
+- classpath/hadoop-mapreduce-client-common-2.7.1.jar
+- classpath/hadoop-mapreduce-client-core-2.7.1.jar
+- classpath/hadoop-mapreduce-client-jobclient-2.7.1.jar
+- classpath/hadoop-mapreduce-client-shuffle-2.7.1.jar
+- classpath/hadoop-yarn-api-2.7.1.jar
+- classpath/hadoop-yarn-client-2.7.1.jar
+- classpath/hadoop-yarn-common-2.7.1.jar
+- classpath/hadoop-yarn-server-common-2.7.1.jar
+- classpath/hadoop-yarn-server-nodemanager-2.7.1.jar
+- classpath/htrace-core-3.1.0-incubating.jar
 - classpath/httpclient-4.2.5.jar
 - classpath/httpcore-4.2.4.jar
 - classpath/jackson-core-asl-1.9.13.jar
 - classpath/jackson-jaxrs-1.9.13.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
 - classpath/jackson-xc-1.9.13.jar
-- classpath/jasper-compiler-5.5.23.jar
-- classpath/jasper-runtime-5.5.23.jar
 - classpath/java-xmlbuilder-0.4.jar
 - classpath/jaxb-api-2.2.2.jar
 - classpath/jaxb-impl-2.2.3-1.jar
@@ -126,18 +124,18 @@ files:
 - classpath/joda-time-2.9.1.jar
 - classpath/jsch-0.1.42.jar
 - classpath/jsp-api-2.1.jar
-- classpath/jsr305-1.3.9.jar
+- classpath/jsr305-3.0.0.jar
 - classpath/leveldbjni-all-1.8.jar
 - classpath/log4j-1.2.17.jar
 - classpath/netty-3.7.0.Final.jar
+- classpath/netty-all-4.0.23.Final.jar
 - classpath/paranamer-2.3.jar
-- classpath/parquet-column-1.5.0.jar
-- classpath/parquet-common-1.5.0.jar
-- classpath/parquet-encoding-1.5.0.jar
-- classpath/parquet-format-2.1.0.jar
-- classpath/parquet-generator-1.5.0.jar
-- classpath/parquet-hadoop-1.5.0.jar
-- classpath/parquet-jackson-1.5.0.jar
+- classpath/parquet-column-1.8.1.jar
+- classpath/parquet-common-1.8.1.jar
+- classpath/parquet-encoding-1.8.1.jar
+- classpath/parquet-format-2.3.0-incubating.jar
+- classpath/parquet-hadoop-1.8.1.jar
+- classpath/parquet-jackson-1.8.1.jar
 - classpath/protobuf-java-2.5.0.jar
 - classpath/servlet-api-2.5.jar
 - classpath/snappy-java-1.1.1.6.jar