embulk-output-parquet 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1ef34fa1ab2ea085e926b70700d4bff09e7bb16
-  data.tar.gz: d2d9df28c5ed603995193552104466da98d400eb
+  metadata.gz: 3aea0935831ddd2ff4368ea66315ad14083d502d
+  data.tar.gz: 847aa476f2f85f8ac7ac0978e12adbcd0fdc636c
 SHA512:
-  metadata.gz: 6a48e6ac6438c1cd56bf431b69ea8e980a54bff54290299d2da7a733c8defe4746230ba7afd2c5446b2d2ce8d42aa7cf74b6ca95c6dd8473cfefa278b891813f
-  data.tar.gz: adcfe86af5337ab4f41b2eb78ae84a2e20f71d40ad63652029d83cb50368ba40139f59f9f38a1bd8a7e7425c6838f96e39f733bbda6230ca9e37c1ae17b09f80
+  metadata.gz: 2af5d5cbc8c0376f8c881a072caa8b45ce245506e6c3330a656e254f7f6f44b0853ffef552bbcbe27c5c2796ee0c55d5d748690f876381ebff95b2e3c3213c72
+  data.tar.gz: 763c60722277b4e0e92446caca2d774ffdc4ae14484eb70754c7fe69251e491bcb555ad3746072c856ced6532985b61aeddbeadb9236885d0fa1e69283750346

data/README.md

@@ -19,8 +19,11 @@
 - **default_timestamp_format**: Format of timestamp columns. This can be overwritten for each column using column_options
 - **column_options**: Specify timezone and timestamp format for each column. Format of this option is the same as the official csv formatter. See [document](
 http://www.embulk.org/docs/built-in.html#csv-formatter-plugin).
+- **config_files**: List of path to Hadoop's configuration files (array of string, default: `[]`)
 - **extra_configurations**: Add extra entries to Configuration which will be passed to ParquetWriter
 - **overwrite**: Overwrite if output files already exist. (default: fail if files exist)
+- **enablesigv4**: Enable Signature Version 4 Signing Process for S3 eu-central-1(Frankfurt) region
+- **addUTF8**: If true, string and timestamp columns are stored with OriginalType.UTF8 (boolean, default false) 
 
 ## Example
 

data/build.gradle

@@ -14,15 +14,15 @@ configurations {
     runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
 }
 
-version = "0.5.0"
+version = "0.6.0"
 
 sourceCompatibility = 1.7
 
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.10"
-    provided "org.embulk:embulk-core:0.7.10"
+    compile  "org.embulk:embulk-core:0.8.28"
+    provided "org.embulk:embulk-core:0.8.28"
 
     compile "org.apache.parquet:parquet-hadoop:1.8.1"
     compile "org.apache.hadoop:hadoop-client:2.7.1"
@@ -30,8 +30,8 @@ dependencies {
     compile "org.xerial.snappy:snappy-java:1.1.1.6"
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.7.7:tests"
-    testCompile "org.embulk:embulk-standards:0.7.7"
+    testCompile "org.embulk:embulk-core:0.8.28:tests"
+    testCompile "org.embulk:embulk-standards:0.8.28"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/config/checkstyle/checkstyle.xml

@@ -66,9 +66,9 @@
                  LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
                  LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
         </module>
-        <module name="RightCurly">
-            <property name="option" value="alone"/>
-        </module>
+        <!--<module name="RightCurly">-->
+            <!--<property name="option" value="alone"/>-->
+        <!--</module>-->
         <module name="GenericWhitespace"/>
         <module name="WhitespaceAfter"/>
         <module name="NoWhitespaceBefore"/>

data/config/checkstyle/default.xml

@@ -56,9 +56,9 @@
                  LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
                  LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
         </module>
-        <module name="RightCurly">
-            <property name="option" value="alone"/>
-        </module>
+        <!--<module name="RightCurly">-->
+            <!--<property name="option" value="alone"/>-->
+        <!--</module>-->
         <module name="GenericWhitespace"/>
         <module name="WhitespaceAfter"/>
         <module name="NoWhitespaceBefore"/>

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 11 00:26:20 PDT 2015
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.7-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip

data/src/main/java/org/embulk/output/EmbulkWriteSupport.java

@@ -5,6 +5,7 @@ import org.apache.parquet.hadoop.api.WriteSupport;
 import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.io.api.RecordConsumer;
 import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.OriginalType;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
 import org.apache.parquet.schema.Type;
@@ -27,11 +28,13 @@ public class EmbulkWriteSupport
     RecordConsumer consumer;
     WriteContext writeContext;
     TimestampFormatter[] timestampFormatters;
+    boolean addUTF8;
 
-    public EmbulkWriteSupport(Schema schema, TimestampFormatter[] timestampFormatters)
+    public EmbulkWriteSupport(Schema schema, TimestampFormatter[] timestampFormatters, boolean addUTF8)
     {
         this.schema = schema;
         this.timestampFormatters = timestampFormatters;
+        this.addUTF8 = addUTF8;
     }
 
     @Override
@@ -73,7 +76,12 @@ public class EmbulkWriteSupport
 
     private MessageType convertSchema(Schema schema)
     {
-        SchemaConvertColumnVisitor visitor = new SchemaConvertColumnVisitor();
+        SchemaConvertColumnVisitor visitor = null;
+        if (addUTF8) {
+            visitor = new SchemaConvertColumnVisitorWithUTF8();
+        } else {
+            visitor = new SchemaConvertColumnVisitor();
+        }
         schema.visitColumns(visitor);
         String messageName = "embulk";
         return new MessageType(messageName, visitor.getConvertedFields());
@@ -123,6 +131,12 @@ public class EmbulkWriteSupport
             }
         }
 
+        @Override
+        public void jsonColumn(Column column)
+        {
+            throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.");
+        }
+
         @Override
         public void timestampColumn(Column column)
         {
@@ -163,6 +177,12 @@ public class EmbulkWriteSupport
             fields.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName()));
         }
 
+        @Override
+        public void jsonColumn(Column column)
+        {
+            throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.");
+        }
+
         @Override
         public void timestampColumn(Column column)
         {
@@ -175,4 +195,21 @@ public class EmbulkWriteSupport
             return fields;
         }
     }
+
+    class SchemaConvertColumnVisitorWithUTF8 extends SchemaConvertColumnVisitor
+    {
+        @Override
+        public void stringColumn(Column column)
+        {
+            fields.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName(), OriginalType.UTF8));
+        }
+
+        @Override
+        public void timestampColumn(Column column)
+        {
+            // formatted as string
+            fields.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName(), OriginalType.UTF8));
+        }
+    }
+
 }

data/src/main/java/org/embulk/output/EmbulkWriterBuilder.java

@@ -13,12 +13,14 @@ public class EmbulkWriterBuilder
 {
     final Schema schema;
     final TimestampFormatter[] timestampFormatters;
+    final boolean addUTF8;
 
-    public EmbulkWriterBuilder(Path file, Schema schema, TimestampFormatter[] timestampFormatters)
+    public EmbulkWriterBuilder(Path file, Schema schema, TimestampFormatter[] timestampFormatters, boolean addUTF8)
     {
         super(file);
         this.schema = schema;
         this.timestampFormatters = timestampFormatters;
+        this.addUTF8 = addUTF8;
     }
 
     @Override
@@ -30,6 +32,6 @@ public class EmbulkWriterBuilder
     @Override
     protected WriteSupport<PageReader> getWriteSupport(Configuration conf)
     {
-        return new EmbulkWriteSupport(schema, timestampFormatters);
+        return new EmbulkWriteSupport(schema, timestampFormatters, addUTF8);
     }
 }

data/src/main/java/org/embulk/output/ParquetOutputPlugin.java

@@ -1,5 +1,6 @@
 package org.embulk.output;
 
+import com.amazonaws.SDKGlobalConfiguration;
 import com.google.common.base.Throwables;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalFileSystem;
@@ -8,13 +9,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.parquet.hadoop.ParquetFileWriter;
 import org.apache.parquet.hadoop.ParquetWriter;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
+import org.embulk.config.*;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
@@ -23,8 +18,11 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.util.Timestamps;
+import org.embulk.output.parquet.ClassLoaderSwap;
 
+import java.io.File;
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.util.List;
 import java.util.Map;
 
@@ -64,6 +62,10 @@ public class ParquetOutputPlugin
         @ConfigDefault("{}")
         Map<String, TimestampColumnOption> getColumnOptions();
 
+        @Config("config_files")
+        @ConfigDefault("[]")
+        List<String> getConfigFiles();
+
         @Config("extra_configurations")
         @ConfigDefault("{}")
         Map<String, String> getExtraConfigurations();
@@ -71,6 +73,14 @@ public class ParquetOutputPlugin
         @Config("overwrite")
         @ConfigDefault("false")
         boolean getOverwrite();
+
+        @Config("enablesigv4")
+        @ConfigDefault("false")
+        String getSignature();
+
+        @Config("addUTF8")
+        @ConfigDefault("false")
+        boolean getAddUTF8();
     }
 
     public interface TimestampColumnOption
@@ -79,27 +89,30 @@ public class ParquetOutputPlugin
     }
 
     public ConfigDiff transaction(ConfigSource config,
-            Schema schema, int processorCount,
-            OutputPlugin.Control control)
+                                  Schema schema, int processorCount,
+                                  OutputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
         //TODO
 
-        control.run(task.dump());
+
+        try (@SuppressWarnings("unchecked") ClassLoaderSwap clswp = new ClassLoaderSwap(this.getClass())) {
+            control.run(task.dump());
+        }
         return Exec.newConfigDiff();
     }
 
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int processorCount,
-            OutputPlugin.Control control)
+                             Schema schema, int processorCount,
+                             OutputPlugin.Control control)
     {
         throw new UnsupportedOperationException("parquet output plugin does not support resuming");
     }
 
     public void cleanup(TaskSource taskSource,
-            Schema schema, int processorCount,
-            List<TaskReport> successTaskReports)
+                        Schema schema, int processorCount,
+                        List<TaskReport> successTaskReports)
     {
         //TODO
     }
@@ -124,18 +137,22 @@ public class ParquetOutputPlugin
 
     private ParquetWriter<PageReader> createWriter(PluginTask task, Schema schema, int processorIndex)
     {
+        //In case of using Frankurt (eu-central-1) with Signature Version 4 Signing Process
+        System.setProperty(SDKGlobalConfiguration.ENABLE_S3_SIGV4_SYSTEM_PROPERTY, task.getSignature());
+
         final TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
+        final boolean addUTF8 = task.getAddUTF8();
 
         final Path path = new Path(buildPath(task, processorIndex));
         final CompressionCodecName codec = CompressionCodecName.valueOf(task.getCompressionCodec());
         final int blockSize = task.getBlockSize();
         final int pageSize = task.getPageSize();
-        final Configuration conf = createConfiguration(task.getExtraConfigurations());
+        final Configuration conf = createConfiguration(task.getExtraConfigurations(), task.getConfigFiles());
         final boolean overwrite = task.getOverwrite();
 
         ParquetWriter<PageReader> writer = null;
         try {
-            EmbulkWriterBuilder builder = new EmbulkWriterBuilder(path, schema, timestampFormatters)
+            EmbulkWriterBuilder builder = new EmbulkWriterBuilder(path, schema, timestampFormatters, addUTF8)
                     .withCompressionCodec(codec)
                     .withRowGroupSize(blockSize)
                     .withPageSize(pageSize)
@@ -147,14 +164,13 @@ public class ParquetOutputPlugin
             }
 
             writer = builder.build();
-        }
-        catch (IOException e) {
+        } catch (IOException e) {
             Throwables.propagate(e);
         }
         return writer;
     }
 
-    private Configuration createConfiguration(Map<String, String> extra)
+    private Configuration createConfiguration(Map<String, String> extra, List<String> configFiles)
     {
         Configuration conf = new Configuration();
 
@@ -162,6 +178,15 @@ public class ParquetOutputPlugin
         conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName());
         conf.set("fs.file.impl", LocalFileSystem.class.getName());
 
+        for (String configFile : configFiles) {
+            File file = new File(configFile);
+            try {
+                conf.addResource(file.toURI().toURL());
+            } catch (MalformedURLException e) {
+                throw new ConfigException(e);
+            }
+        }
+
         // Optional values
         for (Map.Entry<String, String> entry : extra.entrySet()) {
             conf.set(entry.getKey(), entry.getValue());
@@ -192,8 +217,7 @@ public class ParquetOutputPlugin
                 while (reader.nextRecord()) {
                     writer.write(reader);
                 }
-            }
-            catch (IOException e) {
+            } catch (IOException e) {
                 Throwables.propagate(e);
             }
         }
@@ -204,8 +228,7 @@ public class ParquetOutputPlugin
             try {
                 writer.close();
                 writer = null;
-            }
-            catch (IOException e) {
+            } catch (IOException e) {
                 Throwables.propagate(e);
             }
         }

data/src/main/java/org/embulk/output/parquet/ClassLoaderSwap.java

@@ -0,0 +1,25 @@
+package org.embulk.output.parquet;
+
+/**
+ * This class is based on embulk-input-parquet_hadoop PluginClassLoaderScope.java
+ */
+public class ClassLoaderSwap<T> implements AutoCloseable
+{
+    private final ClassLoader pluginClassLoader;
+    private final ClassLoader orgClassLoader;
+    private final Thread curThread;
+
+    public ClassLoaderSwap(Class<T> pluginClass)
+    {
+        this.curThread = Thread.currentThread();
+        this.pluginClassLoader = pluginClass.getClassLoader();
+        this.orgClassLoader = curThread.getContextClassLoader();
+        curThread.setContextClassLoader(pluginClassLoader);
+    }
+
+    @Override
+    public void close()
+    {
+        curThread.setContextClassLoader(orgClassLoader);
+    }
+}

data/src/test/java/org/embulk/output/ParquetOutputPluginTest.java

@@ -10,6 +10,8 @@ import org.junit.Test;
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
@@ -34,6 +36,7 @@ public class ParquetOutputPluginTest
         assertEquals(1048576, task.getPageSize());
         assertEquals("UNCOMPRESSED", task.getCompressionCodec());
         assertFalse(task.getOverwrite());
+        assertEquals("false", task.getSignature());
     }
 
     @Test(expected = ConfigException.class)
@@ -62,9 +65,9 @@ public class ParquetOutputPluginTest
         assertEquals("bar", extra.get("foo"));
 
         ParquetOutputPlugin plugin = new ParquetOutputPlugin();
-        Method method = ParquetOutputPlugin.class.getDeclaredMethod("createConfiguration", Map.class);
+        Method method = ParquetOutputPlugin.class.getDeclaredMethod("createConfiguration", Map.class, List.class);
         method.setAccessible(true);
-        Configuration conf = (Configuration) method.invoke(plugin, extra);
+        Configuration conf = (Configuration) method.invoke(plugin, extra, new ArrayList<String>());
         assertEquals("bar", conf.get("foo"));
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-parquet
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - OKUNO Akihiro
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-11 00:00:00.000000000 Z
+date: 2018-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -60,6 +60,7 @@ files:
 - src/main/java/org/embulk/output/EmbulkWriteSupport.java
 - src/main/java/org/embulk/output/EmbulkWriterBuilder.java
 - src/main/java/org/embulk/output/ParquetOutputPlugin.java
+- src/main/java/org/embulk/output/parquet/ClassLoaderSwap.java
 - src/test/java/org/embulk/output/ParquetOutputPluginTest.java
 - classpath/activation-1.1.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
@@ -85,7 +86,7 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-output-parquet-0.5.0.jar
+- classpath/embulk-output-parquet-0.6.0.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.7.1.jar
 - classpath/hadoop-auth-2.7.1.jar
@@ -123,7 +124,6 @@ files:
 - classpath/jetty-6.1.26.jar
 - classpath/jetty-util-6.1.26.jar
 - classpath/jline-0.9.94.jar
-- classpath/joda-time-2.9.9.jar
 - classpath/jsch-0.1.42.jar
 - classpath/jsp-api-2.1.jar
 - classpath/jsr305-3.0.0.jar