embulk-input-parquet_hadoop 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11d3bfc5cf66805e9ce41966e90759d5acfbed8f
-  data.tar.gz: 234ecd00864d9c122f01a95ab224c18bdff3ccea
+  metadata.gz: 26ed2eaecbcd68dc340a28050283d99e8d4328d8
+  data.tar.gz: cb043093611c02591c8cd013bef9bde4d17ad410
 SHA512:
-  metadata.gz: 05e661e93e1e5c99edec29e2c83cd68d79f45e8c828afb0aeba822e44003057cf5deb1c69e14cf8eebd32755c19a06766c095e9dd0812bc3feee3f3ae4574c0a
-  data.tar.gz: 9b1119067ba7eaeb18ee4ddaac2322881b6177fb3eab92995784745b72d90e6e0c9e60d0bc552afd652f6556392b008628e9065cbd8762ab48a2275cb2a62944
+  metadata.gz: ccd763d5484bbd3e34ea45c217dc3363b5229848194e5061981fba202300b76790fa2a453388ca51ffe81410e8950d8c8647855c784a49461b4537fffc9fa909
+  data.tar.gz: 6f12ead633dba51521b33154c5ae8b237e6372aeb00d689414e8fae89168e17cac5c08059580fca7532f012b7769f13ad804e82b55187c73cebe2377ee03a017

data/src/main/java/org/embulk/input/parquet_hadoop/ConfigurationFactory.java

@@ -63,7 +63,8 @@ public class ConfigurationFactory
             try {
                 logger.trace("embulk-input-parquet_hadoop: load a config file: {}", f);
                 c.addResource(new File(f).toURI().toURL());
-            } catch (MalformedURLException e) {
+            }
+            catch (MalformedURLException e) {
                 throw new ConfigException(e);
             }
         }

data/src/main/java/org/embulk/input/parquet_hadoop/ParquetHadoopInputPlugin.java

@@ -18,6 +18,7 @@ package org.embulk.input.parquet_hadoop;
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
+import jp.co.cyberagent.parquet.msgpack.read.MessagePackReadSupport;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -43,9 +44,9 @@ import org.embulk.spi.type.Types;
 import org.msgpack.value.Value;
 import org.slf4j.Logger;
 import org.slf4j.bridge.SLF4JBridgeHandler;
-import studio.adtech.parquet.msgpack.read.MessagePackReadSupport;
 
 import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.util.List;
 import java.util.logging.Level;
@@ -100,12 +101,14 @@ public class ParquetHadoopInputPlugin
             List<String> files = Lists.transform(statusList, new Function<FileStatus, String>() {
                     @Nullable
                     @Override
-                    public String apply(@Nullable FileStatus input) {
+                    public String apply(@Nullable FileStatus input)
+                    {
                         return input.getPath().toString();
                     }
             });
             task.setFiles(files);
-        } catch (IOException e) {
+        }
+        catch (IOException e) {
             throw Throwables.propagate(e);
         }
 
@@ -152,7 +155,8 @@ public class ParquetHadoopInputPlugin
             ParquetRowReader<Value> reader;
             try (PluginClassLoaderScope ignored = new PluginClassLoaderScope()) {
                 reader = new ParquetRowReader<>(conf, filePath, new MessagePackReadSupport());
-            } catch (ParquetRuntimeException | IOException e) {
+            }
+            catch (ParquetRuntimeException | IOException e) {
                 throw new DataException(e);
             }
 
@@ -160,7 +164,8 @@ public class ParquetHadoopInputPlugin
             while (true) {
                 try (PluginClassLoaderScope ignored = new PluginClassLoaderScope()) {
                     value = reader.read();
-                } catch (ParquetRuntimeException | IOException e) {
+                }
+                catch (ParquetRuntimeException | IOException e) {
                     throw new DataException(e);
                 }
                 if (value == null) {
@@ -175,7 +180,8 @@ public class ParquetHadoopInputPlugin
 
             try (PluginClassLoaderScope ignored = new PluginClassLoaderScope()) {
                 reader.close();
-            } catch (ParquetRuntimeException | IOException e) {
+            }
+            catch (ParquetRuntimeException | IOException e) {
                 throw new DataException(e);
             }
         }
@@ -195,7 +201,8 @@ public class ParquetHadoopInputPlugin
         return new PageBuilder(Exec.getBufferAllocator(), schema, output);
     }
 
-    private List<FileStatus> listFileStatuses(FileSystem fs, Path rootPath) throws IOException {
+    private List<FileStatus> listFileStatuses(FileSystem fs, Path rootPath) throws IOException
+    {
         List<FileStatus> fileStatuses = Lists.newArrayList();
 
         FileStatus[] entries = fs.globStatus(rootPath, HiddenFileFilter.INSTANCE);
@@ -207,7 +214,8 @@ public class ParquetHadoopInputPlugin
             if (entry.isDirectory()) {
                 List<FileStatus> subEntries = listRecursive(fs, entry);
                 fileStatuses.addAll(subEntries);
-            } else {
+            }
+            else {
                 fileStatuses.add(entry);
             }
         }
@@ -223,7 +231,8 @@ public class ParquetHadoopInputPlugin
             for (FileStatus entry : entries) {
                 statusList.addAll(listRecursive(fs, entry));
             }
-        } else {
+        }
+        else {
             statusList.add(status);
         }
         return statusList;
@@ -241,14 +250,16 @@ public class ParquetHadoopInputPlugin
         Level level;
         try {
             level = Level.parse(task.getParquetLogLevel());
-        } catch (IllegalArgumentException e) {
+        }
+        catch (IllegalArgumentException e) {
             logger.warn("embulk-input-parquet_hadoop: Invalid parquet_log_level", e);
             level = Level.WARNING;
         }
         // invoke static initializer that overrides log level.
         try {
             Class.forName("org.apache.parquet.Log");
-        } catch (ClassNotFoundException e) {
+        }
+        catch (ClassNotFoundException e) {
             logger.warn("", e);
         }
 

data/src/main/java/org/embulk/input/parquet_hadoop/ParquetRowReader.java

@@ -48,7 +48,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-public class ParquetRowReader<T> {
+public class ParquetRowReader<T>
+{
     private static final Logger logger = Exec.getLogger(ParquetRowReader.class);
 
     private final Path filePath;
@@ -69,7 +70,8 @@ public class ParquetRowReader<T> {
     private static final boolean strictTypeChecking = true;
     private static final FilterCompat.Filter filter = FilterCompat.NOOP;
 
-    public ParquetRowReader(Configuration configuration, Path filePath, ReadSupport<T> readSupport) throws IOException {
+    public ParquetRowReader(Configuration configuration, Path filePath, ReadSupport<T> readSupport) throws IOException
+    {
         this.filePath = filePath;
 
         ParquetMetadata parquetMetadata = ParquetFileReader.readFooter(configuration, filePath, ParquetMetadataConverter.NO_FILTER);
@@ -100,7 +102,8 @@ public class ParquetRowReader<T> {
         logger.info("ParquetRowReader initialized will read a total of " + total + " records.");
     }
 
-    private void checkRead() throws IOException {
+    private void checkRead() throws IOException
+    {
         if (current == totalCountLoadedSoFar) {
             PageReadStore pages = reader.readNextRowGroup();
             if (pages == null) {
@@ -119,7 +122,8 @@ public class ParquetRowReader<T> {
      * @throws IOException
      * @throws ParquetDecodingException
      */
-    public T read() throws IOException {
+    public T read() throws IOException
+    {
         T currentValue = null;
         boolean recordFound = false;
         while (!recordFound) {
@@ -134,7 +138,8 @@ public class ParquetRowReader<T> {
 
                 try {
                     currentValue = recordReader.read();
-                } catch (RecordMaterializer.RecordMaterializationException e) {
+                }
+                catch (RecordMaterializer.RecordMaterializationException e) {
                     // this might throw, but it's fatal if it does.
                     unmaterializableRecordCounter.incErrors(e);
                     logger.debug("skipping a corrupt record");
@@ -157,7 +162,8 @@ public class ParquetRowReader<T> {
                 recordFound = true;
 
                 logger.debug("read value: {}", currentValue);
-            } catch (RuntimeException e) {
+            }
+            catch (RuntimeException e) {
                 throw new ParquetDecodingException(
                         String.format("Can not read value at %d in block %d in file %s", current, currentBlock, filePath), e);
             }
@@ -166,11 +172,13 @@ public class ParquetRowReader<T> {
         return currentValue;
     }
 
-    public void close() throws IOException {
+    public void close() throws IOException
+    {
         reader.close();
     }
 
-    private static <K, V> Map<K, Set<V>> toSetMultiMap(Map<K, V> map) {
+    private static <K, V> Map<K, Set<V>> toSetMultiMap(Map<K, V> map)
+    {
         Map<K, Set<V>> setMultiMap = new HashMap<>();
         for (Map.Entry<K, V> entry : map.entrySet()) {
             Set<V> set = new HashSet<>();

data/src/main/java/org/embulk/input/parquet_hadoop/PluginClassLoaderScope.java

@@ -25,20 +25,23 @@ package org.embulk.input.parquet_hadoop;
  * hadoop jars is not in classpath of system class loader.
  * So we need to set context class loader to plugins' class loader.
  */
-class PluginClassLoaderScope implements AutoCloseable {
+class PluginClassLoaderScope implements AutoCloseable
+{
     private static final ClassLoader PLUGIN_CLASS_LOADER =
             ParquetHadoopInputPlugin.class.getClassLoader();
 
     private final ClassLoader original;
 
-    public PluginClassLoaderScope() {
+    public PluginClassLoaderScope()
+    {
         Thread current = Thread.currentThread();
         this.original = current.getContextClassLoader();
         Thread.currentThread().setContextClassLoader(PLUGIN_CLASS_LOADER);
     }
 
     @Override
-    public void close() {
+    public void close()
+    {
         Thread.currentThread().setContextClassLoader(original);
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-parquet_hadoop
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Koji AGAWA
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-08 00:00:00.000000000 Z
+date: 2017-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -69,7 +69,7 @@ files:
 - classpath/curator-client-2.7.1.jar
 - classpath/curator-framework-2.7.1.jar
 - classpath/curator-recipes-2.7.1.jar
-- classpath/embulk-input-parquet_hadoop-0.1.0.jar
+- classpath/embulk-input-parquet_hadoop-0.1.1.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.7.3.jar
 - classpath/hadoop-auth-2.7.3.jar
@@ -117,7 +117,7 @@ files:
 - classpath/parquet-format-2.3.0-incubating.jar
 - classpath/parquet-hadoop-1.8.1.jar
 - classpath/parquet-jackson-1.8.1.jar
-- classpath/parquet-msgpack-0.1.0.jar
+- classpath/parquet-msgpack-0.1.1.jar
 - classpath/protobuf-java-2.5.0.jar
 - classpath/servlet-api-2.5.jar
 - classpath/slf4j-api-1.7.24.jar