embulk-input-hdfs 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e125254c0d5e9b6837c42a6dc3a6701c8897395
-  data.tar.gz: 1b2c5aba3844aabd1e7d7eb1af7a177eb9ea8ae0
+  metadata.gz: c428003a976a3148f1b59e7dd54c7ec870ed3bce
+  data.tar.gz: a2941dce02f97452b54938bf73b5d1dceba5f4c5
 SHA512:
-  metadata.gz: 70d2eb9a08bcd17da14a3c4815d2cb88ab6941cd33d7a08d913e1006349d4690a28a3994797120edec3aab7c9a6978aaf4368419cae674b5fce65b5c33b0039c
-  data.tar.gz: 65a03af4289551d279a60b028b77602d606c6a7816ec328fd910c5f6d0a171833260675350766e9f22f5426781a9a9cd0cf6e275780ba43a57c8b7ae105786bf
+  metadata.gz: 02ca4fc8c3c82571296eb8da1d3a61533d0017d49c2712886e0872fbebf58ae8656412c61e3b6f1d6aaf71078ad1e886029a91907df92e236b2c56d3cbbb6083
+  data.tar.gz: f218db27f48822f33427ac0ebcccf5c58c41cdec92486cf9ceb1ddb240f5332744b8d65daf6a672e18488aa675c597ef56d34c994b7c04fe218a5e9c3428c91d

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.1.7"
+version = "0.1.8"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/hdfs/HdfsFileInputPlugin.java

@@ -58,7 +58,7 @@ public class HdfsFileInputPlugin implements FileInputPlugin
 
         @Config("num_partitions") // this parameter is the approximate value.
         @ConfigDefault("-1")      // Default: Runtime.getRuntime().availableProcessors()
-        public int getApproximateNumPartitions();
+        public long getApproximateNumPartitions();
 
         public List<HdfsPartialFile> getFiles();
         public void setFiles(List<HdfsPartialFile> hdfsFiles);
@@ -246,25 +246,25 @@ public class HdfsFileInputPlugin implements FileInputPlugin
             }
         });
 
-        int totalFileLength = 0;
+        long totalFileLength = 0;
         for (Path path : pathList) {
             totalFileLength += fs.getFileStatus(path).getLen();
         }
 
         // TODO: optimum allocation of resources
-        int approximateNumPartitions =
+        long approximateNumPartitions =
                 (task.getApproximateNumPartitions() <= 0) ? Runtime.getRuntime().availableProcessors() : task.getApproximateNumPartitions();
-        int partitionSizeByOneTask = totalFileLength / approximateNumPartitions;
+        long partitionSizeByOneTask = totalFileLength / approximateNumPartitions;
 
         List<HdfsPartialFile> hdfsPartialFiles = new ArrayList<>();
         for (Path path : pathList) {
-            int fileLength = (int) fs.getFileStatus(path).getLen(); // declare `fileLength` here because this is used below.
+            long fileLength = fs.getFileStatus(path).getLen(); // declare `fileLength` here because this is used below.
             if (fileLength <= 0) {
                 logger.info("embulk-input-hdfs: Skip the 0 byte target file: {}", path);
                 continue;
             }
 
-            int numPartitions;
+            long numPartitions;
             if (path.toString().endsWith(".gz") || path.toString().endsWith(".bz2") || path.toString().endsWith(".lzo")) {
                 numPartitions = 1;
             }

data/src/main/java/org/embulk/input/hdfs/HdfsFilePartitioner.java

@@ -14,9 +14,9 @@ public class HdfsFilePartitioner
 {
     private FileSystem fs;
     private Path path;
-    private int numPartitions;
+    private long numPartitions;
 
-    public HdfsFilePartitioner(FileSystem fs, Path path, int numPartitions)
+    public HdfsFilePartitioner(FileSystem fs, Path path, long numPartitions)
     {
         this.fs = fs;
         this.path = path;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-hdfs
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - takahiro.nakayama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-01 00:00:00.000000000 Z
+date: 2015-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -82,7 +82,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-input-hdfs-0.1.7.jar
+- classpath/embulk-input-hdfs-0.1.8.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar