embulk-input-hdfs 0.1.9 → 0.2.1

data/src/test/java/org/embulk/input/hdfs/TestHdfsFileInputPlugin.java

@@ -30,6 +30,7 @@ import javax.annotation.Nullable;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
@@ -69,6 +70,8 @@ public class TestHdfsFileInputPlugin
         assertEquals(true, task.getPartition());
         assertEquals(0, task.getRewindSeconds());
         assertEquals(-1, task.getApproximateNumPartitions());
+        assertEquals(0, task.getSkipHeaderLines());
+        assertEquals(false, task.getDecompression());
     }
 
     @Test(expected = ConfigException.class)
@@ -99,17 +102,14 @@ public class TestHdfsFileInputPlugin
                     }
                 });
 
-                List<String> resultFList = Lists.transform(task.getFiles(), new Function<HdfsPartialFile, String>()
-                {
-                    @Nullable
-                    @Override
-                    public String apply(@Nullable HdfsPartialFile input)
-                    {
-                        assert input != null;
-                        return input.getPath();
+                List<String> resultFList = Lists.newArrayList();
+                for (int i = 0; i < task.getPartialFileList().getTaskCount();i++) {
+                    for (PartialFile partialFile : task.getPartialFileList().get(i)) {
+                        resultFList.add(partialFile.getPath().toString());
                     }
-                });
-                assertEquals(fileList, resultFList);
+                }
+                assertEquals(fileList.size(), resultFList.size());
+                assert fileList.containsAll(resultFList);
                 return emptyTaskReports(taskCount);
             }
         });
@@ -120,8 +120,9 @@ public class TestHdfsFileInputPlugin
     {
         ConfigSource config = getConfigWithDefaultValues();
         config.set("num_partitions", 10);
+        config.set("decompression", true);
         runner.transaction(config, new Control());
-        assertRecords(config, output);
+        assertRecords(config, output, 12);
     }
 
     @Test
@@ -129,8 +130,31 @@ public class TestHdfsFileInputPlugin
     {
         ConfigSource config = getConfigWithDefaultValues();
         config.set("partition", false);
+        config.set("decompression", true);
+        runner.transaction(config, new Control());
+        assertRecords(config, output, 12);
+    }
+
+    @Test
+    public void testHdfsFileInputByOpenWithoutCompressionCodec()
+    {
+        ConfigSource config = getConfigWithDefaultValues();
+        config.set("partition", false);
+        config.set("path", getClass().getResource("/sample_01.csv").getPath());
         runner.transaction(config, new Control());
-        assertRecords(config, output);
+        assertRecords(config, output, 4);
+    }
+
+    @Test
+    public void testStrftime()
+    {
+        ConfigSource config = getConfigWithDefaultValues();
+        config.set("path", "/tmp/%Y-%m-%d");
+        config.set("rewind_seconds", 86400);
+        PluginTask task = config.loadConfig(PluginTask.class);
+        String result = plugin.strftime(task.getJRuby(), task.getPath(), task.getRewindSeconds());
+        String expected = task.getJRuby().runScriptlet("(Time.now - 86400).strftime('/tmp/%Y-%m-%d')").toString();
+        assertEquals(expected, result);
     }
 
     private class Control
@@ -201,10 +225,10 @@ public class TestHdfsFileInputPlugin
         return builder.build();
     }
 
-    private void assertRecords(ConfigSource config, MockPageOutput output)
+    private void assertRecords(ConfigSource config, MockPageOutput output, long size)
     {
         List<Object[]> records = getRecords(config, output);
-        assertEquals(8, records.size());
+        assertEquals(size, records.size());
         {
             Object[] record = records.get(0);
             assertEquals(1L, record[0]);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-hdfs
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.1
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-08 00:00:00.000000000 Z
+date: 2016-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -47,6 +47,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .travis.yml
+- CHENGELOG.md
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -54,18 +55,22 @@ files:
 - config/checkstyle/default.xml
 - example/config.yml
 - example/data.csv
+- example/data2.csv.gz
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - gradlew.bat
 - lib/embulk/input/hdfs.rb
+- src/main/java/org/embulk/input/hdfs/ConfigurationBuilder.java
 - src/main/java/org/embulk/input/hdfs/HdfsFileInputPlugin.java
-- src/main/java/org/embulk/input/hdfs/HdfsFilePartitioner.java
-- src/main/java/org/embulk/input/hdfs/HdfsPartialFile.java
-- src/main/java/org/embulk/input/hdfs/HdfsPartialFileInputStream.java
+- src/main/java/org/embulk/input/hdfs/PartialFile.java
+- src/main/java/org/embulk/input/hdfs/PartialFileInputStream.java
+- src/main/java/org/embulk/input/hdfs/PartialFileInputStreamBuilder.java
+- src/main/java/org/embulk/input/hdfs/PartialFileList.java
 - src/test/java/org/embulk/input/hdfs/TestHdfsFileInputPlugin.java
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
+- src/test/resources/sample_03.csv.gz
 - classpath/activation-1.1.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
@@ -89,23 +94,23 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-input-hdfs-0.1.9.jar
+- classpath/embulk-input-hdfs-0.2.1.jar
 - classpath/gson-2.2.4.jar
-- classpath/hadoop-annotations-2.6.3.jar
-- classpath/hadoop-auth-2.6.3.jar
-- classpath/hadoop-client-2.6.3.jar
-- classpath/hadoop-common-2.6.3.jar
-- classpath/hadoop-hdfs-2.6.3.jar
-- classpath/hadoop-mapreduce-client-app-2.6.3.jar
-- classpath/hadoop-mapreduce-client-common-2.6.3.jar
-- classpath/hadoop-mapreduce-client-core-2.6.3.jar
-- classpath/hadoop-mapreduce-client-jobclient-2.6.3.jar
-- classpath/hadoop-mapreduce-client-shuffle-2.6.3.jar
-- classpath/hadoop-yarn-api-2.6.3.jar
-- classpath/hadoop-yarn-client-2.6.3.jar
-- classpath/hadoop-yarn-common-2.6.3.jar
-- classpath/hadoop-yarn-server-common-2.6.3.jar
-- classpath/hadoop-yarn-server-nodemanager-2.6.3.jar
+- classpath/hadoop-annotations-2.6.4.jar
+- classpath/hadoop-auth-2.6.4.jar
+- classpath/hadoop-client-2.6.4.jar
+- classpath/hadoop-common-2.6.4.jar
+- classpath/hadoop-hdfs-2.6.4.jar
+- classpath/hadoop-mapreduce-client-app-2.6.4.jar
+- classpath/hadoop-mapreduce-client-common-2.6.4.jar
+- classpath/hadoop-mapreduce-client-core-2.6.4.jar
+- classpath/hadoop-mapreduce-client-jobclient-2.6.4.jar
+- classpath/hadoop-mapreduce-client-shuffle-2.6.4.jar
+- classpath/hadoop-yarn-api-2.6.4.jar
+- classpath/hadoop-yarn-client-2.6.4.jar
+- classpath/hadoop-yarn-common-2.6.4.jar
+- classpath/hadoop-yarn-server-common-2.6.4.jar
+- classpath/hadoop-yarn-server-nodemanager-2.6.4.jar
 - classpath/htrace-core-3.0.4.jar
 - classpath/httpclient-4.2.5.jar
 - classpath/httpcore-4.2.4.jar

data/src/main/java/org/embulk/input/hdfs/HdfsFilePartitioner.java

@@ -1,40 +0,0 @@
-package org.embulk.input.hdfs;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Created by takahiro.nakayama on 8/20/15.
- */
-public class HdfsFilePartitioner
-{
-    private FileSystem fs;
-    private Path path;
-    private long numPartitions;
-
-    public HdfsFilePartitioner(FileSystem fs, Path path, long numPartitions)
-    {
-        this.fs = fs;
-        this.path = path;
-        this.numPartitions = numPartitions;
-    }
-
-    public List<HdfsPartialFile> getHdfsPartialFiles()
-            throws IOException
-    {
-        List<HdfsPartialFile> hdfsPartialFiles = new ArrayList<>();
-        long size = fs.getFileStatus(path).getLen();
-        for (int i = 0; i < numPartitions; i++) {
-            long start = size * i / numPartitions;
-            long end = size * (i + 1) / numPartitions;
-            if (start < end) {
-                hdfsPartialFiles.add(new HdfsPartialFile(path.toString(), start, end));
-            }
-        }
-        return hdfsPartialFiles;
-    }
-}

data/src/main/java/org/embulk/input/hdfs/HdfsPartialFile.java

@@ -1,39 +0,0 @@
-package org.embulk.input.hdfs;
-
-/**
- * Created by takahiro.nakayama on 8/20/15.
- */
-// ref. https://github.com/hito4t/embulk-input-filesplit/blob/master/src/main/java/org/embulk/input/filesplit/PartialFile.java
-public class HdfsPartialFile
-{
-    private String path;
-    private long start;
-    private long end;
-
-    public HdfsPartialFile(String path, long start, long end)
-    {
-        this.path = path;
-        this.start = start;
-        this.end = end;
-    }
-
-    // see: http://stackoverflow.com/questions/7625783/jsonmappingexception-no-suitable-constructor-found-for-type-simple-type-class
-    public HdfsPartialFile()
-    {
-    }
-
-    public String getPath()
-    {
-        return path;
-    }
-
-    public long getStart()
-    {
-        return start;
-    }
-
-    public long getEnd()
-    {
-        return end;
-    }
-}