embulk-input-hdfs 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c428003a976a3148f1b59e7dd54c7ec870ed3bce
-  data.tar.gz: a2941dce02f97452b54938bf73b5d1dceba5f4c5
+  metadata.gz: ce120e7049f33e30dd23af9f8b7bcedc1a246457
+  data.tar.gz: a2dc70fee60be2ab535df3549e99304e751a7b7a
 SHA512:
-  metadata.gz: 02ca4fc8c3c82571296eb8da1d3a61533d0017d49c2712886e0872fbebf58ae8656412c61e3b6f1d6aaf71078ad1e886029a91907df92e236b2c56d3cbbb6083
-  data.tar.gz: f218db27f48822f33427ac0ebcccf5c58c41cdec92486cf9ceb1ddb240f5332744b8d65daf6a672e18488aa675c597ef56d34c994b7c04fe218a5e9c3428c91d
+  metadata.gz: a37baf6f948dff41f694457dc9ea9ea9270e41473642114d4dc7a569c61550471b9dbc440478c638fe56ba79956f043097e2129302d3ae12511bdc9d33cef994
+  data.tar.gz: 16922c84dcdb9715cb1b0377886b36192acdda31a037352e18df83895f33b09a9f275cd02b9662f02ee411725a6dae65950cfc256c707f639312810839018037

data/.gitignore

@@ -6,6 +6,9 @@
 /classpath/
 build/
 .idea
+/.settings/
+/.metadata/
+.classpath
+.project
 *.iml
 .ruby-version
-

data/.travis.yml

@@ -0,0 +1,9 @@
+language: java
+jdk:
+  - openjdk7
+  - oraclejdk7
+  - oraclejdk8
+script:
+  - ./gradlew test
+after_success:
+  - ./gradlew jacocoTestReport coveralls

data/README.md

@@ -1,4 +1,6 @@
 # Hdfs file input plugin for Embulk
+[![Build Status](https://travis-ci.org/civitaspo/embulk-input-hdfs.svg)](https://travis-ci.org/civitaspo/embulk-input-hdfs)
+[![Coverage Status](https://coveralls.io/repos/civitaspo/embulk-input-hdfs/badge.svg?branch=master&service=github)](https://coveralls.io/github/civitaspo/embulk-input-hdfs?branch=master)
 
 Read files on Hdfs.
 
@@ -16,6 +18,7 @@ Read files on Hdfs.
 - **rewind_seconds** When you use Date format in input_path property, the format is executed by using the time which is Now minus this property.
 - **partition** when this is true, partition input files and increase task count. (default: `true`)
 - **num_partitions** number of partitions. (default: `Runtime.getRuntime().availableProcessors()`)
+- **skip_header_lines** Skip this number of lines first. Set 1 if the file has header line. (default: `0`)
 
 ## Example
 
@@ -23,8 +26,8 @@ Read files on Hdfs.
 in:
   type: hdfs
   config_files:
-    - /opt/analytics/etc/hadoop/conf/core-site.xml
-    - /opt/analytics/etc/hadoop/conf/hdfs-site.xml
+    - /etc/hadoop/conf/core-site.xml
+    - /etc/hadoop/conf/hdfs-site.xml
   config:
     fs.defaultFS: 'hdfs://hadoop-nn1:8020'
     dfs.replication: 1
@@ -106,4 +109,4 @@ $ ./gradlew gem
 ```
 $ ./gradlew classpath
 $ bundle exec embulk run -I lib example.yml
-```
+```

data/build.gradle

@@ -2,6 +2,9 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
+    id "checkstyle"
+    id "com.github.kt3k.coveralls" version "2.4.0"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -12,18 +15,19 @@ configurations {
     provided
 }
 
-version = "0.1.8"
+version = "0.1.9"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.0"
-    provided "org.embulk:embulk-core:0.7.0"
+    compile  "org.embulk:embulk-core:0.8.+"
+    provided "org.embulk:embulk-core:0.8.+"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    compile 'org.apache.hadoop:hadoop-client:2.6.0'
-    compile 'com.google.guava:guava:15.0'
+    compile 'org.apache.hadoop:hadoop-client:2.6.+'
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.+:tests"
+    testCompile "org.embulk:embulk-standards:0.8.+"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -33,6 +37,29 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete "classpath" }
 
+jacocoTestReport {
+    reports {
+        xml.enabled = true // coveralls plugin depends on xml format report
+        html.enabled = true
+    }
+}
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
     jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
     script "${project.name}.gemspec"
@@ -57,7 +84,7 @@ task gemspec {
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
-  spec.authors       = ["takahiro.nakayama"]
+  spec.authors       = ["Civitaspo"]
   spec.summary       = %[Hdfs file input plugin for Embulk]
   spec.description   = %[Reads files stored on Hdfs.]
   spec.email         = ["civitaspo@gmail.com"]

data/config/checkstyle/checkstyle.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/config/checkstyle/default.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/example/config.yml

@@ -0,0 +1,35 @@
+hdfs_example: &hdfs_example
+  config_files:
+    - /etc/hadoop/conf/core-site.xml
+    - /etc/hadoop/conf/hdfs-site.xml
+  config:
+    fs.defaultFS: 'hdfs://hadoop-nn1:8020'
+    fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
+    fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
+
+local_fs_example: &local_fs_example
+  config:
+    fs.defaultFS: 'file:///'
+    fs.hdfs.impl: 'org.apache.hadoop.fs.LocalFileSystem'
+    fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
+
+in:
+  type: hdfs
+  <<: *local_fs_example
+  path: example/data.csv
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    header_line: true
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: comment, type: string}
+
+out:
+  type: stdout

data/example/data.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 11 00:26:20 PDT 2015
+#Wed Jan 13 12:41:02 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip

data/src/main/java/org/embulk/input/hdfs/HdfsFileInputPlugin.java

@@ -24,18 +24,26 @@ import org.jruby.embed.ScriptingContainer;
 import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.SequenceInputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-public class HdfsFileInputPlugin implements FileInputPlugin
+public class HdfsFileInputPlugin
+        implements FileInputPlugin
 {
     private static final Logger logger = Exec.getLogger(HdfsFileInputPlugin.class);
+    private static FileSystem fs;
 
-    public interface PluginTask extends Task
+    public interface PluginTask
+            extends Task
     {
         @Config("config_files")
         @ConfigDefault("[]")
@@ -60,7 +68,12 @@ public class HdfsFileInputPlugin implements FileInputPlugin
         @ConfigDefault("-1")      // Default: Runtime.getRuntime().availableProcessors()
         public long getApproximateNumPartitions();
 
+        @Config("skip_header_lines") // Skip this number of lines first. Set 1 if the file has header line.
+        @ConfigDefault("0")          // The reason why the parameter is configured is that this plugin splits files.
+        public int getSkipHeaderLines();
+
         public List<HdfsPartialFile> getFiles();
+
         public void setFiles(List<HdfsPartialFile> hdfsFiles);
 
         @ConfigInject
@@ -81,8 +94,8 @@ public class HdfsFileInputPlugin implements FileInputPlugin
                 throw new PathNotFoundException(pathString);
             }
 
+            logger.debug("embulk-input-hdfs: Loading target files: {}", originalFileList);
             task.setFiles(allocateHdfsFilesToTasks(task, getFs(task), originalFileList));
-            logger.info("embulk-input-hdfs: Loading target files: {}", originalFileList);
         }
         catch (IOException e) {
             logger.error(e.getMessage());
@@ -104,8 +117,8 @@ public class HdfsFileInputPlugin implements FileInputPlugin
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-                             int taskCount,
-                             FileInputPlugin.Control control)
+            int taskCount,
+            FileInputPlugin.Control control)
     {
         control.run(taskSource, taskCount);
 
@@ -127,8 +140,8 @@ public class HdfsFileInputPlugin implements FileInputPlugin
 
     @Override
     public void cleanup(TaskSource taskSource,
-                        int taskCount,
-                        List<TaskReport> successTaskReports)
+            int taskCount,
+            List<TaskReport> successTaskReports)
     {
     }
 
@@ -138,15 +151,22 @@ public class HdfsFileInputPlugin implements FileInputPlugin
         final PluginTask task = taskSource.loadTask(PluginTask.class);
 
         InputStream input;
+        final HdfsPartialFile file = task.getFiles().get(taskIndex);
         try {
-            input = openInputStream(task, task.getFiles().get(taskIndex));
+            if (file.getStart() > 0 && task.getSkipHeaderLines() > 0) {
+                input = new SequenceInputStream(getHeadersInputStream(task, file), openInputStream(task, file));
+            }
+            else {
+                input = openInputStream(task, file);
+            }
         }
         catch (IOException e) {
             logger.error(e.getMessage());
             throw new RuntimeException(e);
         }
 
-        return new InputStreamTransactionalFileInput(task.getBufferAllocator(), input) {
+        return new InputStreamTransactionalFileInput(task.getBufferAllocator(), input)
+        {
             @Override
             public void abort()
             { }
@@ -159,6 +179,42 @@ public class HdfsFileInputPlugin implements FileInputPlugin
         };
     }
 
+    private InputStream getHeadersInputStream(PluginTask task, HdfsPartialFile partialFile)
+            throws IOException
+    {
+        FileSystem fs = getFs(task);
+        ByteArrayOutputStream header = new ByteArrayOutputStream();
+        int skippedHeaders = 0;
+
+        try (BufferedInputStream in = new BufferedInputStream(fs.open(new Path(partialFile.getPath())))) {
+            while (true) {
+                int c = in.read();
+                if (c < 0) {
+                    break;
+                }
+
+                header.write(c);
+
+                if (c == '\n') {
+                    skippedHeaders++;
+                }
+                else if (c == '\r') {
+                    int c2 = in.read();
+                    if (c2 == '\n') {
+                        header.write(c2);
+                    }
+                    skippedHeaders++;
+                }
+
+                if (skippedHeaders >= task.getSkipHeaderLines()) {
+                    break;
+                }
+            }
+        }
+        header.close();
+        return new ByteArrayInputStream(header.toByteArray());
+    }
+
     private static HdfsPartialFileInputStream openInputStream(PluginTask task, HdfsPartialFile partialFile)
             throws IOException
     {
@@ -168,6 +224,18 @@ public class HdfsFileInputPlugin implements FileInputPlugin
     }
 
     private static FileSystem getFs(final PluginTask task)
+        throws IOException
+    {
+        if (fs == null) {
+            setFs(task);
+            return fs;
+        }
+        else {
+            return fs;
+        }
+    }
+
+    private static FileSystem setFs(final PluginTask task)
             throws IOException
     {
         Configuration configuration = new Configuration();
@@ -177,18 +245,25 @@ public class HdfsFileInputPlugin implements FileInputPlugin
             configuration.addResource(file.toURI().toURL());
         }
 
-        for (Map.Entry<String, String> entry: task.getConfig().entrySet()) {
+        for (Map.Entry<String, String> entry : task.getConfig().entrySet()) {
             configuration.set(entry.getKey(), entry.getValue());
         }
 
-        return FileSystem.get(configuration);
+        // For debug
+        for (Map.Entry<String, String> entry : configuration) {
+            logger.trace("{}: {}", entry.getKey(), entry.getValue());
+        }
+        logger.debug("Resource Files: {}", configuration);
+
+        fs = FileSystem.get(configuration);
+        return fs;
     }
 
-    private String strftime(final String raw, final int rewind_seconds)
+    private String strftime(final String raw, final int rewindSeconds)
     {
         ScriptingContainer jruby = new ScriptingContainer();
         Object resolved = jruby.runScriptlet(
-                String.format("(Time.now - %s).strftime('%s')", String.valueOf(rewind_seconds), raw));
+                String.format("(Time.now - %s).strftime('%s')", String.valueOf(rewindSeconds), raw));
         return resolved.toString();
     }
 
@@ -255,6 +330,9 @@ public class HdfsFileInputPlugin implements FileInputPlugin
         long approximateNumPartitions =
                 (task.getApproximateNumPartitions() <= 0) ? Runtime.getRuntime().availableProcessors() : task.getApproximateNumPartitions();
         long partitionSizeByOneTask = totalFileLength / approximateNumPartitions;
+        if (partitionSizeByOneTask <= 0) {
+            partitionSizeByOneTask = 1;
+        }
 
         List<HdfsPartialFile> hdfsPartialFiles = new ArrayList<>();
         for (Path path : pathList) {

data/src/main/java/org/embulk/input/hdfs/HdfsFilePartitioner.java

@@ -23,7 +23,8 @@ public class HdfsFilePartitioner
         this.numPartitions = numPartitions;
     }
 
-    public List<HdfsPartialFile> getHdfsPartialFiles() throws IOException
+    public List<HdfsPartialFile> getHdfsPartialFiles()
+            throws IOException
     {
         List<HdfsPartialFile> hdfsPartialFiles = new ArrayList<>();
         long size = fs.getFileStatus(path).getLen();

data/src/main/java/org/embulk/input/hdfs/HdfsPartialFile.java

@@ -1,7 +1,5 @@
 package org.embulk.input.hdfs;
 
-import org.apache.hadoop.fs.Path;
-
 /**
  * Created by takahiro.nakayama on 8/20/15.
  */
@@ -20,7 +18,9 @@ public class HdfsPartialFile
     }
 
     // see: http://stackoverflow.com/questions/7625783/jsonmappingexception-no-suitable-constructor-found-for-type-simple-type-class
-    public HdfsPartialFile() { }
+    public HdfsPartialFile()
+    {
+    }
 
     public String getPath()
     {
@@ -36,5 +36,4 @@ public class HdfsPartialFile
     {
         return end;
     }
-
 }

data/src/main/java/org/embulk/input/hdfs/HdfsPartialFileInputStream.java

@@ -6,7 +6,8 @@ import java.io.InputStream;
 import java.io.PushbackInputStream;
 
 // ref. https://github.com/hito4t/embulk-input-filesplit/blob/master/src/main/java/org/embulk/input/filesplit/PartialFileInputStream.java
-public class HdfsPartialFileInputStream extends InputStream
+public class HdfsPartialFileInputStream
+        extends InputStream
 {
     private final PushbackInputStream original;
     private long start;
@@ -23,13 +24,15 @@ public class HdfsPartialFileInputStream extends InputStream
     }
 
     @Override
-    public int read(byte[] b) throws IOException
+    public int read(byte[] b)
+            throws IOException
     {
         return read(b, 0, b.length);
     }
 
     @Override
-    public int read(byte[] b, int off, int len) throws IOException
+    public int read(byte[] b, int off, int len)
+            throws IOException
     {
         initializeIfNeeded();
 
@@ -45,7 +48,7 @@ public class HdfsPartialFileInputStream extends InputStream
 
         current += read;
         if (current >= end) {
-            for (int i = Math.max((int)(end - 1 - current + read), 0); i < read; i++) {
+            for (int i = Math.max((int) (end - 1 - current + read), 0); i < read; i++) {
                 if (b[off + i] == '\n') {
                     eof = true;
                     return i + 1;
@@ -65,7 +68,8 @@ public class HdfsPartialFileInputStream extends InputStream
     }
 
     @Override
-    public int read() throws IOException
+    public int read()
+            throws IOException
     {
         initializeIfNeeded();
 
@@ -91,7 +95,8 @@ public class HdfsPartialFileInputStream extends InputStream
     }
 
     @Override
-    public long skip(long n) throws IOException
+    public long skip(long n)
+            throws IOException
     {
         throw new IOException("Skip not supported.");
         /*
@@ -102,18 +107,21 @@ public class HdfsPartialFileInputStream extends InputStream
     }
 
     @Override
-    public int available() throws IOException
+    public int available()
+            throws IOException
     {
         return 0;
     }
 
     @Override
-    public void close() throws IOException
+    public void close()
+            throws IOException
     {
         original.close();
     }
 
-    private void initializeIfNeeded() throws IOException
+    private void initializeIfNeeded()
+            throws IOException
     {
         if (current >= start) {
             return;
@@ -144,7 +152,8 @@ public class HdfsPartialFileInputStream extends InputStream
         }
     }
 
-    private int prefetch() throws IOException
+    private int prefetch()
+            throws IOException
     {
         int c = original.read();
         if (c >= 0) {
@@ -152,4 +161,4 @@ public class HdfsPartialFileInputStream extends InputStream
         }
         return c;
     }
-}
+}

data/src/test/java/org/embulk/input/hdfs/TestHdfsFileInputPlugin.java

@@ -1,5 +1,232 @@
 package org.embulk.input.hdfs;
 
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.fs.Path;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.input.hdfs.HdfsFileInputPlugin.PluginTask;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.util.Pages;
+import org.embulk.standards.CsvParserPlugin;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
 public class TestHdfsFileInputPlugin
 {
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Rule
+    public ExpectedException exception = ExpectedException.none();
+
+    private Logger logger = runtime.getExec().getLogger(TestHdfsFileInputPlugin.class);
+    private HdfsFileInputPlugin plugin;
+    private FileInputRunner runner;
+    private MockPageOutput output;
+    private Path path;
+
+    @Before
+    public void createResources()
+    {
+        plugin = new HdfsFileInputPlugin();
+        runner = new FileInputRunner(runtime.getInstance(HdfsFileInputPlugin.class));
+        output = new MockPageOutput();
+        path = new Path(new File(getClass().getResource("/sample_01.csv").getPath()).getParent());
+    }
+
+    @Test
+    public void testDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("path", path.toString());
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals(path.toString(), task.getPath());
+        assertEquals(Lists.newArrayList(), task.getConfigFiles());
+        assertEquals(Maps.newHashMap(), task.getConfig());
+        assertEquals(true, task.getPartition());
+        assertEquals(0, task.getRewindSeconds());
+        assertEquals(-1, task.getApproximateNumPartitions());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testRequiredValues()
+    {
+        ConfigSource config = Exec.newConfigSource();
+        PluginTask task = config.loadConfig(PluginTask.class);
+    }
+
+    @Test
+    public void testFileList()
+    {
+        ConfigSource config = getConfigWithDefaultValues();
+        config.set("num_partitions", 1);
+        plugin.transaction(config, new FileInputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            {
+                PluginTask task = taskSource.loadTask(PluginTask.class);
+                List<String> fileList = Lists.transform(Lists.newArrayList(new File(path.toString()).list()), new Function<String, String>()
+                {
+                    @Nullable
+                    @Override
+                    public String apply(@Nullable String input)
+                    {
+                        return new File(path.toString() + "/" + input).toURI().toString();
+                    }
+                });
+
+                List<String> resultFList = Lists.transform(task.getFiles(), new Function<HdfsPartialFile, String>()
+                {
+                    @Nullable
+                    @Override
+                    public String apply(@Nullable HdfsPartialFile input)
+                    {
+                        assert input != null;
+                        return input.getPath();
+                    }
+                });
+                assertEquals(fileList, resultFList);
+                return emptyTaskReports(taskCount);
+            }
+        });
+    }
+
+    @Test
+    public void testHdfsFileInputByOpen()
+    {
+        ConfigSource config = getConfigWithDefaultValues();
+        config.set("num_partitions", 10);
+        runner.transaction(config, new Control());
+        assertRecords(config, output);
+    }
+
+    @Test
+    public void testHdfsFileInputByOpenWithoutPartition()
+    {
+        ConfigSource config = getConfigWithDefaultValues();
+        config.set("partition", false);
+        runner.transaction(config, new Control());
+        assertRecords(config, output);
+    }
+
+    private class Control
+            implements InputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
+        {
+            List<TaskReport> reports = new ArrayList<>();
+            for (int i = 0; i < taskCount; i++) {
+                reports.add(runner.run(taskSource, schema, i, output));
+            }
+            return reports;
+        }
+    }
+
+    private ConfigSource getConfigWithDefaultValues()
+    {
+        return Exec.newConfigSource()
+                .set("path", path.toString())
+                .set("config", hdfsLocalFSConfig())
+                .set("skip_header_lines", 1)
+                .set("parser", parserConfig(schemaConfig()));
+    }
+
+    static List<TaskReport> emptyTaskReports(int taskCount)
+    {
+        ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
+        for (int i = 0; i < taskCount; i++) {
+            reports.add(Exec.newTaskReport());
+        }
+        return reports.build();
+    }
+
+    private ImmutableMap<String, Object> hdfsLocalFSConfig()
+    {
+        ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+        builder.put("fs.hdfs.impl", "org.apache.hadoop.fs.LocalFileSystem");
+        builder.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem");
+        builder.put("fs.defaultFS", "file:///");
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("newline", "CRLF");
+        builder.put("delimiter", ",");
+        builder.put("quote", "\"");
+        builder.put("escape", "\"");
+        builder.put("trim_if_not_quoted", false);
+        builder.put("skip_header_lines", 1);
+        builder.put("allow_extra_columns", false);
+        builder.put("allow_optional_columns", false);
+        builder.put("columns", schemaConfig);
+        return builder.build();
+    }
+
+    private ImmutableList<Object> schemaConfig()
+    {
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "id", "type", "long"));
+        builder.add(ImmutableMap.of("name", "account", "type", "long"));
+        builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+        builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
+        builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+        return builder.build();
+    }
+
+    private void assertRecords(ConfigSource config, MockPageOutput output)
+    {
+        List<Object[]> records = getRecords(config, output);
+        assertEquals(8, records.size());
+        {
+            Object[] record = records.get(0);
+            assertEquals(1L, record[0]);
+            assertEquals(32864L, record[1]);
+            assertEquals("2015-01-27 19:23:49 UTC", record[2].toString());
+            assertEquals("2015-01-27 00:00:00 UTC", record[3].toString());
+            assertEquals("embulk", record[4]);
+        }
+
+        {
+            Object[] record = records.get(1);
+            assertEquals(2L, record[0]);
+            assertEquals(14824L, record[1]);
+            assertEquals("2015-01-27 19:01:23 UTC", record[2].toString());
+            assertEquals("2015-01-27 00:00:00 UTC", record[3].toString());
+            assertEquals("embulk jruby", record[4]);
+        }
+    }
+
+    private List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
+    {
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        return Pages.toObjects(schema, output.pages);
+    }
 }

data/src/test/resources/sample_01.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/src/test/resources/sample_02.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-hdfs
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
-- takahiro.nakayama
+- Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-22 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,9 +46,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- example/config.yml
+- example/data.csv
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -59,6 +64,8 @@ files:
 - src/main/java/org/embulk/input/hdfs/HdfsPartialFile.java
 - src/main/java/org/embulk/input/hdfs/HdfsPartialFileInputStream.java
 - src/test/java/org/embulk/input/hdfs/TestHdfsFileInputPlugin.java
+- src/test/resources/sample_01.csv
+- src/test/resources/sample_02.csv
 - classpath/activation-1.1.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
@@ -69,7 +76,7 @@ files:
 - classpath/commons-beanutils-1.7.0.jar
 - classpath/commons-cli-1.2.jar
 - classpath/commons-codec-1.6.jar
-- classpath/commons-collections-3.2.1.jar
+- classpath/commons-collections-3.2.2.jar
 - classpath/commons-compress-1.4.1.jar
 - classpath/commons-configuration-1.6.jar
 - classpath/commons-digester-1.8.jar
@@ -82,23 +89,23 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-input-hdfs-0.1.8.jar
+- classpath/embulk-input-hdfs-0.1.9.jar
 - classpath/gson-2.2.4.jar
-- classpath/hadoop-annotations-2.6.0.jar
-- classpath/hadoop-auth-2.6.0.jar
-- classpath/hadoop-client-2.6.0.jar
-- classpath/hadoop-common-2.6.0.jar
-- classpath/hadoop-hdfs-2.6.0.jar
-- classpath/hadoop-mapreduce-client-app-2.6.0.jar
-- classpath/hadoop-mapreduce-client-common-2.6.0.jar
-- classpath/hadoop-mapreduce-client-core-2.6.0.jar
-- classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
-- classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
-- classpath/hadoop-yarn-api-2.6.0.jar
-- classpath/hadoop-yarn-client-2.6.0.jar
-- classpath/hadoop-yarn-common-2.6.0.jar
-- classpath/hadoop-yarn-server-common-2.6.0.jar
-- classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
+- classpath/hadoop-annotations-2.6.3.jar
+- classpath/hadoop-auth-2.6.3.jar
+- classpath/hadoop-client-2.6.3.jar
+- classpath/hadoop-common-2.6.3.jar
+- classpath/hadoop-hdfs-2.6.3.jar
+- classpath/hadoop-mapreduce-client-app-2.6.3.jar
+- classpath/hadoop-mapreduce-client-common-2.6.3.jar
+- classpath/hadoop-mapreduce-client-core-2.6.3.jar
+- classpath/hadoop-mapreduce-client-jobclient-2.6.3.jar
+- classpath/hadoop-mapreduce-client-shuffle-2.6.3.jar
+- classpath/hadoop-yarn-api-2.6.3.jar
+- classpath/hadoop-yarn-client-2.6.3.jar
+- classpath/hadoop-yarn-common-2.6.3.jar
+- classpath/hadoop-yarn-server-common-2.6.3.jar
+- classpath/hadoop-yarn-server-nodemanager-2.6.3.jar
 - classpath/htrace-core-3.0.4.jar
 - classpath/httpclient-4.2.5.jar
 - classpath/httpcore-4.2.4.jar