embulk-input-hdfs 0.2.1 → 0.3.0

data/src/main/java/org/embulk/input/hdfs/TargetFileInputStreamFactory.java

@@ -0,0 +1,128 @@
+package org.embulk.input.hdfs;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+
+public class TargetFileInputStreamFactory
+{
+    private static final Logger logger = Exec.getLogger(TargetFileInputStreamFactory.class);
+    private final FileSystem fs;
+
+    public TargetFileInputStreamFactory(FileSystem fs)
+    {
+        this.fs = fs;
+    }
+
+    public InputStream create(TargetFileInfo t)
+            throws IOException
+    {
+        InputStream is = createSuitableInputStream(t);
+        return createInputStreamWithHeaders(is, t);
+    }
+
+    private InputStream createSuitableInputStream(TargetFileInfo t)
+            throws IOException
+    {
+        if (t.getIsDecompressible()) {
+            logger.debug("embulk-input-hdfs: createDecompressedInputStream: {}", t.getPathString());
+            return createDecompressedInputStream(t);
+        }
+        else if (t.getIsPartitionable()) {
+            logger.debug("embulk-input-hdfs: createPartialInputStream: {}, start:{}, end:{}",
+                    t.getPathString(), t.getStart(), t.getEnd());
+            return createPartialInputStream(t);
+        }
+        else {
+            logger.debug("embulk-input-hdfs: createOriginalInputStream: {}", t.getPathString());
+            return createOriginalInputStream(t);
+        }
+    }
+
+    private InputStream createInputStreamWithHeaders(InputStream original, TargetFileInfo t)
+            throws IOException
+    {
+        if (t.getStart() > 0 && t.getNumHeaderLines() > 0) {
+            logger.debug("embulk-input-hdfs: createInputStreamWithHeaders: {}", t.getPathString());
+            InputStream headers = createHeadersInputStream(t);
+            return new SequenceInputStream(headers, original);
+        }
+        else {
+            return original;
+        }
+    }
+
+    private InputStream createOriginalInputStream(TargetFileInfo t)
+            throws IOException
+    {
+        return fs.open(new Path(t.getPathString()));
+    }
+
+    private InputStream createDecompressedInputStream(TargetFileInfo t)
+            throws IOException
+    {
+        InputStream original = createOriginalInputStream(t);
+        CompressionCodecFactory factory = new CompressionCodecFactory(fs.getConf());
+        CompressionCodec codec = factory.getCodec(new Path(t.getPathString()));
+        if (codec == null) {
+            logger.debug("embulk-input-hdfs: CompressionCodec:　null: {}", t.getPathString());
+            return original;
+        }
+        else {
+            logger.debug("embulk-input-hdfs: CompressionCodec:　{}: {}", codec, t.getPathString());
+            return codec.createInputStream(original);
+        }
+    }
+
+    private InputStream createPartialInputStream(TargetFileInfo t)
+            throws IOException
+    {
+        InputStream original = createOriginalInputStream(t);
+        return new TargetFilePartialInputStream(original, t.getStart(), t.getEnd());
+    }
+
+    private InputStream createHeadersInputStream(TargetFileInfo t)
+            throws IOException
+    {
+        ByteArrayOutputStream header = new ByteArrayOutputStream();
+        int skippedHeaders = 0;
+        InputStream is = createOriginalInputStream(t);
+        try (BufferedInputStream in = new BufferedInputStream(is)) {
+            while (true) {
+                int c = in.read();
+                if (c < 0) {
+                    break;
+                }
+
+                header.write(c);
+
+                if (c == '\n') {
+                    skippedHeaders++;
+                }
+                else if (c == '\r') {
+                    int c2 = in.read();
+                    if (c2 == '\n') {
+                        header.write(c2);
+                    }
+                    skippedHeaders++;
+                }
+
+                if (skippedHeaders >= t.getNumHeaderLines()) {
+                    break;
+                }
+            }
+        }
+        header.close();
+        return new ByteArrayInputStream(header.toByteArray());
+    }
+}

data/src/main/java/org/embulk/input/hdfs/{PartialFileInputStream.java → TargetFilePartialInputStream.java}

@@ -1,15 +1,13 @@
 package org.embulk.input.hdfs;
 
+// Ported from https://github.com/hito4t/embulk-input-filesplit/blob/master/src/main/java/org/embulk/input/filesplit/PartialFileInputStream.java
+
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PushbackInputStream;
 
-/**
- * Created by takahiro.nakayama on 2/13/16.
- * ref. https://github.com/hito4t/embulk-input-filesplit/blob/master/src/main/java/org/embulk/input/filesplit/PartialFileInputStream.java
- */
-public class PartialFileInputStream
+public class TargetFilePartialInputStream
     extends InputStream
 {
     private final PushbackInputStream original;
@@ -18,7 +16,7 @@ public class PartialFileInputStream
     private long current;
     private boolean eof;
 
-    public PartialFileInputStream(InputStream original, long start, long end)
+    public TargetFilePartialInputStream(InputStream original, long start, long end)
     {
         this.original = new PushbackInputStream(new BufferedInputStream(original));
         this.start = start;

data/src/test/java/org/embulk/input/hdfs/TestHdfsFileInputPlugin.java

@@ -30,7 +30,6 @@ import javax.annotation.Nullable;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
@@ -43,7 +42,6 @@ public class TestHdfsFileInputPlugin
     @Rule
     public ExpectedException exception = ExpectedException.none();
 
-    private Logger logger = runtime.getExec().getLogger(TestHdfsFileInputPlugin.class);
     private HdfsFileInputPlugin plugin;
     private FileInputRunner runner;
     private MockPageOutput output;
@@ -67,11 +65,11 @@ public class TestHdfsFileInputPlugin
         assertEquals(path.toString(), task.getPath());
         assertEquals(Lists.newArrayList(), task.getConfigFiles());
         assertEquals(Maps.newHashMap(), task.getConfig());
-        assertEquals(true, task.getPartition());
+        assertEquals(true, task.getWillPartition());
         assertEquals(0, task.getRewindSeconds());
         assertEquals(-1, task.getApproximateNumPartitions());
         assertEquals(0, task.getSkipHeaderLines());
-        assertEquals(false, task.getDecompression());
+        assertEquals(false, task.getWillDecompress());
     }
 
     @Test(expected = ConfigException.class)
@@ -103,9 +101,9 @@ public class TestHdfsFileInputPlugin
                 });
 
                 List<String> resultFList = Lists.newArrayList();
-                for (int i = 0; i < task.getPartialFileList().getTaskCount();i++) {
-                    for (PartialFile partialFile : task.getPartialFileList().get(i)) {
-                        resultFList.add(partialFile.getPath().toString());
+                for (int i = 0; i < task.getTargetFileInfoList().getTaskCount();i++) {
+                    for (TargetFileInfo targetFileInfo : task.getTargetFileInfoList().get(i)) {
+                        resultFList.add(targetFileInfo.getPathString());
                     }
                 }
                 assertEquals(fileList.size(), resultFList.size());
@@ -152,7 +150,7 @@ public class TestHdfsFileInputPlugin
         config.set("path", "/tmp/%Y-%m-%d");
         config.set("rewind_seconds", 86400);
         PluginTask task = config.loadConfig(PluginTask.class);
-        String result = plugin.strftime(task.getJRuby(), task.getPath(), task.getRewindSeconds());
+        String result = new Strftime(task).format(task.getPath());
         String expected = task.getJRuby().runScriptlet("(Time.now - 86400).strftime('/tmp/%Y-%m-%d')").toString();
         assertEquals(expected, result);
     }

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-hdfs
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-25 00:00:00.000000000 Z
+date: 2016-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Reads files stored on Hdfs.
 email:
 - civitaspo@gmail.com
@@ -61,12 +61,13 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/input/hdfs.rb
-- src/main/java/org/embulk/input/hdfs/ConfigurationBuilder.java
+- src/main/java/org/embulk/input/hdfs/ConfigurationFactory.java
 - src/main/java/org/embulk/input/hdfs/HdfsFileInputPlugin.java
-- src/main/java/org/embulk/input/hdfs/PartialFile.java
-- src/main/java/org/embulk/input/hdfs/PartialFileInputStream.java
-- src/main/java/org/embulk/input/hdfs/PartialFileInputStreamBuilder.java
-- src/main/java/org/embulk/input/hdfs/PartialFileList.java
+- src/main/java/org/embulk/input/hdfs/Strftime.java
+- src/main/java/org/embulk/input/hdfs/TargetFileInfo.java
+- src/main/java/org/embulk/input/hdfs/TargetFileInfoList.java
+- src/main/java/org/embulk/input/hdfs/TargetFileInputStreamFactory.java
+- src/main/java/org/embulk/input/hdfs/TargetFilePartialInputStream.java
 - src/test/java/org/embulk/input/hdfs/TestHdfsFileInputPlugin.java
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
@@ -94,7 +95,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-input-hdfs-0.2.1.jar
+- classpath/embulk-input-hdfs-0.3.0.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.4.jar
 - classpath/hadoop-auth-2.6.4.jar

data/src/main/java/org/embulk/input/hdfs/ConfigurationBuilder.java

@@ -1,82 +0,0 @@
-package org.embulk.input.hdfs;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.hadoop.conf.Configuration;
-import org.embulk.config.ConfigException;
-import org.embulk.spi.Exec;
-import org.slf4j.Logger;
-
-import java.io.File;
-import java.net.MalformedURLException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Created by takahiro.nakayama on 2/22/16.
- */
-public class ConfigurationBuilder
-{
-    private static final Logger logger = Exec.getLogger(ConfigurationBuilder.class);
-    private final ImmutableList.Builder<String> configFilesBuilder;
-    private final ImmutableMap.Builder<String, String> configMapBuilder;
-
-    public ConfigurationBuilder()
-    {
-        this.configFilesBuilder = ImmutableList.builder();
-        this.configMapBuilder = ImmutableMap.builder();
-    }
-
-    public ConfigurationBuilder addConfigFiles(List<String> configFiles)
-    {
-        for (String configFile : configFiles) {
-            addConfigFile(configFile);
-        }
-        return this;
-    }
-
-    public ConfigurationBuilder addConfigFile(String configFile)
-    {
-        configFilesBuilder.add(configFile);
-        return this;
-    }
-
-    public ConfigurationBuilder addConfigMap(Map<String, String> configMap)
-    {
-        for (Map.Entry<String, String> entry : configMap.entrySet()) {
-            addConfig(entry.getKey(), entry.getValue());
-        }
-        return this;
-    }
-
-    public ConfigurationBuilder addConfig(String key, String value)
-    {
-        configMapBuilder.put(key, value);
-        return this;
-    }
-
-    public Configuration build()
-    {
-        Configuration configuration = new Configuration();
-        for (String configFile : configFilesBuilder.build()) {
-            File file = new File(configFile);
-            try {
-                configuration.addResource(file.toURI().toURL());
-            }
-            catch (MalformedURLException e) {
-                throw new ConfigException(e);
-            }
-        }
-        for (Map.Entry<String, String> entry : configMapBuilder.build().entrySet()) {
-            configuration.set(entry.getKey(), entry.getValue());
-        }
-        // For debug
-        for (Map.Entry<String, String> entry : configuration) {
-            logger.trace("{}: {}", entry.getKey(), entry.getValue());
-        }
-        logger.trace("Resource Files: {}", configuration);
-        return configuration;
-    }
-}

data/src/main/java/org/embulk/input/hdfs/PartialFile.java

@@ -1,48 +0,0 @@
-package org.embulk.input.hdfs;
-
-import org.apache.hadoop.fs.Path;
-
-/**
- * Created by takahiro.nakayama on 2/20/16.
- * is the same as PartialFileList.Entry, so this class does not need?
- */
-public class PartialFile
-{
-    private final Path path;
-    private final long start;
-    private final long end;
-    private final boolean canDecompress;
-
-    public PartialFile(String path, long start, long end, boolean canDecompress)
-    {
-        this(new Path(path), start, end, canDecompress);
-    }
-
-    public PartialFile(Path path, long start, long end, boolean canDecompress)
-    {
-        this.path = path;
-        this.start = start;
-        this.end = end;
-        this.canDecompress = canDecompress;
-    }
-
-    public Path getPath()
-    {
-        return path;
-    }
-
-    public long getStart()
-    {
-        return start;
-    }
-
-    public long getEnd()
-    {
-        return end;
-    }
-
-    public boolean getCanDecompress()
-    {
-        return canDecompress;
-    }
-}

data/src/main/java/org/embulk/input/hdfs/PartialFileInputStreamBuilder.java

@@ -1,125 +0,0 @@
-package org.embulk.input.hdfs;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.compress.CodecPool;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.io.compress.Decompressor;
-import org.embulk.spi.Exec;
-import org.slf4j.Logger;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.SequenceInputStream;
-
-/**
- * Created by takahiro.nakayama on 2/21/16.
- */
-public class PartialFileInputStreamBuilder
-{
-    private static final Logger logger = Exec.getLogger(PartialFileInputStreamBuilder.class);
-    private final FileSystem fs;
-    private final PartialFile partialFile;
-    private int numHeaderLines = 0;
-
-    public PartialFileInputStreamBuilder(FileSystem fs, PartialFile partialFile)
-    {
-        this.fs = fs;
-        this.partialFile = partialFile;
-    }
-
-    public InputStream build()
-            throws IOException
-    {
-        logger.trace("path: {}, start: {}, end: {}, num_header_lines: {}",
-                partialFile.getPath(), partialFile.getStart(), partialFile.getEnd(), numHeaderLines);
-        if (partialFile.getStart() > 0 && numHeaderLines > 0) {
-            return new SequenceInputStream(createHeadersInputStream(), createPartialFileInputStream());
-        }
-        else {
-            return createPartialFileInputStream();
-        }
-    }
-
-    public PartialFileInputStreamBuilder withHeaders(int numHeaderLines)
-    {
-        this.numHeaderLines = numHeaderLines;
-        return this;
-    }
-
-    private InputStream createOriginalFileWrappedInputStream()
-    {
-        InputStream original = createOriginalFileInputStream();
-        CompressionCodec codec = new CompressionCodecFactory(fs.getConf()).getCodec(partialFile.getPath());
-        if (partialFile.getCanDecompress() && codec != null) {
-            try {
-                return codec.createInputStream(original);
-            }
-            catch (IOException e) {
-                throw Throwables.propagate(e);
-            }
-        }
-        else {
-            return original;
-        }
-    }
-
-    private InputStream createOriginalFileInputStream()
-    {
-        try {
-            return fs.open(partialFile.getPath());
-        }
-        catch (IOException e) {
-            throw Throwables.propagate(e);
-        }
-    }
-
-    // memo: compressioncodec使ったinputstream作る奴いても良いような…
-    // じゃないと、headers もおかしくなるような…ならんか
-
-    private InputStream createPartialFileInputStream()
-    {
-        InputStream original = createOriginalFileWrappedInputStream();
-        return new PartialFileInputStream(original, partialFile.getStart(), partialFile.getEnd());
-    }
-
-    private InputStream createHeadersInputStream()
-            throws IOException
-    {
-        ByteArrayOutputStream header = new ByteArrayOutputStream();
-        int skippedHeaders = 0;
-        InputStream original = createOriginalFileWrappedInputStream();
-        try (BufferedInputStream in = new BufferedInputStream(original)) {
-            while (true) {
-                int c = in.read();
-                if (c < 0) {
-                    break;
-                }
-
-                header.write(c);
-
-                if (c == '\n') {
-                    skippedHeaders++;
-                }
-                else if (c == '\r') {
-                    int c2 = in.read();
-                    if (c2 == '\n') {
-                        header.write(c2);
-                    }
-                    skippedHeaders++;
-                }
-
-                if (skippedHeaders >= numHeaderLines) {
-                    break;
-                }
-            }
-        }
-        header.close();
-        return new ByteArrayInputStream(header.toByteArray());
-    }
-}