embulk-input-hdfs 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e666bbbcb18941dce84889c2ee7fb85d65edbaf4
-  data.tar.gz: 7422b508396787d70e6cea3fc534739c2c20c825
+  metadata.gz: f0ecc7937c596a6725e8c9f95804132af16d94dc
+  data.tar.gz: 0312623c3baf28b75a42839f565a6f0671afc2b8
 SHA512:
-  metadata.gz: c305947dbd3f6bded0a23fbc06efd4d44e6d48cdb4b97c8b0e3861cd4b2a9800f6d8c93cf5280ccb235ca88346e727bb5fb549ae3c7bb2e12a13205e20765085
-  data.tar.gz: 8f33bb06731a3c5a25dd723bef83616992ce5fc8b8d5e1a60d8a1da56421a42b49ae3397feb24134a093bf291af87ddbd208fa866c86fdd997d824a6077434a4
+  metadata.gz: 6a4315e6f9c1dda4f752bcbe930373032b63c019c65bd9356ac385bcce7543fbb8e9279bb5204221f7dd2bef515259c2c19c0e69b1c8d148097567fa599b2c26
+  data.tar.gz: 589a462546b0b1cd376757f61179b857aba5d056b96c6864cc991617929ea6dc5b9255088c79766d54b8784fdc922b4f5192a90a913df0d424bb6fc5e64ea9a8

data/CHENGELOG.md

@@ -1,3 +1,8 @@
+0.3.0 (2016-09-21)
+==================
+- [Incompatible Change] Not partitoning if files are compressed
+  - https://github.com/civitaspo/embulk-input-hdfs/pull/27
+
 0.2.1 (2016-02-25)
 ==================
 - [Fix] does not work

data/README.md

@@ -77,23 +77,26 @@ int partitionSizeByOneTask = totalFileLength / approximateNumPartitions;
 /*
 ...
 */
-
-    long numPartitions;
-    if (task.getPartition()) {
-        if (file.canDecompress()) {
-            numPartitions = ((fileLength - 1) / partitionSizeByOneTask) + 1;
-        }
-        else if (file.getCodec() != null) { // if not null, the file is compressed.
-            numPartitions = 1;
-        }
-        else {
-            numPartitions = ((fileLength - 1) / partitionSizeByOneTask) + 1;
-        }
-    }
-    else {
-        numPartitions = 1;
-    }
-
+            long numPartitions = 1; // default is no partition.
+            if (isPartitionable(task, conf, status)) { // partition: true and (decompression: false or CompressionCodec is null)
+                numPartitions = ((status.getLen() - 1) / partitionSizeByOneTask) + 1;
+            }
+
+            for (long i = 0; i < numPartitions; i++) {
+                long start = status.getLen() * i / numPartitions;
+                long end = status.getLen() * (i + 1) / numPartitions;
+                if (start < end) {
+                    TargetFileInfo targetFileInfo = new TargetFileInfo.Builder()
+                            .pathString(status.getPath().toString())
+                            .start(start)
+                            .end(end)
+                            .isDecompressible(isDecompressible(task, conf, status))
+                            .isPartitionable(isPartitionable(task, conf, status))
+                            .numHeaderLines(task.getSkipHeaderLines())
+                            .build();
+                    builder.add(targetFileInfo);
+                }
+            }
 /*
 ...
 */

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.2.1"
+version = "0.3.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/hdfs/ConfigurationFactory.java

@@ -0,0 +1,60 @@
+package org.embulk.input.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.util.List;
+import java.util.Map;
+
+public class ConfigurationFactory
+{
+    public static final Logger logger = Exec.getLogger(ConfigurationFactory.class);
+
+    interface Task
+    {
+        @Config("config_files")
+        @ConfigDefault("[]")
+        List<String> getConfigFiles();
+
+        @Config("config")
+        @ConfigDefault("{}")
+        Map<String, String> getConfig();
+    }
+
+    private ConfigurationFactory()
+    {
+    }
+
+    public static Configuration create(Task task)
+    {
+        Configuration c = new Configuration();
+        for (String f : task.getConfigFiles()) {
+            try {
+                logger.debug("embulk-input-hdfs: load a config file: {}", f);
+                c.addResource(new File(f).toURI().toURL());
+            }
+            catch (MalformedURLException e) {
+                throw new ConfigException(e);
+            }
+        }
+
+        for (Map.Entry<String, String> entry : task.getConfig().entrySet()) {
+            logger.debug("embulk-input-hdfs: load a config: {}:{}", entry.getKey(), entry.getValue());
+            c.set(entry.getKey(), entry.getValue());
+        }
+
+        // For logging
+        for (Map.Entry<String, String> entry : c) {
+            logger.trace("embulk-input-hdfs: {}: {}", entry.getKey(), entry.getValue());
+        }
+        logger.trace("embulk-input-hdfs: Resource Files: {}", c);
+
+        return c;
+    }
+}

data/src/main/java/org/embulk/input/hdfs/HdfsFileInputPlugin.java

@@ -1,17 +1,12 @@
 package org.embulk.input.hdfs;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -22,43 +17,30 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
-import org.jruby.embed.ScriptingContainer;
 import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 
 public class HdfsFileInputPlugin
         implements FileInputPlugin
 {
     public interface PluginTask
-            extends Task, PartialFileList.Task
+            extends Task, TargetFileInfoList.Task, ConfigurationFactory.Task, Strftime.Task
     {
-        @Config("config_files")
-        @ConfigDefault("[]")
-        List<String> getConfigFiles();
-
-        @Config("config")
-        @ConfigDefault("{}")
-        Map<String, String> getConfig();
-
         @Config("path")
         String getPath();
 
-        @Config("rewind_seconds")
-        @ConfigDefault("0")
-        int getRewindSeconds();
-
         @Config("partition")
         @ConfigDefault("true")
-        boolean getPartition();
+        boolean getWillPartition();
 
         @Config("num_partitions") // this parameter is the approximate value.
         @ConfigDefault("-1")      // Default: Runtime.getRuntime().availableProcessors()
@@ -70,253 +52,175 @@ public class HdfsFileInputPlugin
 
         @Config("decompression") // if true, decompress files by using compression codec
         @ConfigDefault("false")  // when getting FileInputStream.
-        boolean getDecompression();
+        boolean getWillDecompress();
 
-        PartialFileList getPartialFileList();
-        void setPartialFileList(PartialFileList partialFileList);
-
-        @ConfigInject
-        ScriptingContainer getJRuby();
-
-        @ConfigInject
-        BufferAllocator getBufferAllocator();
+        TargetFileInfoList getTargetFileInfoList();
+        void setTargetFileInfoList(TargetFileInfoList targetFileInfoList);
     }
 
     private static final Logger logger = Exec.getLogger(HdfsFileInputPlugin.class);
-    private Optional<Configuration> configurationContainer = Optional.absent();
 
     @Override
     public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-        Configuration configuration = getConfiguration(task);
+
+        if (task.getWillPartition() && task.getWillDecompress()) {
+            logger.info("embulk-input-hdfs: Please be sure that the target files cannot be partitioned if they are compressed.");
+        }
+
+        Configuration conf = ConfigurationFactory.create(task);
 
         // listing Files
         try {
-            FileSystem fs = getFS(configuration);
+            FileSystem fs = FileSystem.get(conf);
 
-            String pathString = strftime(task.getJRuby(), task.getPath(), task.getRewindSeconds());
+            String pathString = new Strftime(task).format(task.getPath());
             Path rootPath = new Path(pathString);
 
-            List<Path> originalFileList = buildOriginalFileList(fs, rootPath);
+            List<FileStatus> statusList = listFileStatuses(fs, rootPath);
 
-            if (originalFileList.isEmpty()) {
+            if (statusList.isEmpty()) {
                 throw new PathNotFoundException(pathString);
             }
 
-            logger.debug("embulk-input-hdfs: Loading target files: {}", originalFileList);
-            PartialFileList list = buildPartialFileList(task, originalFileList);
-            task.setPartialFileList(list);
+            for (FileStatus status : statusList) {
+                logger.debug("embulk-input-hdfs: Loading paths: {}, length: {}", status.getPath(), status.getLen());
+            }
+
+            TargetFileInfoList list = buildTargetFileInfoList(task, statusList);
+            task.setTargetFileInfoList(list);
         }
         catch (IOException e) {
             logger.error(e.getMessage());
-            throw new RuntimeException(e);
+            throw Throwables.propagate(e);
         }
 
         // number of processors is same with number of targets
-        int taskCount = task.getPartialFileList().getTaskCount();
+        int taskCount = task.getTargetFileInfoList().getTaskCount();
         logger.info("embulk-input-hdfs: task size: {}", taskCount);
 
         return resume(task.dump(), taskCount, control);
     }
 
-    private Configuration getConfiguration(PluginTask task)
+    private List<FileStatus> listFileStatuses(FileSystem fs, Path rootPath)
+            throws IOException
     {
-        if (configurationContainer.isPresent()) {
-            return configurationContainer.get();
-        }
-
-        ConfigurationBuilder builder = new ConfigurationBuilder();
-        builder.addConfigFiles(task.getConfigFiles());
-        builder.addConfigMap(task.getConfig());
-        configurationContainer = Optional.of(builder.build());
-        return configurationContainer.get();
-    }
+        List<FileStatus> statusList = Lists.newArrayList();
 
-    private FileSystem getFS(Configuration configuration)
-    {
-        try {
-            return FileSystem.get(configuration);
-        }
-        catch (IOException e) {
-            throw Throwables.propagate(e);
-        }
-    }
-
-    @VisibleForTesting
-    String strftime(final ScriptingContainer jruby, final String format, final int rewindSeconds)
-    {
-        String script = String.format("(Time.now - %d).strftime('%s')", rewindSeconds, format);
-        return jruby.runScriptlet(script).toString();
-    }
-
-    private List<Path> buildOriginalFileList(FileSystem fs, Path rootPath)
-    {
-        List<Path> fileList = Lists.newArrayList();
-
-        final FileStatus[] entries;
-        try {
-            entries = fs.globStatus(rootPath);
-        }
-        catch (IOException e) {
-            throw Throwables.propagate(e);
-        }
+        FileStatus[] entries = fs.globStatus(rootPath);
         // `globStatus` does not throw PathNotFoundException.
         // return null instead.
         // see: https://github.com/apache/hadoop/blob/branch-2.7.0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java#L286
         if (entries == null) {
-            return fileList;
+            return statusList;
         }
 
         for (FileStatus entry : entries) {
             if (entry.isDirectory()) {
-                List<Path> subEntries = listRecursive(fs, entry);
-                fileList.addAll(subEntries);
+                // TODO: use fs.listFiles(entry.getPath(), true); ?
+                List<FileStatus> subEntries = listRecursive(fs, entry);
+                statusList.addAll(subEntries);
             }
             else {
-                fileList.add(entry.getPath());
+                statusList.add(entry);
             }
         }
 
-        return fileList;
+        return statusList;
     }
 
-    private List<Path> listRecursive(final FileSystem fs, FileStatus status)
+    private List<FileStatus> listRecursive(FileSystem fs, FileStatus status)
+            throws IOException
     {
-        List<Path> fileList = Lists.newArrayList();
+        List<FileStatus> statusList = Lists.newArrayList();
         if (status.isDirectory()) {
-            FileStatus[] entries;
-            try {
-                entries = fs.listStatus(status.getPath());
-            }
-            catch (IOException e) {
-                throw Throwables.propagate(e);
-            }
-
+            FileStatus[] entries = fs.listStatus(status.getPath());
             for (FileStatus entry : entries) {
-                fileList.addAll(listRecursive(fs, entry));
+                statusList.addAll(listRecursive(fs, entry));
             }
         }
         else {
-            fileList.add(status.getPath());
+            statusList.add(status);
         }
-        return fileList;
+        return statusList;
     }
 
-    private PartialFileList buildPartialFileList(PluginTask task, List<Path> pathList)
+    private TargetFileInfoList buildTargetFileInfoList(PluginTask task, List<FileStatus> statusList)
+            throws IOException, DataException
     {
-        Configuration configuration = getConfiguration(task);
-        FileSystem fs = getFS(configuration);
-        boolean shouldPartition = task.getPartition();
-        boolean shouldDecompress = task.getDecompression();
-
-        Map<Path, Long> pathLengthMap = Maps.newHashMap();
-        long totalFileLength = 0;
-        for (Path path : pathList) {
-            long fileLength = getHdfsFileLength(fs, path, shouldDecompress);
-
-            if (fileLength <= 0) {
-                logger.info("Skip the 0 byte target file: {}", path);
-                continue;
-            }
-
-            pathLengthMap.put(path, fileLength);
-            totalFileLength += fileLength;
-        }
+        long totalFileLength = calcTotalFilesLength(statusList);
         if (totalFileLength <= 0) {
-            throw Throwables.propagate(new PathIOException(task.getPath(), "All files are empty"));
-        }
-
-        PartialFileList.Builder builder = new PartialFileList.Builder(task);
-
-        // TODO: optimum allocation of resources
-        final long approximateNumPartitions;
-        if (task.getApproximateNumPartitions() <= 0) {
-            approximateNumPartitions = Runtime.getRuntime().availableProcessors();
-        }
-        else {
-            approximateNumPartitions = task.getApproximateNumPartitions();
+            // TODO: skip this error because other file input plugins have no errors if files are empty.
+            throw new DataException("embulk-input-hdfs: All files are empty: " + task.getPath());
         }
 
-        long partitionSizeByOneTask = totalFileLength / approximateNumPartitions;
-        if (partitionSizeByOneTask <= 0) {
-            partitionSizeByOneTask = 1;
-        }
-
-        for (Map.Entry<Path, Long> entry : pathLengthMap.entrySet()) {
-            Path path = entry.getKey();
-            long fileLength = entry.getValue();
+        long partitionSizeByOneTask = calcApproximatePartitionSizeByOneTask(task, totalFileLength);
 
-            long numPartitions;
-            if (shouldPartition) {
-                if (shouldDecompress && getHdfsFileCompressionCodec(fs, path) != null) {
-                    numPartitions = ((fileLength - 1) / partitionSizeByOneTask) + 1;
-                }
-                else if (getHdfsFileCompressionCodec(fs, path) != null) { // if not null, the file is compressed.
-                    numPartitions = 1;
-                }
-                else {
-                    numPartitions = ((fileLength - 1) / partitionSizeByOneTask) + 1;
-                }
+        Configuration conf = ConfigurationFactory.create(task);
+        TargetFileInfoList.Builder builder = TargetFileInfoList.builder(task);
+        for (FileStatus status : statusList) {
+            if (status.getLen() <= 0) {
+                logger.info("embulk-input-hdfs: Skip the 0 byte target file: {}", status.getPath());
+                continue;
             }
-            else {
-                numPartitions = 1;
+
+            long numPartitions = 1; // default is no partition.
+            if (isPartitionable(task, conf, status)) {
+                numPartitions = ((status.getLen() - 1) / partitionSizeByOneTask) + 1;
             }
 
             for (long i = 0; i < numPartitions; i++) {
-                long start = fileLength * i / numPartitions;
-                long end = fileLength * (i + 1) / numPartitions;
+                long start = status.getLen() * i / numPartitions;
+                long end = status.getLen() * (i + 1) / numPartitions;
                 if (start < end) {
-                    logger.debug("PartialFile: path {}, start: {}, end: {}", path, start, end);
-                    builder.add(path.toString(), start, end, shouldDecompress && getHdfsFileCompressionCodec(fs, path) != null);
+                    TargetFileInfo targetFileInfo = new TargetFileInfo.Builder()
+                            .pathString(status.getPath().toString())
+                            .start(start)
+                            .end(end)
+                            .isDecompressible(isDecompressible(task, conf, status))
+                            .isPartitionable(isPartitionable(task, conf, status))
+                            .numHeaderLines(task.getSkipHeaderLines())
+                            .build();
+                    builder.add(targetFileInfo);
                 }
             }
         }
-
         return builder.build();
     }
 
-    private Long getHdfsFileLength(FileSystem fs, Path path, boolean shouldDecompression)
+    private boolean isDecompressible(PluginTask task, Configuration conf, FileStatus status)
     {
-        CompressionCodec codec = getHdfsFileCompressionCodec(fs, path);
-        if (codec == null) {
-            try {
-                return fs.getFileStatus(path).getLen();
-            }
-            catch (IOException e) {
-                throw Throwables.propagate(e);
-            }
-        }
-        else if (!shouldDecompression) {
-            try {
-                return fs.getFileStatus(path).getLen();
-            }
-            catch (IOException e) {
-                throw Throwables.propagate(e);
-            }
-        }
-        else {
-            long fileLength = 0;
-            try (InputStream is = codec.createInputStream(fs.open(path))) {
-                while (is.read() > 0) {
-                    fileLength++;
-                }
-            }
-            catch (IOException e) {
-                throw Throwables.propagate(e);
-            }
-            return fileLength;
-        }
+        return task.getWillDecompress() && new CompressionCodecFactory(conf).getCodec(status.getPath()) != null;
     }
 
-    private CompressionCodec getHdfsFileCompressionCodec(FileSystem fs, Path path)
+    private boolean isPartitionable(PluginTask task, Configuration conf, FileStatus status)
     {
-        return getHdfsFileCompressionCodec(fs.getConf(), path);
+        return task.getWillPartition() && !isDecompressible(task, conf, status);
     }
 
-    private CompressionCodec getHdfsFileCompressionCodec(Configuration configuration, Path path)
+    private long calcTotalFilesLength(List<FileStatus> statusList)
+            throws IOException
     {
-        return new CompressionCodecFactory(configuration).getCodec(path);
+        long total = 0L;
+        for (FileStatus status : statusList) {
+            total += status.getLen();
+        }
+        return total;
+    }
+
+    private long calcApproximatePartitionSizeByOneTask(PluginTask task, long totalFilesLength)
+    {
+        long numPartitions = task.getApproximateNumPartitions();
+        if (numPartitions <= 0) {
+            numPartitions = Runtime.getRuntime().availableProcessors();
+        }
+        // TODO: optimum allocation of resources
+        long partitionSizeByOneTask = totalFilesLength / numPartitions;
+        if (partitionSizeByOneTask <= 0) {
+            partitionSizeByOneTask = 1;
+        }
+        return partitionSizeByOneTask;
     }
 
     @Override
@@ -340,17 +244,22 @@ public class HdfsFileInputPlugin
     public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new HdfsFileInput(task, taskIndex);
+        try {
+            return new HdfsFileInput(task, taskIndex);
+        }
+        catch (IOException e) {
+            throw Throwables.propagate(e);
+        }
     }
 
     public class HdfsFileInput
             extends InputStreamFileInput
             implements TransactionalFileInput
     {
-
         public HdfsFileInput(PluginTask task, int taskIndex)
+                throws IOException
         {
-            super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
+            super(Exec.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
         }
 
         @Override
@@ -369,15 +278,14 @@ public class HdfsFileInputPlugin
     private class SingleFileProvider
             implements InputStreamFileInput.Provider
     {
-        private final FileSystem fs;
-        private final int numHeaderLines;
-        private final Iterator<PartialFile> iterator;
+        private final TargetFileInputStreamFactory factory;
+        private final Iterator<TargetFileInfo> iterator;
 
         public SingleFileProvider(PluginTask task, int taskIndex)
+                throws IOException
         {
-            this.fs = getFS(getConfiguration(task));
-            this.numHeaderLines = task.getSkipHeaderLines();
-            this.iterator = task.getPartialFileList().get(taskIndex).iterator();
+            this.factory = new TargetFileInputStreamFactory(FileSystem.get(ConfigurationFactory.create(task)));
+            this.iterator = task.getTargetFileInfoList().get(taskIndex).iterator();
         }
 
         @Override
@@ -386,8 +294,7 @@ public class HdfsFileInputPlugin
             if (!iterator.hasNext()) {
                 return null;
             }
-            PartialFileInputStreamBuilder builder = new PartialFileInputStreamBuilder(fs, iterator.next()).withHeaders(numHeaderLines);
-            return builder.build();
+            return factory.create(iterator.next());
         }
 
         @Override