embulk-input-filesplit 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6c43f390ee5dd1de19e4481dd2443dc3d3d63c0
-  data.tar.gz: 7fb23bbe679ffa5fc730e3f553a1c14c2633f381
+  metadata.gz: 6c850998d274e9c10441daaac5ec8f25fb06ed7f
+  data.tar.gz: 6ede3a6e8f48a3ac8c4b3568605733cb7bc4b856
 SHA512:
-  metadata.gz: a55da85aa17a112889765c8ec8281e1816c40e00a849032e72ca4618d85c1dda1c759cf178a5bffc5b1f3bd73f918c9eb2e3461f825eedb30b5010810273e833
-  data.tar.gz: eb261354f866e3e4833815fd19c9634a380283dbadf9a47dcb8f12eb6135f502ee0c7d8e13f10c973b91f06fddb77e479f243bf8520e9b2f2338c14b388f35ce
+  metadata.gz: 7bbef333a3b9cacffc614fcb2b3b926d26f0ffde4e1f956568c072c1ac0b1a66db5c04e472b5ad8ed85dee81c20d68e080723feacad0823a2b3bd47536071d40
+  data.tar.gz: ffe27227b3469ad54c91a2e51bc3acb7a1523a8ac09921833d59d5b983ab2759a4b3151a5b1a7414a666a27da224d3cf0527d029e5787031b8dceb07b86d1ebb

data/README.md

@@ -1,40 +1,41 @@
-# Splitting input file plugin for Embulk
-
-This Embulk plugin splits and inputs a text file.
-By splitting a file, input tasks will be executed in multithreads and the performance will be improved.   
-
-Lines of the text file should be separated by CR or LF or CRLF. 
-The plugin searches line separators and splits a file properly.
-
-## Overview
-
-* **Plugin type**: input
-
-## Configuration
-
-- **path**: the path of a text file (string, required)
-- **header_line**: whether the first line is a header or not (boolean, default: false)
-- **tasks**: number of tasks (integer, default: number of available processors * 2)
-
-### Example
-
-```yaml
-in:
-  type: filesplit
-  path: '/data/address.csv'
-  header_line: true
-  tasks: 4
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    header_line: true
-    delimiter: ','
-    ...
-```
-
-### Build
-
-```
-$ ./gradle gem
-```
+# Splitting input file plugin for Embulk
+
+This Embulk plugin splits and inputs a text file.
+By splitting a file, input tasks will be executed in multithreads and the performance will be improved.
+
+Lines of the text file should be separated by CR or LF or CRLF.
+The plugin searches line separators and splits a file properly.
+
+## Overview
+
+* **Plugin type**: input
+
+## Configuration
+
+- **path**: the path of a text file (string, either this or path_prefix is required)
+- **path_prefix**: the path prefix of text files (string, either this or path_prefix is required)
+- **header_line**: whether the first line is a header or not (boolean, default: false)
+- **tasks**: number of tasks (integer, default: number of available processors * 2)
+
+### Example
+
+```yaml
+in:
+  type: filesplit
+  path: '/data/address.csv'
+  header_line: true
+  tasks: 4
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    header_line: true
+    delimiter: ','
+    ...
+```
+
+### Build
+
+```
+$ ./gradle gem
+```

data/build.gradle

@@ -1,64 +1,64 @@
-plugins {
-    id 'com.jfrog.bintray' version '1.1'
-    id 'com.github.jruby-gradle.base' version '0.1.5'
-    id 'java'
-}
-import com.github.jrubygradle.JRubyExec
-
-apply plugin: 'java'
-apply plugin: 'com.github.jruby-gradle.base'
-
-[compileJava, compileTestJava]*.options*.encoding = 'UTF-8'
-
-project.version = '0.1.3'
-
-repositories {
-    mavenCentral()
-    jcenter()
-}
-
-configurations {
-    provided
-}
-
-dependencies {
-    compile     'org.embulk:embulk-core:0.7.4'
-    provided    'org.embulk:embulk-core:0.7.4'
-    testCompile 'org.embulk:embulk-standards:0.7.4'
-    testCompile 'junit:junit:4.+'
-}
-
-task classpath(type: Copy, dependsOn: ["jar"]) {
-    doFirst { file('classpath').deleteDir() }
-    from (configurations.runtime - configurations.provided + files(jar.archivePath))
-    into 'classpath'
-}
-clean { delete 'classpath' }
-
-task gem(type: JRubyExec, dependsOn: ['build', 'gemspec', 'classpath']) {
-    jrubyArgs '-rrubygems/gem_runner', "-eGem::GemRunner.new.run(ARGV)", 'build'
-    script "build/gemspec"
-    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
-}
-
-task gemspec << {
-    file('build').mkdirs();
-    file('build/gemspec').write($/
-Gem::Specification.new do |spec|
-  spec.name          = "${project.name}"
-  spec.version       = "${project.version}"
-  spec.authors       = ["Hitoshi Tanaka"]
-  spec.homepage      = "https://github.com/hito4t/embulk-input-filesplit"
-  spec.summary       = "Embulk plugin for splitting input file"
-  spec.licenses      = ["Apache 2.0"]
-  spec.files         = `git ls-files`.split("\n").grep(%r"^(?!\.)").grep(%r"^(?!depends/)") + Dir["classpath/*.jar"]
-  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
-  spec.require_paths = ["lib"]
-end
-/$)
-}
-
-
-task gempush << {
-    "gem push pkg/embulk-input-filesplit-${project.version}.gem".execute().waitFor()
-}
+plugins {
+    id 'com.jfrog.bintray' version '1.6'
+    id 'com.github.jruby-gradle.base' version '1.2.1'
+    id 'java'
+}
+import com.github.jrubygradle.JRubyExec
+
+apply plugin: 'java'
+apply plugin: 'com.github.jruby-gradle.base'
+
+[compileJava, compileTestJava]*.options*.encoding = 'UTF-8'
+
+project.version = '0.1.4'
+
+repositories {
+    mavenCentral()
+    jcenter()
+}
+
+configurations {
+    provided
+}
+
+dependencies {
+    compile     'org.embulk:embulk-core:0.7.4'
+    provided    'org.embulk:embulk-core:0.7.4'
+    testCompile 'org.embulk:embulk-standards:0.7.4'
+    testCompile 'junit:junit:4.+'
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file('classpath').deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into 'classpath'
+}
+clean { delete 'classpath' }
+
+task gem(type: JRubyExec, dependsOn: ['build', 'gemspec', 'classpath']) {
+    jrubyArgs '-rrubygems/gem_runner', "-eGem::GemRunner.new.run(ARGV)", 'build'
+    scriptArgs "build/gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemspec << {
+    file('build').mkdirs();
+    file('build/gemspec').write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Hitoshi Tanaka"]
+  spec.homepage      = "https://github.com/hito4t/embulk-input-filesplit"
+  spec.summary       = "Embulk plugin for splitting input file"
+  spec.licenses      = ["Apache 2.0"]
+  spec.files         = `git ls-files`.split("\n").grep(%r"^(?!\.)").grep(%r"^(?!depends/)") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+end
+/$)
+}
+
+
+task gempush << {
+    "gem push pkg/embulk-input-filesplit-${project.version}.gem".execute().waitFor()
+}

data/lib/embulk/input/filesplit.rb

@@ -1,3 +1,3 @@
-Embulk::JavaPlugin.register_input(
-  :filesplit, "org.embulk.input.filesplit.LocalFileSplitInputPlugin",
-  File.expand_path('../../../../classpath', __FILE__))
+Embulk::JavaPlugin.register_input(
+  :filesplit, "org.embulk.input.filesplit.LocalFileSplitInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java

@@ -1,187 +1,300 @@
-package org.embulk.input.filesplit;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.SequenceInputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
-
-import com.google.common.base.Optional;
-
-
-public class LocalFileSplitInputPlugin
-        implements FileInputPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @Config("path")
-        public String getPath();
-
-        @Config("tasks")
-        @ConfigDefault("null")
-        public Optional<Integer> getTasks();
-
-        @Config("header_line")
-        @ConfigDefault("false")
-        public boolean getHeaderLine();
-
-        public List<PartialFile> getFiles();
-        public void setFiles(List<PartialFile> files);
-
-        @ConfigInject
-        public BufferAllocator getBufferAllocator();
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        int tasks;
-        if (task.getTasks().isPresent()) {
-            tasks = task.getTasks().get();
-            if (tasks <= 0) {
-                throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
-            }
-        } else {
-            tasks = Runtime.getRuntime().availableProcessors() * 2;
-        }
-
-        long size = new File(task.getPath()).length();
-        List<PartialFile> files = new ArrayList<PartialFile>();
-        for (int i = 0; i < tasks; i++) {
-            long start = size * i / tasks;
-            long end = size * (i + 1) / tasks;
-            if (start < end) {
-                files.add(new PartialFile(task.getPath(), start, end));
-            }
-        }
-
-        task.setFiles(files);
-
-        return resume(task.dump(), task.getFiles().size(), control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control)
-    {
-        control.run(taskSource, taskCount);
-
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<TaskReport> successTaskReports)
-    { }
-
-    @Override
-    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new LocalFileSplitInput(task, taskIndex);
-    }
-
-    public static class LocalFileSplitInput
-            extends InputStreamFileInput
-            implements TransactionalFileInput
-    {
-        public static class FileSplitProvider
-                implements InputStreamFileInput.Provider
-        {
-            private final PartialFile file;
-            private final boolean hasHeader;
-            private boolean opened = false;
-
-            public FileSplitProvider(PartialFile file, boolean hasHeader)
-            {
-                this.file = file;
-                this.hasHeader = hasHeader;
-            }
-
-            @Override
-            public InputStream openNext() throws IOException
-            {
-                if (opened) {
-                    return null;
-                }
-                opened = true;
-
-                InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
-                if (file.getStart() > 0 && hasHeader) {
-                    in = new SequenceInputStream(openHeader(file.getPath()), in);
-                }
-                return in;
-            }
-
-            @Override
-            public void close() { }
-
-            private InputStream openHeader(String path) throws IOException
-            {
-                ByteArrayOutputStream header = new ByteArrayOutputStream();
-                try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
-                    while (true) {
-                        int c = in.read();
-                        if (c < 0) {
-                            break;
-                        }
-
-                        header.write(c);
-
-                        if (c == '\n') {
-                            break;
-                        }
-
-                        if (c == '\r') {
-                            int c2 = in.read();
-                            if (c2 == '\n') {
-                                header.write(c2);
-                            }
-                            break;
-                        }
-                    }
-                }
-                header.close();
-                return new ByteArrayInputStream(header.toByteArray());
-            }
-        }
-
-        public LocalFileSplitInput(PluginTask task, int taskIndex)
-        {
-            super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
-        }
-
-        @Override
-        public void abort() { }
-
-        @Override
-        public TaskReport commit()
-        {
-            return Exec.newTaskReport();
-        }
-    }
-}
+package org.embulk.input.filesplit;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.nio.file.Files;
+import java.nio.file.FileVisitOption;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.FileVisitResult;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+import com.google.common.collect.ImmutableList;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamFileInput;
+
+import com.google.common.base.Optional;
+
+public class LocalFileSplitInputPlugin
+        implements FileInputPlugin
+{
+
+    private final static Path CURRENT_DIR = Paths.get(".").normalize();
+
+    public interface PluginTask
+            extends Task
+    {
+        @Config("path")
+        @ConfigDefault("null")
+        public Optional<String> getPath();
+
+        @Config("path_prefix")
+        @ConfigDefault("null")
+        public Optional<String> getPathPrefix();
+
+        @Config("tasks")
+        @ConfigDefault("null")
+        public Optional<Integer> getTasks();
+
+        @Config("header_line")
+        @ConfigDefault("false")
+        public boolean getHeaderLine();
+
+        public List<PartialFile> getFiles();
+        public void setFiles(List<PartialFile> files);
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        int tasks;
+        if (task.getTasks().isPresent()) {
+            tasks = task.getTasks().get();
+            if (tasks <= 0) {
+                throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
+            }
+        } else {
+            tasks = Runtime.getRuntime().availableProcessors() * 2;
+        }
+
+        List<String> paths = new ArrayList<String>();
+        if (task.getPath().isPresent()) {
+            if (task.getPathPrefix().isPresent()) {
+                throw new IllegalArgumentException("Cannot specify both 'path' and 'path_prefix'");
+            }
+            paths.add(task.getPath().get());
+        } else if (task.getPathPrefix().isPresent()) {
+            paths.addAll(listFiles(task.getPathPrefix().get()));
+        } else {
+            throw new IllegalArgumentException("Specify either 'path' or 'path_prefix'");
+        }
+
+        List<PartialFile> files = new ArrayList<PartialFile>();
+        for (String path : paths) {
+            long size = new File(path).length();
+            for (int i = 0; i < tasks; i++) {
+                long start = size * i / tasks;
+                long end = size * (i + 1) / tasks;
+                if (start < end) {
+                    files.add(new PartialFile(path, start, end));
+                }
+            }
+        }
+
+        task.setFiles(files);
+
+        return resume(task.dump(), task.getFiles().size(), control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control)
+    {
+        control.run(taskSource, taskCount);
+
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports)
+    { }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        return new LocalFileSplitInput(task, taskIndex);
+    }
+
+    /**
+     * Most of this implementation is based on LocalFileInputPlugin hosted at:
+     *   https://github.com/embulk/embulk
+     *
+     */
+    private List<String> listFiles(String prefix)
+    {
+        final Path pathPrefix = Paths.get(prefix).normalize();
+        final Path directory;
+        final String fileNamePrefix;
+        if (Files.isDirectory(pathPrefix)) {
+            directory = pathPrefix;
+            fileNamePrefix = "";
+        } else {
+            fileNamePrefix = pathPrefix.getFileName().toString();
+            Path d = pathPrefix.getParent();
+            directory = (d == null ? CURRENT_DIR : d);
+        }
+
+        final ImmutableList.Builder<String> builder = ImmutableList.builder();
+        try {
+            int maxDepth = Integer.MAX_VALUE;
+            Set<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
+
+            Files.walkFileTree(directory, opts, maxDepth, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs)
+                {
+                    if (path.equals(directory)) {
+                        return FileVisitResult.CONTINUE;
+                    } else {
+                        Path parent = path.getParent();
+                        if (parent == null) {
+                            parent = CURRENT_DIR;
+                        }
+                        if (parent.equals(directory)) {
+                            if (path.getFileName().toString().startsWith(fileNamePrefix)) {
+                                return FileVisitResult.CONTINUE;
+                            } else {
+                                return FileVisitResult.SKIP_SUBTREE;
+                            }
+                        } else {
+                            return FileVisitResult.CONTINUE;
+                        }
+                    }
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
+                {
+                    try {
+                        // Avoid directories from listing.
+                        // Directories are normally unvisited with |FileVisitor#visitFile|, but symbolic links to
+                        // directories are visited like files unless |FOLLOW_LINKS| is set in |Files#walkFileTree|.
+                        // Symbolic links to directories are explicitly skipped here by checking with |Path#toReadlPath|.
+                        if (Files.isDirectory(path.toRealPath())) {
+                            return FileVisitResult.CONTINUE;
+                        }
+                    } catch (IOException ex){
+                        throw new RuntimeException("Can't resolve symbolic link", ex);
+                    }
+                    Path parent = path.getParent();
+                    if (parent == null) {
+                        parent = CURRENT_DIR;
+                    }
+                    if (parent.equals(directory)) {
+                        if (path.getFileName().toString().startsWith(fileNamePrefix)) {
+                            builder.add(path.toString());
+                            return FileVisitResult.CONTINUE;
+                        }
+                    } else {
+                        builder.add(path.toString());
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException ex) {
+            throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex);
+        }
+        return builder.build();
+    }
+
+    public static class LocalFileSplitInput
+            extends InputStreamFileInput
+            implements TransactionalFileInput
+    {
+        public static class FileSplitProvider
+                implements InputStreamFileInput.Provider
+        {
+            private final PartialFile file;
+            private final boolean hasHeader;
+            private boolean opened = false;
+
+            public FileSplitProvider(PartialFile file, boolean hasHeader)
+            {
+                this.file = file;
+                this.hasHeader = hasHeader;
+            }
+
+            @Override
+            public InputStream openNext() throws IOException
+            {
+                if (opened) {
+                    return null;
+                }
+                opened = true;
+
+                InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
+                if (file.getStart() > 0 && hasHeader) {
+                    in = new SequenceInputStream(openHeader(file.getPath()), in);
+                }
+                return in;
+            }
+
+            @Override
+            public void close() { }
+
+            private InputStream openHeader(String path) throws IOException
+            {
+                ByteArrayOutputStream header = new ByteArrayOutputStream();
+                try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
+                    while (true) {
+                        int c = in.read();
+                        if (c < 0) {
+                            break;
+                        }
+
+                        header.write(c);
+
+                        if (c == '\n') {
+                            break;
+                        }
+
+                        if (c == '\r') {
+                            int c2 = in.read();
+                            if (c2 == '\n') {
+                                header.write(c2);
+                            }
+                            break;
+                        }
+                    }
+                }
+                header.close();
+                return new ByteArrayInputStream(header.toByteArray());
+            }
+        }
+
+        public LocalFileSplitInput(PluginTask task, int taskIndex)
+        {
+            super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
+        }
+
+        @Override
+        public void abort() { }
+
+        @Override
+        public TaskReport commit()
+        {
+            return Exec.newTaskReport();
+        }
+    }
+}