embulk-input-sftp 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72aaee2da1604f1a298e13d37ab74be174cddb5f
-  data.tar.gz: 3c062e178443efd8b44874eb9cb2d446385d010b
+  metadata.gz: 66e8dab8cd68fdd0a9844dac72d51eaf070cf93a
+  data.tar.gz: 9de2db631fa24c526e448a2985003063f05a2ae7
 SHA512:
-  metadata.gz: d21a1f92aae66016ea648dc94e2f757e7bc3d968bda4953f35623307f649a49bde0d8b0bbbc84b79cedfb25c35e069f7b6b30134a3959d9211baad7581cb3c59
-  data.tar.gz: 67c3b0a1c2b054093f61aa5ca0946c9fed2f74a0f223e0ee60c44737fa33cbbd5fd90469f3b5baafc62eb45d74c19e504cd87c498a3ebdf8d3bb1dc7a8e5967f
+  metadata.gz: 5079bf38ede9c5f4b611831144cdcc307c3b71acc6f2b2e43f3570d67abf21f4ca762dddfc0273b3be6cb7740072beaad0865fd5e4a98360d8206c1c88cfe3b0
+  data.tar.gz: 538c81c868addc2d141770ab5c88d3d38aa0e9e297214a2a42255e80f4f7de91da7ac646eae6aababe301b611ace77df261e7a43931e98c4d3e76bb0100c9e02

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+## 0.1.1 - 2015-03-18
+
+* [feature] Support last_path_ option [#2](https://github.com/sakama/embulk-input-sftp/pull/2)[#4](https://github.com/sakama/embulk-input-sftp/pull/4)[#7](https://github.com/sakama/embulk-input-sftp/pull/7)
+* [feature] Support path_match_pattern option [#6](https://github.com/sakama/embulk-input-sftp/pull/6)
+* [maintenance] Add unit test [#3](https://github.com/sakama/embulk-input-sftp/pull/3)
+* [maintenance] Skip retry of file downloading when permission denied error happens [#1](https://github.com/sakama/embulk-input-sftp/pull/1)

data/README.md

@@ -19,8 +19,11 @@ Reads files stored on remote server using SFTP
 - **user_directory_is_root**: (boolean, default: `true`)
 - **timeout**: sftp connection timeout seconds (integer, default: `600`)
 - **path_prefix**: Prefix of output paths (string, required)
+- **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)
+- **total_file_count_limit**: maximum number of files to read (integer, optional)
 - **file_ext**: Extension of output files (string, required)
 - **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
+- **min_task_size (experimental)**: minimum size of a task. If this is larger than 0, one task includes multiple input files. This is useful if too many number of tasks impacts performance of output or executor plugins badly. (integer, optional)
 
 ### Proxy configuration
 
@@ -50,6 +53,21 @@ in:
   path_prefix: /data/sftp
 ```
 
+To filter files using regexp:
+
+```yaml
+in:
+  type: sftp
+  path_prefix: logs/csv-
+  ...
+  path_match_pattern: \.csv$   # a file will be skipped if its path doesn't match with this pattern
+
+  ## some examples of regexp:
+  #path_match_pattern: /archive/         # match files in .../archive/... directory
+  #path_match_pattern: /data1/|/data2/   # match files in .../data1/... or .../data2/... directory
+  #path_match_pattern: .csv$|.csv.gz$    # match files whose suffix is .csv or .csv.gz
+```
+
 With proxy
 ```yaml
 in:

data/build.gradle

@@ -3,6 +3,7 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "checkstyle"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -13,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/sftp/FileList.java

@@ -0,0 +1,341 @@
+package org.embulk.input.sftp;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+// this class should be moved to embulk-core
+public class FileList
+{
+    public interface Task
+    {
+        @Config("path_match_pattern")
+        @ConfigDefault("\".*\"")
+        String getPathMatchPattern();
+
+        @Config("total_file_count_limit")
+        @ConfigDefault("2147483647")
+        int getTotalFileCountLimit();
+
+        // TODO support more algorithms to combine tasks
+        @Config("min_task_size")
+        @ConfigDefault("0")
+        long getMinTaskSize();
+    }
+
+    public static class Entry
+    {
+        private int index;
+        private long size;
+
+        @JsonCreator
+        public Entry(
+                @JsonProperty("index") int index,
+                @JsonProperty("size") long size)
+        {
+            this.index = index;
+            this.size = size;
+        }
+
+        @JsonProperty("index")
+        public int getIndex()
+        {
+            return index;
+        }
+
+        @JsonProperty("size")
+        public long getSize()
+        {
+            return size;
+        }
+    }
+
+    public static class Builder
+    {
+        private final Logger log = Exec.getLogger(FileList.class);
+        private final ByteArrayOutputStream binary;
+        private final OutputStream stream;
+        private final List<Entry> entries = new ArrayList<>();
+        private String last = null;
+
+        private int limitCount = Integer.MAX_VALUE;
+        private long minTaskSize = 1;
+        private Pattern pathMatchPattern;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public Builder(Task task)
+        {
+            this();
+            this.pathMatchPattern = Pattern.compile(task.getPathMatchPattern());
+            this.limitCount = task.getTotalFileCountLimit();
+            this.minTaskSize = task.getMinTaskSize();
+        }
+
+        public Builder(ConfigSource config)
+        {
+            this();
+            this.pathMatchPattern = Pattern.compile(config.get(String.class, "path_match_pattern", ".*"));
+            this.limitCount = config.get(int.class, "total_file_count_limit", Integer.MAX_VALUE);
+            this.minTaskSize = config.get(long.class, "min_task_size", 0L);
+        }
+
+        public Builder()
+        {
+            binary = new ByteArrayOutputStream();
+            try {
+                stream = new BufferedOutputStream(new GZIPOutputStream(binary));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        public Builder limitTotalFileCount(int limitCount)
+        {
+            this.limitCount = limitCount;
+            return this;
+        }
+
+        public Builder minTaskSize(long bytes)
+        {
+            this.minTaskSize = bytes;
+            return this;
+        }
+
+        public Builder pathMatchPattern(String pattern)
+        {
+            this.pathMatchPattern = Pattern.compile(pattern);
+            return this;
+        }
+
+        public int size()
+        {
+            return entries.size();
+        }
+
+        public boolean needsMore()
+        {
+            return size() < limitCount;
+        }
+
+        // returns true if this file is used
+        public synchronized boolean add(String path, long size)
+        {
+            // TODO throw IllegalStateException if stream is already closed
+
+            if (!needsMore()) {
+                return false;
+            }
+
+            if (!pathMatchPattern.matcher(path).find()) {
+                return false;
+            }
+
+            int index = entries.size();
+            entries.add(new Entry(index, size));
+            log.info("add file to the request list: {}", path);
+
+            byte[] data = path.getBytes(StandardCharsets.UTF_8);
+            castBuffer.putInt(0, data.length);
+            try {
+                stream.write(castBuffer.array());
+                stream.write(data);
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+
+            last = path;
+            return true;
+        }
+
+        public FileList build()
+        {
+            try {
+                stream.close();
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+        }
+
+        private List<List<Entry>> getSplits(List<Entry> all)
+        {
+            List<List<Entry>> tasks = new ArrayList<>();
+            long currentTaskSize = 0;
+            List<Entry> currentTask = new ArrayList<>();
+            for (Entry entry : all) {
+                currentTask.add(entry);
+                currentTaskSize += entry.getSize();  // TODO consider to multiply the size by cost_per_byte, and add cost_per_file
+                if (currentTaskSize >= minTaskSize) {
+                    tasks.add(currentTask);
+                    currentTask = new ArrayList<>();
+                    currentTaskSize = 0;
+                }
+            }
+            if (!currentTask.isEmpty()) {
+                tasks.add(currentTask);
+            }
+            return tasks;
+        }
+    }
+
+    private final byte[] data;
+    private final List<List<Entry>> tasks;
+    private final Optional<String> last;
+
+    @JsonCreator
+    @Deprecated
+    public FileList(
+            @JsonProperty("data") byte[] data,
+            @JsonProperty("tasks") List<List<Entry>> tasks,
+            @JsonProperty("last") Optional<String> last)
+    {
+        this.data = data;
+        this.tasks = tasks;
+        this.last = last;
+    }
+
+    @JsonIgnore
+    public Optional<String> getLastPath(Optional<String> lastLastPath)
+    {
+        if (last.isPresent()) {
+            return last;
+        }
+        return lastLastPath;
+    }
+
+    @JsonIgnore
+    public int getTaskCount()
+    {
+        return tasks.size();
+    }
+
+    @JsonIgnore
+    public List<String> get(int i)
+    {
+        return new EntryList(data, tasks.get(i));
+    }
+
+    @JsonProperty("data")
+    @Deprecated
+    public byte[] getData()
+    {
+        return data;
+    }
+
+    @JsonProperty("tasks")
+    @Deprecated
+    public List<List<Entry>> getTasks()
+    {
+        return tasks;
+    }
+
+    @JsonProperty("last")
+    @Deprecated
+    public Optional<String> getLast()
+    {
+        return last;
+    }
+
+    private class EntryList
+            extends AbstractList<String>
+    {
+        private final byte[] data;
+        private final List<Entry> entries;
+        private InputStream stream;
+        private int current;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public EntryList(byte[] data, List<Entry> entries)
+        {
+            this.data = data;
+            this.entries = entries;
+            try {
+                this.stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            this.current = 0;
+        }
+
+        @Override
+        public synchronized String get(int i)
+        {
+            Entry e = entries.get(i);
+            if (e.getIndex() < current) {
+                // rewind to the head
+                try {
+                    stream.close();
+                    stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+                }
+                catch (IOException ex) {
+                    throw Throwables.propagate(ex);
+                }
+                current = 0;
+            }
+
+            while (current < e.getIndex()) {
+                readNext();
+            }
+            // now current == e.getIndex()
+            return readNextString();
+        }
+
+        @Override
+        public int size()
+        {
+            return entries.size();
+        }
+
+        private byte[] readNext()
+        {
+            try {
+                stream.read(castBuffer.array());
+                int n = castBuffer.getInt(0);
+                byte[] b = new byte[n];  // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
+                stream.read(b);
+
+                current++;
+
+                return b;
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        private String readNextString()
+        {
+            return new String(readNext(), StandardCharsets.UTF_8);
+        }
+    }
+}

data/src/main/java/org/embulk/input/sftp/PluginTask.java

@@ -8,10 +8,8 @@ import org.embulk.config.Task;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.unit.LocalFile;
 
-import java.util.List;
-
 public interface PluginTask
-        extends Task
+        extends Task, FileList.Task
 {
     @Config("host")
     String getHost();
@@ -59,8 +57,8 @@ public interface PluginTask
     @ConfigDefault("null")
     Optional<ProxyTask> getProxy();
 
-    List<String> getFiles();
-    void setFiles(List<String> files);
+    FileList getFiles();
+    void setFiles(FileList files);
 
     @ConfigInject
     BufferAllocator getBufferAllocator();

data/src/main/java/org/embulk/input/sftp/SftpFileInput.java

@@ -1,8 +1,8 @@
 package org.embulk.input.sftp;
 
 import com.google.common.base.Function;
+import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
@@ -21,17 +21,17 @@ import org.slf4j.Logger;
 import java.io.File;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.List;
 
 public class SftpFileInput
         extends InputStreamFileInput
         implements TransactionalFileInput
 {
     private static final Logger log = Exec.getLogger(SftpFileInput.class);
+    private static boolean isMatchLastKey = false;
 
     public SftpFileInput(PluginTask task, int taskIndex)
     {
-        super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex, initializeStandardFileSystemManager()));
+        super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex, initializeStandardFileSystemManager(), initializeFsOptions(task)));
     }
 
     public void abort()
@@ -124,33 +124,54 @@ public class SftpFileInput
         return fsOptions;
     }
 
-    public static String getSftpFileUri(PluginTask task)
+    public static String getSftpFileUri(PluginTask task, String path)
     {
         try {
-            return new URI("sftp", initializeUserInfo(task), task.getHost(), task.getPort(), task.getPathPrefix(), null, null).toString();
+            return new URI("sftp", initializeUserInfo(task), task.getHost(), task.getPort(), path, null, null).toString();
         }
         catch (URISyntaxException ex) {
             throw new ConfigException(ex);
         }
     }
 
-    public static List<String> listFilesByPrefix(PluginTask task)
+    public static String getRelativePath(Optional<String> path)
     {
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
+        try {
+            if (path.isPresent()) {
+                return new URI(path.get()).getPath();
+            }
+            else {
+                return null;
+            }
+        }
+        catch (URISyntaxException ex) {
+            return null;
+        }
+    }
+
+    public static FileList listFilesByPrefix(PluginTask task)
+    {
+        FileList.Builder builder = new FileList.Builder(task);
         int maxConnectionRetry = task.getMaxConnectionRetry();
+        String lastKey = null;
 
         StandardFileSystemManager manager = null;
         int count = 0;
         while (true) {
             try {
                 manager = initializeStandardFileSystemManager();
-                FileObject files = manager.resolveFile(getSftpFileUri(task), initializeFsOptions(task));
-                String basename = FilenameUtils.getBaseName(task.getPathPrefix());
+                FileSystemOptions fsOptions = initializeFsOptions(task);
 
+                if (task.getLastPath().isPresent() && !task.getLastPath().get().isEmpty()) {
+                    lastKey = manager.resolveFile(getSftpFileUri(task, task.getLastPath().get()), fsOptions).toString();
+                }
+
+                FileObject files = manager.resolveFile(getSftpFileUri(task, task.getPathPrefix()), fsOptions);
+                String basename = FilenameUtils.getBaseName(task.getPathPrefix());
                 if (files.isFolder()) {
                     for (FileObject f : files.getChildren()) {
                         if (f.isFile()) {
-                            addFileToList(builder, f.toString(), "");
+                            addFileToList(builder, f.toString(), f.getContent().getSize(), "", lastKey);
                         }
                     }
                 }
@@ -158,7 +179,7 @@ public class SftpFileInput
                     FileObject parent = files.getParent();
                     for (FileObject f : parent.getChildren()) {
                         if (f.isFile()) {
-                            addFileToList(builder, f.toString(), basename);
+                            addFileToList(builder, f.toString(), f.getContent().getSize(), basename, lastKey);
                         }
                     }
                 }
@@ -189,18 +210,32 @@ public class SftpFileInput
         }
     }
 
-    private static void addFileToList(ImmutableList.Builder<String> builder, String fileName, String basename)
+    private static void addFileToList(FileList.Builder builder, String fileName, long fileSize, String basename, String lastKey)
     {
         if (!basename.isEmpty()) {
             String remoteBasename = FilenameUtils.getBaseName(fileName);
             if (remoteBasename.startsWith(basename)) {
-                builder.add(fileName);
-                log.info("add file to the request list: {}", fileName);
+                if (lastKey != null && !isMatchLastKey) {
+                    if (!fileName.equals(lastKey)) {
+                        return;
+                    }
+                    else {
+                        isMatchLastKey = true;
+                    }
+                }
+                builder.add(fileName, fileSize);
             }
         }
         else {
-            builder.add(fileName);
-            log.info("add file to the request list: {}", fileName);
+            if (lastKey != null && !isMatchLastKey) {
+                if (!fileName.equals(lastKey)) {
+                    return;
+                }
+                else {
+                    isMatchLastKey = true;
+                }
+            }
+            builder.add(fileName, fileSize);
         }
     }