embulk-input-sftp 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 107c7dbf7c423b83b5677db6da16b272c8dccab4
-  data.tar.gz: edcada631a85858a63c57b7663aedbdb2ab365f6
+  metadata.gz: 2afbf35e073d8e31985a7c797395c796828aa675
+  data.tar.gz: 73667da5ec1be837b3c5811969dc5c4ba4014731
 SHA512:
-  metadata.gz: d28c5e3ba7db475342b75df902b2f588e3259384138e3b1ee696e6582aa1d9ea1fecf5e2f0955cda3a2e19454b69c1d1d346c53b1153030b6406e0680cf03519
-  data.tar.gz: 1e7b0e362480f2e610c87871f49f020458054ea5d114a6ab064dde3bf32cf021e44a1954a1ada5f7057fe85d4e780ceaec9b7c5c0e277af2d4f99b283646f26d
+  metadata.gz: 3bb521683b3668ef39f71233755eaabc3f2b67578e711efe942e095dbf97969f1b7aeeb72bb5cbb422a7eeb3d4dc9384e549e0a9f94a331a3829c0d5d779bb8a
+  data.tar.gz: cbb3f79afc30b70fca469bd6012653d13e58983d03934a48d76e474c277e9cc7b79161a03eaf48b38c2ce7389048545cd0dfa77c7d373a216eec556e84941aee

data/.travis.yml

@@ -0,0 +1,10 @@
+language: java
+
+jdk:
+  - oraclejdk8
+  - oraclejdk7
+  - openjdk7
+
+script:
+  - ./gradlew gem
+  - ./gradlew --info check jacocoTestReport

data/CHANGELOG.md

@@ -1,10 +1,16 @@
+## 0.2.0 - 2016-08-19
+
+* [new feature] Support incremental option [#11](https://github.com/sakama/embulk-input-sftp/pull/11)
+* [maintenance] Mask password in the log [#12](https://github.com/sakama/embulk-input-sftp/pull/12)
+* [maintenance] Fix file listing order and handling of last_path [#14](https://github.com/sakama/embulk-input-sftp/pull/14)
+
 ## 0.1.2 - 2015-03-23
 
 * [maintenance] Use RetryExecutor when retrying that is provide by embulk-core [#9](https://github.com/sakama/embulk-input-sftp/pull/9)
 
 ## 0.1.1 - 2015-03-18
 
-* [feature] Support last_path_ option [#2](https://github.com/sakama/embulk-input-sftp/pull/2)[#4](https://github.com/sakama/embulk-input-sftp/pull/4)[#7](https://github.com/sakama/embulk-input-sftp/pull/7)
-* [feature] Support path_match_pattern option [#6](https://github.com/sakama/embulk-input-sftp/pull/6)
+* [new feature] Support last_path_ option [#2](https://github.com/sakama/embulk-input-sftp/pull/2)[#4](https://github.com/sakama/embulk-input-sftp/pull/4)[#7](https://github.com/sakama/embulk-input-sftp/pull/7)
+* [new feature] Support path_match_pattern option [#6](https://github.com/sakama/embulk-input-sftp/pull/6)
 * [maintenance] Add unit test [#3](https://github.com/sakama/embulk-input-sftp/pull/3)
 * [maintenance] Skip retry of file downloading when permission denied error happens [#1](https://github.com/sakama/embulk-input-sftp/pull/1)

data/README.md

@@ -1,4 +1,5 @@
 # SFTP file input plugin for Embulk
+[![Build Status](https://travis-ci.org/embulk/embulk-input-sftp.svg?branch=master)](https://travis-ci.org/embulk/embulk-input-sftp)
 
 Reads files stored on remote server using SFTP
 
@@ -19,6 +20,7 @@ Reads files stored on remote server using SFTP
 - **user_directory_is_root**: (boolean, default: `true`)
 - **timeout**: sftp connection timeout seconds (integer, default: `600`)
 - **path_prefix**: Prefix of output paths (string, required)
+- **incremental**: enables incremental loading(boolean, optional. default: `true`). If incremental loading is enabled, config diff for the next execution will include `last_path` parameter so that next execution skips files before the path. Otherwise, `last_path` will not be included.
 - **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)
 - **total_file_count_limit**: maximum number of files to read (integer, optional)
 - **min_task_size (experimental)**: minimum size of a task. If this is larger than 0, one task includes multiple input files. This is useful if too many number of tasks impacts performance of output or executor plugins badly. (integer, optional)

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -22,7 +22,7 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.6"
     provided "org.embulk:embulk-core:0.8.6"
-    compile "org.apache.commons:commons-vfs2:2.+"
+    compile "org.apache.commons:commons-vfs2:2.1"
     compile "org.apache.commons:commons-io:1.3.2"
     compile "com.jcraft:jsch:0.1.53"
     testCompile "junit:junit:4.+"
@@ -82,7 +82,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[Reads files stored on remote server using SFTP.]
   spec.email         = ["satoshiakama@gmail.com"]
   spec.licenses      = ["Apache-2.0"]
-  spec.homepage      = "https://github.com/sakama/embulk-input-sftp"
+  spec.homepage      = "https://github.com/embulk/embulk-input-sftp"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/src/main/java/org/embulk/input/sftp/FileList.java

@@ -158,7 +158,6 @@ public class FileList
 
             int index = entries.size();
             entries.add(new Entry(index, size));
-            log.info("add file to the request list: {}", path);
 
             byte[] data = path.getBytes(StandardCharsets.UTF_8);
             castBuffer.putInt(0, data.length);

data/src/main/java/org/embulk/input/sftp/PluginTask.java

@@ -49,6 +49,10 @@ public interface PluginTask
     @Config("path_prefix")
     String getPathPrefix();
 
+    @Config("incremental")
+    @ConfigDefault("true")
+    boolean getIncremental();
+
     @Config("last_path")
     @ConfigDefault("null")
     Optional<String> getLastPath();

data/src/main/java/org/embulk/input/sftp/SftpFileInput.java

@@ -25,6 +25,7 @@ import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Arrays;
 
 public class SftpFileInput
         extends InputStreamFileInput
@@ -107,6 +108,7 @@ public class SftpFileInput
                 if (proxy.getHost().isPresent()) {
                     builder.setProxyHost(fsOptions, proxy.getHost().get());
                     builder.setProxyPort(fsOptions, proxy.getPort());
+                    log.info("Using proxy {}:{} proxy_type:{}", proxy.getHost().get(), proxy.getPort(), proxy.getType().toString());
                 }
 
                 if (proxy.getUser().isPresent()) {
@@ -132,7 +134,9 @@ public class SftpFileInput
     public static String getSftpFileUri(PluginTask task, String path)
     {
         try {
-            return new URI("sftp", initializeUserInfo(task), task.getHost(), task.getPort(), path, null, null).toString();
+            String uri = new URI("sftp", initializeUserInfo(task), task.getHost(), task.getPort(), path, null, null).toString();
+            log.info("Connecting to sftp://{}:***@{}:{}/", task.getUser(), task.getHost(), task.getPort());
+            return uri;
         }
         catch (URISyntaxException ex) {
             throw new ConfigException(ex);
@@ -180,7 +184,9 @@ public class SftpFileInput
                             FileObject files = manager.resolveFile(getSftpFileUri(task, task.getPathPrefix()), fsOptions);
                             String basename = FilenameUtils.getBaseName(task.getPathPrefix());
                             if (files.isFolder()) {
-                                for (FileObject f : files.getChildren()) {
+                                FileObject[] children = files.getChildren();
+                                Arrays.sort(children);
+                                for (FileObject f : children) {
                                     if (f.isFile()) {
                                         addFileToList(builder, f.toString(), f.getContent().getSize(), "", lastKey);
                                     }
@@ -188,7 +194,9 @@ public class SftpFileInput
                             }
                             else {
                                 FileObject parent = files.getParent();
-                                for (FileObject f : parent.getChildren()) {
+                                FileObject[] children = parent.getChildren();
+                                Arrays.sort(children);
+                                for (FileObject f : children) {
                                     if (f.isFile()) {
                                         addFileToList(builder, f.toString(), f.getContent().getSize(), basename, lastKey);
                                     }
@@ -238,24 +246,20 @@ public class SftpFileInput
             String remoteBasename = FilenameUtils.getBaseName(fileName);
             if (remoteBasename.startsWith(basename)) {
                 if (lastKey != null && !isMatchLastKey) {
-                    if (!fileName.equals(lastKey)) {
-                        return;
-                    }
-                    else {
+                    if (fileName.equals(lastKey)) {
                         isMatchLastKey = true;
                     }
+                    return;
                 }
                 builder.add(fileName, fileSize);
             }
         }
         else {
             if (lastKey != null && !isMatchLastKey) {
-                if (!fileName.equals(lastKey)) {
-                    return;
-                }
-                else {
+                if (fileName.equals(lastKey)) {
                     isMatchLastKey = true;
                 }
+                return;
             }
             builder.add(fileName, fileSize);
         }

data/src/main/java/org/embulk/input/sftp/SftpFileInputPlugin.java

@@ -33,7 +33,9 @@ public class SftpFileInputPlugin
         control.run(taskSource, taskCount);
 
         ConfigDiff configDiff = Exec.newConfigDiff();
-        configDiff.set("last_path", SftpFileInput.getRelativePath(task.getFiles().getLastPath(task.getLastPath())));
+        if (task.getIncremental()) {
+            configDiff.set("last_path", SftpFileInput.getRelativePath(task.getFiles().getLastPath(task.getLastPath())));
+        }
 
         return configDiff;
     }

data/src/main/java/org/embulk/input/sftp/SingleFileProvider.java

@@ -53,7 +53,6 @@ public class SingleFileProvider
                         @Override
                         public InputStream call() throws FileSystemException
                         {
-                            log.info("Starting to download file {}", key);
                             FileObject file = manager.resolveFile(key, fsOptions);
                             return file.getContent().getInputStream();
                         }

data/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java

@@ -124,6 +124,7 @@ public class TestSftpFileInputPlugin
 
         PluginTask task = config.loadConfig(PluginTask.class);
         assertEquals(22, task.getPort());
+        assertEquals(true, task.getIncremental());
         assertEquals(true, task.getUserDirIsRoot());
         assertEquals(600, task.getSftpConnectionTimeout());
         assertEquals(5, task.getMaxConnectionRetry());
@@ -173,6 +174,23 @@ public class TestSftpFileInputPlugin
         assertEquals("in/aa/a", configDiff.get(String.class, "last_path"));
     }
 
+    @Test
+    public void testResumeIncrementalFalse()
+    {
+        ConfigSource newConfig = config.deepCopy().set("incremental", false);
+        PluginTask task = newConfig.loadConfig(PluginTask.class);
+        task.setFiles(createFileList(Arrays.asList("in/aa/a"), task));
+        ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            {
+                return emptyTaskReports(taskCount);
+            }
+        });
+        assertEquals("{}", configDiff.toString());
+    }
+
     @Test
     public void testCleanup()
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-sftp
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-23 00:00:00.000000000 Z
+date: 2016-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - CHANGELOG.md
 - README.md
 - build.gradle
@@ -70,10 +71,10 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/commons-io-1.3.2.jar
 - classpath/commons-logging-1.2.jar
-- classpath/commons-vfs2-2.1.1660580.2.jar
-- classpath/embulk-input-sftp-0.1.2.jar
+- classpath/commons-vfs2-2.1.jar
+- classpath/embulk-input-sftp-0.2.0.jar
 - classpath/jsch-0.1.53.jar
-homepage: https://github.com/sakama/embulk-input-sftp
+homepage: https://github.com/embulk/embulk-input-sftp
 licenses:
 - Apache-2.0
 metadata: {}