embulk-output-sftp 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c56a0584cf221cab6a3e4caf8f5f6b7fd8c73750
-  data.tar.gz: 9a54d758d53715b2dd6b13ecebbea1306e92ea1d
+  metadata.gz: 98edf71b4a5d7998403671b7e181b250f7db9981
+  data.tar.gz: 940f0d1fea8ed83c957bdfca08a2bb89725b487b
 SHA512:
-  metadata.gz: 10749e577ff285cc6cce4254517f712925e637e959cddcefce74dd086cb54605c675bc272270b50d5291cd8dc36a307dab205dbcba65aaeb1a29814b9b3ad63b
-  data.tar.gz: 9991eb85333675b381725365232947f628862a6e9430e912f3eeeee0d3e6d76e31db40a92fd7b7856706378c049c5b99ace8617980181fa6ec3bd1fc5b6963e9
+  metadata.gz: c4e8426ac217df25329da8a514b1603158bc7d6ed2b630b2e28486fcd36fb81e24aee588e5b1b2f6dea2e0996789a947eaac617330cb6180ca3e6afc35a9f0ed
+  data.tar.gz: 99c6f36a658a92dd3710c9b305892e5961f9d154694570b5f05e7cbf6e4ad2794adfda169e175622f3bb7ff5aa5714e25368e41a0f6224d3dce8acc004b87d9d

data/.travis.yml

@@ -1,8 +1,7 @@
-dist: precise
+dist: trusty
 language: java
 jdk:
-  - openjdk7
-  - oraclejdk7
+  - openjdk8
   - oraclejdk8
 script:
   - ./gradlew test

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+0.1.11 (2018-08-27)
+- Enhance: Add 2 new configs `local_temp_file` (boolean) and `temp_file_threshold` (long)
+  - https://github.com/embulk/embulk-output-sftp/pull/50
 0.1.10 (2018-05-07)
 - Fix: Use java.util.regex.Pattern for host name validation
   - https://github.com/embulk/embulk-output-sftp/pull/49

data/README.md

@@ -24,6 +24,8 @@ Stores files on a SFTP Server
 - **file_ext**: Extension of output files (string, required)
 - **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
 - **rename_file_after_upload**: Upload `file_ext` + ".tmp" first, then rename it after upload finish (boolean, default: `false`)
+- **local_buffering**: Use local temp file to buffer records. If `false`, plugin will buffer records to remote file directly, with ".tmp" as filename suffix (boolean, default: `true`)
+- **temp_file_threshold**: Maximum file size of local temp file, plugin will flush (append) to remote file when local temp file reaches threshold (long, default: `5368709120`, ie. 5GiB, min: 50MiB, max: 10GiB)
 
 ### Proxy configuration
 
@@ -53,6 +55,7 @@ out:
   path_prefix: /data/sftp
   file_ext: _20151020.tsv
   sequence_format: ".%01d%01d"
+  temp_file_threshold: 10737418240 # 10GiB
 ```
 
 With proxy

data/build.gradle

@@ -17,23 +17,24 @@ configurations {
 }
 
 group = "org.embulk.output.sftp"
-version = "0.1.10"
+version = "0.1.11"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.6"
-    provided "org.embulk:embulk-core:0.8.6"
+    compile  "org.embulk:embulk-core:0.9.7"
+    provided "org.embulk:embulk-core:0.9.7"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     compile "org.apache.commons:commons-vfs2:2.2"
     compile "commons-io:commons-io:2.6"
     compile "com.jcraft:jsch:0.1.54"
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.6:tests"
-    testCompile "org.embulk:embulk-standards:0.8.6"
+    testCompile "org.embulk:embulk-core:0.9.7:tests"
+    testCompile "org.embulk:embulk-standards:0.9.7"
     testCompile "org.apache.sshd:apache-sshd:1.1.0+"
     testCompile "org.littleshoot:littleproxy:1.1.0-beta1"
     testCompile "io.netty:netty-all:4.0.34.Final"
+    testCompile "org.mockito:mockito-core:2.+"
 }
 
 jacocoTestReport {

data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java

@@ -13,6 +13,9 @@ import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
 import org.embulk.spi.unit.LocalFile;
 
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+
 import java.util.List;
 import java.util.Map;
 
@@ -47,7 +50,7 @@ public class SftpFileOutputPlugin
 
         @Config("user_directory_is_root")
         @ConfigDefault("true")
-        public Boolean getUserDirIsRoot();
+        public boolean getUserDirIsRoot();
 
         @Config("timeout")
         @ConfigDefault("600") // 10 minutes
@@ -73,7 +76,18 @@ public class SftpFileOutputPlugin
 
         @Config("rename_file_after_upload")
         @ConfigDefault("false")
-        public Boolean getRenameFileAfterUpload();
+        public boolean getRenameFileAfterUpload();
+
+        // if `false`, plugin will use remote file as buffer
+        @Config("local_buffering")
+        @ConfigDefault("true")
+        public boolean getLocalBuffering();
+
+        @Min(50L * 1024 * 1024) // 50MiB
+        @Max(10L * 1024 * 1024 * 1024) // 10GiB
+        @Config("temp_file_threshold")
+        @ConfigDefault("5368709120") // 5GiB
+        public long getTempFileThreshold();
     }
 
     @Override
@@ -141,6 +155,9 @@ public class SftpFileOutputPlugin
     public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new SftpFileOutput(task, taskIndex);
+        if (task.getLocalBuffering()) {
+            return new SftpLocalFileOutput(task, taskIndex);
+        }
+        return new SftpRemoteFileOutput(task, taskIndex);
     }
 }

data/src/main/java/org/embulk/output/sftp/SftpLocalFileOutput.java

@@ -0,0 +1,223 @@
+package org.embulk.output.sftp;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.vfs2.FileObject;
+import org.embulk.config.TaskReport;
+import org.embulk.output.sftp.utils.TimeoutCloser;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.TransactionalFileOutput;
+import org.slf4j.Logger;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
+
+/**
+ * Created by takahiro.nakayama on 10/20/15.
+ */
+public class SftpLocalFileOutput
+        implements FileOutput, TransactionalFileOutput
+{
+    // to make it clear that it is a constant
+    static final String TMP_SUFFIX = ".tmp";
+
+    final Logger logger = Exec.getLogger(getClass());
+    private final String pathPrefix;
+    private final String sequenceFormat;
+    private final String fileNameExtension;
+    private boolean renameFileAfterUpload;
+
+    private final int taskIndex;
+    final SftpUtils sftpUtils;
+    int fileIndex = 0;
+    private File tempFile;
+    private BufferedOutputStream localOutput = null;
+    List<Map<String, String>> fileList = new ArrayList<>();
+    String curFilename;
+    String tempFilename;
+
+    /* for file splitting purpose */
+    private final long threshold; // local file size to flush (upload to server)
+    boolean appending = false; // when local file exceeds threshold, go to append mode
+    FileObject remoteFile;
+    BufferedOutputStream remoteOutput; // to keep output stream open during append mode
+    long bufLen = 0L; // local temp file size
+
+    SftpLocalFileOutput(PluginTask task, int taskIndex)
+    {
+        this.pathPrefix = task.getPathPrefix();
+        this.sequenceFormat = task.getSequenceFormat();
+        this.fileNameExtension = task.getFileNameExtension();
+        this.renameFileAfterUpload = task.getRenameFileAfterUpload();
+        this.taskIndex = taskIndex;
+        this.sftpUtils = new SftpUtils(task);
+        this.threshold = task.getTempFileThreshold();
+    }
+
+    @Override
+    public void nextFile()
+    {
+        closeCurrentFile();
+
+        try {
+            tempFile = Exec.getTempFileSpace().createTempFile();
+            localOutput = new BufferedOutputStream(new FileOutputStream(tempFile));
+            appending = false;
+            curFilename = getOutputFilePath();
+            tempFilename = curFilename + TMP_SUFFIX;
+        }
+        catch (FileNotFoundException e) {
+            logger.error(e.getMessage());
+            throw Throwables.propagate(e);
+        }
+    }
+
+    @Override
+    public void add(final Buffer buffer)
+    {
+        try {
+            final int len = buffer.limit();
+            if (bufLen + len > threshold) {
+                localOutput.close();
+                // into 'append' mode
+                appending = true;
+                flush();
+
+                // reset output stream (overwrite local temp file)
+                localOutput = new BufferedOutputStream(new FileOutputStream(tempFile));
+                bufLen = 0L;
+            }
+            localOutput.write(buffer.array(), buffer.offset(), len);
+            bufLen += len;
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+        finally {
+            buffer.release();
+        }
+    }
+
+    @Override
+    public void finish()
+    {
+        closeCurrentFile();
+        try {
+            flush();
+        }
+        catch (IOException e) {
+            throw Throwables.propagate(e);
+        }
+        closeRemoteFile();
+        // if input config is not `renameFileAfterUpload`
+        // and file is being split, we have to rename it here
+        // otherwise, when it exits, it won't rename
+        if (!renameFileAfterUpload && appending) {
+            sftpUtils.renameFile(tempFilename, curFilename);
+        }
+        fileList.add(fileReport());
+        fileIndex++;
+    }
+
+    @Override
+    public void close()
+    {
+        closeCurrentFile();
+        // TODO
+        sftpUtils.close();
+    }
+
+    @Override
+    public void abort()
+    {
+        // delete incomplete files
+        if (renameFileAfterUpload) {
+            sftpUtils.deleteFile(tempFilename);
+        }
+        else {
+            sftpUtils.deleteFile(curFilename);
+        }
+    }
+
+    @Override
+    public TaskReport commit()
+    {
+        TaskReport report = Exec.newTaskReport();
+        report.set("file_list", fileList);
+        return report;
+    }
+
+    void closeCurrentFile()
+    {
+        try {
+            if (localOutput != null) {
+                localOutput.close();
+                localOutput = null;
+            }
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    void closeRemoteFile()
+    {
+        if (remoteOutput != null) {
+            new TimeoutCloser(remoteOutput).close();
+            remoteOutput = null;
+            remoteFile = null;
+        }
+    }
+
+    String getOutputFilePath()
+    {
+        return pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + fileNameExtension;
+    }
+
+    Map<String, String> fileReport()
+    {
+        return ImmutableMap.of(
+                "temporary_filename", tempFilename,
+                "real_filename", curFilename
+        );
+    }
+
+    private void flush() throws IOException
+    {
+        if (appending) {
+            // open and keep stream open
+            if (remoteOutput == null) {
+                remoteFile = sftpUtils.resolve(tempFilename);
+                remoteOutput = sftpUtils.openStream(remoteFile);
+            }
+            sftpUtils.appendFile(tempFile, remoteFile, remoteOutput);
+        }
+        else {
+            sftpUtils.uploadFile(tempFile, renameFileAfterUpload ? tempFilename : curFilename);
+        }
+    }
+
+    @VisibleForTesting
+    OutputStream getLocalOutput()
+    {
+        return localOutput;
+    }
+
+    @VisibleForTesting
+    OutputStream getRemoteOutput()
+    {
+        return remoteOutput;
+    }
+}

data/src/main/java/org/embulk/output/sftp/SftpRemoteFileOutput.java

@@ -0,0 +1,120 @@
+package org.embulk.output.sftp;
+
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.AbstractScheduledService;
+import com.google.common.util.concurrent.Service;
+import org.apache.commons.vfs2.FileSystemException;
+import org.embulk.output.sftp.utils.TimedCallable;
+import org.embulk.spi.Buffer;
+
+import java.io.BufferedOutputStream;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
+
+public class SftpRemoteFileOutput extends SftpLocalFileOutput
+{
+    private static final int TIMEOUT = 60; // 1min
+    private Service watcher;
+
+    SftpRemoteFileOutput(PluginTask task, int taskIndex)
+    {
+        super(task, taskIndex);
+        appending = true;
+    }
+
+    @Override
+    public void add(final Buffer buffer)
+    {
+        try {
+            final int len = buffer.limit();
+            // time-out write
+            new TimedCallable<Void>()
+            {
+                @Override
+                public Void call() throws Exception
+                {
+                    remoteOutput.write(buffer.array(), buffer.offset(), len);
+                    return null;
+                }
+            }.call(TIMEOUT, TimeUnit.SECONDS);
+            bufLen += len;
+        }
+        catch (InterruptedException | ExecutionException | TimeoutException ex) {
+            logger.error("Failed to write buffer", ex);
+            stopWatcher();
+            throw Throwables.propagate(ex);
+        }
+        finally {
+            buffer.release();
+        }
+    }
+
+    @Override
+    void closeCurrentFile()
+    {
+        super.closeCurrentFile();
+        stopWatcher();
+    }
+
+    void stopWatcher()
+    {
+        if (watcher != null) {
+            watcher.stopAsync();
+        }
+    }
+
+    @Override
+    public void nextFile()
+    {
+        closeCurrentFile();
+
+        try {
+            curFilename = getOutputFilePath();
+            tempFilename = curFilename + TMP_SUFFIX;
+            // resolve remote file & open output stream
+            remoteFile = sftpUtils.newSftpFile(sftpUtils.getSftpFileUri(tempFilename));
+            // this is where it's different from |SftpLocalFileOutput|
+            remoteOutput = new BufferedOutputStream(remoteFile.getContent().getOutputStream());
+            watcher = newProgressWatcher().startAsync();
+        }
+        catch (FileSystemException e) {
+            stopWatcher();
+            throw Throwables.propagate(e);
+        }
+    }
+
+    @Override
+    public void finish()
+    {
+        closeCurrentFile();
+        closeRemoteFile();
+        fileList.add(fileReport());
+        fileIndex++;
+        stopWatcher();
+    }
+
+    private Service newProgressWatcher()
+    {
+        return new AbstractScheduledService()
+        {
+            private static final int PERIOD = 10; // seconds
+            private long prevLen = 0L;
+
+            @Override
+            protected void runOneIteration()
+            {
+                logger.info("Upload progress: {} KB - {} KB/s", bufLen / 1024, (bufLen - prevLen) / 1024 / PERIOD);
+                prevLen = bufLen;
+            }
+
+            @Override
+            protected Scheduler scheduler()
+            {
+                return Scheduler.newFixedRateSchedule(PERIOD, PERIOD, TimeUnit.SECONDS);
+            }
+        };
+    }
+}