embulk-output-sftp 0.1.10 → 0.1.11

data/src/test/java/org/embulk/output/sftp/utils/TestTimedCallable.java

@@ -0,0 +1,36 @@
+package org.embulk.output.sftp.utils;
+
+import org.embulk.EmbulkTestRuntime;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class TestTimedCallable
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void testCallTimeout()
+    {
+        try {
+            new TimedCallable<Void>()
+            {
+                @Override
+                public Void call() throws Exception
+                {
+                    Thread.sleep(200);
+                    return null;
+                }
+            }.call(100, TimeUnit.MILLISECONDS);
+        }
+        catch (Exception e) {
+            assertThat(e, instanceOf(TimeoutException.class));
+        }
+    }
+}

data/src/test/java/org/embulk/output/sftp/utils/TestTimeoutCloser.java

@@ -0,0 +1,45 @@
+package org.embulk.output.sftp.utils;
+
+import org.embulk.EmbulkTestRuntime;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.Closeable;
+import java.util.concurrent.TimeoutException;
+
+import static junit.framework.TestCase.fail;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class TestTimeoutCloser
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void testTimeoutAndAbort()
+    {
+        TimeoutCloser closer = new TimeoutCloser(new Closeable()
+        {
+            @Override
+            public void close()
+            {
+                try {
+                    Thread.sleep(5000);
+                }
+                catch (InterruptedException e) {
+                    fail("Failed to sleep");
+                }
+            }
+        });
+        closer.timeout = 1;
+        try {
+            closer.close();
+            fail("Should not finish");
+        }
+        catch (Exception e) {
+            assertThat(e, instanceOf(RuntimeException.class));
+            assertThat(e.getCause(), instanceOf(TimeoutException.class));
+        }
+    }
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sftp
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - Civitaspo
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-07 00:00:00.000000000 Z
+date: 2018-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,21 +64,27 @@ files:
 - gradlew.bat
 - lib/embulk/output/sftp.rb
 - src/main/java/org/embulk/output/sftp/ProxyTask.java
-- src/main/java/org/embulk/output/sftp/SftpFileOutput.java
 - src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
+- src/main/java/org/embulk/output/sftp/SftpLocalFileOutput.java
+- src/main/java/org/embulk/output/sftp/SftpRemoteFileOutput.java
 - src/main/java/org/embulk/output/sftp/SftpUtils.java
 - src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileObject.java
 - src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileProvider.java
 - src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileSystem.java
 - src/main/java/org/embulk/output/sftp/provider/sftp/package.html
+- src/main/java/org/embulk/output/sftp/utils/DefaultRetry.java
+- src/main/java/org/embulk/output/sftp/utils/TimedCallable.java
+- src/main/java/org/embulk/output/sftp/utils/TimeoutCloser.java
 - src/main/resources/providers.xml
 - src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
+- src/test/java/org/embulk/output/sftp/utils/TestTimedCallable.java
+- src/test/java/org/embulk/output/sftp/utils/TestTimeoutCloser.java
 - src/test/resources/id_rsa
 - src/test/resources/id_rsa.pub
-- classpath/commons-io-2.6.jar
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.2.jar
-- classpath/embulk-output-sftp-0.1.10.jar
+- classpath/embulk-output-sftp-0.1.11.jar
+- classpath/commons-io-2.6.jar
 - classpath/jsch-0.1.54.jar
 homepage: https://github.com/embulk/embulk-output-sftp
 licenses:

data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java

@@ -1,140 +0,0 @@
-package org.embulk.output.sftp;
-
-import com.google.common.base.Throwables;
-import org.embulk.config.TaskReport;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileOutput;
-import org.embulk.spi.TransactionalFileOutput;
-import org.slf4j.Logger;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
-
-/**
- * Created by takahiro.nakayama on 10/20/15.
- */
-public class SftpFileOutput
-    implements FileOutput, TransactionalFileOutput
-{
-    private final Logger logger = Exec.getLogger(SftpFileOutput.class);
-    private final String pathPrefix;
-    private final String sequenceFormat;
-    private final String fileNameExtension;
-    private final boolean renameFileAfterUpload;
-
-    private final int taskIndex;
-    private final SftpUtils sftpUtils;
-    private int fileIndex = 0;
-    private File tempFile;
-    private BufferedOutputStream localOutput = null;
-    private List<Map<String, String>> fileList = new ArrayList<>();
-
-    private final String temporaryFileSuffix = ".tmp";
-
-    SftpFileOutput(PluginTask task, int taskIndex)
-    {
-        this.pathPrefix = task.getPathPrefix();
-        this.sequenceFormat = task.getSequenceFormat();
-        this.fileNameExtension = task.getFileNameExtension();
-        this.renameFileAfterUpload = task.getRenameFileAfterUpload();
-        this.taskIndex = taskIndex;
-        this.sftpUtils = new SftpUtils(task);
-    }
-
-    @Override
-    public void nextFile()
-    {
-        closeCurrentFile();
-
-        try {
-            tempFile = Exec.getTempFileSpace().createTempFile();
-            localOutput = new BufferedOutputStream(new FileOutputStream(tempFile));
-        }
-        catch (FileNotFoundException e) {
-            logger.error(e.getMessage());
-            Throwables.propagate(e);
-        }
-    }
-
-    @Override
-    public void add(final Buffer buffer)
-    {
-        try {
-            localOutput.write(buffer.array(), buffer.offset(), buffer.limit());
-        }
-        catch (IOException ex) {
-            throw Throwables.propagate(ex);
-        }
-        finally {
-            buffer.release();
-        }
-    }
-
-    @Override
-    public void finish()
-    {
-        closeCurrentFile();
-        String fileName = getOutputFilePath();
-        String temporaryFileName = fileName + temporaryFileSuffix;
-        if (renameFileAfterUpload) {
-            sftpUtils.uploadFile(tempFile, temporaryFileName);
-        }
-        else {
-            sftpUtils.uploadFile(tempFile, fileName);
-        }
-
-        Map<String, String> executedFiles = new HashMap<>();
-        executedFiles.put("temporary_filename", temporaryFileName);
-        executedFiles.put("real_filename", fileName);
-        fileList.add(executedFiles);
-        fileIndex++;
-    }
-
-    @Override
-    public void close()
-    {
-        closeCurrentFile();
-        // TODO
-        sftpUtils.close();
-    }
-
-    @Override
-    public void abort()
-    {
-    }
-
-    @Override
-    public TaskReport commit()
-    {
-        TaskReport report = Exec.newTaskReport();
-        report.set("file_list", fileList);
-        return report;
-    }
-
-    private void closeCurrentFile()
-    {
-        if (localOutput != null) {
-            try {
-                localOutput.close();
-            }
-            catch (IOException ex) {
-                throw Throwables.propagate(ex);
-            }
-        }
-    }
-
-    private String getOutputFilePath()
-    {
-        return pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + fileNameExtension;
-    }
-}