embulk-output-sftp 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64ff7608df9289a70721a9c4bf7685a66929e197
-  data.tar.gz: 14fbf8a0b136c44e999f1aa15b333c19f32eca24
+  metadata.gz: bec4dd06576bca7c200d5abf77f0e195c28d998d
+  data.tar.gz: 8ded1287e06e19b4adb8696b4bf6550a4a1ee47e
 SHA512:
-  metadata.gz: e58d44d3f650e31e3a5a88405299b66148e0ca8507330e4ebe093978f275504b48d6969d773394c251ed1653971d051077b5a38119f81b0540d5504514678141
-  data.tar.gz: f2735c0c06d09cdaf6539c96133ffe06b001e3ec4d2a1f3e0f1f690b3898136bfd76f8aae09f0401a5cc4b9545b6112abc6ae2377739c3f55aff26b8688626af
+  metadata.gz: 87a454091a32f4f69c52eabd50a092638bf944edd763d857d2a357c7d11a1375d8bcd92204adeae5cda278549634b6ccfd8b8f7c4ccadd17a8aa89ca4c0c28c6
+  data.tar.gz: 06a421bcea79c0c438f108c20a49a400bac18a1ec914d11d1b4bc5dff182a09401cfeeb31471c70aa861a76baa5374ce2d21d0a8007b2f25aba7910f899391b9

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.1.7 (2018-03-14)
+- Enhance: Upload with ".tmp" suffix and rename file name after upload 
+  - https://github.com/embulk/embulk-output-sftp/pull/43
+  - https://github.com/embulk/embulk-output-sftp/pull/44
+
 0.1.6 (2018-03-13)
 - Fix: Change input stream buffer size to 4MB to fix hanging issue
   - https://github.com/embulk/embulk-output-sftp/pull/46

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 }
 
 group = "org.embulk.output.sftp"
-version = "0.1.6"
+version = "0.1.7"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java

@@ -30,6 +30,7 @@ public class SftpFileOutput
     private final String pathPrefix;
     private final String sequenceFormat;
     private final String fileNameExtension;
+    private final boolean renameFileAfterUpload;
 
     private final int taskIndex;
     private final SftpUtils sftpUtils;
@@ -45,6 +46,7 @@ public class SftpFileOutput
         this.pathPrefix = task.getPathPrefix();
         this.sequenceFormat = task.getSequenceFormat();
         this.fileNameExtension = task.getFileNameExtension();
+        this.renameFileAfterUpload = task.getRenameFileAfterUpload();
         this.taskIndex = taskIndex;
         this.sftpUtils = new SftpUtils(task);
     }
@@ -84,12 +86,12 @@ public class SftpFileOutput
         closeCurrentFile();
         String fileName = getOutputFilePath();
         String temporaryFileName = fileName + temporaryFileSuffix;
-        /*
-          #37 causes permission failure while renaming remote file.
-          https://github.com/embulk/embulk-output-sftp/issues/40
-         */
-        //sftpUtils.uploadFile(tempFile, temporaryFileName);
-        sftpUtils.uploadFile(tempFile, fileName);
+        if (renameFileAfterUpload) {
+            sftpUtils.uploadFile(tempFile, temporaryFileName);
+        }
+        else {
+            sftpUtils.uploadFile(tempFile, fileName);
+        }
 
         Map<String, String> executedFiles = new HashMap<>();
         executedFiles.put("temporary_filename", temporaryFileName);

data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java

@@ -14,6 +14,7 @@ import org.embulk.spi.TransactionalFileOutput;
 import org.embulk.spi.unit.LocalFile;
 
 import java.util.List;
+import java.util.Map;
 
 public class SftpFileOutputPlugin
         implements FileOutputPlugin
@@ -69,6 +70,10 @@ public class SftpFileOutputPlugin
         @Config("proxy")
         @ConfigDefault("null")
         public Optional<ProxyTask> getProxy();
+
+        @Config("rename_file_after_upload")
+        @ConfigDefault("false")
+        public Boolean getRenameFileAfterUpload();
     }
 
     @Override
@@ -98,21 +103,28 @@ public class SftpFileOutputPlugin
             int taskCount,
             List<TaskReport> successTaskReports)
     {
-        /*
-          #37 causes permission failure while renaming remote file.
-          https://github.com/embulk/embulk-output-sftp/issues/40
-         */
-//        SftpUtils sftpUtils = new SftpUtils(taskSource.loadTask(PluginTask.class));
-//        for (TaskReport report : successTaskReports) {
-//            List<Map<String, String>> moveFileList = report.get(List.class, "file_list");
-//            for (Map<String, String> pairFiles : moveFileList) {
-//                String temporaryFileName = pairFiles.get("temporary_filename");
-//                String realFileName = pairFiles.get("real_filename");
-//
-//                sftpUtils.renameFile(temporaryFileName, realFileName);
-//            }
-//        }
-//        sftpUtils.close();
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        if (task.getRenameFileAfterUpload()) {
+            SftpUtils sftpUtils = null;
+            try {
+                sftpUtils = new SftpUtils(task);
+                for (TaskReport report : successTaskReports) {
+                    List<Map<String, String>> moveFileList = report.get(List.class, "file_list");
+                    for (Map<String, String> pairFiles : moveFileList) {
+                        String temporaryFileName = pairFiles.get("temporary_filename");
+                        String realFileName = pairFiles.get("real_filename");
+
+                        sftpUtils.renameFile(temporaryFileName, realFileName);
+                    }
+                }
+            }
+            finally {
+                if (sftpUtils != null) {
+                    sftpUtils.close();
+                }
+            }
+        }
     }
 
     @Override

data/src/main/java/org/embulk/output/sftp/SftpUtils.java

@@ -5,7 +5,7 @@ import com.google.common.base.Throwables;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.commons.vfs2.FileSystemOptions;
-import org.apache.commons.vfs2.impl.StandardFileSystemManager;
+import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
 import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
 import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
 import org.embulk.config.ConfigException;
@@ -32,22 +32,29 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 public class SftpUtils
 {
     private final Logger logger = Exec.getLogger(SftpUtils.class);
-    private final StandardFileSystemManager manager;
+    private final DefaultFileSystemManager manager;
     private final FileSystemOptions fsOptions;
     private final String userInfo;
     private final String host;
     private final int port;
     private final int maxConnectionRetry;
 
-    private StandardFileSystemManager initializeStandardFileSystemManager()
+    private DefaultFileSystemManager initializeStandardFileSystemManager()
     {
         if (!logger.isDebugEnabled()) {
             // TODO: change logging format: org.apache.commons.logging.Log
             System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
         }
-        StandardFileSystemManager manager = new StandardFileSystemManager();
-        manager.setClassLoader(SftpUtils.class.getClassLoader());
+        /*
+         * We can use StandardFileSystemManager instead of DefaultFileSystemManager
+         * when Apache Commons VFS2 removes permission check logic when remote file renaming.
+         * https://github.com/embulk/embulk-output-sftp/issues/40
+         * https://github.com/embulk/embulk-output-sftp/pull/44
+         * https://issues.apache.org/jira/browse/VFS-590
+        */
+        DefaultFileSystemManager manager = new DefaultFileSystemManager();
         try {
+            manager.addProvider("sftp", new org.embulk.output.sftp.provider.sftp.SftpFileProvider());
             manager.init();
         }
         catch (FileSystemException e) {

data/src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileObject.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.output.sftp.provider.sftp;
+
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.provider.AbstractFileName;
+import org.apache.commons.vfs2.provider.sftp.SftpFileSystem;
+
+/*
+ * We can remove this file when Apache Commons VFS2 removes permission check logic when remote file renaming.
+ * https://github.com/embulk/embulk-output-sftp/issues/40
+ * https://github.com/embulk/embulk-output-sftp/pull/44
+ * https://issues.apache.org/jira/browse/VFS-590
+ */
+/**
+ * An SFTP file.
+ */
+public class SftpFileObject extends org.apache.commons.vfs2.provider.sftp.SftpFileObject
+{
+    protected SftpFileObject(final AbstractFileName name, final SftpFileSystem fileSystem) throws FileSystemException
+    {
+        super(name, fileSystem);
+    }
+
+    @Override
+    protected boolean doIsWriteable() throws Exception
+    {
+        return true;
+    }
+}

data/src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileProvider.java

@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.output.sftp.provider.sftp;
+
+import com.jcraft.jsch.Session;
+import org.apache.commons.vfs2.FileName;
+import org.apache.commons.vfs2.FileSystem;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.UserAuthenticationData;
+import org.apache.commons.vfs2.provider.GenericFileName;
+import org.apache.commons.vfs2.provider.sftp.SftpClientFactory;
+import org.apache.commons.vfs2.util.UserAuthenticatorUtils;
+
+/*
+ * We can remove this file when Apache Commons VFS2 removes permission check logic when remote file renaming.
+ * https://github.com/embulk/embulk-output-sftp/issues/40
+ * https://github.com/embulk/embulk-output-sftp/pull/44
+ * https://issues.apache.org/jira/browse/VFS-590
+ */
+/**
+ * A provider for accessing files over SFTP.
+ */
+public class SftpFileProvider extends org.apache.commons.vfs2.provider.sftp.SftpFileProvider
+{
+    /**
+     * Constructs a new provider.
+     */
+    public SftpFileProvider()
+    {
+        super();
+    }
+
+    /**
+     * Creates a {@link FileSystem}.
+     */
+    @Override
+    protected FileSystem doCreateFileSystem(final FileName name, final FileSystemOptions fileSystemOptions)
+            throws FileSystemException
+    {
+        // JSch jsch = createJSch(fileSystemOptions);
+
+        // Create the file system
+        final GenericFileName rootName = (GenericFileName) name;
+
+        Session session;
+        UserAuthenticationData authData = null;
+        try {
+            authData = UserAuthenticatorUtils.authenticate(fileSystemOptions, AUTHENTICATOR_TYPES);
+
+            session = SftpClientFactory.createConnection(rootName.getHostName(), rootName.getPort(),
+                    UserAuthenticatorUtils.getData(authData, UserAuthenticationData.USERNAME,
+                            UserAuthenticatorUtils.toChar(rootName.getUserName())),
+                    UserAuthenticatorUtils.getData(authData, UserAuthenticationData.PASSWORD,
+                            UserAuthenticatorUtils.toChar(rootName.getPassword())),
+                    fileSystemOptions);
+        }
+        catch (final Exception e) {
+            throw new FileSystemException("vfs.provider.sftp/connect.error", name, e);
+        }
+        finally {
+            UserAuthenticatorUtils.cleanup(authData);
+        }
+
+        return new SftpFileSystem(rootName, session, fileSystemOptions);
+    }
+}

data/src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileSystem.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.output.sftp.provider.sftp;
+
+import com.jcraft.jsch.Session;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.commons.vfs2.provider.AbstractFileName;
+import org.apache.commons.vfs2.provider.GenericFileName;
+
+/*
+ * We can remove this file when Apache Commons VFS2 removes permission check logic when remote file renaming.
+ * https://github.com/embulk/embulk-output-sftp/issues/40
+ * https://github.com/embulk/embulk-output-sftp/pull/44
+ * https://issues.apache.org/jira/browse/VFS-590
+ */
+/**
+ * Represents the files on an SFTP server.
+ */
+public class SftpFileSystem extends org.apache.commons.vfs2.provider.sftp.SftpFileSystem
+{
+    protected SftpFileSystem(final GenericFileName rootName, final Session session,
+                             final FileSystemOptions fileSystemOptions)
+    {
+        super(rootName, null, fileSystemOptions);
+    }
+
+    /**
+     * Creates a file object. This method is called only if the requested file is not cached.
+     */
+    @Override
+    protected FileObject createFile(final AbstractFileName name) throws FileSystemException
+    {
+        return new SftpFileObject(name, this);
+    }
+}

data/src/main/java/org/embulk/output/sftp/provider/sftp/package.html

@@ -0,0 +1,19 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+   
+         http://www.apache.org/licenses/LICENSE-2.0
+   
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<body>
+<p>The SFTP Provider.</p>
+</body>

data/src/main/resources/providers.xml

@@ -0,0 +1,29 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<!--
+    We can remove this file when Apache Commons VFS2 removes permission check logic when remote file renaming.
+    https://github.com/embulk/embulk-output-sftp/issues/40
+    https://github.com/embulk/embulk-output-sftp/pull/44
+    https://issues.apache.org/jira/browse/VFS-590
+-->
+<providers>
+    <provider class-name="org.embulk.output.sftp.provider.sftp.SftpFileProvider">
+        <scheme name="sftp"/>
+        <if-available class-name="javax.crypto.Cipher"/>
+        <if-available class-name="com.jcraft.jsch.JSch"/>
+    </provider>
+</providers>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sftp
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Civitaspo
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-13 00:00:00.000000000 Z
+date: 2018-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,13 +67,18 @@ files:
 - src/main/java/org/embulk/output/sftp/SftpFileOutput.java
 - src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
 - src/main/java/org/embulk/output/sftp/SftpUtils.java
+- src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileObject.java
+- src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileProvider.java
+- src/main/java/org/embulk/output/sftp/provider/sftp/SftpFileSystem.java
+- src/main/java/org/embulk/output/sftp/provider/sftp/package.html
+- src/main/resources/providers.xml
 - src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
 - src/test/resources/id_rsa
 - src/test/resources/id_rsa.pub
 - classpath/commons-io-2.6.jar
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.2.jar
-- classpath/embulk-output-sftp-0.1.6.jar
+- classpath/embulk-output-sftp-0.1.7.jar
 - classpath/jsch-0.1.54.jar
 homepage: https://github.com/embulk/embulk-output-sftp
 licenses: