embulk-output-ftp 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fd1fb5019e704b0db01af96b695ee892fb042c52
-  data.tar.gz: 963e27d5c2604becb7aa4fed07876939f106b0a9
+  metadata.gz: db87448f5b289f68ebb6c9d03b36f295a0139ac0
+  data.tar.gz: ab5abab07a3bec49eed5ec362c8fdec549564d88
 SHA512:
-  metadata.gz: ae8ca510952cfe8a2517101ee844a895465e7f2588dccbb795b17cb7ec5ba8f6a350156e99cc0399a9faf54a372bf0db47f3ab96ced3f8eb519323998ff724ae
-  data.tar.gz: 79be6833288b0c5ae9da3319430c966d43c38fa42762f70c6e8d5de3ffc24a14b0712b60e5fbac842e5e715814f06c9518fc402d035392b5ed6df243dca17291
+  metadata.gz: 7281539145640359c6c084c031b7a212efd3044755e57ea5b823f5a4b80635facf0c0f300e59fdf1ced7db7425104a0fdd0f737de7274674a0fba93987b4d8b5
+  data.tar.gz: a2d49e39e67f7c5c680e62651db9ccf6e845bc42e317d81a834d52e9c83a3630003b10667f304fc4351e637f74bb9304b985510d688d5b39db86d837b89eb4de

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.1 - 2016-07-20
+
+* [maintenance] Add unit test, refactoring, change logging logic
+
 ## 0.1.0 - 2016-07-20
 
 * [new feature] First Release

data/README.md

@@ -37,7 +37,7 @@ out:
   path_prefix: /ftp/file/path/prefix
   ext: csv
   formatter:
-    type: csv
+    type: csv.gz
     header_line: false
   encoders:
   - {type: gzip}
@@ -87,7 +87,7 @@ out:
       -----END CERTIFICATE-----
 
   path_prefix: /ftp/file/path/prefix
-    ext: csv
+  ext: csv
 ```
 
 
@@ -96,3 +96,57 @@ out:
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
+
+## Test
+
+```
+$ ./gradlew test  # -t to watch change of files and rebuild continuously
+```
+
+To run unit tests, we need to configure the following environment variables.
+
+When environment variables are not set, skip some test cases.
+
+```
+FTP_TEST_HOST
+FTP_TEST_USER
+FTP_TEST_PASSWORD
+FTP_TEST_SSL_TRUSTED_CA_CERT_FILE
+FTP_TEST_SSL_TRUSTED_CA_CERT_DATA
+```
+
+If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
+```xml
+$ vi ~/Library/LaunchAgents/environment.plist
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>my.startup</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>sh</string>
+    <string>-c</string>
+    <string>
+      launchctl setenv FTP_TEST_HOST ftp.example.com
+      launchctl setenv FTP_TEST_USER username
+      launchctl setenv FTP_TEST_PASSWORD password
+      launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_FILE /path/to/cert.pem
+      launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_DATA /path/to/cert.pem
+    </string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+</dict>
+</plist>
+
+$ launchctl load ~/Library/LaunchAgents/environment.plist
+$ launchctl getenv FTP_TEST_HOST //try to get value.
+
+Then start your applications.
+```
+
+## Acknowledgement
+
+This program is forked from [embulk-input-ftp](https://github.com/embulk/embulk-input-ftp) and originally written by @frsyuki, modified by @sakama.

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -25,6 +25,8 @@ dependencies {
     compile files("libs/ftp4j-1.7.2.jar")
     compile "org.bouncycastle:bcpkix-jdk15on:1.52"
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.9:tests"
+    testCompile "org.embulk:embulk-standards:0.8.9"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/config/checkstyle/checkstyle.xml

@@ -92,7 +92,9 @@
         <module name="TypeName"/>
         <module name="PackageName"/>
         <module name="ParameterName"/>
-        <module name="StaticVariableName"/>
+        <module name="StaticVariableName">
+            <property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
+        </module>
         <module name="ClassTypeParameterName">
             <property name="format" value="^[A-Z][0-9]?$"/>
         </module>

data/config/checkstyle/default.xml

@@ -82,9 +82,8 @@
         <module name="TypeName"/>
         <module name="PackageName"/>
         <module name="ParameterName"/>
-        <module name="StaticVariableName"/>
-        <module name="ClassTypeParameterName">
-            <property name="format" value="^[A-Z][0-9]?$"/>
+        <module name="StaticVariableName">
+            <property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
         </module>
         <module name="MethodTypeParameterName">
             <property name="format" value="^[A-Z][0-9]?$"/>

data/src/main/java/org/embulk/output/ftp/FtpFileOutputPlugin.java

@@ -103,7 +103,7 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
             client = newFTPClient(log, task);
         }
         catch (Exception ex) {
-            throw new ConfigException(ex);
+            throw new ConfigException("Faild to connect to FTP server", ex);
         }
         finally {
             disconnectClient(client);
@@ -168,8 +168,8 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
                     suffix = "." + suffix;
                 }
                 filePath = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + suffix;
-                file = File.createTempFile(filePath, ".tmp");
-                log.info("Writing local file {}", file.getAbsolutePath());
+                file = Exec.getTempFileSpace().createTempFile(filePath, ".tmp");
+                log.info("Writing local temporary file \"{}\"", file.getAbsolutePath());
                 output = new BufferedOutputStream(new FileOutputStream(file));
             }
             catch (IOException ex) {
@@ -225,13 +225,14 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
                                 public Void call() throws FTPIllegalReplyException, FTPException, FTPDataTransferException,
                                                           FTPAbortedException, IOException, RetryGiveupException
                                 {
-                                    log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
-                                    client.upload(filePath, new BufferedInputStream(new FileInputStream(file)), 0L, 0L, new LoggingTransferListener(log, TRANSFER_NOTICE_BYTES));
-                                    log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
+                                    client.upload(filePath,
+                                        new BufferedInputStream(new FileInputStream(file)), 0L, 0L,
+                                        new LoggingTransferListener(file.getAbsolutePath(), filePath, log, TRANSFER_NOTICE_BYTES)
+                                    );
                                     if (!file.delete()) {
                                         throw new ConfigException("Couldn't delete local file " + file.getAbsolutePath());
                                     }
-                                    log.info("Delete local temporary file {}", file.getAbsolutePath());
+                                    log.info("Deleted local temporary file \"{}\"", file.getAbsolutePath());
                                     return null;
                                 }
 
@@ -396,7 +397,7 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
 
         public void received(String statement)
         {
-            log.info("< " + statement);
+            log.debug("< " + statement);
         }
 
         public void sent(String statement)
@@ -405,20 +406,24 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
                 // don't show password
                 return;
             }
-            log.info("> {}", statement);
+            log.debug("> {}", statement);
         }
     }
 
     private static class LoggingTransferListener implements FTPDataTransferListener
     {
+        private final String localPath;
+        private final String remotePath;
         private final Logger log;
         private final long transferNoticeBytes;
 
         private long totalTransfer;
         private long nextTransferNotice;
 
-        public LoggingTransferListener(Logger log, long transferNoticeBytes)
+        public LoggingTransferListener(String localPath, String remotePath, Logger log, long transferNoticeBytes)
         {
+            this.localPath = localPath;
+            this.remotePath = remotePath;
             this.log = log;
             this.transferNoticeBytes = transferNoticeBytes;
             this.nextTransferNotice = transferNoticeBytes;
@@ -426,7 +431,7 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
 
         public void started()
         {
-            log.info("Transfer started");
+            log.info("Transfer started. local path:\"{}\" remote path:\"{}\"", localPath, remotePath);
         }
 
         public void transferred(int length)
@@ -440,7 +445,7 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
 
         public void completed()
         {
-            log.info("Transfer completed {} bytes", totalTransfer);
+            log.info("Transfer completed. remote path:\"{}\", size:{} bytes", remotePath, totalTransfer);
         }
 
         public void aborted()

data/src/main/java/org/embulk/output/ftp/SSLPlugins.java

@@ -63,7 +63,7 @@ public class SSLPlugins
 
     public static class SSLPluginConfig
     {
-        static SSLPluginConfig noVerify = new SSLPluginConfig(VerifyMode.NO_VERIFY, false, ImmutableList.<byte[]>of());
+        static SSLPluginConfig NO_VERIFY = new SSLPluginConfig(VerifyMode.NO_VERIFY, false, ImmutableList.<byte[]>of());
 
         private final VerifyMode verifyMode;
         private final boolean verifyHostname;
@@ -176,7 +176,7 @@ public class SSLPlugins
             }
         }
         else {
-            return SSLPluginConfig.noVerify;
+            return SSLPluginConfig.NO_VERIFY;
         }
     }
 

data/src/test/java/org/embulk/output/ftp/TestFtpFileOutputPlugin.java

@@ -1,5 +1,375 @@
 package org.embulk.output.ftp;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.io.Resources;
+import it.sauronsoftware.ftp4j.FTPClient;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.output.ftp.FtpFileOutputPlugin.PluginTask;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.FileOutputRunner;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.standards.CsvParserPlugin;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Method;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.List;
+
 public class TestFtpFileOutputPlugin
 {
+    private static String FTP_TEST_HOST;
+    private static String FTP_TEST_USER;
+    private static String FTP_TEST_PASSWORD;
+    private static String FTP_TEST_SSL_TRUSTED_CA_CERT_FILE;
+    private static String FTP_TEST_DIRECTORY;
+    private static String FTP_TEST_PATH_PREFIX;
+    private static String LOCAL_PATH_PREFIX;
+    private FileOutputRunner runner;
+
+    /*
+     * This test case requires environment variables
+     *   FTP_TEST_HOST
+     *   FTP_TEST_USER
+     *   FTP_TEST_PASSWORD
+     *   FTP_TEST_SSL_TRUSTED_CA_CERT_FILE
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        FTP_TEST_HOST = System.getenv("FTP_TEST_HOST");
+        FTP_TEST_USER = System.getenv("FTP_TEST_USER");
+        FTP_TEST_PASSWORD = System.getenv("FTP_TEST_PASSWORD");
+        FTP_TEST_SSL_TRUSTED_CA_CERT_FILE = System.getenv("FTP_TEST_SSL_TRUSTED_CA_CERT_FILE");
+        // skip test cases, if environment variables are not set.
+        assumeNotNull(FTP_TEST_HOST, FTP_TEST_USER, FTP_TEST_PASSWORD, FTP_TEST_SSL_TRUSTED_CA_CERT_FILE);
+
+        FTP_TEST_DIRECTORY = System.getenv("FTP_TEST_DIRECTORY") != null ? getDirectory(System.getenv("FTP_TEST_DIRECTORY")) : getDirectory("/unittest/");
+        FTP_TEST_PATH_PREFIX = FTP_TEST_DIRECTORY + "sample_";
+        LOCAL_PATH_PREFIX = Resources.getResource("sample_01.csv").getPath();
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private FtpFileOutputPlugin plugin;
+
+    @Before
+    public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
+    {
+        plugin = new FtpFileOutputPlugin();
+        runner = new FileOutputRunner(runtime.getInstance(FtpFileOutputPlugin.class));
+    }
+
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "ftp")
+                .set("host", FTP_TEST_HOST)
+                .set("user", FTP_TEST_USER)
+                .set("password", FTP_TEST_PASSWORD)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        assertEquals(FTP_TEST_HOST, task.getHost());
+        assertEquals(FTP_TEST_USER, task.getUser().get());
+        assertEquals(FTP_TEST_PASSWORD, task.getPassword().get());
+        assertEquals(10, task.getMaxConnectionRetry());
+    }
+
+    @Test
+    public void testTransaction()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "ftp")
+                .set("host", FTP_TEST_HOST)
+                .set("user", FTP_TEST_USER)
+                .set("password", FTP_TEST_PASSWORD)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testTransactionWithInvalidHost()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "ftp")
+                .set("host", "non-exists.example.com")
+                .set("user", FTP_TEST_USER)
+                .set("password", FTP_TEST_PASSWORD)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
+//    @Test
+//    public void testTransactionWithSsl()
+//    {
+//        ConfigSource config = Exec.newConfigSource()
+//                .set("in", inputConfig())
+//                .set("parser", parserConfig(schemaConfig()))
+//                .set("type", "ftp")
+//                .set("host", FTP_TEST_HOST)
+//                .set("port", 990)
+//                .set("user", FTP_TEST_USER)
+//                .set("password", FTP_TEST_PASSWORD)
+//                .set("ssl", true)
+//                .set("ssl_verify", false)
+//                .set("ssl_verify_hostname", false)
+//                .set("ssl_trusted_ca_cert_file", FTP_TEST_SSL_TRUSTED_CA_CERT_FILE)
+//                .set("path_prefix", "my-prefix")
+//                .set("file_ext", ".csv")
+//                .set("formatter", formatterConfig());
+//
+//        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+//
+//        runner.transaction(config, schema, 0, new Control());
+//    }
+
+    @Test
+    public void testResume()
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileOutputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource)
+            {
+                return Lists.newArrayList(Exec.newTaskReport());
+            }
+        });
+        //assertEquals("in/aa/a", configDiff.get(String.class, "last_path"));
+    }
+
+    @Test
+    public void testCleanup()
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
+    }
+
+    @Test
+    public void testFtpFileOutputByOpen() throws Exception
+    {
+        ConfigSource configSource = config();
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        task.setSSLConfig(SSLPlugins.configure(task));
+        Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        runner.transaction(configSource, schema, 0, new Control());
+
+        TransactionalFileOutput output = plugin.open(task.dump(), 0);
+
+        output.nextFile();
+
+        FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
+        byte[] bytes = convertInputStreamToByte(is);
+        Buffer buffer = Buffer.wrap(bytes);
+        output.add(buffer);
+
+        output.finish();
+        output.commit();
+
+        String remotePath = FTP_TEST_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
+        assertRecords(remotePath, task);
+    }
+
+    public ConfigSource config()
+    {
+        return Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "ftp")
+                .set("host", FTP_TEST_HOST)
+                .set("user", FTP_TEST_USER)
+                .set("password", FTP_TEST_PASSWORD)
+                .set("path_prefix", FTP_TEST_PATH_PREFIX)
+                .set("last_path", "")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+    }
+
+    private class Control implements OutputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource)
+        {
+            return Lists.newArrayList(Exec.newTaskReport());
+        }
+    }
+
+    private ImmutableMap<String, Object> inputConfig()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "file");
+        builder.put("path_prefix", LOCAL_PATH_PREFIX);
+        builder.put("last_path", "");
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("newline", "CRLF");
+        builder.put("delimiter", ",");
+        builder.put("quote", "\"");
+        builder.put("escape", "\"");
+        builder.put("trim_if_not_quoted", false);
+        builder.put("skip_header_lines", 1);
+        builder.put("allow_extra_columns", false);
+        builder.put("allow_optional_columns", false);
+        builder.put("columns", schemaConfig);
+        return builder.build();
+    }
+
+    private ImmutableList<Object> schemaConfig()
+    {
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "id", "type", "long"));
+        builder.add(ImmutableMap.of("name", "account", "type", "long"));
+        builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+        builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
+        builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+        builder.add(ImmutableMap.of("name", "json_column", "type", "json"));
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> formatterConfig()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("header_line", "false");
+        builder.put("timezone", "Asia/Tokyo");
+        return builder.build();
+    }
+
+    private void assertRecords(String remotePath, PluginTask task) throws Exception
+    {
+        ImmutableList<List<String>> records = getFileContentsFromFtp(remotePath, task);
+        assertEquals(6, records.size());
+        {
+            List<String> record = records.get(1);
+            assertEquals("1", record.get(0));
+            assertEquals("32864", record.get(1));
+            assertEquals("2015-01-27 19:23:49", record.get(2));
+            assertEquals("20150127", record.get(3));
+            assertEquals("embulk", record.get(4));
+            assertEquals("{\"k\":true}", record.get(5));
+        }
+
+        {
+            List<String> record = records.get(2);
+            assertEquals("2", record.get(0));
+            assertEquals("14824", record.get(1));
+            assertEquals("2015-01-27 19:01:23", record.get(2));
+            assertEquals("20150127", record.get(3));
+            assertEquals("embulk jruby", record.get(4));
+            assertEquals("{\"k\":1}", record.get(5));
+        }
+
+        {
+            List<String> record = records.get(3);
+            assertEquals("{\"k\":1.23}", record.get(5));
+        }
+
+        {
+            List<String> record = records.get(4);
+            assertEquals("{\"k\":\"v\"}", record.get(5));
+        }
+
+        {
+            List<String> record = records.get(5);
+            assertEquals("{\"k\":\"2015-02-03 08:13:45\"}", record.get(5));
+        }
+    }
+
+    private ImmutableList<List<String>> getFileContentsFromFtp(String path, PluginTask task) throws Exception
+    {
+        Method method = FtpFileOutputPlugin.class.getDeclaredMethod("newFTPClient", Logger.class, PluginTask.class);
+        method.setAccessible(true);
+        FTPClient client = (FTPClient) method.invoke(plugin, Exec.getLogger(FtpFileOutputPlugin.class), task);
+
+        ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
+        File localFile = Exec.getTempFileSpace().createTempFile();
+        client.download(path, localFile);
+        InputStream is = new BufferedInputStream(new FileInputStream(localFile));
+        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+        String line;
+        while ((line = reader.readLine()) != null) {
+            List<String> records = Arrays.asList(line.split(",", 0));
+
+            builder.add(records);
+        }
+        return builder.build();
+    }
+
+    private static String getDirectory(String dir)
+    {
+        if (!dir.isEmpty() && !dir.endsWith("/")) {
+            dir = dir + "/";
+        }
+        if (dir.startsWith("/")) {
+            dir = dir.replaceFirst("/", "");
+        }
+        return dir;
+    }
+
+    private byte[] convertInputStreamToByte(InputStream is) throws IOException
+    {
+        ByteArrayOutputStream bo = new ByteArrayOutputStream();
+        byte [] buffer = new byte[1024];
+        while (true) {
+            int len = is.read(buffer);
+            if (len < 0) {
+                break;
+            }
+            bo.write(buffer, 0, len);
+        }
+        return bo.toByteArray();
+    }
 }

data/src/test/resources/sample_01.csv

@@ -0,0 +1,6 @@
+id,account,time,purchase,comment,json_column
+1,32864,2015-01-27 19:23:49,20150127,embulk,{"k":true}
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby,{"k":1}
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",{"k":1.23}
+4,11270,2015-01-29 11:54:36,20150129,NULL,{"k":"v"}
+5,53231,2015-01-30 13:48:12,20150130,NULL,{"k":"2015-02-03 08:13:45"}

data/src/test/resources/sample_02.csv

@@ -0,0 +1,6 @@
+id,account,time,purchase,comment,json_column
+1,32864,2015-01-27 19:23:49,20150127,embulk,{"k":true}
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby,{"k":1}
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",{"k":1.23}
+4,11270,2015-01-29 11:54:36,20150129,NULL,{"k":"v"}
+5,53231,2015-01-30 13:48:12,20150130,NULL,{"k":"2015-02-03 08:13:45"}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-ftp
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Satoshi Akama
@@ -59,17 +59,17 @@ files:
 - gradlew.bat
 - lib/embulk/output/ftp.rb
 - libs/ftp4j-1.7.2.jar
-- src/main/java/org/embulk/output/ftp/BlockingTransfer.java
 - src/main/java/org/embulk/output/ftp/FtpFileOutputPlugin.java
 - src/main/java/org/embulk/output/ftp/SSLPlugins.java
 - src/main/java/org/embulk/output/ftp/TrustManagers.java
-- src/test/java/org/embulk/output/ftp/TestBlockingTransfer.java
 - src/test/java/org/embulk/output/ftp/TestFtpFileOutputPlugin.java
 - src/test/java/org/embulk/output/ftp/TestSSLPlugins.java
 - src/test/java/org/embulk/output/ftp/TrustedManagers.java
+- src/test/resources/sample_01.csv
+- src/test/resources/sample_02.csv
 - classpath/bcpkix-jdk15on-1.52.jar
 - classpath/bcprov-jdk15on-1.52.jar
-- classpath/embulk-output-ftp-0.1.0.jar
+- classpath/embulk-output-ftp-0.1.1.jar
 - classpath/ftp4j-1.7.2.jar
 homepage: https://github.com/sakama/embulk-output-ftp
 licenses:

data/src/main/java/org/embulk/output/ftp/BlockingTransfer.java

@@ -1,277 +0,0 @@
-package org.embulk.output.ftp;
-
-import com.google.common.base.Function;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-public class BlockingTransfer
-{
-    private final WriterChannel writerChannel;
-    private final ReaderChannel readerChannel;
-    private Future<?> transferCompletionFuture;
-
-    public static BlockingTransfer submit(ExecutorService executor,
-            Function<BlockingTransfer, Runnable> starterFactory)
-    {
-        BlockingTransfer transfer = new BlockingTransfer();
-        final Runnable starter = starterFactory.apply(transfer);
-        transfer.setTransferCompletionFuture(
-                executor.submit(new Callable<Void>() {
-                    public Void call() throws Exception
-                    {
-                        starter.run();
-                        return null;
-                    }
-                })
-            );
-        return transfer;
-    }
-
-    private BlockingTransfer()
-    {
-        this.writerChannel = new WriterChannel();
-        this.readerChannel = new ReaderChannel();
-    }
-
-    private void setTransferCompletionFuture(Future<?> future)
-    {
-        this.transferCompletionFuture = future;
-    }
-
-    public ReadableByteChannel getReaderChannel()
-    {
-        return readerChannel;
-    }
-
-    public WritableByteChannel getWriterChannel()
-    {
-        return writerChannel;
-    }
-
-    public void transferFailed(Throwable exception)
-    {
-        readerChannel.overwriteException(exception);
-    }
-
-    void waitForTransferCompletion() throws IOException
-    {
-        Future<?> f = transferCompletionFuture;
-        if (f != null) {
-            try {
-                f.get();
-            }
-            catch (CancellationException | InterruptedException ex) {
-                throw new InterruptedIOException();
-            }
-            catch (ExecutionException ex) {
-                // transfer failed
-                Throwable e = ex.getCause();
-                if (e instanceof IOException) {
-                    throw (IOException) e;
-                }
-                else if (e instanceof RuntimeException) {
-                    throw (RuntimeException) e;
-                }
-                else if (e instanceof Error) {
-                    throw (Error) e;
-                }
-                else {
-                    throw new IOException(e);
-                }
-            }
-        }
-    }
-
-    public class WriterChannel implements WritableByteChannel
-    {
-        public int write(ByteBuffer src) throws IOException
-        {
-            int sz = src.remaining();
-            if (sz <= 0) {
-                return sz;
-            }
-
-            synchronized (readerChannel) {
-                if (!readerChannel.waitForWritable()) {
-                    return -1;
-                }
-
-                readerChannel.setBuffer(src);
-
-                if (!readerChannel.waitForWritable()) {  // wait for complete processing src
-                    return -1;
-                }
-            }
-
-            return sz - src.remaining();
-        }
-
-        public boolean isOpen()
-        {
-            return readerChannel.isOpen();
-        }
-
-        public void close() throws IOException
-        {
-            readerChannel.closePeer();
-            waitForTransferCompletion();
-        }
-    }
-
-    private static int transferByteBuffer(ByteBuffer src, ByteBuffer dst)
-    {
-        int pos = dst.position();
-
-        int srcrem = src.remaining();
-        int dstrem = dst.remaining();
-        if (dstrem < srcrem) {
-            int lim = src.limit();
-            try {
-                src.limit(src.position() + dstrem);
-                dst.put(src);
-            }
-            finally {
-                src.limit(lim);
-            }
-        }
-        else {
-            dst.put(src);
-        }
-
-        return dst.position() - pos;
-    }
-
-    public class ReaderChannel implements ReadableByteChannel
-    {
-        private ByteBuffer buffer;
-        private Throwable exception;
-
-        public synchronized int read(ByteBuffer dst) throws IOException
-        {
-            if (!waitForReadable()) {
-                return -1;
-            }
-
-            int len = transferByteBuffer(buffer, dst);
-            if (!buffer.hasRemaining()) {
-                setBuffer(null);
-                notifyAll();
-            }
-
-            return len;
-        }
-
-        public synchronized boolean isOpen()
-        {
-            return exception == null;
-        }
-
-        public void close() throws IOException
-        {
-            setException(new EOFException("reader closed channel"));
-        }
-
-        private void setBuffer(ByteBuffer buffer)
-        {
-            this.buffer = buffer;
-            notifyAll();
-        }
-
-        private synchronized boolean waitForWritable() throws IOException
-        {
-            while (buffer != null) {
-                if (exception != null) {
-                    if (exception instanceof EOFException) {
-                        return false;
-                    }
-                    throwException();
-                }
-
-                try {
-                    wait();
-                }
-                catch (InterruptedException ex) {
-                    // TODO throws ClosedByInterruptException or InterruptedIOException?
-                }
-            }
-
-            return true;
-        }
-
-        private boolean waitForReadable() throws IOException
-        {
-            while (buffer == null) {
-                if (exception != null) {
-                    if (exception instanceof EOFException) {
-                        return false;
-                    }
-                    throwException();
-                }
-
-                try {
-                    wait();
-                }
-                catch (InterruptedException ex) {
-                    // TODO throws ClosedByInterruptException or InterruptedIOException?
-                }
-            }
-
-            return true;
-        }
-
-        public synchronized void closePeer() throws IOException
-        {
-            waitForWritable();
-            if (exception != null && !(exception instanceof EOFException)) {
-                throwException();
-            }
-            setException(new EOFException("writer closed channel"));
-        }
-
-        public synchronized void setException(Throwable exception)
-        {
-            if (this.exception == null) {
-                this.exception = exception;
-            }
-            notifyAll();
-        }
-
-        public synchronized void overwriteException(Throwable exception)
-        {
-            this.exception = exception;
-            notifyAll();
-        }
-
-        public boolean hasException()
-        {
-            return exception != null;
-        }
-
-        public void throwException() throws IOException
-        {
-            Throwable ex = exception;
-            if (ex instanceof IOException) {
-                throw (IOException) ex;
-            }
-            else if (ex instanceof RuntimeException) {
-                throw (RuntimeException) ex;
-            }
-            else if (ex instanceof Error) {
-                throw (Error) ex;
-            }
-            else {
-                throw new IOException(ex);
-            }
-        }
-    }
-}

data/src/test/java/org/embulk/output/ftp/TestBlockingTransfer.java

@@ -1,5 +0,0 @@
-package org.embulk.output.ftp;
-
-public class TestBlockingTransfer
-{
-}