embulk-output-azure_blob_storage 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e7f5671df4655218d40ef364f9475e05f3cdaa4
-  data.tar.gz: 5ba40cd551788bdde07a5a08ea05998a3ee7f0cb
+  metadata.gz: 5cc1a4b0540a0db2bb640ffff57daef127b63eec
+  data.tar.gz: 09115e5d83c014bac1b3d8d19b12fb8adad3a4be
 SHA512:
-  metadata.gz: c3330eefd6eab6363a399a09905c374c0b8c427fa367f44d545d2b7eba77c91e4480b7d76b360fa06307603532de9da227e90b5ca3380aa650087165de4c5149
-  data.tar.gz: 76c557136ad97309d1175433879ad7ae92ff2d5b826b72b772c724b624130685c2e53a748b1b079f6c3d914f90dda9ef86311a9536d2543f60cd69d546d10576
+  metadata.gz: 6b167e589e571d77ed3b33552505090aa58bdc1c4470fd36fc66bbc90b8dca8129a0b2f031b1832640a2b0154855fd207a2100477f26ef34fec7ac6a8c4ee818
+  data.tar.gz: 9aedb87b4cdb6750ded5ef47108fd9574392f3a244ac11c051c287595c04195abd54ccbccb0c84f92a961e0aa48f8f015a6bfd94110252865dc437238e2515f5

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+## 0.1.3 - 2015-03-15
+
+* [maintenance] Add retry logic using expotential backoff [#6](https://github.com/sakama/embulk-output-azure_blob_storage/pull/6)
+* [maintenance] Change local file output path to temporary directory
+[#5](https://github.com/sakama/embulk-output-azure_blob_storage/pull/5)
+
+## 0.1.2 - 2015-02-02
+
+* [maintenance] Upgrade embulk version to v0.8.2 [#3](https://github.com/sakama/embulk-output-azure_blob_storage/pull/3)
+
+## 0.1.1 - 2015-11-16
+
+* [maintenance] Added unit tests [#2](https://github.com/sakama/embulk-output-azure_blob_storage/pull/2)
+* [maintenance] Refactored code [#1](https://github.com/sakama/embulk-output-azure_blob_storage/pull/1)

data/README.md

@@ -70,7 +70,7 @@ AZURE_CONTAINER_DIRECTORY (optional, if needed)
 ```
 
 If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
-```
+```xml
 $ vi ~/Library/LaunchAgents/environment.plist
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
@@ -98,4 +98,4 @@ $ launchctl load ~/Library/LaunchAgents/environment.plist
 $ launchctl getenv AZURE_ACCOUNT_NAME //try to get value.
 
 Then start your applications.
-```
+```

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.2"
+version = "0.1.3"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java

@@ -56,6 +56,10 @@ public class AzureBlobStorageFileOutputPlugin
         @Config("sequence_format")
         @ConfigDefault("\"%03d.%02d\"")
         String getSequenceFormat();
+
+        @Config("max_connection_retry")
+        @ConfigDefault("5") // 5 times retry to connect sftp server if failed.
+        int getMaxConnectionRetry();
     }
 
     private static final Logger log = Exec.getLogger(AzureBlobStorageFileOutputPlugin.class);
@@ -71,7 +75,7 @@ public class AzureBlobStorageFileOutputPlugin
             String containerName = task.getContainer();
             CloudBlobContainer container = blobClient.getContainerReference(containerName);
             if (!container.exists()) {
-                log.info(String.format("container [%s] is not exists and created.", containerName));
+                log.info("container {} doesn't exists and created.", containerName);
                 container.createIfNotExists();
             }
         }
@@ -124,6 +128,7 @@ public class AzureBlobStorageFileOutputPlugin
         private final String sequenceFormat;
         private final String pathSuffix;
         private final CloudBlobClient client;
+        private final int maxConnectionRetry;
         private CloudBlobContainer container = null;
         private BufferedOutputStream output = null;
         private int fileIndex;
@@ -138,6 +143,7 @@ public class AzureBlobStorageFileOutputPlugin
             this.sequenceFormat = task.getSequenceFormat();
             this.pathSuffix = task.getFileNameExtension();
             this.client = newAzureClient(task.getAccountName(), task.getAccountKey());
+            this.maxConnectionRetry = task.getMaxConnectionRetry();
             try {
                 this.container = client.getContainerReference(task.getContainer());
             }
@@ -157,17 +163,11 @@ public class AzureBlobStorageFileOutputPlugin
                     suffix = "." + suffix;
                 }
                 filePath = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + suffix;
-                file = new File(filePath);
-
-                String parentPath = file.getParent();
-                File dir = new File(parentPath);
-                if (!dir.exists()) {
-                    dir.mkdir();
-                }
-                log.info(String.format("Writing local file [%s]", filePath));
-                output = new BufferedOutputStream(new FileOutputStream(filePath));
+                file = File.createTempFile(filePath, ".tmp");
+                log.info("Writing local file {}", file.getAbsolutePath());
+                output = new BufferedOutputStream(new FileOutputStream(file));
             }
-            catch (FileNotFoundException ex) {
+            catch (IOException ex) {
                 throw Throwables.propagate(ex);
             }
         }
@@ -202,18 +202,47 @@ public class AzureBlobStorageFileOutputPlugin
         @Override
         public void finish()
         {
-            closeFile();
+            close();
+            try {
+                Thread.sleep(1000 * 10);
+            }
+            catch (Exception ex) {
+                // null;
+            }
             if (filePath != null) {
-                try {
-                    CloudBlockBlob blob = container.getBlockBlobReference(filePath);
-                    log.info(String.format("Upload start [%s]", filePath));
-                    blob.upload(new FileInputStream(file), file.length());
-                    log.info(String.format("Upload completed [%s]", filePath));
-                    file.delete();
-                    log.info(String.format("Delete completed local file [%s]", filePath));
-                }
-                catch (StorageException | URISyntaxException | IOException ex) {
-                    Throwables.propagate(ex);
+                int count = 0;
+                while (true) {
+                    try {
+                        CloudBlockBlob blob = container.getBlockBlobReference(filePath);
+                        log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
+                        blob.upload(new FileInputStream(file), file.length());
+                        log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
+                        log.info("Delete completed local file {}", file.getAbsolutePath());
+                        if (!file.delete()) {
+                            throw new ConfigException("Couldn't delete local file " + file.getAbsolutePath());
+                        }
+                        break;
+                    }
+                    catch (FileNotFoundException | URISyntaxException ex) {
+                        throw new ConfigException(ex);
+                    }
+                    catch (StorageException | IOException ex) {
+                        if (++count == maxConnectionRetry) {
+                            Throwables.propagate(ex);
+                        }
+                        log.warn("failed to connect SFTP server: " + ex.getMessage(), ex);
+
+                        try {
+                            long sleepTime = ((long) Math.pow(2, count) * 1000);
+                            log.warn("sleep in next connection retry: {} milliseconds", sleepTime);
+                            Thread.sleep(sleepTime); // milliseconds
+                        }
+                        catch (InterruptedException ex2) {
+                            // Ignore this exception because this exception is just about `sleep`.
+                            log.warn(ex2.getMessage(), ex2);
+                        }
+                        log.warn("retrying to connect SFTP server: " + count + " times");
+                    }
                 }
             }
         }

data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java

@@ -3,11 +3,13 @@ package org.embulk.output.azure_blob_storage;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import com.google.common.io.Resources;
 import com.microsoft.azure.storage.blob.CloudBlob;
 import com.microsoft.azure.storage.blob.CloudBlobClient;
 import com.microsoft.azure.storage.blob.CloudBlobContainer;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
@@ -30,10 +32,12 @@ import static org.junit.Assume.assumeNotNull;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.security.GeneralSecurityException;
 import java.util.Arrays;
@@ -67,7 +71,7 @@ public class TestAzureBlobStorageFileOutputPlugin
 
         AZURE_CONTAINER_DIRECTORY = System.getenv("AZURE_CONTAINER_DIRECTORY") != null ? getDirectory(System.getenv("AZURE_CONTAINER_DIRECTORY")) : getDirectory("");
         AZURE_PATH_PREFIX = AZURE_CONTAINER_DIRECTORY + "sample_";
-        LOCAL_PATH_PREFIX = AzureBlobStorageFileOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
+        LOCAL_PATH_PREFIX = Resources.getResource("sample_01.csv").getPath();
     }
 
     @Rule
@@ -96,7 +100,7 @@ public class TestAzureBlobStorageFileOutputPlugin
                 .set("formatter", formatterConfig());
 
         PluginTask task = config.loadConfig(PluginTask.class);
-        assertEquals(AZURE_ACCOUNT_NAME, task.getAccountName().toString());
+        assertEquals(AZURE_ACCOUNT_NAME, task.getAccountName());
     }
 
     @Test
@@ -210,6 +214,64 @@ public class TestAzureBlobStorageFileOutputPlugin
         assertRecords(remotePath);
     }
 
+    @Test(expected = ConfigException.class)
+    public void testAzureFileOutputByOpenWithNonReadableFile() throws Exception
+    {
+        ConfigSource configSource = config();
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        runner.transaction(configSource, schema, 0, new Control());
+
+        TransactionalFileOutput output = plugin.open(task.dump(), 0);
+
+        output.nextFile();
+
+        FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
+        byte[] bytes = convertInputStreamToByte(is);
+        Buffer buffer = Buffer.wrap(bytes);
+        output.add(buffer);
+
+        Field field = AzureBlobStorageFileOutputPlugin.AzureFileOutput.class.getDeclaredField("file");
+        field.setAccessible(true);
+        File file = File.createTempFile("non-readable-file", ".tmp");
+        file.setReadable(false);
+        field.set(output, file);
+        output.finish();
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testAzureFileOutputByOpenWithRetry() throws Exception
+    {
+        ConfigSource configSource = config();
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        runner.transaction(configSource, schema, 0, new Control());
+
+        TransactionalFileOutput output = plugin.open(task.dump(), 0);
+
+        output.nextFile();
+
+        FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
+        byte[] bytes = convertInputStreamToByte(is);
+        Buffer buffer = Buffer.wrap(bytes);
+        output.add(buffer);
+
+        // set maxConnectionRetry = 1 for Test
+        Field maxConnectionRetry = AzureBlobStorageFileOutputPlugin.AzureFileOutput.class.getDeclaredField("maxConnectionRetry");
+        maxConnectionRetry.setAccessible(true);
+        maxConnectionRetry.set(output, 1);
+
+        // set non-existing-container for Test
+        Method method = AzureBlobStorageFileOutputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
+        method.setAccessible(true);
+        CloudBlobClient client = (CloudBlobClient) method.invoke(plugin, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY);
+
+        Field container = AzureBlobStorageFileOutputPlugin.AzureFileOutput.class.getDeclaredField("container");
+        container.setAccessible(true);
+        container.set(output, client.getContainerReference("non-exists-container"));
+        output.finish();
+    }
+
     public ConfigSource config()
     {
         return Exec.newConfigSource()
@@ -268,6 +330,7 @@ public class TestAzureBlobStorageFileOutputPlugin
         builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
         builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
         builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+        builder.add(ImmutableMap.of("name", "json_column", "type", "json"));
         return builder.build();
     }
 
@@ -283,7 +346,7 @@ public class TestAzureBlobStorageFileOutputPlugin
     private void assertRecords(String azurePath) throws Exception
     {
         ImmutableList<List<String>> records = getFileContentsFromAzure(azurePath);
-        assertEquals(5, records.size());
+        assertEquals(6, records.size());
         {
             List<String> record = records.get(1);
             assertEquals("1", record.get(0));
@@ -291,6 +354,7 @@ public class TestAzureBlobStorageFileOutputPlugin
             assertEquals("2015-01-27 19:23:49", record.get(2));
             assertEquals("20150127", record.get(3));
             assertEquals("embulk", record.get(4));
+            assertEquals("{\"k\":true}", record.get(5));
         }
 
         {
@@ -300,6 +364,22 @@ public class TestAzureBlobStorageFileOutputPlugin
             assertEquals("2015-01-27 19:01:23", record.get(2));
             assertEquals("20150127", record.get(3));
             assertEquals("embulk jruby", record.get(4));
+            assertEquals("{\"k\":1}", record.get(5));
+        }
+
+        {
+            List<String> record = records.get(3);
+            assertEquals("{\"k\":1.23}", record.get(5));
+        }
+
+        {
+            List<String> record = records.get(4);
+            assertEquals("{\"k\":\"v\"}", record.get(5));
+        }
+
+        {
+            List<String> record = records.get(5);
+            assertEquals("{\"k\":\"2015-02-03 08:13:45\"}", record.get(5));
         }
     }
 
@@ -330,11 +410,7 @@ public class TestAzureBlobStorageFileOutputPlugin
         method.setAccessible(true);
         CloudBlobClient client = (CloudBlobClient) method.invoke(plugin, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY);
         CloudBlobContainer container = client.getContainerReference(containerName);
-        if (container.exists()) {
-            return true;
-        }
-
-        return false;
+        return container.exists();
     }
 
     private void deleteContainerIfExists(String containerName) throws Exception
@@ -352,7 +428,7 @@ public class TestAzureBlobStorageFileOutputPlugin
 
     private static String getDirectory(String dir)
     {
-        if (dir != null && !dir.endsWith("/")) {
+        if (!dir.isEmpty() && !dir.endsWith("/")) {
             dir = dir + "/";
         }
         if (dir.startsWith("/")) {

data/src/test/resources/sample_01.csv

@@ -1,5 +1,6 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL
+id,account,time,purchase,comment,json_column
+1,32864,2015-01-27 19:23:49,20150127,embulk,{"k":true}
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby,{"k":1}
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",{"k":1.23}
+4,11270,2015-01-29 11:54:36,20150129,NULL,{"k":"v"}
+5,53231,2015-01-30 13:48:12,20150130,NULL,{"k":"2015-02-03 08:13:45"}

data/src/test/resources/sample_02.csv

@@ -1,5 +1,6 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL
+id,account,time,purchase,comment,json_column
+1,32864,2015-01-27 19:23:49,20150127,embulk,{"k":true}
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby,{"k":1}
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",{"k":1.23}
+4,11270,2015-01-29 11:54:36,20150129,NULL,{"k":"v"}
+5,53231,2015-01-30 13:48:12,20150130,NULL,{"k":"2015-02-03 08:13:45"}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-02 00:00:00.000000000 Z
+date: 2016-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -47,7 +47,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .travis.yml
-- ChangeLog
+- CHANGELOG.md
 - README.md
 - build.gradle
 - config/checkstyle/checkstyle.xml
@@ -63,7 +63,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/azure-storage-4.0.0.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-output-azure_blob_storage-0.1.2.jar
+- classpath/embulk-output-azure_blob_storage-0.1.3.jar
 - classpath/jackson-core-2.6.0.jar
 homepage: https://github.com/sakama/embulk-output-azure_blob_storage
 licenses:

data/ChangeLog

@@ -1,4 +0,0 @@
-Release 0.1.1 - 2015-11-16
-
-* Added unit tests
-* Refactored code