embulk-output-azure_blob_storage 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5cc1a4b0540a0db2bb640ffff57daef127b63eec
-  data.tar.gz: 09115e5d83c014bac1b3d8d19b12fb8adad3a4be
+  metadata.gz: 6f0517ecae2588f48dd8266de574626f89534a7c
+  data.tar.gz: 2487729f05d378c29cf76b81a8a921d1710df3c6
 SHA512:
-  metadata.gz: 6b167e589e571d77ed3b33552505090aa58bdc1c4470fd36fc66bbc90b8dca8129a0b2f031b1832640a2b0154855fd207a2100477f26ef34fec7ac6a8c4ee818
-  data.tar.gz: 9aedb87b4cdb6750ded5ef47108fd9574392f3a244ac11c051c287595c04195abd54ccbccb0c84f92a961e0aa48f8f015a6bfd94110252865dc437238e2515f5
+  metadata.gz: 0bcd72b765c0d90581a38ae5b6b095c3eaa18e3adf5819469f3da9eb61d899c01401b68be87e01c4d1ec0940d200bbd5a602e56ad05b94e1ce7b0d72594d6907
+  data.tar.gz: 6677c17408925c38d2ac0896686fae03fcfdcb89a7777f1fb4f33d26de9b1d9321c6462fde2bda05a7e7c51e1088c35f2ade0c1b4ef5004cb8773ede29aefde1

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.4 - 2015-03-20
+
+* [maintenance] Use RetryExecutor when retry is needed [#7](https://github.com/sakama/embulk-output-azure_blob_storage/pull/7)
+
 ## 0.1.3 - 2015-03-15
 
 * [maintenance] Add retry logic using expotential backoff [#6](https://github.com/sakama/embulk-output-azure_blob_storage/pull/6)

data/README.md

@@ -1,6 +1,6 @@
-# Azure blob Storage file output plugin for Embulk
+# Azure Blob Storage file output plugin for Embulk
 
-[Embulk](http://www.embulk.org/) file output plugin stores files on [Microsoft Azure](https://azure.microsoft.com/) [blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
+[Embulk](http://www.embulk.org/) file output plugin stores files on [Microsoft Azure](https://azure.microsoft.com/) [Blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
 
 
 ## Overview
@@ -22,10 +22,10 @@ First, create Azure [Storage Account](https://azure.microsoft.com/en-us/document
 
 ### Auto create container
 
-container will create automatically when container isn't exists.
+container will create automatically when container doesn't exists.
  
-When a container is deleted, a container with the same name cannot be created for at least 30 seconds.
-It's a [service specification](https://technet.microsoft.com/en-us/library/dd179408.aspx#Anchor_3) of Azure blob Storage.
+When a container was deleted, a container with same name cannot be created for at least 30 seconds.
+It's a [service specification](https://technet.microsoft.com/en-us/library/dd179408.aspx#Anchor_3) of Azure Blob Storage.
 
 ## Example
 

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.3"
+version = "0.1.4"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"
@@ -65,8 +65,8 @@ Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
   spec.authors       = ["Satoshi Akama"]
-  spec.summary       = %[Microsoft Azure blob Storage file output plugin for Embulk]
-  spec.description   = %[Stores files on Microsoft Azure blob Storage.]
+  spec.summary       = %[Microsoft Azure Blob Storage file output plugin for Embulk]
+  spec.description   = %[Stores files on Microsoft Azure Blob Storage.]
   spec.email         = ["satoshiakama@gmail.com"]
   spec.licenses      = ["Apache-2.0"]
   spec.homepage      = "https://github.com/sakama/embulk-output-azure_blob_storage"

data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java

@@ -18,8 +18,10 @@ import org.embulk.spi.Buffer;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
-
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.BufferedOutputStream;
 import java.io.File;
@@ -58,7 +60,7 @@ public class AzureBlobStorageFileOutputPlugin
         String getSequenceFormat();
 
         @Config("max_connection_retry")
-        @ConfigDefault("5") // 5 times retry to connect sftp server if failed.
+        @ConfigDefault("10") // 5 times retry to connect sftp server if failed.
         int getMaxConnectionRetry();
     }
 
@@ -203,48 +205,71 @@ public class AzureBlobStorageFileOutputPlugin
         public void finish()
         {
             close();
-            try {
-                Thread.sleep(1000 * 10);
-            }
-            catch (Exception ex) {
-                // null;
-            }
+            uploadFile();
+        }
+
+        private Void uploadFile()
+        {
             if (filePath != null) {
-                int count = 0;
-                while (true) {
-                    try {
-                        CloudBlockBlob blob = container.getBlockBlobReference(filePath);
-                        log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
-                        blob.upload(new FileInputStream(file), file.length());
-                        log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
-                        log.info("Delete completed local file {}", file.getAbsolutePath());
-                        if (!file.delete()) {
-                            throw new ConfigException("Couldn't delete local file " + file.getAbsolutePath());
-                        }
-                        break;
-                    }
-                    catch (FileNotFoundException | URISyntaxException ex) {
-                        throw new ConfigException(ex);
-                    }
-                    catch (StorageException | IOException ex) {
-                        if (++count == maxConnectionRetry) {
-                            Throwables.propagate(ex);
-                        }
-                        log.warn("failed to connect SFTP server: " + ex.getMessage(), ex);
-
-                        try {
-                            long sleepTime = ((long) Math.pow(2, count) * 1000);
-                            log.warn("sleep in next connection retry: {} milliseconds", sleepTime);
-                            Thread.sleep(sleepTime); // milliseconds
-                        }
-                        catch (InterruptedException ex2) {
-                            // Ignore this exception because this exception is just about `sleep`.
-                            log.warn(ex2.getMessage(), ex2);
-                        }
-                        log.warn("retrying to connect SFTP server: " + count + " times");
-                    }
+                try {
+                    return retryExecutor()
+                            .withRetryLimit(maxConnectionRetry)
+                            .withInitialRetryWait(500)
+                            .withMaxRetryWait(30 * 1000)
+                            .runInterruptible(new Retryable<Void>() {
+                                @Override
+                                public Void call() throws StorageException, URISyntaxException, IOException, RetryGiveupException
+                                {
+                                    CloudBlockBlob blob = container.getBlockBlobReference(filePath);
+                                    log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
+                                    blob.upload(new FileInputStream(file), file.length());
+                                    log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
+                                    log.info("Delete completed local file {}", file.getAbsolutePath());
+                                    if (!file.delete()) {
+                                        throw new ConfigException("Couldn't delete local file " + file.getAbsolutePath());
+                                    }
+                                    return null;
+                                }
+
+                                @Override
+                                public boolean isRetryableException(Exception exception)
+                                {
+                                    return true;
+                                }
+
+                                @Override
+                                public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                        throws RetryGiveupException
+                                {
+                                    Class clazz = exception.getClass();
+                                    if (clazz.equals(FileNotFoundException.class) || clazz.equals(URISyntaxException.class) || clazz.equals(ConfigException.class)) {
+                                        throw new RetryGiveupException(exception);
+                                    }
+                                    String message = String.format("Azure Blob Storage put request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                            retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                                    if (retryCount % 3 == 0) {
+                                        log.warn(message, exception);
+                                    }
+                                    else {
+                                        log.warn(message);
+                                    }
+                                }
+
+                                @Override
+                                public void onGiveup(Exception firstException, Exception lastException)
+                                        throws RetryGiveupException
+                                {
+                                }
+                            });
+                }
+                catch (RetryGiveupException ex) {
+                    throw Throwables.propagate(ex.getCause());
+                }
+                catch (InterruptedException ex) {
+                    throw Throwables.propagate(ex);
                 }
             }
+            return null;
         }
 
         @Override

data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java

@@ -9,7 +9,6 @@ import com.microsoft.azure.storage.blob.CloudBlobClient;
 import com.microsoft.azure.storage.blob.CloudBlobContainer;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
@@ -34,6 +33,7 @@ import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -100,7 +100,11 @@ public class TestAzureBlobStorageFileOutputPlugin
                 .set("formatter", formatterConfig());
 
         PluginTask task = config.loadConfig(PluginTask.class);
+
         assertEquals(AZURE_ACCOUNT_NAME, task.getAccountName());
+        assertEquals(AZURE_ACCOUNT_KEY, task.getAccountKey());
+        assertEquals(AZURE_CONTAINER, task.getContainer());
+        assertEquals(10, task.getMaxConnectionRetry());
     }
 
     @Test
@@ -214,7 +218,7 @@ public class TestAzureBlobStorageFileOutputPlugin
         assertRecords(remotePath);
     }
 
-    @Test(expected = ConfigException.class)
+    @Test
     public void testAzureFileOutputByOpenWithNonReadableFile() throws Exception
     {
         ConfigSource configSource = config();
@@ -236,7 +240,12 @@ public class TestAzureBlobStorageFileOutputPlugin
         File file = File.createTempFile("non-readable-file", ".tmp");
         file.setReadable(false);
         field.set(output, file);
-        output.finish();
+        try {
+            output.finish();
+        }
+        catch (Exception ex) {
+            assertEquals(FileNotFoundException.class, ex.getCause().getClass());
+        }
     }
 
     @Test(expected = RuntimeException.class)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-14 00:00:00.000000000 Z
+date: 2016-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Stores files on Microsoft Azure blob Storage.
+description: Stores files on Microsoft Azure Blob Storage.
 email:
 - satoshiakama@gmail.com
 executables: []
@@ -63,7 +63,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/azure-storage-4.0.0.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-output-azure_blob_storage-0.1.3.jar
+- classpath/embulk-output-azure_blob_storage-0.1.4.jar
 - classpath/jackson-core-2.6.0.jar
 homepage: https://github.com/sakama/embulk-output-azure_blob_storage
 licenses:
@@ -88,5 +88,5 @@ rubyforge_project:
 rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
-summary: Microsoft Azure blob Storage file output plugin for Embulk
+summary: Microsoft Azure Blob Storage file output plugin for Embulk
 test_files: []