embulk-input-azure_blob_storage 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49b71df05579f8c392ec70089aaeca95f1b81144
-  data.tar.gz: a63ffb1663b9bbed5827211dbb62eeb04b20a987
+  metadata.gz: 60cc4b85fafbfd719fdb70232f8c7b44c3153d9f
+  data.tar.gz: 50af2362a8edbc752bbc1116a36ede8a3f64aa84
 SHA512:
-  metadata.gz: 1a43a45c850746c1a50b9f12be823f0081ea9deddd4403e831f2533e6af4ab4861405e45cc62824fcb637b6509178fceeb6cad4f583542677e0ee3383beb8e43
-  data.tar.gz: d5e6066c047f60034d50ff6084919157476f962adf5c9f2b1a59685a97d83a254637ed0037276629714e2b1223c229c0564b58c162cfffbb58b10a4822f563aa
+  metadata.gz: b11039548309b1d49bc7b87e54a6ff60eb36863e8f75127f087f71520a957d833aac197f577dbbf262ae0d68a656edd92b0c5fa5ed42fb7c90c62f8e5f551b15
+  data.tar.gz: ecb4b533a64440f78a07ebdb8e1a7b5a4dba91a79a4c7121e32463ecdb4e5746d26186078ae938dd73774273001d917ac09b914d20fa312690ccde91a49986d5

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.5 - 2015-03-30
+
+* [maintenance] Use RetryExecutor when retry is needed [#8](https://github.com/sakama/embulk-input-azure_blob_storage/pull/8)
+
 ## 0.1.4 - 2015-03-22
 
 * [new feature] Support `last_path` option [#7](https://github.com/sakama/embulk-input-azure_blob_storage/pull/7)
@@ -15,4 +19,4 @@
 ## 0.1.1 - 2015-10-11
 
 * [maintenance] Change 'DefaultEndpointsProtocol' from http to https [#1](https://github.com/sakama/embulk-input-azure_blob_storage/pull/1)
-* [maintenance] Specify targetCompatibility = 1.7
+* [maintenance] Specify targetCompatibility = 1.7

data/README.md

@@ -1,6 +1,6 @@
-# Azure blob Storage file input plugin for Embulk
+# Azure Blob Storage file input plugin for Embulk
 
-[Embulk](http://www.embulk.org/) file input plugin read files stored on [Microsoft Azure](https://azure.microsoft.com/) [blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
+[Embulk](http://www.embulk.org/) file input plugin read files stored on [Microsoft Azure](https://azure.microsoft.com/) [Blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
 
 ## Overview
 
@@ -129,4 +129,4 @@ $ launchctl load ~/Library/LaunchAgents/environment.plist
 $ launchctl getenv AZURE_ACCOUNT_NAME //try to get value.
 
 Then start your applications.
-```
+```

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.4"
+version = "0.1.5"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"
@@ -65,8 +65,8 @@ Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
   spec.authors       = ["Satoshi Akama"]
-  spec.summary       = %[Microsoft Azure blob Storage file input plugin for Embulk]
-  spec.description   = %[Reads files stored on Microsoft Azure blob Storage.]
+  spec.summary       = %[Microsoft Azure Blob Storage file input plugin for Embulk]
+  spec.description   = %[Reads files stored on Microsoft Azure Blob Storage.]
   spec.email         = ["satoshiakama@gmail.com"]
   spec.licenses      = ["Apache-2.0"]
   spec.homepage      = "https://github.com/sakama/embulk-input-azure_blob_storage"

data/src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java

@@ -27,7 +27,10 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -63,7 +66,7 @@ public class AzureBlobStorageFileInputPlugin
         int getMaxResults();
 
         @Config("max_connection_retry")
-        @ConfigDefault("5") // 5 times retry to connect sftp server if failed.
+        @ConfigDefault("10") // 10 times retry to connect sftp server if failed.
         int getMaxConnectionRetry();
 
         FileList getFiles();
@@ -126,44 +129,85 @@ public class AzureBlobStorageFileInputPlugin
         }
         FileList.Builder builder = new FileList.Builder(task);
 
-        return listFilesWithPrefix(builder, client, task.getContainer(), task.getPathPrefix(), task.getLastPath(), task.getMaxResults());
+        return listFilesWithPrefix(builder, client, task.getContainer(), task.getPathPrefix(),
+                                    task.getLastPath(), task.getMaxResults(), task.getMaxConnectionRetry());
     }
 
-    private static FileList listFilesWithPrefix(FileList.Builder builder, CloudBlobClient client, String containerName,
-                                             String prefix, Optional<String> lastPath, int maxResults)
+    private static FileList listFilesWithPrefix(final FileList.Builder builder, final CloudBlobClient client,
+                                                final String containerName, final String prefix, final Optional<String> lastPath,
+                                                final int maxResults, final int maxConnectionRetry)
     {
-        String lastKey = (lastPath.isPresent() && !lastPath.get().isEmpty()) ? createNextToken(lastPath.get()) : null;
-        ResultContinuation token = null;
-        if (lastKey != null) {
-            token = new ResultContinuation();
-            token.setContinuationType(ResultContinuationType.BLOB);
-            log.debug("lastPath: {}", lastPath.get());
-            log.debug("lastPath(Base64encoded): {}", lastKey);
-            token.setNextMarker(lastKey);
-        }
-
+        final String lastKey = (lastPath.isPresent() && !lastPath.get().isEmpty()) ? createNextToken(lastPath.get()) : null;
         try {
-            CloudBlobContainer container = client.getContainerReference(containerName);
-            ResultSegment<ListBlobItem> blobs;
-            do {
-                blobs = container.listBlobsSegmented(prefix, true, null, maxResults, token, null, null);
-                log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength(), blobs.getContinuationToken()));
-                for (ListBlobItem blobItem : blobs.getResults()) {
-                    if (blobItem instanceof CloudBlob) {
-                        CloudBlob blob = (CloudBlob) blobItem;
-                        if (blob.exists() && !blob.getUri().toString().endsWith("/")) {
-                            builder.add(blob.getName(), blob.getProperties().getLength());
-                            log.debug(String.format("name:%s, class:%s, uri:%s", blob.getName(), blob.getClass(), blob.getUri()));
+            return retryExecutor()
+                    .withRetryLimit(maxConnectionRetry)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<FileList>() {
+                        @Override
+                        public FileList call() throws StorageException, URISyntaxException, IOException
+                        {
+                            ResultContinuation token = null;
+                            if (lastKey != null) {
+                                token = new ResultContinuation();
+                                token.setContinuationType(ResultContinuationType.BLOB);
+                                log.debug("lastPath: {}", lastPath.get());
+                                log.debug("lastPath(Base64encoded): {}", lastKey);
+                                token.setNextMarker(lastKey);
+                            }
+
+                            CloudBlobContainer container = client.getContainerReference(containerName);
+                            ResultSegment<ListBlobItem> blobs;
+                            do {
+                                blobs = container.listBlobsSegmented(prefix, true, null, maxResults, token, null, null);
+                                log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength(), blobs.getContinuationToken()));
+                                for (ListBlobItem blobItem : blobs.getResults()) {
+                                    if (blobItem instanceof CloudBlob) {
+                                        CloudBlob blob = (CloudBlob) blobItem;
+                                        if (blob.exists() && !blob.getUri().toString().endsWith("/")) {
+                                            builder.add(blob.getName(), blob.getProperties().getLength());
+                                            log.debug(String.format("name:%s, class:%s, uri:%s", blob.getName(), blob.getClass(), blob.getUri()));
+                                        }
+                                    }
+                                }
+                                token = blobs.getContinuationToken();
+                            } while (blobs.getContinuationToken() != null);
+                            return builder.build();
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            return true;
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryGiveupException
+                        {
+                            String message = String.format("SFTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            }
+                            else {
+                                log.warn(message);
+                            }
                         }
-                    }
-                }
-                token = blobs.getContinuationToken();
-            } while (blobs.getContinuationToken() != null);
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryGiveupException
+                        {
+                        }
+                    });
+        }
+        catch (RetryGiveupException ex) {
+            throw Throwables.propagate(ex.getCause());
         }
-        catch (URISyntaxException | StorageException ex) {
+        catch (InterruptedException ex) {
             throw Throwables.propagate(ex);
         }
-        return builder.build();
     }
 
     @Override
@@ -215,31 +259,54 @@ public class AzureBlobStorageFileInputPlugin
             if (opened || !iterator.hasNext()) {
                 return null;
             }
+            final String key = iterator.next();
             opened = true;
-            int count = 0;
-
-            while (true) {
-                try {
-                    CloudBlobContainer container = client.getContainerReference(containerName);
-                    CloudBlob blob = container.getBlockBlobReference(iterator.next());
-                    return blob.openInputStream();
-                }
-                catch (StorageException | URISyntaxException ex) {
-                    if (++count == maxConnectionRetry) {
-                        Throwables.propagate(ex);
-                    }
-
-                    try {
-                        long sleepTime = ((long) Math.pow(2, count) * 1000);
-                        log.warn("Sleep in next connection retry: {} milliseconds", sleepTime);
-                        Thread.sleep(sleepTime);
-                    }
-                    catch (InterruptedException ex2) {
-                        // Ignore this exception because this exception is just about `sleep`.
-                        log.warn(ex2.getMessage(), ex2);
-                    }
-                    log.warn("Retrying to connect Azure server: " + count + " times");
-                }
+            try {
+                return retryExecutor()
+                        .withRetryLimit(maxConnectionRetry)
+                        .withInitialRetryWait(500)
+                        .withMaxRetryWait(30 * 1000)
+                        .runInterruptible(new Retryable<InputStream>() {
+                            @Override
+                            public InputStream call() throws StorageException, URISyntaxException, IOException
+                            {
+                                CloudBlobContainer container = client.getContainerReference(containerName);
+                                CloudBlob blob = container.getBlockBlobReference(key);
+                                return blob.openInputStream();
+                            }
+
+                            @Override
+                            public boolean isRetryableException(Exception exception)
+                            {
+                                return true;
+                            }
+
+                            @Override
+                            public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                    throws RetryGiveupException
+                            {
+                                String message = String.format("Azure Blob Storage GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                        retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                                if (retryCount % 3 == 0) {
+                                    log.warn(message, exception);
+                                }
+                                else {
+                                    log.warn(message);
+                                }
+                            }
+
+                            @Override
+                            public void onGiveup(Exception firstException, Exception lastException)
+                                    throws RetryGiveupException
+                            {
+                            }
+                        });
+            }
+            catch (RetryGiveupException ex) {
+                throw Throwables.propagate(ex.getCause());
+            }
+            catch (InterruptedException ex) {
+                throw Throwables.propagate(ex);
             }
         }
 

data/src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java

@@ -90,7 +90,7 @@ public class TestAzureBlobStorageFileInputPlugin
 
         PluginTask task = config.loadConfig(PluginTask.class);
         assertEquals(5000, task.getMaxResults());
-        assertEquals(5, task.getMaxConnectionRetry());
+        assertEquals(10, task.getMaxConnectionRetry());
     }
 
     @Test(expected = ConfigException.class)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-22 00:00:00.000000000 Z
+date: 2016-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Reads files stored on Microsoft Azure blob Storage.
+description: Reads files stored on Microsoft Azure Blob Storage.
 email:
 - satoshiakama@gmail.com
 executables: []
@@ -64,7 +64,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/azure-storage-4.0.0.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-input-azure_blob_storage-0.1.4.jar
+- classpath/embulk-input-azure_blob_storage-0.1.5.jar
 - classpath/jackson-core-2.6.0.jar
 homepage: https://github.com/sakama/embulk-input-azure_blob_storage
 licenses:
@@ -89,5 +89,5 @@ rubyforge_project:
 rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
-summary: Microsoft Azure blob Storage file input plugin for Embulk
+summary: Microsoft Azure Blob Storage file input plugin for Embulk
 test_files: []