embulk-input-gcs 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fd9bbc407e6e93967a7d08fcd72b4d4f0c419659
-  data.tar.gz: 86ebeaf4dace9de06146a4671d6fe3db69089b5c
+  metadata.gz: a7f7692d41284e6ac5299bb159a47416b4a31b5a
+  data.tar.gz: c970445b2be3449e3691e63cc139ca3f362333c8
 SHA512:
-  metadata.gz: 783728ec908c8f598acfac3ec555075a55b7249943afc76d030f2fe57d9353ddbb78cbecabdba9504189d943b9574555d3ba301a4ee654f5c78b81c179abdef4
-  data.tar.gz: 0e8815c728a060f749e8d797938235b72eed889c2155b635256646e5816432326555648cf229ef76a01e969aca6ec659eb5ce976c43f32ee64f6a3d3ce451d69
+  metadata.gz: 9220e5404d2edfd40193813563ce40d487fabab48e78073c256193eac383edd1cae8e84700e97eb47e7513ba0ec8002d19ae7879b63c7a57961852fec0a40348
+  data.tar.gz: 104febfa4425cac2622d381222c18c3c11033e965731087b7a9fa69f34907cd7f543747b84ffaac83ca4b23b3c2680cf413fc858661c9bd74cbc87d0a6be3342

data/ChangeLog

@@ -1,3 +1,6 @@
+Release 0.1.11 - 2016-01-25
+* Added retry logic
+
 Release 0.1.10 - 2015-11-07
 
 * Fix resume download logics

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.10"
+version = "0.1.11"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -4,6 +4,7 @@ import java.util.List;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.io.InputStream;
 import java.math.BigInteger;
 
@@ -14,6 +15,7 @@ import com.google.common.base.Function;
 import com.google.common.base.Throwables;
 import com.google.common.base.Charsets;
 import com.google.common.io.BaseEncoding;
+import com.google.common.annotations.VisibleForTesting;
 import java.security.GeneralSecurityException;
 
 import org.embulk.config.TaskReport;
@@ -31,6 +33,10 @@ import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.unit.LocalFile;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.ResumableInputStream;
+import org.embulk.spi.util.RetryExecutor.Retryable;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import org.slf4j.Logger;
 
@@ -278,6 +284,73 @@ public class GcsFileInputPlugin
         return new GcsFileInput(task, taskIndex);
     }
 
+    @VisibleForTesting
+    static class GcsInputStreamReopener
+            implements ResumableInputStream.Reopener
+    {
+        private final Logger log = Exec.getLogger(GcsInputStreamReopener.class);
+        private final Storage client;
+        private final String bucket;
+        private final String key;
+
+        public GcsInputStreamReopener(Storage client, String bucket, String key)
+        {
+            this.client = client;
+            this.bucket = bucket;
+            this.key = key;
+        }
+
+        @Override
+        public InputStream reopen(final long offset, final Exception closedCause) throws IOException
+        {
+            try {
+                return retryExecutor()
+                    .withRetryLimit(3)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<InputStream>() {
+                        @Override
+                        public InputStream call() throws InterruptedIOException, IOException
+                        {
+                            log.warn(String.format("GCS read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
+                            Storage.Objects.Get getObject = client.objects().get(bucket, key);
+                            return getObject.executeMediaAsInputStream();
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            return true;  // TODO
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryGiveupException
+                        {
+                            String message = String.format("GCS GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait/1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            } else {
+                                log.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryGiveupException
+                        {
+                        }
+                    });
+            } catch (RetryGiveupException ex) {
+                Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
+                throw Throwables.propagate(ex.getCause());
+            } catch (InterruptedException ex) {
+                throw new InterruptedIOException();
+            }
+        }
+    }
+
     public class GcsFileInput
             extends InputStreamFileInput
             implements TransactionalFileInput
@@ -322,7 +395,7 @@ public class GcsFileInputPlugin
             opened = true;
             Storage.Objects.Get getObject = client.objects().get(bucket, key);
 
-            return getObject.executeMediaAsInputStream();
+            return new ResumableInputStream(getObject.executeMediaAsInputStream(), new GcsInputStreamReopener(client, bucket, key));
         }
 
         @Override

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -7,6 +7,9 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.IOException;
 import java.security.GeneralSecurityException;
 
@@ -199,7 +202,7 @@ public class TestGcsFileInputPlugin
                 .set("json_keyfile", GCP_JSON_KEYFILE)
                 .set("parser", parserConfig(schemaConfig()));
 
-        PluginTask task = config().loadConfig(PluginTask.class);
+        PluginTask task = config.loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
 
         Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
@@ -292,6 +295,34 @@ public class TestGcsFileInputPlugin
         assertRecords(config, output);
     }
 
+    @Test
+    public void testGcsFileInputByReopen()
+            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", GCP_PATH_PREFIX)
+                .set("auth_method", "json_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("json_keyfile", GCP_JSON_KEYFILE)
+                .set("parser", parserConfig(schemaConfig()));
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+        runner.transaction(config, new Control());
+
+        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
+        method.setAccessible(true);
+        Storage client = plugin.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
+        task.setFiles(plugin.listFiles(task, client));
+
+        String key = GCP_BUCKET_DIRECTORY + "sample_01.csv";
+        GcsFileInputPlugin.GcsInputStreamReopener opener = new GcsFileInputPlugin.GcsInputStreamReopener(client, GCP_BUCKET, key);
+        try (InputStream in = opener.reopen(0, new RuntimeException())) {
+            BufferedReader r = new BufferedReader(new InputStreamReader(in));
+            assertEquals("id,account,time,purchase,comment", r.readLine());
+        }
+    }
+
     @Test
     public void testBase64()
             throws NoSuchMethodException, IllegalAccessException, InvocationTargetException

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-07 00:00:00.000000000 Z
+date: 2016-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,7 +63,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.1.10.jar
+- classpath/embulk-input-gcs-0.1.11.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev27-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar