embulk-input-gcs 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 04cb5b37d8fb8c70e1c9c4c306cf792ac1ad1ec9
-  data.tar.gz: 44ec9518fc188320a19ffbe2ce7db0b07cda30f3
+  metadata.gz: 5c7a8f8dcd33afc9c58b25bc3604dae90a407fb4
+  data.tar.gz: bd5afe31028d098c805e7096a6939e877bc2633e
 SHA512:
-  metadata.gz: 3d2a59336002f07d48bcf8b5b22b3626e9c59df7aad93a4f91721667f6f7e1552bd1db72ebd1323e19ce1dc67f32da64b9ad88bc6d6785591904b8d87059249f
-  data.tar.gz: 1533eccae86b7355303ab91bc5ab08cf49c816438fef3541ad1318fc24f0eab948667132952746173a2dd3dabea41bc06cfc2493b8b4701c1908d2e07ff9b635
+  metadata.gz: 9e761bc69cbb53342b4f7cc943bff476e8678a6dc4f22306039d2269fd91d4651c797d51d6891580af148d53d4cc54fb3a7a45710c9cada571dd59db88b12842
+  data.tar.gz: 2e03a2ca66c9daae76190f70c5dda99c437cf783e77ff5dd6ed1fe15dbe6633e4644350f53d303eb7e0aa06af84540d4bd6d5f6ffb65299819e508a397b5f9d6

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 0.2.7 - 2018-03-22
+* [maintenance] Fix retry logic to avoid IOException happens while IOUtils.copy() [#33](https://github.com/embulk/embulk-input-gcs/pull/33)
+
 ## 0.2.6 - 2018-03-05
-* [maintenance] Support "path_match_pattern" option [#32](https://github.com/embulk/embulk-input-gcs/pull/32)
+* [new feature] Support "path_match_pattern" option [#32](https://github.com/embulk/embulk-input-gcs/pull/32)
 
 ## 0.2.5 - 2017-05-19
 * [maintenance] Fix InputStream handling to avoid plugin get less records than expected [#27](https://github.com/embulk/embulk-input-gcs/pull/27)

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.2.6"
+version = "0.2.7"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/src/main/java/org/embulk/input/gcs/SingleFileProvider.java

@@ -2,11 +2,9 @@ package org.embulk.input.gcs;
 
 import com.google.api.client.util.IOUtils;
 import com.google.api.services.storage.Storage;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Throwables;
 import org.embulk.spi.Exec;
 import org.embulk.spi.util.InputStreamFileInput;
-import org.embulk.spi.util.ResumableInputStream;
 import org.embulk.spi.util.RetryExecutor;
 import org.slf4j.Logger;
 
@@ -17,7 +15,6 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InterruptedIOException;
 import java.util.Iterator;
 
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
@@ -30,6 +27,7 @@ public class SingleFileProvider
     private final Iterator<String> iterator;
     private final int maxConnectionRetry;
     private boolean opened = false;
+    private final Logger log = Exec.getLogger(SingleFileProvider.class);
 
     public SingleFileProvider(PluginTask task, int taskIndex)
     {
@@ -50,12 +48,9 @@ public class SingleFileProvider
             return null;
         }
         String key = iterator.next();
-        Storage.Objects.Get getObject = client.objects().get(bucket, key);
         File tempFile = Exec.getTempFileSpace().createTempFile();
-        try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) {
-            IOUtils.copy(getObject.executeMediaAsInputStream(), outputStream);
-        }
-        return new ResumableInputStream(new BufferedInputStream(new FileInputStream(tempFile)), new GcsInputStreamReopener(tempFile, client, bucket, key, maxConnectionRetry));
+        getRemoteContentsWithRetry(tempFile, client, bucket, key, maxConnectionRetry);
+        return new BufferedInputStream(new FileInputStream(tempFile));
     }
 
     @Override
@@ -63,81 +58,53 @@ public class SingleFileProvider
     {
     }
 
-    @VisibleForTesting
-    static class GcsInputStreamReopener
-            implements ResumableInputStream.Reopener
+    private Void getRemoteContentsWithRetry(final File tempFile, final Storage client, final String bucket, final String key, int maxConnectionRetry)
     {
-        private final Logger log = Exec.getLogger(GcsInputStreamReopener.class);
-        private final File tempFile;
-        private final Storage client;
-        private final String bucket;
-        private final String key;
-        private final int maxConnectionRetry;
-
-        public GcsInputStreamReopener(File tempFile, Storage client, String bucket, String key, int maxConnectionRetry)
-        {
-            this.tempFile = tempFile;
-            this.client = client;
-            this.bucket = bucket;
-            this.key = key;
-            this.maxConnectionRetry = maxConnectionRetry;
-        }
-
-        @Override
-        public InputStream reopen(final long offset, final Exception closedCause) throws IOException
-        {
-            try {
-                return retryExecutor()
-                        .withRetryLimit(maxConnectionRetry)
-                        .withInitialRetryWait(500)
-                        .withMaxRetryWait(30 * 1000)
-                        .runInterruptible(new RetryExecutor.Retryable<InputStream>() {
-                            @Override
-                            public InputStream call() throws IOException
-                            {
-                                log.warn(String.format("GCS read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
-                                Storage.Objects.Get getObject = client.objects().get(bucket, key);
-
-                                try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) {
-                                    IOUtils.copy(getObject.executeMediaAsInputStream(), outputStream);
-                                }
-                                return new BufferedInputStream(new FileInputStream(tempFile));
+        try {
+            return retryExecutor()
+                    .withRetryLimit(maxConnectionRetry)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new RetryExecutor.Retryable<Void>() {
+                        @Override
+                        public Void call() throws IOException
+                        {
+                            Storage.Objects.Get getObject = client.objects().get(bucket, key);
+                            try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) {
+                                IOUtils.copy(getObject.executeMediaAsInputStream(), outputStream);
                             }
+                            return null;
+                        }
 
-                            @Override
-                            public boolean isRetryableException(Exception exception)
-                            {
-                                return true;  // TODO
-                            }
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            return true;  // TODO
+                        }
 
-                            @Override
-                            public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-                                    throws RetryExecutor.RetryGiveupException
-                            {
-                                String message = String.format("GCS GET request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                        retryCount, retryLimit, retryWait / 1000, exception.getMessage());
-                                if (retryCount % 3 == 0) {
-                                    log.warn(message, exception);
-                                }
-                                else {
-                                    log.warn(message);
-                                }
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryExecutor.RetryGiveupException
+                        {
+                            String message = String.format("GCS GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
                             }
-
-                            @Override
-                            public void onGiveup(Exception firstException, Exception lastException)
-                                    throws RetryExecutor.RetryGiveupException
-                            {
+                            else {
+                                log.warn(message);
                             }
-                        });
-            }
-            catch (RetryExecutor.RetryGiveupException ex) {
-                Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
-                throw Throwables.propagate(ex.getCause());
-            }
-            catch (InterruptedException ex) {
-                throw new InterruptedIOException();
-            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryExecutor.RetryGiveupException
+                        {
+                        }
+                    });
+        }
+        catch (RetryExecutor.RetryGiveupException | InterruptedException ex) {
+            throw Throwables.propagate(ex.getCause());
         }
     }
 }

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -24,12 +24,7 @@ import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.BufferedReader;
-
-import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -420,35 +415,6 @@ public class TestGcsFileInputPlugin
         assertRecords(config, output);
     }
 
-    @Test
-    public void testGcsFileInputByReopen()
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
-    {
-        ConfigSource config = Exec.newConfigSource()
-                .set("bucket", GCP_BUCKET)
-                .set("path_prefix", GCP_PATH_PREFIX)
-                .set("auth_method", "json_key")
-                .set("service_account_email", GCP_EMAIL)
-                .set("json_keyfile", GCP_JSON_KEYFILE)
-                .set("parser", parserConfig(schemaConfig()));
-
-        PluginTask task = config.loadConfig(PluginTask.class);
-        runner.transaction(config, new Control());
-
-        Method method = GcsFileInput.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
-        method.setAccessible(true);
-        Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
-        File tempFile = Exec.getTempFileSpace().createTempFile();
-        task.setFiles(GcsFileInput.listFiles(task, client));
-
-        String key = GCP_BUCKET_DIRECTORY + "sample_01.csv";
-        SingleFileProvider.GcsInputStreamReopener opener = new SingleFileProvider.GcsInputStreamReopener(tempFile, client, GCP_BUCKET, key, MAX_CONNECTION_RETRY);
-        try (InputStream in = opener.reopen(0, new RuntimeException())) {
-            BufferedReader r = new BufferedReader(new InputStreamReader(in));
-            assertEquals("id,account,time,purchase,comment", r.readLine());
-        }
-    }
-
     @Test
     public void testBase64()
             throws NoSuchMethodException, IllegalAccessException, InvocationTargetException

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-05 00:00:00.000000000 Z
+date: 2018-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +71,7 @@ files:
 - src/test/resources/secretkeys.tar.enc
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.2.6.jar
+- classpath/embulk-input-gcs-0.2.7.jar
 - classpath/google-api-client-1.21.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar
 - classpath/google-http-client-1.21.0.jar