embulk-input-gcs 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84c5f3e02ea5ad9dd6b8536ad9c35bc5e893f785
-  data.tar.gz: 6e5cc8dda9c62e0515db17cf1c486b902609b853
+  metadata.gz: 8e9248a8e1b730f9885caa93b24cbf47dfb8a8c0
+  data.tar.gz: 6405b783b5d7ea6a6b74f017946f69e9b7c5b3a6
 SHA512:
-  metadata.gz: ad92f9ff0c884436d4e667d01f23ea088e6a68793941ec8efb87e7694d2770ecde5a0a2ad59da1e58649477be2bd1237843b729004113d447bdeba9c359f0d36
-  data.tar.gz: 15087aa4dbd3365c88b3ab75cbf02de30cd9e0250bf3b372aeee155b2155239db68b75934d920120771e126b14825d7e4544bbca0bf5fb2dff3f483a6837eee1
+  metadata.gz: fbbce0c9ba052477d392ba00586d6951f00924a9d163409b3394ac393035b9b33c5acb2aeecc2111b1a3febc8d59ab0b9401a4b680ed6e578cfc5e57ab844e3d
+  data.tar.gz: 23604910383ac32a7aa66b0afb94144cd23bea0b26866748d24ec8189f27ea313e4a9cda0a599a1aefce8be84e0bcc3957754dbe95e6d5755dfc517666685806

data/.travis.yml

@@ -0,0 +1,23 @@
+language: java
+
+jdk:
+  - oraclejdk8
+  - oraclejdk7
+  - openjdk7
+
+before_install:
+  - openssl aes-256-cbc -K $encrypted_79f1af2a2546_key -iv $encrypted_79f1af2a2546_iv
+    -in src/test/resources/secretkeys.tar.enc -out secretkeys.tar -d
+  - tar xvf secretkeys.tar
+
+env:
+  global:
+    - GCP_EMAIL=unittest@embulk-input-gcs-test.iam.gserviceaccount.com
+    - GCP_BUCKET=embulk-input-gcs-test
+    - GCP_BUCKET_DIRECTORY=unittests_import
+    - GCP_P12_KEYFILE=./embulk-input-gcs-test-018324286daf.p12
+    - GCP_JSON_KEYFILE=./embulk-input-gcs-test-841948b819cf.json
+
+script:
+  - ./gradlew gem
+  - ./gradlew --info check jacocoTestReport

data/CHANGELOG.md

@@ -0,0 +1,49 @@
+## 0.2.1 - 2016-08-04
+* [maintenance] Use retry logic when generate GCS client [#21](https://github.com/embulk/embulk-input-gcs/pull/21)
+
+## 0.2.0 - 2016-06-03
+* [new feature] Support path option to allow to specify list of target objects directly @sonots thanks! [#17](https://github.com/embulk/embulk-input-gcs/pull/17)
+
+## 0.1.13 - 2016-02-04
+* [maintenance] Upgraded embulk to v0.8.2  [#14](https://github.com/embulk/embulk-input-gcs/pull/14)
+* [maintenance] Updated Google HTTP Client Library from 1.19.0 to 2.1.21.0 [#15](https://github.com/embulk/embulk-input-gcs/pull/15)
+* [maintenance] Updated Google Cloud Storage API Client Library from v1-rev27-1.19.1 to v1-rev59-1.21.0  [#15](https://github.com/embulk/embulk-input-gcs/pull/15)
+
+## 0.1.11 - 2016-01-25
+* [maintenance] Added retry logic [#11](https://github.com/embulk/embulk-input-gcs/pull/11)
+
+## 0.1.10 - 2015-11-07
+
+* [maintenance] Fix resume download logics [#10](https://github.com/embulk/embulk-input-gcs/pull/10)
+* [maintenance] Throw ConfigException when files listing failed. @muga thanks! [#9](https://github.com/embulk/embulk-input-gcs/pull/9)
+
+## 0.1.9 - 2015-10-30
+
+* [maintenance] Fix GcsAuthentication object initialization for mapreduce executor. @muga thanks!  [#7](https://github.com/embulk/embulk-input-gcs/pull/7)
+
+## 0.1.8 - 2015-10-29
+
+* [maintenance] Added unit tests [#8](https://github.com/embulk/embulk-input-gcs/pull/8)
+
+## 0.1.7 - 2015-10-06
+
+* [new feature] Added new auth method - json_keyfile of GCP(Google Cloud Platform)'s service account [#5](https://github.com/embulk/embulk-input-gcs/pull/5)
+* [maintenance] Supported mapreduce-executor [#4](https://github.com/embulk/embulk-input-gcs/pull/4)
+
+## 0.1.6 - 2015-09-05
+
+* [new feature] Added new auth method - pre-defined access token of GCE(Google Compute Engine) [#3](https://github.com/embulk/embulk-input-gcs/pull/3)
+
+## 0.1.5 - 2015-08-19
+
+* [maintenance] Upgraded embulk version to 0.7.0
+* [maintenance] Refactored
+
+## 0.1.4 - 2015-06-27
+
+* [maintenance] Keep last last_path when input files is empty. @frsyuki thanks! [#1](https://github.com/embulk/embulk-input-gcs/pull/1)
+* [maintenance] Refactored error handling logics.
+
+## 0.1.3 - 2015-03-16
+
+* [maintenance] Changed supported Java version from 8 to 7

data/README.md

@@ -1,4 +1,5 @@
 # Google Cloud Storage file input plugin for Embulk
+[![Build Status](https://travis-ci.org/embulk/embulk-input-gcs.svg?branch=master)](https://travis-ci.org/embulk/embulk-input-gcs)
 
 ## Overview
 

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.2.0"
+version = "0.2.1"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/src/main/java/org/embulk/input/gcs/GcsAuthentication.java

@@ -1,5 +1,6 @@
 package org.embulk.input.gcs;
 
+import com.google.api.client.auth.oauth2.TokenResponseException;
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.googleapis.compute.ComputeCredential;
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
@@ -10,16 +11,21 @@ import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.api.services.storage.Storage;
 import com.google.api.services.storage.StorageScopes;
-import com.google.api.services.storage.model.Objects;
 import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
+import org.embulk.config.ConfigException;
 import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.File;
 import java.io.FileInputStream;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.security.GeneralSecurityException;
 import java.util.Collections;
 
@@ -74,13 +80,13 @@ public class GcsAuthentication
                                 StorageScopes.DEVSTORAGE_READ_ONLY
                         )
                 )
-                .setServiceAccountPrivateKeyFromP12File(new File(p12KeyFilePath.orNull()))
+                .setServiceAccountPrivateKeyFromP12File(new File(p12KeyFilePath.get()))
                 .build();
     }
 
     private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
     {
-        FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
+        FileInputStream stream = new FileInputStream(jsonKeyFilePath.get());
 
         return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
                 .createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_ONLY));
@@ -99,16 +105,84 @@ public class GcsAuthentication
         return credential;
     }
 
-    public Storage getGcsClient(String bucket) throws GoogleJsonResponseException, IOException
+    public Storage getGcsClient(final String bucket, int maxConnectionRetry) throws ConfigException, IOException
     {
-        Storage client = new Storage.Builder(httpTransport, jsonFactory, credentials)
-                .setApplicationName(applicationName)
-                .build();
+        try {
+            return retryExecutor()
+                    .withRetryLimit(maxConnectionRetry)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<Storage>() {
+                        @Override
+                        public Storage call() throws IOException, RetryGiveupException
+                        {
+                            Storage client = new Storage.Builder(httpTransport, jsonFactory, credentials)
+                                    .setApplicationName(applicationName)
+                                    .build();
+
+                            // For throw ConfigException when authentication is fail.
+                            long maxResults = 1;
+                            client.objects().list(bucket).setMaxResults(maxResults).execute();
+
+                            return client;
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            if (exception instanceof GoogleJsonResponseException || exception instanceof TokenResponseException) {
+                                int statusCode;
+                                if (exception instanceof GoogleJsonResponseException) {
+                                    statusCode = ((GoogleJsonResponseException) exception).getDetails().getCode();
+                                }
+                                else {
+                                    statusCode = ((TokenResponseException) exception).getStatusCode();
+                                }
+                                if (statusCode / 100 == 4) {
+                                    return false;
+                                }
+                            }
+                            return true;
+                        }
 
-        // For throw IOException when authentication is fail.
-        long maxResults = 1;
-        Objects objects = client.objects().list(bucket).setMaxResults(maxResults).execute();
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryGiveupException
+                        {
+                            String message = String.format("GCS GET request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            }
+                            else {
+                                log.warn(message);
+                            }
+                        }
 
-        return client;
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryGiveupException
+                        {
+                        }
+                    });
+        }
+        catch (RetryGiveupException ex) {
+            if (ex.getCause() instanceof GoogleJsonResponseException || ex.getCause() instanceof TokenResponseException) {
+                int statusCode = 0;
+                if (ex.getCause() instanceof GoogleJsonResponseException) {
+                    statusCode = ((GoogleJsonResponseException) ex.getCause()).getDetails().getCode();
+                }
+                else if (ex.getCause() instanceof TokenResponseException) {
+                    statusCode = ((TokenResponseException) ex.getCause()).getStatusCode();
+                }
+                if (statusCode / 100 == 4) {
+                    throw new ConfigException(ex);
+                }
+            }
+            throw Throwables.propagate(ex);
+        }
+        catch (InterruptedException ex) {
+            throw new InterruptedIOException();
+        }
     }
 }

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -90,6 +90,10 @@ public class GcsFileInputPlugin
         List<String> getFiles();
         void setFiles(List<String> files);
 
+        @Config("max_connection_retry")
+        @ConfigDefault("10") // 10 times retry to connect GCS server if failed.
+        int getMaxConnectionRetry();
+
         @ConfigInject
         BufferAllocator getBufferAllocator();
     }
@@ -198,7 +202,7 @@ public class GcsFileInputPlugin
     {
         Storage client = null;
         try {
-            client = auth.getGcsClient(task.getBucket());
+            client = auth.getGcsClient(task.getBucket(), task.getMaxConnectionRetry());
         }
         catch (IOException ex) {
             throw new ConfigException(ex);
@@ -303,12 +307,14 @@ public class GcsFileInputPlugin
         private final Storage client;
         private final String bucket;
         private final String key;
+        private final int maxConnectionRetry;
 
-        public GcsInputStreamReopener(Storage client, String bucket, String key)
+        public GcsInputStreamReopener(Storage client, String bucket, String key, int maxConnectionRetry)
         {
             this.client = client;
             this.bucket = bucket;
             this.key = key;
+            this.maxConnectionRetry = maxConnectionRetry;
         }
 
         @Override
@@ -316,7 +322,7 @@ public class GcsFileInputPlugin
         {
             try {
                 return retryExecutor()
-                    .withRetryLimit(3)
+                    .withRetryLimit(maxConnectionRetry)
                     .withInitialRetryWait(500)
                     .withMaxRetryWait(30 * 1000)
                     .runInterruptible(new Retryable<InputStream>() {
@@ -395,6 +401,7 @@ public class GcsFileInputPlugin
         private final Storage client;
         private final String bucket;
         private final String key;
+        private final int maxConnectionRetry;
         private boolean opened = false;
 
         public SingleFileProvider(PluginTask task, int taskIndex)
@@ -402,6 +409,7 @@ public class GcsFileInputPlugin
             this.client = newGcsClient(task, newGcsAuth(task));
             this.bucket = task.getBucket();
             this.key = task.getFiles().get(taskIndex);
+            this.maxConnectionRetry = task.getMaxConnectionRetry();
         }
 
         @Override
@@ -413,7 +421,7 @@ public class GcsFileInputPlugin
             opened = true;
             Storage.Objects.Get getObject = client.objects().get(bucket, key);
 
-            return new ResumableInputStream(getObject.executeMediaAsInputStream(), new GcsInputStreamReopener(client, bucket, key));
+            return new ResumableInputStream(getObject.executeMediaAsInputStream(), new GcsInputStreamReopener(client, bucket, key, maxConnectionRetry));
         }
 
         @Override

data/src/test/java/org/embulk/input/gcs/TestGcsAuthentication.java

@@ -1,10 +1,10 @@
 package org.embulk.input.gcs;
 
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.services.storage.Storage;
 import com.google.common.base.Optional;
 import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
@@ -25,6 +25,7 @@ public class TestGcsAuthentication
     private static Optional<String> GCP_JSON_KEYFILE;
     private static String GCP_BUCKET;
     private static final String GCP_APPLICATION_NAME = "embulk-input-gcs";
+    private static int MAX_CONNECTION_RETRY = 3;
 
     /*
      * This test case requires environment variables
@@ -97,12 +98,12 @@ public class TestGcsAuthentication
                 GCP_APPLICATION_NAME
         );
 
-        Storage client = auth.getGcsClient(GCP_BUCKET);
+        Storage client = auth.getGcsClient(GCP_BUCKET, MAX_CONNECTION_RETRY);
 
         assertEquals(Storage.class, client.getClass());
     }
 
-    @Test(expected = GoogleJsonResponseException.class)
+    @Test(expected = ConfigException.class)
     public void testGetGcsClientUsingServiceAccountCredentialThrowJsonResponseException()
             throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
     {
@@ -114,7 +115,7 @@ public class TestGcsAuthentication
                 GCP_APPLICATION_NAME
         );
 
-        Storage client = auth.getGcsClient("non-exists-bucket");
+        Storage client = auth.getGcsClient("non-exists-bucket", MAX_CONNECTION_RETRY);
 
         assertEquals(Storage.class, client.getClass());
     }
@@ -162,12 +163,12 @@ public class TestGcsAuthentication
                 GCP_APPLICATION_NAME
         );
 
-        Storage client = auth.getGcsClient(GCP_BUCKET);
+        Storage client = auth.getGcsClient(GCP_BUCKET, MAX_CONNECTION_RETRY);
 
         assertEquals(Storage.class, client.getClass());
     }
 
-    @Test(expected = GoogleJsonResponseException.class)
+    @Test(expected = ConfigException.class)
     public void testGetServiceAccountCredentialFromJsonThrowGoogleJsonResponseException()
             throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
     {
@@ -179,6 +180,6 @@ public class TestGcsAuthentication
                 GCP_APPLICATION_NAME
         );
 
-        Storage client = auth.getGcsClient("non-exists-bucket");
+        Storage client = auth.getGcsClient("non-exists-bucket", MAX_CONNECTION_RETRY);
     }
 }

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -52,6 +52,7 @@ public class TestGcsFileInputPlugin
     private static String GCP_BUCKET_DIRECTORY;
     private static String GCP_PATH_PREFIX;
     private static String GCP_APPLICATION_NAME;
+    private static int MAX_CONNECTION_RETRY = 3;
     private FileInputRunner runner;
     private MockPageOutput output;
 
@@ -358,7 +359,7 @@ public class TestGcsFileInputPlugin
         task.setFiles(plugin.listFiles(task, client));
 
         String key = GCP_BUCKET_DIRECTORY + "sample_01.csv";
-        GcsFileInputPlugin.GcsInputStreamReopener opener = new GcsFileInputPlugin.GcsInputStreamReopener(client, GCP_BUCKET, key);
+        GcsFileInputPlugin.GcsInputStreamReopener opener = new GcsFileInputPlugin.GcsInputStreamReopener(client, GCP_BUCKET, key, MAX_CONNECTION_RETRY);
         try (InputStream in = opener.reopen(0, new RuntimeException())) {
             BufferedReader r = new BufferedReader(new InputStreamReader(in));
             assertEquals("id,account,time,purchase,comment", r.readLine());

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-02 00:00:00.000000000 Z
+date: 2016-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,7 +46,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
-- ChangeLog
+- .travis.yml
+- CHANGELOG.md
 - README.md
 - build.gradle
 - config/checkstyle/checkstyle.xml
@@ -63,9 +64,10 @@ files:
 - src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
+- src/test/resources/secretkeys.tar.enc
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.2.0.jar
+- classpath/embulk-input-gcs-0.2.1.jar
 - classpath/google-api-client-1.21.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar
 - classpath/google-http-client-1.21.0.jar

data/ChangeLog

@@ -1,48 +0,0 @@
-Release 0.2.0 - 2016-06-03
-* Support path option to allow to specify list of target objects directly @sonots thanks! [#17](https://github.com/embulk/embulk-input-gcs/pull/17)
-
-Release 0.1.13 - 2016-02-04
-* Upgraded embulk to v0.8.2
-* Updated Google HTTP Client Library from 1.19.0 to 2.1.21.0
-* Updated Google Cloud Storage API Client Library from v1-rev27-1.19.1 to v1-rev59-1.21.0
-
-Release 0.1.11 - 2016-01-25
-* Added retry logic
-
-* Added retry logic
-
-Release 0.1.10 - 2015-11-07
-
-* Fix resume download logics
-* Throw ConfigException when files listing failed. @muga thanks!
-
-Release 0.1.9 - 2015-10-30
-
-* Fix GcsAuthentication object initialization for mapreduce executor. @muga thanks!
-
-Release 0.1.8 - 2015-10-29
-
-* Added unit tests
-
-Release 0.1.7 - 2015-10-06
-
-* Added new auth method - json_keyfile of GCP(Google Cloud Platform)'s service account
-* Supported mapreduce-executor
-
-Release 0.1.6 - 2015-09-05
-
-* Added new auth method - pre-defined access token of GCE(Google Compute Engine)
-
-Release 0.1.5 - 2015-08-19
-
-* Upgraded embulk version to 0.7.0
-* Refactored
-
-Release 0.1.4 - 2015-06-27
-
-* Keep last last_path when input files is empty. @frsyuki thanks!
-* Refactored error handling logics.
-
-Release 0.1.3 - 2015-03-16
-
-* Changed supported Java version from 8 to 7