embulk-input-gcs 0.3.0 → 0.3.1

data/src/test/java/org/embulk/input/gcs/TestInputStreamReopener.java

@@ -0,0 +1,145 @@
+package org.embulk.input.gcs;
+
+import com.google.cloud.ReadChannel;
+import com.google.cloud.RestorableState;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.Storage;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.spi.util.ResumableInputStream;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+
+public class TestInputStreamReopener
+{
+    private static class MockReadChannel implements ReadChannel
+    {
+        private FileChannel ch;
+
+        MockReadChannel(FileChannel ch)
+        {
+            this.ch = ch;
+        }
+
+        @Override
+        public boolean isOpen()
+        {
+            return this.ch.isOpen();
+        }
+
+        @Override
+        public void close()
+        {
+            try {
+                this.ch.close();
+            }
+            catch (IOException ignored) {
+            }
+        }
+
+        @Override
+        public void seek(long position) throws IOException
+        {
+            this.ch.position(position);
+        }
+
+        @Override
+        public void setChunkSize(int chunkSize)
+        {
+            // no-op
+        }
+
+        @Override
+        public RestorableState<ReadChannel> capture()
+        {
+            return null;
+        }
+
+        @Override
+        public int read(ByteBuffer dst) throws IOException
+        {
+            return this.ch.read(dst);
+        }
+    }
+
+    private static final String SAMPLE_PATH = TestInputStreamReopener.class.getResource("/sample_01.csv").getPath();
+
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void testResume()
+    {
+        final String bucket = "any_bucket";
+        final String key = "any_file";
+
+        final Storage client = mockStorage();
+
+        final SingleFileProvider.InputStreamReopener reopener = new SingleFileProvider.InputStreamReopener(client, bucket, key);
+        final byte[] buf = new byte[200];
+        final ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try (final ResumableInputStream ris = new ResumableInputStream(reopener)) {
+            int len;
+            while ((len = ris.read(buf)) != -1) {
+                out.write(buf, 0, len);
+            }
+            // read from resumable input stream
+            String content = out.toString("UTF-8");
+            // assert content
+            assertString(content);
+        }
+        catch (IOException e) {
+            e.printStackTrace();
+            fail("Should not throw");
+        }
+    }
+
+    private Storage mockStorage()
+    {
+        Blob blob = Mockito.mock(Blob.class);
+        // mock Storage to return ReadChannel
+        Storage client = Mockito.mock(Storage.class);
+        Mockito.doReturn(blob).when(client).get(eq("any_bucket"), eq("any_file"));
+        // to return new instance every time (can't re-use channel, because it'll be closed)
+        Mockito.doAnswer(invocation -> new MockReadChannel(mockChannel())).when(blob).reader();
+        return client;
+    }
+
+    /**
+     * Return a mock FileChannel, with simulated error during reads
+     *
+     * @return
+     * @throws IOException
+     */
+    private static FileChannel mockChannel() throws IOException
+    {
+        FileChannel ch = Mockito.spy(FileChannel.open(Paths.get(SAMPLE_PATH)));
+        // success -> error -> success -> error...
+        Mockito.doCallRealMethod()
+                .doThrow(new IOException("Fake IOException, going to resume"))
+                .when(ch).read(any(ByteBuffer.class));
+        return ch;
+    }
+
+    private static void assertString(final String actual) throws IOException
+    {
+        final String expected = Files.asCharSource(new File(SAMPLE_PATH), Charsets.UTF_8).read();
+        assertEquals(expected, actual);
+    }
+}

data/src/test/java/org/embulk/input/gcs/TestRetryUtils.java

@@ -0,0 +1,164 @@
+package org.embulk.input.gcs;
+
+import com.google.api.client.auth.oauth2.TokenResponseException;
+import com.google.api.client.googleapis.json.GoogleJsonError;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.http.LowLevelHttpResponse;
+import com.google.api.client.json.Json;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.testing.http.HttpTesting;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import com.google.cloud.storage.StorageException;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.spi.Exec;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+
+import static org.embulk.input.gcs.RetryUtils.withRetry;
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class TestRetryUtils
+{
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private RetryUtils.DefaultRetryable<Object> mock;
+
+    @Before
+    public void setUp()
+    {
+        mock = new RetryUtils.DefaultRetryable<Object>()
+        {
+            @Override
+            public Object call()
+            {
+                return null;
+            }
+        };
+    }
+
+    @Test
+    public void testRetryable() throws IOException
+    {
+        // verify that #isRetryable() returns false for below cases:
+        // - GoogleJsonResponseException && details.code == 4xx
+        assertFalse(mock.isRetryableException(fakeJsonException(400, "fake_400_ex", null)));
+        // - TokenResponseException && statusCode == 4xx
+        assertFalse(mock.isRetryableException(fakeTokenException(400, "{}")));
+        assertFalse(mock.isRetryableException(fakeTokenException(401, "{\"foo\":\"bar\"}")));
+        assertFalse(mock.isRetryableException(fakeTokenException(403, "{ \"error_description\": \"Invalid...\"}")));
+        // return true
+        // - GoogleJsonResponseException && details.code = 5xx
+        assertTrue(mock.isRetryableException(fakeJsonException(500, "fake_500_ex", null)));
+        // - GoogleJsonResponseException && details == null && content != null
+        assertTrue(mock.isRetryableException(fakeJsonExceptionWithoutDetails(400, "fake_400_ex", "this content will make it retry-able")));
+        // - TokenResponseException && statusCode = 5xx
+        assertTrue(mock.isRetryableException(fakeTokenException(500, "{}")));
+        // - TokenResponseException && details.errorDescription contains 'Invalid JWT'
+        assertTrue(mock.isRetryableException(fakeTokenException(403, "{ \"error_description\": \"Invalid JWT...\"}")));
+    }
+
+    @Test
+    public void testWithRetry() throws Exception
+    {
+        mock = Mockito.spy(mock);
+        Exception ex = new StorageException(403, "Fake Exception");
+        Mockito.doThrow(ex).doThrow(ex).doReturn(null).when(mock).call();
+
+        Object result = withRetry(params(), mock);
+        assertNull(result);
+        Mockito.verify(mock, Mockito.times(3)).call();
+    }
+
+    @Test
+    public void testWithRetryGiveUp()
+    {
+        final String expectMsg = "Will retry and give up";
+        mock = new RetryUtils.DefaultRetryable<Object>()
+        {
+            @Override
+            public Object call()
+            {
+                throw new IllegalStateException(expectMsg);
+            }
+        };
+        try {
+            withRetry(params(), mock);
+        }
+        catch (RuntimeException e) {
+            // root cause -> RetryGiveUpException -> RuntimeException
+            Throwable rootCause = e.getCause().getCause();
+            assertEquals(expectMsg, rootCause.getMessage());
+            assertThat(rootCause, instanceOf(IllegalStateException.class));
+        }
+    }
+
+    private static RetryUtils.Task params()
+    {
+        return Exec.newConfigSource().set("initial_retry_interval_millis", 1).loadConfig(RetryUtils.Task.class);
+    }
+
+    private static GoogleJsonResponseException fakeJsonException(final int code, final String message, final String content)
+    {
+        GoogleJsonResponseException.Builder builder = new GoogleJsonResponseException.Builder(code, message, new HttpHeaders());
+        builder.setContent(content);
+        return new GoogleJsonResponseException(builder, fakeJsonError(code, message));
+    }
+
+    private static GoogleJsonResponseException fakeJsonExceptionWithoutDetails(final int code, final String message, final String content)
+    {
+        GoogleJsonResponseException.Builder builder = new GoogleJsonResponseException.Builder(code, message, new HttpHeaders());
+        builder.setContent(content);
+        return new GoogleJsonResponseException(builder, null);
+    }
+
+    private static GoogleJsonError fakeJsonError(final int code, final String message)
+    {
+        GoogleJsonError error = new GoogleJsonError();
+        error.setCode(code);
+        error.setMessage(message);
+        return error;
+    }
+
+    private static TokenResponseException fakeTokenException(final int code, final String content) throws IOException
+    {
+        HttpTransport transport = new MockHttpTransport() {
+            @Override
+            public LowLevelHttpRequest buildRequest(String method, String url)
+            {
+                return new MockLowLevelHttpRequest() {
+                    @Override
+                    public LowLevelHttpResponse execute()
+                    {
+                        MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
+                        response.addHeader("custom_header", "value");
+                        response.setStatusCode(code);
+                        response.setContentType(Json.MEDIA_TYPE);
+                        response.setContent(content);
+                        return response;
+                    }
+                };
+            }
+        };
+        HttpRequest request = transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL);
+        request.setThrowExceptionOnExecuteError(false);
+        return TokenResponseException.from(JacksonFactory.getDefaultInstance(), request.execute());
+    }
+}

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-26 00:00:00.000000000 Z
+date: 2019-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -21,13 +21,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -35,21 +35,56 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Reads files stored on Google Cloud Storage (Standard, Durable Reduced Availability or Nearline)
+description: Reads files stored on Google Cloud Storage (Standard, Durable Reduced
+  Availability or Nearline)
 email:
 - satoshiakama@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - README.md
 - build.gradle
+- classpath/animal-sniffer-annotations-1.14.jar
+- classpath/api-common-1.7.0.jar
+- classpath/checker-compat-qual-2.5.2.jar
+- classpath/commons-codec-1.10.jar
+- classpath/commons-logging-1.2.jar
+- classpath/embulk-input-gcs-0.3.1-shadow.jar
+- classpath/error_prone_annotations-2.1.3.jar
+- classpath/gax-1.35.1.jar
+- classpath/gax-httpjson-0.52.1.jar
+- classpath/google-api-client-1.27.0.jar
+- classpath/google-api-services-storage-v1-rev20181013-1.27.0.jar
+- classpath/google-auth-library-credentials-0.12.0.jar
+- classpath/google-auth-library-oauth2-http-0.12.0.jar
+- classpath/google-cloud-core-1.56.0.jar
+- classpath/google-cloud-core-http-1.56.0.jar
+- classpath/google-http-client-1.27.0.jar
+- classpath/google-http-client-appengine-1.27.0.jar
+- classpath/google-http-client-jackson2-1.27.0.jar
+- classpath/google-oauth-client-1.27.0.jar
+- classpath/grpc-context-1.12.0.jar
+- classpath/gson-2.7.jar
+- classpath/httpclient-4.5.5.jar
+- classpath/httpcore-4.4.9.jar
+- classpath/j2objc-annotations-1.1.jar
+- classpath/jackson-core-2.9.6.jar
+- classpath/javax.annotation-api-1.2.jar
+- classpath/jsr305-3.0.2.jar
+- classpath/opencensus-api-0.15.0.jar
+- classpath/opencensus-contrib-http-util-0.15.0.jar
+- classpath/proto-google-common-protos-1.12.0.jar
+- classpath/proto-google-iam-v1-0.12.0.jar
+- classpath/protobuf-java-3.6.1.jar
+- classpath/protobuf-java-util-3.6.1.jar
+- classpath/threetenbp-1.3.3.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - gradle/wrapper/gradle-wrapper.jar
@@ -57,31 +92,21 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/input/gcs.rb
-- secretkeys.tar
-- secretkeys.tar.enc
 - settings.gradle
+- src/main/java/org/embulk/input/gcs/AuthUtils.java
 - src/main/java/org/embulk/input/gcs/FileList.java
-- src/main/java/org/embulk/input/gcs/GcsAuthentication.java
 - src/main/java/org/embulk/input/gcs/GcsFileInput.java
 - src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java
 - src/main/java/org/embulk/input/gcs/PluginTask.java
+- src/main/java/org/embulk/input/gcs/RetryUtils.java
 - src/main/java/org/embulk/input/gcs/SingleFileProvider.java
-- src/test/java/org/embulk/input/gcs/TestGcsAuthentication.java
+- src/test/java/org/embulk/input/gcs/TestAuthUtils.java
 - src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java
+- src/test/java/org/embulk/input/gcs/TestInputStreamReopener.java
+- src/test/java/org/embulk/input/gcs/TestRetryUtils.java
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
-- src/test/resources/secretkeys.tar.enc
-- classpath/google-api-client-1.21.0.jar
-- classpath/httpclient-4.0.1.jar
-- classpath/jsr305-1.3.9.jar
-- classpath/commons-logging-1.1.1.jar
-- classpath/google-api-services-storage-v1-rev59-1.21.0.jar
-- classpath/google-http-client-1.21.0.jar
-- classpath/google-oauth-client-1.21.0.jar
-- classpath/commons-codec-1.3.jar
-- classpath/google-http-client-jackson2-1.21.0.jar
-- classpath/httpcore-4.0.1.jar
-- classpath/embulk-input-gcs-0.3.0.jar
+- src/test/resources/secrets.tar.enc
 homepage: https://github.com/embulk/embulk-input-gcs
 licenses:
 - Apache-2.0
@@ -92,17 +117,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Google Cloud Storage input plugin for Embulk