embulk-input-gcs 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed0415e9a36b04db434dd09b655ac159ce062b2a
-  data.tar.gz: 868028f5e59ddaf23771a962b635ecc34c344f28
+  metadata.gz: 01e9c70bf2d1f4c9a25784ab64828913c056078d
+  data.tar.gz: 22d6b84e12d965045719b49026924f5247a6609a
 SHA512:
-  metadata.gz: b1d8f6ee7e69eddaad96547196bfd0b193c5c8141323aad8b84945f197fbe2f464feb273f20f7a6da6d978559ff8a3ce0e683e622984e76188a77d0f4ac82a91
-  data.tar.gz: f2c7f5db67082badf48838b6fc4b1af372cc525ca70b5668656e5d9faf8eadf0dbd346f010433a824ad220a8394ee86a5cbc2758f3ef70082e8c6b578340026a
+  metadata.gz: d090628aec5d9512976ceff29215f99e04eb939e4e927106bca4bee47fe95cef7e636fc74b239cde7222cc71f5b259b43905b36ad90d3c515f8b97f8b6c9c220
+  data.tar.gz: 0ad764f0f9f85818b5924b3477bfe7d65806113ca56e689b45e919d3e8c0b156bfbb39cf60e9f349bb1005264655eaf0ad4446115f0254c554c5e425edac746a

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.5 - 2017-05-19
+* [maintenance] Fix InputStream handling to avoid plugin get less records than expected [#27](https://github.com/embulk/embulk-input-gcs/pull/27)
+
 ## 0.2.4 - 2017-03-16
 * [maintenance] Fix possibility to throw NPE when authentication [#25](https://github.com/embulk/embulk-input-gcs/pull/25)
 

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.2.4"
+version = "0.2.5"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -1,6 +1,7 @@
 package org.embulk.input.gcs;
 
 import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.util.IOUtils;
 import com.google.api.services.storage.Storage;
 import com.google.api.services.storage.model.Bucket;
 import com.google.api.services.storage.model.Objects;
@@ -33,6 +34,11 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InterruptedIOException;
@@ -250,19 +256,21 @@ public class GcsFileInputPlugin
         String lastKey = lastPath.isPresent() ? base64Encode(lastPath.get()) : null;
 
         // @see https://cloud.google.com/storage/docs/json_api/v1/objects#resource
-        try {
-            Storage.Buckets.Get getBucket = client.buckets().get(bucket);
-            getBucket.setProjection("full");
-            Bucket bk = getBucket.execute();
-
-            log.debug("bucket name: " + bucket);
-            log.debug("bucket location: " + bk.getLocation());
-            log.debug("bucket timeCreated: " + bk.getTimeCreated());
-            log.debug("bucket owner: " + bk.getOwner());
-        }
-        catch (IOException e) {
-            log.warn("Could not access to bucket:" + bucket);
-            log.warn(e.getMessage());
+        if (log.isDebugEnabled()) {
+            try {
+                Storage.Buckets.Get getBucket = client.buckets().get(bucket);
+                getBucket.setProjection("full");
+                Bucket bk = getBucket.execute();
+
+                log.debug("bucket name: " + bucket);
+                log.debug("bucket location: " + bk.getLocation());
+                log.debug("bucket timeCreated: " + bk.getTimeCreated());
+                log.debug("bucket owner: " + bk.getOwner());
+            }
+            catch (IOException e) {
+                log.warn("Could not access to bucket:" + bucket);
+                log.warn(e.getMessage());
+            }
         }
 
         try {
@@ -312,13 +320,15 @@ public class GcsFileInputPlugin
             implements ResumableInputStream.Reopener
     {
         private final Logger log = Exec.getLogger(GcsInputStreamReopener.class);
+        private final File tempFile;
         private final Storage client;
         private final String bucket;
         private final String key;
         private final int maxConnectionRetry;
 
-        public GcsInputStreamReopener(Storage client, String bucket, String key, int maxConnectionRetry)
+        public GcsInputStreamReopener(File tempFile, Storage client, String bucket, String key, int maxConnectionRetry)
         {
+            this.tempFile = tempFile;
             this.client = client;
             this.bucket = bucket;
             this.key = key;
@@ -335,11 +345,15 @@ public class GcsFileInputPlugin
                     .withMaxRetryWait(30 * 1000)
                     .runInterruptible(new Retryable<InputStream>() {
                         @Override
-                        public InputStream call() throws InterruptedIOException, IOException
+                        public InputStream call() throws IOException
                         {
                             log.warn(String.format("GCS read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
                             Storage.Objects.Get getObject = client.objects().get(bucket, key);
-                            return getObject.executeMediaAsInputStream();
+
+                            try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) {
+                                IOUtils.copy(getObject.executeMediaAsInputStream(), outputStream);
+                            }
+                            return new BufferedInputStream(new FileInputStream(tempFile));
                         }
 
                         @Override
@@ -428,8 +442,11 @@ public class GcsFileInputPlugin
             }
             opened = true;
             Storage.Objects.Get getObject = client.objects().get(bucket, key);
-
-            return new ResumableInputStream(getObject.executeMediaAsInputStream(), new GcsInputStreamReopener(client, bucket, key, maxConnectionRetry));
+            File tempFile = Exec.getTempFileSpace().createTempFile();
+            try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) {
+                IOUtils.copy(getObject.executeMediaAsInputStream(), outputStream);
+            }
+            return new ResumableInputStream(new BufferedInputStream(new FileInputStream(tempFile)), new GcsInputStreamReopener(tempFile, client, bucket, key, maxConnectionRetry));
         }
 
         @Override

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -27,6 +27,7 @@ import org.junit.Test;
 
 import java.io.BufferedReader;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -404,10 +405,11 @@ public class TestGcsFileInputPlugin
         Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
         method.setAccessible(true);
         Storage client = plugin.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
+        File tempFile = Exec.getTempFileSpace().createTempFile();
         task.setFiles(plugin.listFiles(task, client));
 
         String key = GCP_BUCKET_DIRECTORY + "sample_01.csv";
-        GcsFileInputPlugin.GcsInputStreamReopener opener = new GcsFileInputPlugin.GcsInputStreamReopener(client, GCP_BUCKET, key, MAX_CONNECTION_RETRY);
+        GcsFileInputPlugin.GcsInputStreamReopener opener = new GcsFileInputPlugin.GcsInputStreamReopener(tempFile, client, GCP_BUCKET, key, MAX_CONNECTION_RETRY);
         try (InputStream in = opener.reopen(0, new RuntimeException())) {
             BufferedReader r = new BufferedReader(new InputStreamReader(in));
             assertEquals("id,account,time,purchase,comment", r.readLine());

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-16 00:00:00.000000000 Z
+date: 2017-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ files:
 - src/test/resources/secretkeys.tar.enc
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.2.4.jar
+- classpath/embulk-input-gcs-0.2.5.jar
 - classpath/google-api-client-1.21.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar
 - classpath/google-http-client-1.21.0.jar