embulk-input-gcs 0.2.8 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5659bab7542e13aad7999cf6f389a65583b4e98
-  data.tar.gz: dd924ef90a423c1144d41a3807529ca06573aef5
+  metadata.gz: dfc015455f0851ec196018a6ede5835653ef7f5a
+  data.tar.gz: 47185bd68f3bbe58070094fd9d87bda3635476e4
 SHA512:
-  metadata.gz: e9bd79f9ee60895f4af84dcf72422f13659f074fbb97ddaf07a900daf7d184283eee91e3a51c2edc21774715eba0b83357fe1dc4e6993eaaf67f512ebd2efa45
-  data.tar.gz: a5bba84dab3456e94048f9e8416b881e6323a22fdd64af7a5cd8f78ffbd2c345be2b79d00834bf112396dfe2dbdeb7c58afcdc3a315f91b182711e082263182a
+  metadata.gz: ef1608fd099a302e4977b3808d7adb39ea2718937b893545f147167579b9d0ff0dab8b231ab033650c340f0c66588f4fb4568704ea19af499979f0cb7c51276a
+  data.tar.gz: e1a57cf92f6c81bb79bd7476d08eaecfab9c9d8c21f61f5d5b32e680cf70b0c9503c8bbc2ff7f2331f50a073c60010a55d105c37c0559cf66c82c439c47d7d5a

data/.travis.yml

@@ -4,8 +4,8 @@ jdk:
   - oraclejdk8
 
 before_install:
-  - openssl aes-256-cbc -K $encrypted_79f1af2a2546_key -iv $encrypted_79f1af2a2546_iv
-    -in src/test/resources/secretkeys.tar.enc -out secretkeys.tar -d
+  - openssl aes-256-cbc -K $encrypted_407f38382059_key -iv $encrypted_407f38382059_iv
+    -in src/test/resources/secretkeys.tar.enc -out ./secretkeys.tar -d
   - tar xvf secretkeys.tar
 
 env:
@@ -13,8 +13,8 @@ env:
     - GCP_EMAIL=unittest@embulk-input-gcs-test.iam.gserviceaccount.com
     - GCP_BUCKET=embulk-input-gcs-test
     - GCP_BUCKET_DIRECTORY=unittests_import
-    - GCP_P12_KEYFILE=./embulk-input-gcs-test-018324286daf.p12
-    - GCP_JSON_KEYFILE=./embulk-input-gcs-test-841948b819cf.json
+    - GCP_P12_KEYFILE=./embulk-input-gcs-test-9d51de6767a8.p12
+    - GCP_JSON_KEYFILE=./embulk-input-gcs-test-65cfe930dca1.json
 
 script:
   - ./gradlew gem

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.0 - 2018-12-26
+* [maintenance] Use file name output feature in Embulk core to show file name in cmdout [#37](https://github.com/embulk/embulk-input-gcs/pull/37)
+
 ## 0.2.8 - 2018-06-29
 * [maintenance] Improve retry logic to retry "400 Bad Request" "Invalid JWT: No valid verifier found for issuer" [#34](https://github.com/embulk/embulk-input-gcs/pull/34)
 

data/README.md

@@ -3,6 +3,8 @@
 
 ## Overview
 
+embulk-input-gcs v0.3.0+ requires Embulk v0.9.12+
+
 * Plugin type: **file input**
 * Resume supported: **yes**
 * Cleanup supported: **yes**

data/build.gradle

@@ -17,18 +17,18 @@ configurations {
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-version = "0.2.8"
+version = "0.3.0"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.2"
-    provided "org.embulk:embulk-core:0.8.2"
+    compile  "org.embulk:embulk-core:0.9.12"
+    provided "org.embulk:embulk-core:0.9.12"
 
     compile "com.google.http-client:google-http-client-jackson2:1.21.0"
     compile ("com.google.apis:google-api-services-storage:v1-rev59-1.21.0") {exclude module: "guava-jdk5"}
 
     testCompile "junit:junit:4.12"
-    testCompile "org.embulk:embulk-core:0.8.2:tests"
-    testCompile "org.embulk:embulk-standards:0.8.2"
+    testCompile "org.embulk:embulk-core:0.9.12:tests"
+    testCompile "org.embulk:embulk-standards:0.9.12"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/input/gcs/FileList.java

@@ -3,7 +3,6 @@ package org.embulk.input.gcs;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -21,6 +20,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
@@ -176,7 +176,7 @@ public class FileList
             catch (IOException ex) {
                 throw Throwables.propagate(ex);
             }
-            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.ofNullable(last));
         }
 
         private List<List<Entry>> getSplits(List<Entry> all)

data/src/main/java/org/embulk/input/gcs/GcsAuthentication.java

@@ -11,7 +11,6 @@ import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.api.services.storage.Storage;
 import com.google.api.services.storage.StorageScopes;
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import org.embulk.config.ConfigException;
@@ -28,6 +27,7 @@ import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.security.GeneralSecurityException;
 import java.util.Collections;
+import java.util.Optional;
 
 public class GcsAuthentication
 {
@@ -74,7 +74,7 @@ public class GcsAuthentication
         return new GoogleCredential.Builder()
                 .setTransport(httpTransport)
                 .setJsonFactory(jsonFactory)
-                .setServiceAccountId(serviceAccountEmail.orNull())
+                .setServiceAccountId(serviceAccountEmail.orElseGet(null))
                 .setServiceAccountScopes(
                         ImmutableList.of(
                                 StorageScopes.DEVSTORAGE_READ_ONLY

data/src/main/java/org/embulk/input/gcs/GcsFileInput.java

@@ -6,8 +6,6 @@ import com.google.api.services.storage.model.Bucket;
 import com.google.api.services.storage.model.Objects;
 import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Charsets;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
 import com.google.common.io.BaseEncoding;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskReport;
@@ -21,6 +19,8 @@ import java.io.IOException;
 import java.math.BigInteger;
 import java.security.GeneralSecurityException;
 import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
 
 public class GcsFileInput
         extends InputStreamFileInput
@@ -53,8 +53,8 @@ public class GcsFileInput
             return new GcsAuthentication(
                     task.getAuthMethod().getString(),
                     task.getServiceAccountEmail(),
-                    task.getP12Keyfile().transform(localFileToPathString()),
-                    task.getJsonKeyfile().transform(localFileToPathString()),
+                    task.getP12Keyfile().map(localFileToPathString()),
+                    task.getJsonKeyfile().map(localFileToPathString()),
                     task.getApplicationName()
             );
         }

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -1,8 +1,6 @@
 package org.embulk.input.gcs;
 
 import com.google.api.services.storage.Storage;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
@@ -18,6 +16,8 @@ import org.slf4j.Logger;
 import java.io.IOException;
 import java.security.GeneralSecurityException;
 import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
 
 public class GcsFileInputPlugin
         implements FileInputPlugin
@@ -86,8 +86,8 @@ public class GcsFileInputPlugin
             return new GcsAuthentication(
                     task.getAuthMethod().getString(),
                     task.getServiceAccountEmail(),
-                    task.getP12Keyfile().transform(localFileToPathString()),
-                    task.getJsonKeyfile().transform(localFileToPathString()),
+                    task.getP12Keyfile().map(localFileToPathString()),
+                    task.getJsonKeyfile().map(localFileToPathString()),
                     task.getApplicationName()
             );
         }

data/src/main/java/org/embulk/input/gcs/PluginTask.java

@@ -1,6 +1,5 @@
 package org.embulk.input.gcs;
 
-import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigInject;
@@ -9,6 +8,7 @@ import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.unit.LocalFile;
 
 import java.util.List;
+import java.util.Optional;
 
 public interface PluginTask
         extends Task, FileList.Task

data/src/main/java/org/embulk/input/gcs/SingleFileProvider.java

@@ -5,6 +5,7 @@ import com.google.api.services.storage.Storage;
 import com.google.common.base.Throwables;
 import org.embulk.spi.Exec;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.InputStreamFileInput.InputStreamWithHints;
 import org.embulk.spi.util.RetryExecutor;
 import org.slf4j.Logger;
 
@@ -14,7 +15,6 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Iterator;
 
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
@@ -38,7 +38,7 @@ public class SingleFileProvider
     }
 
     @Override
-    public InputStream openNext() throws IOException
+    public InputStreamWithHints openNextWithHints() throws IOException
     {
         if (opened) {
             return null;
@@ -50,7 +50,10 @@ public class SingleFileProvider
         String key = iterator.next();
         File tempFile = Exec.getTempFileSpace().createTempFile();
         getRemoteContentsWithRetry(tempFile, client, bucket, key, maxConnectionRetry);
-        return new BufferedInputStream(new FileInputStream(tempFile));
+        return new InputStreamWithHints(
+                new BufferedInputStream(new FileInputStream(tempFile)),
+                String.format("gcs://%s/%s", bucket, key)
+        );
     }
 
     @Override

data/src/test/java/org/embulk/input/gcs/TestGcsAuthentication.java

@@ -2,7 +2,6 @@ package org.embulk.input.gcs;
 
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.services.storage.Storage;
-import com.google.common.base.Optional;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
 import org.junit.BeforeClass;
@@ -14,6 +13,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.security.GeneralSecurityException;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assume.assumeNotNull;

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -1,7 +1,6 @@
 package org.embulk.input.gcs;
 
 import com.google.api.services.storage.Storage;
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -29,6 +28,7 @@ import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assume.assumeNotNull;
@@ -295,7 +295,7 @@ public class TestGcsFileInputPlugin
         method.setAccessible(true);
         Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
         FileList.Builder builder = new FileList.Builder(config);
-        GcsFileInput.listGcsFilesByPrefix(builder, client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.<String>absent());
+        GcsFileInput.listGcsFilesByPrefix(builder, client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.empty());
         FileList fileList = builder.build();
         assertEquals(expected.get(0), fileList.get(0).get(0));
         assertEquals(expected.get(1), fileList.get(1).get(0));
@@ -325,7 +325,7 @@ public class TestGcsFileInputPlugin
         method.setAccessible(true);
         Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
         FileList.Builder builder = new FileList.Builder(configWithPattern);
-        GcsFileInput.listGcsFilesByPrefix(builder, client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.<String>absent());
+        GcsFileInput.listGcsFilesByPrefix(builder, client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.empty());
         FileList fileList = builder.build();
         assertEquals(expected.get(0), fileList.get(0).get(0));
         assertEquals(GCP_BUCKET_DIRECTORY + "sample_01.csv", configDiff.get(String.class, "last_path"));
@@ -353,7 +353,7 @@ public class TestGcsFileInputPlugin
         method.setAccessible(true);
         Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
         FileList.Builder builder = new FileList.Builder(config);
-        GcsFileInput.listGcsFilesByPrefix(builder, client, "non-exists-bucket", "prefix", Optional.<String>absent()); // no errors happens
+        GcsFileInput.listGcsFilesByPrefix(builder, client, "non-exists-bucket", "prefix", Optional.empty()); // no errors happens
     }
 
     @Test

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.0
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-29 00:00:00.000000000 Z
+date: 2018-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -57,6 +57,8 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/input/gcs.rb
+- secretkeys.tar
+- secretkeys.tar.enc
 - settings.gradle
 - src/main/java/org/embulk/input/gcs/FileList.java
 - src/main/java/org/embulk/input/gcs/GcsAuthentication.java
@@ -75,11 +77,11 @@ files:
 - classpath/commons-logging-1.1.1.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar
 - classpath/google-http-client-1.21.0.jar
-- classpath/embulk-input-gcs-0.2.8.jar
 - classpath/google-oauth-client-1.21.0.jar
 - classpath/commons-codec-1.3.jar
 - classpath/google-http-client-jackson2-1.21.0.jar
 - classpath/httpcore-4.0.1.jar
+- classpath/embulk-input-gcs-0.3.0.jar
 homepage: https://github.com/embulk/embulk-input-gcs
 licenses:
 - Apache-2.0