embulk-output-gcs 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b41fdc69131fe0099ff844875bafa6d160510e10
-  data.tar.gz: 5b084d56f12509c0c273ea295519a1534d310579
+  metadata.gz: c0d6f9617394f320742be74ac318d99ef0f93c06
+  data.tar.gz: af9f3a3679a2d76b5c2944fd7e8264ecab3b3a27
 SHA512:
-  metadata.gz: a7f51eec122063edaffe94aa065f9bdd1dcc09752d46296f3f8d276d5af250ef82f1a2f56b7e0163f646c98731954d4bf5ce2b901f4e22b4e09033fd20fbf627
-  data.tar.gz: 7ed822140abca79f035a9bf59af3d472472b6bdfa4c1db505a22ae4ca0a80ca1fd380635487f72eb8fd74f706fd1e531fdf421f63b1614f6fcc5731b319c1489
+  metadata.gz: 086960d449e95821defa56b6860eaf2a45fe753db3c7d763272c694ae5fd87b07aba937bb170f748b6d779775e71145c9ff350528d12ea6d770e64d40b67f3ec
+  data.tar.gz: 99a7bc4c683d61b500780c316be1c7861181cccde8f20daf6b674e96068a2fc9414db5b830013c0f6c223b33d2cc7a4d164bc62536ed0082ba419754ac8a7980

data/.travis.yml

@@ -1,8 +1,6 @@
 language: java
 jdk:
 - oraclejdk8
-- oraclejdk7
-- openjdk7
 env:
   global:
   - GCP_EMAIL=account-2@embulk-output-gcs-test.iam.gserviceaccount.com

data/build.gradle

@@ -14,10 +14,10 @@ configurations {
     provided
 }
 
-sourceCompatibility = 1.7
-targetCompatibility = 1.7
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
 
-version = "0.4.1"
+version = "0.4.2"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.6"

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 13 12:41:02 JST 2016
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/src/main/java/org/embulk/output/AuthMethod.java

@@ -0,0 +1,20 @@
+package org.embulk.output;
+
+public enum AuthMethod
+{
+    private_key("private_key"),
+    compute_engine("compute_engine"),
+    json_key("json_key");
+
+    private final String string;
+
+    AuthMethod(String string)
+    {
+        this.string = string;
+    }
+
+    public String getString()
+    {
+        return string;
+    }
+}

data/src/main/java/org/embulk/output/GcsOutputPlugin.java

@@ -1,102 +1,25 @@
 package org.embulk.output;
 
-import com.google.api.client.http.InputStreamContent;
-import com.google.api.client.repackaged.org.apache.commons.codec.binary.Base64;
 import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.model.StorageObject;
 import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.spi.Buffer;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
 import org.embulk.spi.unit.LocalFile;
-import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
-import org.embulk.spi.util.RetryExecutor.Retryable;
-import org.slf4j.Logger;
-import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InterruptedIOException;
 import java.security.GeneralSecurityException;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 
 public class GcsOutputPlugin implements FileOutputPlugin
 {
-    private static final Logger logger = Exec.getLogger(GcsOutputPlugin.class);
-
-    public interface PluginTask extends Task
-    {
-        @Config("bucket")
-        String getBucket();
-
-        @Config("path_prefix")
-        String getPathPrefix();
-
-        @Config("file_ext")
-        String getFileNameExtension();
-
-        @Config("sequence_format")
-        @ConfigDefault("\".%03d.%02d\"")
-        String getSequenceFormat();
-
-        @Config("content_type")
-        @ConfigDefault("\"application/octet-stream\"")
-        String getContentType();
-
-        @Config("auth_method")
-        @ConfigDefault("\"private_key\"")
-        AuthMethod getAuthMethod();
-
-        @Config("service_account_email")
-        @ConfigDefault("null")
-        Optional<String> getServiceAccountEmail();
-
-        // kept for backward compatibility
-        @Config("p12_keyfile_path")
-        @ConfigDefault("null")
-        Optional<String> getP12KeyfilePath();
-
-        @Config("p12_keyfile")
-        @ConfigDefault("null")
-        Optional<LocalFile> getP12Keyfile();
-        void setP12Keyfile(Optional<LocalFile> p12Keyfile);
-
-        @Config("json_keyfile")
-        @ConfigDefault("null")
-        Optional<LocalFile> getJsonKeyfile();
-
-        @Config("application_name")
-        @ConfigDefault("\"embulk-output-gcs\"")
-        String getApplicationName();
-
-        @Config("max_connection_retry")
-        @ConfigDefault("10") // 10 times retry to connect GCS server if failed.
-        int getMaxConnectionRetry();
-    }
-
     @Override
     public ConfigDiff transaction(ConfigSource config,
                                   int taskCount,
@@ -152,7 +75,7 @@ public class GcsOutputPlugin implements FileOutputPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
         Storage client = createClient(task);
-        return new TransactionalGcsFileOutput(task, client, taskIndex);
+        return new GcsTransactionalFileOutput(task, client, taskIndex);
     }
 
     private GcsAuthentication newGcsAuth(PluginTask task)
@@ -192,262 +115,4 @@ public class GcsOutputPlugin implements FileOutputPlugin
             }
         };
     }
-
-    static class TransactionalGcsFileOutput implements TransactionalFileOutput
-    {
-        private final int taskIndex;
-        private final Storage client;
-        private final String bucket;
-        private final String pathPrefix;
-        private final String pathSuffix;
-        private final String sequenceFormat;
-        private final String contentType;
-        private final int maxConnectionRetry;
-        private final List<StorageObject> storageObjects = new ArrayList<>();
-
-        private int fileIndex = 0;
-        private int callCount = 0;
-        private BufferedOutputStream currentStream = null;
-        private Future<StorageObject> currentUpload = null;
-        private File tempFile = null;
-
-        TransactionalGcsFileOutput(PluginTask task, Storage client, int taskIndex)
-        {
-            this.taskIndex = taskIndex;
-            this.client = client;
-            this.bucket = task.getBucket();
-            this.pathPrefix = task.getPathPrefix();
-            this.pathSuffix = task.getFileNameExtension();
-            this.sequenceFormat = task.getSequenceFormat();
-            this.contentType = task.getContentType();
-            this.maxConnectionRetry = task.getMaxConnectionRetry();
-        }
-
-        public void nextFile()
-        {
-            closeCurrentUpload();
-            try {
-                tempFile = Exec.getTempFileSpace().createTempFile();
-                currentStream = new BufferedOutputStream(new FileOutputStream(tempFile));
-                fileIndex++;
-            }
-            catch (IOException ex) {
-                Throwables.propagate(ex);
-            }
-        }
-
-        @Override
-        public void add(Buffer buffer)
-        {
-            try {
-                logger.debug("#add called {} times for taskIndex {}", callCount, taskIndex);
-                currentStream.write(buffer.array(), buffer.offset(), buffer.limit());
-                callCount++;
-            }
-            catch (IOException ex) {
-                throw new RuntimeException(ex);
-            }
-            finally {
-                buffer.release();
-            }
-        }
-
-        @Override
-        public void finish()
-        {
-            String path = generateRemotePath(pathPrefix, sequenceFormat, taskIndex, fileIndex, pathSuffix);
-            close();
-            if (tempFile != null) {
-                currentUpload = startUpload(path);
-            }
-
-            closeCurrentUpload();
-        }
-
-        @Override
-        public void close()
-        {
-            try {
-                if (currentStream != null) {
-                    currentStream.close();
-                    currentStream = null;
-                }
-            }
-            catch (IOException ex) {
-                throw Throwables.propagate(ex);
-            }
-        }
-
-        @Override
-        public void abort()
-        {
-        }
-
-        @Override
-        public TaskReport commit()
-        {
-            TaskReport report = Exec.newTaskReport();
-            report.set("files", storageObjects);
-            return report;
-        }
-
-        private void closeCurrentUpload()
-        {
-            try {
-                if (currentUpload != null) {
-                    StorageObject obj = currentUpload.get();
-                    storageObjects.add(obj);
-                    logger.info("Uploaded '{}/{}' to {}bytes", obj.getBucket(), obj.getName(), obj.getSize());
-                    currentUpload = null;
-                }
-
-                callCount = 0;
-            }
-            catch (InterruptedException | ExecutionException ex) {
-                throw Throwables.propagate(ex);
-            }
-        }
-
-        private Future<StorageObject> startUpload(final String path)
-        {
-            try {
-                final ExecutorService executor = Executors.newCachedThreadPool();
-                final String hash = getLocalMd5hash(tempFile.getAbsolutePath());
-
-                return executor.submit(new Callable<StorageObject>() {
-                    @Override
-                    public StorageObject call() throws IOException
-                    {
-                        try {
-                            logger.info("Uploading '{}/{}'", bucket, path);
-                            return execUploadWithRetry(path, hash);
-                        }
-                        finally {
-                            executor.shutdown();
-                        }
-                    }
-                });
-            }
-            catch (IOException ex) {
-                throw Throwables.propagate(ex);
-            }
-        }
-
-        private StorageObject execUploadWithRetry(final String path, final String localHash) throws IOException
-        {
-            try {
-                return retryExecutor()
-                    .withRetryLimit(maxConnectionRetry)
-                    .withInitialRetryWait(500)
-                    .withMaxRetryWait(30 * 1000)
-                    .runInterruptible(new Retryable<StorageObject>() {
-                    @Override
-                    public StorageObject call() throws IOException, RetryGiveupException
-                    {
-                        try (final BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(tempFile))) {
-                            InputStreamContent mediaContent = new InputStreamContent(contentType, inputStream);
-                            mediaContent.setCloseInputStream(true);
-
-                            StorageObject objectMetadata = new StorageObject();
-                            objectMetadata.setName(path);
-
-                            final Storage.Objects.Insert insert = client.objects().insert(bucket, objectMetadata, mediaContent);
-                            insert.setDisableGZipContent(true);
-                            StorageObject obj = insert.execute();
-
-                            logger.info(String.format("Local Hash(MD5): %s / Remote Hash(MD5): %s", localHash, obj.getMd5Hash()));
-                            return obj;
-                        }
-                    }
-
-                    @Override
-                    public boolean isRetryableException(Exception exception)
-                    {
-                        return true;
-                    }
-
-                    @Override
-                    public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
-                    {
-                        String message = String.format("GCS put request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
-                                        retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
-                        if (retryCount % 3 == 0) {
-                            logger.warn(message, exception);
-                        }
-                        else {
-                            logger.warn(message);
-                        }
-                    }
-
-                    @Override
-                    public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
-                    {
-                    }
-                });
-            }
-            catch (RetryGiveupException ex) {
-                throw Throwables.propagate(ex.getCause());
-            }
-            catch (InterruptedException ex) {
-                throw new InterruptedIOException();
-            }
-        }
-
-        /*
-        MD5 hash sum on GCS bucket is encoded with base64.
-        You can get same hash with following commands.
-        $ openssl dgst -md5 -binary /path/to/file.txt | openssl enc -base64
-        or
-        $ gsutil hash -m /path/to/file.txt
-         */
-        private String getLocalMd5hash(String filePath) throws IOException
-        {
-            try {
-                MessageDigest md = MessageDigest.getInstance("MD5");
-                try (BufferedInputStream input = new BufferedInputStream(new FileInputStream(new File(filePath)))) {
-                    byte[] buffer = new byte[256];
-                    int len;
-                    while ((len = input.read(buffer, 0, buffer.length)) >= 0) {
-                        md.update(buffer, 0, len);
-                    }
-                    return new String(Base64.encodeBase64(md.digest()));
-                }
-            }
-            catch (NoSuchAlgorithmException ex) {
-                throw new ConfigException("MD5 algorism not found");
-            }
-        }
-    }
-
-    /**
-     * GCS has character limitation in object names.
-     * @see https://cloud.google.com/storage/docs/naming#objectnames
-     * Although "." isn't listed at above pages, we can't access "./" path from GUI console.
-     * And in many cases, user don't intend of creating "/" directory under the bucket.
-     * This method normalizes path when it contains "./" and "/" and its variations at the beginning
-     */
-    private static String generateRemotePath(String pathPrefix, String sequenceFormat, int taskIndex, int fileIndex, String pathSuffix)
-    {
-        String path = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix;
-        return path.replaceFirst("^\\.*/*", "");
-    }
-
-    public enum AuthMethod
-    {
-        private_key("private_key"),
-        compute_engine("compute_engine"),
-        json_key("json_key");
-
-        private final String string;
-
-        AuthMethod(String string)
-        {
-            this.string = string;
-        }
-
-        public String getString()
-        {
-            return string;
-        }
-    }
 }

data/src/main/java/org/embulk/output/GcsTransactionalFileOutput.java

@@ -0,0 +1,252 @@
+package org.embulk.output;
+
+import com.google.api.client.http.InputStreamContent;
+import com.google.api.client.repackaged.org.apache.commons.codec.binary.Base64;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.Throwables;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
+import org.slf4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
+public class GcsTransactionalFileOutput implements TransactionalFileOutput
+{
+    private static final Logger logger = Exec.getLogger(GcsTransactionalFileOutput.class);
+
+    private final int taskIndex;
+    private final Storage client;
+    private final String bucket;
+    private final String pathPrefix;
+    private final String pathSuffix;
+    private final String sequenceFormat;
+    private final String contentType;
+    private final int maxConnectionRetry;
+    private final List<StorageObject> storageObjects = new ArrayList<>();
+
+    private int fileIndex = 0;
+    private int callCount = 0;
+    private BufferedOutputStream currentStream = null;
+    private StorageObject currentUpload = null;
+    private File tempFile = null;
+
+    GcsTransactionalFileOutput(PluginTask task, Storage client, int taskIndex)
+    {
+        this.taskIndex = taskIndex;
+        this.client = client;
+        this.bucket = task.getBucket();
+        this.pathPrefix = task.getPathPrefix();
+        this.pathSuffix = task.getFileNameExtension();
+        this.sequenceFormat = task.getSequenceFormat();
+        this.contentType = task.getContentType();
+        this.maxConnectionRetry = task.getMaxConnectionRetry();
+    }
+
+    public void nextFile()
+    {
+        closeCurrentUpload();
+        try {
+            tempFile = Exec.getTempFileSpace().createTempFile();
+            currentStream = new BufferedOutputStream(new FileOutputStream(tempFile));
+            fileIndex++;
+        }
+        catch (IOException ex) {
+            Throwables.propagate(ex);
+        }
+    }
+
+    @Override
+    public void add(Buffer buffer)
+    {
+        try {
+            logger.debug("#add called {} times for taskIndex {}", callCount, taskIndex);
+            currentStream.write(buffer.array(), buffer.offset(), buffer.limit());
+            callCount++;
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+        finally {
+            buffer.release();
+        }
+    }
+
+    @Override
+    public void finish()
+    {
+        String path = generateRemotePath(pathPrefix, sequenceFormat, taskIndex, fileIndex, pathSuffix);
+        close();
+        if (tempFile != null) {
+            currentUpload = startUpload(path);
+        }
+
+        closeCurrentUpload();
+    }
+
+    @Override
+    public void close()
+    {
+        try {
+            if (currentStream != null) {
+                currentStream.close();
+                currentStream = null;
+            }
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    @Override
+    public void abort()
+    {
+    }
+
+    @Override
+    public TaskReport commit()
+    {
+        TaskReport report = Exec.newTaskReport();
+        report.set("files", storageObjects);
+        return report;
+    }
+
+    private void closeCurrentUpload()
+    {
+        if (currentUpload != null) {
+            StorageObject obj = currentUpload;
+            storageObjects.add(obj);
+            logger.info("Uploaded '{}/{}' to {}bytes", obj.getBucket(), obj.getName(), obj.getSize());
+            currentUpload = null;
+        }
+
+        callCount = 0;
+    }
+
+    private StorageObject startUpload(final String path)
+    {
+        try {
+            final String hash = getLocalMd5hash(tempFile.getAbsolutePath());
+
+            return execUploadWithRetry(path, hash);
+        }
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    private StorageObject execUploadWithRetry(final String path, final String localHash) throws IOException
+    {
+        try {
+            return retryExecutor()
+                .withRetryLimit(maxConnectionRetry)
+                .withInitialRetryWait(500)
+                .withMaxRetryWait(30 * 1000)
+                .runInterruptible(new Retryable<StorageObject>() {
+                @Override
+                public StorageObject call() throws IOException
+                {
+                    try (final BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(tempFile))) {
+                        InputStreamContent mediaContent = new InputStreamContent(contentType, inputStream);
+                        mediaContent.setCloseInputStream(true);
+
+                        StorageObject objectMetadata = new StorageObject();
+                        objectMetadata.setName(path);
+
+                        final Storage.Objects.Insert insert = client.objects().insert(bucket, objectMetadata, mediaContent);
+                        insert.setDisableGZipContent(true);
+                        StorageObject obj = insert.execute();
+
+                        logger.info(String.format("Local Hash(MD5): %s / Remote Hash(MD5): %s", localHash, obj.getMd5Hash()));
+                        return obj;
+                    }
+                }
+
+                @Override
+                public boolean isRetryableException(Exception exception)
+                {
+                    return true;
+                }
+
+                @Override
+                public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
+                {
+                    String message = String.format("GCS put request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
+                    if (retryCount % 3 == 0) {
+                        logger.warn(message, exception);
+                    }
+                    else {
+                        logger.warn(message);
+                    }
+                }
+
+                @Override
+                public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
+                {
+                }
+            });
+        }
+        catch (RetryGiveupException ex) {
+            throw Throwables.propagate(ex.getCause());
+        }
+        catch (InterruptedException ex) {
+            throw new InterruptedIOException();
+        }
+    }
+
+    /*
+    MD5 hash sum on GCS bucket is encoded with base64.
+    You can get same hash with following commands.
+    $ openssl dgst -md5 -binary /path/to/file.txt | openssl enc -base64
+    or
+    $ gsutil hash -m /path/to/file.txt
+     */
+    private String getLocalMd5hash(String filePath) throws IOException
+    {
+        try {
+            MessageDigest md = MessageDigest.getInstance("MD5");
+            try (BufferedInputStream input = new BufferedInputStream(new FileInputStream(new File(filePath)))) {
+                byte[] buffer = new byte[256];
+                int len;
+                while ((len = input.read(buffer, 0, buffer.length)) >= 0) {
+                    md.update(buffer, 0, len);
+                }
+                return new String(Base64.encodeBase64(md.digest()));
+            }
+        }
+        catch (NoSuchAlgorithmException ex) {
+            throw new ConfigException("MD5 algorism not found");
+        }
+    }
+
+    /**
+     * GCS has character limitation in object names.
+     * @see https://cloud.google.com/storage/docs/naming#objectnames
+     * Although "." isn't listed at above pages, we can't access "./" path from GUI console.
+     * And in many cases, user don't intend of creating "/" directory under the bucket.
+     * This method normalizes path when it contains "./" and "/" and its variations at the beginning
+     */
+    private static String generateRemotePath(String pathPrefix, String sequenceFormat, int taskIndex, int fileIndex, String pathSuffix)
+    {
+        String path = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix;
+        return path.replaceFirst("^\\.*/*", "");
+    }
+}

data/src/main/java/org/embulk/output/PluginTask.java

@@ -0,0 +1,57 @@
+package org.embulk.output;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.spi.unit.LocalFile;
+
+public interface PluginTask extends Task
+{
+    @Config("bucket")
+    String getBucket();
+
+    @Config("path_prefix")
+    String getPathPrefix();
+
+    @Config("file_ext")
+    String getFileNameExtension();
+
+    @Config("sequence_format")
+    @ConfigDefault("\".%03d.%02d\"")
+    String getSequenceFormat();
+
+    @Config("content_type")
+    @ConfigDefault("\"application/octet-stream\"")
+    String getContentType();
+
+    @Config("auth_method")
+    @ConfigDefault("\"private_key\"")
+    AuthMethod getAuthMethod();
+
+    @Config("service_account_email")
+    @ConfigDefault("null")
+    Optional<String> getServiceAccountEmail();
+
+    // kept for backward compatibility
+    @Config("p12_keyfile_path")
+    @ConfigDefault("null")
+    Optional<String> getP12KeyfilePath();
+
+    @Config("p12_keyfile")
+    @ConfigDefault("null")
+    Optional<LocalFile> getP12Keyfile();
+    void setP12Keyfile(Optional<LocalFile> p12Keyfile);
+
+    @Config("json_keyfile")
+    @ConfigDefault("null")
+    Optional<LocalFile> getJsonKeyfile();
+
+    @Config("application_name")
+    @ConfigDefault("\"embulk-output-gcs\"")
+    String getApplicationName();
+
+    @Config("max_connection_retry")
+    @ConfigDefault("10") // 10 times retry to connect GCS server if failed.
+    int getMaxConnectionRetry();
+}

data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java

@@ -11,7 +11,6 @@ import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.output.GcsOutputPlugin.PluginTask;
 import org.embulk.spi.Buffer;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutputPlugin;
@@ -98,7 +97,7 @@ public class TestGcsOutputPlugin
                 .set("file_ext", ".csv")
                 .set("formatter", formatterConfig());
 
-        GcsOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
+        PluginTask task = config.loadConfig(PluginTask.class);
         assertEquals("private_key", task.getAuthMethod().toString());
     }
 
@@ -284,7 +283,7 @@ public class TestGcsOutputPlugin
     {
         ConfigSource configSource = config();
         PluginTask task = configSource.loadConfig(PluginTask.class);
-        Method method = GcsOutputPlugin.class.getDeclaredMethod("generateRemotePath", String.class, String.class, int.class, int.class, String.class);
+        Method method = GcsTransactionalFileOutput.class.getDeclaredMethod("generateRemotePath", String.class, String.class, int.class, int.class, String.class);
         method.setAccessible(true);
         assertEquals("sample.000.01.csv", method.invoke(plugin, "/sample", task.getSequenceFormat(), 0, 1, ".csv"));
         assertEquals("sample.000.01.csv", method.invoke(plugin, "./sample", task.getSequenceFormat(), 0, 1, ".csv"));

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-gcs
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-29 00:00:00.000000000 Z
+date: 2018-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,24 +62,27 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/output/gcs.rb
+- src/main/java/org/embulk/output/AuthMethod.java
 - src/main/java/org/embulk/output/GcsAuthentication.java
 - src/main/java/org/embulk/output/GcsOutputPlugin.java
+- src/main/java/org/embulk/output/GcsTransactionalFileOutput.java
+- src/main/java/org/embulk/output/PluginTask.java
 - src/test/java/org/embulk/output/TestGcsAuthentication.java
 - src/test/java/org/embulk/output/TestGcsOutputPlugin.java
 - src/test/resources/keys.tar.enc
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
-- classpath/commons-codec-1.3.jar
-- classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-gcs-0.4.1.jar
-- classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev28-1.19.1.jar
+- classpath/httpclient-4.0.1.jar
+- classpath/jsr305-1.3.9.jar
+- classpath/embulk-output-gcs-0.4.2.jar
+- classpath/commons-logging-1.1.1.jar
 - classpath/google-http-client-1.19.0.jar
+- classpath/google-api-client-1.19.1.jar
+- classpath/commons-codec-1.3.jar
+- classpath/httpcore-4.0.1.jar
 - classpath/google-http-client-jackson2-1.19.0.jar
 - classpath/google-oauth-client-1.19.0.jar
-- classpath/httpclient-4.0.1.jar
-- classpath/httpcore-4.0.1.jar
-- classpath/jsr305-1.3.9.jar
 homepage: https://github.com/hakobera/embulk-output-gcs
 licenses:
 - MIT