embulk-input-azure_blob_storage 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f1caa223dfe66430c7a479aceb4a34859bbfcac7
-  data.tar.gz: 911394ff73af6685d62fcfd560736b1669383e0b
+  metadata.gz: 023f5fb334a7684e46155d3a3538168ca6c10d6c
+  data.tar.gz: 2a7ed41ce1b0f56bca0a3134add627295d5d359d
 SHA512:
-  metadata.gz: 6b52d4764546a1a083a340c1fe0de7ac1b2674c5ea03758c7eb94d3704165564aa0dfb8b23ec4ac4d592c333031dff180e990419fbd3d15813e8cedd169d1e82
-  data.tar.gz: 1cf0d786da368c7c195b79a3f2909bf8092b3f32aaad70fd28deb29de47833a60217daa92c3db26745373f0742e13bed5a2a3650ad63342cc15e3d4f2fe902a5
+  metadata.gz: fa524db4147ff03b7fe7ae509c8096c0cb1b191239a2959b9e969217ec48398bb552e588fe2b05f4f3610cbab0934dd87b14eae288f3ca0ff445959134e7879a
+  data.tar.gz: af78a9bc48198e812795189134bcaf449fd246d7ee90fc2f6e9ac9e70bfa1ddbf4192dd9f7863545b22a522591de91e0f67881a9a2dee65d2fed947d498f25b1

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.0 - 2018-12-26
+* [new feature] Use file name output feature in Embulk core to show file name instead of logging [#17](https://github.com/sakama/embulk-input-azure_blob_storage/pull/17)
+
 ## 0.1.8 - 2018-10-18
 * [maintenance] Update dependencies [#15](https://github.com/sakama/embulk-input-azure_blob_storage/pull/15)
 

data/README.md

@@ -3,6 +3,8 @@
 
 [Embulk](http://www.embulk.org/) file input plugin read files stored on [Microsoft Azure](https://azure.microsoft.com/) [Blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
 
+embulk-input-azure_blog_storage v0.2.0+ requires Embulk v0.9.12+
+
 ## Overview
 
 * **Plugin type**: file input

data/build.gradle

@@ -17,17 +17,17 @@ configurations {
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-version = "0.1.8"
+version = "0.2.0"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.9.8"
-    provided "org.embulk:embulk-core:0.9.8"
+    compile  "org.embulk:embulk-core:0.9.12"
+    provided "org.embulk:embulk-core:0.9.12"
 
     compile "com.microsoft.azure:azure-storage:8.0.0"
 
     testCompile "junit:junit:4.12"
-    testCompile "org.embulk:embulk-core:0.9.8:tests"
-    testCompile "org.embulk:embulk-standards:0.9.8"
+    testCompile "org.embulk:embulk-core:0.9.12:tests"
+    testCompile "org.embulk:embulk-standards:0.9.12"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java

@@ -1,7 +1,6 @@
 package org.embulk.input.azure_blob_storage;
 
 import com.google.common.base.Charsets;
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.io.BaseEncoding;
 import com.microsoft.azure.storage.CloudStorageAccount;
@@ -27,17 +26,18 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.InputStreamFileInput.InputStreamWithHints;
 import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.net.URISyntaxException;
 import java.security.InvalidKeyException;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
 
 public class AzureBlobStorageFileInputPlugin
         implements FileInputPlugin
@@ -260,7 +260,7 @@ public class AzureBlobStorageFileInputPlugin
         }
 
         @Override
-        public InputStream openNext() throws IOException
+        public InputStreamWithHints openNextWithHints()
         {
             if (opened || !iterator.hasNext()) {
                 return null;
@@ -272,13 +272,16 @@ public class AzureBlobStorageFileInputPlugin
                         .withRetryLimit(maxConnectionRetry)
                         .withInitialRetryWait(500)
                         .withMaxRetryWait(30 * 1000)
-                        .runInterruptible(new Retryable<InputStream>() {
+                        .runInterruptible(new Retryable<InputStreamWithHints>() {
                             @Override
-                            public InputStream call() throws StorageException, URISyntaxException, IOException
+                            public InputStreamWithHints call() throws StorageException, URISyntaxException, IOException
                             {
                                 CloudBlobContainer container = client.getContainerReference(containerName);
                                 CloudBlob blob = container.getBlockBlobReference(key);
-                                return blob.openInputStream();
+                                return new InputStreamWithHints(
+                                        blob.openInputStream(),
+                                        String.format("%s/%s", containerName, key)
+                                );
                             }
 
                             @Override

data/src/main/java/org/embulk/input/azure_blob_storage/FileList.java

@@ -3,7 +3,6 @@ package org.embulk.input.azure_blob_storage;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -24,6 +23,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
@@ -182,7 +182,7 @@ public class FileList
             catch (IOException ex) {
                 throw Throwables.propagate(ex);
             }
-            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.ofNullable(last));
         }
 
         private List<List<Entry>> getSplits(List<Entry> all)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-18 00:00:00.000000000 Z
+date: 2018-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,9 +65,9 @@ files:
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
 - classpath/azure-storage-8.0.0.jar
+- classpath/embulk-input-azure_blob_storage-0.2.0.jar
 - classpath/guava-20.0.jar
 - classpath/azure-keyvault-core-1.0.0.jar
-- classpath/embulk-input-azure_blob_storage-0.1.8.jar
 - classpath/jackson-core-2.9.4.jar
 homepage: https://github.com/embulk/embulk-input-azure_blob_storage
 licenses: