embulk-input-azure_blob_storage 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a85c5e311d426a1f67c47b4947683827a3bb2140
-  data.tar.gz: 2ae6c9a16e143e4731e4de7518fa504d2cb2f8c1
+  metadata.gz: 49b71df05579f8c392ec70089aaeca95f1b81144
+  data.tar.gz: a63ffb1663b9bbed5827211dbb62eeb04b20a987
 SHA512:
-  metadata.gz: 68588df49d434087541ea542a4731236d5496356df5b66bdb81f620803c305f76c33b39ff7340d176378f80c0f41e0275efedc95b180198828776218a3148f55
-  data.tar.gz: 7cc8c4758c67360d7cbbd5cf4ca12bfe8f8b975641fcc31fa2b3308e4ba3c8093e7aa0edf577e078d9f3441e8636bae700dc07fbb16c0b181ec71742b3ae963c
+  metadata.gz: 1a43a45c850746c1a50b9f12be823f0081ea9deddd4403e831f2533e6af4ab4861405e45cc62824fcb637b6509178fceeb6cad4f583542677e0ee3383beb8e43
+  data.tar.gz: d5e6066c047f60034d50ff6084919157476f962adf5c9f2b1a59685a97d83a254637ed0037276629714e2b1223c229c0564b58c162cfffbb58b10a4822f563aa

data/CHANGELOG.md

@@ -1,6 +1,11 @@
+## 0.1.4 - 2015-03-22
+
+* [new feature] Support `last_path` option [#7](https://github.com/sakama/embulk-input-azure_blob_storage/pull/7)
+* [new feature] Support `path_match_pattern` option [#6](https://github.com/sakama/embulk-input-azure_blob_storage/pull/6)
+
 ## 0.1.3 - 2015-03-16
 
-* [maintenance] Add unit test[#4](https://github.com/sakama/embulk-input-azure_blob_storage/pull/4)
+* [maintenance] Add unit test [#4](https://github.com/sakama/embulk-input-azure_blob_storage/pull/4)
 * [maintenance] Add retry logic [#3](https://github.com/sakama/embulk-input-azure_blob_storage/pull/3)
 
 ## 0.1.2 - 2015-10-11

data/README.md

@@ -16,6 +16,8 @@ First, create Azure [Storage Account](https://azure.microsoft.com/en-us/document
 - **account_key**: primary access key (string, required)
 - **container**: container name data stored (string, required)
 - **path_prefix**: prefix of target keys (string, required) (string, required)
+- **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)
+- **total_file_count_limit**: maximum number of files to read (integer, optional)
 
 ## Example
 
@@ -55,6 +57,21 @@ in:
 out: {type: stdout}
 ```
 
+To filter files using regexp:
+
+```yaml
+in:
+  type: sftp
+  path_prefix: logs/csv-
+  ...
+  path_match_pattern: \.csv$   # a file will be skipped if its path doesn't match with this pattern
+
+  ## some examples of regexp:
+  #path_match_pattern: /archive/         # match files in .../archive/... directory
+  #path_match_pattern: /data1/|/data2/   # match files in .../data1/... or .../data2/... directory
+  #path_match_pattern: .csv$|.csv.gz$    # match files whose suffix is .csv or .csv.gz
+```
+
 ## Build
 
 ```

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.3"
+version = "0.1.4"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java

@@ -1,10 +1,12 @@
 package org.embulk.input.azure_blob_storage;
 
+import com.google.common.base.Charsets;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
+import com.google.common.io.BaseEncoding;
 import com.microsoft.azure.storage.CloudStorageAccount;
 import com.microsoft.azure.storage.ResultContinuation;
+import com.microsoft.azure.storage.ResultContinuationType;
 import com.microsoft.azure.storage.ResultSegment;
 import com.microsoft.azure.storage.StorageException;
 import com.microsoft.azure.storage.blob.CloudBlob;
@@ -31,15 +33,14 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URISyntaxException;
 import java.security.InvalidKeyException;
-import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 
 public class AzureBlobStorageFileInputPlugin
         implements FileInputPlugin
 {
     public interface PluginTask
-            extends Task
+            extends Task, FileList.Task
     {
         @Config("account_name")
         String getAccountName();
@@ -65,9 +66,8 @@ public class AzureBlobStorageFileInputPlugin
         @ConfigDefault("5") // 5 times retry to connect sftp server if failed.
         int getMaxConnectionRetry();
 
-        List<String> getFiles();
-
-        void setFiles(List<String> files);
+        FileList getFiles();
+        void setFiles(FileList files);
 
         @ConfigInject
         BufferAllocator getBufferAllocator();
@@ -83,28 +83,18 @@ public class AzureBlobStorageFileInputPlugin
         CloudBlobClient blobClient = newAzureClient(task.getAccountName(), task.getAccountKey());
         task.setFiles(listFiles(blobClient, task));
 
-        return resume(task.dump(), task.getFiles().size(), control);
+        return resume(task.dump(), task.getFiles().getTaskCount(), control);
     }
 
     @Override
     public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
-
         control.run(taskSource, taskCount);
 
         ConfigDiff configDiff = Exec.newConfigDiff();
+        configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
 
-        List<String> files = new ArrayList<>(task.getFiles());
-        if (files.isEmpty()) {
-            if (task.getLastPath().isPresent()) {
-                configDiff.set("last_path", task.getLastPath().get());
-            }
-        }
-        else {
-            Collections.sort(files);
-            configDiff.set("last_path", files.get(files.size() - 1));
-        }
         return configDiff;
     }
 
@@ -129,22 +119,28 @@ public class AzureBlobStorageFileInputPlugin
         return account.createCloudBlobClient();
     }
 
-    private List<String> listFiles(CloudBlobClient client, PluginTask task)
+    private FileList listFiles(CloudBlobClient client, PluginTask task)
     {
         if (task.getPathPrefix().equals("/")) {
             log.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
         }
+        FileList.Builder builder = new FileList.Builder(task);
 
-        return listFilesWithPrefix(client, task.getContainer(), task.getPathPrefix(), task.getLastPath(), task.getMaxResults());
+        return listFilesWithPrefix(builder, client, task.getContainer(), task.getPathPrefix(), task.getLastPath(), task.getMaxResults());
     }
 
-    private static List<String> listFilesWithPrefix(CloudBlobClient client, String containerName,
+    private static FileList listFilesWithPrefix(FileList.Builder builder, CloudBlobClient client, String containerName,
                                              String prefix, Optional<String> lastPath, int maxResults)
     {
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-        // It seems I can't cast lastKey<String> to token<ResultContinuation> by Azure SDK for Java
-        String lastKey = lastPath.orNull();
+        String lastKey = (lastPath.isPresent() && !lastPath.get().isEmpty()) ? createNextToken(lastPath.get()) : null;
         ResultContinuation token = null;
+        if (lastKey != null) {
+            token = new ResultContinuation();
+            token.setContinuationType(ResultContinuationType.BLOB);
+            log.debug("lastPath: {}", lastPath.get());
+            log.debug("lastPath(Base64encoded): {}", lastKey);
+            token.setNextMarker(lastKey);
+        }
 
         try {
             CloudBlobContainer container = client.getContainerReference(containerName);
@@ -156,7 +152,7 @@ public class AzureBlobStorageFileInputPlugin
                     if (blobItem instanceof CloudBlob) {
                         CloudBlob blob = (CloudBlob) blobItem;
                         if (blob.exists() && !blob.getUri().toString().endsWith("/")) {
-                            builder.add(blob.getName());
+                            builder.add(blob.getName(), blob.getProperties().getLength());
                             log.debug(String.format("name:%s, class:%s, uri:%s", blob.getName(), blob.getClass(), blob.getUri()));
                         }
                     }
@@ -201,7 +197,7 @@ public class AzureBlobStorageFileInputPlugin
     {
         private CloudBlobClient client;
         private final String containerName;
-        private final String key;
+        private final Iterator<String> iterator;
         private final int maxConnectionRetry;
         private boolean opened = false;
 
@@ -209,14 +205,14 @@ public class AzureBlobStorageFileInputPlugin
         {
             this.client = newAzureClient(task.getAccountName(), task.getAccountKey());
             this.containerName = task.getContainer();
-            this.key = task.getFiles().get(taskIndex);
+            this.iterator = task.getFiles().get(taskIndex).iterator();
             this.maxConnectionRetry = task.getMaxConnectionRetry();
         }
 
         @Override
         public InputStream openNext() throws IOException
         {
-            if (opened) {
+            if (opened || !iterator.hasNext()) {
                 return null;
             }
             opened = true;
@@ -225,7 +221,7 @@ public class AzureBlobStorageFileInputPlugin
             while (true) {
                 try {
                     CloudBlobContainer container = client.getContainerReference(containerName);
-                    CloudBlob blob = container.getBlockBlobReference(key);
+                    CloudBlob blob = container.getBlockBlobReference(iterator.next());
                     return blob.openInputStream();
                 }
                 catch (StorageException | URISyntaxException ex) {
@@ -250,4 +246,23 @@ public class AzureBlobStorageFileInputPlugin
         @Override
         public void close() {}
     }
+
+    private static String createNextToken(String path)
+    {
+        StringBuilder sb = new StringBuilder()
+                .append(String.format("%06d", path.length()))
+                .append("!")
+                .append(path)
+                .append("!000028!9999-12-31T23:59:59.9999999Z!");
+
+        String encodedString = BaseEncoding.base64().encode(sb.toString().getBytes(Charsets.UTF_8));
+
+        StringBuilder marker = new StringBuilder()
+                .append("2")
+                .append("!")
+                .append(encodedString.length())
+                .append("!")
+                .append(encodedString);
+        return marker.toString();
+    }
 }

data/src/main/java/org/embulk/input/azure_blob_storage/FileList.java

@@ -0,0 +1,341 @@
+package org.embulk.input.azure_blob_storage;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+// this class should be moved to embulk-core
+public class FileList
+{
+    public interface Task
+    {
+        @Config("path_match_pattern")
+        @ConfigDefault("\".*\"")
+        String getPathMatchPattern();
+
+        @Config("total_file_count_limit")
+        @ConfigDefault("2147483647")
+        int getTotalFileCountLimit();
+
+        // TODO support more algorithms to combine tasks
+        @Config("min_task_size")
+        @ConfigDefault("0")
+        long getMinTaskSize();
+    }
+
+    public static class Entry
+    {
+        private int index;
+        private long size;
+
+        @JsonCreator
+        public Entry(
+                @JsonProperty("index") int index,
+                @JsonProperty("size") long size)
+        {
+            this.index = index;
+            this.size = size;
+        }
+
+        @JsonProperty("index")
+        public int getIndex()
+        {
+            return index;
+        }
+
+        @JsonProperty("size")
+        public long getSize()
+        {
+            return size;
+        }
+    }
+
+    public static class Builder
+    {
+        private final Logger log = Exec.getLogger(FileList.class);
+        private final ByteArrayOutputStream binary;
+        private final OutputStream stream;
+        private final List<Entry> entries = new ArrayList<>();
+        private String last = null;
+
+        private int limitCount = Integer.MAX_VALUE;
+        private long minTaskSize = 1;
+        private Pattern pathMatchPattern;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public Builder(Task task)
+        {
+            this();
+            this.pathMatchPattern = Pattern.compile(task.getPathMatchPattern());
+            this.limitCount = task.getTotalFileCountLimit();
+            this.minTaskSize = task.getMinTaskSize();
+        }
+
+        public Builder(ConfigSource config)
+        {
+            this();
+            this.pathMatchPattern = Pattern.compile(config.get(String.class, "path_match_pattern", ".*"));
+            this.limitCount = config.get(int.class, "total_file_count_limit", Integer.MAX_VALUE);
+            this.minTaskSize = config.get(long.class, "min_task_size", 0L);
+        }
+
+        public Builder()
+        {
+            binary = new ByteArrayOutputStream();
+            try {
+                stream = new BufferedOutputStream(new GZIPOutputStream(binary));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        public Builder limitTotalFileCount(int limitCount)
+        {
+            this.limitCount = limitCount;
+            return this;
+        }
+
+        public Builder minTaskSize(long bytes)
+        {
+            this.minTaskSize = bytes;
+            return this;
+        }
+
+        public Builder pathMatchPattern(String pattern)
+        {
+            this.pathMatchPattern = Pattern.compile(pattern);
+            return this;
+        }
+
+        public int size()
+        {
+            return entries.size();
+        }
+
+        public boolean needsMore()
+        {
+            return size() < limitCount;
+        }
+
+        // returns true if this file is used
+        public synchronized boolean add(String path, long size)
+        {
+            // TODO throw IllegalStateException if stream is already closed
+
+            if (!needsMore()) {
+                return false;
+            }
+
+            if (!pathMatchPattern.matcher(path).find()) {
+                return false;
+            }
+
+            int index = entries.size();
+            entries.add(new Entry(index, size));
+            log.info("add file to the request list: {}", path);
+
+            byte[] data = path.getBytes(StandardCharsets.UTF_8);
+            castBuffer.putInt(0, data.length);
+            try {
+                stream.write(castBuffer.array());
+                stream.write(data);
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+
+            last = path;
+            return true;
+        }
+
+        public FileList build()
+        {
+            try {
+                stream.close();
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+        }
+
+        private List<List<Entry>> getSplits(List<Entry> all)
+        {
+            List<List<Entry>> tasks = new ArrayList<>();
+            long currentTaskSize = 0;
+            List<Entry> currentTask = new ArrayList<>();
+            for (Entry entry : all) {
+                currentTask.add(entry);
+                currentTaskSize += entry.getSize();  // TODO consider to multiply the size by cost_per_byte, and add cost_per_file
+                if (currentTaskSize >= minTaskSize) {
+                    tasks.add(currentTask);
+                    currentTask = new ArrayList<>();
+                    currentTaskSize = 0;
+                }
+            }
+            if (!currentTask.isEmpty()) {
+                tasks.add(currentTask);
+            }
+            return tasks;
+        }
+    }
+
+    private final byte[] data;
+    private final List<List<Entry>> tasks;
+    private final Optional<String> last;
+
+    @JsonCreator
+    @Deprecated
+    public FileList(
+            @JsonProperty("data") byte[] data,
+            @JsonProperty("tasks") List<List<Entry>> tasks,
+            @JsonProperty("last") Optional<String> last)
+    {
+        this.data = data;
+        this.tasks = tasks;
+        this.last = last;
+    }
+
+    @JsonIgnore
+    public Optional<String> getLastPath(Optional<String> lastLastPath)
+    {
+        if (last.isPresent()) {
+            return last;
+        }
+        return lastLastPath;
+    }
+
+    @JsonIgnore
+    public int getTaskCount()
+    {
+        return tasks.size();
+    }
+
+    @JsonIgnore
+    public List<String> get(int i)
+    {
+        return new EntryList(data, tasks.get(i));
+    }
+
+    @JsonProperty("data")
+    @Deprecated
+    public byte[] getData()
+    {
+        return data;
+    }
+
+    @JsonProperty("tasks")
+    @Deprecated
+    public List<List<Entry>> getTasks()
+    {
+        return tasks;
+    }
+
+    @JsonProperty("last")
+    @Deprecated
+    public Optional<String> getLast()
+    {
+        return last;
+    }
+
+    private class EntryList
+            extends AbstractList<String>
+    {
+        private final byte[] data;
+        private final List<Entry> entries;
+        private InputStream stream;
+        private int current;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public EntryList(byte[] data, List<Entry> entries)
+        {
+            this.data = data;
+            this.entries = entries;
+            try {
+                this.stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            this.current = 0;
+        }
+
+        @Override
+        public synchronized String get(int i)
+        {
+            Entry e = entries.get(i);
+            if (e.getIndex() < current) {
+                // rewind to the head
+                try {
+                    stream.close();
+                    stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+                }
+                catch (IOException ex) {
+                    throw Throwables.propagate(ex);
+                }
+                current = 0;
+            }
+
+            while (current < e.getIndex()) {
+                readNext();
+            }
+            // now current == e.getIndex()
+            return readNextString();
+        }
+
+        @Override
+        public int size()
+        {
+            return entries.size();
+        }
+
+        private byte[] readNext()
+        {
+            try {
+                stream.read(castBuffer.array());
+                int n = castBuffer.getInt(0);
+                byte[] b = new byte[n];  // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
+                stream.read(b);
+
+                current++;
+
+                return b;
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        private String readNextString()
+        {
+            return new String(readNext(), StandardCharsets.UTF_8);
+        }
+    }
+}

data/src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java

@@ -93,18 +93,6 @@ public class TestAzureBlobStorageFileInputPlugin
         assertEquals(5, task.getMaxConnectionRetry());
     }
 
-    public ConfigSource config()
-    {
-        return Exec.newConfigSource()
-                .set("account_name", AZURE_ACCOUNT_NAME)
-                .set("account_key", AZURE_ACCOUNT_KEY)
-                .set("container", AZURE_CONTAINER)
-                .set("path_prefix", AZURE_PATH_PREFIX)
-                .set("last_path", "")
-                .set("file_ext", ".csv")
-                .set("parser", parserConfig(schemaConfig()));
-    }
-
     @Test(expected = ConfigException.class)
     public void checkDefaultValuesAccountNameIsNull()
     {
@@ -114,7 +102,6 @@ public class TestAzureBlobStorageFileInputPlugin
                 .set("container", AZURE_CONTAINER)
                 .set("path_prefix", AZURE_PATH_PREFIX)
                 .set("last_path", "")
-                .set("file_ext", ".csv")
                 .set("parser", parserConfig(schemaConfig()));
 
         runner.transaction(config, new Control());
@@ -129,7 +116,6 @@ public class TestAzureBlobStorageFileInputPlugin
                 .set("container", AZURE_CONTAINER)
                 .set("path_prefix", AZURE_PATH_PREFIX)
                 .set("last_path", "")
-                .set("file_ext", ".csv")
                 .set("parser", parserConfig(schemaConfig()));
 
         runner.transaction(config, new Control());
@@ -144,7 +130,6 @@ public class TestAzureBlobStorageFileInputPlugin
                 .set("container", null)
                 .set("path_prefix", AZURE_PATH_PREFIX)
                 .set("last_path", "")
-                .set("file_ext", ".csv")
                 .set("parser", parserConfig(schemaConfig()));
 
         runner.transaction(config, new Control());
@@ -166,7 +151,7 @@ public class TestAzureBlobStorageFileInputPlugin
     public void testResume()
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-        task.setFiles(Arrays.asList("in/aa/a"));
+        task.setFiles(createFileList(Arrays.asList("in/aa/a"), task));
         ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
         {
             @Override
@@ -190,8 +175,8 @@ public class TestAzureBlobStorageFileInputPlugin
             throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
     {
         List<String> expected = Arrays.asList(
-                AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_01.csv",
-                AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_02.csv"
+            AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_01.csv",
+            AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_02.csv"
         );
 
         PluginTask task = config.loadConfig(PluginTask.class);
@@ -210,8 +195,9 @@ public class TestAzureBlobStorageFileInputPlugin
 
         Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", CloudBlobClient.class, PluginTask.class);
         listFiles.setAccessible(true);
-        List<String> actual = (List<String>) listFiles.invoke(plugin, client, task);
-        assertEquals(expected, actual);
+        FileList actual = (FileList) listFiles.invoke(plugin, client, task);
+        assertEquals(expected.get(0), actual.get(0).get(0));
+        assertEquals(expected.get(1), actual.get(1).get(0));
         assertEquals(AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_02.csv", configDiff.get(String.class, "last_path"));
     }
 
@@ -228,11 +214,26 @@ public class TestAzureBlobStorageFileInputPlugin
 
         Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", CloudBlobClient.class, PluginTask.class);
         listFiles.setAccessible(true);
-        task.setFiles((List<String>) listFiles.invoke(plugin, client, task));
+        task.setFiles((FileList) listFiles.invoke(plugin, client, task));
 
         assertRecords(config, output);
     }
 
+    @Test
+    public void testCreateNextToken() throws Exception
+    {
+        Method base64Encode = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("createNextToken", String.class);
+        base64Encode.setAccessible(true);
+
+        String expected = "2!92!MDAwMDI1IXJlYWRvbmx5L3NhbXBsZV8wMS50c3YuZ3ohMDAwMDI4ITk5OTktMTItMzFUMjM6NTk6NTkuOTk5OTk5OVoh";
+        String lastPath = "readonly/sample_01.tsv.gz";
+        assertEquals(expected, base64Encode.invoke(plugin, lastPath).toString());
+
+        expected = "2!120!MDAwMDQ2IXBhdGgvdGhhdC9oYXZlL2xvbmcvcGF0aC9uYW1lL3NhbXBsZV8wMS50c3YuZ3ohMDAwMDI4ITk5OTktMTItMzFUMjM6NTk6NTkuOTk5OTk5OVoh";
+        lastPath = "path/that/have/long/path/name/sample_01.tsv.gz";
+        assertEquals(expected, base64Encode.invoke(plugin, lastPath).toString());
+    }
+
     static List<TaskReport> emptyTaskReports(int taskCount)
     {
         ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
@@ -256,6 +257,17 @@ public class TestAzureBlobStorageFileInputPlugin
         }
     }
 
+    public ConfigSource config()
+    {
+        return Exec.newConfigSource()
+                .set("account_name", AZURE_ACCOUNT_NAME)
+                .set("account_key", AZURE_ACCOUNT_KEY)
+                .set("container", AZURE_CONTAINER)
+                .set("path_prefix", AZURE_PATH_PREFIX)
+                .set("last_path", "")
+                .set("parser", parserConfig(schemaConfig()));
+    }
+
     private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
     {
         ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
@@ -340,4 +352,13 @@ public class TestAzureBlobStorageFileInputPlugin
         }
         return dir;
     }
+
+    private FileList createFileList(List<String> fileList, PluginTask task)
+    {
+        FileList.Builder builder = new FileList.Builder(task);
+        for (String file : fileList) {
+            builder.add(file, 0);
+        }
+        return builder.build();
+    }
 }

data/src/test/java/org/embulk/input/azure_blob_storage/TestFileList.java

@@ -0,0 +1,87 @@
+package org.embulk.input.azure_blob_storage;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestFileList
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+
+    @Before
+    public void createConfigSource()
+    {
+        config = runtime.getExec().newConfigSource();
+    }
+
+    @Test
+    public void checkMinTaskSize()
+            throws Exception
+    {
+        { // not specify min_task_size
+            FileList fileList = newFileList(config.deepCopy(),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(3, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(1).get(0));
+            assertEquals("sample_02", fileList.get(2).get(0));
+        }
+
+        {
+            FileList fileList = newFileList(config.deepCopy().set("min_task_size", 100),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(3, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(1).get(0));
+            assertEquals("sample_02", fileList.get(2).get(0));
+        }
+
+        {
+            FileList fileList = newFileList(config.deepCopy().set("min_task_size", 200),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(2, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(0).get(1));
+            assertEquals("sample_02", fileList.get(1).get(0));
+        }
+
+        {
+            FileList fileList = newFileList(config.deepCopy().set("min_task_size", 700),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(1, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(0).get(1));
+            assertEquals("sample_02", fileList.get(0).get(2));
+        }
+    }
+
+    private static FileList newFileList(ConfigSource config, Object... nameAndSize)
+    {
+        FileList.Builder builder = new FileList.Builder(config);
+
+        for (int i = 0; i < nameAndSize.length; i += 2) {
+            builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]);
+        }
+
+        return builder.build();
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-16 00:00:00.000000000 Z
+date: 2016-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -57,12 +57,14 @@ files:
 - gradlew.bat
 - lib/embulk/input/azure_blob_storage.rb
 - src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java
+- src/main/java/org/embulk/input/azure_blob_storage/FileList.java
 - src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java
+- src/test/java/org/embulk/input/azure_blob_storage/TestFileList.java
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
 - classpath/azure-storage-4.0.0.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-input-azure_blob_storage-0.1.3.jar
+- classpath/embulk-input-azure_blob_storage-0.1.4.jar
 - classpath/jackson-core-2.6.0.jar
 homepage: https://github.com/sakama/embulk-input-azure_blob_storage
 licenses: