embulk-output-azure_blob_storage 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c606ebbfa50cbbc803f2ff2f47d6f4dd445eb9d3
-  data.tar.gz: e36fc67a7ce53216367c0c5fb8e0c1aa629fb8d4
+  metadata.gz: 780301179e35e336690948c80eb1f0be40969fda
+  data.tar.gz: 0820d6fe2606320543583146126c0df179ff8dcf
 SHA512:
-  metadata.gz: beb7e8ec55ce584f88de927aa6fedf75dc62513636e71d60604b1440d525f04e93b196c95518e1f1d132c1a42d0bdd52090f30facb6a59683b7ce45ece2ac2ce
-  data.tar.gz: b45f0a27f6024c703c3297962dcb7c8957588df0be1f8a8ba27ac17e57385518bd1aa387bbf9e5e560dfd977f1c91af230ea3e4b76019720281881eff13a3cef
+  metadata.gz: 77f5049f878c6a82809226ebe49873ec31e6a16d475dd7991b9a729d3348be98885052bfa83c5bf97777bac053014532dec2c028e9add5b2f21d3729abe919f0
+  data.tar.gz: 75dee87d6947f777e5a8e0211bbbb8dc9922bd6cbb7784a2853bb4a9f907f66ce05e572bb431e2d34a7ffe23c096b24b04a30c40d0075007b3f301f5e3b5f8b5

data/.travis.yml

@@ -0,0 +1,5 @@
+language: java
+jdk:
+- oraclejdk7
+script:
+- ./gradlew --info check jacocoTestReport

data/README.md

@@ -50,3 +50,56 @@ out:
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
+
+
+## Test
+
+```
+$ ./gradlew test  # -t to watch change of files and rebuild continuously
+```
+
+To run unit tests, we need to configure the following environment variables.
+
+Additionally, following files will be needed to upload to existing GCS bucket.
+* [sample_01.csv](./src/test/resources/sample_01.csv)
+* [sample_02.csv](./src/test/resources/sample_02.csv)
+
+When environment variables are not set, skip some test cases.
+
+```
+AZURE_ACCOUNT_NAME
+AZURE_ACCOUNT_KEY
+AZURE_CONTAINER
+AZURE_CONTAINER_DIRECTORY (optional, if needed)
+```
+
+If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
+```
+$ vi ~/Library/LaunchAgents/environment.plist
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>my.startup</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>sh</string>
+    <string>-c</string>
+    <string>
+      launchctl setenv AZURE_ACCOUNT_NAME my-account-name
+      launchctl setenv AZURE_ACCOUNT_KEY my-account-key
+      launchctl setenv AZURE_CONTAINER my-container
+      launchctl setenv AZURE_CONTAINER_DIRECTORY unittests
+    </string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+</dict>
+</plist>
+
+$ launchctl load ~/Library/LaunchAgents/environment.plist
+$ launchctl getenv AZURE_ACCOUNT_NAME //try to get value.
+
+Then start your applications.
+```

data/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -15,7 +16,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.0"
+version = "0.1.1"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"
@@ -23,7 +24,9 @@ dependencies {
 
     compile "com.microsoft.azure:azure-storage:4.0.0"
 
-    testCompile "junit:junit:4.+"
+    testCompile "junit:junit:4.12"
+    testCompile "org.embulk:embulk-core:0.7.5:tests"
+    testCompile "org.embulk:embulk-standards:0.7.5"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/embulk-output-azure_blob_storage.gemspec

@@ -0,0 +1,18 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-azure_blob_storage"
+  spec.version       = "0.1.1"
+  spec.authors       = ["Satoshi Akama"]
+  spec.summary       = %[Microsoft Azure blob Storage file output plugin for Embulk]
+  spec.description   = %[Stores files on Microsoft Azure blob Storage.]
+  spec.email         = ["satoshiakama@gmail.com"]
+  spec.licenses      = ["Apache-2.0"]
+  spec.homepage      = "https://github.com/sakama/embulk-output-azure_blob_storage"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java

@@ -53,24 +53,22 @@ public class AzureBlobStorageFileOutputPlugin
     }
 
     private static final Logger log = Exec.getLogger(AzureBlobStorageFileOutputPlugin.class);
-    private static CloudBlobClient blobClient;
-    private static CloudBlobContainer container;
 
     @Override
     public ConfigDiff transaction(ConfigSource config, int taskCount,
             FileOutputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-        blobClient = newAzureClient(task.getAccountName(), task.getAccountKey());
-        String containerName = task.getContainer();
 
         try {
-            container = blobClient.getContainerReference(containerName);
+            CloudBlobClient blobClient = newAzureClient(task.getAccountName(), task.getAccountKey());
+            String containerName = task.getContainer();
+            CloudBlobContainer container = blobClient.getContainerReference(containerName);
             if (!container.exists()) {
                 log.info(String.format("container [%s] is not exists and created.", containerName));
                 container.createIfNotExists();
             }
-        } catch (StorageException | URISyntaxException ex) {
+        } catch (StorageException | URISyntaxException | ConfigException ex) {
             Throwables.propagate(ex);
         }
 
@@ -106,94 +104,118 @@ public class AzureBlobStorageFileOutputPlugin
     }
 
     @Override
-    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
-    {
+    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex) {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
+        return new AzureFileOutput(task, taskIndex);
+    }
 
-        final String pathPrefix = task.getPathPrefix();
-        final String sequenceFormat = task.getSequenceFormat();
-        final String pathSuffix = task.getFileNameExtension();
-
-        return new TransactionalFileOutput() {
-            private int fileIndex = 0;
-            private BufferedOutputStream output = null;
-            private File file;
-            private String filePath;
+    public static class AzureFileOutput implements TransactionalFileOutput
+    {
+        private final String pathPrefix;
+        private final String sequenceFormat;
+        private final String pathSuffix;
+        private final CloudBlobClient client;
+        private CloudBlobContainer container = null;
+        private BufferedOutputStream output = null;
+        private int fileIndex;
+        private File file;
+        private String filePath;
+        private int taskIndex;
+
+        public AzureFileOutput(PluginTask task, int taskIndex)
+        {
+            this.taskIndex = taskIndex;
+            this.pathPrefix = task.getPathPrefix();
+            this.sequenceFormat = task.getSequenceFormat();
+            this.pathSuffix = task.getFileNameExtension();
+            this.client = newAzureClient(task.getAccountName(), task.getAccountKey());
+            try {
+                this.container = client.getContainerReference(task.getContainer());
+            } catch (URISyntaxException | StorageException ex) {
+                Throwables.propagate(ex);
+            }
+        }
 
-            public void nextFile()
-            {
-                closeFile();
+        @Override
+        public void nextFile()
+        {
+            closeFile();
 
-                try {
-                    String suffix = pathSuffix;
-                    if (!suffix.startsWith(".")) {
-                        suffix = "." + suffix;
-                    }
-                    filePath = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + suffix;
-                    file = new File(filePath);
-
-                    String parentPath = file.getParent();
-                    File dir = new File(parentPath);
-                    if (!dir.exists()) {
-                        dir.mkdir();
-                    }
-                    log.info(String.format("Writing local file [%s]", filePath));
-                    output = new BufferedOutputStream(new FileOutputStream(filePath));
-                } catch (FileNotFoundException ex) {
-                    throw Throwables.propagate(ex);
+            try {
+                String suffix = pathSuffix;
+                if (!suffix.startsWith(".")) {
+                    suffix = "." + suffix;
                 }
-            }
+                filePath = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + suffix;
+                file = new File(filePath);
 
-            private void closeFile()
-            {
-                if (output != null) {
-                    try {
-                        output.close();
-                    } catch (IOException ex) {
-                        throw Throwables.propagate(ex);
-                    }
+                String parentPath = file.getParent();
+                File dir = new File(parentPath);
+                if (!dir.exists()) {
+                    dir.mkdir();
                 }
+                log.info(String.format("Writing local file [%s]", filePath));
+                output = new BufferedOutputStream(new FileOutputStream(filePath));
+            } catch (FileNotFoundException ex) {
+                throw Throwables.propagate(ex);
             }
+        }
 
-            public void add(Buffer buffer)
-            {
+        private void closeFile()
+        {
+            if (output != null) {
                 try {
-                    output.write(buffer.array(), buffer.offset(), buffer.limit());
+                    output.close();
+                    fileIndex++;
                 } catch (IOException ex) {
                     throw Throwables.propagate(ex);
-                } finally {
-                    buffer.release();
                 }
             }
+        }
 
-            public void finish()
-            {
-                closeFile();
-                if (filePath != null) {
-                    try {
-                        CloudBlockBlob blob = container.getBlockBlobReference(filePath);
-                        log.info(String.format("Upload start [%s]", filePath));
-                        blob.upload(new FileInputStream(file), file.length());
-                        log.info(String.format("Upload completed [%s]", filePath));
-                        file.delete();
-                        log.info(String.format("Delete completed local file [%s]", filePath));
-                    } catch (StorageException | URISyntaxException | IOException ex) {
-                        Throwables.propagate(ex);
-                    }
-                }
+        @Override
+        public void add(Buffer buffer)
+        {
+            try {
+                output.write(buffer.array(), buffer.offset(), buffer.limit());
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            } finally {
+                buffer.release();
             }
+        }
 
-            public void close()
-            {
-                closeFile();
+        @Override
+        public void finish()
+        {
+            closeFile();
+            if (filePath != null) {
+                try {
+                    CloudBlockBlob blob = container.getBlockBlobReference(filePath);
+                    log.info(String.format("Upload start [%s]", filePath));
+                    blob.upload(new FileInputStream(file), file.length());
+                    log.info(String.format("Upload completed [%s]", filePath));
+                    file.delete();
+                    log.info(String.format("Delete completed local file [%s]", filePath));
+                } catch (StorageException | URISyntaxException | IOException ex) {
+                    Throwables.propagate(ex);
+                }
             }
+        }
 
-            public void abort() {}
+        @Override
+        public void close()
+        {
+            closeFile();
+        }
 
-            public TaskReport commit()
-            {
-                return Exec.newTaskReport();
-            }
-        };
+        @Override
+        public void abort() {}
+
+        @Override
+        public TaskReport commit()
+        {
+            return Exec.newTaskReport();
+        }
     }
 }

data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java

@@ -1,5 +1,379 @@
 package org.embulk.output.azure_blob_storage;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Method;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.FileOutputRunner;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.standards.CsvParserPlugin;
+
+import org.junit.BeforeClass;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+import com.microsoft.azure.storage.blob.CloudBlob;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import org.embulk.output.azure_blob_storage.AzureBlobStorageFileOutputPlugin.PluginTask;
+
 public class TestAzureBlobStorageFileOutputPlugin
 {
+    private static String AZURE_ACCOUNT_NAME;
+    private static String AZURE_ACCOUNT_KEY;
+    private static String AZURE_CONTAINER;
+    private static String AZURE_CONTAINER_DIRECTORY;
+    private static String AZURE_PATH_PREFIX;
+    private static String LOCAL_PATH_PREFIX;
+    private FileOutputRunner runner;
+
+    /*
+     * This test case requires environment variables
+     *   AZURE_ACCOUNT_NAME
+     *   AZURE_ACCOUNT_KEY
+     *   AZURE_CONTAINER
+     *   AZURE_CONTAINER_DIRECTORY
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        AZURE_ACCOUNT_NAME = System.getenv("AZURE_ACCOUNT_NAME");
+        AZURE_ACCOUNT_KEY = System.getenv("AZURE_ACCOUNT_KEY");
+        AZURE_CONTAINER = System.getenv("AZURE_CONTAINER");
+        // skip test cases, if environment variables are not set.
+        assumeNotNull(AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY, AZURE_CONTAINER);
+
+        AZURE_CONTAINER_DIRECTORY = System.getenv("AZURE_CONTAINER_DIRECTORY") != null ? getDirectory(System.getenv("AZURE_CONTAINER_DIRECTORY")) : getDirectory("");
+        AZURE_PATH_PREFIX = AZURE_CONTAINER_DIRECTORY + "sample_";
+        LOCAL_PATH_PREFIX = AzureBlobStorageFileOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private AzureBlobStorageFileOutputPlugin plugin;
+
+    @Before
+    public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
+    {
+        plugin = new AzureBlobStorageFileOutputPlugin();
+        runner = new FileOutputRunner(runtime.getInstance(AzureBlobStorageFileOutputPlugin.class));
+    }
+
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "azure_blob_storage")
+                .set("account_name", AZURE_ACCOUNT_NAME)
+                .set("account_key", AZURE_ACCOUNT_KEY)
+                .set("container", AZURE_CONTAINER)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals(AZURE_ACCOUNT_NAME, task.getAccountName().toString());
+    }
+
+    @Test
+    public void testTransaction()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "azure_blob_storage")
+                .set("account_name", AZURE_ACCOUNT_NAME)
+                .set("account_key", AZURE_ACCOUNT_KEY)
+                .set("container", AZURE_CONTAINER)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
+    @Test
+    public void testTransactionCreateNonexistsContainer() throws Exception
+    {
+        String container = "non-exists-container";
+        deleteContainerIfExists(container);
+
+        assertEquals(false, isExistsContainer(container));
+
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "azure_blob_storage")
+                .set("account_name", AZURE_ACCOUNT_NAME)
+                .set("account_key", AZURE_ACCOUNT_KEY)
+                .set("container", container)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+
+        assertEquals(true, isExistsContainer(container));
+        deleteContainerIfExists(container);
+    }
+
+    @Test
+    public void testResume()
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileOutputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource)
+            {
+                return Lists.newArrayList(Exec.newTaskReport());
+            }
+        });
+        //assertEquals("in/aa/a", configDiff.get(String.class, "last_path"));
+    }
+
+    @Test
+    public void testCleanup()
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
+    }
+
+    @Test(expected = RuntimeException.class)
+    public void testCreateAzureClientThrowsConfigException()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "azure_blob_storage")
+                .set("account_name", "invalid-account-name")
+                .set("account_key", AZURE_ACCOUNT_KEY)
+                .set("container", AZURE_CONTAINER)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
+    @Test
+    public void testAzureFileOutputByOpen() throws Exception
+    {
+        ConfigSource configSource = config();
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        runner.transaction(configSource, schema, 0, new Control());
+
+        TransactionalFileOutput output = plugin.open(task.dump(), 0);
+
+        output.nextFile();
+
+        FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
+        byte[] bytes = convertInputStreamToByte(is);
+        Buffer buffer = Buffer.wrap(bytes);
+        output.add(buffer);
+
+        output.finish();
+        output.commit();
+
+        String remotePath = AZURE_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
+        assertRecords(remotePath);
+    }
+
+    public ConfigSource config()
+    {
+        return Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "azure_blob_storage")
+                .set("account_name", AZURE_ACCOUNT_NAME)
+                .set("account_key", AZURE_ACCOUNT_KEY)
+                .set("container", AZURE_CONTAINER)
+                .set("path_prefix", AZURE_PATH_PREFIX)
+                .set("last_path", "")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+    }
+
+    private class Control
+            implements OutputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource)
+        {
+            return Lists.newArrayList(Exec.newTaskReport());
+        }
+    }
+
+    private ImmutableMap<String, Object> inputConfig()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "file");
+        builder.put("path_prefix", LOCAL_PATH_PREFIX);
+        builder.put("last_path", "");
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("newline", "CRLF");
+        builder.put("delimiter", ",");
+        builder.put("quote", "\"");
+        builder.put("escape", "\"");
+        builder.put("trim_if_not_quoted", false);
+        builder.put("skip_header_lines", 1);
+        builder.put("allow_extra_columns", false);
+        builder.put("allow_optional_columns", false);
+        builder.put("columns", schemaConfig);
+        return builder.build();
+    }
+
+    private ImmutableList<Object> schemaConfig()
+    {
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "id", "type", "long"));
+        builder.add(ImmutableMap.of("name", "account", "type", "long"));
+        builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+        builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
+        builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> formatterConfig()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("header_line", "false");
+        builder.put("timezone", "Asia/Tokyo");
+        return builder.build();
+    }
+
+    private void assertRecords(String azurePath) throws Exception
+    {
+        ImmutableList<List<String>> records = getFileContentsFromAzure(azurePath);
+        assertEquals(5, records.size());
+        {
+            List<String> record = records.get(1);
+            assertEquals("1", record.get(0));
+            assertEquals("32864", record.get(1));
+            assertEquals("2015-01-27 19:23:49", record.get(2));
+            assertEquals("20150127", record.get(3));
+            assertEquals("embulk", record.get(4));
+        }
+
+        {
+            List<String> record = records.get(2);
+            assertEquals("2", record.get(0));
+            assertEquals("14824", record.get(1));
+            assertEquals("2015-01-27 19:01:23", record.get(2));
+            assertEquals("20150127", record.get(3));
+            assertEquals("embulk jruby", record.get(4));
+        }
+    }
+
+    private ImmutableList<List<String>> getFileContentsFromAzure(String path) throws Exception
+    {
+        Method method = AzureBlobStorageFileOutputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
+        method.setAccessible(true);
+        CloudBlobClient client = (CloudBlobClient) method.invoke(plugin, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY);
+        CloudBlobContainer container = client.getContainerReference(AZURE_CONTAINER);
+        CloudBlob blob = container.getBlockBlobReference(path);
+
+        ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
+
+        InputStream is =  blob.openInputStream();
+        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+        String line;
+        while ((line = reader.readLine()) != null) {
+            List<String> records = Arrays.asList(line.split(",", 0));
+
+            builder.add(records);
+        }
+        return builder.build();
+    }
+
+    private boolean isExistsContainer(String containerName) throws Exception
+    {
+        Method method = AzureBlobStorageFileOutputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
+        method.setAccessible(true);
+        CloudBlobClient client = (CloudBlobClient) method.invoke(plugin, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY);
+        CloudBlobContainer container = client.getContainerReference(containerName);
+        if (container.exists()) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private void deleteContainerIfExists(String containerName) throws Exception
+    {
+        Method method = AzureBlobStorageFileOutputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
+        method.setAccessible(true);
+        CloudBlobClient client = (CloudBlobClient) method.invoke(plugin, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY);
+        CloudBlobContainer container = client.getContainerReference(containerName);
+        if (container.exists()) {
+            container.delete();
+            // container could not create same name after deletion.
+            Thread.sleep(30000);
+        }
+    }
+
+    private static String getDirectory(String dir)
+    {
+        if (dir != null && !dir.endsWith("/")) {
+            dir = dir + "/";
+        }
+        if (dir.startsWith("/")) {
+            dir = dir.replaceFirst("/", "");
+        }
+        return dir;
+    }
+
+    private byte[] convertInputStreamToByte(InputStream is) throws IOException
+    {
+        ByteArrayOutputStream bo = new ByteArrayOutputStream();
+        byte [] buffer = new byte[1024];
+        while(true) {
+            int len = is.read(buffer);
+            if(len < 0) {
+                break;
+            }
+            bo.write(buffer, 0, len);
+        }
+        return bo.toByteArray();
+    }
 }

data/src/test/resources/sample_01.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/src/test/resources/sample_02.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-11 00:00:00.000000000 Z
+date: 2015-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,8 +46,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - README.md
 - build.gradle
+- embulk-output-azure_blob_storage.gemspec
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -55,9 +57,11 @@ files:
 - lib/embulk/output/azure_blob_storage.rb
 - src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java
 - src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java
+- src/test/resources/sample_01.csv
+- src/test/resources/sample_02.csv
 - classpath/azure-storage-4.0.0.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-output-azure_blob_storage-0.1.0.jar
+- classpath/embulk-output-azure_blob_storage-0.1.1.jar
 - classpath/jackson-core-2.6.0.jar
 homepage: https://github.com/sakama/embulk-output-azure_blob_storage
 licenses: