embulk-output-hdfs 0.2.4 → 0.3.0

data/src/main/java/org/embulk/output/hdfs/compat/ModeCompat.java

@@ -0,0 +1,76 @@
+package org.embulk.output.hdfs.compat;
+
+import com.google.common.base.Optional;
+import org.embulk.config.ConfigException;
+import org.embulk.output.hdfs.ModeTask;
+import org.embulk.output.hdfs.ModeTask.Mode;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import static org.embulk.output.hdfs.HdfsFileOutputPlugin.PluginTask.DeleteInAdvancePolicy;
+import static org.embulk.output.hdfs.ModeTask.Mode.ABORT_IF_EXIST;
+import static org.embulk.output.hdfs.ModeTask.Mode.DELETE_FILES_IN_ADVANCE;
+import static org.embulk.output.hdfs.ModeTask.Mode.DELETE_RECURSIVE_IN_ADVANCE;
+import static org.embulk.output.hdfs.ModeTask.Mode.OVERWRITE;
+
+@Deprecated
+public class ModeCompat
+{
+    private static final Logger logger = Exec.getLogger(ModeCompat.class);
+
+    private ModeCompat()
+    {
+    }
+
+    @Deprecated
+    public static Mode getMode(
+            ModeTask task,
+            Optional<Boolean> overwrite,
+            Optional<DeleteInAdvancePolicy> deleteInAdvancePolicy)
+    {
+        if (!overwrite.isPresent() && !deleteInAdvancePolicy.isPresent()) {
+            return task.getMode();
+        }
+
+        // Display Deprecated Messages
+        if (overwrite.isPresent()) {
+            logger.warn("`overwrite` option is Deprecated. Please use `mode` option instead.");
+        }
+        if (deleteInAdvancePolicy.isPresent()) {
+            logger.warn("`delete_in_advance` is Deprecated. Please use `mode` option instead.");
+        }
+        if (!task.getMode().isDefaultMode()) {
+            String msg = "`mode` option cannot be used with `overwrite` option or `delete_in_advance` option.";
+            logger.error(msg);
+            throw new ConfigException(msg);
+        }
+
+        // Select Mode for Compatibility
+        if (!deleteInAdvancePolicy.isPresent()) {
+            if (overwrite.get()) {
+                logger.warn(String.format("Select `mode: %s` for compatibility", OVERWRITE.toString()));
+                return OVERWRITE;
+            }
+            logger.warn(String.format("Select `mode: %s` for compatibility", ABORT_IF_EXIST.toString()));
+            return ABORT_IF_EXIST;
+        }
+
+        switch (deleteInAdvancePolicy.get()) {  // deleteInAdvancePolicy is always present.
+            case NONE:
+                if (overwrite.isPresent() && overwrite.get()) {
+                    logger.warn(String.format("Select `mode: %s` for compatibility", OVERWRITE.toString()));
+                    return OVERWRITE;
+                }
+                logger.warn(String.format("Select `mode: %s` for compatibility", ABORT_IF_EXIST.toString()));
+                return ABORT_IF_EXIST;
+            case FILE_ONLY:
+                logger.warn(String.format("Select `mode: %s` for compatibility", DELETE_FILES_IN_ADVANCE.toString()));
+                return DELETE_FILES_IN_ADVANCE;
+            case RECURSIVE:
+                logger.warn(String.format("Select `mode: %s` for compatibility", DELETE_RECURSIVE_IN_ADVANCE.toString()));
+                return DELETE_RECURSIVE_IN_ADVANCE;
+            default:
+                throw new ConfigException(String.format("Unknown policy: %s", deleteInAdvancePolicy.get()));
+        }
+    }
+}

data/src/main/java/org/embulk/output/hdfs/transaction/AbortIfExistTx.java

@@ -0,0 +1,6 @@
+package org.embulk.output.hdfs.transaction;
+
+public class AbortIfExistTx
+        extends AbstractTx
+{
+}

data/src/main/java/org/embulk/output/hdfs/transaction/AbstractTx.java

@@ -0,0 +1,53 @@
+package org.embulk.output.hdfs.transaction;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.output.hdfs.HdfsFileOutput;
+import org.embulk.output.hdfs.util.StrftimeUtil;
+import org.embulk.spi.Exec;
+import org.embulk.spi.TransactionalFileOutput;
+
+import java.util.List;
+
+import static org.embulk.output.hdfs.HdfsFileOutputPlugin.PluginTask;
+
+abstract class AbstractTx
+        implements Tx
+{
+    protected void beforeRun(PluginTask task)
+    {
+    }
+
+    protected void afterRun(PluginTask task, List<TaskReport> reports)
+    {
+    }
+
+    protected ConfigDiff newConfigDiff()
+    {
+        return Exec.newConfigDiff();
+    }
+
+    public ConfigDiff transaction(PluginTask task, ControlRun control)
+    {
+        beforeRun(task);
+        List<TaskReport> reports = control.run();
+        afterRun(task, reports);
+        return newConfigDiff();
+    }
+
+    protected String getPathPrefix(PluginTask task)
+    {
+        return StrftimeUtil.strftime(task.getPathPrefix(), task.getRewindSeconds());
+    }
+
+    protected boolean canOverwrite()
+    {
+        return false;
+    }
+
+    public TransactionalFileOutput newOutput(PluginTask task, TaskSource taskSource, int taskIndex)
+    {
+        return new HdfsFileOutput(task, getPathPrefix(task), canOverwrite(), taskIndex);
+    }
+}

data/src/main/java/org/embulk/output/hdfs/transaction/ControlRun.java

@@ -0,0 +1,10 @@
+package org.embulk.output.hdfs.transaction;
+
+import org.embulk.config.TaskReport;
+
+import java.util.List;
+
+public interface ControlRun
+{
+    List<TaskReport> run();
+}

data/src/main/java/org/embulk/output/hdfs/transaction/DeleteFilesInAdvanceTx.java

@@ -0,0 +1,22 @@
+package org.embulk.output.hdfs.transaction;
+
+import org.apache.hadoop.fs.Path;
+import org.embulk.output.hdfs.HdfsFileOutputPlugin;
+import org.embulk.output.hdfs.client.HdfsClient;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+public class DeleteFilesInAdvanceTx
+        extends AbstractTx
+{
+    private static final Logger logger = Exec.getLogger(DeleteFilesInAdvanceTx.class);
+
+    @Override
+    protected void beforeRun(HdfsFileOutputPlugin.PluginTask task)
+    {
+        HdfsClient hdfsClient = HdfsClient.build(task);
+        Path globPath = new Path(getPathPrefix(task) + "*");
+        logger.info("Delete {} (File Only) in advance", globPath);
+        hdfsClient.globFilesAndTrash(globPath);
+    }
+}

data/src/main/java/org/embulk/output/hdfs/transaction/DeleteRecursiveInAdvanceTx.java

@@ -0,0 +1,22 @@
+package org.embulk.output.hdfs.transaction;
+
+import org.apache.hadoop.fs.Path;
+import org.embulk.output.hdfs.HdfsFileOutputPlugin;
+import org.embulk.output.hdfs.client.HdfsClient;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+public class DeleteRecursiveInAdvanceTx
+        extends AbstractTx
+{
+    private static final Logger logger = Exec.getLogger(DeleteRecursiveInAdvanceTx.class);
+
+    @Override
+    protected void beforeRun(HdfsFileOutputPlugin.PluginTask task)
+    {
+        HdfsClient hdfsClient = HdfsClient.build(task);
+        Path globPath = new Path(getPathPrefix(task) + "*");
+        logger.info("Delete {} (Recursive) in advance", globPath);
+        hdfsClient.globAndTrash(globPath);
+    }
+}

data/src/main/java/org/embulk/output/hdfs/transaction/OverwriteTx.java

@@ -0,0 +1,11 @@
+package org.embulk.output.hdfs.transaction;
+
+public class OverwriteTx
+        extends AbstractTx
+{
+    @Override
+    protected boolean canOverwrite()
+    {
+        return true;
+    }
+}

data/src/main/java/org/embulk/output/hdfs/transaction/ReplaceTx.java

@@ -0,0 +1,62 @@
+package org.embulk.output.hdfs.transaction;
+
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.output.hdfs.HdfsFileOutputPlugin.PluginTask;
+import org.embulk.output.hdfs.client.HdfsClient;
+import org.embulk.output.hdfs.util.SafeWorkspaceName;
+import org.embulk.output.hdfs.util.SamplePath;
+import org.embulk.output.hdfs.util.StrftimeUtil;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.nio.file.Paths;
+import java.util.List;
+
+public class ReplaceTx
+        extends AbstractTx
+{
+    private static final Logger logger = Exec.getLogger(ReplaceTx.class);
+
+    @Override
+    protected String getPathPrefix(PluginTask task)
+    {
+        return Paths.get(task.getSafeWorkspace(), super.getPathPrefix(task)).toString();
+    }
+
+    @Override
+    protected void beforeRun(PluginTask task)
+    {
+        HdfsClient hdfsClient = HdfsClient.build(task);
+        if (task.getSequenceFormat().contains("/")) {
+            throw new ConfigException("Must not include `/` in `sequence_format` if atomic is true.");
+        }
+
+        String safeWorkspace = SafeWorkspaceName.build(task.getWorkspace());
+        logger.info("Use as a workspace: {}", safeWorkspace);
+
+        String safeWsWithOutput = Paths.get(safeWorkspace, getOutputSampleDir(task)).toString();
+        logger.debug("The actual workspace must be with output dirs: {}", safeWsWithOutput);
+        if (!hdfsClient.mkdirs(safeWsWithOutput)) {
+            throw new ConfigException(String.format("Failed to make a directory: %s", safeWsWithOutput));
+        }
+        task.setSafeWorkspace(safeWorkspace);
+    }
+
+    @Override
+    protected void afterRun(PluginTask task, List<TaskReport> reports)
+    {
+        HdfsClient hdfsClient = HdfsClient.build(task);
+        String outputDir = getOutputSampleDir(task);
+        String safeWsWithOutput = Paths.get(task.getSafeWorkspace(), getOutputSampleDir(task)).toString();
+
+        hdfsClient.renameDirectory(safeWsWithOutput, outputDir, true);
+        logger.info("Store: {} >>> {}", safeWsWithOutput, outputDir);
+    }
+
+    private String getOutputSampleDir(PluginTask task)
+    {
+        String pathPrefix = StrftimeUtil.strftime(task.getPathPrefix(), task.getRewindSeconds());
+        return SamplePath.getDir(pathPrefix, task.getSequenceFormat(), task.getFileExt());
+    }
+}

data/src/main/java/org/embulk/output/hdfs/transaction/Tx.java

@@ -0,0 +1,13 @@
+package org.embulk.output.hdfs.transaction;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskSource;
+import org.embulk.output.hdfs.HdfsFileOutputPlugin.PluginTask;
+import org.embulk.spi.TransactionalFileOutput;
+
+public interface Tx
+{
+    ConfigDiff transaction(PluginTask task, ControlRun control);
+
+    TransactionalFileOutput newOutput(PluginTask task, TaskSource taskSource, int taskIndex);
+}

data/src/main/java/org/embulk/output/hdfs/util/SafeWorkspaceName.java

@@ -0,0 +1,21 @@
+package org.embulk.output.hdfs.util;
+
+import java.nio.file.Paths;
+import java.util.UUID;
+
+public class SafeWorkspaceName
+{
+    private static final String prefix = "embulk-output-hdfs";
+
+    private SafeWorkspaceName()
+    {
+    }
+
+    public static String build(String workspace)
+    {
+        long nanoTime = System.nanoTime();
+        String uuid = UUID.randomUUID().toString();
+        String dirname = String.format("%s_%d_%s", prefix, nanoTime, uuid);
+        return Paths.get(workspace, dirname).toString();
+    }
+}

data/src/main/java/org/embulk/output/hdfs/util/SamplePath.java

@@ -0,0 +1,21 @@
+package org.embulk.output.hdfs.util;
+
+import java.nio.file.Paths;
+
+public class SamplePath
+{
+    private SamplePath()
+    {
+    }
+
+    public static String getFile(String pathPrefix, String sequenceFormat, String fileExt)
+    {
+        return pathPrefix + String.format(sequenceFormat, 0, 0) + fileExt;
+    }
+
+    public static String getDir(String pathPrefix, String sequenceFormat, String fileExt)
+    {
+        String sampleFile = getFile(pathPrefix, sequenceFormat, fileExt);
+        return Paths.get(sampleFile).getParent().toString();
+    }
+}

data/src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java

@@ -0,0 +1,23 @@
+package org.embulk.output.hdfs.util;
+
+import org.jruby.embed.ScriptingContainer;
+
+public class StrftimeUtil
+{
+    private static final String scriptTemplate = "(Time.now - %d).strftime('%s')";
+
+    private StrftimeUtil()
+    {
+    }
+
+    public static String strftime(String format, int rewindSeconds)
+    {
+        String script = buildScript(format, rewindSeconds);
+        return new ScriptingContainer().runScriptlet(script).toString();
+    }
+
+    private static String buildScript(String format, int rewindSeconds)
+    {
+        return String.format(scriptTemplate, rewindSeconds, format);
+    }
+}

data/src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java

@@ -4,6 +4,7 @@ import com.google.common.base.Charsets;
 import com.google.common.base.Optional;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
@@ -17,6 +18,7 @@ import org.embulk.spi.PageTestUtils;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.Timestamp;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -37,9 +39,11 @@ import static org.embulk.spi.type.Types.*;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.core.Is.isA;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 import static org.msgpack.value.ValueFactory.newMap;
 import static org.msgpack.value.ValueFactory.newString;
 
@@ -107,9 +111,9 @@ public class TestHdfsFileOutputPlugin
         assertEquals(Lists.newArrayList(), task.getConfigFiles());
         assertEquals(Maps.newHashMap(), task.getConfig());
         assertEquals(0, task.getRewindSeconds());
-        assertEquals(false, task.getOverwrite());
+        assertEquals(Optional.absent(), task.getOverwrite());
         assertEquals(Optional.absent(), task.getDoas());
-        assertEquals(PluginTask.DeleteInAdvancePolicy.NONE, task.getDeleteInAdvance());
+        assertEquals(Optional.absent(), task.getDeleteInAdvance());
     }
 
     @Test(expected = ConfigException.class)
@@ -119,6 +123,14 @@ public class TestHdfsFileOutputPlugin
         PluginTask task = config.loadConfig(PluginTask.class);
     }
 
+    @Test(expected = ConfigException.class)
+    public void testSequenceFormatMode_replace()
+    {
+        run(getBaseConfigSource()
+                .set("mode", "replace")
+                .set("sequence_format", "%d/%d"));
+    }
+
     private List<String> lsR(List<String> names, java.nio.file.Path dir)
     {
         try (DirectoryStream<java.nio.file.Path> stream = Files.newDirectoryStream(dir)) {
@@ -201,14 +213,20 @@ public class TestHdfsFileOutputPlugin
         }
     }
 
+    private ConfigSource getDefaultFsConfig()
+    {
+        return Exec.newConfigSource()
+                .set("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
+                .set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
+                .set("fs.trash.interval", "3600")
+                .set("fs.defaultFS", "file:///");
+    }
+
     @Test
     public void testBulkLoad()
     {
         ConfigSource config = getBaseConfigSource()
-                .setNested("config", Exec.newConfigSource()
-                        .set("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
-                        .set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
-                        .set("fs.defaultFS", "file:///"));
+                .setNested("config", getDefaultFsConfig());
 
         run(config);
         List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
@@ -219,28 +237,25 @@ public class TestHdfsFileOutputPlugin
     }
 
     @Test
-    public void testDeleteRECURSIVEInAdvance()
+    public void testDeleteInAdvance_RECURSIVE()
             throws IOException
     {
         for (int n = 0; n <= 10; n++) {
-            tmpFolder.newFile("embulk-output-hdfs_file_" + n + ".txt");
-            tmpFolder.newFolder("embulk-output-hdfs_directory_" + n);
+            tmpFolder.newFile("embulk-output-hdfs_testDeleteInAdvance_RECURSIVE_file_" + n + ".txt");
+            tmpFolder.newFolder("embulk-output-hdfs_testDeleteInAdvance_RECURSIVE_directory_" + n);
         }
 
         List<String> fileListBeforeRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
 
         ConfigSource config = getBaseConfigSource()
-                .setNested("config", Exec.newConfigSource()
-                        .set("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
-                        .set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
-                        .set("fs.defaultFS", "file:///"))
+                .setNested("config", getDefaultFsConfig())
                 .set("delete_in_advance", "RECURSIVE");
 
         run(config);
 
         List<String> fileListAfterRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
         assertNotEquals(fileListBeforeRun, fileListAfterRun);
-        assertThat(fileListAfterRun, not(hasItem(containsString("embulk-output-hdfs_directory"))));
+        assertThat(fileListAfterRun, not(hasItem(containsString("embulk-output-hdfs_testDeleteInAdvance_RECURSIVE_directory_"))));
         assertThat(fileListAfterRun, not(hasItem(containsString("txt"))));
         assertThat(fileListAfterRun, hasItem(containsString(pathPrefix + "001.00.csv")));
         assertRecordsInFile(String.format("%s/%s001.00.csv",
@@ -249,21 +264,18 @@ public class TestHdfsFileOutputPlugin
     }
 
     @Test
-    public void testDeleteFILE_ONLYInAdvance()
+    public void testDeleteInAdvance_FILE_ONLY()
             throws IOException
     {
         for (int n = 0; n <= 10; n++) {
-            tmpFolder.newFile("embulk-output-hdfs_file_" + n + ".txt");
-            tmpFolder.newFolder("embulk-output-hdfs_directory_" + n);
+            tmpFolder.newFile("embulk-output-hdfs_testDeleteInAdvance_FILE_ONLY_file_" + n + ".txt");
+            tmpFolder.newFolder("embulk-output-hdfs_testDeleteInAdvance_FILE_ONLY_directory_" + n);
         }
 
         List<String> fileListBeforeRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
 
         ConfigSource config = getBaseConfigSource()
-                .setNested("config", Exec.newConfigSource()
-                        .set("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
-                        .set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
-                        .set("fs.defaultFS", "file:///"))
+                .setNested("config", getDefaultFsConfig())
                 .set("delete_in_advance", "FILE_ONLY");
 
         run(config);
@@ -271,7 +283,126 @@ public class TestHdfsFileOutputPlugin
         List<String> fileListAfterRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
         assertNotEquals(fileListBeforeRun, fileListAfterRun);
         assertThat(fileListAfterRun, not(hasItem(containsString("txt"))));
-        assertThat(fileListAfterRun, hasItem(containsString("embulk-output-hdfs_directory")));
+        assertThat(fileListAfterRun, hasItem(containsString("embulk-output-hdfs_testDeleteInAdvance_FILE_ONLY_directory_")));
+        assertThat(fileListAfterRun, hasItem(containsString(pathPrefix + "001.00.csv")));
+        assertRecordsInFile(String.format("%s/%s001.00.csv",
+                tmpFolder.getRoot().getAbsolutePath(),
+                pathPrefix));
+    }
+
+    @Test
+    public void testMode_delete_recursive_in_advance()
+            throws IOException
+    {
+        for (int n = 0; n <= 10; n++) {
+            tmpFolder.newFile("embulk-output-hdfs_testMode_delete_recursive_in_advance_file_" + n + ".txt");
+            tmpFolder.newFolder("embulk-output-hdfs_testMode_delete_recursive_in_advance_directory_" + n);
+        }
+
+        List<String> fileListBeforeRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
+
+        ConfigSource config = getBaseConfigSource()
+                .setNested("config", getDefaultFsConfig())
+                .set("mode", "delete_recursive_in_advance");
+
+        run(config);
+
+        List<String> fileListAfterRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
+        assertNotEquals(fileListBeforeRun, fileListAfterRun);
+        assertThat(fileListAfterRun, not(hasItem(containsString("embulk-output-hdfs_testMode_delete_recursive_in_advance_directory_"))));
+        assertThat(fileListAfterRun, not(hasItem(containsString("txt"))));
+        assertThat(fileListAfterRun, hasItem(containsString(pathPrefix + "001.00.csv")));
+        assertRecordsInFile(String.format("%s/%s001.00.csv",
+                tmpFolder.getRoot().getAbsolutePath(),
+                pathPrefix));
+    }
+
+    @Test
+    public void testMode_delete_files_in_advance()
+            throws IOException
+    {
+        for (int n = 0; n <= 10; n++) {
+            tmpFolder.newFile("embulk-output-hdfs_testMode_delete_files_in_advance_file_" + n + ".txt");
+            tmpFolder.newFolder("embulk-output-hdfs_testMode_delete_files_in_advance_directory_" + n);
+        }
+
+        List<String> fileListBeforeRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
+
+        ConfigSource config = getBaseConfigSource()
+                .setNested("config", getDefaultFsConfig())
+                .set("mode", "delete_files_in_advance");
+
+        run(config);
+
+        List<String> fileListAfterRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
+        assertNotEquals(fileListBeforeRun, fileListAfterRun);
+        assertThat(fileListAfterRun, not(hasItem(containsString("txt"))));
+        assertThat(fileListAfterRun, hasItem(containsString("embulk-output-hdfs_testMode_delete_files_in_advance_directory_")));
+        assertThat(fileListAfterRun, hasItem(containsString(pathPrefix + "001.00.csv")));
+        assertRecordsInFile(String.format("%s/%s001.00.csv",
+                tmpFolder.getRoot().getAbsolutePath(),
+                pathPrefix));
+    }
+
+    @Test
+    public void testMode_abort_if_exist()
+            throws IOException
+    {
+        ConfigSource config = getBaseConfigSource()
+                .setNested("config", getDefaultFsConfig())
+                .set("mode", "abort_if_exist");
+
+        run(config);
+        try {
+            run(config);
+        }
+        catch (Exception e) {
+            Throwable t = e;
+            while (t != null) {
+                t = t.getCause();
+                if (t.getCause() instanceof FileAlreadyExistsException) {
+                    Assert.assertTrue(true);
+                    return;
+                }
+            }
+            Assert.fail("FileAlreadyExistsException is not cause.");
+        }
+
+    }
+
+    @Test
+    public void testMode_overwrite()
+            throws IOException
+    {
+        ConfigSource config = getBaseConfigSource()
+                .setNested("config", getDefaultFsConfig())
+                .set("mode", "overwrite");
+
+        run(config);
+        run(config);
+        Assert.assertTrue(true);
+    }
+
+    @Test
+    public void testMode_replace()
+            throws IOException
+    {
+        for (int n = 0; n <= 10; n++) {
+            tmpFolder.newFile("embulk-output-hdfs_testMode_delete_recursive_in_advance_file_" + n + ".txt");
+            tmpFolder.newFolder("embulk-output-hdfs_testMode_delete_recursive_in_advance_directory_" + n);
+        }
+
+        List<String> fileListBeforeRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
+
+
+        run(getBaseConfigSource()
+                .set("config", getDefaultFsConfig())
+                .set("mode", "replace"));
+
+        List<String> fileListAfterRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
+        assertNotEquals(fileListBeforeRun, fileListAfterRun);
+        assertThat(fileListAfterRun, not(hasItem(containsString("embulk-output-hdfs_testMode_delete_recursive_in_advance_directory_"))));
+        assertThat(fileListAfterRun, not(hasItem(containsString("txt"))));
         assertThat(fileListAfterRun, hasItem(containsString(pathPrefix + "001.00.csv")));
         assertRecordsInFile(String.format("%s/%s001.00.csv",
                 tmpFolder.getRoot().getAbsolutePath(),