embulk-output-hdfs 0.2.4 → 0.3.0

data/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java

@@ -1,43 +1,32 @@
 package org.embulk.output.hdfs;
 
 import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.spi.Buffer;
+import org.embulk.output.hdfs.ModeTask.Mode;
+import org.embulk.output.hdfs.compat.ModeCompat;
+import org.embulk.output.hdfs.transaction.ControlRun;
+import org.embulk.output.hdfs.transaction.Tx;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
-import org.jruby.embed.ScriptingContainer;
 import org.slf4j.Logger;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-import static org.embulk.output.hdfs.HdfsFileOutputPlugin.PluginTask.*;
-
 public class HdfsFileOutputPlugin
         implements FileOutputPlugin
 {
     private static final Logger logger = Exec.getLogger(HdfsFileOutputPlugin.class);
 
     public interface PluginTask
-            extends Task
+            extends Task, ModeTask
     {
         @Config("config_files")
         @ConfigDefault("[]")
@@ -61,197 +50,88 @@ public class HdfsFileOutputPlugin
         @ConfigDefault("0")
         int getRewindSeconds();
 
+        @Deprecated  // Please Use `mode` option
         @Config("overwrite")
-        @ConfigDefault("false")
-        boolean getOverwrite();
+        @ConfigDefault("null")
+        Optional<Boolean> getOverwrite();
 
         @Config("doas")
         @ConfigDefault("null")
         Optional<String> getDoas();
 
-        enum DeleteInAdvancePolicy{ NONE, FILE_ONLY, RECURSIVE}
-        @Config("delete_in_advance")
-        @ConfigDefault("\"NONE\"")
-        DeleteInAdvancePolicy getDeleteInAdvance();
-    }
+        @Deprecated
+        enum DeleteInAdvancePolicy
+        {
+            NONE, FILE_ONLY, RECURSIVE
+        }
 
-    @Override
-    public ConfigDiff transaction(ConfigSource config, int taskCount,
-            FileOutputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
+        @Deprecated  // Please Use `mode` option
+        @Config("delete_in_advance")
+        @ConfigDefault("null")
+        Optional<DeleteInAdvancePolicy> getDeleteInAdvance();
 
-        try {
-            String pathPrefix = strftime(task.getPathPrefix(), task.getRewindSeconds());
-            FileSystem fs = getFs(task);
-            deleteInAdvance(fs, pathPrefix, task.getDeleteInAdvance());
-        }
-        catch (IOException e) {
-            throw Throwables.propagate(e);
-        }
+        @Config("workspace")
+        @ConfigDefault("\"/tmp\"")
+        String getWorkspace();
 
-        control.run(task.dump());
-        return Exec.newConfigDiff();
+        String getSafeWorkspace();
+        void setSafeWorkspace(String safeWorkspace);
     }
 
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileOutputPlugin.Control control)
+    private void compat(PluginTask task)
     {
-        throw new UnsupportedOperationException("hdfs output plugin does not support resuming");
+        Mode modeCompat = ModeCompat.getMode(task, task.getOverwrite(), task.getDeleteInAdvance());
+        task.setMode(modeCompat);
     }
 
-    @Override
-    public void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<TaskReport> successTaskReports)
+    // NOTE: This is to avoid the following error.
+    // Error: java.lang.RuntimeException: com.fasterxml.jackson.databind.JsonMappingException: Field 'SafeWorkspace' is required but not set
+    //  at [Source: N/A; line: -1, column: -1]
+    private void avoidDatabindError(PluginTask task)
     {
+        // Set default value
+        task.setSafeWorkspace("");
     }
 
     @Override
-    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
+    public ConfigDiff transaction(ConfigSource config, int taskCount,
+            final FileOutputPlugin.Control control)
     {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        compat(task);
+        avoidDatabindError(task);
 
-        final String pathPrefix = strftime(task.getPathPrefix(), task.getRewindSeconds());
-        final String pathSuffix = task.getFileExt();
-        final String sequenceFormat = task.getSequenceFormat();
-
-        return new TransactionalFileOutput()
+        Tx tx = task.getMode().newTx();
+        return tx.transaction(task, new ControlRun()
         {
-            private final List<String> hdfsFileNames = new ArrayList<>();
-            private int fileIndex = 0;
-            private Path currentPath = null;
-            private OutputStream output = null;
-
-            @Override
-            public void nextFile()
-            {
-                closeCurrentStream();
-                currentPath = new Path(pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix);
-                fileIndex++;
-            }
-
-            @Override
-            public void add(Buffer buffer)
-            {
-                try {
-                    // this implementation is for creating file when there is data.
-                    if (output == null) {
-                        FileSystem fs = getFs(task);
-                        output = fs.create(currentPath, task.getOverwrite());
-                        logger.info("Uploading '{}'", currentPath);
-                        hdfsFileNames.add(currentPath.toString());
-                    }
-                    output.write(buffer.array(), buffer.offset(), buffer.limit());
-                }
-                catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-                finally {
-                    buffer.release();
-                }
-            }
-
             @Override
-            public void finish()
+            public List<TaskReport> run()
             {
-                closeCurrentStream();
+                return control.run(task.dump());
             }
-
-            @Override
-            public void close()
-            {
-                closeCurrentStream();
-            }
-
-            @Override
-            public void abort()
-            {
-            }
-
-            @Override
-            public TaskReport commit()
-            {
-                TaskReport report = Exec.newTaskReport();
-                report.set("hdfs_file_names", hdfsFileNames);
-                return report;
-            }
-
-            private void closeCurrentStream()
-            {
-                if (output != null) {
-                    try {
-                        output.close();
-                        output = null;
-                    }
-                    catch (IOException e) {
-                        throw new RuntimeException(e);
-                    }
-                }
-            }
-        };
+        });
     }
 
-    private static FileSystem getFs(final PluginTask task)
-            throws IOException
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileOutputPlugin.Control control)
     {
-        Configuration configuration = new Configuration();
-
-        for (String configFile : task.getConfigFiles()) {
-            File file = new File(configFile);
-            configuration.addResource(file.toURI().toURL());
-        }
-
-        for (Map.Entry<String, String> entry: task.getConfig().entrySet()) {
-            configuration.set(entry.getKey(), entry.getValue());
-        }
-
-        if (task.getDoas().isPresent()) {
-            URI uri = FileSystem.getDefaultUri(configuration);
-            try {
-                return FileSystem.get(uri, configuration, task.getDoas().get());
-            }
-            catch (InterruptedException e) {
-                throw Throwables.propagate(e);
-            }
-        }
-        return FileSystem.get(configuration);
+        throw new UnsupportedOperationException("hdfs output plugin does not support resuming");
     }
 
-    private String strftime(final String raw, final int rewind_seconds)
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports)
     {
-        ScriptingContainer jruby = new ScriptingContainer();
-        Object resolved = jruby.runScriptlet(
-                String.format("(Time.now - %s).strftime('%s')", String.valueOf(rewind_seconds), raw));
-        return resolved.toString();
     }
 
-    private void deleteInAdvance(FileSystem fs, String pathPrefix, DeleteInAdvancePolicy deleteInAdvancePolicy)
-            throws IOException
+    @Override
+    public TransactionalFileOutput open(TaskSource taskSource, int taskIndex)
     {
-        final Path globPath = new Path(pathPrefix + "*");
-        switch (deleteInAdvancePolicy) {
-            case NONE:
-                // do nothing
-                break;
-            case FILE_ONLY:
-                for (FileStatus status : fs.globStatus(globPath)) {
-                    if (status.isFile()) {
-                        logger.debug("delete in advance: {}", status.getPath());
-                        fs.delete(status.getPath(), false);
-                    }
-                }
-                break;
-            case RECURSIVE:
-                for (FileStatus status : fs.globStatus(globPath)) {
-                    logger.debug("delete in advance: {}", status.getPath());
-                    fs.delete(status.getPath(), true);
-                }
-                break;
-            default:
-                throw new ConfigException("`delete_in_advance` must not null.");
-        }
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        Tx tx = task.getMode().newTx();
+        return tx.newOutput(task, taskSource, taskIndex);
     }
 }

data/src/main/java/org/embulk/output/hdfs/ModeTask.java

@@ -0,0 +1,111 @@
+package org.embulk.output.hdfs;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.Task;
+import org.embulk.output.hdfs.transaction.AbortIfExistTx;
+import org.embulk.output.hdfs.transaction.DeleteFilesInAdvanceTx;
+import org.embulk.output.hdfs.transaction.DeleteRecursiveInAdvanceTx;
+import org.embulk.output.hdfs.transaction.OverwriteTx;
+import org.embulk.output.hdfs.transaction.ReplaceTx;
+import org.embulk.output.hdfs.transaction.Tx;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import javax.annotation.Nullable;
+
+import java.util.Locale;
+
+public interface ModeTask
+        extends Task
+{
+    @Config("mode")
+    @ConfigDefault("\"abort_if_exist\"")
+    Mode getMode();
+
+    void setMode(Mode mode);
+
+    enum Mode
+    {
+        ABORT_IF_EXIST,
+        OVERWRITE,
+        DELETE_FILES_IN_ADVANCE,
+        DELETE_RECURSIVE_IN_ADVANCE,
+        REPLACE;
+
+        private static final Logger logger = Exec.getLogger(Mode.class);
+
+        @Deprecated  // For compat
+        public boolean isDefaultMode()
+        {
+            return this.equals(ABORT_IF_EXIST);
+        }
+
+        @JsonValue
+        @Override
+        public String toString()
+        {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+
+        @JsonCreator
+        @SuppressWarnings("unused")
+        public static Mode fromString(String value)
+        {
+            switch (value) {
+                case "abort_if_exist":
+                    return ABORT_IF_EXIST;
+                case "overwrite":
+                    return OVERWRITE;
+                case "delete_files_in_advance":
+                    return DELETE_FILES_IN_ADVANCE;
+                case "delete_recursive_in_advance":
+                    return DELETE_RECURSIVE_IN_ADVANCE;
+                case "replace":
+                    return REPLACE;
+                default:
+                    throw new ConfigException(String.format(
+                            "Unknown mode `%s`. Supported mode is %s",
+                            value,
+                            Joiner.on(", ").join(
+                                    Lists.transform(Lists.newArrayList(Mode.values()), new Function<Mode, String>()
+                                    {
+                                        @Nullable
+                                        @Override
+                                        public String apply(@Nullable Mode input)
+                                        {
+                                            assert input != null;
+                                            return String.format("`%s`", input.toString());
+                                        }
+                                    })
+                            )
+                    ));
+            }
+        }
+
+        public Tx newTx()
+        {
+            switch (this) {
+                case ABORT_IF_EXIST:
+                    return new AbortIfExistTx();
+                case DELETE_FILES_IN_ADVANCE:
+                    return new DeleteFilesInAdvanceTx();
+                case DELETE_RECURSIVE_IN_ADVANCE:
+                    return new DeleteRecursiveInAdvanceTx();
+                case OVERWRITE:
+                    return new OverwriteTx();
+                case REPLACE:
+                    return new ReplaceTx();
+                default:
+                    throw new IllegalStateException();
+            }
+        }
+
+    }
+}

data/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java

@@ -0,0 +1,269 @@
+package org.embulk.output.hdfs.client;
+
+import com.google.common.base.Optional;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
+import org.embulk.config.ConfigException;
+import org.embulk.output.hdfs.HdfsFileOutputPlugin;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+public class HdfsClient
+{
+    public static HdfsClient build(HdfsFileOutputPlugin.PluginTask task)
+    {
+        Configuration conf = buildConfiguration(task.getConfigFiles(), task.getConfig());
+        return new HdfsClient(conf, task.getDoas());
+    }
+
+    ;
+
+    public static Configuration buildConfiguration(List<String> configFiles, Map<String, String> configs)
+    {
+        Configuration c = new Configuration();
+        for (String configFile : configFiles) {
+            File file = new File(configFile);
+            try {
+                c.addResource(file.toURI().toURL());
+            }
+            catch (MalformedURLException e) {
+                throw new ConfigException(e);
+            }
+        }
+        for (Map.Entry<String, String> config : configs.entrySet()) {
+            c.set(config.getKey(), config.getValue());
+        }
+        return c;
+    }
+
+    private static Logger logger = Exec.getLogger(HdfsClient.class);
+    private final Configuration conf;
+    private final FileSystem fs;
+    private final Optional<String> user;
+    private final RetryExecutor re = RetryExecutor.retryExecutor()
+            .withRetryLimit(3)
+            .withMaxRetryWait(500)             // ms
+            .withMaxRetryWait(10 * 60 * 1000); // ms
+
+    private HdfsClient(Configuration conf, Optional<String> user)
+    {
+        this.conf = conf;
+        this.user = user;
+        this.fs = getFs(conf, user);
+    }
+
+    private abstract static class Retryable<T>
+            implements RetryExecutor.Retryable<T>
+    {
+        @Override
+        public boolean isRetryableException(Exception exception)
+        {
+            return true; // TODO: which Exception is retryable?
+        }
+
+        @Override
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                throws RetryExecutor.RetryGiveupException
+        {
+            String m = String.format(
+                    "%s. (Retry: Count: %d, Limit: %d, Wait: %d ms)",
+                    exception.getMessage(),
+                    retryCount,
+                    retryLimit,
+                    retryWait);
+            logger.warn(m, exception);
+        }
+
+        @Override
+        public void onGiveup(Exception firstException, Exception lastException)
+                throws RetryExecutor.RetryGiveupException
+        {
+        }
+    }
+
+    private <T> T run(Retryable<T> retryable)
+    {
+        try {
+            return re.run(retryable);
+        }
+        catch (RetryExecutor.RetryGiveupException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private FileSystem getFs(Configuration conf, Optional<String> user)
+    {
+        if (user.isPresent()) {
+            return getFs(conf, user.get());
+        }
+        else {
+            return getFs(conf);
+        }
+    }
+
+    private FileSystem getFs(final Configuration conf, final String user)
+    {
+        return run(new Retryable<FileSystem>()
+        {
+            @Override
+            public FileSystem call()
+                    throws Exception
+            {
+                URI uri = FileSystem.getDefaultUri(conf);
+                return FileSystem.get(uri, conf, user);
+            }
+        });
+    }
+
+    private FileSystem getFs(final Configuration conf)
+    {
+        return run(new Retryable<FileSystem>()
+        {
+            @Override
+            public FileSystem call()
+                    throws Exception
+            {
+                return FileSystem.get(conf);
+            }
+        });
+    }
+
+    public FileStatus[] glob(final Path globPath)
+    {
+        return run(new Retryable<FileStatus[]>()
+        {
+            @Override
+            public FileStatus[] call()
+                    throws Exception
+            {
+                return fs.globStatus(globPath);
+            }
+        });
+    }
+
+    public boolean trash(final Path path)
+    {
+        return run(new Retryable<Boolean>()
+        {
+            @Override
+            public Boolean call()
+                    throws Exception
+            {
+                return Trash.moveToAppropriateTrash(fs, path, conf);
+            }
+        });
+    }
+
+    public void globFilesAndTrash(final Path globPath)
+    {
+        for (final FileStatus fileStatus : glob(globPath)) {
+            if (fileStatus.isDirectory()) {
+                logger.debug("Skip {} because {} is a directory.",
+                        fileStatus.getPath(), fileStatus.getPath());
+                continue;
+            }
+            logger.debug("Move To Trash: {}", fileStatus.getPath());
+            if (!trash(fileStatus.getPath())) {
+                throw new RuntimeException(String.format("Failed to Move To Trash: %s", fileStatus.getPath()));
+            }
+        }
+    }
+
+    public void globAndTrash(final Path globPath)
+    {
+        for (final FileStatus fileStatus : glob(globPath)) {
+            logger.debug("Move To Trash: {}", fileStatus.getPath());
+            if (!trash(fileStatus.getPath())) {
+                throw new RuntimeException(String.format("Failed to Move To Trash: %s", fileStatus.getPath()));
+            }
+        }
+    }
+
+    public OutputStream create(final Path path, final boolean overwrite)
+    {
+        return run(new Retryable<OutputStream>()
+        {
+            @Override
+            public OutputStream call()
+                    throws Exception
+            {
+                return fs.create(path, overwrite);
+            }
+        });
+    }
+
+    public boolean mkdirs(String path)
+    {
+        return mkdirs(new Path(path));
+    }
+
+    public boolean mkdirs(final Path path)
+    {
+        return run(new Retryable<Boolean>()
+        {
+            @Override
+            public Boolean call()
+                    throws Exception
+            {
+                return fs.mkdirs(path);
+            }
+        });
+    }
+
+    public void close()
+    {
+        run(new Retryable<Void>()
+        {
+            @Override
+            public Void call()
+                    throws Exception
+            {
+                fs.close();
+                return null;
+            }
+        });
+    }
+
+    public void renameDirectory(String src, String dst, boolean trashIfExists)
+    {
+        renameDirectory(new Path(src), new Path(dst), trashIfExists);
+    }
+
+    public void renameDirectory(final Path src, final Path dst, final boolean trashIfExists)
+    {
+        run(new Retryable<Void>()
+        {
+            @Override
+            public Void call()
+                    throws Exception
+            {
+                if (fs.exists(dst)) {
+                    if (!trashIfExists) {
+                        throw new DataException(String.format("Directory Exists: %s", dst.toString()));
+                    }
+                    logger.info("Move To Trash: {}", dst);
+                    if (!trash(dst)) {
+                        throw new IllegalStateException(String.format("Failed to Move To Trash: %s", dst.toString()));
+                    }
+                }
+                FileContext.getFileContext(conf).rename(src, dst, Options.Rename.NONE);
+                logger.debug("Rename: {} >>> {}", src, dst);
+                return null;
+            }
+        });
+    }
+}