embulk-output-hdfs 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84d24b454173e755db38f3b87e8fb5808d4788b2
-  data.tar.gz: 2f031bc26b79c4483e1cea88133251c199e51f37
+  metadata.gz: e23e49bd7a7a7cb1587c929faf9979fe3ae5cf94
+  data.tar.gz: 13833b306d2d69da411177c6f4a094955b572f79
 SHA512:
-  metadata.gz: 3db39115ff402039d21530cb7251f2def2c4e4f5ac1aeb735dc69e6f444d35390be6f560d452d69318b7025de70a421d8fd62cae4dff19e78002b5c670cdfd3e
-  data.tar.gz: c4ebc62b415d33caad0713d41d5277c85b033467486a4a680cc109fd2fe2f1e6e2975034b2de046fba8136345bae112a64dcf501646d149ddd8860b0fca19091
+  metadata.gz: 6ac5ec5966dd880ed625d85afdaa8bd587dcf03aadfa4c2464f5bb78034dea247d263197ced5a1130d3d394a801dd9ea3d5a91180973cc4ced4dd43c332c1a10
+  data.tar.gz: 82f2d8d22029f356925e7412d9a891317d5b6c4fa667787ce2df46fd7195db4aaf813410c705e590ba21958fcb9242a81ca6b4e04d2a4904c4770eacf23f95eb

data/.gitignore

@@ -6,3 +6,6 @@
 /classpath/
 build/
 .idea
+*.iml
+.ruby-version
+

data/README.md

@@ -1,11 +1,11 @@
-# Hdfs output plugin for Embulk
+# Hdfs file output plugin for Embulk
 
 A File Output Plugin for Embulk to write HDFS.
 
 ## Overview
 
 * **Plugin type**: file output
-* **Load all or nothing**: no
+* **Load all or nothing**: yes
 * **Resume supported**: no
 * **Cleanup supported**: no
 
@@ -13,8 +13,12 @@ A File Output Plugin for Embulk to write HDFS.
 
 - **config_files** list of paths to Hadoop's configuration files (array of strings, default: `[]`)
 - **config** overwrites configuration parameters (hash, default: `{}`)
-- **output_path** the path finally stored files. (string, default: `"/tmp/embulk.output.hdfs_output.%Y%m%d_%s"`)
-- **working_path** the path temporary stored files. (string, default: `"/tmp/embulk.working.hdfs_output.%Y%m%d_%s"`)
+- **path_prefix** prefix of target files (string, required)
+- **file_ext** suffix of target files (string, required)
+- **sequence_format** format for sequence part of target files (string, default: `'.%03d.%02d'`)
+- **rewind_seconds** When you use Date format in path_prefix property(like `/tmp/embulk/%Y-%m-%d/out`), the format is interpreted by using the time which is Now minus this property. (int, default: `0`)
+- **overwrite** overwrite files when the same filenames already exists (boolean, default: `false`)
+    - *caution*: even if this property is `true`, this does not mean ensuring the idempotence. if you want to ensure the idempotence, you need the procedures to remove output files after or before running. 
 
 ## Example
 
@@ -24,14 +28,13 @@ out:
   config_files:
     - /etc/hadoop/conf/core-site.xml
     - /etc/hadoop/conf/hdfs-site.xml
-    - /etc/hadoop/conf/mapred-site.xml
-    - /etc/hadoop/conf/yarn-site.xml
   config:
     fs.defaultFS: 'hdfs://hdp-nn1:8020'
-    dfs.replication: 1
-    mapreduce.client.submit.file.replication: 1
     fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
     fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
+  path_prefix: '/tmp/embulk/hdfs_output/%Y-%m-%d/out'
+  file_ext: 'txt'
+  overwrite: true
   formatter:
     type: csv
     encoding: UTF-8

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -22,7 +22,7 @@ dependencies {
     provided "org.embulk:embulk-core:0.7.0"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     compile 'org.apache.hadoop:hadoop-client:2.6.0'
-    compile 'com.google.guava:guava:14.0'
+    compile 'com.google.guava:guava:15.0'
     testCompile "junit:junit:4.+"
 }
 
@@ -57,9 +57,9 @@ task gemspec {
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
-  spec.authors       = ["takahiro.nakayama"]
-  spec.summary       = %[Hdfs output plugin for Embulk]
-  spec.description   = %[Dumps records to Hdfs.]
+  spec.authors       = ["Civitaspo"]
+  spec.summary       = %[Hdfs file output plugin for Embulk]
+  spec.description   = %[Stores files on Hdfs.]
   spec.email         = ["civitaspo@gmail.com"]
   spec.licenses      = ["MIT"]
   spec.homepage      = "https://github.com/civitaspo/embulk-output-hdfs"

data/lib/embulk/output/hdfs.rb

@@ -1,3 +1,3 @@
 Embulk::JavaPlugin.register_output(
-  "hdfs", "org.embulk.output.HdfsOutputPlugin",
+  "hdfs", "org.embulk.output.hdfs.HdfsFileOutputPlugin",
   File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java

@@ -0,0 +1,198 @@
+package org.embulk.output.hdfs;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.embulk.config.TaskReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.TransactionalFileOutput;
+import org.jruby.embed.ScriptingContainer;
+import org.slf4j.Logger;
+
+public class HdfsFileOutputPlugin
+        implements FileOutputPlugin
+{
+    private static final Logger logger = Exec.getLogger(HdfsFileOutputPlugin.class);
+
+    public interface PluginTask
+            extends Task
+    {
+        @Config("config_files")
+        @ConfigDefault("[]")
+        public List<String> getConfigFiles();
+
+        @Config("config")
+        @ConfigDefault("{}")
+        public Map<String, String> getConfig();
+
+        @Config("path_prefix")
+        public String getPathPrefix();
+
+        @Config("file_ext")
+        public String getFileNameExtension();
+
+        @Config("sequence_format")
+        @ConfigDefault("\"%03d.%02d.\"")
+        public String getSequenceFormat();
+
+        @Config("rewind_seconds")
+        @ConfigDefault("0")
+        public int getRewindSeconds();
+
+        @Config("overwrite")
+        @ConfigDefault("false")
+        public boolean getOverwrite();
+
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, int taskCount,
+            FileOutputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        control.run(task.dump());
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileOutputPlugin.Control control)
+    {
+        throw new UnsupportedOperationException("hdfs output plugin does not support resuming");
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        final String pathPrefix = strftime(task.getPathPrefix(), task.getRewindSeconds());
+        final String pathSuffix = task.getFileNameExtension();
+        final String sequenceFormat = task.getSequenceFormat();
+
+        return new TransactionalFileOutput()
+        {
+            private final List<String> hdfsFileNames = new ArrayList<>();
+            private int fileIndex = 0;
+            private OutputStream output = null;
+
+            @Override
+            public void nextFile()
+            {
+                closeCurrentStream();
+                Path path = new Path(pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix);
+                try {
+                    FileSystem fs = getFs(task);
+                    output = fs.create(path, task.getOverwrite());
+                    logger.info("Uploading '{}'", path);
+                }
+                catch (IOException e) {
+                    logger.error(e.getMessage());
+                    throw new RuntimeException(e);
+                }
+                hdfsFileNames.add(path.toString());
+                fileIndex++;
+            }
+
+            @Override
+            public void add(Buffer buffer)
+            {
+                try {
+                    output.write(buffer.array(), buffer.offset(), buffer.limit());
+                }
+                catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                finally {
+                    buffer.release();
+                }
+            }
+
+            @Override
+            public void finish()
+            {
+                closeCurrentStream();
+            }
+
+            @Override
+            public void close()
+            {
+                closeCurrentStream();
+            }
+
+            @Override
+            public void abort()
+            {
+            }
+
+            @Override
+            public TaskReport commit()
+            {
+                TaskReport report = Exec.newTaskReport();
+                report.set("hdfs_file_names", hdfsFileNames);
+                return report;
+            }
+
+            private void closeCurrentStream()
+            {
+                if (output != null) {
+                    try {
+                        output.close();
+                        output = null;
+                    }
+                    catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+        };
+    }
+
+    private static FileSystem getFs(final PluginTask task)
+            throws IOException
+    {
+        Configuration configuration = new Configuration();
+
+        for (Object configFile : task.getConfigFiles()) {
+            configuration.addResource(configFile.toString());
+        }
+        configuration.reloadConfiguration();
+
+        for (Map.Entry<String, String> entry: task.getConfig().entrySet()) {
+            configuration.set(entry.getKey(), entry.getValue());
+        }
+
+        return FileSystem.get(configuration);
+    }
+
+    private String strftime(final String raw, final int rewind_seconds)
+    {
+        ScriptingContainer jruby = new ScriptingContainer();
+        Object resolved = jruby.runScriptlet(
+                String.format("(Time.now - %s).strftime('%s')", String.valueOf(rewind_seconds), raw));
+        return resolved.toString();
+    }
+}

data/src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.output.hdfs;
+
+public class TestHdfsFileOutputPlugin
+{
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-hdfs
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
-- takahiro.nakayama
+- Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2015-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,7 +38,7 @@ dependencies:
         version: '10.0'
   prerelease: false
   type: :development
-description: Dumps records to Hdfs.
+description: Stores files on Hdfs.
 email:
 - civitaspo@gmail.com
 executables: []
@@ -54,8 +54,8 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/output/hdfs.rb
-- src/main/java/org/embulk/output/HdfsOutputPlugin.java
-- src/test/java/org/embulk/output/TestHdfsOutputPlugin.java
+- src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java
+- src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java
 - classpath/activation-1.1.jar
 - classpath/apacheds-i18n-2.0.0-M15.jar
 - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
@@ -79,7 +79,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-output-hdfs-0.1.2.jar
+- classpath/embulk-output-hdfs-0.2.0.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar
@@ -151,5 +151,5 @@ rubyforge_project:
 rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
-summary: Hdfs output plugin for Embulk
+summary: Hdfs file output plugin for Embulk
 test_files: []

data/src/main/java/org/embulk/output/HdfsOutputPlugin.java

@@ -1,219 +0,0 @@
-package org.embulk.output;
-
-import com.google.common.base.Throwables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.embulk.config.*;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileOutputPlugin;
-import org.embulk.spi.TransactionalFileOutput;
-import org.jruby.embed.ScriptingContainer;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.List;
-import java.util.Map;
-
-public class HdfsOutputPlugin implements FileOutputPlugin
-{
-    private static final Logger logger = Exec.getLogger(HdfsOutputPlugin.class);
-
-    public interface PluginTask extends Task
-    {
-        @Config("config_files")
-        @ConfigDefault("[]")
-        public List<String> getConfigFiles();
-
-        @Config("config")
-        @ConfigDefault("{}")
-        public Map<String, String> getConfig();
-
-        @Config("sequence_format")
-        @ConfigDefault("\"%03d.%02d\"")
-        public String getSequenceFormat();
-
-        @Config("output_path")
-        @ConfigDefault("\"/tmp/embulk.output.hdfs_output.%Y%m%d_%s\"")
-        public String getOutputPath();
-
-        @Config("working_path")
-        @ConfigDefault("\"/tmp/embulk.working.hdfs_output.%Y%m%d_%s\"")
-        public String getWorkingPath();
-
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-                                  int taskCount,
-                                  FileOutputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-        return resume(task.dump(), taskCount, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-                             int taskCount,
-                             FileOutputPlugin.Control control)
-    {
-        control.run(taskSource);
-        return Exec.newConfigDiff();
-    }
-
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-                        int taskCount,
-                        List<TaskReport> successTaskReports)
-    {
-    }
-
-    @Override
-    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        Configuration configuration = getHdfsConfiguration(task);
-        FileSystem fs = getFs(configuration);
-        String workingPath = strftime(task.getWorkingPath());
-        String outputPath = strftime(task.getOutputPath());
-        return new TransactionalHdfsFileOutput(task, fs, workingPath, outputPath, taskIndex);
-    }
-
-    private Configuration getHdfsConfiguration(final PluginTask task)
-    {
-        Configuration configuration = new Configuration();
-
-        List configFiles = task.getConfigFiles();
-        for (Object configFile : configFiles) {
-            configuration.addResource(configFile.toString());
-        }
-
-        for (Map.Entry<String, String> entry: task.getConfig().entrySet()) {
-            configuration.set(entry.getKey(), entry.getValue());
-        }
-
-        return configuration;
-    }
-
-    private FileSystem getFs(final Configuration configuration) {
-        try {
-            FileSystem fs = FileSystem.get(configuration);
-            return fs;
-        }
-        catch (IOException e) {
-            logger.error(e.getMessage());
-            throw Throwables.propagate(e);
-        }
-    }
-
-    private String strftime(final String path)
-    {
-        // strftime
-        ScriptingContainer jruby = new ScriptingContainer();
-        Object result = jruby.runScriptlet("Time.now.strftime('" + path + "')");
-        return result.toString();
-    }
-
-    static class TransactionalHdfsFileOutput implements TransactionalFileOutput
-    {
-        private final int taskIndex;
-        private final FileSystem fs;
-        private final String workingPath;
-        private final String outputPath;
-        private final String sequenceFormat;
-
-        private int fileIndex = 0;
-        private int callCount = 0;
-        private Path currentPath = null;
-        private OutputStream currentStream = null;
-
-        public TransactionalHdfsFileOutput(PluginTask task, FileSystem fs, String workingPath, String outputPath, int taskIndex)
-        {
-            this.taskIndex = taskIndex;
-            this.fs = fs;
-            this.workingPath = workingPath;
-            this.outputPath = outputPath;
-            this.sequenceFormat = task.getSequenceFormat();
-        }
-
-        public void nextFile() {
-            closeCurrentStream();
-            currentPath = new Path(workingPath + '/' + String.format(sequenceFormat, taskIndex, fileIndex));
-            try {
-                if (fs.exists(currentPath)) {
-                    throw new IllegalAccessException(currentPath.toString() + "already exists.");
-                }
-                currentStream = fs.create(currentPath);
-                logger.info("Uploading '{}'", currentPath.toString());
-            }
-            catch (IOException | IllegalAccessException e) {
-                logger.error(e.getMessage());
-                throw Throwables.propagate(e);
-            }
-            fileIndex++;
-        }
-
-        @Override
-        public void add(Buffer buffer) {
-            if (currentStream == null) {
-                throw new IllegalStateException("nextFile() must be called before poll()");
-            }
-            try {
-                logger.debug("#add called {} times for taskIndex {}", callCount, taskIndex);
-                currentStream.write(buffer.array(), buffer.offset(), buffer.limit());
-                callCount++;
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            } finally {
-                buffer.release();
-            }
-        }
-
-        @Override
-        public void finish() {
-            closeCurrentStream();
-        }
-
-        @Override
-        public void close() {
-            closeCurrentStream();
-        }
-
-        @Override
-        public void abort() {
-        }
-
-        @Override
-        public TaskReport commit() {
-            try {
-                fs.rename(new Path(workingPath), new Path(outputPath));
-                logger.info("rename {} => {}", workingPath, outputPath);
-            } catch (IOException e) {
-                logger.error(e.getMessage());
-                throw Throwables.propagate(e);
-            }
-
-            TaskReport report = Exec.newTaskReport();
-            report.set("files", currentPath);
-            return report;
-        }
-
-        private void closeCurrentStream() {
-            try {
-                if (currentStream != null) {
-                    currentStream.close();
-                    currentStream = null;
-                }
-
-                callCount = 0;
-            } catch (IOException e) {
-                logger.error(e.getMessage());
-                throw Throwables.propagate(e);
-            }
-        }
-    }
-}

data/src/test/java/org/embulk/output/TestHdfsOutputPlugin.java

@@ -1,5 +0,0 @@
-package org.embulk.output;
-
-public class TestHdfsOutputPlugin
-{
-}