embulk-output-hdfs 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90bae20ab751bea6d3807b44b252062d137a710d
-  data.tar.gz: 482d46a137ba2fad65988bd98ec88613a884e590
+  metadata.gz: dd16ecf369d77f2a3dab20ebbb305dfbfca1a2a9
+  data.tar.gz: e89f1b6e282e461abc6116c42dfe393abff4032e
 SHA512:
-  metadata.gz: b23e3d09a38d4dd493e965bd3229e87ff89d43828ce48cd71c2b0ae996575d6a8a6fd9e404b6b075c3bb7719961d45af856dc6a21e778516d3783e06b4c92cd9
-  data.tar.gz: c5ad1e1f16d5c632a5352c25dc3e065945a40fcdac135aeb8cc8150332d4812cc262c2869fe9308337b246ffdf0bf08448d1e8f47f27c08eb542be529491515b
+  metadata.gz: e8e335c24da1f14cd05492baaddb1c2de0c57d0cfa9a15eef17b85c288e1bb0f57ba91cf95f1006a612b16b624a57ac7d422a7853fe6ccf44d66caabc9984523
+  data.tar.gz: 0049d9041ee1796b7e6e1cbc8dd1f7e0e738509f67abceaf0e7220ada5c95d9b63820cd38a5d37c720ee1e303e6504d3d38ba2524e9f9412143c5329cdba3186

data/.travis.yml

@@ -1,3 +1,4 @@
+dist: precise
 language: java
 jdk:
   - openjdk7
@@ -6,4 +7,8 @@ jdk:
 script:
   - ./gradlew test
 after_success:
-  - ./gradlew jacocoTestReport coveralls
+  - ./gradlew jacocoTestReport coveralls
+addons:
+  hosts:
+    - example.com
+  hostname: example.com

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+0.3.0 (2017-12-03)
+==================
+* Add: `mode` option.
+    * Add `replace` behaviour.
+* Deprecated: `delete_in_advance` option. Please use `mode` instead.
+* Deprecated: `overwrite` option. Please use `mode` instead.
+* Enhancement: Delete behaviour become safely.
+* Enhancement: Update embulk 0.8.38
+
 0.2.4 (2016-04-27)
 ==================
 - Enhancement: Avoid to create 0 byte files

data/README.md

@@ -14,19 +14,21 @@ A File Output Plugin for Embulk to write HDFS.
 
 ## Configuration
 
-- **config_files** list of paths to Hadoop's configuration files (array of strings, default: `[]`)
-- **config** overwrites configuration parameters (hash, default: `{}`)
-- **path_prefix** prefix of target files (string, required)
-- **file_ext** suffix of target files (string, required)
-- **sequence_format** format for sequence part of target files (string, default: `'.%03d.%02d'`)
-- **rewind_seconds** When you use Date format in path_prefix property(like `/tmp/embulk/%Y-%m-%d/out`), the format is interpreted by using the time which is Now minus this property. (int, default: `0`)
-- **overwrite** overwrite files when the same filenames already exists (boolean, default: `false`)
+- **config_files**: list of paths to Hadoop's configuration files (array of strings, default: `[]`)
+- **config**: overwrites configuration parameters (hash, default: `{}`)
+- **path_prefix**: prefix of target files (string, required)
+- **file_ext**: suffix of target files (string, required)
+- **sequence_format**: format for sequence part of target files (string, default: `'%03d.%02d.'`)
+- **rewind_seconds**: When you use Date format in path_prefix property(like `/tmp/embulk/%Y-%m-%d/out`), the format is interpreted by using the time which is Now minus this property. (int, default: `0`)
+- **doas**: username which access to Hdfs (string, default: executed user)
+- **overwrite** *(Deprecated: Please use `mode` option instead)*: overwrite files when the same filenames already exists (boolean, default: `false`)
     - *caution*: even if this property is `true`, this does not mean ensuring the idempotence. if you want to ensure the idempotence, you need the procedures to remove output files after or before running. 
-- **doas** username which access to Hdfs (string, default: executed user)
-- **delete_in_advance** delete files and directories having `path_prefix` in advance (enum, default: `NONE`)
+- **delete_in_advance** *(Deprecated: Please use `mode` option instead)*: delete files and directories having `path_prefix` in advance (enum, default: `NONE`)
     - `NONE`: do nothing
     - `FILE_ONLY`: delete files
     - `RECURSIVE`: delete files and directories
+- **mode**: "abort_if_exist", "overwrite", "delete_files_in_advance", "delete_recursive_in_advance", or "replace". See below. (string, optional, default: `"abort_if_exist"`)
+    * In the future, default mode will become `"replace"`.
 
 ## CAUTION
 If you use `hadoop` user (hdfs admin user) as `doas`, and if `delete_in_advance` is `RECURSIVE`,
@@ -34,6 +36,33 @@ If you use `hadoop` user (hdfs admin user) as `doas`, and if `delete_in_advance`
 this means `embulk-output-hdfs` can destroy your hdfs.
 So, please be careful when you use `delete_in_advance` option and `doas` option ...
 
+## About DELETE
+
+When this plugin deletes files or directories, use [`Hadoop Trash API`](https://hadoop.apache.org/docs/r2.8.0/api/org/apache/hadoop/fs/Trash.html). So, you can find them in the trash during `fs.trash.interval`.
+
+## Modes
+
+* **abort_if_exist**:
+    * Behavior: This mode writes rows to the target files in order. If target files already exist, abort the transaction.
+    * Transactional: No. If fails, the target files could have some rows written.
+    * Resumable: No.
+* **overwrite**:
+    * Behavior: This mode writes rows to the target files in order. If target files already exist, this re-write from the beginning of the file.
+    * Transactional: No. If fails, the target files could have some rows written.
+    * Resumable: No.
+* **delete_files_in_advance**:
+    * Behavior: This mode delete files at first, then writes rows to the target files in order.
+    * Transactional: No. If fails, the target files could be removed.
+    * Resumable: No.
+* **delete_recursive_in_advance**:
+    * Behavior: This mode delete directories recursively at first, then writes rows to the target files in order.
+    * Transactional: No. If fails, the target files could be removed.
+    * Resumable: No.
+* **replace**:
+    * Behavior: This mode writes rows to the workspace files in order, then replace them to target directories. This **replace** is not **atomic** because hdfs api does not have atomic replace. 
+    * Transactional: No. If fails, the target files could be removed. 
+    * Resumable: No.
+
 ## Example
 
 ```yaml

data/build.gradle

@@ -15,20 +15,20 @@ configurations {
     provided
 }
 
-version = "0.2.4"
+version = "0.3.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.8"
-    provided "org.embulk:embulk-core:0.8.8"
+    compile  "org.embulk:embulk-core:0.8.38"
+    provided "org.embulk:embulk-core:0.8.38"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     compile 'org.apache.hadoop:hadoop-client:2.6.0'
     compile 'com.google.guava:guava:15.0'
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.8:tests"
-    testCompile "org.embulk:embulk-standards:0.8.8"
+    testCompile "org.embulk:embulk-core:0.8.38:tests"
+    testCompile "org.embulk:embulk-standards:0.8.38"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -72,9 +72,11 @@ task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
     script "pkg/${project.name}-${project.version}.gem"
 }
 
-task "package"(dependsOn: ["gemspec", "classpath"]) << {
-    println "> Build succeeded."
-    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
 }
 
 task gemspec {

data/example/config.yml

@@ -6,12 +6,14 @@ hdfs_example: &hdfs_example
     fs.defaultFS: 'hdfs://hadoop-nn1:8020'
     fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
     fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
+    fs.trash.interval: 3600
 
 local_fs_example: &local_fs_example
   config:
     fs.defaultFS: 'file:///'
     fs.hdfs.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
     fs.file.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
+    fs.trash.interval: 3600
     io.compression.codecs: 'org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec'
 
 in:
@@ -38,7 +40,7 @@ out:
   <<: *local_fs_example
   path_prefix: /tmp/embulk-output-hdfs_example/file_
   file_ext: csv
-  delete_in_advance: FILE_ONLY
+  mode: replace
   formatter:
     type: csv
     newline: CRLF

data/example/config_deprecated_option.yml

@@ -0,0 +1,52 @@
+hdfs_example: &hdfs_example
+  config_files:
+    - /etc/hadoop/conf/core-site.xml
+    - /etc/hadoop/conf/hdfs-site.xml
+  config:
+    fs.defaultFS: 'hdfs://hadoop-nn1:8020'
+    fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
+    fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
+
+local_fs_example: &local_fs_example
+  config:
+    fs.defaultFS: 'file:///'
+    fs.hdfs.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
+    fs.file.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
+    io.compression.codecs: 'org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec'
+
+in:
+  type: file
+  path_prefix: example/data
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    header_line: true
+    stop_on_invalid_record: true
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: comment, type: string}
+
+
+out:
+  type: hdfs
+  <<: *local_fs_example
+  path_prefix: /tmp/embulk-output-hdfs_example/file_
+  file_ext: csv
+  delete_in_advance: FILE_ONLY
+  formatter:
+    type: csv
+    newline: CRLF
+    newline_in_field: LF
+    header_line: true
+    charset: UTF-8
+    quote_policy: NONE
+    quote: '"'
+    escape: '\'
+    null_string: ''
+    default_timezone: UTC

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Wed Jan 13 12:41:02 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -6,20 +6,38 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'embulk-output-hdfs'

data/src/main/java/org/embulk/output/hdfs/HdfsFileOutput.java

@@ -0,0 +1,160 @@
+package org.embulk.output.hdfs;
+
+import org.apache.hadoop.fs.Path;
+import org.embulk.config.TaskReport;
+import org.embulk.output.hdfs.HdfsFileOutputPlugin.PluginTask;
+import org.embulk.output.hdfs.client.HdfsClient;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class HdfsFileOutput
+        implements FileOutput, TransactionalFileOutput
+{
+    private static final Logger logger = Exec.getLogger(HdfsFileOutput.class);
+    private final RetryExecutor re = RetryExecutor.retryExecutor()
+            .withRetryLimit(3)
+            .withMaxRetryWait(500) // ms
+            .withMaxRetryWait(10 * 60 * 1000); // ms
+
+    private final HdfsClient hdfsClient;
+    private final int taskIdx;
+    private final String pathPrefix;
+    private final String sequenceFormat;
+    private final String fileExt;
+    private final boolean overwrite;
+
+    private int fileIdx = 0;
+    private Path currentPath = null;
+    private OutputStream o = null;
+
+    public HdfsFileOutput(PluginTask task, String pathPrefix, boolean overwrite, int taskIdx)
+    {
+        this.hdfsClient = HdfsClient.build(task);
+        this.pathPrefix = pathPrefix;
+        this.taskIdx = taskIdx;
+        this.sequenceFormat = task.getSequenceFormat();
+        this.fileExt = task.getFileExt();
+        this.overwrite = overwrite;
+    }
+
+    @Override
+    public void abort()
+    {
+    }
+
+    @Override
+    public TaskReport commit()
+    {
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public void nextFile()
+    {
+        closeCurrentStream();
+        currentPath = newPath();
+        fileIdx++;
+    }
+
+    @Override
+    public void add(Buffer buffer)
+    {
+        try {
+            // this implementation is for creating file when there is data.
+            if (o == null) {
+                o = hdfsClient.create(currentPath, overwrite);
+                logger.info("Uploading '{}'", currentPath);
+            }
+            write(buffer);
+        }
+        catch (RetryExecutor.RetryGiveupException e) {
+            throw new RuntimeException(e);
+        }
+        finally {
+            buffer.release();
+        }
+    }
+
+    @Override
+    public void finish()
+    {
+        closeCurrentStream();
+    }
+
+    @Override
+    public void close()
+    {
+        closeCurrentStream();
+        hdfsClient.close();
+    }
+
+    private void write(final Buffer buffer)
+            throws RetryExecutor.RetryGiveupException
+    {
+        re.run(new RetryExecutor.Retryable<Void>()
+        {
+            @Override
+            public Void call()
+                    throws Exception
+            {
+                o.write(buffer.array(), buffer.offset(), buffer.limit());
+                return null;
+            }
+
+            @Override
+            public boolean isRetryableException(Exception exception)
+            {
+                return true; // TODO: which Exception is retryable?
+            }
+
+            @Override
+            public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                    throws RetryExecutor.RetryGiveupException
+            {
+                String m = String.format(
+                        "%s. (Retry: Count: %d, Limit: %d, Wait: %d ms)",
+                        exception.getMessage(),
+                        retryCount,
+                        retryLimit,
+                        retryWait);
+                logger.warn(m, exception);
+            }
+
+            @Override
+            public void onGiveup(Exception firstException, Exception lastException)
+                    throws RetryExecutor.RetryGiveupException
+            {
+            }
+        });
+    }
+
+    private Path newPath()
+    {
+        return new Path(pathPrefix + getSequence() + fileExt);
+    }
+
+    private String getSequence()
+    {
+        return String.format(sequenceFormat, taskIdx, fileIdx);
+    }
+
+    private void closeCurrentStream()
+    {
+        if (o != null) {
+            try {
+                o.close();
+                o = null;
+            }
+            catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}