embulk-input-s3 0.1.7 → 0.2.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-27 00:00:00.000000000 Z
+date: 2015-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -45,17 +45,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- ChangeLog
-- README.md
 - build.gradle
-- gradle/wrapper/gradle-wrapper.jar
-- gradle/wrapper/gradle-wrapper.properties
-- gradlew
-- gradlew.bat
 - lib/embulk/input/s3.rb
-- src/main/java/org/embulk/input/s3/RetryExecutor.java
-- src/main/java/org/embulk/input/s3/RetryableInputStream.java
+- src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - classpath/aws-java-sdk-core-1.9.22.jar
@@ -63,9 +55,10 @@ files:
 - classpath/aws-java-sdk-s3-1.9.22.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-input-s3-0.1.7.jar
+- classpath/embulk-input-s3-0.2.0.jar
 - classpath/httpclient-4.3.4.jar
 - classpath/httpcore-4.3.2.jar
+- classpath/joda-time-2.8.1.jar
 homepage: https://github.com/embulk/embulk-input-s3
 licenses:
 - Apache 2.0

data/.gitignore

@@ -1,6 +0,0 @@
-*~
-.idea
-build/
-/classpath/
-/.gradle
-/*.gem

data/ChangeLog

@@ -1,25 +0,0 @@
-Release 0.1.7 - 2015-03-27
-
-* Keep last_path in case of no input files
-
-Release 0.1.6 - 2015-03-27
-
-* Fixed error when there are no input files
-
-Release 0.1.5 - 2015-03-23
-
-* Implemented retrying
-
-Release 0.1.3 - 2015-02-25
-
-* Upgraded aws-sdk to 1.9.22
-
-Release 0.1.2 - 2015-02-20
-
-* Sets last_path to the next config.
-* Upgraded embulk version to 0.4.6.
-
-Release 0.1.1 - 2015-02-20
-
-* Upgraded embulk version to 0.4.2.
-

data/README.md

@@ -1,34 +0,0 @@
-# S3 file input plugin for Embulk
-
-## Overview
-
-* Plugin type: **file input**
-* Resume supported: **yes**
-* Cleanup supported: **yes**
-
-## Configuration
-
-- **bucket** S3 bucket name (string, required)
-- **path_prefix** prefix of target keys (string, required)
-- **endpoint** S3 endpoint login user name (string, optional)
-- **access_key_id** AWS access key id (string, required)
-- **secret_access_key** AWS secret key (string, required)
-
-## Example
-
-```yaml
-in:
-  type: s3
-  bucket: my-s3-bucket
-  path_prefix: logs/csv-
-  endpoint: s3-us-west-1.amazonaws.com
-  access_key_id: ABCXYZ123ABCXYZ123
-  secret_access_key: AbCxYz123aBcXyZ123
-```
-
-## Build
-
-```
-./gradlew gem
-```
-

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +0,0 @@
-#Wed Feb 04 13:46:12 PST 2015
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip

data/gradlew

@@ -1,164 +0,0 @@
-#!/usr/bin/env bash
-
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
-##############################################################################
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn ( ) {
-    echo "$*"
-}
-
-die ( ) {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-esac
-
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=$((i+1))
-    done
-    case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
-}
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
-
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

data/gradlew.bat

@@ -1,90 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega

data/src/main/java/org/embulk/input/s3/RetryExecutor.java

@@ -1,130 +0,0 @@
-package org.embulk.input.s3;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-
-public class RetryExecutor
-{
-    public static RetryExecutor retryExecutor()
-    {
-        // TODO default configuration
-        return new RetryExecutor(3, 500, 30*60*1000);
-    }
-
-    public static class RetryGiveupException
-            extends ExecutionException
-    {
-        public RetryGiveupException(String message, Exception cause)
-        {
-            super(cause);
-        }
-
-        public RetryGiveupException(Exception cause)
-        {
-            super(cause);
-        }
-
-        public Exception getCause()
-        {
-            return (Exception) super.getCause();
-        }
-    }
-
-    public static interface Retryable<T>
-            extends Callable<T>
-    {
-        public T call()
-            throws Exception;
-
-        public boolean isRetryableException(Exception exception);
-
-        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-            throws RetryGiveupException;
-
-        public void onGiveup(Exception firstException, Exception lastException)
-            throws RetryGiveupException;
-    }
-
-    private final int retryLimit;
-    private final int initialRetryWait;
-    private final int maxRetryWait;
-
-    private RetryExecutor(int retryLimit, int initialRetryWait, int maxRetryWait)
-    {
-        this.retryLimit = retryLimit;
-        this.initialRetryWait = initialRetryWait;
-        this.maxRetryWait = maxRetryWait;
-    }
-
-    public RetryExecutor withRetryLimit(int count)
-    {
-        return new RetryExecutor(count, initialRetryWait, maxRetryWait);
-    }
-
-    public RetryExecutor withInitialRetryWait(int msec)
-    {
-        return new RetryExecutor(retryLimit, msec, maxRetryWait);
-    }
-
-    public RetryExecutor withMaxRetryWait(int msec)
-    {
-        return new RetryExecutor(retryLimit, initialRetryWait, msec);
-    }
-
-    public <T> T runInterruptible(Retryable<T> op)
-            throws InterruptedException, RetryGiveupException
-    {
-        return run(op, true);
-    }
-
-    public <T> T run(Retryable<T> op)
-            throws RetryGiveupException
-    {
-        try {
-            return run(op, false);
-        } catch (InterruptedException ex) {
-            throw new RetryGiveupException("Unexpected interruption", ex);
-        }
-    }
-
-    private <T> T run(Retryable<T> op, boolean interruptible)
-            throws InterruptedException, RetryGiveupException
-    {
-        int retryWait = initialRetryWait;
-        int retryCount = 0;
-
-        Exception firstException = null;
-
-        while(true) {
-            try {
-                return op.call();
-            } catch (Exception exception) {
-                if (firstException == null) {
-                    firstException = exception;
-                }
-                if (!op.isRetryableException(exception) || retryCount >= retryLimit) {
-                    op.onGiveup(firstException, exception);
-                    throw new RetryGiveupException(firstException);
-                }
-
-                retryCount++;
-                op.onRetry(exception, retryCount, retryLimit, retryWait);
-
-                try {
-                    Thread.sleep(retryWait);
-                } catch (InterruptedException ex) {
-                    if (interruptible) {
-                        throw ex;
-                    }
-                }
-
-                // exponential back-off with hard limit
-                retryWait *= 2;
-                if (retryWait > maxRetryWait) {
-                    retryWait = maxRetryWait;
-                }
-            }
-        }
-    }
-}
-