embulk-input-http 0.0.14 → 0.0.15

data/gradlew

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched.
+if $cygwin ; then
+    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >&-
+APP_HOME="`pwd -P`"
+cd "$SAVED" >&-
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

data/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/input/http.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_input(
+  "http", "org.embulk.input.http.HttpFileInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'embulk-input-http'

data/src/main/java/org/embulk/input/http/BasicAuthConfig.java

@@ -0,0 +1,39 @@
+package org.embulk.input.http;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+
+public class BasicAuthConfig {
+
+    private final String user;
+    private final String password;
+
+    @JsonCreator
+    public BasicAuthConfig(
+            @JsonProperty("user") String user,
+            @JsonProperty("password") String password) {
+        this.user = user;
+        this.password = password;
+    }
+
+    @JsonProperty("user")
+    public String getUser() {
+        return user;
+    }
+
+    @JsonProperty("password")
+    public String getPassword() {
+        return password;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(user, password);
+    }
+
+    @Override
+    public String toString() {
+        return String.format("BasicAuthConfig[%s, %s]", getUser(), getPassword());
+    }
+}

data/src/main/java/org/embulk/input/http/HttpFileInputPlugin.java

@@ -0,0 +1,333 @@
+package org.embulk.input.http;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import org.apache.http.Header;
+import org.apache.http.NameValuePair;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.message.BasicNameValuePair;
+import org.embulk.config.*;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class HttpFileInputPlugin implements FileInputPlugin {
+
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public interface PluginTask extends Task {
+        @Config("url")
+        public String getUrl();
+
+        @Config("charset")
+        @ConfigDefault("\"utf-8\"")
+        public String getCharset();
+
+        @Config("method")
+        @ConfigDefault("\"get\"")
+        public String getMethod();
+
+        @Config("user_agent")
+        @ConfigDefault("\"Embulk::Input::HttpFileInputPlugin\"")
+        public String getUserAgent();
+
+        @Config("open_timeout")
+        @ConfigDefault("2000")
+        public int getOpenTimeout();
+
+        @Config("read_timeout")
+        @ConfigDefault("10000")
+        public int getReadTimeout();
+
+        @Config("max_retries")
+        @ConfigDefault("5")
+        public int getMaxRetries();
+
+        @Config("retry_interval")
+        @ConfigDefault("10000")
+        public int getRetryInterval();
+
+        @Config("request_interval")
+        @ConfigDefault("0")
+        public int getRequestInterval();
+
+        public void setRequestInterval(int requestInterval);
+
+        @Config("interval_includes_response_time")
+        @ConfigDefault("null")
+        public boolean getIntervalIncludesResponseTime();
+
+        @Config("params")
+        @ConfigDefault("null")
+        public Optional<ParamsConfig> getParams();
+
+        @Config("basic_auth")
+        @ConfigDefault("null")
+        public Optional<BasicAuthConfig> getBasicAuth();
+
+        @Config("pager")
+        @ConfigDefault("null")
+        public Optional<PagerConfig> getPager();
+
+        @Config("request_headers")
+        @ConfigDefault("{}")
+        public Map<String, String> getRequestHeaders();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+
+        public List<List<QueryConfig.Query>> getQueries();
+
+        public void setQueries(List<List<QueryConfig.Query>> queries);
+
+        public HttpMethod getHttpMethod();
+
+        public void setHttpMethod(HttpMethod httpMethod);
+    }
+
+    public enum HttpMethod {
+        POST,
+        GET
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        final int tasks;
+        if (task.getParams().isPresent()) {
+            List<List<QueryConfig.Query>> queries = task.getParams().get().generateQueries(task.getPager());
+            task.setQueries(queries);
+            tasks = queries.size();
+        } else if (task.getPager().isPresent()) {
+            List<List<QueryConfig.Query>> queries = task.getPager().get().expand();
+            task.setQueries(queries);
+            tasks = queries.size();
+        } else {
+            task.setQueries(Lists.<List<QueryConfig.Query>>newArrayList());
+            task.setRequestInterval(0);
+            tasks = 1;
+        }
+
+        task.setHttpMethod(HttpMethod.valueOf(task.getMethod().toUpperCase()));
+
+        return resume(task.dump(), tasks, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control) {
+        control.run(taskSource, taskCount);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {
+    }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        HttpRequestBase request;
+        try {
+            request = makeRequest(task, taskIndex);
+        } catch (URISyntaxException | UnsupportedEncodingException e) {
+            throw Throwables.propagate(e);
+        }
+
+        HttpClientBuilder builder = HttpClientBuilder.create()
+                .disableAutomaticRetries()
+                .setDefaultRequestConfig(makeRequestConfig(task))
+                .setDefaultHeaders(makeHeaders(task));
+
+        if (task.getBasicAuth().isPresent()) {
+            builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(), request));
+        }
+
+        HttpClient client = builder.build();
+
+        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
+                request.getURI().toString()));
+
+        RetryableHandler retryable = new RetryableHandler(client, request);
+        long startTimeMills = System.currentTimeMillis();
+        try {
+            RetryExecutor.retryExecutor().
+                    withRetryLimit(task.getMaxRetries()).
+                    withInitialRetryWait(task.getRetryInterval()).
+                    withMaxRetryWait(30 * 60 * 1000).
+                    runInterruptible(retryable);
+            InputStream stream = retryable.getResponse().getEntity().getContent();
+            PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
+            stream = null;
+            return input;
+        } catch (Exception e) {
+            throw Throwables.propagate(e);
+        }
+    }
+
+    private CredentialsProvider makeCredentialsProvider(BasicAuthConfig config, HttpRequestBase scopeRequest) {
+        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+        final AuthScope authScope = new AuthScope(scopeRequest.getURI().getHost(),
+                scopeRequest.getURI().getPort());
+        credentialsProvider.setCredentials(authScope,
+                new UsernamePasswordCredentials(config.getUser(), config.getPassword()));
+        return credentialsProvider;
+    }
+
+    private HttpRequestBase makeRequest(PluginTask task, int taskIndex)
+            throws URISyntaxException, UnsupportedEncodingException {
+        final List<QueryConfig.Query> queries = (task.getQueries().isEmpty()) ?
+                null : task.getQueries().get(taskIndex);
+        if (task.getHttpMethod() == HttpMethod.GET) {
+            HttpGet request = new HttpGet(task.getUrl());
+            if (queries != null) {
+                URIBuilder builder = new URIBuilder(request.getURI());
+                for (QueryConfig.Query q : queries) {
+                    for (String v : q.getValues()) {
+                        builder.addParameter(q.getName(), v);
+                    }
+                }
+                request.setURI(builder.build());
+            }
+            return request;
+        } else if (task.getHttpMethod() == HttpMethod.POST) {
+            HttpPost request = new HttpPost(task.getUrl());
+            if (queries != null) {
+                List<NameValuePair> pairs = new ArrayList<>();
+                for (QueryConfig.Query q : queries) {
+                    for (String v : q.getValues()) {
+                        pairs.add(new BasicNameValuePair(q.getName(), v));
+                    }
+                }
+                request.setEntity(new UrlEncodedFormEntity(pairs));
+            }
+            return request;
+        }
+        throw new IllegalArgumentException(String.format("Unsupported http method %s", task.getMethod()));
+    }
+
+    private List<Header> makeHeaders(PluginTask task) {
+        List<Header> headers = new ArrayList<>();
+        headers.add(new BasicHeader("Accept", "*/*"));
+        headers.add(new BasicHeader("Accept-Charset", task.getCharset()));
+        headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
+        headers.add(new BasicHeader("Accept-Language", "en-us,en;q=0.5"));
+        headers.add(new BasicHeader("User-Agent", task.getUserAgent()));
+        for (Map.Entry<String, String> entry : task.getRequestHeaders().entrySet()) {
+            headers.add(new BasicHeader(entry.getKey(), entry.getValue()));
+        }
+        return headers;
+    }
+
+    private RequestConfig makeRequestConfig(PluginTask task) {
+        return RequestConfig.custom()
+                .setCircularRedirectsAllowed(true)
+                .setMaxRedirects(10)
+                .setRedirectsEnabled(true)
+                .setConnectTimeout(task.getOpenTimeout())
+                .setSocketTimeout(task.getReadTimeout())
+                .build();
+    }
+
+    public static class PluginFileInput extends InputStreamFileInput
+            implements TransactionalFileInput {
+
+        private final Logger logger = Exec.getLogger(getClass());
+
+        private final long startTimeMills;
+        private final PluginTask task;
+
+        public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills) {
+            super(task.getBufferAllocator(), new SingleFileProvider(stream));
+            this.startTimeMills = startTimeMills;
+            this.task = task;
+        }
+
+        public TaskReport commit() {
+            return Exec.newTaskReport();
+        }
+
+        @Override
+        public void close() {
+            super.close();
+            handleInterval();
+        }
+
+        @Override
+        public void abort() {
+        }
+
+        protected void handleInterval() {
+            if (task.getRequestInterval() <= 0) {
+                return;
+            }
+            long interval = task.getRequestInterval();
+            if (task.getIntervalIncludesResponseTime()) {
+                interval = interval - (System.currentTimeMillis() - startTimeMills);
+            }
+            if (interval > 0) {
+                logger.info(String.format("waiting %d msec ...", interval));
+                try {
+                    Thread.sleep(interval);
+                } catch (InterruptedException e) {
+                    Throwables.propagate(e);
+                }
+            }
+        }
+
+        private static class SingleFileProvider
+                implements InputStreamFileInput.Provider {
+
+            private final InputStream stream;
+            private boolean opened = false;
+
+            public SingleFileProvider(InputStream stream) {
+                this.stream = stream;
+            }
+
+            @Override
+            public InputStream openNext() throws IOException {
+                if (opened) {
+                    return null;
+                }
+                opened = true;
+                return stream;
+            }
+
+            @Override
+            public void close() throws IOException {
+                if (!opened) {
+                    stream.close();
+                }
+            }
+        }
+    }
+}