embulk-input-http 0.0.13 → 0.0.14

data/gradlew

@@ -1,164 +0,0 @@
-#!/usr/bin/env bash
-
-##############################################################################
-##
-##  Gradle start up script for UN*X
-##
-##############################################################################
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
-
-APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
-
-# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
-
-warn ( ) {
-    echo "$*"
-}
-
-die ( ) {
-    echo
-    echo "$*"
-    echo
-    exit 1
-}
-
-# OS specific support (must be 'true' or 'false').
-cygwin=false
-msys=false
-darwin=false
-case "`uname`" in
-  CYGWIN* )
-    cygwin=true
-    ;;
-  Darwin* )
-    darwin=true
-    ;;
-  MINGW* )
-    msys=true
-    ;;
-esac
-
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
-# Determine the Java command to use to start the JVM.
-if [ -n "$JAVA_HOME" ] ; then
-    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
-        # IBM's JDK on AIX uses strange locations for the executables
-        JAVACMD="$JAVA_HOME/jre/sh/java"
-    else
-        JAVACMD="$JAVA_HOME/bin/java"
-    fi
-    if [ ! -x "$JAVACMD" ] ; then
-        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-    fi
-else
-    JAVACMD="java"
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-
-Please set the JAVA_HOME variable in your environment to match the
-location of your Java installation."
-fi
-
-# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
-    MAX_FD_LIMIT=`ulimit -H -n`
-    if [ $? -eq 0 ] ; then
-        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
-            MAX_FD="$MAX_FD_LIMIT"
-        fi
-        ulimit -n $MAX_FD
-        if [ $? -ne 0 ] ; then
-            warn "Could not set maximum file descriptor limit: $MAX_FD"
-        fi
-    else
-        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
-    fi
-fi
-
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
-    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
-
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-
-    # We build the pattern for arguments to be converted via cygpath
-    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
-    SEP=""
-    for dir in $ROOTDIRSRAW ; do
-        ROOTDIRS="$ROOTDIRS$SEP$dir"
-        SEP="|"
-    done
-    OURCYGPATTERN="(^($ROOTDIRS))"
-    # Add a user-defined pattern to the cygpath arguments
-    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
-        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
-    fi
-    # Now convert the arguments - kludge to limit ourselves to /bin/sh
-    i=0
-    for arg in "$@" ; do
-        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
-        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
-
-        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
-        else
-            eval `echo args$i`="\"$arg\""
-        fi
-        i=$((i+1))
-    done
-    case $i in
-        (0) set -- ;;
-        (1) set -- "$args0" ;;
-        (2) set -- "$args0" "$args1" ;;
-        (3) set -- "$args0" "$args1" "$args2" ;;
-        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
-        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
-        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
-        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
-        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
-        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
-    esac
-fi
-
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
-}
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
-
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

data/gradlew.bat

@@ -1,90 +0,0 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega

data/lib/embulk/input/http.rb

@@ -1,3 +0,0 @@
-Embulk::JavaPlugin.register_input(
-  "http", "org.embulk.input.HttpInputPlugin",
-  File.expand_path('../../../../classpath', __FILE__))

data/settings.gradle

@@ -1 +0,0 @@
-rootProject.name = 'embulk-input-http'

data/src/main/java/org/embulk/input/BasicAuthConfig.java

@@ -1,39 +0,0 @@
-package org.embulk.input;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
-public class BasicAuthConfig {
-
-    private final String user;
-    private final String password;
-
-    @JsonCreator
-    public BasicAuthConfig(
-            @JsonProperty("user") String user,
-            @JsonProperty("password") String password) {
-        this.user = user;
-        this.password = password;
-    }
-
-    @JsonProperty("user")
-    public String getUser() {
-        return user;
-    }
-
-    @JsonProperty("password")
-    public String getPassword() {
-        return password;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hashCode(user, password);
-    }
-
-    @Override
-    public String toString() {
-        return String.format("BasicAuthConfig[%s, %s]", getUser(), getPassword());
-    }
-}

data/src/main/java/org/embulk/input/HttpInputPlugin.java

@@ -1,332 +0,0 @@
-package org.embulk.input;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Lists;
-import org.apache.http.Header;
-import org.apache.http.NameValuePair;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.message.BasicHeader;
-import org.apache.http.message.BasicNameValuePair;
-import org.embulk.config.*;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
-import org.embulk.spi.util.RetryExecutor;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class HttpInputPlugin implements FileInputPlugin {
-
-    private final Logger logger = Exec.getLogger(getClass());
-
-    public interface PluginTask extends Task {
-        @Config("url")
-        public String getUrl();
-
-        @Config("charset")
-        @ConfigDefault("\"utf-8\"")
-        public String getCharset();
-
-        @Config("method")
-        @ConfigDefault("\"get\"")
-        public String getMethod();
-
-        @Config("user_agent")
-        @ConfigDefault("\"Embulk::Input::HttpInputPlugin\"")
-        public String getUserAgent();
-
-        @Config("open_timeout")
-        @ConfigDefault("2000")
-        public int getOpenTimeout();
-
-        @Config("read_timeout")
-        @ConfigDefault("10000")
-        public int getReadTimeout();
-
-        @Config("max_retries")
-        @ConfigDefault("5")
-        public int getMaxRetries();
-
-        @Config("retry_interval")
-        @ConfigDefault("10000")
-        public int getRetryInterval();
-
-        @Config("request_interval")
-        @ConfigDefault("0")
-        public int getRequestInterval();
-
-        public void setRequestInterval(int requestInterval);
-
-        @Config("interval_includes_response_time")
-        @ConfigDefault("null")
-        public boolean getIntervalIncludesResponseTime();
-
-        @Config("params")
-        @ConfigDefault("null")
-        public Optional<ParamsConfig> getParams();
-
-        @Config("basic_auth")
-        @ConfigDefault("null")
-        public Optional<BasicAuthConfig> getBasicAuth();
-
-        @Config("pager")
-        @ConfigDefault("null")
-        public Optional<PagerConfig> getPager();
-
-        @Config("request_headers")
-        @ConfigDefault("{}")
-        public Map<String, String> getRequestHeaders();
-
-        @ConfigInject
-        public BufferAllocator getBufferAllocator();
-
-        public List<List<QueryConfig.Query>> getQueries();
-
-        public void setQueries(List<List<QueryConfig.Query>> queries);
-
-        public HttpMethod getHttpMethod();
-
-        public void setHttpMethod(HttpMethod httpMethod);
-    }
-
-    public enum HttpMethod {
-        POST,
-        GET
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        final int tasks;
-        if (task.getParams().isPresent()) {
-            List<List<QueryConfig.Query>> queries = task.getParams().get().generateQueries(task.getPager());
-            task.setQueries(queries);
-            tasks = queries.size();
-        } else if (task.getPager().isPresent()) {
-            List<List<QueryConfig.Query>> queries = task.getPager().get().expand();
-            task.setQueries(queries);
-            tasks = queries.size();
-        } else {
-            task.setQueries(Lists.<List<QueryConfig.Query>>newArrayList());
-            task.setRequestInterval(0);
-            tasks = 1;
-        }
-
-        task.setHttpMethod(HttpMethod.valueOf(task.getMethod().toUpperCase()));
-
-        return resume(task.dump(), tasks, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control) {
-        control.run(taskSource, taskCount);
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {
-    }
-
-    @Override
-    public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        HttpRequestBase request;
-        try {
-            request = makeRequest(task, taskIndex);
-        } catch (URISyntaxException | UnsupportedEncodingException e) {
-            throw Throwables.propagate(e);
-        }
-
-        HttpClientBuilder builder = HttpClientBuilder.create()
-                .disableAutomaticRetries()
-                .setDefaultRequestConfig(makeRequestConfig(task))
-                .setDefaultHeaders(makeHeaders(task));
-
-        if (task.getBasicAuth().isPresent()) {
-            builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(), request));
-        }
-
-        HttpClient client = builder.build();
-
-        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(), request.getURI().toString()));
-
-        RetryableHandler retryable = new RetryableHandler(client, request);
-        long startTimeMills = System.currentTimeMillis();
-        try {
-            RetryExecutor.retryExecutor().
-                    withRetryLimit(task.getMaxRetries()).
-                    withInitialRetryWait(task.getRetryInterval()).
-                    withMaxRetryWait(30 * 60 * 1000).
-                    runInterruptible(retryable);
-            InputStream stream = retryable.getResponse().getEntity().getContent();
-            PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
-            stream = null;
-            return input;
-        } catch (Exception e) {
-            throw Throwables.propagate(e);
-        }
-    }
-
-    private CredentialsProvider makeCredentialsProvider(BasicAuthConfig config, HttpRequestBase scopeRequest) {
-        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-        final AuthScope authScope = new AuthScope(scopeRequest.getURI().getHost(),
-                scopeRequest.getURI().getPort());
-        credentialsProvider.setCredentials(authScope,
-                new UsernamePasswordCredentials(config.getUser(), config.getPassword()));
-        return credentialsProvider;
-    }
-
-    private HttpRequestBase makeRequest(PluginTask task, int taskIndex)
-            throws URISyntaxException, UnsupportedEncodingException {
-        final List<QueryConfig.Query> queries = (task.getQueries().isEmpty()) ?
-                null : task.getQueries().get(taskIndex);
-        if (task.getHttpMethod() == HttpMethod.GET) {
-            HttpGet request = new HttpGet(task.getUrl());
-            if (queries != null) {
-                URIBuilder builder = new URIBuilder(request.getURI());
-                for (QueryConfig.Query q : queries) {
-                    for (String v : q.getValues()) {
-                        builder.addParameter(q.getName(), v);
-                    }
-                }
-                request.setURI(builder.build());
-            }
-            return request;
-        } else if (task.getHttpMethod() == HttpMethod.POST) {
-            HttpPost request = new HttpPost(task.getUrl());
-            if (queries != null) {
-                List<NameValuePair> pairs = new ArrayList<>();
-                for (QueryConfig.Query q : queries) {
-                    for (String v : q.getValues()) {
-                        pairs.add(new BasicNameValuePair(q.getName(), v));
-                    }
-                }
-                request.setEntity(new UrlEncodedFormEntity(pairs));
-            }
-            return request;
-        }
-        throw new IllegalArgumentException(String.format("Unsupported http method %s", task.getMethod()));
-    }
-
-    private List<Header> makeHeaders(PluginTask task) {
-        List<Header> headers = new ArrayList<>();
-        headers.add(new BasicHeader("Accept", "*/*"));
-        headers.add(new BasicHeader("Accept-Charset", task.getCharset()));
-        headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
-        headers.add(new BasicHeader("Accept-Language", "en-us,en;q=0.5"));
-        headers.add(new BasicHeader("User-Agent", task.getUserAgent()));
-        for (Map.Entry<String, String> entry : task.getRequestHeaders().entrySet()) {
-            headers.add(new BasicHeader(entry.getKey(), entry.getValue()));
-        }
-        return headers;
-    }
-
-    private RequestConfig makeRequestConfig(PluginTask task) {
-        return RequestConfig.custom()
-                .setCircularRedirectsAllowed(true)
-                .setMaxRedirects(10)
-                .setRedirectsEnabled(true)
-                .setConnectTimeout(task.getOpenTimeout())
-                .setSocketTimeout(task.getReadTimeout())
-                .build();
-    }
-
-    public static class PluginFileInput extends InputStreamFileInput
-            implements TransactionalFileInput {
-
-        private final Logger logger = Exec.getLogger(getClass());
-
-        private final long startTimeMills;
-        private final PluginTask task;
-
-        public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills) {
-            super(task.getBufferAllocator(), new SingleFileProvider(stream));
-            this.startTimeMills = startTimeMills;
-            this.task = task;
-        }
-
-        public TaskReport commit() {
-            return Exec.newTaskReport();
-        }
-
-        @Override
-        public void close() {
-            super.close();
-            handleInterval();
-        }
-
-        @Override
-        public void abort() {
-        }
-
-        protected void handleInterval() {
-            if (task.getRequestInterval() <= 0) {
-                return;
-            }
-            long interval = task.getRequestInterval();
-            if (task.getIntervalIncludesResponseTime()) {
-                interval = interval - (System.currentTimeMillis() - startTimeMills);
-            }
-            if (interval > 0) {
-                logger.info(String.format("waiting %d msec ...", interval));
-                try {
-                    Thread.sleep(interval);
-                } catch (InterruptedException e) {
-                    Throwables.propagate(e);
-                }
-            }
-        }
-
-        private static class SingleFileProvider
-                implements InputStreamFileInput.Provider {
-
-            private final InputStream stream;
-            private boolean opened = false;
-
-            public SingleFileProvider(InputStream stream) {
-                this.stream = stream;
-            }
-
-            @Override
-            public InputStream openNext() throws IOException {
-                if (opened) {
-                    return null;
-                }
-                opened = true;
-                return stream;
-            }
-
-            @Override
-            public void close() throws IOException {
-                if (!opened) {
-                    stream.close();
-                }
-            }
-        }
-    }
-}