embulk-executor-remoteserver 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 471122040cf026eb287627d106e4b7b014bde970
+  data.tar.gz: cda196374b2e6737d73bbccb3ec5e33826cf8f45
+SHA512:
+  metadata.gz: 0bba4418aaf83000aec6cad419e32c28eef1598a2a25441400e165527b5558956df2d28d7bd6a1eb1fb3c19a5d7f961470b7c5fd5a3dc42b5ca6315f5ba50a58
+  data.tar.gz: 8f28f104c3d2ba6424e5534adc62b95be593433cf77ad3a70bd4703263f6974e3fa72ed8afffed0390b3334b398c9817089efaaab2e9c8851e7f12f8ad7c66ce

data/.circleci/config.yml

@@ -0,0 +1,42 @@
+version: 2
+jobs:
+  build:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-gradle-{{ checksum "gradle/dependency-locks/compileClasspath.lockfile" }}-{{ checksum "gradle/dependency-locks/testCompileClasspath.lockfile" }}
+            - v1-gradle-
+      - run: ./test/setup.sh
+      - run: ./gradlew check --info
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/junit/
+            find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/junit/ \;
+          when: always
+      - store_test_results:
+          path: ~/junit
+      - store_artifacts:
+          path: ~/junit
+
+      - save_cache:
+          paths:
+            - "~/.gradle"
+            - "~/.m2"
+          key: v1-gradle-{{ checksum "gradle/dependency-locks/compileClasspath.lockfile" }}-{{ checksum "gradle/dependency-locks/testCompileClasspath.lockfile" }}
+
+      - deploy:
+          name: Push Gem to RubyGems.org and bump up
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "release" ]; then
+              mkdir -p ~/.gem
+              curl -f -u ${RUBYGEMS_USER}:${RUBYGEMS_PASSWORD} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials
+              git config --global user.email "shiketaudonko41@gmail.com"
+              git config --global user.name "kamatama41"
+              git checkout master
+              git reset --hard origin/master
+              ./gradlew release -Prelease.useAutomaticVersion=true
+            fi

data/.gitignore

@@ -0,0 +1,14 @@
+*~
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project
+*.iml
+.bundle

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM openjdk:8
+WORKDIR /usr/src/app
+COPY build.gradle settings.gradle gradlew gradle.properties ./
+COPY ./gradle ./gradle
+COPY ./src ./src
+COPY ./.git ./.git
+RUN ./gradlew --no-daemon executableEmbulkServer
+
+FROM openjdk:8-jre
+WORKDIR /root/
+COPY ./docker .
+COPY --from=0 /usr/src/app/build/libs/embulk-server-*.jar ./embulk-server.jar
+ENTRYPOINT ["/root/run_embulk_server.sh"]
+EXPOSE 30001

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Shinichi Ishimura
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# Remote server executor plugin for Embulk
+
+Embulk executor plugin to run plugins on remote server.
+
+## Overview
+
+* **Plugin type**: executor
+
+## Configuration
+
+- **hosts**: List of remote servers. If not specified, the executor runs as local mode, which start Embulk server on its own process (array of string)
+- **timeout_seconds**: Timeout seconds of the whole execution (integer, default: `3600`)
+
+## Example
+
+```yaml
+exec:
+  type: remoteserver
+  hosts:
+    - {name:remote-server1.com, port:30001}
+    - {name:remote-server2.com, port:30001}
+  timeout_seconds: 86400
+```
+
+
+## Build
+
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```

data/build.gradle

@@ -0,0 +1,70 @@
+plugins {
+    id "java"
+    id "com.github.kamatama41.embulk" version "0.5.2"
+    id "net.researchgate.release" version "2.8.0"
+}
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+    maven { url 'http://kamatama41.github.com/maven-repository/repository' }
+}
+
+configurations {
+    serverRuntime
+    serverRuntime.extendsFrom runtime, testRuntime
+
+    [compileClasspath, testCompileClasspath].each {
+        it.resolutionStrategy.activateDependencyLocking()
+    }
+}
+
+dependencies {
+    compile("com.github.kamatama41:nsocket:0.2.10") {
+        // Use embulk-core's Jackson (2.6.7) instead
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+    }
+    // Followings are needed for running EmbulkServer
+    implementation "org.embulk:embulk-standards:0.9.16"
+    serverRuntime "org.embulk:embulk-standards:0.9.16"
+    implementation "ch.qos.logback:logback-classic:1.2.3"
+    serverRuntime "ch.qos.logback:logback-classic:1.2.3"
+    implementation "ch.qos.logback:logback-core:1.2.3"
+    serverRuntime "ch.qos.logback:logback-core:1.2.3"
+
+    testImplementation "org.embulk:embulk-standards:0.9.16"
+    testImplementation "org.embulk:embulk-test:0.9.16"
+    testImplementation "com.github.kamatama41:embulk-test-helpers:0.8.0"
+    testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
+    testRuntime "org.junit.jupiter:junit-jupiter-engine:5.3.2"
+}
+
+task executableEmbulkServer(type: Jar) {
+    manifest {
+        attributes 'Main-Class': "org.embulk.executor.remoteserver.Launcher"
+    }
+    baseName = 'embulk-server'
+    from { (sourceSets.main.output + configurations.serverRuntime).collect { it.isDirectory() ? it : zipTree(it) } }
+    with jar
+}
+
+embulk {
+    version = "0.9.16"
+    category = "executor"
+    name = "docker"
+    authors = ["Shinichi Ishimura"]
+    email = "shiketaudonko41@gmail.com"
+    description = "Embulk executor plugin to run plugins on remote server."
+    licenses = ["MIT"]
+    homepage = "https://github.com/kamatama41/embulk-executor-remoteserver"
+}
+
+test {
+    useJUnitPlatform()
+}
+
+release {
+    git { requireBranch = 'master' }
+}
+afterReleaseBuild.dependsOn gemPush

data/docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.2'
+services:
+  server1:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      LOG_LEVEL: debug
+    ports:
+      - "24224:30001"
+    volumes:
+      - ./tmp/output:/output
+      - ./src/test/resources/json:/root/src/test/resources/json
+  server2:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      LOG_LEVEL: debug
+    ports:
+      - "24225:30001"
+    volumes:
+      - ./tmp/output:/output
+      - ./src/test/resources/json:/root/src/test/resources/json

data/docker/run_embulk_server.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cmd="java ${JVM_OPTS} -jar /root/embulk-server.jar"
+exec ${cmd}

data/gradle.properties

@@ -0,0 +1 @@
+version=0.1.0

data/gradle/dependency-locks/compileClasspath.lockfile

@@ -0,0 +1,40 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+aopalliance:aopalliance:1.0
+ch.qos.logback:logback-classic:1.2.3
+ch.qos.logback:logback-core:1.2.3
+com.fasterxml.jackson.core:jackson-annotations:2.6.7
+com.fasterxml.jackson.core:jackson-core:2.6.7
+com.fasterxml.jackson.core:jackson-databind:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
+com.fasterxml.jackson.module:jackson-module-guice:2.6.7
+com.github.kamatama41:nsocket:0.2.10
+com.google.code.findbugs:annotations:3.0.0
+com.google.guava:guava:18.0
+com.google.inject.extensions:guice-multibindings:4.0
+com.google.inject:guice:4.0
+com.ibm.icu:icu4j:54.1.1
+commons-beanutils:commons-beanutils-core:1.8.3
+commons-cli:commons-cli:1.3.1
+commons-collections:commons-collections:3.2.1
+commons-lang:commons-lang:2.4
+io.airlift:slice:0.9
+io.netty:netty-buffer:4.0.44.Final
+io.netty:netty-common:4.0.44.Final
+javax.inject:javax.inject:1
+javax.validation:validation-api:1.1.0.Final
+joda-time:joda-time:2.9.2
+org.apache.bval:bval-core:0.5
+org.apache.bval:bval-jsr303:0.5
+org.apache.commons:commons-compress:1.10
+org.apache.commons:commons-lang3:3.4
+org.apache.velocity:velocity:1.7
+org.embulk:embulk-core:0.9.16
+org.embulk:embulk-standards:0.9.16
+org.jruby:jruby-complete:9.1.15.0
+org.msgpack:msgpack-core:0.8.11
+org.slf4j:slf4j-api:1.7.25
+org.yaml:snakeyaml:1.18

data/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -0,0 +1,49 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+aopalliance:aopalliance:1.0
+ch.qos.logback:logback-classic:1.2.3
+ch.qos.logback:logback-core:1.2.3
+com.fasterxml.jackson.core:jackson-annotations:2.6.7
+com.fasterxml.jackson.core:jackson-core:2.6.7
+com.fasterxml.jackson.core:jackson-databind:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
+com.fasterxml.jackson.module:jackson-module-guice:2.6.7
+com.github.kamatama41:embulk-test-helpers:0.8.0
+com.github.kamatama41:nsocket:0.2.10
+com.google.code.findbugs:annotations:3.0.0
+com.google.guava:guava:18.0
+com.google.inject.extensions:guice-multibindings:4.0
+com.google.inject:guice:4.0
+com.ibm.icu:icu4j:54.1.1
+commons-beanutils:commons-beanutils-core:1.8.3
+commons-cli:commons-cli:1.3.1
+commons-collections:commons-collections:3.2.1
+commons-lang:commons-lang:2.4
+io.airlift:slice:0.9
+io.netty:netty-buffer:4.0.44.Final
+io.netty:netty-common:4.0.44.Final
+javax.inject:javax.inject:1
+javax.validation:validation-api:1.1.0.Final
+joda-time:joda-time:2.9.2
+junit:junit:4.12
+org.apache.bval:bval-core:0.5
+org.apache.bval:bval-jsr303:0.5
+org.apache.commons:commons-compress:1.10
+org.apache.commons:commons-lang3:3.4
+org.apache.velocity:velocity:1.7
+org.apiguardian:apiguardian-api:1.0.0
+org.embulk:embulk-core:0.9.16
+org.embulk:embulk-standards:0.9.16
+org.embulk:embulk-test:0.9.16
+org.hamcrest:hamcrest-core:1.3
+org.hamcrest:hamcrest-library:1.3
+org.jruby:jruby-complete:9.1.15.0
+org.junit.jupiter:junit-jupiter-api:5.3.2
+org.junit.platform:junit-platform-commons:1.3.2
+org.msgpack:msgpack-core:0.8.11
+org.opentest4j:opentest4j:1.1.1
+org.slf4j:slf4j-api:1.7.25
+org.yaml:snakeyaml:1.18

data/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

data/gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"