embulk-executor-remoteserver 0.1.0

data/src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java

@@ -0,0 +1,51 @@
+package org.embulk.executor.remoteserver;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.github.kamatama41.nsocket.Command;
+import com.github.kamatama41.nsocket.Connection;
+
+class StartTaskCommand implements Command<StartTaskCommand.Data> {
+    static final String ID = "start_task";
+    private final SessionManager sessionManager;
+
+    StartTaskCommand(SessionManager sessionManager) {
+        this.sessionManager = sessionManager;
+    }
+
+    @Override
+    public void execute(Data data, Connection connection) {
+        Session session = sessionManager.getSession(data.getSessionId());
+        if (session == null) {
+            throw new IllegalStateException("Session is not created.");
+        }
+        session.runTaskAsynchronously(data.getTaskIndex());
+    }
+
+    @Override
+    public String getId() {
+        return ID;
+    }
+
+    static class Data {
+        private String sessionId;
+        private int taskIndex;
+
+        @JsonCreator
+        Data(@JsonProperty("sessionId") String sessionId,
+             @JsonProperty("taskIndex") int taskIndex) {
+            this.sessionId = sessionId;
+            this.taskIndex = taskIndex;
+        }
+
+        @JsonProperty
+        String getSessionId() {
+            return sessionId;
+        }
+
+        @JsonProperty
+        int getTaskIndex() {
+            return taskIndex;
+        }
+    }
+}

data/src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java

@@ -0,0 +1,11 @@
+package org.embulk.executor.remoteserver;
+
+class TaskExecutionException extends RuntimeException {
+    TaskExecutionException(String message) {
+        super(message);
+    }
+
+    TaskExecutionException(Throwable cause) {
+        super(cause);
+    }
+}

data/src/main/java/org/embulk/executor/remoteserver/TaskState.java

@@ -0,0 +1,5 @@
+package org.embulk.executor.remoteserver;
+
+public enum TaskState {
+    STARTED, INPUT_COMMITTED, OUTPUT_COMMITTED, FINISHED, FAILED
+}

data/src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateData.java

@@ -0,0 +1,55 @@
+package org.embulk.executor.remoteserver;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+class UpdateTaskStateData {
+    private String sessionId;
+    private int taskIndex;
+    private TaskState taskState;
+    private String taskReport;
+    private String errorMessage;
+
+    @JsonCreator
+    UpdateTaskStateData(
+            @JsonProperty("sessionId") String sessionId,
+            @JsonProperty("taskIndex") int taskIndex,
+            @JsonProperty("taskState") TaskState taskState) {
+        this.sessionId = sessionId;
+        this.taskIndex = taskIndex;
+        this.taskState = taskState;
+    }
+
+    @JsonProperty
+    String getSessionId() {
+        return sessionId;
+    }
+
+    @JsonProperty
+    int getTaskIndex() {
+        return taskIndex;
+    }
+
+    @JsonProperty
+    TaskState getTaskState() {
+        return taskState;
+    }
+
+    @JsonProperty
+    String getTaskReport() {
+        return taskReport;
+    }
+
+    void setTaskReport(String taskReport) {
+        this.taskReport = taskReport;
+    }
+
+    @JsonProperty
+    String getErrorMessage() {
+        return errorMessage;
+    }
+
+    void setErrorMessage(String errorMessage) {
+        this.errorMessage = errorMessage;
+    }
+}

data/src/main/resources/logback.xml

@@ -0,0 +1,11 @@
+<configuration>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="debug">
+        <appender-ref ref="STDOUT"/>
+    </root>
+</configuration>

data/src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java

@@ -0,0 +1,80 @@
+package org.embulk.executor.remoteserver;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.test.EmbulkPluginTest;
+import org.embulk.test.EmbulkTest;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@EmbulkTest(value = RemoteServerExecutor.class, name = "remoteserver")
+class TestRemoteServerExecutor extends EmbulkPluginTest {
+    private static final List<Host> HOSTS = Arrays.asList(new Host("localhost", 24224), new Host("localhost", 24225));
+    private static final Path OUTPUT_DIR = Paths.get("tmp", "output");
+    private static final Path TEST_DIR = Paths.get("test");
+
+    @Test
+    void testSimpleCase() {
+        setSystemConfig(config().set("jruby_global_bundler_plugin_source_directory", TEST_DIR.toFile().getAbsolutePath()));
+
+        ConfigSource inConfig = config().set("type", "file")
+                .set("path_prefix", "src/test/resources/json/test")
+                .set("parser", config().set("type", "json")
+                        .set("columns", Collections.singletonList(config()
+                                .set("name", "a").set("type", "long")
+                        ))
+                );
+
+        ConfigSource execConfig = config().set("type", "remoteserver")
+                .set("hosts", HOSTS)
+                .set("timeout_seconds", 5);
+
+        ConfigSource filterConfig = config().set("type", "hash")
+                .set("columns", Collections.singletonList(config()
+                        .set("name", "a").set("algorithm", "MD5")
+                ));
+
+        ConfigSource outConfig = config().set("type", "file")
+                .set("path_prefix", "/output/out_file_")
+                .set("file_ext", "json")
+                .set("formatter", config()
+                        .set("type", "csv")
+                        .set("header_line", false)
+                        .set("quote_policy", "NONE")
+                );
+
+        runConfig(inConfig)
+                .execConfig(execConfig)
+                .filterConfig(filterConfig)
+                .outConfig(outConfig).run();
+
+        File[] files = OUTPUT_DIR.toFile().listFiles();
+        assertEquals(2, files.length);
+        Set<String> outputs = Arrays.stream(files).map(f -> {
+            try {
+                return String.join("", Files.readAllLines(f.toPath()));
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }).collect(Collectors.toSet());
+
+        Set<String> expected = new HashSet<String>(){{
+            add("c4ca4238a0b923820dcc509a6f75849b"); // "1" of MD5
+            add("c81e728d9d4c2f636f067f89cc14862c"); // "2" of MD5
+        }};
+        assertEquals(expected, outputs);
+    }
+}

data/src/test/resources/json/test1.json

@@ -0,0 +1 @@
+{"a": 1}

data/src/test/resources/json/test2.json

@@ -0,0 +1 @@
+{"a": 2}

data/test/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+
+gem 'embulk'
+gem 'embulk-filter-hash', '= 0.5.0'

data/test/Gemfile.lock

@@ -0,0 +1,20 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    embulk (0.9.16-java)
+      bundler (>= 1.10.6)
+      liquid (~> 4.0.0)
+      msgpack (~> 1.1.0)
+    embulk-filter-hash (0.5.0)
+    liquid (4.0.0)
+    msgpack (1.1.0-java)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  embulk
+  embulk-filter-hash (= 0.5.0)
+
+BUNDLED WITH
+   1.16.0

data/test/setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+project_root="$(cd $(dirname $0)/..; pwd -P)"
+cd ${project_root}
+
+rm -rf tmp/output && mkdir -p tmp/output
+./gradlew embulk_bundle_--clean -Pgemfile=test/Gemfile -PbundlePath=${project_root}/tmp/vendor/bundle
+docker-compose up -d --build

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: embulk-executor-remoteserver
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shinichi Ishimura
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-03-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Embulk executor plugin to run plugins on remote server.
+email:
+- shiketaudonko41@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- Dockerfile
+- LICENSE
+- README.md
+- build.gradle
+- classpath/embulk-executor-remoteserver-0.1.0.jar
+- classpath/msgpack-core-0.8.16.jar
+- classpath/nsocket-0.2.10.jar
+- classpath/slf4j-api-1.7.26.jar
+- docker-compose.yml
+- docker/run_embulk_server.sh
+- gradle.properties
+- gradle/dependency-locks/compileClasspath.lockfile
+- gradle/dependency-locks/testCompileClasspath.lockfile
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/executor/remoteserver.rb
+- settings.gradle
+- src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java
+- src/main/java/org/embulk/executor/remoteserver/EmbulkServer.java
+- src/main/java/org/embulk/executor/remoteserver/Host.java
+- src/main/java/org/embulk/executor/remoteserver/InitializeSessionCommand.java
+- src/main/java/org/embulk/executor/remoteserver/Launcher.java
+- src/main/java/org/embulk/executor/remoteserver/NotifyTaskStateCommand.java
+- src/main/java/org/embulk/executor/remoteserver/PluginArchive.java
+- src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java
+- src/main/java/org/embulk/executor/remoteserver/RemoveSessionCommand.java
+- src/main/java/org/embulk/executor/remoteserver/Session.java
+- src/main/java/org/embulk/executor/remoteserver/SessionManager.java
+- src/main/java/org/embulk/executor/remoteserver/SessionState.java
+- src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java
+- src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java
+- src/main/java/org/embulk/executor/remoteserver/TaskState.java
+- src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateData.java
+- src/main/resources/logback.xml
+- src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java
+- src/test/resources/json/test1.json
+- src/test/resources/json/test2.json
+- test/Gemfile
+- test/Gemfile.lock
+- test/setup.sh
+homepage: https://github.com/kamatama41/embulk-executor-remoteserver
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Docker executor plugin for Embulk
+test_files:
+- test/Gemfile
+- test/Gemfile.lock
+- test/setup.sh