embulk-executor-remoteserver 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1e201bcff4c6f7ec2cf013c0f8dc02ae9f35bf8
-  data.tar.gz: 91c0ae41c7453450075e834c5cccffac28b5c0c8
+  metadata.gz: 2c835aba3fc2c0ce9c17fa5d9fe70c756097081a
+  data.tar.gz: 5831d9e3268af8ef640e1910c0ed1b86eaec88d6
 SHA512:
-  metadata.gz: 8f0aa143d8add016ae8cb70303ca782783d476766d7896a321caaa9da015aecf3098c7f5271385644caea74b3be2a9a49e6e62f39c4afd1a3a44fceffd9da848
-  data.tar.gz: fa81fb681acf7097ef38b5699faa3cfc2777fd14ad12156527db201894d929dad8c4ac3320cd81322416e723bbeef23c300c5d2898477019a5bad1efd12d08e8
+  metadata.gz: 46a1920c3c054f3d7b68c05308c38d4ab3ff3d4b6dd940733e29b5917803f2ce7a5a031efa214ebce2a76f8a3eb1929756fe9f7bc5d5bc9bccbcee148497b1ef
+  data.tar.gz: 0de6447bb0d0b7756fdb6e2dcc30eeb82a52f589a51e50fc67b1cbfa7308a5bce4e648d4420079cc8711b5c0ec00912f024bbb1508f1b490be64accabac82d63

data/.circleci/config.yml

@@ -10,6 +10,7 @@ jobs:
             - v1-gradle-{{ checksum "gradle/dependency-locks/compileClasspath.lockfile" }}-{{ checksum "gradle/dependency-locks/testCompileClasspath.lockfile" }}
             - v1-gradle-
       - run: ./test/setup.sh
+      - run: sudo chown -R circleci.circleci tmp/certs
       - run: ./gradlew check --info
       - run:
           name: Save test results

data/.gitignore

@@ -12,3 +12,5 @@ build/
 .project
 *.iml
 .bundle
+
+test/Gemfile.lock

data/README.md

@@ -8,12 +8,19 @@ Embulk executor plugin to run Embulk tasks on remote servers.
 
 ### Notes
 - It's still very experimental version, so might change its spec without notification. 
-- It might have performance issues or bugs. I would appreciate it if you use this and give me reports/feedback!
+- It might have some issues or bugs. I would appreciate it if you use this and give me reports/feedback!
 
 ## Configuration
 
-- **hosts**: List of remote servers (`hostname` or `hostname:port`, default port is `30001`). If not specified, the executor runs as local mode, which start Embulk server on its own process (array of string)
+- **hosts**: List of remote servers (`hostname` or `hostname:port`, default port is `30001`). If not specified, the executor runs as the local mode, which starts an Embulk server on its own process (array of string)
 - **timeout_seconds**: Timeout seconds of the whole execution (integer, default: `3600`)
+- **tls**: Enable to connect server over TLS (boolean, default: `false`)
+- **key_store_p12**: Information of a P12 file used as a [KeyManager](https://docs.oracle.com/javase/8/docs/api/javax/net/ssl/KeyManager.html) to register your client certificate. It would be needed when client auth mode is enabled on Embulk server.
+  - **path**: Path of the file (string, required)
+  - **password**: Password of the file (string, required)
+- **trust_store_p12**: Information of a P12 file used as a [TrustManager](https://docs.oracle.com/javase/8/docs/api/javax/net/ssl/TrustManager.html) to register a CA certificate. It would be needed when Embulk server uses unknown CA's certificate.
+  - **path**: Path of the file (string, required)
+  - **password**: Password of the file (string, required)
 
 ## Example
 
@@ -27,7 +34,7 @@ exec:
 ```
 
 ## Embulk server
-The server recieves requests from client (Embulk) and run Embulk tasks, then returns results to client. It communicates with clients via `TCP 30001 port`. 
+The server receives requests from client (Embulk) and run Embulk tasks, then returns results to client. It communicates with clients via `TCP 30001 port`. 
 
 ### Running Embulk server as a Docker container
 The image is hosted by [DockerHub](https://hub.docker.com/r/kamatama41/embulk-executor-remoteserver).  
@@ -37,6 +44,18 @@ You can try running Embulk server by the following command.
 $ docker run --rm -p 30001:30001 kamatama41/embulk-executor-remoteserver
 ```
 
+### Configure Embulk server
+There are some environment variables to configure the server
+
+- `BIND_ADDRESS`: Bind address of the server (default: `0.0.0.0`)
+- `PORT`: Port number to listen (default: `30001`)
+- `ENABLE_TLS`: Try to connect to client via TLS if `true` (default: `false`)
+- `ENABLE_TLS_CLIENT_AUTH`: Require client authentication if `true` (default: `false`) 
+- `KEY_P12_PATH`: Path of the P12 file used as a KeyManager
+- `KEY_P12_PASSWORD`: Password of the Key P12 file
+- `TRUST_P12_PATH`: Path of the P12 file used as a TrustManager
+- `TRUST_P12_PASSWORD`: Password of the Trust P12 file
+
 ## Build
 
 ```

data/build.gradle

@@ -21,20 +21,20 @@ configurations {
 }
 
 dependencies {
-    compile("com.github.kamatama41:nsocket:0.2.13") {
+    compile("com.github.kamatama41:nsocket:0.3.4") {
         // Use embulk-core's Jackson (2.6.7) instead
         exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
     }
     // Followings are needed for running EmbulkServer
-    implementation "org.embulk:embulk-standards:0.9.16"
-    serverRuntime "org.embulk:embulk-standards:0.9.16"
+    implementation "org.embulk:embulk-standards:0.9.17"
+    serverRuntime "org.embulk:embulk-standards:0.9.17"
     implementation "ch.qos.logback:logback-classic:1.2.3"
     serverRuntime "ch.qos.logback:logback-classic:1.2.3"
     implementation "ch.qos.logback:logback-core:1.2.3"
     serverRuntime "ch.qos.logback:logback-core:1.2.3"
 
-    testImplementation "org.embulk:embulk-standards:0.9.16"
-    testImplementation "org.embulk:embulk-test:0.9.16"
+    testImplementation "org.embulk:embulk-standards:0.9.17"
+    testImplementation "org.embulk:embulk-test:0.9.17"
     testImplementation "com.github.kamatama41:embulk-test-helpers:0.8.0"
     testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
     testRuntime "org.junit.jupiter:junit-jupiter-engine:5.3.2"
@@ -50,7 +50,7 @@ task executableEmbulkServer(type: Jar) {
 }
 
 embulk {
-    version = "0.9.16"
+    version = "0.9.17"
     category = "executor"
     name = "docker"
     authors = ["Shinichi Ishimura"]
@@ -67,4 +67,4 @@ test {
 release {
     git { requireBranch = 'master' }
 }
-afterReleaseBuild.dependsOn gemPush
+afterReleaseBuild.dependsOn gemPush

data/docker-compose.test.yml

@@ -0,0 +1,49 @@
+version: '3.2'
+services:
+  server1: &server
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      LOG_LEVEL: debug
+    ports:
+      - "30001:30001"
+    volumes:
+      - ./tmp/output:/output
+      - ./src/test/resources/json:/root/src/test/resources/json
+
+  server2:
+    <<: *server
+    ports:
+      - "30002:30001"
+
+  tls-server1: &tls-server
+    <<: *server
+    environment:
+      LOG_LEVEL: debug
+      ENABLE_TLS: "true"
+      ENABLE_TLS_CLIENT_AUTH: "true"
+      KEY_P12_PATH: /certs/embulk-server.local.p12
+      KEY_P12_PASSWORD: abcde
+      TRUST_P12_PATH: /certs/ca-chain.p12
+      TRUST_P12_PASSWORD: p@ssw0rd
+    ports:
+        - "30003:30001"
+    volumes:
+      - ./tmp/output:/output
+      - ./tmp/certs:/certs
+      - ./src/test/resources/json:/root/src/test/resources/json
+
+  tls-server2:
+    <<: *tls-server
+    ports:
+      - "30004:30001"
+
+  cert-generator:
+    image: kamatama41/test-cert-generator
+    volumes:
+      - ./tmp/certs:/root/work
+    environment:
+      SERVER_COMMON_NAME: embulk-server.local
+      SERVER_P12_PASSWORD: abcde
+      CLIENT_P12_PASSWORD: fghij

data/example/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org/'
+
+gem 'embulk'
+gem 'embulk-filter-add_time'
+gem 'embulk-executor-remoteserver'

data/example/README.md

@@ -0,0 +1,32 @@
+## How to run
+
+Run `embulk bundle`
+
+```sh
+$ embulk bundle --path vendor/bundle
+```
+
+Run `docker-compose up`
+
+```sh
+$ docker-compose -f docker-compse.yml up
+Creating example_remote1_1 ... done
+Attaching to example_remote1_1
+remote1_1  | 15:11:59.045 [main] INFO  c.g.kamatama41.nsocket.SocketServer - Starting server..
+```
+
+In another session, run `embulk run`
+
+```sh
+$ embulk run config.yml -b .
+```
+
+So that 2 output files would be generated in `work` directory
+
+```sh
+$ cat work/output_*
+id,name,time
+1,Scott,2019-04-06 15:12:42.029000 +0000
+id,name,time
+2,Tiger,2019-04-06 15:12:42.029000 +0000
+```

data/example/config.yml

@@ -0,0 +1,25 @@
+in:
+  type: file
+  path_prefix: work/input_
+  parser:
+    type: json
+    columns:
+      - {name: id, type: long}
+      - {name: name, type: string}
+filters:
+  - type: add_time
+    to_column:
+      name: time
+      type: timestamp
+    from_value:
+      mode: upload_time
+exec:
+  type: remoteserver
+  hosts:
+    - localhost:30001
+out:
+  type: file
+  path_prefix: work/output_
+  file_ext: csv
+  formatter:
+    type: csv

data/example/docker-compse.yml

@@ -0,0 +1,8 @@
+version: '3.2'
+services:
+  remote1:
+    image: kamatama41/embulk-executor-remoteserver
+    ports:
+      - "30001:30001"
+    volumes:
+      - ./work:/root/work

data/example/work/input_1.json

@@ -0,0 +1 @@
+{"id":  1, "name":  "Scott"}

data/example/work/input_2.json

@@ -0,0 +1 @@
+{"id":  2, "name":  "Tiger"}

data/gradle.properties

@@ -1 +1 @@
-version=0.2.1
+version=0.3.0

data/gradle/dependency-locks/compileClasspath.lockfile

@@ -11,16 +11,13 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
-com.github.kamatama41:nsocket:0.2.13
+com.github.kamatama41:nsocket:0.3.4
 com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0
 com.google.inject:guice:4.0
 com.ibm.icu:icu4j:54.1.1
 commons-beanutils:commons-beanutils-core:1.8.3
-commons-cli:commons-cli:1.3.1
-commons-collections:commons-collections:3.2.1
-commons-lang:commons-lang:2.4
 io.airlift:slice:0.9
 io.netty:netty-buffer:4.0.44.Final
 io.netty:netty-common:4.0.44.Final
@@ -31,9 +28,8 @@ org.apache.bval:bval-core:0.5
 org.apache.bval:bval-jsr303:0.5
 org.apache.commons:commons-compress:1.10
 org.apache.commons:commons-lang3:3.4
-org.apache.velocity:velocity:1.7
-org.embulk:embulk-core:0.9.16
-org.embulk:embulk-standards:0.9.16
+org.embulk:embulk-core:0.9.17
+org.embulk:embulk-standards:0.9.17
 org.jruby:jruby-complete:9.1.15.0
 org.msgpack:msgpack-core:0.8.11
 org.slf4j:slf4j-api:1.7.25

data/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -12,16 +12,13 @@ com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
 com.github.kamatama41:embulk-test-helpers:0.8.0
-com.github.kamatama41:nsocket:0.2.13
+com.github.kamatama41:nsocket:0.3.4
 com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0
 com.google.inject:guice:4.0
 com.ibm.icu:icu4j:54.1.1
 commons-beanutils:commons-beanutils-core:1.8.3
-commons-cli:commons-cli:1.3.1
-commons-collections:commons-collections:3.2.1
-commons-lang:commons-lang:2.4
 io.airlift:slice:0.9
 io.netty:netty-buffer:4.0.44.Final
 io.netty:netty-common:4.0.44.Final
@@ -33,11 +30,10 @@ org.apache.bval:bval-core:0.5
 org.apache.bval:bval-jsr303:0.5
 org.apache.commons:commons-compress:1.10
 org.apache.commons:commons-lang3:3.4
-org.apache.velocity:velocity:1.7
 org.apiguardian:apiguardian-api:1.0.0
-org.embulk:embulk-core:0.9.16
-org.embulk:embulk-standards:0.9.16
-org.embulk:embulk-test:0.9.16
+org.embulk:embulk-core:0.9.17
+org.embulk:embulk-standards:0.9.17
+org.embulk:embulk-test:0.9.17
 org.hamcrest:hamcrest-core:1.3
 org.hamcrest:hamcrest-library:1.3
 org.jruby:jruby-complete:9.1.15.0

data/src/main/java/org/embulk/executor/remoteserver/ClientSession.java

@@ -15,7 +15,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
-class ClientSession {
+class ClientSession implements AutoCloseable {
     private static final Logger log = LoggerFactory.getLogger(ClientSession.class);
 
     private final String id;
@@ -78,6 +78,11 @@ class ClientSession {
         return isFinished;
     }
 
+    @Override
+    public void close() {
+        isFinished = true;
+    }
+
     synchronized void update(UpdateTaskStateData data) {
         switch (data.getTaskState()) {
             case STARTED:
@@ -115,7 +120,7 @@ class ClientSession {
                 throw new TaskExecutionException(message);
             }
         } finally {
-            isFinished = true;
+            close();
         }
     }
 

data/src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java

@@ -29,8 +29,12 @@ class EmbulkClient extends SocketClient implements AutoCloseable {
 
     static EmbulkClient open(
             ClientSession session,
-            List<Host> hosts) throws IOException {
+            List<Host> hosts,
+            TLSConfig tlsConfig) throws IOException {
         EmbulkClient client = new EmbulkClient(session);
+        if (tlsConfig != null) {
+            client.setSslContext(tlsConfig.getSSLContext());
+        }
         client.open();
 
         for (Host host : hosts) {
@@ -90,6 +94,7 @@ class EmbulkClient extends SocketClient implements AutoCloseable {
         for (Connection connection : getActiveConnections()) {
             connection.sendSyncCommand(RemoveSessionCommand.ID, session.getId());
         }
+        session.close();
         super.close();
     }
 
@@ -99,7 +104,7 @@ class EmbulkClient extends SocketClient implements AutoCloseable {
             if(!session.isFinished()) {
                 try {
                     // Try reconnecting
-                    Connection newConnection = reconnect(connection);
+                    Connection newConnection = reconnect(connection, i -> !session.isFinished());
                     newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
                 } catch (IOException e) {
                     log.warn(String.format("A connection to %s could not be reconnected.", connection.getRemoteSocketAddress()), e);

data/src/main/java/org/embulk/executor/remoteserver/EmbulkServer.java

@@ -11,7 +11,7 @@ public class EmbulkServer implements AutoCloseable {
         this.server = server;
     }
 
-    static EmbulkServer start(String host, int port, int numOfWorkers) throws IOException {
+    static EmbulkServer start(String host, int port, int numOfWorkers, TLSConfig tlsConfig) throws IOException {
         SocketServer server = new SocketServer();
         ServerSessionRegistry sessionRegistry = new ServerSessionRegistry();
         server.setHost(host);
@@ -21,6 +21,12 @@ public class EmbulkServer implements AutoCloseable {
         server.registerSyncCommand(new InitializeSessionCommand(sessionRegistry));
         server.registerSyncCommand(new RemoveSessionCommand(sessionRegistry));
         server.registerCommand(new StartTaskCommand(sessionRegistry));
+        if (tlsConfig != null) {
+            server.setSslContext(tlsConfig.getSSLContext());
+            if (tlsConfig.isEnableClientAuth()) {
+                server.enableSslClientAuth();
+            }
+        }
         server.start();
         return new EmbulkServer(server);
     }

data/src/main/java/org/embulk/executor/remoteserver/Launcher.java

@@ -12,13 +12,39 @@ public class Launcher {
         Map<String, String> envVars = System.getenv();
         String host = envVars.getOrDefault("BIND_ADDRESS", "0.0.0.0");
         int port = Integer.parseInt(envVars.getOrDefault("PORT", "30001"));
-        int numOfWorkers = Integer.parseInt(envVars.getOrDefault("NUM_OF_WORKERS", "1"));
-        Level logLevel = Level.toLevel(envVars.getOrDefault("LOG_LEVEL", "info"));
-        configureLogLevel(logLevel);
-        EmbulkServer.start(host, port, numOfWorkers);
+        int numOfWorkers = Integer.parseInt(envVars.getOrDefault("NUM_OF_WORKERS", "5"));
+        TLSConfig tlsConfig = createTLSConfig(envVars);
+        configureLogLevel(envVars);
+
+        EmbulkServer.start(host, port, numOfWorkers, tlsConfig);
+    }
+
+    private static TLSConfig createTLSConfig(Map<String, String> envVars) {
+        if (!"true".equals(envVars.get("ENABLE_TLS"))) {
+            return null;
+        }
+
+        TLSConfig tlsConfig = new TLSConfig();
+        String keyP12Path = envVars.get("KEY_P12_PATH");
+        String keyP12Password = envVars.get("KEY_P12_PASSWORD");
+        if (keyP12Path != null && keyP12Password != null) {
+            tlsConfig.keyStore(new P12File(keyP12Path, keyP12Password));
+        }
+
+        String trustP12Path = envVars.get("TRUST_P12_PATH");
+        String trustP12Password = envVars.get("TRUST_P12_PASSWORD");
+        if (trustP12Path != null && trustP12Password != null) {
+            tlsConfig.trustStore(new P12File(trustP12Path, trustP12Password));
+        }
+
+        if ("true".equals(envVars.get("ENABLE_TLS_CLIENT_AUTH"))) {
+            tlsConfig.enableClientAuth(true);
+        }
+        return tlsConfig;
     }
 
-    private static void configureLogLevel(Level logLevel) {
+    private static void configureLogLevel(Map<String, String> envVars) {
+        Level logLevel = Level.toLevel(envVars.getOrDefault("LOG_LEVEL", "info"));
         Logger rootLogger = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
         rootLogger.setLevel(logLevel);
     }

data/src/main/java/org/embulk/executor/remoteserver/P12File.java

@@ -0,0 +1,28 @@
+package org.embulk.executor.remoteserver;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+class P12File {
+    private final String path;
+    private final String password;
+
+    @JsonCreator
+    P12File(@JsonProperty("path") String path, @JsonProperty("password") String password) {
+        if (path == null || password == null) {
+            throw new NullPointerException("Path and password must not be null");
+        }
+        this.path = path;
+        this.password = password;
+    }
+
+    @JsonProperty
+    String getPath() {
+        return path;
+    }
+
+    @JsonProperty
+    String getPassword() {
+        return password;
+    }
+}

data/src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java

@@ -24,6 +24,7 @@ import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
@@ -42,8 +43,33 @@ public class RemoteServerExecutor implements ExecutorPlugin {
         @ConfigDefault("3600")
         int getTimeoutSeconds();
 
+        @Config("tls")
+        @ConfigDefault("false")
+        boolean getTLS();
+
+        @Config("key_store_p12")
+        @ConfigDefault("null")
+        Optional<P12File> getKeyStoreP12();
+
+        @Config("trust_store_p12")
+        @ConfigDefault("null")
+        Optional<P12File> getTrustStoreP12();
+
         @ConfigInject
         ModelManager getModelManager();
+
+        // Used for the local mode (mainly for testing)
+        @Config("__server_key_store_p12")
+        @ConfigDefault("null")
+        Optional<P12File> getServerKeyStoreP12();
+
+        @Config("__server_trust_store_p12")
+        @ConfigDefault("null")
+        Optional<P12File> getServerTrustStoreP12();
+
+        @Config("__server_enable_client_auth")
+        @ConfigDefault("false")
+        boolean getServerEnableClientAuth();
     }
 
     @Inject
@@ -56,8 +82,8 @@ public class RemoteServerExecutor implements ExecutorPlugin {
     public void transaction(ConfigSource config, Schema outputSchema, int inputTaskCount, Control control) {
         PluginTask task = config.loadConfig(PluginTask.class);
         if (task.getHosts().isEmpty()) {
-            log.info("Hosts is empty. Run with a local server.");
-            try (EmbulkServer _autoclosed = EmbulkServer.start(DEFAULT_HOST.getName(), DEFAULT_HOST.getPort(), 1)) {
+            log.info("Hosts is empty. Run as the local mode.");
+            try (EmbulkServer _autoclosed = startEmbulkServer(task)) {
                 control.transaction(outputSchema, inputTaskCount, new ExecutorImpl(inputTaskCount, task, Collections.singletonList(DEFAULT_HOST)));
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
@@ -103,7 +129,15 @@ public class RemoteServerExecutor implements ExecutorPlugin {
 
             ClientSession session = new ClientSession(
                     systemConfigJson, pluginTaskJson, processTaskJson, gemSpecs, pluginArchiveBytes, state, inputTaskCount, modelManager);
-            try (EmbulkClient client = EmbulkClient.open(session, hosts)) {
+
+            TLSConfig tlsConfig = null;
+            if (pluginTask.getTLS()) {
+                tlsConfig = new TLSConfig();
+                pluginTask.getKeyStoreP12().ifPresent(tlsConfig::keyStore);
+                pluginTask.getTrustStoreP12().ifPresent(tlsConfig::trustStore);
+            }
+
+            try (EmbulkClient client = EmbulkClient.open(session, hosts, tlsConfig)) {
                 client.createSession();
 
                 state.initialize(inputTaskCount, inputTaskCount);
@@ -132,4 +166,17 @@ public class RemoteServerExecutor implements ExecutorPlugin {
             }
         }
     }
+
+    private EmbulkServer startEmbulkServer(PluginTask task) throws IOException {
+        TLSConfig tlsConfig = null;
+        if (task.getTLS()) {
+            tlsConfig = new TLSConfig();
+            task.getServerKeyStoreP12().ifPresent(tlsConfig::keyStore);
+            task.getServerTrustStoreP12().ifPresent(tlsConfig::trustStore);
+            if (task.getServerEnableClientAuth()) {
+                tlsConfig.enableClientAuth(true);
+            }
+        }
+        return EmbulkServer.start(DEFAULT_HOST.getName(), DEFAULT_HOST.getPort(), 1, tlsConfig);
+    }
 }

data/src/main/java/org/embulk/executor/remoteserver/ServerSession.java

@@ -173,7 +173,7 @@ class ServerSession implements AutoCloseable {
 
     @Override
     public void close() {
-        log.debug("Closing the session {}", id);
+        log.info("Closing the session {}", id);
         sessionRunner.shutdownNow();
     }
 }

data/src/main/java/org/embulk/executor/remoteserver/TLSConfig.java

@@ -0,0 +1,74 @@
+package org.embulk.executor.remoteserver;
+
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.security.KeyStore;
+
+class TLSConfig {
+    private P12File keyStore = null;
+    private P12File trustStore = null;
+    private boolean enableClientAuth = false;
+
+    TLSConfig() {
+    }
+
+    TLSConfig keyStore(P12File keyStore) {
+        this.keyStore = keyStore;
+        return this;
+    }
+
+    TLSConfig trustStore(P12File trustStore) {
+        this.trustStore = trustStore;
+        return this;
+    }
+
+    TLSConfig enableClientAuth(boolean enableClientAuth) {
+        this.enableClientAuth = enableClientAuth;
+        return this;
+    }
+
+    SSLContext getSSLContext() {
+        try {
+            KeyManager[] keyManagers = null;
+            if (keyStore != null) {
+                KeyStore ks = load(keyStore);
+                KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
+                kmf.init(ks, keyStore.getPassword().toCharArray());
+                keyManagers = kmf.getKeyManagers();
+            }
+
+            TrustManager[] trustManagers = null;
+            if (trustStore != null) {
+                KeyStore ts = load(trustStore);
+                TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+                tmf.init(ts);
+                trustManagers = tmf.getTrustManagers();
+            }
+
+            SSLContext context = SSLContext.getInstance("TLS");
+            context.init(keyManagers, trustManagers, null);
+            return context;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    boolean isEnableClientAuth() {
+        return enableClientAuth;
+    }
+
+    private static KeyStore load(P12File file) {
+        try (InputStream keyStoreIS = new FileInputStream(file.getPath())) {
+            KeyStore ks = KeyStore.getInstance("PKCS12");
+            ks.load(keyStoreIS, file.getPassword().toCharArray());
+            return ks;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

data/src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java

@@ -3,6 +3,7 @@ package org.embulk.executor.remoteserver;
 import org.embulk.config.ConfigSource;
 import org.embulk.test.EmbulkPluginTest;
 import org.embulk.test.EmbulkTest;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
@@ -12,49 +13,65 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.embulk.test.Utils.configFromResource;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 @EmbulkTest(value = RemoteServerExecutor.class, name = "remoteserver")
 class TestRemoteServerExecutor extends EmbulkPluginTest {
-    private static final List<String> HOSTS = Arrays.asList("localhost", "localhost:30002");
     private static final Path OUTPUT_DIR = Paths.get("tmp", "output");
     private static final Path TEST_DIR = Paths.get("test");
 
+    @BeforeEach
+    void cleanupOutputDir() {
+        File outputDir = OUTPUT_DIR.toFile();
+        if (outputDir.exists()) {
+            Arrays.stream(outputDir.listFiles()).forEach(File::delete);
+        }
+    }
+
     @Test
     void testSimpleCase() {
         setSystemConfig(config().set("jruby_global_bundler_plugin_source_directory", TEST_DIR.toFile().getAbsolutePath()));
 
-        ConfigSource inConfig = config().set("type", "file")
-                .set("path_prefix", "src/test/resources/json/test")
-                .set("parser", config().set("type", "json")
-                        .set("columns", Collections.singletonList(config()
-                                .set("name", "a").set("type", "long")
-                        ))
-                );
+        ConfigSource inConfig = configFromResource("config/input.yml");
+        ConfigSource execConfig = configFromResource("config/exec_base.yml");
+        ConfigSource filterConfig = configFromResource("config/filter.yml");
+        ConfigSource outConfig = configFromResource("config/output.yml");
 
-        ConfigSource execConfig = config().set("type", "remoteserver")
-                .set("hosts", HOSTS)
-                .set("timeout_seconds", 5);
+        runConfig(inConfig)
+                .execConfig(execConfig)
+                .filterConfig(filterConfig)
+                .outConfig(outConfig).run();
 
-        ConfigSource filterConfig = config().set("type", "hash")
-                .set("columns", Collections.singletonList(config()
-                        .set("name", "a").set("algorithm", "MD5")
-                ));
+        File[] files = OUTPUT_DIR.toFile().listFiles();
+        assertEquals(2, files.length);
+        Set<String> outputs = Arrays.stream(files).map(f -> {
+            try {
+                return String.join("", Files.readAllLines(f.toPath()));
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        }).collect(Collectors.toSet());
+
+        Set<String> expected = new HashSet<String>(){{
+            add("c4ca4238a0b923820dcc509a6f75849b"); // "1" of MD5
+            add("c81e728d9d4c2f636f067f89cc14862c"); // "2" of MD5
+        }};
+        assertEquals(expected, outputs);
+    }
+
+    @Test
+    void testConnectWithTLS() {
+        setSystemConfig(config().set("jruby_global_bundler_plugin_source_directory", TEST_DIR.toFile().getAbsolutePath()));
 
-        ConfigSource outConfig = config().set("type", "file")
-                .set("path_prefix", "/output/out_file_")
-                .set("file_ext", "json")
-                .set("formatter", config()
-                        .set("type", "csv")
-                        .set("header_line", false)
-                        .set("quote_policy", "NONE")
-                );
+        ConfigSource inConfig = configFromResource("config/input.yml");
+        ConfigSource execConfig = configFromResource("config/exec_tls.yml");
+        ConfigSource filterConfig = configFromResource("config/filter.yml");
+        ConfigSource outConfig = configFromResource("config/output.yml");
 
         runConfig(inConfig)
                 .execConfig(execConfig)

data/src/test/resources/config/exec_base.yml

@@ -0,0 +1,5 @@
+type: remoteserver
+timeout_seconds: 5
+hosts:
+  - localhost
+  - localhost:30002

data/src/test/resources/config/exec_tls.yml

@@ -0,0 +1,13 @@
+type: remoteserver
+timeout_seconds: 5
+hosts:
+  - localhost:30003
+  - localhost:30004
+tls: true
+key_store_p12:
+  path: tmp/certs/client.p12
+  password: fghij
+trust_store_p12:
+  path: tmp/certs/ca-chain.p12
+  password: p@ssw0rd
+

data/src/test/resources/config/filter.yml

@@ -0,0 +1,3 @@
+type: hash
+columns:
+  - {name: a, algorithm: MD5}

data/src/test/resources/config/input.yml

@@ -0,0 +1,6 @@
+type: file
+path_prefix: src/test/resources/json/test
+parser:
+  type: json
+  columns:
+    - {name: a, type: long}

data/src/test/resources/config/output.yml

@@ -0,0 +1,7 @@
+type: file
+path_prefix: /output/out_file_
+file_ext: csv
+formatter:
+  type: csv
+  header_line: false
+  quote_policy: NONE

data/test/setup.sh

@@ -4,5 +4,11 @@ project_root="$(cd $(dirname $0)/..; pwd -P)"
 cd ${project_root}
 
 rm -rf tmp/output && mkdir -p tmp/output
+rm -rf tmp/certs && mkdir -p tmp/certs
 ./gradlew embulk_bundle_--clean -Pgemfile=test/Gemfile -PbundlePath=${project_root}/tmp/vendor/bundle
-docker-compose up -d --build
+
+if [[ "${SKIP_DOCKER_BUILD}" != "true" ]]; then
+  docker-compose -f docker-compose.test.yml build
+fi
+docker-compose -f docker-compose.test.yml run cert-generator
+docker-compose -f docker-compose.test.yml up -d server1 server2 tls-server1 tls-server2

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-remoteserver
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-06 00:00:00.000000000 Z
+date: 2019-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -51,12 +51,18 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-executor-remoteserver-0.2.1.jar
+- classpath/embulk-executor-remoteserver-0.3.0.jar
 - classpath/msgpack-core-0.8.16.jar
-- classpath/nsocket-0.2.13.jar
+- classpath/nsocket-0.3.4.jar
 - classpath/slf4j-api-1.7.26.jar
-- docker-compose.yml
+- docker-compose.test.yml
 - docker/run_embulk_server.sh
+- example/Gemfile
+- example/README.md
+- example/config.yml
+- example/docker-compse.yml
+- example/work/input_1.json
+- example/work/input_2.json
 - gradle.properties
 - gradle/dependency-locks/compileClasspath.lockfile
 - gradle/dependency-locks/testCompileClasspath.lockfile
@@ -72,22 +78,28 @@ files:
 - src/main/java/org/embulk/executor/remoteserver/Host.java
 - src/main/java/org/embulk/executor/remoteserver/InitializeSessionCommand.java
 - src/main/java/org/embulk/executor/remoteserver/Launcher.java
+- src/main/java/org/embulk/executor/remoteserver/P12File.java
 - src/main/java/org/embulk/executor/remoteserver/PluginArchive.java
 - src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java
 - src/main/java/org/embulk/executor/remoteserver/RemoveSessionCommand.java
 - src/main/java/org/embulk/executor/remoteserver/ServerSession.java
 - src/main/java/org/embulk/executor/remoteserver/ServerSessionRegistry.java
 - src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java
+- src/main/java/org/embulk/executor/remoteserver/TLSConfig.java
 - src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java
 - src/main/java/org/embulk/executor/remoteserver/TaskState.java
 - src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateCommand.java
 - src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateData.java
 - src/main/resources/logback.xml
 - src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java
+- src/test/resources/config/exec_base.yml
+- src/test/resources/config/exec_tls.yml
+- src/test/resources/config/filter.yml
+- src/test/resources/config/input.yml
+- src/test/resources/config/output.yml
 - src/test/resources/json/test1.json
 - src/test/resources/json/test2.json
 - test/Gemfile
-- test/Gemfile.lock
 - test/setup.sh
 homepage: https://github.com/kamatama41/embulk-executor-remoteserver
 licenses:
@@ -115,5 +127,4 @@ specification_version: 4
 summary: Docker executor plugin for Embulk
 test_files:
 - test/Gemfile
-- test/Gemfile.lock
 - test/setup.sh

data/docker-compose.yml

@@ -1,24 +0,0 @@
-version: '3.2'
-services:
-  server1:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    environment:
-      LOG_LEVEL: debug
-    ports:
-      - "30001:30001"
-    volumes:
-      - ./tmp/output:/output
-      - ./src/test/resources/json:/root/src/test/resources/json
-  server2:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    environment:
-      LOG_LEVEL: debug
-    ports:
-      - "30002:30001"
-    volumes:
-      - ./tmp/output:/output
-      - ./src/test/resources/json:/root/src/test/resources/json

data/test/Gemfile.lock

@@ -1,20 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    embulk (0.9.16-java)
-      bundler (>= 1.10.6)
-      liquid (~> 4.0.0)
-      msgpack (~> 1.1.0)
-    embulk-filter-hash (0.5.0)
-    liquid (4.0.0)
-    msgpack (1.1.0-java)
-
-PLATFORMS
-  java
-
-DEPENDENCIES
-  embulk
-  embulk-filter-hash (= 0.5.0)
-
-BUNDLED WITH
-   1.16.0