embulk-executor-remoteserver 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b18f93356633018febab0d307085bb42dc4a5f79
-  data.tar.gz: f0d31304ca00c26b415c5e7a018179067a98741a
+  metadata.gz: d1e201bcff4c6f7ec2cf013c0f8dc02ae9f35bf8
+  data.tar.gz: 91c0ae41c7453450075e834c5cccffac28b5c0c8
 SHA512:
-  metadata.gz: 076b0b9291241b5195e49d572638d87a7d91a40f4908cc437eb372ce454f171c1cacc2ab481240081d241fea15e2bc93aa52462a56600509a879c92964fa3cbd
-  data.tar.gz: a3f6d984008e5815bfce5780e6e5b017621078ae4e8fb6b38f3de5bd70f2b8626c10c5f8216150fdae7fcc5003313b16fa7356e0f7daa4c9d94a04973e80b1ad
+  metadata.gz: 8f0aa143d8add016ae8cb70303ca782783d476766d7896a321caaa9da015aecf3098c7f5271385644caea74b3be2a9a49e6e62f39c4afd1a3a44fceffd9da848
+  data.tar.gz: fa81fb681acf7097ef38b5699faa3cfc2777fd14ad12156527db201894d929dad8c4ac3320cd81322416e723bbeef23c300c5d2898477019a5bad1efd12d08e8

data/build.gradle

@@ -21,7 +21,7 @@ configurations {
 }
 
 dependencies {
-    compile("com.github.kamatama41:nsocket:0.2.10") {
+    compile("com.github.kamatama41:nsocket:0.2.13") {
         // Use embulk-core's Jackson (2.6.7) instead
         exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
     }

data/gradle.properties

@@ -1 +1 @@
-version=0.2.0
+version=0.2.1

data/gradle/dependency-locks/compileClasspath.lockfile

@@ -11,7 +11,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
-com.github.kamatama41:nsocket:0.2.10
+com.github.kamatama41:nsocket:0.2.13
 com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0

data/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -12,7 +12,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
 com.github.kamatama41:embulk-test-helpers:0.8.0
-com.github.kamatama41:nsocket:0.2.10
+com.github.kamatama41:nsocket:0.2.13
 com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0

data/src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java

@@ -4,9 +4,10 @@ import com.github.kamatama41.nsocket.CommandListener;
 import com.github.kamatama41.nsocket.Connection;
 import com.github.kamatama41.nsocket.SocketClient;
 import org.embulk.config.ConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
@@ -15,39 +16,52 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicInteger;
 
-class EmbulkClient implements AutoCloseable {
-    private final SocketClient client;
-    private final List<Host> hosts;
+class EmbulkClient extends SocketClient implements AutoCloseable {
+    private static final Logger log = LoggerFactory.getLogger(EmbulkClient.class);
     private final ClientSession session;
-    private final AtomicInteger counter = new AtomicInteger(1);
+    private final AtomicInteger counter;
 
-    private EmbulkClient(SocketClient client, List<Host> hosts, ClientSession session) {
-        this.client = client;
-        this.hosts = hosts;
+    private EmbulkClient(ClientSession session) {
+        super();
         this.session = session;
+        this.counter = new AtomicInteger(0);
     }
 
     static EmbulkClient open(
             ClientSession session,
             List<Host> hosts) throws IOException {
-        SocketClient client = new SocketClient();
-        client.registerSyncCommand(new InitializeSessionCommand(null));
-        client.registerSyncCommand(new RemoveSessionCommand(null));
-        client.registerCommand(new UpdateTaskStateCommand(session));
-        client.registerListener(new Reconnector(client, session));
+        EmbulkClient client = new EmbulkClient(session);
         client.open();
+
         for (Host host : hosts) {
-            client.addNode(host.toAddress());
+            try {
+                client.addNode(host.toAddress());
+            } catch (IOException e) {
+                log.warn(String.format("Failed to connect to %s", host.toString()), e);
+            }
+        }
+
+        if (client.getActiveConnections().isEmpty()) {
+            throw new IOException("Failed to connect all hosts");
         }
-        return new EmbulkClient(client, hosts, session);
+        return client;
+    }
+
+    @Override
+    public synchronized void open() throws IOException {
+        registerSyncCommand(new InitializeSessionCommand(null));
+        registerSyncCommand(new RemoveSessionCommand(null));
+        registerCommand(new UpdateTaskStateCommand(session));
+        registerListener(new Reconnector());
+        super.open();
     }
 
     void createSession() {
-        ExecutorService es = Executors.newFixedThreadPool(hosts.size());
+        List<Connection> activeConnections = getActiveConnections();
+        ExecutorService es = Executors.newFixedThreadPool(activeConnections.size());
         List<Future> futures = new ArrayList<>();
-        for (Host host : hosts) {
+        for (Connection connection : activeConnections) {
             futures.add(es.submit(() -> {
-                Connection connection = client.getConnection(host.toAddress());
                 connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
             }));
         }
@@ -66,34 +80,30 @@ class EmbulkClient implements AutoCloseable {
 
     void startTask(int taskIndex) {
         // Round robin (more smart logic needed?)
-        InetSocketAddress target = hosts.get(counter.getAndIncrement() % hosts.size()).toAddress();
-        client.getConnection(target).sendCommand(
-                StartTaskCommand.ID, new StartTaskCommand.Data(session.getId(), taskIndex));
+        List<Connection> activeConnections = getActiveConnections();
+        Connection target = activeConnections.get(counter.getAndIncrement() % activeConnections.size());
+        target.sendCommand(StartTaskCommand.ID, new StartTaskCommand.Data(session.getId(), taskIndex));
     }
 
     @Override
     public void close() throws IOException {
-        for (Host host : hosts) {
-            Connection connection = client.getConnection(host.toAddress());
+        for (Connection connection : getActiveConnections()) {
             connection.sendSyncCommand(RemoveSessionCommand.ID, session.getId());
         }
-        client.close();
+        super.close();
     }
 
-    private static class Reconnector implements CommandListener {
-        private final SocketClient client;
-        private final ClientSession session;
-
-        Reconnector(SocketClient client, ClientSession session) {
-            this.client = client;
-            this.session = session;
-        }
-
+    private class Reconnector implements CommandListener {
         @Override
         public void onDisconnected(Connection connection) {
             if(!session.isFinished()) {
-                Connection newConnection = client.getConnection((InetSocketAddress) connection.getRemoteSocketAddress());
-                newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
+                try {
+                    // Try reconnecting
+                    Connection newConnection = reconnect(connection);
+                    newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
+                } catch (IOException e) {
+                    log.warn(String.format("A connection to %s could not be reconnected.", connection.getRemoteSocketAddress()), e);
+                }
             }
         }
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-remoteserver
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Shinichi Ishimura
@@ -51,9 +51,9 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-executor-remoteserver-0.2.0.jar
+- classpath/embulk-executor-remoteserver-0.2.1.jar
 - classpath/msgpack-core-0.8.16.jar
-- classpath/nsocket-0.2.10.jar
+- classpath/nsocket-0.2.13.jar
 - classpath/slf4j-api-1.7.26.jar
 - docker-compose.yml
 - docker/run_embulk_server.sh