embulk-executor-remoteserver 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b18f93356633018febab0d307085bb42dc4a5f79
4
- data.tar.gz: f0d31304ca00c26b415c5e7a018179067a98741a
3
+ metadata.gz: d1e201bcff4c6f7ec2cf013c0f8dc02ae9f35bf8
4
+ data.tar.gz: 91c0ae41c7453450075e834c5cccffac28b5c0c8
5
5
  SHA512:
6
- metadata.gz: 076b0b9291241b5195e49d572638d87a7d91a40f4908cc437eb372ce454f171c1cacc2ab481240081d241fea15e2bc93aa52462a56600509a879c92964fa3cbd
7
- data.tar.gz: a3f6d984008e5815bfce5780e6e5b017621078ae4e8fb6b38f3de5bd70f2b8626c10c5f8216150fdae7fcc5003313b16fa7356e0f7daa4c9d94a04973e80b1ad
6
+ metadata.gz: 8f0aa143d8add016ae8cb70303ca782783d476766d7896a321caaa9da015aecf3098c7f5271385644caea74b3be2a9a49e6e62f39c4afd1a3a44fceffd9da848
7
+ data.tar.gz: fa81fb681acf7097ef38b5699faa3cfc2777fd14ad12156527db201894d929dad8c4ac3320cd81322416e723bbeef23c300c5d2898477019a5bad1efd12d08e8
data/build.gradle CHANGED
@@ -21,7 +21,7 @@ configurations {
21
21
  }
22
22
 
23
23
  dependencies {
24
- compile("com.github.kamatama41:nsocket:0.2.10") {
24
+ compile("com.github.kamatama41:nsocket:0.2.13") {
25
25
  // Use embulk-core's Jackson (2.6.7) instead
26
26
  exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
27
27
  }
data/gradle.properties CHANGED
@@ -1 +1 @@
1
- version=0.2.0
1
+ version=0.2.1
@@ -11,7 +11,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
11
11
  com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
12
12
  com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
13
13
  com.fasterxml.jackson.module:jackson-module-guice:2.6.7
14
- com.github.kamatama41:nsocket:0.2.10
14
+ com.github.kamatama41:nsocket:0.2.13
15
15
  com.google.code.findbugs:annotations:3.0.0
16
16
  com.google.guava:guava:18.0
17
17
  com.google.inject.extensions:guice-multibindings:4.0
@@ -12,7 +12,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
12
12
  com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
13
13
  com.fasterxml.jackson.module:jackson-module-guice:2.6.7
14
14
  com.github.kamatama41:embulk-test-helpers:0.8.0
15
- com.github.kamatama41:nsocket:0.2.10
15
+ com.github.kamatama41:nsocket:0.2.13
16
16
  com.google.code.findbugs:annotations:3.0.0
17
17
  com.google.guava:guava:18.0
18
18
  com.google.inject.extensions:guice-multibindings:4.0
@@ -4,9 +4,10 @@ import com.github.kamatama41.nsocket.CommandListener;
4
4
  import com.github.kamatama41.nsocket.Connection;
5
5
  import com.github.kamatama41.nsocket.SocketClient;
6
6
  import org.embulk.config.ConfigException;
7
+ import org.slf4j.Logger;
8
+ import org.slf4j.LoggerFactory;
7
9
 
8
10
  import java.io.IOException;
9
- import java.net.InetSocketAddress;
10
11
  import java.util.ArrayList;
11
12
  import java.util.List;
12
13
  import java.util.concurrent.ExecutionException;
@@ -15,39 +16,52 @@ import java.util.concurrent.Executors;
15
16
  import java.util.concurrent.Future;
16
17
  import java.util.concurrent.atomic.AtomicInteger;
17
18
 
18
- class EmbulkClient implements AutoCloseable {
19
- private final SocketClient client;
20
- private final List<Host> hosts;
19
+ class EmbulkClient extends SocketClient implements AutoCloseable {
20
+ private static final Logger log = LoggerFactory.getLogger(EmbulkClient.class);
21
21
  private final ClientSession session;
22
- private final AtomicInteger counter = new AtomicInteger(1);
22
+ private final AtomicInteger counter;
23
23
 
24
- private EmbulkClient(SocketClient client, List<Host> hosts, ClientSession session) {
25
- this.client = client;
26
- this.hosts = hosts;
24
+ private EmbulkClient(ClientSession session) {
25
+ super();
27
26
  this.session = session;
27
+ this.counter = new AtomicInteger(0);
28
28
  }
29
29
 
30
30
  static EmbulkClient open(
31
31
  ClientSession session,
32
32
  List<Host> hosts) throws IOException {
33
- SocketClient client = new SocketClient();
34
- client.registerSyncCommand(new InitializeSessionCommand(null));
35
- client.registerSyncCommand(new RemoveSessionCommand(null));
36
- client.registerCommand(new UpdateTaskStateCommand(session));
37
- client.registerListener(new Reconnector(client, session));
33
+ EmbulkClient client = new EmbulkClient(session);
38
34
  client.open();
35
+
39
36
  for (Host host : hosts) {
40
- client.addNode(host.toAddress());
37
+ try {
38
+ client.addNode(host.toAddress());
39
+ } catch (IOException e) {
40
+ log.warn(String.format("Failed to connect to %s", host.toString()), e);
41
+ }
42
+ }
43
+
44
+ if (client.getActiveConnections().isEmpty()) {
45
+ throw new IOException("Failed to connect all hosts");
41
46
  }
42
- return new EmbulkClient(client, hosts, session);
47
+ return client;
48
+ }
49
+
50
+ @Override
51
+ public synchronized void open() throws IOException {
52
+ registerSyncCommand(new InitializeSessionCommand(null));
53
+ registerSyncCommand(new RemoveSessionCommand(null));
54
+ registerCommand(new UpdateTaskStateCommand(session));
55
+ registerListener(new Reconnector());
56
+ super.open();
43
57
  }
44
58
 
45
59
  void createSession() {
46
- ExecutorService es = Executors.newFixedThreadPool(hosts.size());
60
+ List<Connection> activeConnections = getActiveConnections();
61
+ ExecutorService es = Executors.newFixedThreadPool(activeConnections.size());
47
62
  List<Future> futures = new ArrayList<>();
48
- for (Host host : hosts) {
63
+ for (Connection connection : activeConnections) {
49
64
  futures.add(es.submit(() -> {
50
- Connection connection = client.getConnection(host.toAddress());
51
65
  connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
52
66
  }));
53
67
  }
@@ -66,34 +80,30 @@ class EmbulkClient implements AutoCloseable {
66
80
 
67
81
  void startTask(int taskIndex) {
68
82
  // Round robin (more smart logic needed?)
69
- InetSocketAddress target = hosts.get(counter.getAndIncrement() % hosts.size()).toAddress();
70
- client.getConnection(target).sendCommand(
71
- StartTaskCommand.ID, new StartTaskCommand.Data(session.getId(), taskIndex));
83
+ List<Connection> activeConnections = getActiveConnections();
84
+ Connection target = activeConnections.get(counter.getAndIncrement() % activeConnections.size());
85
+ target.sendCommand(StartTaskCommand.ID, new StartTaskCommand.Data(session.getId(), taskIndex));
72
86
  }
73
87
 
74
88
  @Override
75
89
  public void close() throws IOException {
76
- for (Host host : hosts) {
77
- Connection connection = client.getConnection(host.toAddress());
90
+ for (Connection connection : getActiveConnections()) {
78
91
  connection.sendSyncCommand(RemoveSessionCommand.ID, session.getId());
79
92
  }
80
- client.close();
93
+ super.close();
81
94
  }
82
95
 
83
- private static class Reconnector implements CommandListener {
84
- private final SocketClient client;
85
- private final ClientSession session;
86
-
87
- Reconnector(SocketClient client, ClientSession session) {
88
- this.client = client;
89
- this.session = session;
90
- }
91
-
96
+ private class Reconnector implements CommandListener {
92
97
  @Override
93
98
  public void onDisconnected(Connection connection) {
94
99
  if(!session.isFinished()) {
95
- Connection newConnection = client.getConnection((InetSocketAddress) connection.getRemoteSocketAddress());
96
- newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
100
+ try {
101
+ // Try reconnecting
102
+ Connection newConnection = reconnect(connection);
103
+ newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
104
+ } catch (IOException e) {
105
+ log.warn(String.format("A connection to %s could not be reconnected.", connection.getRemoteSocketAddress()), e);
106
+ }
97
107
  }
98
108
  }
99
109
  }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-executor-remoteserver
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shinichi Ishimura
@@ -51,9 +51,9 @@ files:
51
51
  - LICENSE
52
52
  - README.md
53
53
  - build.gradle
54
- - classpath/embulk-executor-remoteserver-0.2.0.jar
54
+ - classpath/embulk-executor-remoteserver-0.2.1.jar
55
55
  - classpath/msgpack-core-0.8.16.jar
56
- - classpath/nsocket-0.2.10.jar
56
+ - classpath/nsocket-0.2.13.jar
57
57
  - classpath/slf4j-api-1.7.26.jar
58
58
  - docker-compose.yml
59
59
  - docker/run_embulk_server.sh