embulk-executor-remoteserver 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/classpath/{embulk-executor-remoteserver-0.2.0.jar → embulk-executor-remoteserver-0.2.1.jar} +0 -0
- data/classpath/{nsocket-0.2.10.jar → nsocket-0.2.13.jar} +0 -0
- data/gradle.properties +1 -1
- data/gradle/dependency-locks/compileClasspath.lockfile +1 -1
- data/gradle/dependency-locks/testCompileClasspath.lockfile +1 -1
- data/src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java +45 -35
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d1e201bcff4c6f7ec2cf013c0f8dc02ae9f35bf8
|
4
|
+
data.tar.gz: 91c0ae41c7453450075e834c5cccffac28b5c0c8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f0aa143d8add016ae8cb70303ca782783d476766d7896a321caaa9da015aecf3098c7f5271385644caea74b3be2a9a49e6e62f39c4afd1a3a44fceffd9da848
|
7
|
+
data.tar.gz: fa81fb681acf7097ef38b5699faa3cfc2777fd14ad12156527db201894d929dad8c4ac3320cd81322416e723bbeef23c300c5d2898477019a5bad1efd12d08e8
|
data/build.gradle
CHANGED
@@ -21,7 +21,7 @@ configurations {
|
|
21
21
|
}
|
22
22
|
|
23
23
|
dependencies {
|
24
|
-
compile("com.github.kamatama41:nsocket:0.2.
|
24
|
+
compile("com.github.kamatama41:nsocket:0.2.13") {
|
25
25
|
// Use embulk-core's Jackson (2.6.7) instead
|
26
26
|
exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
|
27
27
|
}
|
data/classpath/{embulk-executor-remoteserver-0.2.0.jar → embulk-executor-remoteserver-0.2.1.jar}
RENAMED
Binary file
|
Binary file
|
data/gradle.properties
CHANGED
@@ -1 +1 @@
|
|
1
|
-
version=0.2.
|
1
|
+
version=0.2.1
|
@@ -11,7 +11,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
|
|
11
11
|
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
|
12
12
|
com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
|
13
13
|
com.fasterxml.jackson.module:jackson-module-guice:2.6.7
|
14
|
-
com.github.kamatama41:nsocket:0.2.
|
14
|
+
com.github.kamatama41:nsocket:0.2.13
|
15
15
|
com.google.code.findbugs:annotations:3.0.0
|
16
16
|
com.google.guava:guava:18.0
|
17
17
|
com.google.inject.extensions:guice-multibindings:4.0
|
@@ -12,7 +12,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
|
|
12
12
|
com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
|
13
13
|
com.fasterxml.jackson.module:jackson-module-guice:2.6.7
|
14
14
|
com.github.kamatama41:embulk-test-helpers:0.8.0
|
15
|
-
com.github.kamatama41:nsocket:0.2.
|
15
|
+
com.github.kamatama41:nsocket:0.2.13
|
16
16
|
com.google.code.findbugs:annotations:3.0.0
|
17
17
|
com.google.guava:guava:18.0
|
18
18
|
com.google.inject.extensions:guice-multibindings:4.0
|
@@ -4,9 +4,10 @@ import com.github.kamatama41.nsocket.CommandListener;
|
|
4
4
|
import com.github.kamatama41.nsocket.Connection;
|
5
5
|
import com.github.kamatama41.nsocket.SocketClient;
|
6
6
|
import org.embulk.config.ConfigException;
|
7
|
+
import org.slf4j.Logger;
|
8
|
+
import org.slf4j.LoggerFactory;
|
7
9
|
|
8
10
|
import java.io.IOException;
|
9
|
-
import java.net.InetSocketAddress;
|
10
11
|
import java.util.ArrayList;
|
11
12
|
import java.util.List;
|
12
13
|
import java.util.concurrent.ExecutionException;
|
@@ -15,39 +16,52 @@ import java.util.concurrent.Executors;
|
|
15
16
|
import java.util.concurrent.Future;
|
16
17
|
import java.util.concurrent.atomic.AtomicInteger;
|
17
18
|
|
18
|
-
class EmbulkClient implements AutoCloseable {
|
19
|
-
private final
|
20
|
-
private final List<Host> hosts;
|
19
|
+
class EmbulkClient extends SocketClient implements AutoCloseable {
|
20
|
+
private static final Logger log = LoggerFactory.getLogger(EmbulkClient.class);
|
21
21
|
private final ClientSession session;
|
22
|
-
private final AtomicInteger counter
|
22
|
+
private final AtomicInteger counter;
|
23
23
|
|
24
|
-
private EmbulkClient(
|
25
|
-
|
26
|
-
this.hosts = hosts;
|
24
|
+
private EmbulkClient(ClientSession session) {
|
25
|
+
super();
|
27
26
|
this.session = session;
|
27
|
+
this.counter = new AtomicInteger(0);
|
28
28
|
}
|
29
29
|
|
30
30
|
static EmbulkClient open(
|
31
31
|
ClientSession session,
|
32
32
|
List<Host> hosts) throws IOException {
|
33
|
-
|
34
|
-
client.registerSyncCommand(new InitializeSessionCommand(null));
|
35
|
-
client.registerSyncCommand(new RemoveSessionCommand(null));
|
36
|
-
client.registerCommand(new UpdateTaskStateCommand(session));
|
37
|
-
client.registerListener(new Reconnector(client, session));
|
33
|
+
EmbulkClient client = new EmbulkClient(session);
|
38
34
|
client.open();
|
35
|
+
|
39
36
|
for (Host host : hosts) {
|
40
|
-
|
37
|
+
try {
|
38
|
+
client.addNode(host.toAddress());
|
39
|
+
} catch (IOException e) {
|
40
|
+
log.warn(String.format("Failed to connect to %s", host.toString()), e);
|
41
|
+
}
|
42
|
+
}
|
43
|
+
|
44
|
+
if (client.getActiveConnections().isEmpty()) {
|
45
|
+
throw new IOException("Failed to connect all hosts");
|
41
46
|
}
|
42
|
-
return
|
47
|
+
return client;
|
48
|
+
}
|
49
|
+
|
50
|
+
@Override
|
51
|
+
public synchronized void open() throws IOException {
|
52
|
+
registerSyncCommand(new InitializeSessionCommand(null));
|
53
|
+
registerSyncCommand(new RemoveSessionCommand(null));
|
54
|
+
registerCommand(new UpdateTaskStateCommand(session));
|
55
|
+
registerListener(new Reconnector());
|
56
|
+
super.open();
|
43
57
|
}
|
44
58
|
|
45
59
|
void createSession() {
|
46
|
-
|
60
|
+
List<Connection> activeConnections = getActiveConnections();
|
61
|
+
ExecutorService es = Executors.newFixedThreadPool(activeConnections.size());
|
47
62
|
List<Future> futures = new ArrayList<>();
|
48
|
-
for (
|
63
|
+
for (Connection connection : activeConnections) {
|
49
64
|
futures.add(es.submit(() -> {
|
50
|
-
Connection connection = client.getConnection(host.toAddress());
|
51
65
|
connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
|
52
66
|
}));
|
53
67
|
}
|
@@ -66,34 +80,30 @@ class EmbulkClient implements AutoCloseable {
|
|
66
80
|
|
67
81
|
void startTask(int taskIndex) {
|
68
82
|
// Round robin (more smart logic needed?)
|
69
|
-
|
70
|
-
|
71
|
-
|
83
|
+
List<Connection> activeConnections = getActiveConnections();
|
84
|
+
Connection target = activeConnections.get(counter.getAndIncrement() % activeConnections.size());
|
85
|
+
target.sendCommand(StartTaskCommand.ID, new StartTaskCommand.Data(session.getId(), taskIndex));
|
72
86
|
}
|
73
87
|
|
74
88
|
@Override
|
75
89
|
public void close() throws IOException {
|
76
|
-
for (
|
77
|
-
Connection connection = client.getConnection(host.toAddress());
|
90
|
+
for (Connection connection : getActiveConnections()) {
|
78
91
|
connection.sendSyncCommand(RemoveSessionCommand.ID, session.getId());
|
79
92
|
}
|
80
|
-
|
93
|
+
super.close();
|
81
94
|
}
|
82
95
|
|
83
|
-
private
|
84
|
-
private final SocketClient client;
|
85
|
-
private final ClientSession session;
|
86
|
-
|
87
|
-
Reconnector(SocketClient client, ClientSession session) {
|
88
|
-
this.client = client;
|
89
|
-
this.session = session;
|
90
|
-
}
|
91
|
-
|
96
|
+
private class Reconnector implements CommandListener {
|
92
97
|
@Override
|
93
98
|
public void onDisconnected(Connection connection) {
|
94
99
|
if(!session.isFinished()) {
|
95
|
-
|
96
|
-
|
100
|
+
try {
|
101
|
+
// Try reconnecting
|
102
|
+
Connection newConnection = reconnect(connection);
|
103
|
+
newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
|
104
|
+
} catch (IOException e) {
|
105
|
+
log.warn(String.format("A connection to %s could not be reconnected.", connection.getRemoteSocketAddress()), e);
|
106
|
+
}
|
97
107
|
}
|
98
108
|
}
|
99
109
|
}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-executor-remoteserver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinichi Ishimura
|
@@ -51,9 +51,9 @@ files:
|
|
51
51
|
- LICENSE
|
52
52
|
- README.md
|
53
53
|
- build.gradle
|
54
|
-
- classpath/embulk-executor-remoteserver-0.2.
|
54
|
+
- classpath/embulk-executor-remoteserver-0.2.1.jar
|
55
55
|
- classpath/msgpack-core-0.8.16.jar
|
56
|
-
- classpath/nsocket-0.2.
|
56
|
+
- classpath/nsocket-0.2.13.jar
|
57
57
|
- classpath/slf4j-api-1.7.26.jar
|
58
58
|
- docker-compose.yml
|
59
59
|
- docker/run_embulk_server.sh
|