embulk-executor-remoteserver 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +3 -3
- data/README.md +9 -6
- data/docker-compose.yml +2 -2
- data/gradle.properties +1 -1
- data/src/main/java/org/embulk/executor/remoteserver/{SessionState.java → ClientSession.java} +8 -12
- data/src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java +22 -22
- data/src/main/java/org/embulk/executor/remoteserver/EmbulkServer.java +5 -5
- data/src/main/java/org/embulk/executor/remoteserver/Host.java +13 -8
- data/src/main/java/org/embulk/executor/remoteserver/InitializeSessionCommand.java +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java +10 -6
- data/src/main/java/org/embulk/executor/remoteserver/RemoveSessionCommand.java +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/{Session.java → ServerSession.java} +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/ServerSessionRegistry.java +37 -0
- data/src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java +0 -4
- data/src/main/java/org/embulk/executor/remoteserver/{NotifyTaskStateCommand.java → UpdateTaskStateCommand.java} +6 -6
- data/src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java +1 -1
- metadata +7 -7
- data/src/main/java/org/embulk/executor/remoteserver/SessionManager.java +0 -37
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b18f93356633018febab0d307085bb42dc4a5f79
|
4
|
+
data.tar.gz: f0d31304ca00c26b415c5e7a018179067a98741a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 076b0b9291241b5195e49d572638d87a7d91a40f4908cc437eb372ce454f171c1cacc2ab481240081d241fea15e2bc93aa52462a56600509a879c92964fa3cbd
|
7
|
+
data.tar.gz: a3f6d984008e5815bfce5780e6e5b017621078ae4e8fb6b38f3de5bd70f2b8626c10c5f8216150fdae7fcc5003313b16fa7356e0f7daa4c9d94a04973e80b1ad
|
data/Dockerfile
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
FROM openjdk:8
|
1
|
+
FROM openjdk:8 as builder
|
2
2
|
WORKDIR /usr/src/app
|
3
3
|
COPY build.gradle settings.gradle gradlew gradle.properties ./
|
4
4
|
COPY ./gradle ./gradle
|
@@ -9,6 +9,6 @@ RUN ./gradlew --no-daemon executableEmbulkServer
|
|
9
9
|
FROM openjdk:8-jre
|
10
10
|
WORKDIR /root/
|
11
11
|
COPY ./docker .
|
12
|
-
COPY --from=
|
13
|
-
|
12
|
+
COPY --from=builder /usr/src/app/build/libs/embulk-server-*.jar ./embulk-server.jar
|
13
|
+
CMD ["/root/run_embulk_server.sh"]
|
14
14
|
EXPOSE 30001
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Remote server executor plugin for Embulk
|
2
2
|
|
3
|
-
Embulk executor plugin to run
|
3
|
+
Embulk executor plugin to run Embulk tasks on remote servers.
|
4
4
|
|
5
5
|
## Overview
|
6
6
|
|
@@ -12,7 +12,7 @@ Embulk executor plugin to run plugins on remote server.
|
|
12
12
|
|
13
13
|
## Configuration
|
14
14
|
|
15
|
-
- **hosts**: List of remote servers. If not specified, the executor runs as local mode, which start Embulk server on its own process (array of string)
|
15
|
+
- **hosts**: List of remote servers (`hostname` or `hostname:port`, default port is `30001`). If not specified, the executor runs as local mode, which start Embulk server on its own process (array of string)
|
16
16
|
- **timeout_seconds**: Timeout seconds of the whole execution (integer, default: `3600`)
|
17
17
|
|
18
18
|
## Example
|
@@ -21,13 +21,16 @@ Embulk executor plugin to run plugins on remote server.
|
|
21
21
|
exec:
|
22
22
|
type: remoteserver
|
23
23
|
hosts:
|
24
|
-
-
|
25
|
-
-
|
24
|
+
- embulk-server1.local
|
25
|
+
- embulk-server2.local:30002
|
26
26
|
timeout_seconds: 86400
|
27
27
|
```
|
28
28
|
|
29
|
-
##
|
30
|
-
|
29
|
+
## Embulk server
|
30
|
+
The server recieves requests from client (Embulk) and run Embulk tasks, then returns results to client. It communicates with clients via `TCP 30001 port`.
|
31
|
+
|
32
|
+
### Running Embulk server as a Docker container
|
33
|
+
The image is hosted by [DockerHub](https://hub.docker.com/r/kamatama41/embulk-executor-remoteserver).
|
31
34
|
You can try running Embulk server by the following command.
|
32
35
|
|
33
36
|
```sh
|
data/docker-compose.yml
CHANGED
@@ -7,7 +7,7 @@ services:
|
|
7
7
|
environment:
|
8
8
|
LOG_LEVEL: debug
|
9
9
|
ports:
|
10
|
-
- "
|
10
|
+
- "30001:30001"
|
11
11
|
volumes:
|
12
12
|
- ./tmp/output:/output
|
13
13
|
- ./src/test/resources/json:/root/src/test/resources/json
|
@@ -18,7 +18,7 @@ services:
|
|
18
18
|
environment:
|
19
19
|
LOG_LEVEL: debug
|
20
20
|
ports:
|
21
|
-
- "
|
21
|
+
- "30002:30001"
|
22
22
|
volumes:
|
23
23
|
- ./tmp/output:/output
|
24
24
|
- ./src/test/resources/json:/root/src/test/resources/json
|
data/gradle.properties
CHANGED
@@ -1 +1 @@
|
|
1
|
-
version=0.
|
1
|
+
version=0.2.0
|
data/src/main/java/org/embulk/executor/remoteserver/{SessionState.java → ClientSession.java}
RENAMED
@@ -15,10 +15,10 @@ import java.util.concurrent.TimeUnit;
|
|
15
15
|
import java.util.concurrent.TimeoutException;
|
16
16
|
import java.util.stream.Collectors;
|
17
17
|
|
18
|
-
class
|
19
|
-
private static final Logger log = LoggerFactory.getLogger(
|
18
|
+
class ClientSession {
|
19
|
+
private static final Logger log = LoggerFactory.getLogger(ClientSession.class);
|
20
20
|
|
21
|
-
private String
|
21
|
+
private final String id;
|
22
22
|
private final String systemConfigJson;
|
23
23
|
private final String pluginTaskJson;
|
24
24
|
private final String processTaskJson;
|
@@ -32,11 +32,11 @@ class SessionState {
|
|
32
32
|
private volatile boolean isFinished;
|
33
33
|
private final Map<Integer, String> errorMessages;
|
34
34
|
|
35
|
-
|
35
|
+
ClientSession(
|
36
36
|
String systemConfigJson, String pluginTaskJson, String processTaskJson,
|
37
37
|
List<PluginArchive.GemSpec> gemSpecs, byte[] pluginArchiveBytes,
|
38
38
|
ProcessState state, int inputTaskCount, ModelManager modelManager) {
|
39
|
-
this.
|
39
|
+
this.id = UUID.randomUUID().toString();
|
40
40
|
this.systemConfigJson = systemConfigJson;
|
41
41
|
this.pluginTaskJson = pluginTaskJson;
|
42
42
|
this.processTaskJson = processTaskJson;
|
@@ -50,8 +50,8 @@ class SessionState {
|
|
50
50
|
this.errorMessages = new ConcurrentHashMap<>();
|
51
51
|
}
|
52
52
|
|
53
|
-
String
|
54
|
-
return
|
53
|
+
String getId() {
|
54
|
+
return id;
|
55
55
|
}
|
56
56
|
|
57
57
|
String getSystemConfigJson() {
|
@@ -74,10 +74,6 @@ class SessionState {
|
|
74
74
|
return pluginArchiveBytes;
|
75
75
|
}
|
76
76
|
|
77
|
-
ProcessState getState() {
|
78
|
-
return state;
|
79
|
-
}
|
80
|
-
|
81
77
|
boolean isFinished() {
|
82
78
|
return isFinished;
|
83
79
|
}
|
@@ -110,7 +106,7 @@ class SessionState {
|
|
110
106
|
void waitUntilCompleted(int timeoutSeconds) throws InterruptedException, TimeoutException {
|
111
107
|
try {
|
112
108
|
if (!timer.await(timeoutSeconds, TimeUnit.SECONDS)) {
|
113
|
-
throw new TimeoutException(String.format("The session (%s) was time-out.",
|
109
|
+
throw new TimeoutException(String.format("The session (%s) was time-out.", id));
|
114
110
|
}
|
115
111
|
if (!errorMessages.isEmpty()) {
|
116
112
|
String message = errorMessages.entrySet().stream()
|
@@ -18,28 +18,28 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|
18
18
|
class EmbulkClient implements AutoCloseable {
|
19
19
|
private final SocketClient client;
|
20
20
|
private final List<Host> hosts;
|
21
|
-
private final
|
21
|
+
private final ClientSession session;
|
22
22
|
private final AtomicInteger counter = new AtomicInteger(1);
|
23
23
|
|
24
|
-
private EmbulkClient(SocketClient client, List<Host> hosts,
|
24
|
+
private EmbulkClient(SocketClient client, List<Host> hosts, ClientSession session) {
|
25
25
|
this.client = client;
|
26
26
|
this.hosts = hosts;
|
27
|
-
this.
|
27
|
+
this.session = session;
|
28
28
|
}
|
29
29
|
|
30
30
|
static EmbulkClient open(
|
31
|
-
|
31
|
+
ClientSession session,
|
32
32
|
List<Host> hosts) throws IOException {
|
33
33
|
SocketClient client = new SocketClient();
|
34
34
|
client.registerSyncCommand(new InitializeSessionCommand(null));
|
35
35
|
client.registerSyncCommand(new RemoveSessionCommand(null));
|
36
|
-
client.registerCommand(new
|
37
|
-
client.registerListener(new Reconnector(client,
|
36
|
+
client.registerCommand(new UpdateTaskStateCommand(session));
|
37
|
+
client.registerListener(new Reconnector(client, session));
|
38
38
|
client.open();
|
39
39
|
for (Host host : hosts) {
|
40
40
|
client.addNode(host.toAddress());
|
41
41
|
}
|
42
|
-
return new EmbulkClient(client, hosts,
|
42
|
+
return new EmbulkClient(client, hosts, session);
|
43
43
|
}
|
44
44
|
|
45
45
|
void createSession() {
|
@@ -48,7 +48,7 @@ class EmbulkClient implements AutoCloseable {
|
|
48
48
|
for (Host host : hosts) {
|
49
49
|
futures.add(es.submit(() -> {
|
50
50
|
Connection connection = client.getConnection(host.toAddress());
|
51
|
-
connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(
|
51
|
+
connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
|
52
52
|
}));
|
53
53
|
}
|
54
54
|
try {
|
@@ -68,44 +68,44 @@ class EmbulkClient implements AutoCloseable {
|
|
68
68
|
// Round robin (more smart logic needed?)
|
69
69
|
InetSocketAddress target = hosts.get(counter.getAndIncrement() % hosts.size()).toAddress();
|
70
70
|
client.getConnection(target).sendCommand(
|
71
|
-
StartTaskCommand.ID, new StartTaskCommand.Data(
|
71
|
+
StartTaskCommand.ID, new StartTaskCommand.Data(session.getId(), taskIndex));
|
72
72
|
}
|
73
73
|
|
74
74
|
@Override
|
75
75
|
public void close() throws IOException {
|
76
76
|
for (Host host : hosts) {
|
77
77
|
Connection connection = client.getConnection(host.toAddress());
|
78
|
-
connection.sendSyncCommand(RemoveSessionCommand.ID,
|
78
|
+
connection.sendSyncCommand(RemoveSessionCommand.ID, session.getId());
|
79
79
|
}
|
80
80
|
client.close();
|
81
81
|
}
|
82
82
|
|
83
83
|
private static class Reconnector implements CommandListener {
|
84
84
|
private final SocketClient client;
|
85
|
-
private final
|
85
|
+
private final ClientSession session;
|
86
86
|
|
87
|
-
Reconnector(SocketClient client,
|
87
|
+
Reconnector(SocketClient client, ClientSession session) {
|
88
88
|
this.client = client;
|
89
|
-
this.
|
89
|
+
this.session = session;
|
90
90
|
}
|
91
91
|
|
92
92
|
@Override
|
93
93
|
public void onDisconnected(Connection connection) {
|
94
|
-
if(!
|
94
|
+
if(!session.isFinished()) {
|
95
95
|
Connection newConnection = client.getConnection((InetSocketAddress) connection.getRemoteSocketAddress());
|
96
|
-
newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(
|
96
|
+
newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
|
97
97
|
}
|
98
98
|
}
|
99
99
|
}
|
100
100
|
|
101
|
-
private static InitializeSessionCommand.Data toInitializeSessionData(
|
101
|
+
private static InitializeSessionCommand.Data toInitializeSessionData(ClientSession session) {
|
102
102
|
return new InitializeSessionCommand.Data(
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
103
|
+
session.getId(),
|
104
|
+
session.getSystemConfigJson(),
|
105
|
+
session.getPluginTaskJson(),
|
106
|
+
session.getProcessTaskJson(),
|
107
|
+
session.getGemSpecs(),
|
108
|
+
session.getPluginArchiveBytes()
|
109
109
|
);
|
110
110
|
}
|
111
111
|
}
|
@@ -7,20 +7,20 @@ import java.io.IOException;
|
|
7
7
|
public class EmbulkServer implements AutoCloseable {
|
8
8
|
private SocketServer server;
|
9
9
|
|
10
|
-
EmbulkServer(SocketServer server) {
|
10
|
+
private EmbulkServer(SocketServer server) {
|
11
11
|
this.server = server;
|
12
12
|
}
|
13
13
|
|
14
14
|
static EmbulkServer start(String host, int port, int numOfWorkers) throws IOException {
|
15
15
|
SocketServer server = new SocketServer();
|
16
|
-
|
16
|
+
ServerSessionRegistry sessionRegistry = new ServerSessionRegistry();
|
17
17
|
server.setHost(host);
|
18
18
|
server.setPort(port);
|
19
19
|
server.setDefaultContentBufferSize(4 * 1024 * 1024); // 4MB
|
20
20
|
server.setNumOfWorkers(numOfWorkers);
|
21
|
-
server.registerSyncCommand(new InitializeSessionCommand(
|
22
|
-
server.registerSyncCommand(new RemoveSessionCommand(
|
23
|
-
server.registerCommand(new StartTaskCommand(
|
21
|
+
server.registerSyncCommand(new InitializeSessionCommand(sessionRegistry));
|
22
|
+
server.registerSyncCommand(new RemoveSessionCommand(sessionRegistry));
|
23
|
+
server.registerCommand(new StartTaskCommand(sessionRegistry));
|
24
24
|
server.start();
|
25
25
|
return new EmbulkServer(server);
|
26
26
|
}
|
@@ -1,27 +1,20 @@
|
|
1
1
|
package org.embulk.executor.remoteserver;
|
2
2
|
|
3
|
-
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
-
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
-
|
6
3
|
import java.net.InetSocketAddress;
|
7
4
|
|
8
5
|
class Host {
|
9
6
|
private String name;
|
10
7
|
private int port;
|
11
8
|
|
12
|
-
|
13
|
-
Host(@JsonProperty("name") String name,
|
14
|
-
@JsonProperty("port") int port) {
|
9
|
+
Host(String name, int port) {
|
15
10
|
this.name = name;
|
16
11
|
this.port = port;
|
17
12
|
}
|
18
13
|
|
19
|
-
@JsonProperty
|
20
14
|
String getName() {
|
21
15
|
return name;
|
22
16
|
}
|
23
17
|
|
24
|
-
@JsonProperty
|
25
18
|
int getPort() {
|
26
19
|
return port;
|
27
20
|
}
|
@@ -29,4 +22,16 @@ class Host {
|
|
29
22
|
InetSocketAddress toAddress() {
|
30
23
|
return new InetSocketAddress(name, port);
|
31
24
|
}
|
25
|
+
|
26
|
+
static Host of(String host) {
|
27
|
+
String[] split = host.split(":");
|
28
|
+
if (split.length > 2) {
|
29
|
+
throw new IllegalArgumentException("Host must be the format 'hostname(:port)' but " + host);
|
30
|
+
}
|
31
|
+
if (split.length == 1) {
|
32
|
+
return new Host(split[0], 30001);
|
33
|
+
} else {
|
34
|
+
return new Host(split[0], Integer.parseInt(split[1]));
|
35
|
+
}
|
36
|
+
}
|
32
37
|
}
|
@@ -9,15 +9,15 @@ import java.util.List;
|
|
9
9
|
|
10
10
|
class InitializeSessionCommand implements SyncCommand<InitializeSessionCommand.Data, Void> {
|
11
11
|
static final String ID = "initialize_session";
|
12
|
-
private final
|
12
|
+
private final ServerSessionRegistry sessionRegistry;
|
13
13
|
|
14
|
-
InitializeSessionCommand(
|
15
|
-
this.
|
14
|
+
InitializeSessionCommand(ServerSessionRegistry sessionRegistry) {
|
15
|
+
this.sessionRegistry = sessionRegistry;
|
16
16
|
}
|
17
17
|
|
18
18
|
@Override
|
19
19
|
public Void apply(Data data, Connection connection) {
|
20
|
-
|
20
|
+
sessionRegistry.register(
|
21
21
|
data.getSessionId(),
|
22
22
|
data.getSystemConfigJson(),
|
23
23
|
data.getPluginTaskJson(),
|
@@ -25,17 +25,18 @@ import java.nio.file.Files;
|
|
25
25
|
import java.util.Collections;
|
26
26
|
import java.util.List;
|
27
27
|
import java.util.concurrent.TimeoutException;
|
28
|
+
import java.util.stream.Collectors;
|
28
29
|
|
29
30
|
public class RemoteServerExecutor implements ExecutorPlugin {
|
30
31
|
private static final Logger log = LoggerFactory.getLogger(RemoteServerExecutor.class);
|
31
|
-
private static final Host DEFAULT_HOST = new Host("localhost",
|
32
|
+
private static final Host DEFAULT_HOST = new Host("localhost", 30001);
|
32
33
|
private final ConfigSource systemConfig;
|
33
34
|
private final ScriptingContainer jruby;
|
34
35
|
|
35
36
|
interface PluginTask extends Task {
|
36
37
|
@Config("hosts")
|
37
38
|
@ConfigDefault("[]")
|
38
|
-
List<
|
39
|
+
List<String> getHosts();
|
39
40
|
|
40
41
|
@Config("timeout_seconds")
|
41
42
|
@ConfigDefault("3600")
|
@@ -62,7 +63,10 @@ public class RemoteServerExecutor implements ExecutorPlugin {
|
|
62
63
|
throw new UncheckedIOException(e);
|
63
64
|
}
|
64
65
|
} else {
|
65
|
-
control.transaction(
|
66
|
+
control.transaction(
|
67
|
+
outputSchema,
|
68
|
+
inputTaskCount,
|
69
|
+
new ExecutorImpl(inputTaskCount, task, task.getHosts().stream().map(Host::of).collect(Collectors.toList())));
|
66
70
|
}
|
67
71
|
}
|
68
72
|
|
@@ -97,9 +101,9 @@ public class RemoteServerExecutor implements ExecutorPlugin {
|
|
97
101
|
String pluginTaskJson = modelManager.writeObject(pluginTask);
|
98
102
|
String processTaskJson = modelManager.writeObject(processTask);
|
99
103
|
|
100
|
-
|
104
|
+
ClientSession session = new ClientSession(
|
101
105
|
systemConfigJson, pluginTaskJson, processTaskJson, gemSpecs, pluginArchiveBytes, state, inputTaskCount, modelManager);
|
102
|
-
try (EmbulkClient client = EmbulkClient.open(
|
106
|
+
try (EmbulkClient client = EmbulkClient.open(session, hosts)) {
|
103
107
|
client.createSession();
|
104
108
|
|
105
109
|
state.initialize(inputTaskCount, inputTaskCount);
|
@@ -110,7 +114,7 @@ public class RemoteServerExecutor implements ExecutorPlugin {
|
|
110
114
|
}
|
111
115
|
client.startTask(i);
|
112
116
|
}
|
113
|
-
|
117
|
+
session.waitUntilCompleted(pluginTask.getTimeoutSeconds() + 1); // Add 1 sec to consider network latency
|
114
118
|
} catch (InterruptedException | TimeoutException e) {
|
115
119
|
throw new IllegalStateException(e);
|
116
120
|
} catch (IOException e) {
|
@@ -5,15 +5,15 @@ import com.github.kamatama41.nsocket.SyncCommand;
|
|
5
5
|
|
6
6
|
public class RemoveSessionCommand implements SyncCommand<String, Void> {
|
7
7
|
static final String ID = "remove_session";
|
8
|
-
private final
|
8
|
+
private final ServerSessionRegistry sessionRegistry;
|
9
9
|
|
10
|
-
RemoveSessionCommand(
|
11
|
-
this.
|
10
|
+
RemoveSessionCommand(ServerSessionRegistry sessionRegistry) {
|
11
|
+
this.sessionRegistry = sessionRegistry;
|
12
12
|
}
|
13
13
|
|
14
14
|
@Override
|
15
15
|
public Void apply(String sessionId, Connection connection) {
|
16
|
-
|
16
|
+
sessionRegistry.remove(sessionId);
|
17
17
|
return null;
|
18
18
|
}
|
19
19
|
|
@@ -29,8 +29,8 @@ import java.util.concurrent.TimeUnit;
|
|
29
29
|
|
30
30
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
31
31
|
|
32
|
-
class
|
33
|
-
private static final Logger log = LoggerFactory.getLogger(
|
32
|
+
class ServerSession implements AutoCloseable {
|
33
|
+
private static final Logger log = LoggerFactory.getLogger(ServerSession.class);
|
34
34
|
private final String id;
|
35
35
|
private final EmbulkEmbed embed;
|
36
36
|
private final ScriptingContainer jruby;
|
@@ -44,7 +44,7 @@ class Session implements AutoCloseable {
|
|
44
44
|
private final ExecutorService sessionRunner;
|
45
45
|
private volatile Connection connection;
|
46
46
|
|
47
|
-
|
47
|
+
ServerSession(
|
48
48
|
String id,
|
49
49
|
String systemConfig,
|
50
50
|
String pluginTaskConfig,
|
@@ -145,7 +145,7 @@ class Session implements AutoCloseable {
|
|
145
145
|
UpdateTaskStateData data;
|
146
146
|
Queue<UpdateTaskStateData> buffer = bufferMap.get(taskIndex);
|
147
147
|
while ((data = buffer.poll()) != null) {
|
148
|
-
connection.sendCommand(
|
148
|
+
connection.sendCommand(UpdateTaskStateCommand.ID, data);
|
149
149
|
}
|
150
150
|
}
|
151
151
|
|
@@ -0,0 +1,37 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.github.kamatama41.nsocket.Connection;
|
4
|
+
|
5
|
+
import java.util.List;
|
6
|
+
import java.util.concurrent.ConcurrentHashMap;
|
7
|
+
import java.util.concurrent.ConcurrentMap;
|
8
|
+
|
9
|
+
class ServerSessionRegistry {
|
10
|
+
private final ConcurrentMap<String, ServerSession> sessionMap;
|
11
|
+
|
12
|
+
ServerSessionRegistry() {
|
13
|
+
this.sessionMap = new ConcurrentHashMap<>();
|
14
|
+
}
|
15
|
+
|
16
|
+
void register(String sessionId,
|
17
|
+
String systemConfig,
|
18
|
+
String pluginTaskConfig,
|
19
|
+
String processTaskConfig,
|
20
|
+
List<PluginArchive.GemSpec> gemSpecs,
|
21
|
+
byte[] pluginArchive,
|
22
|
+
Connection connection) {
|
23
|
+
ServerSession session = sessionMap.computeIfAbsent(
|
24
|
+
sessionId, (k) -> new ServerSession(
|
25
|
+
sessionId, systemConfig, pluginTaskConfig, processTaskConfig, gemSpecs, pluginArchive));
|
26
|
+
session.updateConnection(connection);
|
27
|
+
}
|
28
|
+
|
29
|
+
ServerSession get(String sessionId) {
|
30
|
+
return sessionMap.get(sessionId);
|
31
|
+
}
|
32
|
+
|
33
|
+
void remove(String sessionId) {
|
34
|
+
ServerSession removed = sessionMap.remove(sessionId);
|
35
|
+
removed.close();
|
36
|
+
}
|
37
|
+
}
|
@@ -7,15 +7,15 @@ import com.github.kamatama41.nsocket.Connection;
|
|
7
7
|
|
8
8
|
class StartTaskCommand implements Command<StartTaskCommand.Data> {
|
9
9
|
static final String ID = "start_task";
|
10
|
-
private final
|
10
|
+
private final ServerSessionRegistry sessionRegistry;
|
11
11
|
|
12
|
-
StartTaskCommand(
|
13
|
-
this.
|
12
|
+
StartTaskCommand(ServerSessionRegistry sessionRegistry) {
|
13
|
+
this.sessionRegistry = sessionRegistry;
|
14
14
|
}
|
15
15
|
|
16
16
|
@Override
|
17
17
|
public void execute(Data data, Connection connection) {
|
18
|
-
|
18
|
+
ServerSession session = sessionRegistry.get(data.getSessionId());
|
19
19
|
if (session == null) {
|
20
20
|
throw new IllegalStateException("Session is not created.");
|
21
21
|
}
|
@@ -3,17 +3,17 @@ package org.embulk.executor.remoteserver;
|
|
3
3
|
import com.github.kamatama41.nsocket.Command;
|
4
4
|
import com.github.kamatama41.nsocket.Connection;
|
5
5
|
|
6
|
-
class
|
7
|
-
static final String ID = "
|
8
|
-
private final
|
6
|
+
class UpdateTaskStateCommand implements Command<UpdateTaskStateData> {
|
7
|
+
static final String ID = "update_task_state";
|
8
|
+
private final ClientSession session;
|
9
9
|
|
10
|
-
|
11
|
-
this.
|
10
|
+
UpdateTaskStateCommand(ClientSession session) {
|
11
|
+
this.session = session;
|
12
12
|
}
|
13
13
|
|
14
14
|
@Override
|
15
15
|
public void execute(UpdateTaskStateData data, Connection connection) throws Exception {
|
16
|
-
|
16
|
+
session.update(data);
|
17
17
|
}
|
18
18
|
|
19
19
|
@Override
|
@@ -22,7 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
22
22
|
|
23
23
|
@EmbulkTest(value = RemoteServerExecutor.class, name = "remoteserver")
|
24
24
|
class TestRemoteServerExecutor extends EmbulkPluginTest {
|
25
|
-
private static final List<
|
25
|
+
private static final List<String> HOSTS = Arrays.asList("localhost", "localhost:30002");
|
26
26
|
private static final Path OUTPUT_DIR = Paths.get("tmp", "output");
|
27
27
|
private static final Path TEST_DIR = Paths.get("test");
|
28
28
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-executor-remoteserver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinichi Ishimura
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-04-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -51,7 +51,7 @@ files:
|
|
51
51
|
- LICENSE
|
52
52
|
- README.md
|
53
53
|
- build.gradle
|
54
|
-
- classpath/embulk-executor-remoteserver-0.
|
54
|
+
- classpath/embulk-executor-remoteserver-0.2.0.jar
|
55
55
|
- classpath/msgpack-core-0.8.16.jar
|
56
56
|
- classpath/nsocket-0.2.10.jar
|
57
57
|
- classpath/slf4j-api-1.7.26.jar
|
@@ -66,21 +66,21 @@ files:
|
|
66
66
|
- gradlew.bat
|
67
67
|
- lib/embulk/executor/remoteserver.rb
|
68
68
|
- settings.gradle
|
69
|
+
- src/main/java/org/embulk/executor/remoteserver/ClientSession.java
|
69
70
|
- src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java
|
70
71
|
- src/main/java/org/embulk/executor/remoteserver/EmbulkServer.java
|
71
72
|
- src/main/java/org/embulk/executor/remoteserver/Host.java
|
72
73
|
- src/main/java/org/embulk/executor/remoteserver/InitializeSessionCommand.java
|
73
74
|
- src/main/java/org/embulk/executor/remoteserver/Launcher.java
|
74
|
-
- src/main/java/org/embulk/executor/remoteserver/NotifyTaskStateCommand.java
|
75
75
|
- src/main/java/org/embulk/executor/remoteserver/PluginArchive.java
|
76
76
|
- src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java
|
77
77
|
- src/main/java/org/embulk/executor/remoteserver/RemoveSessionCommand.java
|
78
|
-
- src/main/java/org/embulk/executor/remoteserver/
|
79
|
-
- src/main/java/org/embulk/executor/remoteserver/
|
80
|
-
- src/main/java/org/embulk/executor/remoteserver/SessionState.java
|
78
|
+
- src/main/java/org/embulk/executor/remoteserver/ServerSession.java
|
79
|
+
- src/main/java/org/embulk/executor/remoteserver/ServerSessionRegistry.java
|
81
80
|
- src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java
|
82
81
|
- src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java
|
83
82
|
- src/main/java/org/embulk/executor/remoteserver/TaskState.java
|
83
|
+
- src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateCommand.java
|
84
84
|
- src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateData.java
|
85
85
|
- src/main/resources/logback.xml
|
86
86
|
- src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java
|
@@ -1,37 +0,0 @@
|
|
1
|
-
package org.embulk.executor.remoteserver;
|
2
|
-
|
3
|
-
import com.github.kamatama41.nsocket.Connection;
|
4
|
-
|
5
|
-
import java.util.List;
|
6
|
-
import java.util.concurrent.ConcurrentHashMap;
|
7
|
-
import java.util.concurrent.ConcurrentMap;
|
8
|
-
|
9
|
-
class SessionManager {
|
10
|
-
private final ConcurrentMap<String, Session> sessionMap;
|
11
|
-
|
12
|
-
SessionManager() {
|
13
|
-
this.sessionMap = new ConcurrentHashMap<>();
|
14
|
-
}
|
15
|
-
|
16
|
-
void registerNewSession(String sessionId,
|
17
|
-
String systemConfig,
|
18
|
-
String pluginTaskConfig,
|
19
|
-
String processTaskConfig,
|
20
|
-
List<PluginArchive.GemSpec> gemSpecs,
|
21
|
-
byte[] pluginArchive,
|
22
|
-
Connection connection) {
|
23
|
-
Session session = sessionMap.computeIfAbsent(
|
24
|
-
sessionId, (k) -> new Session(
|
25
|
-
sessionId, systemConfig, pluginTaskConfig, processTaskConfig, gemSpecs, pluginArchive));
|
26
|
-
session.updateConnection(connection);
|
27
|
-
}
|
28
|
-
|
29
|
-
Session getSession(String sessionId) {
|
30
|
-
return sessionMap.get(sessionId);
|
31
|
-
}
|
32
|
-
|
33
|
-
void removeSession(String sessionId) {
|
34
|
-
Session removed = sessionMap.remove(sessionId);
|
35
|
-
removed.close();
|
36
|
-
}
|
37
|
-
}
|