embulk-executor-remoteserver 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +3 -3
- data/README.md +9 -6
- data/docker-compose.yml +2 -2
- data/gradle.properties +1 -1
- data/src/main/java/org/embulk/executor/remoteserver/{SessionState.java → ClientSession.java} +8 -12
- data/src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java +22 -22
- data/src/main/java/org/embulk/executor/remoteserver/EmbulkServer.java +5 -5
- data/src/main/java/org/embulk/executor/remoteserver/Host.java +13 -8
- data/src/main/java/org/embulk/executor/remoteserver/InitializeSessionCommand.java +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java +10 -6
- data/src/main/java/org/embulk/executor/remoteserver/RemoveSessionCommand.java +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/{Session.java → ServerSession.java} +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/ServerSessionRegistry.java +37 -0
- data/src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java +4 -4
- data/src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java +0 -4
- data/src/main/java/org/embulk/executor/remoteserver/{NotifyTaskStateCommand.java → UpdateTaskStateCommand.java} +6 -6
- data/src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java +1 -1
- metadata +7 -7
- data/src/main/java/org/embulk/executor/remoteserver/SessionManager.java +0 -37
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b18f93356633018febab0d307085bb42dc4a5f79
|
4
|
+
data.tar.gz: f0d31304ca00c26b415c5e7a018179067a98741a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 076b0b9291241b5195e49d572638d87a7d91a40f4908cc437eb372ce454f171c1cacc2ab481240081d241fea15e2bc93aa52462a56600509a879c92964fa3cbd
|
7
|
+
data.tar.gz: a3f6d984008e5815bfce5780e6e5b017621078ae4e8fb6b38f3de5bd70f2b8626c10c5f8216150fdae7fcc5003313b16fa7356e0f7daa4c9d94a04973e80b1ad
|
data/Dockerfile
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
FROM openjdk:8
|
1
|
+
FROM openjdk:8 as builder
|
2
2
|
WORKDIR /usr/src/app
|
3
3
|
COPY build.gradle settings.gradle gradlew gradle.properties ./
|
4
4
|
COPY ./gradle ./gradle
|
@@ -9,6 +9,6 @@ RUN ./gradlew --no-daemon executableEmbulkServer
|
|
9
9
|
FROM openjdk:8-jre
|
10
10
|
WORKDIR /root/
|
11
11
|
COPY ./docker .
|
12
|
-
COPY --from=
|
13
|
-
|
12
|
+
COPY --from=builder /usr/src/app/build/libs/embulk-server-*.jar ./embulk-server.jar
|
13
|
+
CMD ["/root/run_embulk_server.sh"]
|
14
14
|
EXPOSE 30001
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Remote server executor plugin for Embulk
|
2
2
|
|
3
|
-
Embulk executor plugin to run
|
3
|
+
Embulk executor plugin to run Embulk tasks on remote servers.
|
4
4
|
|
5
5
|
## Overview
|
6
6
|
|
@@ -12,7 +12,7 @@ Embulk executor plugin to run plugins on remote server.
|
|
12
12
|
|
13
13
|
## Configuration
|
14
14
|
|
15
|
-
- **hosts**: List of remote servers. If not specified, the executor runs as local mode, which start Embulk server on its own process (array of string)
|
15
|
+
- **hosts**: List of remote servers (`hostname` or `hostname:port`, default port is `30001`). If not specified, the executor runs as local mode, which start Embulk server on its own process (array of string)
|
16
16
|
- **timeout_seconds**: Timeout seconds of the whole execution (integer, default: `3600`)
|
17
17
|
|
18
18
|
## Example
|
@@ -21,13 +21,16 @@ Embulk executor plugin to run plugins on remote server.
|
|
21
21
|
exec:
|
22
22
|
type: remoteserver
|
23
23
|
hosts:
|
24
|
-
-
|
25
|
-
-
|
24
|
+
- embulk-server1.local
|
25
|
+
- embulk-server2.local:30002
|
26
26
|
timeout_seconds: 86400
|
27
27
|
```
|
28
28
|
|
29
|
-
##
|
30
|
-
|
29
|
+
## Embulk server
|
30
|
+
The server recieves requests from client (Embulk) and run Embulk tasks, then returns results to client. It communicates with clients via `TCP 30001 port`.
|
31
|
+
|
32
|
+
### Running Embulk server as a Docker container
|
33
|
+
The image is hosted by [DockerHub](https://hub.docker.com/r/kamatama41/embulk-executor-remoteserver).
|
31
34
|
You can try running Embulk server by the following command.
|
32
35
|
|
33
36
|
```sh
|
data/docker-compose.yml
CHANGED
@@ -7,7 +7,7 @@ services:
|
|
7
7
|
environment:
|
8
8
|
LOG_LEVEL: debug
|
9
9
|
ports:
|
10
|
-
- "
|
10
|
+
- "30001:30001"
|
11
11
|
volumes:
|
12
12
|
- ./tmp/output:/output
|
13
13
|
- ./src/test/resources/json:/root/src/test/resources/json
|
@@ -18,7 +18,7 @@ services:
|
|
18
18
|
environment:
|
19
19
|
LOG_LEVEL: debug
|
20
20
|
ports:
|
21
|
-
- "
|
21
|
+
- "30002:30001"
|
22
22
|
volumes:
|
23
23
|
- ./tmp/output:/output
|
24
24
|
- ./src/test/resources/json:/root/src/test/resources/json
|
data/gradle.properties
CHANGED
@@ -1 +1 @@
|
|
1
|
-
version=0.
|
1
|
+
version=0.2.0
|
data/src/main/java/org/embulk/executor/remoteserver/{SessionState.java → ClientSession.java}
RENAMED
@@ -15,10 +15,10 @@ import java.util.concurrent.TimeUnit;
|
|
15
15
|
import java.util.concurrent.TimeoutException;
|
16
16
|
import java.util.stream.Collectors;
|
17
17
|
|
18
|
-
class
|
19
|
-
private static final Logger log = LoggerFactory.getLogger(
|
18
|
+
class ClientSession {
|
19
|
+
private static final Logger log = LoggerFactory.getLogger(ClientSession.class);
|
20
20
|
|
21
|
-
private String
|
21
|
+
private final String id;
|
22
22
|
private final String systemConfigJson;
|
23
23
|
private final String pluginTaskJson;
|
24
24
|
private final String processTaskJson;
|
@@ -32,11 +32,11 @@ class SessionState {
|
|
32
32
|
private volatile boolean isFinished;
|
33
33
|
private final Map<Integer, String> errorMessages;
|
34
34
|
|
35
|
-
|
35
|
+
ClientSession(
|
36
36
|
String systemConfigJson, String pluginTaskJson, String processTaskJson,
|
37
37
|
List<PluginArchive.GemSpec> gemSpecs, byte[] pluginArchiveBytes,
|
38
38
|
ProcessState state, int inputTaskCount, ModelManager modelManager) {
|
39
|
-
this.
|
39
|
+
this.id = UUID.randomUUID().toString();
|
40
40
|
this.systemConfigJson = systemConfigJson;
|
41
41
|
this.pluginTaskJson = pluginTaskJson;
|
42
42
|
this.processTaskJson = processTaskJson;
|
@@ -50,8 +50,8 @@ class SessionState {
|
|
50
50
|
this.errorMessages = new ConcurrentHashMap<>();
|
51
51
|
}
|
52
52
|
|
53
|
-
String
|
54
|
-
return
|
53
|
+
String getId() {
|
54
|
+
return id;
|
55
55
|
}
|
56
56
|
|
57
57
|
String getSystemConfigJson() {
|
@@ -74,10 +74,6 @@ class SessionState {
|
|
74
74
|
return pluginArchiveBytes;
|
75
75
|
}
|
76
76
|
|
77
|
-
ProcessState getState() {
|
78
|
-
return state;
|
79
|
-
}
|
80
|
-
|
81
77
|
boolean isFinished() {
|
82
78
|
return isFinished;
|
83
79
|
}
|
@@ -110,7 +106,7 @@ class SessionState {
|
|
110
106
|
void waitUntilCompleted(int timeoutSeconds) throws InterruptedException, TimeoutException {
|
111
107
|
try {
|
112
108
|
if (!timer.await(timeoutSeconds, TimeUnit.SECONDS)) {
|
113
|
-
throw new TimeoutException(String.format("The session (%s) was time-out.",
|
109
|
+
throw new TimeoutException(String.format("The session (%s) was time-out.", id));
|
114
110
|
}
|
115
111
|
if (!errorMessages.isEmpty()) {
|
116
112
|
String message = errorMessages.entrySet().stream()
|
@@ -18,28 +18,28 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|
18
18
|
class EmbulkClient implements AutoCloseable {
|
19
19
|
private final SocketClient client;
|
20
20
|
private final List<Host> hosts;
|
21
|
-
private final
|
21
|
+
private final ClientSession session;
|
22
22
|
private final AtomicInteger counter = new AtomicInteger(1);
|
23
23
|
|
24
|
-
private EmbulkClient(SocketClient client, List<Host> hosts,
|
24
|
+
private EmbulkClient(SocketClient client, List<Host> hosts, ClientSession session) {
|
25
25
|
this.client = client;
|
26
26
|
this.hosts = hosts;
|
27
|
-
this.
|
27
|
+
this.session = session;
|
28
28
|
}
|
29
29
|
|
30
30
|
static EmbulkClient open(
|
31
|
-
|
31
|
+
ClientSession session,
|
32
32
|
List<Host> hosts) throws IOException {
|
33
33
|
SocketClient client = new SocketClient();
|
34
34
|
client.registerSyncCommand(new InitializeSessionCommand(null));
|
35
35
|
client.registerSyncCommand(new RemoveSessionCommand(null));
|
36
|
-
client.registerCommand(new
|
37
|
-
client.registerListener(new Reconnector(client,
|
36
|
+
client.registerCommand(new UpdateTaskStateCommand(session));
|
37
|
+
client.registerListener(new Reconnector(client, session));
|
38
38
|
client.open();
|
39
39
|
for (Host host : hosts) {
|
40
40
|
client.addNode(host.toAddress());
|
41
41
|
}
|
42
|
-
return new EmbulkClient(client, hosts,
|
42
|
+
return new EmbulkClient(client, hosts, session);
|
43
43
|
}
|
44
44
|
|
45
45
|
void createSession() {
|
@@ -48,7 +48,7 @@ class EmbulkClient implements AutoCloseable {
|
|
48
48
|
for (Host host : hosts) {
|
49
49
|
futures.add(es.submit(() -> {
|
50
50
|
Connection connection = client.getConnection(host.toAddress());
|
51
|
-
connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(
|
51
|
+
connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
|
52
52
|
}));
|
53
53
|
}
|
54
54
|
try {
|
@@ -68,44 +68,44 @@ class EmbulkClient implements AutoCloseable {
|
|
68
68
|
// Round robin (more smart logic needed?)
|
69
69
|
InetSocketAddress target = hosts.get(counter.getAndIncrement() % hosts.size()).toAddress();
|
70
70
|
client.getConnection(target).sendCommand(
|
71
|
-
StartTaskCommand.ID, new StartTaskCommand.Data(
|
71
|
+
StartTaskCommand.ID, new StartTaskCommand.Data(session.getId(), taskIndex));
|
72
72
|
}
|
73
73
|
|
74
74
|
@Override
|
75
75
|
public void close() throws IOException {
|
76
76
|
for (Host host : hosts) {
|
77
77
|
Connection connection = client.getConnection(host.toAddress());
|
78
|
-
connection.sendSyncCommand(RemoveSessionCommand.ID,
|
78
|
+
connection.sendSyncCommand(RemoveSessionCommand.ID, session.getId());
|
79
79
|
}
|
80
80
|
client.close();
|
81
81
|
}
|
82
82
|
|
83
83
|
private static class Reconnector implements CommandListener {
|
84
84
|
private final SocketClient client;
|
85
|
-
private final
|
85
|
+
private final ClientSession session;
|
86
86
|
|
87
|
-
Reconnector(SocketClient client,
|
87
|
+
Reconnector(SocketClient client, ClientSession session) {
|
88
88
|
this.client = client;
|
89
|
-
this.
|
89
|
+
this.session = session;
|
90
90
|
}
|
91
91
|
|
92
92
|
@Override
|
93
93
|
public void onDisconnected(Connection connection) {
|
94
|
-
if(!
|
94
|
+
if(!session.isFinished()) {
|
95
95
|
Connection newConnection = client.getConnection((InetSocketAddress) connection.getRemoteSocketAddress());
|
96
|
-
newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(
|
96
|
+
newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(session));
|
97
97
|
}
|
98
98
|
}
|
99
99
|
}
|
100
100
|
|
101
|
-
private static InitializeSessionCommand.Data toInitializeSessionData(
|
101
|
+
private static InitializeSessionCommand.Data toInitializeSessionData(ClientSession session) {
|
102
102
|
return new InitializeSessionCommand.Data(
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
103
|
+
session.getId(),
|
104
|
+
session.getSystemConfigJson(),
|
105
|
+
session.getPluginTaskJson(),
|
106
|
+
session.getProcessTaskJson(),
|
107
|
+
session.getGemSpecs(),
|
108
|
+
session.getPluginArchiveBytes()
|
109
109
|
);
|
110
110
|
}
|
111
111
|
}
|
@@ -7,20 +7,20 @@ import java.io.IOException;
|
|
7
7
|
public class EmbulkServer implements AutoCloseable {
|
8
8
|
private SocketServer server;
|
9
9
|
|
10
|
-
EmbulkServer(SocketServer server) {
|
10
|
+
private EmbulkServer(SocketServer server) {
|
11
11
|
this.server = server;
|
12
12
|
}
|
13
13
|
|
14
14
|
static EmbulkServer start(String host, int port, int numOfWorkers) throws IOException {
|
15
15
|
SocketServer server = new SocketServer();
|
16
|
-
|
16
|
+
ServerSessionRegistry sessionRegistry = new ServerSessionRegistry();
|
17
17
|
server.setHost(host);
|
18
18
|
server.setPort(port);
|
19
19
|
server.setDefaultContentBufferSize(4 * 1024 * 1024); // 4MB
|
20
20
|
server.setNumOfWorkers(numOfWorkers);
|
21
|
-
server.registerSyncCommand(new InitializeSessionCommand(
|
22
|
-
server.registerSyncCommand(new RemoveSessionCommand(
|
23
|
-
server.registerCommand(new StartTaskCommand(
|
21
|
+
server.registerSyncCommand(new InitializeSessionCommand(sessionRegistry));
|
22
|
+
server.registerSyncCommand(new RemoveSessionCommand(sessionRegistry));
|
23
|
+
server.registerCommand(new StartTaskCommand(sessionRegistry));
|
24
24
|
server.start();
|
25
25
|
return new EmbulkServer(server);
|
26
26
|
}
|
@@ -1,27 +1,20 @@
|
|
1
1
|
package org.embulk.executor.remoteserver;
|
2
2
|
|
3
|
-
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
-
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
-
|
6
3
|
import java.net.InetSocketAddress;
|
7
4
|
|
8
5
|
class Host {
|
9
6
|
private String name;
|
10
7
|
private int port;
|
11
8
|
|
12
|
-
|
13
|
-
Host(@JsonProperty("name") String name,
|
14
|
-
@JsonProperty("port") int port) {
|
9
|
+
Host(String name, int port) {
|
15
10
|
this.name = name;
|
16
11
|
this.port = port;
|
17
12
|
}
|
18
13
|
|
19
|
-
@JsonProperty
|
20
14
|
String getName() {
|
21
15
|
return name;
|
22
16
|
}
|
23
17
|
|
24
|
-
@JsonProperty
|
25
18
|
int getPort() {
|
26
19
|
return port;
|
27
20
|
}
|
@@ -29,4 +22,16 @@ class Host {
|
|
29
22
|
InetSocketAddress toAddress() {
|
30
23
|
return new InetSocketAddress(name, port);
|
31
24
|
}
|
25
|
+
|
26
|
+
static Host of(String host) {
|
27
|
+
String[] split = host.split(":");
|
28
|
+
if (split.length > 2) {
|
29
|
+
throw new IllegalArgumentException("Host must be the format 'hostname(:port)' but " + host);
|
30
|
+
}
|
31
|
+
if (split.length == 1) {
|
32
|
+
return new Host(split[0], 30001);
|
33
|
+
} else {
|
34
|
+
return new Host(split[0], Integer.parseInt(split[1]));
|
35
|
+
}
|
36
|
+
}
|
32
37
|
}
|
@@ -9,15 +9,15 @@ import java.util.List;
|
|
9
9
|
|
10
10
|
class InitializeSessionCommand implements SyncCommand<InitializeSessionCommand.Data, Void> {
|
11
11
|
static final String ID = "initialize_session";
|
12
|
-
private final
|
12
|
+
private final ServerSessionRegistry sessionRegistry;
|
13
13
|
|
14
|
-
InitializeSessionCommand(
|
15
|
-
this.
|
14
|
+
InitializeSessionCommand(ServerSessionRegistry sessionRegistry) {
|
15
|
+
this.sessionRegistry = sessionRegistry;
|
16
16
|
}
|
17
17
|
|
18
18
|
@Override
|
19
19
|
public Void apply(Data data, Connection connection) {
|
20
|
-
|
20
|
+
sessionRegistry.register(
|
21
21
|
data.getSessionId(),
|
22
22
|
data.getSystemConfigJson(),
|
23
23
|
data.getPluginTaskJson(),
|
@@ -25,17 +25,18 @@ import java.nio.file.Files;
|
|
25
25
|
import java.util.Collections;
|
26
26
|
import java.util.List;
|
27
27
|
import java.util.concurrent.TimeoutException;
|
28
|
+
import java.util.stream.Collectors;
|
28
29
|
|
29
30
|
public class RemoteServerExecutor implements ExecutorPlugin {
|
30
31
|
private static final Logger log = LoggerFactory.getLogger(RemoteServerExecutor.class);
|
31
|
-
private static final Host DEFAULT_HOST = new Host("localhost",
|
32
|
+
private static final Host DEFAULT_HOST = new Host("localhost", 30001);
|
32
33
|
private final ConfigSource systemConfig;
|
33
34
|
private final ScriptingContainer jruby;
|
34
35
|
|
35
36
|
interface PluginTask extends Task {
|
36
37
|
@Config("hosts")
|
37
38
|
@ConfigDefault("[]")
|
38
|
-
List<
|
39
|
+
List<String> getHosts();
|
39
40
|
|
40
41
|
@Config("timeout_seconds")
|
41
42
|
@ConfigDefault("3600")
|
@@ -62,7 +63,10 @@ public class RemoteServerExecutor implements ExecutorPlugin {
|
|
62
63
|
throw new UncheckedIOException(e);
|
63
64
|
}
|
64
65
|
} else {
|
65
|
-
control.transaction(
|
66
|
+
control.transaction(
|
67
|
+
outputSchema,
|
68
|
+
inputTaskCount,
|
69
|
+
new ExecutorImpl(inputTaskCount, task, task.getHosts().stream().map(Host::of).collect(Collectors.toList())));
|
66
70
|
}
|
67
71
|
}
|
68
72
|
|
@@ -97,9 +101,9 @@ public class RemoteServerExecutor implements ExecutorPlugin {
|
|
97
101
|
String pluginTaskJson = modelManager.writeObject(pluginTask);
|
98
102
|
String processTaskJson = modelManager.writeObject(processTask);
|
99
103
|
|
100
|
-
|
104
|
+
ClientSession session = new ClientSession(
|
101
105
|
systemConfigJson, pluginTaskJson, processTaskJson, gemSpecs, pluginArchiveBytes, state, inputTaskCount, modelManager);
|
102
|
-
try (EmbulkClient client = EmbulkClient.open(
|
106
|
+
try (EmbulkClient client = EmbulkClient.open(session, hosts)) {
|
103
107
|
client.createSession();
|
104
108
|
|
105
109
|
state.initialize(inputTaskCount, inputTaskCount);
|
@@ -110,7 +114,7 @@ public class RemoteServerExecutor implements ExecutorPlugin {
|
|
110
114
|
}
|
111
115
|
client.startTask(i);
|
112
116
|
}
|
113
|
-
|
117
|
+
session.waitUntilCompleted(pluginTask.getTimeoutSeconds() + 1); // Add 1 sec to consider network latency
|
114
118
|
} catch (InterruptedException | TimeoutException e) {
|
115
119
|
throw new IllegalStateException(e);
|
116
120
|
} catch (IOException e) {
|
@@ -5,15 +5,15 @@ import com.github.kamatama41.nsocket.SyncCommand;
|
|
5
5
|
|
6
6
|
public class RemoveSessionCommand implements SyncCommand<String, Void> {
|
7
7
|
static final String ID = "remove_session";
|
8
|
-
private final
|
8
|
+
private final ServerSessionRegistry sessionRegistry;
|
9
9
|
|
10
|
-
RemoveSessionCommand(
|
11
|
-
this.
|
10
|
+
RemoveSessionCommand(ServerSessionRegistry sessionRegistry) {
|
11
|
+
this.sessionRegistry = sessionRegistry;
|
12
12
|
}
|
13
13
|
|
14
14
|
@Override
|
15
15
|
public Void apply(String sessionId, Connection connection) {
|
16
|
-
|
16
|
+
sessionRegistry.remove(sessionId);
|
17
17
|
return null;
|
18
18
|
}
|
19
19
|
|
@@ -29,8 +29,8 @@ import java.util.concurrent.TimeUnit;
|
|
29
29
|
|
30
30
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
31
31
|
|
32
|
-
class
|
33
|
-
private static final Logger log = LoggerFactory.getLogger(
|
32
|
+
class ServerSession implements AutoCloseable {
|
33
|
+
private static final Logger log = LoggerFactory.getLogger(ServerSession.class);
|
34
34
|
private final String id;
|
35
35
|
private final EmbulkEmbed embed;
|
36
36
|
private final ScriptingContainer jruby;
|
@@ -44,7 +44,7 @@ class Session implements AutoCloseable {
|
|
44
44
|
private final ExecutorService sessionRunner;
|
45
45
|
private volatile Connection connection;
|
46
46
|
|
47
|
-
|
47
|
+
ServerSession(
|
48
48
|
String id,
|
49
49
|
String systemConfig,
|
50
50
|
String pluginTaskConfig,
|
@@ -145,7 +145,7 @@ class Session implements AutoCloseable {
|
|
145
145
|
UpdateTaskStateData data;
|
146
146
|
Queue<UpdateTaskStateData> buffer = bufferMap.get(taskIndex);
|
147
147
|
while ((data = buffer.poll()) != null) {
|
148
|
-
connection.sendCommand(
|
148
|
+
connection.sendCommand(UpdateTaskStateCommand.ID, data);
|
149
149
|
}
|
150
150
|
}
|
151
151
|
|
@@ -0,0 +1,37 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.github.kamatama41.nsocket.Connection;
|
4
|
+
|
5
|
+
import java.util.List;
|
6
|
+
import java.util.concurrent.ConcurrentHashMap;
|
7
|
+
import java.util.concurrent.ConcurrentMap;
|
8
|
+
|
9
|
+
class ServerSessionRegistry {
|
10
|
+
private final ConcurrentMap<String, ServerSession> sessionMap;
|
11
|
+
|
12
|
+
ServerSessionRegistry() {
|
13
|
+
this.sessionMap = new ConcurrentHashMap<>();
|
14
|
+
}
|
15
|
+
|
16
|
+
void register(String sessionId,
|
17
|
+
String systemConfig,
|
18
|
+
String pluginTaskConfig,
|
19
|
+
String processTaskConfig,
|
20
|
+
List<PluginArchive.GemSpec> gemSpecs,
|
21
|
+
byte[] pluginArchive,
|
22
|
+
Connection connection) {
|
23
|
+
ServerSession session = sessionMap.computeIfAbsent(
|
24
|
+
sessionId, (k) -> new ServerSession(
|
25
|
+
sessionId, systemConfig, pluginTaskConfig, processTaskConfig, gemSpecs, pluginArchive));
|
26
|
+
session.updateConnection(connection);
|
27
|
+
}
|
28
|
+
|
29
|
+
ServerSession get(String sessionId) {
|
30
|
+
return sessionMap.get(sessionId);
|
31
|
+
}
|
32
|
+
|
33
|
+
void remove(String sessionId) {
|
34
|
+
ServerSession removed = sessionMap.remove(sessionId);
|
35
|
+
removed.close();
|
36
|
+
}
|
37
|
+
}
|
@@ -7,15 +7,15 @@ import com.github.kamatama41.nsocket.Connection;
|
|
7
7
|
|
8
8
|
class StartTaskCommand implements Command<StartTaskCommand.Data> {
|
9
9
|
static final String ID = "start_task";
|
10
|
-
private final
|
10
|
+
private final ServerSessionRegistry sessionRegistry;
|
11
11
|
|
12
|
-
StartTaskCommand(
|
13
|
-
this.
|
12
|
+
StartTaskCommand(ServerSessionRegistry sessionRegistry) {
|
13
|
+
this.sessionRegistry = sessionRegistry;
|
14
14
|
}
|
15
15
|
|
16
16
|
@Override
|
17
17
|
public void execute(Data data, Connection connection) {
|
18
|
-
|
18
|
+
ServerSession session = sessionRegistry.get(data.getSessionId());
|
19
19
|
if (session == null) {
|
20
20
|
throw new IllegalStateException("Session is not created.");
|
21
21
|
}
|
@@ -3,17 +3,17 @@ package org.embulk.executor.remoteserver;
|
|
3
3
|
import com.github.kamatama41.nsocket.Command;
|
4
4
|
import com.github.kamatama41.nsocket.Connection;
|
5
5
|
|
6
|
-
class
|
7
|
-
static final String ID = "
|
8
|
-
private final
|
6
|
+
class UpdateTaskStateCommand implements Command<UpdateTaskStateData> {
|
7
|
+
static final String ID = "update_task_state";
|
8
|
+
private final ClientSession session;
|
9
9
|
|
10
|
-
|
11
|
-
this.
|
10
|
+
UpdateTaskStateCommand(ClientSession session) {
|
11
|
+
this.session = session;
|
12
12
|
}
|
13
13
|
|
14
14
|
@Override
|
15
15
|
public void execute(UpdateTaskStateData data, Connection connection) throws Exception {
|
16
|
-
|
16
|
+
session.update(data);
|
17
17
|
}
|
18
18
|
|
19
19
|
@Override
|
@@ -22,7 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
22
22
|
|
23
23
|
@EmbulkTest(value = RemoteServerExecutor.class, name = "remoteserver")
|
24
24
|
class TestRemoteServerExecutor extends EmbulkPluginTest {
|
25
|
-
private static final List<
|
25
|
+
private static final List<String> HOSTS = Arrays.asList("localhost", "localhost:30002");
|
26
26
|
private static final Path OUTPUT_DIR = Paths.get("tmp", "output");
|
27
27
|
private static final Path TEST_DIR = Paths.get("test");
|
28
28
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-executor-remoteserver
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinichi Ishimura
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-04-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -51,7 +51,7 @@ files:
|
|
51
51
|
- LICENSE
|
52
52
|
- README.md
|
53
53
|
- build.gradle
|
54
|
-
- classpath/embulk-executor-remoteserver-0.
|
54
|
+
- classpath/embulk-executor-remoteserver-0.2.0.jar
|
55
55
|
- classpath/msgpack-core-0.8.16.jar
|
56
56
|
- classpath/nsocket-0.2.10.jar
|
57
57
|
- classpath/slf4j-api-1.7.26.jar
|
@@ -66,21 +66,21 @@ files:
|
|
66
66
|
- gradlew.bat
|
67
67
|
- lib/embulk/executor/remoteserver.rb
|
68
68
|
- settings.gradle
|
69
|
+
- src/main/java/org/embulk/executor/remoteserver/ClientSession.java
|
69
70
|
- src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java
|
70
71
|
- src/main/java/org/embulk/executor/remoteserver/EmbulkServer.java
|
71
72
|
- src/main/java/org/embulk/executor/remoteserver/Host.java
|
72
73
|
- src/main/java/org/embulk/executor/remoteserver/InitializeSessionCommand.java
|
73
74
|
- src/main/java/org/embulk/executor/remoteserver/Launcher.java
|
74
|
-
- src/main/java/org/embulk/executor/remoteserver/NotifyTaskStateCommand.java
|
75
75
|
- src/main/java/org/embulk/executor/remoteserver/PluginArchive.java
|
76
76
|
- src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java
|
77
77
|
- src/main/java/org/embulk/executor/remoteserver/RemoveSessionCommand.java
|
78
|
-
- src/main/java/org/embulk/executor/remoteserver/
|
79
|
-
- src/main/java/org/embulk/executor/remoteserver/
|
80
|
-
- src/main/java/org/embulk/executor/remoteserver/SessionState.java
|
78
|
+
- src/main/java/org/embulk/executor/remoteserver/ServerSession.java
|
79
|
+
- src/main/java/org/embulk/executor/remoteserver/ServerSessionRegistry.java
|
81
80
|
- src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java
|
82
81
|
- src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java
|
83
82
|
- src/main/java/org/embulk/executor/remoteserver/TaskState.java
|
83
|
+
- src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateCommand.java
|
84
84
|
- src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateData.java
|
85
85
|
- src/main/resources/logback.xml
|
86
86
|
- src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java
|
@@ -1,37 +0,0 @@
|
|
1
|
-
package org.embulk.executor.remoteserver;
|
2
|
-
|
3
|
-
import com.github.kamatama41.nsocket.Connection;
|
4
|
-
|
5
|
-
import java.util.List;
|
6
|
-
import java.util.concurrent.ConcurrentHashMap;
|
7
|
-
import java.util.concurrent.ConcurrentMap;
|
8
|
-
|
9
|
-
class SessionManager {
|
10
|
-
private final ConcurrentMap<String, Session> sessionMap;
|
11
|
-
|
12
|
-
SessionManager() {
|
13
|
-
this.sessionMap = new ConcurrentHashMap<>();
|
14
|
-
}
|
15
|
-
|
16
|
-
void registerNewSession(String sessionId,
|
17
|
-
String systemConfig,
|
18
|
-
String pluginTaskConfig,
|
19
|
-
String processTaskConfig,
|
20
|
-
List<PluginArchive.GemSpec> gemSpecs,
|
21
|
-
byte[] pluginArchive,
|
22
|
-
Connection connection) {
|
23
|
-
Session session = sessionMap.computeIfAbsent(
|
24
|
-
sessionId, (k) -> new Session(
|
25
|
-
sessionId, systemConfig, pluginTaskConfig, processTaskConfig, gemSpecs, pluginArchive));
|
26
|
-
session.updateConnection(connection);
|
27
|
-
}
|
28
|
-
|
29
|
-
Session getSession(String sessionId) {
|
30
|
-
return sessionMap.get(sessionId);
|
31
|
-
}
|
32
|
-
|
33
|
-
void removeSession(String sessionId) {
|
34
|
-
Session removed = sessionMap.remove(sessionId);
|
35
|
-
removed.close();
|
36
|
-
}
|
37
|
-
}
|