embulk-executor-remoteserver 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.circleci/config.yml +42 -0
- data/.gitignore +14 -0
- data/Dockerfile +14 -0
- data/LICENSE +21 -0
- data/README.md +30 -0
- data/build.gradle +70 -0
- data/docker-compose.yml +24 -0
- data/docker/run_embulk_server.sh +3 -0
- data/gradle.properties +1 -0
- data/gradle/dependency-locks/compileClasspath.lockfile +40 -0
- data/gradle/dependency-locks/testCompileClasspath.lockfile +49 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +5 -0
- data/gradlew +172 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/executor/remoteserver.rb +3 -0
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/executor/remoteserver/EmbulkClient.java +111 -0
- data/src/main/java/org/embulk/executor/remoteserver/EmbulkServer.java +32 -0
- data/src/main/java/org/embulk/executor/remoteserver/Host.java +32 -0
- data/src/main/java/org/embulk/executor/remoteserver/InitializeSessionCommand.java +94 -0
- data/src/main/java/org/embulk/executor/remoteserver/Launcher.java +25 -0
- data/src/main/java/org/embulk/executor/remoteserver/NotifyTaskStateCommand.java +23 -0
- data/src/main/java/org/embulk/executor/remoteserver/PluginArchive.java +170 -0
- data/src/main/java/org/embulk/executor/remoteserver/RemoteServerExecutor.java +131 -0
- data/src/main/java/org/embulk/executor/remoteserver/RemoveSessionCommand.java +24 -0
- data/src/main/java/org/embulk/executor/remoteserver/Session.java +177 -0
- data/src/main/java/org/embulk/executor/remoteserver/SessionManager.java +37 -0
- data/src/main/java/org/embulk/executor/remoteserver/SessionState.java +143 -0
- data/src/main/java/org/embulk/executor/remoteserver/StartTaskCommand.java +51 -0
- data/src/main/java/org/embulk/executor/remoteserver/TaskExecutionException.java +11 -0
- data/src/main/java/org/embulk/executor/remoteserver/TaskState.java +5 -0
- data/src/main/java/org/embulk/executor/remoteserver/UpdateTaskStateData.java +55 -0
- data/src/main/resources/logback.xml +11 -0
- data/src/test/java/org/embulk/executor/remoteserver/TestRemoteServerExecutor.java +80 -0
- data/src/test/resources/json/test1.json +1 -0
- data/src/test/resources/json/test2.json +1 -0
- data/test/Gemfile +4 -0
- data/test/Gemfile.lock +20 -0
- data/test/setup.sh +8 -0
- metadata +119 -0
data/gradlew.bat
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
set DIRNAME=%~dp0
|
12
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
+
set APP_BASE_NAME=%~n0
|
14
|
+
set APP_HOME=%DIRNAME%
|
15
|
+
|
16
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
+
set DEFAULT_JVM_OPTS="-Xmx64m"
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windows variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
|
53
|
+
:win9xME_args
|
54
|
+
@rem Slurp the command line arguments.
|
55
|
+
set CMD_LINE_ARGS=
|
56
|
+
set _SKIP=2
|
57
|
+
|
58
|
+
:win9xME_args_slurp
|
59
|
+
if "x%~1" == "x" goto execute
|
60
|
+
|
61
|
+
set CMD_LINE_ARGS=%*
|
62
|
+
|
63
|
+
:execute
|
64
|
+
@rem Setup the command line
|
65
|
+
|
66
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
67
|
+
|
68
|
+
@rem Execute Gradle
|
69
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
70
|
+
|
71
|
+
:end
|
72
|
+
@rem End local scope for the variables with windows NT shell
|
73
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
74
|
+
|
75
|
+
:fail
|
76
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
77
|
+
rem the _cmd.exe /c_ return code!
|
78
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
79
|
+
exit /b 1
|
80
|
+
|
81
|
+
:mainEnd
|
82
|
+
if "%OS%"=="Windows_NT" endlocal
|
83
|
+
|
84
|
+
:omega
|
data/settings.gradle
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rootProject.name = 'embulk-executor-remoteserver'
|
@@ -0,0 +1,111 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.github.kamatama41.nsocket.CommandListener;
|
4
|
+
import com.github.kamatama41.nsocket.Connection;
|
5
|
+
import com.github.kamatama41.nsocket.SocketClient;
|
6
|
+
import org.embulk.config.ConfigException;
|
7
|
+
|
8
|
+
import java.io.IOException;
|
9
|
+
import java.net.InetSocketAddress;
|
10
|
+
import java.util.ArrayList;
|
11
|
+
import java.util.List;
|
12
|
+
import java.util.concurrent.ExecutionException;
|
13
|
+
import java.util.concurrent.ExecutorService;
|
14
|
+
import java.util.concurrent.Executors;
|
15
|
+
import java.util.concurrent.Future;
|
16
|
+
import java.util.concurrent.atomic.AtomicInteger;
|
17
|
+
|
18
|
+
class EmbulkClient implements AutoCloseable {
|
19
|
+
private final SocketClient client;
|
20
|
+
private final List<Host> hosts;
|
21
|
+
private final SessionState sessionState;
|
22
|
+
private final AtomicInteger counter = new AtomicInteger(1);
|
23
|
+
|
24
|
+
private EmbulkClient(SocketClient client, List<Host> hosts, SessionState sessionState) {
|
25
|
+
this.client = client;
|
26
|
+
this.hosts = hosts;
|
27
|
+
this.sessionState = sessionState;
|
28
|
+
}
|
29
|
+
|
30
|
+
static EmbulkClient open(
|
31
|
+
SessionState sessionState,
|
32
|
+
List<Host> hosts) throws IOException {
|
33
|
+
SocketClient client = new SocketClient();
|
34
|
+
client.registerSyncCommand(new InitializeSessionCommand(null));
|
35
|
+
client.registerSyncCommand(new RemoveSessionCommand(null));
|
36
|
+
client.registerCommand(new NotifyTaskStateCommand(sessionState));
|
37
|
+
client.registerListener(new Reconnector(client, sessionState));
|
38
|
+
client.open();
|
39
|
+
for (Host host : hosts) {
|
40
|
+
client.addNode(host.toAddress());
|
41
|
+
}
|
42
|
+
return new EmbulkClient(client, hosts, sessionState);
|
43
|
+
}
|
44
|
+
|
45
|
+
void createSession() {
|
46
|
+
ExecutorService es = Executors.newFixedThreadPool(hosts.size());
|
47
|
+
List<Future> futures = new ArrayList<>();
|
48
|
+
for (Host host : hosts) {
|
49
|
+
futures.add(es.submit(() -> {
|
50
|
+
Connection connection = client.getConnection(host.toAddress());
|
51
|
+
connection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(sessionState));
|
52
|
+
}));
|
53
|
+
}
|
54
|
+
try {
|
55
|
+
for (Future future : futures) {
|
56
|
+
future.get();
|
57
|
+
}
|
58
|
+
} catch (ExecutionException e) {
|
59
|
+
throw new ConfigException(e.getCause());
|
60
|
+
} catch (Exception e) {
|
61
|
+
throw new ConfigException(e);
|
62
|
+
} finally {
|
63
|
+
es.shutdown();
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
67
|
+
void startTask(int taskIndex) {
|
68
|
+
// Round robin (more smart logic needed?)
|
69
|
+
InetSocketAddress target = hosts.get(counter.getAndIncrement() % hosts.size()).toAddress();
|
70
|
+
client.getConnection(target).sendCommand(
|
71
|
+
StartTaskCommand.ID, new StartTaskCommand.Data(sessionState.getSessionId(), taskIndex));
|
72
|
+
}
|
73
|
+
|
74
|
+
@Override
|
75
|
+
public void close() throws IOException {
|
76
|
+
for (Host host : hosts) {
|
77
|
+
Connection connection = client.getConnection(host.toAddress());
|
78
|
+
connection.sendSyncCommand(RemoveSessionCommand.ID, sessionState.getSessionId());
|
79
|
+
}
|
80
|
+
client.close();
|
81
|
+
}
|
82
|
+
|
83
|
+
private static class Reconnector implements CommandListener {
|
84
|
+
private final SocketClient client;
|
85
|
+
private final SessionState sessionState;
|
86
|
+
|
87
|
+
Reconnector(SocketClient client, SessionState sessionState) {
|
88
|
+
this.client = client;
|
89
|
+
this.sessionState = sessionState;
|
90
|
+
}
|
91
|
+
|
92
|
+
@Override
|
93
|
+
public void onDisconnected(Connection connection) {
|
94
|
+
if(!sessionState.isFinished()) {
|
95
|
+
Connection newConnection = client.getConnection((InetSocketAddress) connection.getRemoteSocketAddress());
|
96
|
+
newConnection.sendSyncCommand(InitializeSessionCommand.ID, toInitializeSessionData(sessionState));
|
97
|
+
}
|
98
|
+
}
|
99
|
+
}
|
100
|
+
|
101
|
+
private static InitializeSessionCommand.Data toInitializeSessionData(SessionState sessionState) {
|
102
|
+
return new InitializeSessionCommand.Data(
|
103
|
+
sessionState.getSessionId(),
|
104
|
+
sessionState.getSystemConfigJson(),
|
105
|
+
sessionState.getPluginTaskJson(),
|
106
|
+
sessionState.getProcessTaskJson(),
|
107
|
+
sessionState.getGemSpecs(),
|
108
|
+
sessionState.getPluginArchiveBytes()
|
109
|
+
);
|
110
|
+
}
|
111
|
+
}
|
@@ -0,0 +1,32 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.github.kamatama41.nsocket.SocketServer;
|
4
|
+
|
5
|
+
import java.io.IOException;
|
6
|
+
|
7
|
+
public class EmbulkServer implements AutoCloseable {
|
8
|
+
private SocketServer server;
|
9
|
+
|
10
|
+
EmbulkServer(SocketServer server) {
|
11
|
+
this.server = server;
|
12
|
+
}
|
13
|
+
|
14
|
+
static EmbulkServer start(String host, int port, int numOfWorkers) throws IOException {
|
15
|
+
SocketServer server = new SocketServer();
|
16
|
+
SessionManager sessionManager = new SessionManager();
|
17
|
+
server.setHost(host);
|
18
|
+
server.setPort(port);
|
19
|
+
server.setDefaultContentBufferSize(4 * 1024 * 1024); // 4MB
|
20
|
+
server.setNumOfWorkers(numOfWorkers);
|
21
|
+
server.registerSyncCommand(new InitializeSessionCommand(sessionManager));
|
22
|
+
server.registerSyncCommand(new RemoveSessionCommand(sessionManager));
|
23
|
+
server.registerCommand(new StartTaskCommand(sessionManager));
|
24
|
+
server.start();
|
25
|
+
return new EmbulkServer(server);
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
public void close() throws IOException {
|
30
|
+
server.stop();
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,32 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
+
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
+
|
6
|
+
import java.net.InetSocketAddress;
|
7
|
+
|
8
|
+
class Host {
|
9
|
+
private String name;
|
10
|
+
private int port;
|
11
|
+
|
12
|
+
@JsonCreator
|
13
|
+
Host(@JsonProperty("name") String name,
|
14
|
+
@JsonProperty("port") int port) {
|
15
|
+
this.name = name;
|
16
|
+
this.port = port;
|
17
|
+
}
|
18
|
+
|
19
|
+
@JsonProperty
|
20
|
+
String getName() {
|
21
|
+
return name;
|
22
|
+
}
|
23
|
+
|
24
|
+
@JsonProperty
|
25
|
+
int getPort() {
|
26
|
+
return port;
|
27
|
+
}
|
28
|
+
|
29
|
+
InetSocketAddress toAddress() {
|
30
|
+
return new InetSocketAddress(name, port);
|
31
|
+
}
|
32
|
+
}
|
@@ -0,0 +1,94 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
+
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
+
import com.github.kamatama41.nsocket.Connection;
|
6
|
+
import com.github.kamatama41.nsocket.SyncCommand;
|
7
|
+
|
8
|
+
import java.util.List;
|
9
|
+
|
10
|
+
class InitializeSessionCommand implements SyncCommand<InitializeSessionCommand.Data, Void> {
|
11
|
+
static final String ID = "initialize_session";
|
12
|
+
private final SessionManager sessionManager;
|
13
|
+
|
14
|
+
InitializeSessionCommand(SessionManager sessionManager) {
|
15
|
+
this.sessionManager = sessionManager;
|
16
|
+
}
|
17
|
+
|
18
|
+
@Override
|
19
|
+
public Void apply(Data data, Connection connection) {
|
20
|
+
sessionManager.registerNewSession(
|
21
|
+
data.getSessionId(),
|
22
|
+
data.getSystemConfigJson(),
|
23
|
+
data.getPluginTaskJson(),
|
24
|
+
data.getProcessTaskJson(),
|
25
|
+
data.getGemSpecs(),
|
26
|
+
data.getPluginArchive(),
|
27
|
+
connection);
|
28
|
+
return null;
|
29
|
+
}
|
30
|
+
|
31
|
+
@Override
|
32
|
+
public long getTimeoutMillis() {
|
33
|
+
return 60000L;
|
34
|
+
}
|
35
|
+
|
36
|
+
@Override
|
37
|
+
public String getId() {
|
38
|
+
return ID;
|
39
|
+
}
|
40
|
+
|
41
|
+
static class Data {
|
42
|
+
private String sessionId;
|
43
|
+
private String systemConfigJson;
|
44
|
+
private String pluginTaskJson;
|
45
|
+
private String processTaskJson;
|
46
|
+
private List<PluginArchive.GemSpec> gemSpecs;
|
47
|
+
private byte[] pluginArchive;
|
48
|
+
|
49
|
+
@JsonCreator
|
50
|
+
Data(@JsonProperty("sessionId") String sessionId,
|
51
|
+
@JsonProperty("systemConfigJson") String systemConfigJson,
|
52
|
+
@JsonProperty("pluginTaskJson") String pluginTaskJson,
|
53
|
+
@JsonProperty("processTaskJson") String processTaskJson,
|
54
|
+
@JsonProperty("gemSpecs") List<PluginArchive.GemSpec> gemSpecs,
|
55
|
+
@JsonProperty("pluginArchive") byte[] pluginArchive) {
|
56
|
+
this.sessionId = sessionId;
|
57
|
+
this.systemConfigJson = systemConfigJson;
|
58
|
+
this.pluginTaskJson = pluginTaskJson;
|
59
|
+
this.processTaskJson = processTaskJson;
|
60
|
+
this.gemSpecs = gemSpecs;
|
61
|
+
this.pluginArchive = pluginArchive;
|
62
|
+
}
|
63
|
+
|
64
|
+
@JsonProperty
|
65
|
+
String getSessionId() {
|
66
|
+
return sessionId;
|
67
|
+
}
|
68
|
+
|
69
|
+
@JsonProperty
|
70
|
+
String getSystemConfigJson() {
|
71
|
+
return systemConfigJson;
|
72
|
+
}
|
73
|
+
|
74
|
+
@JsonProperty
|
75
|
+
String getPluginTaskJson() {
|
76
|
+
return pluginTaskJson;
|
77
|
+
}
|
78
|
+
|
79
|
+
@JsonProperty
|
80
|
+
String getProcessTaskJson() {
|
81
|
+
return processTaskJson;
|
82
|
+
}
|
83
|
+
|
84
|
+
@JsonProperty
|
85
|
+
List<PluginArchive.GemSpec> getGemSpecs() {
|
86
|
+
return gemSpecs;
|
87
|
+
}
|
88
|
+
|
89
|
+
@JsonProperty
|
90
|
+
byte[] getPluginArchive() {
|
91
|
+
return pluginArchive;
|
92
|
+
}
|
93
|
+
}
|
94
|
+
}
|
@@ -0,0 +1,25 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import ch.qos.logback.classic.Level;
|
4
|
+
import ch.qos.logback.classic.Logger;
|
5
|
+
import org.slf4j.LoggerFactory;
|
6
|
+
|
7
|
+
import java.io.IOException;
|
8
|
+
import java.util.Map;
|
9
|
+
|
10
|
+
public class Launcher {
|
11
|
+
public static void main(String[] args) throws IOException {
|
12
|
+
Map<String, String> envVars = System.getenv();
|
13
|
+
String host = envVars.getOrDefault("BIND_ADDRESS", "0.0.0.0");
|
14
|
+
int port = Integer.parseInt(envVars.getOrDefault("PORT", "30001"));
|
15
|
+
int numOfWorkers = Integer.parseInt(envVars.getOrDefault("NUM_OF_WORKERS", "1"));
|
16
|
+
Level logLevel = Level.toLevel(envVars.getOrDefault("LOG_LEVEL", "info"));
|
17
|
+
configureLogLevel(logLevel);
|
18
|
+
EmbulkServer.start(host, port, numOfWorkers);
|
19
|
+
}
|
20
|
+
|
21
|
+
private static void configureLogLevel(Level logLevel) {
|
22
|
+
Logger rootLogger = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
|
23
|
+
rootLogger.setLevel(logLevel);
|
24
|
+
}
|
25
|
+
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.github.kamatama41.nsocket.Command;
|
4
|
+
import com.github.kamatama41.nsocket.Connection;
|
5
|
+
|
6
|
+
class NotifyTaskStateCommand implements Command<UpdateTaskStateData> {
|
7
|
+
static final String ID = "notify_task_state";
|
8
|
+
private final SessionState sessionState;
|
9
|
+
|
10
|
+
NotifyTaskStateCommand(SessionState sessionState) {
|
11
|
+
this.sessionState = sessionState;
|
12
|
+
}
|
13
|
+
|
14
|
+
@Override
|
15
|
+
public void execute(UpdateTaskStateData data, Connection connection) throws Exception {
|
16
|
+
sessionState.update(data);
|
17
|
+
}
|
18
|
+
|
19
|
+
@Override
|
20
|
+
public String getId() {
|
21
|
+
return ID;
|
22
|
+
}
|
23
|
+
}
|
@@ -0,0 +1,170 @@
|
|
1
|
+
package org.embulk.executor.remoteserver;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
+
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
+
import com.google.common.collect.ImmutableList;
|
6
|
+
import com.google.common.io.ByteStreams;
|
7
|
+
import org.jruby.embed.ScriptingContainer;
|
8
|
+
|
9
|
+
import java.io.File;
|
10
|
+
import java.io.IOException;
|
11
|
+
import java.io.InputStream;
|
12
|
+
import java.io.OutputStream;
|
13
|
+
import java.nio.file.DirectoryStream;
|
14
|
+
import java.nio.file.Files;
|
15
|
+
import java.nio.file.NoSuchFileException;
|
16
|
+
import java.nio.file.NotDirectoryException;
|
17
|
+
import java.nio.file.Path;
|
18
|
+
import java.util.List;
|
19
|
+
import java.util.zip.ZipEntry;
|
20
|
+
import java.util.zip.ZipInputStream;
|
21
|
+
import java.util.zip.ZipOutputStream;
|
22
|
+
|
23
|
+
// Copied from embulk-executor-mapreduce
|
24
|
+
public class PluginArchive {
|
25
|
+
public static class GemSpec {
|
26
|
+
private final String name;
|
27
|
+
private final List<String> requirePaths;
|
28
|
+
|
29
|
+
@JsonCreator
|
30
|
+
public GemSpec(
|
31
|
+
@JsonProperty("name") String name,
|
32
|
+
@JsonProperty("requirePaths") List<String> requirePaths) {
|
33
|
+
this.name = name;
|
34
|
+
this.requirePaths = requirePaths;
|
35
|
+
}
|
36
|
+
|
37
|
+
@JsonProperty("name")
|
38
|
+
public String getName() {
|
39
|
+
return name;
|
40
|
+
}
|
41
|
+
|
42
|
+
@JsonProperty("requirePaths")
|
43
|
+
public List<String> getRequirePaths() {
|
44
|
+
return requirePaths;
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
private static class LocalGem
|
49
|
+
extends GemSpec {
|
50
|
+
private final File localPath;
|
51
|
+
|
52
|
+
public LocalGem(File localPath, String name, List<String> requirePaths) {
|
53
|
+
super(name, requirePaths);
|
54
|
+
this.localPath = localPath;
|
55
|
+
}
|
56
|
+
|
57
|
+
public File getLocalPath() {
|
58
|
+
return localPath;
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
public static class Builder {
|
63
|
+
private final ImmutableList.Builder<LocalGem> localGems = ImmutableList.builder();
|
64
|
+
|
65
|
+
@SuppressWarnings("unchecked")
|
66
|
+
public Builder addLoadedRubyGems(ScriptingContainer jruby) {
|
67
|
+
List<List<String>> tuples = (List<List<String>>) jruby.runScriptlet("Gem.loaded_specs.map {|k,v| [k, v.full_gem_path, v.require_paths].flatten }");
|
68
|
+
for (List<String> tuple : tuples) {
|
69
|
+
String name = tuple.remove(0);
|
70
|
+
String fullGemPath = tuple.remove(0);
|
71
|
+
List<String> requirePaths = ImmutableList.copyOf(tuple);
|
72
|
+
addSpec(new File(fullGemPath), name, requirePaths);
|
73
|
+
}
|
74
|
+
return this;
|
75
|
+
}
|
76
|
+
|
77
|
+
public Builder addSpec(File localPath, String name, List<String> requirePaths) {
|
78
|
+
localGems.add(new LocalGem(localPath, name, requirePaths));
|
79
|
+
return this;
|
80
|
+
}
|
81
|
+
|
82
|
+
public PluginArchive build() {
|
83
|
+
return new PluginArchive(localGems.build());
|
84
|
+
}
|
85
|
+
}
|
86
|
+
|
87
|
+
private final List<LocalGem> localGems;
|
88
|
+
|
89
|
+
private PluginArchive(List<LocalGem> localGems) {
|
90
|
+
this.localGems = localGems;
|
91
|
+
}
|
92
|
+
|
93
|
+
@SuppressWarnings("unchecked")
|
94
|
+
public void restoreLoadPathsTo(ScriptingContainer jruby) {
|
95
|
+
List<String> loadPaths = (List<String>) jruby.runScriptlet("$LOAD_PATH");
|
96
|
+
for (LocalGem localGem : localGems) {
|
97
|
+
Path localGemPath = localGem.getLocalPath().toPath();
|
98
|
+
for (String requirePath : localGem.getRequirePaths()) {
|
99
|
+
loadPaths.add(localGemPath.resolve(requirePath).toString());
|
100
|
+
}
|
101
|
+
}
|
102
|
+
jruby.setLoadPaths(loadPaths);
|
103
|
+
}
|
104
|
+
|
105
|
+
public List<GemSpec> dump(OutputStream out)
|
106
|
+
throws IOException {
|
107
|
+
ImmutableList.Builder<GemSpec> builder = ImmutableList.builder();
|
108
|
+
try (ZipOutputStream zip = new ZipOutputStream(out)) {
|
109
|
+
for (LocalGem localGem : localGems) {
|
110
|
+
zipDirectory(zip, localGem.getLocalPath().toPath(), localGem.getName() + "/");
|
111
|
+
builder.add(new GemSpec(localGem.getName(), localGem.getRequirePaths()));
|
112
|
+
}
|
113
|
+
}
|
114
|
+
return builder.build();
|
115
|
+
}
|
116
|
+
|
117
|
+
private static void zipDirectory(ZipOutputStream zip, Path directory, String name)
|
118
|
+
throws IOException {
|
119
|
+
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(directory)) {
|
120
|
+
for (Path path : dirStream) {
|
121
|
+
if (Files.isDirectory(path)) {
|
122
|
+
zipDirectory(zip, path, name + path.getFileName() + "/");
|
123
|
+
} else {
|
124
|
+
zip.putNextEntry(new ZipEntry(name + path.getFileName()));
|
125
|
+
try (InputStream in = Files.newInputStream(path)) {
|
126
|
+
ByteStreams.copy(in, zip);
|
127
|
+
}
|
128
|
+
zip.closeEntry();
|
129
|
+
}
|
130
|
+
}
|
131
|
+
} catch (NoSuchFileException | NotDirectoryException ex) {
|
132
|
+
// ignore
|
133
|
+
}
|
134
|
+
}
|
135
|
+
|
136
|
+
public static PluginArchive load(File localDirectory, List<GemSpec> gemSpecs,
|
137
|
+
InputStream in) throws IOException {
|
138
|
+
try (ZipInputStream zip = new ZipInputStream(in)) {
|
139
|
+
unzipDirectory(zip, localDirectory.toPath());
|
140
|
+
}
|
141
|
+
|
142
|
+
ImmutableList.Builder<LocalGem> builder = ImmutableList.builder();
|
143
|
+
for (GemSpec gemSpec : gemSpecs) {
|
144
|
+
builder.add(new LocalGem(
|
145
|
+
new File(localDirectory, gemSpec.getName()),
|
146
|
+
gemSpec.getName(),
|
147
|
+
gemSpec.getRequirePaths()));
|
148
|
+
}
|
149
|
+
return new PluginArchive(builder.build());
|
150
|
+
}
|
151
|
+
|
152
|
+
private static void unzipDirectory(ZipInputStream zip, Path directory)
|
153
|
+
throws IOException {
|
154
|
+
while (true) {
|
155
|
+
ZipEntry entry = zip.getNextEntry();
|
156
|
+
if (entry == null) {
|
157
|
+
break;
|
158
|
+
}
|
159
|
+
Path path = directory.resolve(entry.getName());
|
160
|
+
if (entry.getName().endsWith("/")) {
|
161
|
+
Files.createDirectories(path);
|
162
|
+
} else {
|
163
|
+
Files.createDirectories(path.getParent());
|
164
|
+
try (OutputStream out = Files.newOutputStream(path)) {
|
165
|
+
ByteStreams.copy(zip, out);
|
166
|
+
}
|
167
|
+
}
|
168
|
+
}
|
169
|
+
}
|
170
|
+
}
|