embulk-input-remote 0.1.10 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 05d25bac2bc3ef17ff27004d9fce3e6317b8be80
4
- data.tar.gz: 502556efd3275e388fc7ebb072dacc9bfa524231
3
+ metadata.gz: ea3bfc2c0371ae959be124623397149f674a3228
4
+ data.tar.gz: 7b80c2e4233a95201f1481991f7a536ba1bebf9c
5
5
  SHA512:
6
- metadata.gz: 8a99a040b1900a2458787b658699ea6b449b5c2d95de42b23554b44a152a88f2ef73efa2a3d53299e2a31e51d3147b187166905901c75e78c2787dfc767c920d
7
- data.tar.gz: f7203448e89983ca048cd4c85dbc8a503dbbda92bf49f793f5f46ce81c1d4291a3d2e86f0a693e84f515d7731a51a64fe94b57ff71c978c6bc271736251f8733
6
+ metadata.gz: c635b206f4fdfe0f6e794bada76a2f900ad47f7cd4cbc17f2b6797d6661d5d4f8c279a551c2a20115358b4aa5162683f618504d8a4d7bef30ee7b72a6591ec7c
7
+ data.tar.gz: b70b926b21825bc5563634e36a77c9155110fd48f56b5f18f28a84553eb101ac8fb73d4bdf6f279d6144577e2257299e1dc41364bf9ce3cc097acc8f5ddf5f81
@@ -6,15 +6,16 @@ jdk:
6
6
  - oraclejdk8
7
7
 
8
8
  env:
9
- - YAML_TEST01=$PWD/src/test/resources/yaml/test01.yml
9
+ - >-
10
+ KEY_PATH=$PWD/id_rsa_test
10
11
 
11
12
  services:
12
13
  - docker
13
14
 
14
15
  before_install:
15
- - docker build -t eg_sshd .
16
- - docker run -d -p 10022:22 -v $PWD/src/test/resources/input:/mount --name test_sshd eg_sshd
17
- - docker ps -a
16
+ - ssh-keygen -t ecdsa -f ${KEY_PATH} -N ''
17
+ - docker-compose up -d
18
+ - docker-compose ps
18
19
 
19
20
  script:
20
21
  - ./gradlew --info check
data/Dockerfile CHANGED
@@ -1,16 +1,17 @@
1
- # ref: https://docs.docker.com/engine/examples/running_ssh_service/
2
- FROM ubuntu:16.04
3
-
4
- RUN apt-get update && apt-get install -y openssh-server
5
- RUN mkdir /var/run/sshd
6
- RUN echo 'root:screencast' | chpasswd
7
- RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
8
-
9
- # SSH login fix. Otherwise user is kicked off after login
10
- RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
11
-
12
- ENV NOTVISIBLE "in users profile"
13
- RUN echo "export VISIBLE=now" >> /etc/profile
14
-
15
- EXPOSE 22
16
- CMD ["/usr/sbin/sshd", "-D"]
1
+ # Deprecated
2
+ ## ref: https://docs.docker.com/engine/examples/running_ssh_service/
3
+ #FROM ubuntu:16.04
4
+ #
5
+ #RUN apt-get update && apt-get install -y openssh-server
6
+ #RUN mkdir /var/run/sshd
7
+ #RUN echo 'root:screencast' | chpasswd
8
+ #RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
9
+ #
10
+ ## SSH login fix. Otherwise user is kicked off after login
11
+ #RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
12
+ #
13
+ #ENV NOTVISIBLE "in users profile"
14
+ #RUN echo "export VISIBLE=now" >> /etc/profile
15
+ #
16
+ #EXPOSE 22
17
+ #CMD ["/usr/sbin/sshd", "-D"]
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- [![Build Status](https://travis-ci.org/kamatama41/tfenv.svg?branch=master)](https://travis-ci.org/kamatama41/tfenv)
1
+ [![Build Status](https://travis-ci.org/kamatama41/embulk-input-remote.svg?branch=master)](https://travis-ci.org/kamatama41/embulk-input-remote)
2
2
 
3
3
  # Remote file input plugin for [Embulk](https://github.com/embulk/embulk)
4
4
 
@@ -12,10 +12,10 @@ This plugin load data from Remote hosts by SCP
12
12
 
13
13
  ## Configuration
14
14
 
15
- - **hosts**: Target hosts (list, default: [])
15
+ - **hosts**: Target hosts, format should be `host` or `host:port` (overrides `default_port`) (list, default: [])
16
16
  - **hosts_command**: Command for getting hosts(Windows not supported). If given the option, "hosts" is overwritten. (string, default: null)
17
17
  - **hosts_separator**: Separator for "hosts_command" result (string, default: " ")
18
- - **port**: Port number for SSH (integer, default: 22)
18
+ - **default_port**: Default port number for SSH (integer, default: 22)
19
19
  - **path**: Path of remote host (File or Directory) (string, default: "")
20
20
  - **path_command**: Command for getting path (Windows not supported). If given the option "path" is overwritten. (string, default: null)
21
21
  - **ignore_not_found_hosts**: If the option is true, Hosts which meet the following conditions are skipped. (Means they are not included into resume target.) (boolean, default: false)
@@ -35,7 +35,7 @@ in:
35
35
  type: remote
36
36
  hosts:
37
37
  - host1
38
- - host2
38
+ - host2:10022
39
39
  # hosts_command: echo 'host1,host2'
40
40
  # hosts_separator: ','
41
41
  path: /some/path/20150414125923
@@ -50,7 +50,7 @@ in:
50
50
  ```
51
51
 
52
52
  ## Note
53
- When this plugin run on Linux, task is sometimes blocked.
53
+ When this plugin run on Linux, a task might be blocked.
54
54
  The cause is java.security.SecureRandom. Please try one of the followings.
55
55
 
56
56
  ### set JVM_OPTION "-Djava.security.egd"
@@ -71,6 +71,18 @@ securerandom.source=file:/dev/./urandom # after
71
71
 
72
72
  http://stackoverflow.com/questions/137212/how-to-solve-performance-problem-with-java-securerandom
73
73
 
74
+ ## Development on local machine
75
+ - Install Docker and then we can create SSH-able containers
76
+ ```sh
77
+ $ ssh-keygen -t ecdsa -f ./id_rsa_test -N ''
78
+ $ docker-compose up -d
79
+ $ docker-compose ps
80
+ Name Command State Ports
81
+ --------------------------------------------------------------------------
82
+ embulkinputremote_host1_1 /entrypoint.sh Up 0.0.0.0:10022->22/tcp
83
+ embulkinputremote_host2_1 /entrypoint.sh Up 0.0.0.0:10023->22/tcp
84
+ ```
85
+
74
86
  ## Build
75
87
 
76
88
  ```
@@ -18,15 +18,16 @@ configurations {
18
18
  provided
19
19
  }
20
20
 
21
- version = "0.1.10"
21
+ version = "0.2.0"
22
22
 
23
23
  dependencies {
24
24
  compile "org.embulk:embulk-core:0.8.15"
25
- compile "com.hierynomus:sshj:0.15.0"
25
+ compile "com.hierynomus:sshj:0.19.1"
26
26
  compile "com.jcraft:jzlib:1.1.3"
27
27
  provided "org.embulk:embulk-core:0.8.15"
28
28
  testCompile "org.embulk:embulk-standards:0.8.15"
29
29
  testCompile "org.embulk:embulk-test:0.8.15"
30
+ testCompile "com.github.docker-java:docker-java:3.0.7"
30
31
  // for local testing
31
32
  // testCompile files('../embulk/embulk-test/build/libs/embulk-test-0.8.15.jar')
32
33
  // testCompile 'junit:junit:4.12'
@@ -0,0 +1,16 @@
1
+ version: '2'
2
+ services:
3
+ host1:
4
+ image: "sickp/alpine-sshd:7.2"
5
+ ports:
6
+ - "10022:22"
7
+ volumes:
8
+ - ./src/test/resources/input/host1/:/mount
9
+ - ./id_rsa_test.pub:/root/.ssh/authorized_keys
10
+ host2:
11
+ image: "sickp/alpine-sshd:7.2"
12
+ ports:
13
+ - "10023:22"
14
+ volumes:
15
+ - ./src/test/resources/input/host2/:/mount
16
+ - ./id_rsa_test.pub:/root/.ssh/authorized_keys
@@ -8,9 +8,8 @@ import java.io.InputStream;
8
8
  import java.io.InputStreamReader;
9
9
  import java.util.ArrayList;
10
10
  import java.util.Arrays;
11
- import java.util.Collections;
12
11
  import java.util.List;
13
- import java.util.Map;
12
+ import java.util.Objects;
14
13
 
15
14
  import com.fasterxml.jackson.annotation.JsonCreator;
16
15
  import com.fasterxml.jackson.annotation.JsonProperty;
@@ -32,8 +31,6 @@ import org.embulk.spi.TransactionalFileInput;
32
31
  import org.embulk.spi.util.InputStreamTransactionalFileInput;
33
32
  import org.slf4j.Logger;
34
33
 
35
- import javax.annotation.Nullable;
36
-
37
34
  public class RemoteFileInputPlugin
38
35
  implements FileInputPlugin {
39
36
  public interface PluginTask
@@ -50,9 +47,9 @@ public class RemoteFileInputPlugin
50
47
  @ConfigDefault("\" \"")
51
48
  String getHostsSeparator();
52
49
 
53
- @Config("port")
50
+ @Config("default_port")
54
51
  @ConfigDefault("22")
55
- int getPort();
52
+ int getDefaultPort();
56
53
 
57
54
  @Config("path")
58
55
  @ConfigDefault("\"\"")
@@ -63,18 +60,17 @@ public class RemoteFileInputPlugin
63
60
  Optional<String> getPathCommand();
64
61
 
65
62
  @Config("auth")
66
- @ConfigDefault("{}")
67
- Map<String, String> getAuth();
63
+ AuthConfig getAuthConfig();
68
64
 
69
65
  @Config("ignore_not_found_hosts")
70
66
  @ConfigDefault("false")
71
67
  boolean getIgnoreNotFoundHosts();
72
68
 
73
- @Config("last_target")
74
- @ConfigDefault("null")
75
- Optional<Target> getLastTarget();
69
+ @Config("done_targets")
70
+ @ConfigDefault("[]")
71
+ List<Target> getDoneTargets();
76
72
 
77
- void setLastTarget(Optional<Target> lastTarget);
73
+ void setDoneTargets(List<Target> lastTarget);
78
74
 
79
75
  List<Target> getTargets();
80
76
 
@@ -84,6 +80,28 @@ public class RemoteFileInputPlugin
84
80
  BufferAllocator getBufferAllocator();
85
81
  }
86
82
 
83
+ public interface AuthConfig extends Task {
84
+ @Config("type")
85
+ @ConfigDefault("\"public_key\"")
86
+ String getType();
87
+
88
+ @Config("user")
89
+ @ConfigDefault("null")
90
+ Optional<String> getUser();
91
+
92
+ @Config("key_path")
93
+ @ConfigDefault("null")
94
+ Optional<String> getKeyPath();
95
+
96
+ @Config("password")
97
+ @ConfigDefault("null")
98
+ Optional<String> getPassword();
99
+
100
+ @Config("skip_host_key_verification")
101
+ @ConfigDefault("false")
102
+ boolean getSkipHostKeyVerification();
103
+ }
104
+
87
105
  private final Logger log = Exec.getLogger(getClass());
88
106
 
89
107
  @Override
@@ -103,11 +121,17 @@ public class RemoteFileInputPlugin
103
121
  final String path = getPath(task);
104
122
 
105
123
  final ImmutableList.Builder<Target> builder = ImmutableList.builder();
106
- Target lastTarget = task.getLastTarget().orNull();
124
+ List<Target> doneTargets = task.getDoneTargets();
107
125
  for (String host : hosts) {
108
- Target target = new Target(host, path);
126
+ String[] split = host.split(":");
127
+ final String targetHost = split[0];
128
+ int targetPort = task.getDefaultPort();
129
+ if (split.length > 1) {
130
+ targetPort = Integer.valueOf(split[1]);
131
+ }
132
+ Target target = new Target(targetHost, targetPort, path);
109
133
 
110
- if (lastTarget == null || target.compareTo(lastTarget) > 0) {
134
+ if (!doneTargets.contains(target)) {
111
135
  if (task.getIgnoreNotFoundHosts()) {
112
136
  try {
113
137
  final boolean exists = exists(target, task);
@@ -180,13 +204,8 @@ public class RemoteFileInputPlugin
180
204
  control.run(taskSource, taskCount);
181
205
 
182
206
  List<Target> targets = new ArrayList<>(task.getTargets());
183
- Collections.sort(targets);
184
-
185
- if (targets.isEmpty()) {
186
- return Exec.newConfigDiff();
187
- }
188
207
 
189
- return Exec.newConfigDiff().set("last_target", targets.get(targets.size() - 1));
208
+ return Exec.newConfigDiff().set("done_targets", targets);
190
209
  }
191
210
 
192
211
  @Override
@@ -222,9 +241,7 @@ public class RemoteFileInputPlugin
222
241
  }
223
242
 
224
243
  private boolean exists(Target target, PluginTask task) throws IOException {
225
- try (SSHClient client = SSHClient.getInstance()) {
226
- client.connect(target.getHost(), task.getPort(), task.getAuth());
227
-
244
+ try (SSHClient client = SSHClient.connect(target.getHost(), target.getPort(), task.getAuthConfig())) {
228
245
  final String checkCmd = "ls " + target.getPath(); // TODO: windows
229
246
  final int timeout = 5/* second */;
230
247
  final SSHClient.CommandResult commandResult = client.execCommand(checkCmd, timeout);
@@ -239,23 +256,26 @@ public class RemoteFileInputPlugin
239
256
  }
240
257
 
241
258
  private InputStream download(Target target, PluginTask task) throws IOException {
242
- try (SSHClient client = SSHClient.getInstance()) {
243
- client.connect(target.getHost(), task.getPort(), task.getAuth());
259
+ try (SSHClient client = SSHClient.connect(target.getHost(), target.getPort(), task.getAuthConfig())) {
244
260
  final ByteArrayOutputStream stream = new ByteArrayOutputStream();
245
261
  client.scpDownload(target.getPath(), stream);
246
262
  return new ByteArrayInputStream(stream.toByteArray());
247
263
  }
248
264
  }
249
265
 
250
- public static class Target implements Comparable<Target> {
266
+ public static class Target {
251
267
  private final String host;
268
+ private final int port;
252
269
  private final String path;
253
270
 
254
271
  @JsonCreator
255
272
  public Target(
256
273
  @JsonProperty("host") String host,
257
- @JsonProperty("path") String path) {
274
+ @JsonProperty("port") int port,
275
+ @JsonProperty("path") String path
276
+ ) {
258
277
  this.host = host;
278
+ this.port = port;
259
279
  this.path = path;
260
280
  }
261
281
 
@@ -263,21 +283,32 @@ public class RemoteFileInputPlugin
263
283
  return host;
264
284
  }
265
285
 
286
+ public int getPort() {
287
+ return port;
288
+ }
289
+
266
290
  public String getPath() {
267
291
  return path;
268
292
  }
269
293
 
270
294
  @Override
271
- public int compareTo(@Nullable Target other) {
272
- if (other == null) {
273
- throw new NullPointerException();
274
- }
275
- return this.toString().compareTo(other.toString());
295
+ public boolean equals(Object o) {
296
+ if (this == o) return true;
297
+ if (o == null || getClass() != o.getClass()) return false;
298
+ Target target = (Target) o;
299
+ return port == target.port &&
300
+ Objects.equals(host, target.host) &&
301
+ Objects.equals(path, target.path);
302
+ }
303
+
304
+ @Override
305
+ public int hashCode() {
306
+ return Objects.hash(host, port, path);
276
307
  }
277
308
 
278
309
  @Override
279
310
  public String toString() {
280
- return host + ":" + path;
311
+ return host + ":" + port + ":" + path;
281
312
  }
282
313
  }
283
314
  }
@@ -1,63 +1,56 @@
1
1
  package org.embulk.input.remote;
2
2
 
3
- import com.hierynomus.sshj.signature.SignatureEdDSA;
4
3
  import net.schmizz.sshj.DefaultConfig;
5
4
  import net.schmizz.sshj.connection.channel.direct.Session;
6
- import net.schmizz.sshj.signature.SignatureDSA;
7
- import net.schmizz.sshj.signature.SignatureECDSA;
8
- import net.schmizz.sshj.signature.SignatureRSA;
9
5
  import net.schmizz.sshj.transport.verification.PromiscuousVerifier;
10
6
  import net.schmizz.sshj.xfer.InMemoryDestFile;
11
7
  import net.schmizz.sshj.xfer.LocalDestFile;
8
+ import org.embulk.input.RemoteFileInputPlugin;
12
9
 
13
10
  import java.io.Closeable;
14
11
  import java.io.IOException;
15
12
  import java.io.InputStream;
16
13
  import java.io.OutputStream;
17
- import java.util.Arrays;
18
- import java.util.Map;
19
14
  import java.util.concurrent.TimeUnit;
20
15
 
21
16
  public class SSHClient implements Closeable {
22
17
 
23
18
  private final net.schmizz.sshj.SSHClient client;
24
19
 
25
- public static SSHClient getInstance() {
26
- return new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig(){
27
- @Override
28
- protected void initSignatureFactories() {
29
- setSignatureFactories(Arrays.asList(
30
- new SignatureRSA.Factory(),
31
- new SignatureECDSA.Factory(),
32
- new SignatureDSA.Factory(),
33
- new SignatureEdDSA.Factory()
34
- ));
35
- }
36
- }));
20
+ public static SSHClient connect(
21
+ String host, int port, RemoteFileInputPlugin.AuthConfig authConfig
22
+ ) throws IOException {
23
+
24
+ SSHClient client = new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig()));
25
+ client.connectToHost(host, port, authConfig);
26
+ return client;
37
27
  }
38
28
 
39
29
  private SSHClient(net.schmizz.sshj.SSHClient client) {
40
30
  this.client = client;
41
31
  }
42
32
 
43
- public void connect(String host, int port, Map<String, String> authConfig) throws IOException {
44
- if (Boolean.valueOf(authConfig.get("skip_host_key_verification"))) {
33
+ private void connectToHost(String host, int port, RemoteFileInputPlugin.AuthConfig authConfig) throws IOException {
34
+ if (authConfig.getSkipHostKeyVerification()) {
45
35
  client.addHostKeyVerifier(new PromiscuousVerifier());
46
36
  }
47
37
  client.loadKnownHosts();
48
38
  client.connect(host, port);
49
39
 
50
- final String type = authConfig.get("type") != null ? authConfig.get("type") : "public_key";
51
- final String user = authConfig.get("user") != null ? authConfig.get("user") : System.getProperty("user.name");
40
+ final String type = authConfig.getType();
41
+ final String user = authConfig.getUser().or(System.getProperty("user.name"));
52
42
 
53
43
  if ("password".equals(type)) {
54
- client.authPassword(user, authConfig.get("password"));
44
+ if (authConfig.getPassword().isPresent()) {
45
+ client.authPassword(user, authConfig.getPassword().get());
46
+ } else {
47
+ throw new IllegalStateException("Password is not set.");
48
+ }
55
49
  } else if ("public_key".equals(type)) {
56
- final String key_path = authConfig.get("key_path");
57
- if (key_path == null) {
58
- client.authPublickey(user);
50
+ if (authConfig.getKeyPath().isPresent()) {
51
+ client.authPublickey(user, authConfig.getKeyPath().get());
59
52
  } else {
60
- client.authPublickey(user, key_path);
53
+ client.authPublickey(user);
61
54
  }
62
55
  } else {
63
56
  throw new UnsupportedOperationException("Unsupported auth type : " + type);
@@ -1,38 +1,255 @@
1
1
  package org.embulk.input;
2
2
 
3
+ import ch.qos.logback.classic.Level;
4
+ import ch.qos.logback.classic.Logger;
5
+ import com.github.dockerjava.api.DockerClient;
6
+ import com.github.dockerjava.api.model.Container;
7
+ import com.github.dockerjava.core.DockerClientBuilder;
8
+ import org.embulk.EmbulkEmbed;
3
9
  import org.embulk.config.ConfigSource;
4
10
  import org.embulk.spi.InputPlugin;
5
- import org.embulk.test.EmbulkTests;
11
+ import org.embulk.test.MemoryOutputPlugin;
12
+ import org.embulk.test.MyEmbulkTests;
13
+ import org.embulk.test.MyTestingEmbulk;
6
14
  import org.embulk.test.TestingEmbulk;
15
+ import org.junit.Before;
16
+ import org.junit.Ignore;
7
17
  import org.junit.Rule;
8
18
  import org.junit.Test;
19
+ import org.slf4j.LoggerFactory;
9
20
 
10
- import java.nio.file.Path;
21
+ import java.util.Arrays;
22
+ import java.util.Collections;
23
+ import java.util.HashSet;
24
+ import java.util.List;
25
+ import java.util.Set;
11
26
 
12
- import static org.embulk.test.EmbulkTests.readResource;
13
- import static org.embulk.test.EmbulkTests.readSortedFile;
14
27
  import static org.hamcrest.Matchers.is;
15
28
  import static org.junit.Assert.assertThat;
16
29
 
17
- public class TestRemoteFileInputPlugin
18
- {
30
+ public class TestRemoteFileInputPlugin {
31
+ private static final String CONTAINER_ID_HOST1 = "embulkinputremote_host1_1";
32
+ private static final String CONTAINER_ID_HOST2 = "embulkinputremote_host2_1";
33
+ private static final DockerClient dockerClient = DockerClientBuilder.getInstance().build();
34
+
19
35
  @Rule
20
- public TestingEmbulk embulk = TestingEmbulk
36
+ public MyTestingEmbulk embulk = (MyTestingEmbulk) MyTestingEmbulk
21
37
  .builder()
22
38
  .registerPlugin(InputPlugin.class, "remote", RemoteFileInputPlugin.class)
23
39
  .build();
24
40
 
41
+ @Before
42
+ public void prepare() {
43
+ startContainer(CONTAINER_ID_HOST1);
44
+ startContainer(CONTAINER_ID_HOST2);
45
+
46
+ String logLevel = System.getenv("LOG_LEVEL");
47
+ if (logLevel != null) {
48
+ // Set log level
49
+ Logger rootLogger = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
50
+ rootLogger.setLevel(Level.toLevel(logLevel));
51
+ }
52
+ }
53
+
25
54
  @Test
26
55
  public void loadFromRemote() throws Exception
27
56
  {
28
- ConfigSource config = EmbulkTests.config("YAML_TEST01");
29
- Path out = embulk.createTempFile("csv");
57
+ embulk.runInput(baseConfig());
58
+ assertValues(values(1L, "user1"));
59
+ }
60
+
61
+ @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
62
+ @Test
63
+ public void loadFromRemoteViaPublicKey() throws Exception
64
+ {
65
+ String keyPath = System.getenv("KEY_PATH");
66
+ if (keyPath == null) {
67
+ keyPath = "./id_rsa_test";
68
+ }
69
+
70
+ final ConfigSource publicKeyAuth = newConfig().set("auth", newConfig()
71
+ .set("type", "public_key")
72
+ .set("key_path", keyPath)
73
+ );
74
+ embulk.runInput(baseConfig().merge(publicKeyAuth));
75
+
76
+ assertValues(values(1L, "user1"));
77
+ }
78
+
79
+ @Test
80
+ public void testMultiHosts() throws Exception
81
+ {
82
+ final ConfigSource multiHosts = newConfig()
83
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
84
+ final ConfigSource config = baseConfig().merge(multiHosts);
85
+
86
+ // Run
87
+ embulk.runInput(config);
88
+ assertValues(
89
+ values(1L, "user1"),
90
+ values(2L, "user2")
91
+ );
92
+ }
93
+
94
+ @Test
95
+ public void loadAllFilesInDirectory() throws Exception
96
+ {
97
+ final ConfigSource multiHosts = newConfig()
98
+ .set("path", "/mount");
99
+ final ConfigSource config = baseConfig().merge(multiHosts);
100
+
101
+ embulk.runInput(config);
102
+ assertValues(
103
+ values(1L, "user1"),
104
+ values(1L, "command_user1")
105
+ );
106
+ }
107
+
108
+ @Test
109
+ public void testDefaultPort() throws Exception
110
+ {
111
+ final ConfigSource defaultPort = newConfig()
112
+ .set("hosts", Collections.singletonList("localhost"))
113
+ .set("default_port", 10022);
114
+
115
+ embulk.runInput(baseConfig().merge(defaultPort));
116
+
117
+ assertValues(values(1L, "user1"));
118
+ }
119
+
120
+ @Test
121
+ public void testConfDiff() throws Exception
122
+ {
123
+ final ConfigSource host2Config = newConfig()
124
+ .set("hosts", Collections.singletonList("localhost:10023"));
125
+ ConfigSource config = baseConfig().merge(host2Config);
126
+
127
+ // Run
128
+ TestingEmbulk.RunResult runResult = embulk.runInput(config);
129
+ assertValues(values(2L, "user2"));
130
+
131
+ // Re-run with additional host1
132
+ final ConfigSource multiHost = newConfig()
133
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
134
+ config = baseConfig().merge(multiHost);
30
135
 
31
- embulk.runInput(config, out);
136
+ embulk.runInput(config, runResult.getConfigDiff());
32
137
 
33
- assertThat(
34
- readSortedFile(out),
35
- is(readResource("expect/test01.csv")));
138
+ assertValues(values(1L, "user1"));
36
139
  }
37
140
 
141
+ @Test
142
+ public void testResume() throws Exception
143
+ {
144
+ final ConfigSource multiHost = newConfig()
145
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
146
+ final ConfigSource config = baseConfig().merge(multiHost);
147
+
148
+ // Stop host2 temporarily
149
+ stopContainer(CONTAINER_ID_HOST2);
150
+
151
+ // Run (but will fail)
152
+ EmbulkEmbed.ResumableResult resumableResult = embulk.resume(config);
153
+
154
+ assertThat(resumableResult.isSuccessful(), is(false));
155
+ assertValues(values(1L, "user1"));
156
+
157
+ // Start host2 again
158
+ startContainer(CONTAINER_ID_HOST2);
159
+
160
+ // Resume
161
+ resumableResult = embulk.resume(config, resumableResult.getResumeState());
162
+
163
+ assertThat(resumableResult.isSuccessful(), is(true));
164
+ assertValues(values(2L, "user2"));
165
+ }
166
+
167
+ @Test
168
+ public void testIgnoreNotFoundHosts() throws Exception
169
+ {
170
+ final ConfigSource ignoreNotFoundHosts = newConfig()
171
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
172
+ .set("ignore_not_found_hosts", true);
173
+ final ConfigSource config = baseConfig().merge(ignoreNotFoundHosts);
174
+
175
+ // Stop host2
176
+ stopContainer(CONTAINER_ID_HOST2);
177
+
178
+ // Run (host2 will be ignored)
179
+ EmbulkEmbed.ResumableResult resumableResult = embulk.resume(config);
180
+
181
+ assertThat(resumableResult.isSuccessful(), is(true));
182
+ assertValues(values(1L, "user1"));
183
+ }
184
+
185
+ @Test
186
+ public void testCommandOptions() throws Exception
187
+ {
188
+ final ConfigSource ignoreNotFoundHosts = newConfig()
189
+ .set("hosts_command", "echo localhost:10022,localhost:10023")
190
+ .set("hosts_separator", ",")
191
+ .set("path_command", "echo /mount/test_command.csv");
192
+ final ConfigSource config = baseConfig().merge(ignoreNotFoundHosts);
193
+
194
+ embulk.runInput(config);
195
+
196
+ assertValues(
197
+ values(1L, "command_user1"),
198
+ values(2L, "command_user2")
199
+ );
200
+ }
201
+
202
+ private ConfigSource baseConfig() {
203
+ return MyEmbulkTests.configFromResource("yaml/base.yml");
204
+ }
205
+
206
+ private ConfigSource newConfig() {
207
+ return embulk.newConfig();
208
+ }
209
+
210
+ private void assertValues(List... valuesList) {
211
+ Set<List> actual = new HashSet<>();
212
+ for (MemoryOutputPlugin.Record record : MemoryOutputPlugin.getRecords()) {
213
+ actual.add(record.getValues());
214
+ }
215
+
216
+ Set<List> expected = new HashSet<>();
217
+ Collections.addAll(expected, valuesList);
218
+
219
+ assertThat(actual, is(expected));
220
+ }
221
+
222
+ private List values(Object... values) {
223
+ return Arrays.asList(values);
224
+ }
225
+
226
+ //////////////////////////////
227
+ // Methods for Docker
228
+ //////////////////////////////
229
+
230
+ private static void stopContainer(String containerId) {
231
+ if (isRunning(containerId)) {
232
+ dockerClient.stopContainerCmd(containerId).exec();
233
+ }
234
+ }
235
+
236
+ private static void startContainer(String containerId) {
237
+ if (!isRunning(containerId)) {
238
+ dockerClient.startContainerCmd(containerId).exec();
239
+ }
240
+ }
241
+
242
+ private static boolean isRunning(String containerId) {
243
+ List<Container> containers = dockerClient.listContainersCmd().exec();
244
+ for (Container container : containers) {
245
+ for (String name : container.getNames()) {
246
+ if (name.contains(containerId)) {
247
+ System.out.println("Found " + containerId);
248
+ return true;
249
+ }
250
+ }
251
+ }
252
+ System.out.println("Not Found " + containerId);
253
+ return false;
254
+ }
38
255
  }
@@ -0,0 +1,143 @@
1
+ package org.embulk.test;
2
+
3
+ import com.google.common.collect.ImmutableList;
4
+ import org.embulk.config.ConfigDiff;
5
+ import org.embulk.config.ConfigSource;
6
+ import org.embulk.config.Task;
7
+ import org.embulk.config.TaskReport;
8
+ import org.embulk.config.TaskSource;
9
+ import org.embulk.spi.Column;
10
+ import org.embulk.spi.Exec;
11
+ import org.embulk.spi.OutputPlugin;
12
+ import org.embulk.spi.Page;
13
+ import org.embulk.spi.PageReader;
14
+ import org.embulk.spi.Schema;
15
+ import org.embulk.spi.TransactionalPageOutput;
16
+ import org.embulk.spi.util.Pages;
17
+
18
+ import java.util.ArrayList;
19
+ import java.util.List;
20
+
21
+ public class MemoryOutputPlugin implements OutputPlugin
22
+ {
23
+ public interface PluginTask extends Task { }
24
+
25
+ @Override
26
+ public ConfigDiff transaction(ConfigSource config,
27
+ Schema schema, int taskCount,
28
+ OutputPlugin.Control control)
29
+ {
30
+ final PluginTask task = config.loadConfig(PluginTask.class);
31
+ return resume(task.dump(), schema, taskCount, control);
32
+ }
33
+
34
+ @Override
35
+ public ConfigDiff resume(TaskSource taskSource,
36
+ Schema schema, int taskCount,
37
+ OutputPlugin.Control control)
38
+ {
39
+ control.run(taskSource);
40
+ return Exec.newConfigDiff();
41
+ }
42
+
43
+ @Override
44
+ public void cleanup(TaskSource taskSource,
45
+ Schema schema, int taskCount,
46
+ List<TaskReport> successTaskReports)
47
+ { }
48
+
49
+ @Override
50
+ public TransactionalPageOutput open(final TaskSource taskSource, final Schema schema, final int taskIndex)
51
+ {
52
+ return new TransactionalPageOutput()
53
+ {
54
+ private final PageReader reader = new PageReader(schema);
55
+
56
+ public void add(Page page)
57
+ {
58
+ reader.setPage(page);
59
+ while (reader.nextRecord())
60
+ {
61
+ Recorder.addRecord(reader);
62
+ }
63
+ }
64
+
65
+ public void finish() { }
66
+
67
+ public void close()
68
+ {
69
+ reader.close();
70
+ }
71
+
72
+ public void abort() { }
73
+
74
+ public TaskReport commit()
75
+ {
76
+ return Exec.newTaskReport();
77
+ }
78
+ };
79
+ }
80
+
81
+ public static void clearRecords()
82
+ {
83
+ Recorder.clearRecords();
84
+ }
85
+
86
+ public static List<Record> getRecords()
87
+ {
88
+ return Recorder.getRecords();
89
+ }
90
+
91
+ private static class Recorder
92
+ {
93
+ private static List<Record> records = new ArrayList<>();
94
+
95
+ private Recorder() { }
96
+
97
+ private synchronized static void addRecord(PageReader reader)
98
+ {
99
+ final ImmutableList.Builder<Object> values = ImmutableList.builder();
100
+ final ImmutableList.Builder<Column> columns = ImmutableList.builder();
101
+ reader.getSchema().visitColumns(new Pages.ObjectColumnVisitor(reader) {
102
+ @Override
103
+ public void visit(org.embulk.spi.Column column, Object value) {
104
+ values.add(value);
105
+ columns.add(column);
106
+ }
107
+ });
108
+ records.add(new Record(values.build(), columns.build()));
109
+ }
110
+
111
+ static void clearRecords()
112
+ {
113
+ records = new ArrayList<>();
114
+ }
115
+
116
+ static List<Record> getRecords()
117
+ {
118
+ return new ArrayList<>(records);
119
+ }
120
+ }
121
+
122
+ public static class Record
123
+ {
124
+ private final List<Object> values;
125
+ private final List<Column> columns;
126
+
127
+ Record(List<Object> values, List<Column> columns)
128
+ {
129
+ this.values = values;
130
+ this.columns = columns;
131
+ }
132
+
133
+ public List<Object> getValues()
134
+ {
135
+ return values;
136
+ }
137
+
138
+ public List<Column> getColumns()
139
+ {
140
+ return columns;
141
+ }
142
+ }
143
+ }
@@ -0,0 +1,23 @@
1
+ package org.embulk.test;
2
+
3
+ import org.embulk.EmbulkEmbed;
4
+ import org.embulk.config.ConfigSource;
5
+
6
+ import static com.google.common.base.Strings.isNullOrEmpty;
7
+ import static org.embulk.test.EmbulkTests.readResource;
8
+ import static org.hamcrest.Matchers.is;
9
+ import static org.junit.Assume.assumeThat;
10
+
11
+ public class MyEmbulkTests {
12
+ private MyEmbulkTests() {
13
+ }
14
+
15
+ public static ConfigSource configFromString(String yaml) {
16
+ assumeThat(isNullOrEmpty(yaml), is(false));
17
+ return EmbulkEmbed.newSystemConfigLoader().fromYamlString(yaml);
18
+ }
19
+
20
+ public static ConfigSource configFromResource(String name) {
21
+ return configFromString(readResource(name));
22
+ }
23
+ }
@@ -0,0 +1,92 @@
1
+ package org.embulk.test;
2
+
3
+ import org.embulk.EmbulkEmbed;
4
+ import org.embulk.config.ConfigDiff;
5
+ import org.embulk.config.ConfigSource;
6
+ import org.embulk.exec.ResumeState;
7
+ import org.embulk.spi.OutputPlugin;
8
+
9
+ import java.io.IOException;
10
+ import java.lang.reflect.Field;
11
+
12
+ public class MyTestingEmbulk extends TestingEmbulk {
13
+
14
+ public static class Builder extends TestingEmbulk.Builder {
15
+ public TestingEmbulk build() {
16
+ this.registerPlugin(OutputPlugin.class, "memory", MemoryOutputPlugin.class);
17
+ return new MyTestingEmbulk(this);
18
+ }
19
+ }
20
+
21
+ public static TestingEmbulk.Builder builder()
22
+ {
23
+ return new MyTestingEmbulk.Builder();
24
+ }
25
+
26
+ private final EmbulkEmbed superEmbed;
27
+
28
+ MyTestingEmbulk(Builder builder) {
29
+ super(builder);
30
+ this.superEmbed = extractSuperField("embed");
31
+ }
32
+
33
+ public RunResult runInput(ConfigSource inConfig) throws IOException {
34
+ return runInput(inConfig, (ConfigDiff) null);
35
+ }
36
+
37
+ public RunResult runInput(ConfigSource inConfig, ConfigDiff confDiff) throws IOException {
38
+ ConfigSource execConfig = newConfig()
39
+ .set("min_output_tasks", 1);
40
+
41
+ ConfigSource outConfig = newConfig()
42
+ .set("type", "memory");
43
+
44
+ ConfigSource config = newConfig()
45
+ .set("exec", execConfig)
46
+ .set("in", inConfig)
47
+ .set("out", outConfig);
48
+
49
+ // embed.run returns TestingBulkLoader.TestingExecutionResult because
50
+ MemoryOutputPlugin.clearRecords();
51
+ if (confDiff == null) {
52
+ return (RunResult) superEmbed.run(config);
53
+ } else {
54
+ return (RunResult) superEmbed.run(config.merge(confDiff));
55
+ }
56
+ }
57
+
58
+ public EmbulkEmbed.ResumableResult resume(ConfigSource inConfig) throws IOException {
59
+ return resume(inConfig, null);
60
+ }
61
+
62
+ public EmbulkEmbed.ResumableResult resume(ConfigSource inConfig, ResumeState resumeState) throws IOException {
63
+ ConfigSource execConfig = newConfig()
64
+ .set("min_output_tasks", 1);
65
+
66
+ ConfigSource outConfig = newConfig()
67
+ .set("type", "memory");
68
+
69
+ ConfigSource config = newConfig()
70
+ .set("exec", execConfig)
71
+ .set("in", inConfig)
72
+ .set("out", outConfig);
73
+
74
+ MemoryOutputPlugin.clearRecords();
75
+ if (resumeState == null) {
76
+ return superEmbed.runResumable(config);
77
+ } else {
78
+ return superEmbed.new ResumeStateAction(config, resumeState).resume();
79
+ }
80
+ }
81
+
82
+ @SuppressWarnings("unchecked")
83
+ private <T> T extractSuperField(String fieldName) {
84
+ try {
85
+ Field field = TestingEmbulk.class.getDeclaredField(fieldName);
86
+ field.setAccessible(true);
87
+ return (T) field.get(this);
88
+ } catch (NoSuchFieldException | IllegalAccessException e) {
89
+ throw new RuntimeException(e);
90
+ }
91
+ }
92
+ }
@@ -0,0 +1 @@
1
+ 1,command_user1
@@ -0,0 +1 @@
1
+ 2,command_user2
@@ -1,12 +1,11 @@
1
1
  type: remote
2
2
  hosts:
3
- - localhost
4
- port: 10022
5
- path: /mount/test01.csv
3
+ - localhost:10022
4
+ path: /mount/test.csv
6
5
  auth:
7
6
  user: root
8
7
  type: password
9
- password: screencast
8
+ password: root
10
9
  skip_host_key_verification: true
11
10
  parser:
12
11
  type: csv
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-remote
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.10
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shinichi Ishimura
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-25 00:00:00.000000000 Z
11
+ date: 2017-01-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -51,6 +51,7 @@ files:
51
51
  - LICENSE.txt
52
52
  - README.md
53
53
  - build.gradle
54
+ - docker-compose.yml
54
55
  - example/csv/sample_01.csv.gz
55
56
  - example/example.yml.liquid
56
57
  - gradle/wrapper/gradle-wrapper.jar
@@ -61,15 +62,20 @@ files:
61
62
  - src/main/java/org/embulk/input/RemoteFileInputPlugin.java
62
63
  - src/main/java/org/embulk/input/remote/SSHClient.java
63
64
  - src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java
64
- - src/test/resources/expect/test01.csv
65
- - src/test/resources/input/test01.csv
66
- - src/test/resources/yaml/test01.yml
65
+ - src/test/java/org/embulk/test/MemoryOutputPlugin.java
66
+ - src/test/java/org/embulk/test/MyEmbulkTests.java
67
+ - src/test/java/org/embulk/test/MyTestingEmbulk.java
68
+ - src/test/resources/input/host1/test.csv
69
+ - src/test/resources/input/host1/test_command.csv
70
+ - src/test/resources/input/host2/test.csv
71
+ - src/test/resources/input/host2/test_command.csv
72
+ - src/test/resources/yaml/base.yml
67
73
  - classpath/bcpkix-jdk15on-1.51.jar
68
74
  - classpath/bcprov-jdk15on-1.51.jar
69
- - classpath/ecc-25519-java-1.0.1.jar
70
- - classpath/embulk-input-remote-0.1.10.jar
75
+ - classpath/eddsa-0.1.0.jar
76
+ - classpath/embulk-input-remote-0.2.0.jar
71
77
  - classpath/jzlib-1.1.3.jar
72
- - classpath/sshj-0.15.0.jar
78
+ - classpath/sshj-0.19.1.jar
73
79
  homepage: https://github.com/kamatama41/embulk-input-remote
74
80
  licenses:
75
81
  - MIT
@@ -1,2 +0,0 @@
1
- 1,kamatama41
2
- 2,kamatama42
@@ -1,2 +0,0 @@
1
- 1,kamatama41
2
- 2,kamatama42