embulk-input-remote 0.2.0 → 0.3.0

data/src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java

@@ -1,255 +0,0 @@
-package org.embulk.input;
-
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.Logger;
-import com.github.dockerjava.api.DockerClient;
-import com.github.dockerjava.api.model.Container;
-import com.github.dockerjava.core.DockerClientBuilder;
-import org.embulk.EmbulkEmbed;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.InputPlugin;
-import org.embulk.test.MemoryOutputPlugin;
-import org.embulk.test.MyEmbulkTests;
-import org.embulk.test.MyTestingEmbulk;
-import org.embulk.test.TestingEmbulk;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-public class TestRemoteFileInputPlugin {
-    private static final String CONTAINER_ID_HOST1 = "embulkinputremote_host1_1";
-    private static final String CONTAINER_ID_HOST2 = "embulkinputremote_host2_1";
-    private static final DockerClient dockerClient = DockerClientBuilder.getInstance().build();
-
-    @Rule
-    public MyTestingEmbulk embulk = (MyTestingEmbulk) MyTestingEmbulk
-            .builder()
-            .registerPlugin(InputPlugin.class, "remote", RemoteFileInputPlugin.class)
-            .build();
-
-    @Before
-    public void prepare() {
-        startContainer(CONTAINER_ID_HOST1);
-        startContainer(CONTAINER_ID_HOST2);
-
-        String logLevel = System.getenv("LOG_LEVEL");
-        if (logLevel != null) {
-            // Set log level
-            Logger rootLogger = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
-            rootLogger.setLevel(Level.toLevel(logLevel));
-        }
-    }
-
-    @Test
-    public void loadFromRemote() throws Exception
-    {
-        embulk.runInput(baseConfig());
-        assertValues(values(1L, "user1"));
-    }
-
-    @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
-    @Test
-    public void loadFromRemoteViaPublicKey() throws Exception
-    {
-        String keyPath = System.getenv("KEY_PATH");
-        if (keyPath == null) {
-            keyPath = "./id_rsa_test";
-        }
-
-        final ConfigSource publicKeyAuth = newConfig().set("auth", newConfig()
-                .set("type", "public_key")
-                .set("key_path", keyPath)
-        );
-        embulk.runInput(baseConfig().merge(publicKeyAuth));
-
-        assertValues(values(1L, "user1"));
-    }
-
-    @Test
-    public void testMultiHosts() throws Exception
-    {
-        final ConfigSource multiHosts = newConfig()
-                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
-        final ConfigSource config = baseConfig().merge(multiHosts);
-
-        // Run
-        embulk.runInput(config);
-        assertValues(
-                values(1L, "user1"),
-                values(2L, "user2")
-        );
-    }
-
-    @Test
-    public void loadAllFilesInDirectory() throws Exception
-    {
-        final ConfigSource multiHosts = newConfig()
-                .set("path", "/mount");
-        final ConfigSource config = baseConfig().merge(multiHosts);
-
-        embulk.runInput(config);
-        assertValues(
-                values(1L, "user1"),
-                values(1L, "command_user1")
-        );
-    }
-
-    @Test
-    public void testDefaultPort() throws Exception
-    {
-        final ConfigSource defaultPort = newConfig()
-                .set("hosts", Collections.singletonList("localhost"))
-                .set("default_port", 10022);
-
-        embulk.runInput(baseConfig().merge(defaultPort));
-
-        assertValues(values(1L, "user1"));
-    }
-
-    @Test
-    public void testConfDiff() throws Exception
-    {
-        final ConfigSource host2Config = newConfig()
-                .set("hosts", Collections.singletonList("localhost:10023"));
-        ConfigSource config = baseConfig().merge(host2Config);
-
-        // Run
-        TestingEmbulk.RunResult runResult = embulk.runInput(config);
-        assertValues(values(2L, "user2"));
-
-        // Re-run with additional host1
-        final ConfigSource multiHost = newConfig()
-                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
-        config = baseConfig().merge(multiHost);
-
-        embulk.runInput(config, runResult.getConfigDiff());
-
-        assertValues(values(1L, "user1"));
-    }
-
-    @Test
-    public void testResume() throws Exception
-    {
-        final ConfigSource multiHost = newConfig()
-                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
-        final ConfigSource config = baseConfig().merge(multiHost);
-
-        // Stop host2 temporarily
-        stopContainer(CONTAINER_ID_HOST2);
-
-        // Run (but will fail)
-        EmbulkEmbed.ResumableResult resumableResult = embulk.resume(config);
-
-        assertThat(resumableResult.isSuccessful(), is(false));
-        assertValues(values(1L, "user1"));
-
-        // Start host2 again
-        startContainer(CONTAINER_ID_HOST2);
-
-        // Resume
-        resumableResult = embulk.resume(config, resumableResult.getResumeState());
-
-        assertThat(resumableResult.isSuccessful(), is(true));
-        assertValues(values(2L, "user2"));
-    }
-
-    @Test
-    public void testIgnoreNotFoundHosts() throws Exception
-    {
-        final ConfigSource ignoreNotFoundHosts = newConfig()
-                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
-                .set("ignore_not_found_hosts", true);
-        final ConfigSource config = baseConfig().merge(ignoreNotFoundHosts);
-
-        // Stop host2
-        stopContainer(CONTAINER_ID_HOST2);
-
-        // Run (host2 will be ignored)
-        EmbulkEmbed.ResumableResult resumableResult = embulk.resume(config);
-
-        assertThat(resumableResult.isSuccessful(), is(true));
-        assertValues(values(1L, "user1"));
-    }
-
-    @Test
-    public void testCommandOptions() throws Exception
-    {
-        final ConfigSource ignoreNotFoundHosts = newConfig()
-                .set("hosts_command", "echo localhost:10022,localhost:10023")
-                .set("hosts_separator", ",")
-                .set("path_command", "echo /mount/test_command.csv");
-        final ConfigSource config = baseConfig().merge(ignoreNotFoundHosts);
-
-        embulk.runInput(config);
-
-        assertValues(
-                values(1L, "command_user1"),
-                values(2L, "command_user2")
-        );
-    }
-
-    private ConfigSource baseConfig() {
-        return MyEmbulkTests.configFromResource("yaml/base.yml");
-    }
-
-    private ConfigSource newConfig() {
-        return embulk.newConfig();
-    }
-
-    private void assertValues(List... valuesList) {
-        Set<List> actual = new HashSet<>();
-        for (MemoryOutputPlugin.Record record : MemoryOutputPlugin.getRecords()) {
-            actual.add(record.getValues());
-        }
-
-        Set<List> expected = new HashSet<>();
-        Collections.addAll(expected, valuesList);
-
-        assertThat(actual, is(expected));
-    }
-
-    private List values(Object... values) {
-        return Arrays.asList(values);
-    }
-
-    //////////////////////////////
-    // Methods for Docker
-    //////////////////////////////
-
-    private static void stopContainer(String containerId) {
-        if (isRunning(containerId)) {
-            dockerClient.stopContainerCmd(containerId).exec();
-        }
-    }
-
-    private static void startContainer(String containerId) {
-        if (!isRunning(containerId)) {
-            dockerClient.startContainerCmd(containerId).exec();
-        }
-    }
-
-    private static boolean isRunning(String containerId) {
-        List<Container> containers = dockerClient.listContainersCmd().exec();
-        for (Container container : containers) {
-            for (String name : container.getNames()) {
-                if (name.contains(containerId)) {
-                    System.out.println("Found " + containerId);
-                    return true;
-                }
-            }
-        }
-        System.out.println("Not Found " + containerId);
-        return false;
-    }
-}

data/src/test/java/org/embulk/test/MemoryOutputPlugin.java

@@ -1,143 +0,0 @@
-package org.embulk.test;
-
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.Column;
-import org.embulk.spi.Exec;
-import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.Page;
-import org.embulk.spi.PageReader;
-import org.embulk.spi.Schema;
-import org.embulk.spi.TransactionalPageOutput;
-import org.embulk.spi.util.Pages;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class MemoryOutputPlugin implements OutputPlugin
-{
-    public interface PluginTask extends Task { }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-                                  Schema schema, int taskCount,
-                                  OutputPlugin.Control control)
-    {
-        final PluginTask task = config.loadConfig(PluginTask.class);
-        return resume(task.dump(), schema, taskCount, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-                             Schema schema, int taskCount,
-                             OutputPlugin.Control control)
-    {
-        control.run(taskSource);
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-                        Schema schema, int taskCount,
-                        List<TaskReport> successTaskReports)
-    { }
-
-    @Override
-    public TransactionalPageOutput open(final TaskSource taskSource, final Schema schema, final int taskIndex)
-    {
-        return new TransactionalPageOutput()
-        {
-            private final PageReader reader = new PageReader(schema);
-
-            public void add(Page page)
-            {
-                reader.setPage(page);
-                while (reader.nextRecord())
-                {
-                    Recorder.addRecord(reader);
-                }
-            }
-
-            public void finish() { }
-
-            public void close()
-            {
-                reader.close();
-            }
-
-            public void abort() { }
-
-            public TaskReport commit()
-            {
-                return Exec.newTaskReport();
-            }
-        };
-    }
-
-    public static void clearRecords()
-    {
-        Recorder.clearRecords();
-    }
-
-    public static List<Record> getRecords()
-    {
-        return Recorder.getRecords();
-    }
-
-    private static class Recorder
-    {
-        private static List<Record> records = new ArrayList<>();
-
-        private Recorder() { }
-
-        private synchronized static void addRecord(PageReader reader)
-        {
-            final ImmutableList.Builder<Object> values = ImmutableList.builder();
-            final ImmutableList.Builder<Column> columns = ImmutableList.builder();
-            reader.getSchema().visitColumns(new Pages.ObjectColumnVisitor(reader) {
-                @Override
-                public void visit(org.embulk.spi.Column column, Object value) {
-                    values.add(value);
-                    columns.add(column);
-                }
-            });
-            records.add(new Record(values.build(), columns.build()));
-        }
-
-        static void clearRecords()
-        {
-            records = new ArrayList<>();
-        }
-
-        static List<Record> getRecords()
-        {
-            return new ArrayList<>(records);
-        }
-    }
-
-    public static class Record
-    {
-        private final List<Object> values;
-        private final List<Column> columns;
-
-        Record(List<Object> values, List<Column> columns)
-        {
-            this.values = values;
-            this.columns = columns;
-        }
-
-        public List<Object> getValues()
-        {
-            return values;
-        }
-
-        public List<Column> getColumns()
-        {
-            return columns;
-        }
-    }
-}

data/src/test/java/org/embulk/test/MyEmbulkTests.java

@@ -1,23 +0,0 @@
-package org.embulk.test;
-
-import org.embulk.EmbulkEmbed;
-import org.embulk.config.ConfigSource;
-
-import static com.google.common.base.Strings.isNullOrEmpty;
-import static org.embulk.test.EmbulkTests.readResource;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assume.assumeThat;
-
-public class MyEmbulkTests {
-    private MyEmbulkTests() {
-    }
-
-    public static ConfigSource configFromString(String yaml) {
-        assumeThat(isNullOrEmpty(yaml), is(false));
-        return EmbulkEmbed.newSystemConfigLoader().fromYamlString(yaml);
-    }
-
-    public static ConfigSource configFromResource(String name) {
-        return configFromString(readResource(name));
-    }
-}

data/src/test/java/org/embulk/test/MyTestingEmbulk.java

@@ -1,92 +0,0 @@
-package org.embulk.test;
-
-import org.embulk.EmbulkEmbed;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.exec.ResumeState;
-import org.embulk.spi.OutputPlugin;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-
-public class MyTestingEmbulk extends TestingEmbulk {
-
-    public static class Builder extends TestingEmbulk.Builder {
-        public TestingEmbulk build() {
-            this.registerPlugin(OutputPlugin.class, "memory", MemoryOutputPlugin.class);
-            return new MyTestingEmbulk(this);
-        }
-    }
-
-    public static TestingEmbulk.Builder builder()
-    {
-        return new MyTestingEmbulk.Builder();
-    }
-
-    private final EmbulkEmbed superEmbed;
-
-    MyTestingEmbulk(Builder builder) {
-        super(builder);
-        this.superEmbed = extractSuperField("embed");
-    }
-
-    public RunResult runInput(ConfigSource inConfig) throws IOException {
-        return runInput(inConfig, (ConfigDiff) null);
-    }
-
-    public RunResult runInput(ConfigSource inConfig, ConfigDiff confDiff) throws IOException {
-        ConfigSource execConfig = newConfig()
-                .set("min_output_tasks", 1);
-
-        ConfigSource outConfig = newConfig()
-                .set("type", "memory");
-
-        ConfigSource config = newConfig()
-                .set("exec", execConfig)
-                .set("in", inConfig)
-                .set("out", outConfig);
-
-        // embed.run returns TestingBulkLoader.TestingExecutionResult because
-        MemoryOutputPlugin.clearRecords();
-        if (confDiff == null) {
-            return (RunResult) superEmbed.run(config);
-        } else {
-            return (RunResult) superEmbed.run(config.merge(confDiff));
-        }
-    }
-
-    public EmbulkEmbed.ResumableResult resume(ConfigSource inConfig) throws IOException {
-        return resume(inConfig, null);
-    }
-
-    public EmbulkEmbed.ResumableResult resume(ConfigSource inConfig, ResumeState resumeState) throws IOException {
-        ConfigSource execConfig = newConfig()
-                .set("min_output_tasks", 1);
-
-        ConfigSource outConfig = newConfig()
-                .set("type", "memory");
-
-        ConfigSource config = newConfig()
-                .set("exec", execConfig)
-                .set("in", inConfig)
-                .set("out", outConfig);
-
-        MemoryOutputPlugin.clearRecords();
-        if (resumeState == null) {
-            return superEmbed.runResumable(config);
-        } else {
-            return superEmbed.new ResumeStateAction(config, resumeState).resume();
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private <T> T extractSuperField(String fieldName) {
-        try {
-            Field field = TestingEmbulk.class.getDeclaredField(fieldName);
-            field.setAccessible(true);
-            return (T) field.get(this);
-        } catch (NoSuchFieldException | IllegalAccessException e) {
-            throw new RuntimeException(e);
-        }
-    }
-}