embulk 0.7.0-java

data/embulk-core/src/test/java/org/embulk/TestPluginSourceModule.java

@@ -0,0 +1,23 @@
+package org.embulk;
+
+import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
+
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.MockFormatterPlugin;
+import org.embulk.spi.MockParserPlugin;
+import org.embulk.spi.ParserPlugin;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+public class TestPluginSourceModule implements Module
+{
+    @Override
+    public void configure(Binder binder)
+    {
+        registerPluginTo(binder, ParserPlugin.class, "mock",
+                MockParserPlugin.class);
+        registerPluginTo(binder, FormatterPlugin.class, "mock",
+                MockFormatterPlugin.class);
+    }
+}

data/embulk-core/src/test/java/org/embulk/TestUtilityModule.java

@@ -0,0 +1,17 @@
+package org.embulk;
+
+//import org.embulk.record.RandomRecordGenerator;
+//import org.embulk.record.RandomSchemaGenerator;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+public class TestUtilityModule
+        implements Module
+{
+    @Override
+    public void configure(Binder binder) {
+        binder.bind(RandomManager.class);
+        //binder.bind(RandomRecordGenerator.class);
+        //binder.bind(RandomSchemaGenerator.class);
+    }
+}

data/embulk-core/src/test/java/org/embulk/config/TestConfigLoader.java

@@ -0,0 +1,66 @@
+package org.embulk.config;
+
+import java.util.Properties;
+import java.io.InputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import com.google.inject.Inject;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestConfigLoader
+{
+    private ConfigLoader loader;
+
+    @Before
+    public void setup() throws Exception
+    {
+        this.loader = new ConfigLoader(new ModelManager(null, new ObjectMapper()));
+    }
+
+    @Test
+    public void testFromEmptyJson() throws IOException
+    {
+        ConfigSource config = loader.fromJson(newInputStream("{\"type\":\"test\",\"data\":1}"));
+        assertEquals("test", config.get(String.class, "type"));
+        assertEquals(1, (int) config.get(Integer.class, "data"));
+    }
+
+    @Test
+    public void testFromYamlProperties() throws IOException
+    {
+        Properties props = new Properties();
+        props.setProperty("type", "test");
+        props.setProperty("data", "1");
+
+        ConfigSource config = loader.fromPropertiesYamlLiteral(props, "");
+        assertEquals("test", config.get(String.class, "type"));
+        assertEquals(1, (int) config.get(Integer.class, "data"));
+    }
+
+    @Test
+    public void testFromYamlPropertiesNested() throws IOException
+    {
+        Properties props = new Properties();
+        props.setProperty("type", "test");
+        props.setProperty("columns.k1", "1");
+        props.setProperty("values.myval.data", "2");
+
+        ConfigSource config = loader.fromPropertiesYamlLiteral(props, "");
+        System.out.println("config: "+config);
+        assertEquals("test", config.get(String.class, "type"));
+        assertEquals(1, (int) config.getNested("columns").get(Integer.class, "k1"));
+        assertEquals(2, (int) config.getNested("values").getNested("myval").get(Integer.class, "data"));
+    }
+
+    private static InputStream newInputStream(String string)
+    {
+        byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
+        return new ByteArrayInputStream(bytes);
+    }
+}

data/embulk-core/src/test/java/org/embulk/config/TestConfigSource.java

@@ -0,0 +1,114 @@
+package org.embulk.config;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import com.google.inject.Inject;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestConfigSource
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+
+    @Before
+    public void setup() throws Exception
+    {
+        config = Exec.newConfigSource();
+    }
+
+    private static interface TypeFields
+            extends Task
+    {
+        @Config("boolean")
+        public boolean getBoolean();
+
+        @Config("double")
+        public double getDouble();
+
+        @Config("int")
+        public int getInt();
+
+        @Config("long")
+        public long getLong();
+
+        @Config("string")
+        public String getString();
+    }
+
+    @Test
+    public void testSetGet()
+    {
+        config.set("boolean", true);
+        config.set("int", 3);
+        config.set("double", 0.2);
+        config.set("long", Long.MAX_VALUE);
+        config.set("string", "sf");
+
+        assertEquals(true, (boolean) config.get(boolean.class, "boolean"));
+        assertEquals(3, (int) config.get(int.class, "int"));
+        assertEquals(0.2, (double) config.get(double.class, "double"), 0.001);
+        assertEquals(Long.MAX_VALUE, (long) config.get(long.class, "long"));
+        assertEquals("sf", config.get(String.class, "string"));
+    }
+
+    @Test
+    public void testLoadConfig()
+    {
+        config.set("boolean", true);
+        config.set("int", 3);
+        config.set("double", 0.2);
+        config.set("long", Long.MAX_VALUE);
+        config.set("string", "sf");
+
+        TypeFields task = config.loadConfig(TypeFields.class);
+        assertEquals(true, task.getBoolean());
+        assertEquals(3, task.getInt());
+        assertEquals(0.2, task.getDouble(), 0.001);
+        assertEquals(Long.MAX_VALUE, task.getLong());
+        assertEquals("sf", task.getString());
+    }
+
+    private static interface ValidateFields
+            extends Task
+    {
+        @Config("valid")
+        public String getValid();
+    }
+
+    @Test
+    public void testValidatePasses()
+    {
+        config.set("valid", "data");
+        ValidateFields task = config.loadConfig(ValidateFields.class);
+        task.validate();
+        assertEquals("data", task.getValid());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testDefaultValueValidateFails()
+    {
+        ValidateFields task = config.loadConfig(ValidateFields.class);
+        task.validate();
+    }
+
+    // TODO test Min, Max, and other validations
+
+    private static interface SimpleFields
+            extends Task
+    {
+        @Config("type")
+        public String getType();
+    }
+
+    @Test
+    public void testFromJson()
+    {
+        String json = "{\"type\":\"test\"}";
+        // TODO
+    }
+}

data/embulk-core/src/test/java/org/embulk/config/TestTaskSource.java

@@ -0,0 +1,70 @@
+package org.embulk.config;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import com.google.inject.Inject;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestTaskSource
+{
+    private static interface TypeFields
+            extends Task
+    {
+        public boolean getBoolean();
+        public void setBoolean(boolean v);
+
+        public double getDouble();
+        public void setDouble(double v);
+
+        public int getInt();
+        public void setInt(int v);
+
+        public long getLong();
+        public void setLong(long v);
+
+        public String getString();
+        public void setString(String v);
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private TaskSource taskSource;
+
+    @Before
+    public void setup() throws Exception
+    {
+        taskSource = Exec.newTaskSource();
+        taskSource.set("Boolean", false);
+        taskSource.set("Double", 0.5);
+        taskSource.set("Int", 0);
+        taskSource.set("Long", 0);
+        taskSource.set("String", "");
+    }
+
+    @Test
+    public void testEqualsOfLoadedTasks()
+    {
+        TypeFields task = taskSource.loadTask(TypeFields.class);
+        task.setBoolean(true);
+        task.setDouble(0.2);
+        task.setInt(3);
+        task.setLong(Long.MAX_VALUE);
+        task.setString("sf");
+
+        TaskSource taskSource2 = task.dump();
+        TypeFields task2 = taskSource2.loadTask(TypeFields.class);
+
+        assertTrue(task.equals(task2));
+        assertTrue(task.hashCode() == task2.hashCode());
+
+        task.setBoolean(false);
+        assertFalse(task.equals(task2));
+        assertFalse(task.hashCode() == task2.hashCode());
+    }
+}

data/embulk-core/src/test/java/org/embulk/plugin/MockPluginSource.java

@@ -0,0 +1,58 @@
+package org.embulk.plugin;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import com.google.inject.multibindings.Multibinder;
+
+public class MockPluginSource
+        implements PluginSource
+{
+    private final Class<?> expectedIface;
+    private final Object plugin;
+    private PluginType typeConfig;
+
+    public <T> MockPluginSource(Class<T> expectedIface, T plugin)
+    {
+        this.expectedIface = expectedIface;
+        this.plugin = plugin;
+    }
+
+    public PluginType getTypeConfig()
+    {
+        return typeConfig;
+    }
+
+    @SuppressWarnings("unchecked")
+    public <T> T newPlugin(Class<T> iface, PluginType typeConfig) throws PluginSourceNotMatchException
+    {
+        if (expectedIface.equals(iface)) {
+            this.typeConfig = typeConfig;
+            return (T) plugin;
+        } else {
+            throw new PluginSourceNotMatchException();
+        }
+    }
+
+    public static <T> Module newInjectModule(final Class<T> expectedIface, final T plugin)
+    {
+        return new InjectModule(expectedIface, plugin);
+    }
+
+    public static class InjectModule
+            implements Module
+    {
+        private final PluginSource pluginSource;
+
+        public <T> InjectModule(Class<T> expectedIface, T plugin)
+        {
+            this.pluginSource = new MockPluginSource(expectedIface, plugin);
+        }
+
+        @Override
+        public void configure(Binder binder)
+        {
+            Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+            multibinder.addBinding().toInstance(pluginSource);
+        }
+    }
+}

data/embulk-core/src/test/java/org/embulk/plugin/TestPluginType.java

@@ -0,0 +1,18 @@
+package org.embulk.plugin;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class TestPluginType
+{
+    @Test
+    public void testEquals()
+    {
+        PluginType type = new PluginType("a");
+        assertEquals(true, (type.equals(type)));
+
+        assertEquals(true, (type.equals(new PluginType("a"))));
+        assertEquals(false, (type.equals(new PluginType("b"))));
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/MockFileOutput.java

@@ -0,0 +1,63 @@
+package org.embulk.spi;
+
+import java.util.List;
+import java.util.ArrayList;
+
+public class MockFileOutput
+        implements FileOutput
+{
+    private List<List<Buffer>> files = new ArrayList<List<Buffer>>();
+    private List<Buffer> lastBuffers = null;
+    private boolean finished = false;
+    private boolean closed = false;
+
+    public List<List<Buffer>> getFiles()
+    {
+        return files;
+    }
+
+    public List<Buffer> getLastBuffers()
+    {
+        return lastBuffers;
+    }
+
+    public void nextFile()
+    {
+        lastBuffers = new ArrayList<Buffer>();
+        files.add(lastBuffers);
+    }
+
+    public boolean isFinished()
+    {
+        return finished;
+    }
+
+    public boolean isClosed()
+    {
+        return closed;
+    }
+
+    public void add(Buffer buffer)
+    {
+        if (lastBuffers == null) {
+            throw new IllegalStateException("FileOutput.nextFile is not called");
+        }
+        if (finished) {
+            throw new IllegalStateException("FileOutput is already finished");
+        }
+        if (closed) {
+            throw new IllegalStateException("FileOutput is already closed");
+        }
+        lastBuffers.add(buffer);
+    }
+
+    public void finish()
+    {
+        finished = true;
+    }
+
+    public void close()
+    {
+        closed = true;
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java

@@ -0,0 +1,101 @@
+package org.embulk.spi;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.ColumnVisitor;
+
+public class MockFormatterPlugin implements FormatterPlugin
+{
+    public static List<List<Object>> records;
+
+    public interface PluginTask extends Task
+    {
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema schema,
+            FormatterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump());
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, final Schema schema,
+            FileOutput output)
+    {
+        return new PageOutput()
+        {
+            public void add(Page page)
+            {
+                records = readPage(schema, page);
+            }
+
+            @Override
+            public void finish()
+            {
+            }
+
+            @Override
+            public void close()
+            {
+            }
+        };
+    }
+
+    public static List<List<Object>> readPage(final Schema schema, Page page)
+    {
+        List<List<Object>> records = new ArrayList<>();
+        try (final PageReader pageReader = new PageReader(schema)) {
+            pageReader.setPage(page);
+            while (pageReader.nextRecord()) {
+                final List<Object> record = new ArrayList<>();
+                schema.visitColumns(new ColumnVisitor()
+                {
+                    public void booleanColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getBoolean(column));
+                        }
+                    }
+
+                    public void longColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getLong(column));
+                        }
+                    }
+
+                    public void doubleColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getDouble(column));
+                        }
+                    }
+
+                    public void stringColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getString(column));
+                        }
+                    }
+
+                    public void timestampColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getTimestamp(column));
+                        }
+                    }
+                });
+                records.add(record);
+            }
+        }
+        return records;
+    }
+}