embulk-parser-csv_with_default_value 0.1.0

data/src/test/java/org/embulk/GuiceBinder.java

@@ -0,0 +1,72 @@
+package org.embulk;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+public class GuiceBinder
+        implements TestRule
+{
+    private final List<Module> baseModules;
+    private List<Module> extraModules;
+    private Injector injector;
+
+    public GuiceBinder(Module... baseModules)
+    {
+        this.baseModules = ImmutableList.copyOf(baseModules);
+        reset();
+    }
+
+    private void reset()
+    {
+        extraModules = new ArrayList<Module>();
+        injector = null;
+    }
+
+    public synchronized void addModule(Module module)
+    {
+        if (injector != null) {
+            throw new IllegalStateException("Injector is already initialized. Call addModule before getInjector or getInstance");
+        }
+        extraModules.add(module);
+    }
+
+    public synchronized Injector getInjector()
+    {
+        if (injector == null) {
+            ImmutableList.Builder<Module> modules = ImmutableList.builder();
+            modules.addAll(baseModules);
+            modules.addAll(extraModules);
+            injector = Guice.createInjector(modules.build());
+        }
+        return injector;
+    }
+
+    public <T> T getInstance(Class<T> klass)
+    {
+        return getInjector().getInstance(klass);
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        return new GuceBinderWatcher().apply(base, description);
+    }
+
+    private class GuceBinderWatcher
+            extends TestWatcher
+    {
+        @Override
+        protected void starting(Description description)
+        {
+            reset();
+        }
+    }
+}

data/src/test/java/org/embulk/RandomManager.java

@@ -0,0 +1,53 @@
+package org.embulk;
+
+import java.util.Map;
+import java.util.Random;
+
+public class RandomManager
+{
+    protected long seed;
+    protected Random random;
+
+    public RandomManager()
+    {
+        this(getDefaultSeed());
+    }
+
+    public RandomManager(long seed)
+    {
+        this.seed = seed;
+        this.random = new Random(seed);
+        System.out.println(" Random seed: 0x"+Long.toHexString(seed)+"L");
+    }
+
+    public long getRandomSeed()
+    {
+        return seed;
+    }
+
+    public void setRandomSeed(long seed)
+    {
+        random.setSeed(seed);
+        this.seed = seed;
+        System.out.println(" Set random seed: 0x"+Long.toHexString(this.seed)+"L");
+    }
+
+    public Random getRandom()
+    {
+        return random;
+    }
+
+    private static long getDefaultSeed() {
+        Map<String, String> env = System.getenv();
+        String s = env.get("RANDOM_SEED");
+        try {
+            if(s != null) {
+                return Long.parseLong(s);
+            }
+        } catch (NumberFormatException e) {
+            System.out.println("RANDOM_SEED variable is wrong: "+e);
+        }
+
+        return new Random().nextLong();
+    }
+}

data/src/test/java/org/embulk/TestPluginSourceModule.java

@@ -0,0 +1,23 @@
+package org.embulk;
+
+import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
+
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.MockFormatterPlugin;
+import org.embulk.spi.MockParserPlugin;
+import org.embulk.spi.ParserPlugin;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+public class TestPluginSourceModule implements Module
+{
+    @Override
+    public void configure(Binder binder)
+    {
+        registerPluginTo(binder, ParserPlugin.class, "mock",
+                MockParserPlugin.class);
+        registerPluginTo(binder, FormatterPlugin.class, "mock",
+                MockFormatterPlugin.class);
+    }
+}

data/src/test/java/org/embulk/TestUtilityModule.java

@@ -0,0 +1,17 @@
+package org.embulk;
+
+//import org.embulk.record.RandomRecordGenerator;
+//import org.embulk.record.RandomSchemaGenerator;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+public class TestUtilityModule
+        implements Module
+{
+    @Override
+    public void configure(Binder binder) {
+        binder.bind(RandomManager.class);
+        //binder.bind(RandomRecordGenerator.class);
+        //binder.bind(RandomSchemaGenerator.class);
+    }
+}

data/src/test/java/org/embulk/parser/csv_with_default_value/TestCsvWithDefaultValueParserPlugin.java

@@ -0,0 +1,97 @@
+package org.embulk.parser.csv_with_default_value;
+
+import com.google.common.collect.Maps;
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import java.nio.charset.Charset;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.joda.time.DateTimeZone;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.Newline;
+import org.embulk.EmbulkTestRuntime;
+
+
+public class TestCsvWithDefaultValueParserPlugin {
+
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("columns", ImmutableList.of(
+                        ImmutableMap.of(
+                                "name", "date_code",
+                                "type", "string"))
+                );
+
+        CsvWithDefaultValueParserPlugin.PluginTask task = config.loadConfig(CsvWithDefaultValueParserPlugin.PluginTask.class);
+        assertEquals(Charset.forName("utf-8"), task.getCharset());
+        assertEquals(Newline.CRLF, task.getNewline());
+        assertEquals(false, task.getHeaderLine().or(false));
+        assertEquals(",", task.getDelimiter());
+        assertEquals(Optional.of(new CsvWithDefaultValueParserPlugin.QuoteCharacter('\"')), task.getQuoteChar());
+        assertEquals(false, task.getAllowOptionalColumns());
+        assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
+        assertEquals("%Y-%m-%d %H:%M:%S.%N %z", task.getDefaultTimestampFormat());
+        assertEquals(Maps.newHashMap(), task.getDefaultValues());
+    }
+    @Test(expected = ConfigException.class)
+    public void checkColumnsRequired()
+    {
+        ConfigSource config = Exec.newConfigSource();
+
+        config.loadConfig(CsvWithDefaultValueParserPlugin.PluginTask.class);
+    }
+
+    @Test
+    public void checkLoadConfig()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("charset", "utf-16")
+                .set("newline", "LF")
+                .set("header_line", true)
+                .set("delimiter", "\t")
+                .set("quote", "\\")
+                .set("allow_optional_columns", true)
+                .set("columns", ImmutableList.of(
+                        ImmutableMap.of(
+                                "name", "date_code",
+                                "type", "string"))
+
+                )
+                .set("default_values", ImmutableMap.of(
+                        "longCol", ImmutableMap.of("default_value", 123, "type", "immediate"),
+                        "doubleCol", ImmutableMap.of("default_value", 123, "type", "immediate"),
+                        "timestampCol", ImmutableMap.of("default_value", "1900-01-01 00:00:00", "type", "immediate")
+                ));
+
+        CsvWithDefaultValueParserPlugin.PluginTask task = config.loadConfig(CsvWithDefaultValueParserPlugin.PluginTask.class);
+        assertEquals(Charset.forName("utf-16"), task.getCharset());
+        assertEquals(Newline.LF, task.getNewline());
+        assertEquals(true, task.getHeaderLine().or(false));
+        assertEquals("\t", task.getDelimiter());
+        assertEquals(Optional.of(new CsvWithDefaultValueParserPlugin.QuoteCharacter('\\')), task.getQuoteChar());
+        assertEquals(true, task.getAllowOptionalColumns());
+
+        assertEquals(ImmutableMap.of(
+                "longCol", new ColumnDefaultValueImpl("123", ColumnDefaultValue.ValueType.IMMEDIATE),
+                "doubleCol", new ColumnDefaultValueImpl("123", ColumnDefaultValue.ValueType.IMMEDIATE),
+                "timestampCol", new ColumnDefaultValueImpl("1900-01-01 00:00:00", ColumnDefaultValue.ValueType.IMMEDIATE)),
+                task.getDefaultValues());
+    }
+
+    @Test
+    public void checkColumnDefaultValues(){
+
+    }
+
+
+}

data/src/test/java/org/embulk/parser/csv_with_default_value/ValueTypeTest.java

@@ -0,0 +1,47 @@
+package org.embulk.parser.csv_with_default_value;
+
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Types;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.*;
+import static org.embulk.parser.csv_with_default_value.ColumnDefaultValue.ValueType.*;
+import static org.mockito.Mockito.*;
+public class ValueTypeTest {
+
+    @Test
+    public void testFromString(){
+        assertThat(ColumnDefaultValue.ValueType.fromString("immediate"), equalTo(ColumnDefaultValue.ValueType.IMMEDIATE));
+        assertThat(ColumnDefaultValue.ValueType.fromString("null"), equalTo(ColumnDefaultValue.ValueType.NULL));
+        //TODO add assert for timestamp
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testFromStringThrowsException(){
+        ColumnDefaultValue.ValueType.fromString("hoge");
+    }
+
+    @Test
+    public void testGetValueMethods(){
+        PageBuilder pageBuilder = mock(PageBuilder.class);
+
+        IMMEDIATE.doubleValue(new ColumnDefaultValueImpl("123", IMMEDIATE), pageBuilder, new Column(0, "doubleCol", Types.DOUBLE));
+        IMMEDIATE.longValue(new ColumnDefaultValueImpl("123", IMMEDIATE), pageBuilder, new Column(1, "longCol", Types.LONG));
+        verify(pageBuilder).setDouble(new Column(0, "doubleCol", Types.DOUBLE),123.0);
+        verify(pageBuilder).setLong(new Column(1, "longCol", Types.LONG),123L);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testGetLongValueFromNullFails(){
+        NULL.longValue(new ColumnDefaultValueImpl("123", NULL), mock(PageBuilder.class), new Column(0, "", Types.DOUBLE));
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testGetDoubleValueFromNullFails(){
+        NULL.doubleValue(new ColumnDefaultValueImpl("123", NULL), mock(PageBuilder.class), new Column(0, "", Types.DOUBLE));
+    }
+
+}

data/src/test/java/org/embulk/spi/MockFormatterPlugin.java

@@ -0,0 +1,108 @@
+package org.embulk.spi;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.ColumnVisitor;
+
+public class MockFormatterPlugin implements FormatterPlugin
+{
+    public static List<List<Object>> records;
+
+    public interface PluginTask extends Task
+    {
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema schema,
+                            FormatterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump());
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, final Schema schema,
+                           FileOutput output)
+    {
+        return new PageOutput()
+        {
+            public void add(Page page)
+            {
+                records = readPage(schema, page);
+            }
+
+            @Override
+            public void finish()
+            {
+            }
+
+            @Override
+            public void close()
+            {
+            }
+        };
+    }
+
+    public static List<List<Object>> readPage(final Schema schema, Page page)
+    {
+        List<List<Object>> records = new ArrayList<>();
+        try (final PageReader pageReader = new PageReader(schema)) {
+            pageReader.setPage(page);
+            while (pageReader.nextRecord()) {
+                final List<Object> record = new ArrayList<>();
+                schema.visitColumns(new ColumnVisitor()
+                {
+                    public void booleanColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getBoolean(column));
+                        }
+                    }
+
+                    public void longColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getLong(column));
+                        }
+                    }
+
+                    public void doubleColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getDouble(column));
+                        }
+                    }
+
+                    public void stringColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getString(column));
+                        }
+                    }
+
+                    public void timestampColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getTimestamp(column));
+                        }
+                    }
+
+                    public void jsonColumn(Column column)
+                    {
+                        if (!pageReader.isNull(column)) {
+                            record.add(pageReader.getJson(column));
+                        }
+                    }
+                });
+                records.add(record);
+            }
+        }
+        return records;
+    }
+}

data/src/test/java/org/embulk/spi/MockParserPlugin.java

@@ -0,0 +1,80 @@
+package org.embulk.spi;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.PageOutput;
+
+public class MockParserPlugin implements ParserPlugin
+{
+    public static boolean raiseException = false;
+
+    public interface PluginTask extends Task
+    {
+        @Config("columns")
+        SchemaConfig getSchemaConfig();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump(), task.getSchemaConfig().toSchema());
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+                    FileInput input, PageOutput output)
+    {
+        try (final PageBuilder pageBuilder = new PageBuilder(
+                Exec.getBufferAllocator(), schema, output)) {
+            while (input.nextFile()) {
+                Buffer buffer = input.poll();
+                if (buffer != null) {
+                    for (Column column : schema.getColumns()) {
+                        Type type = column.getType();
+                        switch (type.getName()) {
+                            case "boolean":
+                                pageBuilder.setBoolean(column, true);
+                                break;
+                            case "long":
+                                pageBuilder.setLong(column, 2L);
+                                break;
+                            case "double":
+                                pageBuilder.setDouble(column, 3.0D);
+                                break;
+                            case "string":
+                                pageBuilder.setString(column, "45");
+                                break;
+                            case "timestamp":
+                                pageBuilder.setTimestamp(column,
+                                        Timestamp.ofEpochMilli(678L));
+                                break;
+                            case "json":
+                                pageBuilder.setJson(
+                                        column,
+                                        new JsonParser().parse("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}")
+                                );
+                                break;
+                            default:
+                                throw new IllegalStateException("Unknown getType: "
+                                        + type.getName());
+                        }
+                    }
+                    pageBuilder.addRecord();
+                    if (raiseException) {
+                        throw new RuntimeException("emulated exception");
+                    }
+                }
+            }
+            pageBuilder.finish();
+        }
+    }
+}