embulk-parser-jsonline 0.2.2

data/src/main/java/org/embulk/parser/jsonl/cast/StringCast.java

@@ -0,0 +1,82 @@
+package org.embulk.parser.jsonl.cast;
+
+import com.google.common.collect.ImmutableSet;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.time.TimestampParser;
+
+public class StringCast
+{
+    // copy from csv plugin
+    public static final ImmutableSet<String> TRUE_STRINGS =
+            ImmutableSet.of(
+                    "true", "True", "TRUE",
+                    "yes", "Yes", "YES",
+                    "t", "T", "y", "Y",
+                    "on", "On", "ON",
+                    "1");
+
+    public static final ImmutableSet<String> FALSE_STRINGS =
+            ImmutableSet.of(
+                    "false", "False", "FALSE",
+                    "no", "No", "NO",
+                    "f", "F", "n", "N",
+                    "off", "Off", "OFF",
+                    "0");
+
+    private StringCast() {}
+
+    private static String buildErrorMessage(String as, String value)
+    {
+        return String.format("cannot cast String to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(String value) throws DataException
+    {
+        if (TRUE_STRINGS.contains(value)) {
+            return true;
+        }
+        else if (FALSE_STRINGS.contains(value)) {
+            return false;
+        }
+        else {
+            throw new DataException(buildErrorMessage("boolean", value));
+        }
+    }
+
+    public static long asLong(String value) throws DataException
+    {
+        try {
+            return Long.parseLong(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("long", value), ex);
+        }
+    }
+
+    public static double asDouble(String value) throws DataException
+    {
+        try {
+            return Double.parseDouble(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("double", value), ex);
+        }
+    }
+
+    public static String asString(String value) throws DataException
+    {
+        return value;
+    }
+
+    public static Timestamp asTimestamp(String value, TimestampParser parser) throws DataException
+    {
+        try {
+            return parser.parse(value);
+        }
+        catch (TimestampParseException ex) {
+            throw new DataException(buildErrorMessage("timestamp", value), ex);
+        }
+    }
+}

data/src/test/java/org/embulk/parser/jsonl/TestColumnCaster.java

@@ -0,0 +1,256 @@
+package org.embulk.parser.jsonl;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.msgpack.value.MapValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestColumnCaster
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    public MapValue mapValue;
+    public DataException thrown;
+    public ScriptingContainer jruby;
+    public TimestampParser parser;
+
+    @Before
+    public void createResource()
+    {
+        jruby = new ScriptingContainer();
+        thrown = new DataException("any");
+        Value[] kvs = new Value[2];
+        kvs[0] = ValueFactory.newString("k");
+        kvs[1] = ValueFactory.newString("v");
+        mapValue = ValueFactory.newMap(kvs);
+        parser = new TimestampParser(jruby, "%Y-%m-%d %H:%M:%S.%N", DateTimeZone.UTC);
+    }
+
+    @Test
+    public void asBooleanFromBoolean()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asBooleanFromInteger()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newInteger(1)));
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newInteger(2));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromFloat()
+    {
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newFloat(1.1));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromString()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newString("true")));
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromJson()
+    {
+        try {
+            ColumnCaster.asBoolean(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asLongFromBoolean()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asLongFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asLongFromFloat()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asLongFromString()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1")));
+        try {
+            ColumnCaster.asLong(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asLongFromJson()
+    {
+        try {
+            ColumnCaster.asLong(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asDoubleFromBoolean()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asDoubleFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asDoubleFromFloat()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asDoubleFromString()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1")));
+        try {
+            ColumnCaster.asLong(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asDoubleFromJson()
+    {
+        try {
+            ColumnCaster.asLong(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asStringFromBoolean()
+    {
+        assertEquals("true", ColumnCaster.asString(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asStringFromInteger()
+    {
+        assertEquals("1", ColumnCaster.asString(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asStringFromFloat()
+    {
+        assertEquals("1.5", ColumnCaster.asString(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asStringFromString()
+    {
+        assertEquals("1", ColumnCaster.asString(ValueFactory.newString("1")));
+    }
+
+    @Test
+    public void asStringFromJson()
+    {
+        assertEquals("{\"k\":\"v\"}", ColumnCaster.asString(mapValue));
+    }
+
+    @Test
+    public void asTimestampFromBoolean()
+    {
+        try {
+            ColumnCaster.asTimestamp(ValueFactory.newBoolean(true), parser);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asTimestampFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asTimestamp(ValueFactory.newInteger(1), parser).getEpochSecond());
+    }
+
+    @Test
+    public void asTimestampFromFloat()
+    {
+        Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000);
+        assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newFloat(1463084053.5), parser));
+    }
+
+    @Test
+    public void asTimestampFromString()
+    {
+        Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000);
+        assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newString("2016-05-12 20:14:13.5"), parser));
+    }
+
+    @Test
+    public void asTimestampFromJson()
+    {
+        try {
+            ColumnCaster.asTimestamp(mapValue, parser);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+}

data/src/test/java/org/embulk/parser/jsonl/TestJsonlParserPlugin.java

@@ -0,0 +1,278 @@
+package org.embulk.parser.jsonl;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.Pages;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import static org.embulk.spi.type.Types.BOOLEAN;
+import static org.embulk.spi.type.Types.DOUBLE;
+import static org.embulk.spi.type.Types.JSON;
+import static org.embulk.spi.type.Types.LONG;
+import static org.embulk.spi.type.Types.STRING;
+import static org.embulk.spi.type.Types.TIMESTAMP;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.msgpack.value.ValueFactory.newArray;
+import static org.msgpack.value.ValueFactory.newMap;
+import static org.msgpack.value.ValueFactory.newString;
+
+public class TestJsonlParserPlugin
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+    private JsonlParserPlugin plugin;
+    private MockPageOutput output;
+
+    @Before
+    public void createResource()
+    {
+        config = config().set("type", "jsonl");
+        plugin = new JsonlParserPlugin();
+        recreatePageOutput();
+    }
+
+    private void recreatePageOutput()
+    {
+        output = new MockPageOutput();
+    }
+
+    @Test
+    public void skipRecords()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema);
+
+        transaction(config, fileInput(
+                "[]",
+                "\"embulk\"",
+                "10",
+                "true",
+                "false",
+                "null",
+                " "
+        ));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(0, records.size());
+    }
+
+    @Test
+    public void throwDataException()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema).set("stop_on_invalid_record", true);
+
+        try {
+            transaction(config, fileInput(
+                    "\"not_map_value\""
+            ));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void writeNils()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema);
+
+        transaction(config, fileInput(
+                "{}",
+                "{\"_c0\":null,\"_c1\":null,\"_c2\":null}",
+                "{\"_c3\":null,\"_c4\":null,\"_c5\":null}",
+                "{}"
+        ));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(4, records.size());
+
+        for (Object[] record : records) {
+            for (int i = 0; i < 6; i++) {
+                assertNull(record[i]);
+            }
+        }
+    }
+
+    @Test
+    public void useNormal()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S %Z")), column("_c5", JSON));
+        List<ConfigSource> configs = Lists.newArrayList(
+                this.config.deepCopy().set("columns", schema),
+                this.config.deepCopy().set("schema", schema)
+        );
+
+        for (ConfigSource config : configs) {
+            transaction(config, fileInput(
+                    "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}}",
+                    "[1, 2, 3]",
+                    "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}"
+            ));
+
+            List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+            assertEquals(2, records.size());
+
+            Object[] record;
+            {
+                record = records.get(0);
+                assertEquals(true, record[0]);
+                assertEquals(10L, record[1]);
+                assertEquals(0.1, (Double) record[2], 0.0001);
+                assertEquals("embulk", record[3]);
+                assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]);
+                assertEquals(newMap(newString("k"), newString("v")), record[5]);
+            }
+            {
+                record = records.get(1);
+                assertEquals(false, record[0]);
+                assertEquals(-10L, record[1]);
+                assertEquals(1.0, (Double) record[2], 0.0001);
+                assertEquals("エンバルク", record[3]);
+                assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]);
+                assertEquals(newArray(newString("e0"), newString("e1")), record[5]);
+            }
+
+            recreatePageOutput();
+        }
+    }
+
+    @Test
+    public void useColumnOptions()
+            throws Exception
+    {
+
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE));
+        File yamlFile = getResourceFile("use_column_options.yml");
+        ConfigSource config = getConfigFromYamlFile(yamlFile);
+
+        transaction(config, fileInput(
+                "{\"_c0\":\"true\",\"_c1\":\"10\",\"_c2\":\"0.1\"}",
+                "{\"_c0\":\"false\",\"_c1\":\"-10\",\"_c2\":\"1.0\"}"
+        ));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(2, records.size());
+
+        Object[] record;
+        {
+            record = records.get(0);
+            assertEquals(true, record[0]);
+            assertEquals(10L, record[1]);
+            assertEquals(0.1, (Double) record[2], 0.0001);
+        }
+        {
+            record = records.get(1);
+            assertEquals(false, record[0]);
+            assertEquals(-10L, record[1]);
+            assertEquals(1.0, (Double) record[2], 0.0001);
+        }
+    }
+
+    private ConfigSource config()
+    {
+        return runtime.getExec().newConfigSource();
+    }
+
+    private File getResourceFile(String resourceName)
+            throws IOException
+    {
+        return new File(this.getClass().getResource(resourceName).getFile());
+    }
+
+    private ConfigSource getConfigFromYamlFile(File yamlFile)
+            throws IOException
+    {
+        ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
+        return loader.fromYamlFile(yamlFile);
+    }
+
+    private void transaction(ConfigSource config, final FileInput input)
+    {
+        plugin.transaction(config, new ParserPlugin.Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                plugin.run(taskSource, schema, input, output);
+            }
+        });
+    }
+
+    private FileInput fileInput(String... lines)
+            throws Exception
+    {
+        StringBuilder sb = new StringBuilder();
+        for (String line : lines) {
+            sb.append(line).append("\n");
+        }
+
+        ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes());
+        return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
+    }
+
+    private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams)
+            throws IOException
+    {
+        return new InputStreamFileInput.IteratorProvider(
+                ImmutableList.copyOf(inputStreams));
+    }
+
+    private SchemaConfig schema(ColumnConfig... columns)
+    {
+        return new SchemaConfig(Lists.newArrayList(columns));
+    }
+
+    private ColumnConfig column(String name, Type type)
+    {
+        return column(name, type, config());
+    }
+
+    private ColumnConfig column(String name, Type type, ConfigSource option)
+    {
+        return new ColumnConfig(name, type, option);
+    }
+}

data/src/test/java/org/embulk/parser/jsonl/cast/TestBooleanCast.java

@@ -0,0 +1,56 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestBooleanCast
+{
+    @Test
+    public void asBoolean()
+    {
+        assertEquals(true, BooleanCast.asBoolean(true));
+        assertEquals(false, BooleanCast.asBoolean(false));
+    }
+
+    @Test
+    public void asLong()
+    {
+        assertEquals(1, BooleanCast.asLong(true));
+        assertEquals(0, BooleanCast.asLong(false));
+    }
+
+    @Test
+    public void asDouble()
+    {
+        try {
+            BooleanCast.asDouble(true);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asString()
+    {
+        assertEquals("true", BooleanCast.asString(true));
+        assertEquals("false", BooleanCast.asString(false));
+    }
+
+    @Test
+    public void asTimestamp()
+    {
+        try {
+            BooleanCast.asTimestamp(true);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+}