embulk-parser-jsonl 0.1.2 → 0.2.0

data/src/main/java/org/embulk/parser/jsonl/JsonRecordValidateException.java

@@ -3,12 +3,20 @@ package org.embulk.parser.jsonl;
 import org.embulk.spi.DataException;
 
 public class JsonRecordValidateException
-        extends DataException {
-    JsonRecordValidateException(String message) {
+        extends DataException
+{
+    public JsonRecordValidateException(String message)
+    {
         super(message);
     }
 
-    public JsonRecordValidateException(Throwable cause) {
+    public JsonRecordValidateException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
+
+    public JsonRecordValidateException(Throwable cause)
+    {
         super(cause);
     }
 }

data/src/main/java/org/embulk/parser/jsonl/JsonlParserPlugin.java

@@ -10,8 +10,6 @@ import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
-import org.embulk.parser.jsonl.getter.CommonColumnGetter;
-import org.embulk.parser.jsonl.getter.ColumnGetterFactory;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.DataException;
@@ -25,6 +23,7 @@ import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.json.JsonParseException;
 import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.Type;
 import org.embulk.spi.util.LineDecoder;
 import org.embulk.spi.util.Timestamps;
 import org.msgpack.value.Value;
@@ -37,6 +36,23 @@ import static org.msgpack.value.ValueFactory.newString;
 public class JsonlParserPlugin
         implements ParserPlugin
 {
+    @Deprecated
+    public interface JsonlColumnOption
+            extends Task
+    {
+        @Config("type")
+        @ConfigDefault("null")
+        Optional<Type> getType();
+    }
+
+    public interface TypecastColumnOption
+            extends Task
+    {
+        @Config("typecast")
+        @ConfigDefault("null")
+        public Optional<Boolean> getTypecast();
+    }
+
     public interface PluginTask
             extends Task, LineDecoder.DecoderTask, TimestampParser.Task
     {
@@ -53,8 +69,13 @@ public class JsonlParserPlugin
         @ConfigDefault("false")
         boolean getStopOnInvalidRecord();
 
+        @Config("default_typecast")
+        @ConfigDefault("true")
+        Boolean getDefaultTypecast();
+
         @Config("column_options")
         @ConfigDefault("{}")
+        @Deprecated
         Map<String, JsonlColumnOption> getColumnOptions();
     }
 
@@ -73,16 +94,27 @@ public class JsonlParserPlugin
     public void transaction(ConfigSource configSource, Control control)
     {
         PluginTask task = configSource.loadConfig(PluginTask.class);
+
+        if (! task.getColumnOptions().isEmpty()) {
+            log.warn("embulk-parser-jsonl: \"column_options\" option is deprecated, specify type directly to \"columns\" option with typecast: true (default: true).");
+        }
+
         SchemaConfig schemaConfig = getSchemaConfig(task);
         ImmutableList.Builder<Column> columns = ImmutableList.builder();
         for (int i = 0; i < schemaConfig.getColumnCount(); i++) {
             ColumnConfig columnConfig = schemaConfig.getColumn(i);
-            JsonlColumnOption columnOption = columnOptionOf(task.getColumnOptions(), columnConfig.getName());
-            columns.add(new Column(i, columnConfig.getName(), columnOption.getType().or(columnConfig.getType())));
+            Type type = getType(task, columnConfig);
+            columns.add(new Column(i, columnConfig.getName(), type));
         }
         control.run(task.dump(), new Schema(columns.build()));
     }
 
+    private static Type getType(PluginTask task, ColumnConfig columnConfig)
+    {
+        JsonlColumnOption columnOption = columnOptionOf(task.getColumnOptions(), columnConfig.getName());
+        return columnOption.getType().or(columnConfig.getType());
+    }
+
     // this method is to keep the backward compatibility of 'schema' option.
     private SchemaConfig getSchemaConfig(PluginTask task)
     {
@@ -115,13 +147,7 @@ public class JsonlParserPlugin
         final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
-            ColumnGetterFactory factory = new ColumnGetterFactory(pageBuilder, timestampParsers);
-            ImmutableMap.Builder<String, CommonColumnGetter> columnGettersBuilder = ImmutableMap.builder();
-            for (ColumnConfig columnConfig : schemaConfig.getColumns()) {
-                CommonColumnGetter columnGetter = factory.newColumnGetter(columnConfig);
-                columnGettersBuilder.put(columnConfig.getName(), columnGetter);
-            }
-            ImmutableMap<String, CommonColumnGetter> columnGetters = columnGettersBuilder.build();
+            ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, schema, pageBuilder, timestampParsers);
 
             while (decoder.nextFile()) { // TODO this implementation should be improved with new JsonParser API on Embulk v0.8.3
                 lineNumber = 0;
@@ -139,9 +165,8 @@ public class JsonlParserPlugin
                         final Map<Value, Value> record = value.asMapValue().map();
                         for (Column column : schema.getColumns()) {
                             Value v = record.get(getColumnNameValue(column));
-                            CommonColumnGetter columnGetter = columnGetters.get(column.getName());
-                            columnGetter.setValue(v);
-                            column.visit(columnGetter);
+                            visitor.setValue(v);
+                            column.visit(visitor);
                         }
 
                         pageBuilder.addRecord();

data/src/main/java/org/embulk/parser/jsonl/cast/BooleanCast.java

@@ -0,0 +1,39 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class BooleanCast
+{
+    private BooleanCast() {}
+
+    private static String buildErrorMessage(String as, boolean value)
+    {
+        return String.format("cannot cast boolean to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(boolean value) throws DataException
+    {
+        return value;
+    }
+
+    public static long asLong(boolean value) throws DataException
+    {
+        return value ? 1 : 0;
+    }
+
+    public static double asDouble(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(boolean value) throws DataException
+    {
+        return value ? "true" : "false";
+    }
+
+    public static Timestamp asTimestamp(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/cast/DoubleCast.java

@@ -0,0 +1,41 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class DoubleCast
+{
+    private DoubleCast() {}
+
+    private static String buildErrorMessage(String as, double value)
+    {
+        return String.format("cannot cast double to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(double value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(double value) throws DataException
+    {
+        return (long) value;
+    }
+
+    public static double asDouble(double value) throws DataException
+    {
+        return value;
+    }
+
+    public static String asString(double value) throws DataException
+    {
+        return String.valueOf(value);
+    }
+
+    public static Timestamp asTimestamp(double value) throws DataException
+    {
+        long epochSecond = (long) value;
+        long nanoAdjustMent = (long) ((value - epochSecond) * 1000000000);
+        return Timestamp.ofEpochSecond(epochSecond, nanoAdjustMent);
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/cast/JsonCast.java

@@ -0,0 +1,40 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public class JsonCast
+{
+    private JsonCast() {}
+
+    private static String buildErrorMessage(String as, Value value)
+    {
+        return String.format("cannot cast Json to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("long", value));
+    }
+
+    public static double asDouble(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(Value value) throws DataException
+    {
+        return value.toString();
+    }
+
+    public static Timestamp asTimestamp(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/cast/LongCast.java

@@ -0,0 +1,47 @@
+package org.embulk.parser.jsonl.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class LongCast
+{
+    private LongCast() {}
+
+    private static String buildErrorMessage(String as, long value)
+    {
+        return String.format("cannot cast long to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(long value) throws DataException
+    {
+        if (value == 1) {
+            return true;
+        }
+        else if (value == 0) {
+            return false;
+        }
+        else {
+            throw new DataException(buildErrorMessage("boolean", value));
+        }
+    }
+
+    public static long asLong(long value) throws DataException
+    {
+        return value;
+    }
+
+    public static double asDouble(long value) throws DataException
+    {
+        return (double) value;
+    }
+
+    public static String asString(long value) throws DataException
+    {
+        return String.valueOf(value);
+    }
+
+    public static Timestamp asTimestamp(long value) throws DataException
+    {
+        return Timestamp.ofEpochSecond(value);
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/cast/StringCast.java

@@ -0,0 +1,82 @@
+package org.embulk.parser.jsonl.cast;
+
+import com.google.common.collect.ImmutableSet;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.time.TimestampParser;
+
+public class StringCast
+{
+    // copy from csv plugin
+    public static final ImmutableSet<String> TRUE_STRINGS =
+            ImmutableSet.of(
+                    "true", "True", "TRUE",
+                    "yes", "Yes", "YES",
+                    "t", "T", "y", "Y",
+                    "on", "On", "ON",
+                    "1");
+
+    public static final ImmutableSet<String> FALSE_STRINGS =
+            ImmutableSet.of(
+                    "false", "False", "FALSE",
+                    "no", "No", "NO",
+                    "f", "F", "n", "N",
+                    "off", "Off", "OFF",
+                    "0");
+
+    private StringCast() {}
+
+    private static String buildErrorMessage(String as, String value)
+    {
+        return String.format("cannot cast String to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(String value) throws DataException
+    {
+        if (TRUE_STRINGS.contains(value)) {
+            return true;
+        }
+        else if (FALSE_STRINGS.contains(value)) {
+            return false;
+        }
+        else {
+            throw new DataException(buildErrorMessage("boolean", value));
+        }
+    }
+
+    public static long asLong(String value) throws DataException
+    {
+        try {
+            return Long.parseLong(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("long", value), ex);
+        }
+    }
+
+    public static double asDouble(String value) throws DataException
+    {
+        try {
+            return Double.parseDouble(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("double", value), ex);
+        }
+    }
+
+    public static String asString(String value) throws DataException
+    {
+        return value;
+    }
+
+    public static Timestamp asTimestamp(String value, TimestampParser parser) throws DataException
+    {
+        try {
+            return parser.parse(value);
+        }
+        catch (TimestampParseException ex) {
+            throw new DataException(buildErrorMessage("timestamp", value), ex);
+        }
+    }
+}

data/src/test/java/org/embulk/parser/jsonl/TestColumnCaster.java

@@ -0,0 +1,256 @@
+package org.embulk.parser.jsonl;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.msgpack.value.MapValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestColumnCaster
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    public MapValue mapValue;
+    public DataException thrown;
+    public ScriptingContainer jruby;
+    public TimestampParser parser;
+
+    @Before
+    public void createResource()
+    {
+        jruby = new ScriptingContainer();
+        thrown = new DataException("any");
+        Value[] kvs = new Value[2];
+        kvs[0] = ValueFactory.newString("k");
+        kvs[1] = ValueFactory.newString("v");
+        mapValue = ValueFactory.newMap(kvs);
+        parser = new TimestampParser(jruby, "%Y-%m-%d %H:%M:%S.%N", DateTimeZone.UTC);
+    }
+
+    @Test
+    public void asBooleanFromBoolean()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asBooleanFromInteger()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newInteger(1)));
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newInteger(2));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromFloat()
+    {
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newFloat(1.1));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromString()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newString("true")));
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromJson()
+    {
+        try {
+            ColumnCaster.asBoolean(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asLongFromBoolean()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asLongFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asLongFromFloat()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asLongFromString()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1")));
+        try {
+            ColumnCaster.asLong(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asLongFromJson()
+    {
+        try {
+            ColumnCaster.asLong(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asDoubleFromBoolean()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asDoubleFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asDoubleFromFloat()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asDoubleFromString()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1")));
+        try {
+            ColumnCaster.asLong(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asDoubleFromJson()
+    {
+        try {
+            ColumnCaster.asLong(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asStringFromBoolean()
+    {
+        assertEquals("true", ColumnCaster.asString(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asStringFromInteger()
+    {
+        assertEquals("1", ColumnCaster.asString(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asStringFromFloat()
+    {
+        assertEquals("1.5", ColumnCaster.asString(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asStringFromString()
+    {
+        assertEquals("1", ColumnCaster.asString(ValueFactory.newString("1")));
+    }
+
+    @Test
+    public void asStringFromJson()
+    {
+        assertEquals("{\"k\":\"v\"}", ColumnCaster.asString(mapValue));
+    }
+
+    @Test
+    public void asTimestampFromBoolean()
+    {
+        try {
+            ColumnCaster.asTimestamp(ValueFactory.newBoolean(true), parser);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asTimestampFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asTimestamp(ValueFactory.newInteger(1), parser).getEpochSecond());
+    }
+
+    @Test
+    public void asTimestampFromFloat()
+    {
+        Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000);
+        assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newFloat(1463084053.5), parser));
+    }
+
+    @Test
+    public void asTimestampFromString()
+    {
+        Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000);
+        assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newString("2016-05-12 20:14:13.5"), parser));
+    }
+
+    @Test
+    public void asTimestampFromJson()
+    {
+        try {
+            ColumnCaster.asTimestamp(mapValue, parser);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+}