embulk-parser-jsonpath 0.1.0

data/src/main/java/org/embulk/parser/jsonpath/JsonRecordValidateException.java

@@ -0,0 +1,22 @@
+package org.embulk.parser.jsonpath;
+
+import org.embulk.spi.DataException;
+
+public class JsonRecordValidateException
+        extends DataException
+{
+    public JsonRecordValidateException(String message)
+    {
+        super(message);
+    }
+
+    public JsonRecordValidateException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
+
+    public JsonRecordValidateException(Throwable cause)
+    {
+        super(cause);
+    }
+}

data/src/main/java/org/embulk/parser/jsonpath/JsonpathParserPlugin.java

@@ -0,0 +1,148 @@
+package org.embulk.parser.jsonpath;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+import com.jayway.jsonpath.JsonPath;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.FileInputInputStream;
+import org.embulk.spi.util.Timestamps;
+import org.msgpack.value.Value;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static java.util.Locale.ENGLISH;
+import static org.msgpack.value.ValueFactory.newString;
+
+public class JsonpathParserPlugin
+        implements ParserPlugin
+{
+
+    private static final Logger logger = Exec.getLogger(JsonpathParserPlugin.class);
+
+    private Map<String, Value> columnNameValues;
+
+    public interface TypecastColumnOption
+            extends Task
+    {
+        @Config("typecast")
+        @ConfigDefault("null")
+        public Optional<Boolean> getTypecast();
+    }
+
+    public interface PluginTask
+            extends Task, TimestampParser.Task
+    {
+        @Config("root")
+        @ConfigDefault("\"$\"")
+        public String getRoot();
+
+        @Config("columns")
+        SchemaConfig getSchemaConfig();
+
+        @Config("default_typecast")
+        @ConfigDefault("true")
+        Boolean getDefaultTypecast();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        Schema schema = task.getSchemaConfig().toSchema();
+
+        control.run(task.dump(), schema);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        String jsonRoot = task.getRoot();
+
+        setColumnNameValues(schema);
+
+        logger.debug("JSONPath = " + jsonRoot);
+        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
+        final JsonParser jsonParser = new JsonParser();
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
+
+        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
+            ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, schema, pageBuilder, timestampParsers);
+
+            try (FileInputInputStream is = new FileInputInputStream(input)) {
+                while (is.nextFile()) {
+                    // TODO more efficient handling.
+                    String json = JsonPath.read(is, jsonRoot).toString();
+                    Value value = jsonParser.parse(json);
+                    if (!value.isArrayValue()) {
+                        throw new JsonRecordValidateException("Json string is not representing array value.");
+                    }
+
+                    for (Value recordValue : value.asArrayValue()) {
+                        if (!recordValue.isMapValue()) {
+                            if (stopOnInvalidRecord) {
+                                throw new JsonRecordValidateException("Json string is not representing map value.");
+                            }
+                            logger.warn(String.format(ENGLISH, "Skipped invalid record  %s", recordValue));
+                            continue;
+                        }
+
+                        logger.debug("recordValue = " + recordValue.toString());
+                        final Map<Value, Value> record = recordValue.asMapValue().map();
+                        for (Column column : schema.getColumns()) {
+                            Value v = record.get(getColumnNameValue(column));
+                            visitor.setValue(v);
+                            column.visit(visitor);
+                        }
+
+                        pageBuilder.addRecord();
+                    }
+                }
+            }
+            catch (IOException e) {
+                // TODO more efficient exception handling.
+                throw new DataException("catch IOException " + e);
+            }
+
+            pageBuilder.finish();
+        }
+    }
+
+    private void setColumnNameValues(Schema schema)
+    {
+        ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
+        for (Column column : schema.getColumns()) {
+            String name = column.getName();
+            builder.put(name, newString(name));
+        }
+        columnNameValues = builder.build();
+    }
+
+    private Value getColumnNameValue(Column column)
+    {
+        return columnNameValues.get(column.getName());
+    }
+}

data/src/main/java/org/embulk/parser/jsonpath/cast/BooleanCast.java

@@ -0,0 +1,39 @@
+package org.embulk.parser.jsonpath.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class BooleanCast
+{
+    private BooleanCast() {}
+
+    private static String buildErrorMessage(String as, boolean value)
+    {
+        return String.format("cannot cast boolean to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(boolean value) throws DataException
+    {
+        return value;
+    }
+
+    public static long asLong(boolean value) throws DataException
+    {
+        return value ? 1 : 0;
+    }
+
+    public static double asDouble(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(boolean value) throws DataException
+    {
+        return value ? "true" : "false";
+    }
+
+    public static Timestamp asTimestamp(boolean value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}

data/src/main/java/org/embulk/parser/jsonpath/cast/DoubleCast.java

@@ -0,0 +1,41 @@
+package org.embulk.parser.jsonpath.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class DoubleCast
+{
+    private DoubleCast() {}
+
+    private static String buildErrorMessage(String as, double value)
+    {
+        return String.format("cannot cast double to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(double value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(double value) throws DataException
+    {
+        return (long) value;
+    }
+
+    public static double asDouble(double value) throws DataException
+    {
+        return value;
+    }
+
+    public static String asString(double value) throws DataException
+    {
+        return String.valueOf(value);
+    }
+
+    public static Timestamp asTimestamp(double value) throws DataException
+    {
+        long epochSecond = (long) value;
+        long nanoAdjustMent = (long) ((value - epochSecond) * 1000000000);
+        return Timestamp.ofEpochSecond(epochSecond, nanoAdjustMent);
+    }
+}

data/src/main/java/org/embulk/parser/jsonpath/cast/JsonCast.java

@@ -0,0 +1,40 @@
+package org.embulk.parser.jsonpath.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public class JsonCast
+{
+    private JsonCast() {}
+
+    private static String buildErrorMessage(String as, Value value)
+    {
+        return String.format("cannot cast Json to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("boolean", value));
+    }
+
+    public static long asLong(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("long", value));
+    }
+
+    public static double asDouble(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("double", value));
+    }
+
+    public static String asString(Value value) throws DataException
+    {
+        return value.toString();
+    }
+
+    public static Timestamp asTimestamp(Value value) throws DataException
+    {
+        throw new DataException(buildErrorMessage("timestamp", value));
+    }
+}

data/src/main/java/org/embulk/parser/jsonpath/cast/LongCast.java

@@ -0,0 +1,47 @@
+package org.embulk.parser.jsonpath.cast;
+
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class LongCast
+{
+    private LongCast() {}
+
+    private static String buildErrorMessage(String as, long value)
+    {
+        return String.format("cannot cast long to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(long value) throws DataException
+    {
+        if (value == 1) {
+            return true;
+        }
+        else if (value == 0) {
+            return false;
+        }
+        else {
+            throw new DataException(buildErrorMessage("boolean", value));
+        }
+    }
+
+    public static long asLong(long value) throws DataException
+    {
+        return value;
+    }
+
+    public static double asDouble(long value) throws DataException
+    {
+        return (double) value;
+    }
+
+    public static String asString(long value) throws DataException
+    {
+        return String.valueOf(value);
+    }
+
+    public static Timestamp asTimestamp(long value) throws DataException
+    {
+        return Timestamp.ofEpochSecond(value);
+    }
+}

data/src/main/java/org/embulk/parser/jsonpath/cast/StringCast.java

@@ -0,0 +1,82 @@
+package org.embulk.parser.jsonpath.cast;
+
+import com.google.common.collect.ImmutableSet;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.time.TimestampParser;
+
+public class StringCast
+{
+    // copy from csv plugin
+    public static final ImmutableSet<String> TRUE_STRINGS =
+            ImmutableSet.of(
+                    "true", "True", "TRUE",
+                    "yes", "Yes", "YES",
+                    "t", "T", "y", "Y",
+                    "on", "On", "ON",
+                    "1");
+
+    public static final ImmutableSet<String> FALSE_STRINGS =
+            ImmutableSet.of(
+                    "false", "False", "FALSE",
+                    "no", "No", "NO",
+                    "f", "F", "n", "N",
+                    "off", "Off", "OFF",
+                    "0");
+
+    private StringCast() {}
+
+    private static String buildErrorMessage(String as, String value)
+    {
+        return String.format("cannot cast String to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(String value) throws DataException
+    {
+        if (TRUE_STRINGS.contains(value)) {
+            return true;
+        }
+        else if (FALSE_STRINGS.contains(value)) {
+            return false;
+        }
+        else {
+            throw new DataException(buildErrorMessage("boolean", value));
+        }
+    }
+
+    public static long asLong(String value) throws DataException
+    {
+        try {
+            return Long.parseLong(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("long", value), ex);
+        }
+    }
+
+    public static double asDouble(String value) throws DataException
+    {
+        try {
+            return Double.parseDouble(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("double", value), ex);
+        }
+    }
+
+    public static String asString(String value) throws DataException
+    {
+        return value;
+    }
+
+    public static Timestamp asTimestamp(String value, TimestampParser parser) throws DataException
+    {
+        try {
+            return parser.parse(value);
+        }
+        catch (TimestampParseException ex) {
+            throw new DataException(buildErrorMessage("timestamp", value), ex);
+        }
+    }
+}

data/src/test/java/org/embulk/parser/jsonpath/TestColumnCaster.java

@@ -0,0 +1,256 @@
+package org.embulk.parser.jsonpath;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.msgpack.value.MapValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestColumnCaster
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    public MapValue mapValue;
+    public DataException thrown;
+    public ScriptingContainer jruby;
+    public TimestampParser parser;
+
+    @Before
+    public void createResource()
+    {
+        jruby = new ScriptingContainer();
+        thrown = new DataException("any");
+        Value[] kvs = new Value[2];
+        kvs[0] = ValueFactory.newString("k");
+        kvs[1] = ValueFactory.newString("v");
+        mapValue = ValueFactory.newMap(kvs);
+        parser = new TimestampParser(jruby, "%Y-%m-%d %H:%M:%S.%N", DateTimeZone.UTC);
+    }
+
+    @Test
+    public void asBooleanFromBoolean()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asBooleanFromInteger()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newInteger(1)));
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newInteger(2));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromFloat()
+    {
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newFloat(1.1));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromString()
+    {
+        assertEquals(true, ColumnCaster.asBoolean(ValueFactory.newString("true")));
+        try {
+            ColumnCaster.asBoolean(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asBooleanFromJson()
+    {
+        try {
+            ColumnCaster.asBoolean(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asLongFromBoolean()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asLongFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asLongFromFloat()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asLongFromString()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1")));
+        try {
+            ColumnCaster.asLong(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asLongFromJson()
+    {
+        try {
+            ColumnCaster.asLong(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asDoubleFromBoolean()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asDoubleFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asDoubleFromFloat()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asDoubleFromString()
+    {
+        assertEquals(1, ColumnCaster.asLong(ValueFactory.newString("1")));
+        try {
+            ColumnCaster.asLong(ValueFactory.newString("foo"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asDoubleFromJson()
+    {
+        try {
+            ColumnCaster.asLong(mapValue);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asStringFromBoolean()
+    {
+        assertEquals("true", ColumnCaster.asString(ValueFactory.newBoolean(true)));
+    }
+
+    @Test
+    public void asStringFromInteger()
+    {
+        assertEquals("1", ColumnCaster.asString(ValueFactory.newInteger(1)));
+    }
+
+    @Test
+    public void asStringFromFloat()
+    {
+        assertEquals("1.5", ColumnCaster.asString(ValueFactory.newFloat(1.5)));
+    }
+
+    @Test
+    public void asStringFromString()
+    {
+        assertEquals("1", ColumnCaster.asString(ValueFactory.newString("1")));
+    }
+
+    @Test
+    public void asStringFromJson()
+    {
+        assertEquals("{\"k\":\"v\"}", ColumnCaster.asString(mapValue));
+    }
+
+    @Test
+    public void asTimestampFromBoolean()
+    {
+        try {
+            ColumnCaster.asTimestamp(ValueFactory.newBoolean(true), parser);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void asTimestampFromInteger()
+    {
+        assertEquals(1, ColumnCaster.asTimestamp(ValueFactory.newInteger(1), parser).getEpochSecond());
+    }
+
+    @Test
+    public void asTimestampFromFloat()
+    {
+        Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000);
+        assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newFloat(1463084053.5), parser));
+    }
+
+    @Test
+    public void asTimestampFromString()
+    {
+        Timestamp expected = Timestamp.ofEpochSecond(1463084053, 500000000);
+        assertEquals(expected, ColumnCaster.asTimestamp(ValueFactory.newString("2016-05-12 20:14:13.5"), parser));
+    }
+
+    @Test
+    public void asTimestampFromJson()
+    {
+        try {
+            ColumnCaster.asTimestamp(mapValue, parser);
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+}