embulk-parser-jsonpath 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 183e0da4c58eb9f093c8f2f650b3b405e43cf753
-  data.tar.gz: 5bb2a2cfa9045013efc7b1b4dd2b48f03dc6956d
+  metadata.gz: 5a5c5cf3bdba81a3a1bb16dae74d6c4a7af3379c
+  data.tar.gz: e041030833ba05bcc5967e84d4d0e6604b10094c
 SHA512:
-  metadata.gz: 858fd75dadb25d362e479669df01c68b2b9b521788824da5c48d3f6c684376a8969176f1fcfb8c8c8efcccb3ad38602b79c14ff503ca2b1c87344ef1298573a1
-  data.tar.gz: 17e5e84ccb3e7441a096fc35d451e0c544364021ae048038f8f7f267fff5da4d8ddc5c9bae0a5e3aa5f476438a371b6d9dff60642cdf8b8d76ac09b1a749c00f
+  metadata.gz: 2a532592bc7672a092c43da076b51a0773284a55db1e1b3a4a547247031f981a3acdde9afa53ed5653f4742b94d7abc9ed76bb93ac4cae1ed58bff6eb3242a54
+  data.tar.gz: 428416e6b448f50458c7e36e0b1de078fb110737ae081b3a92e70751639216efd49f91017743ecfdcb20d23bdfab16d168621daf45981b7c8c70318a5af74604

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # ChangeLog
 
+## 0.2.0 (2017-03-13)
+
+* Support `path` parameter in column config. (@takumakanari)
+* Allow some strings convert to boolean.(@takumakanari)
+* Support `schema` parameter for compatibility [embulk-parser-json](https://github.com/takumakanari/embulk-parser-json).
+
 ## 0.1.3 (2017-03-07)
 
 * Make enable to skip invalid records/columns(@takumakanari)

data/README.md

@@ -17,8 +17,19 @@ The JSON with [JSONPath](http://goessner.net/articles/JsonPath/) parser plugin f
 * **default_timestamp_format**: Default timestamp format of the timestamp (string, default: `%Y-%m-%d %H:%M:%S.%N %z`)
 * **default_typecast**: Specify whether to cast values automatically to the specified types or not (boolean, default: true)
 
+### columns
+
+* **name**: Name of the column (string, required)
+* **type**: Type of the column (string, required)
+* **timezone**: Timezone of the timestamp if type is timestamp (string, default: default_timestamp)
+* **format**: Format of the timestamp if type is timestamp (string, default: default_format)
+* **typecast**: Whether cast values or not (boolean, default: default_typecast)
+* **path**: JSON ppath for specific column. (string, default: `null`)
+
 ## Example
 
+### Basic Usage
+
 ```json
 {
   "count": 100,
@@ -88,12 +99,104 @@ registered_at (timestamp) : 2014-06-30 19:25:27 UTC
           age (     long) : 73
         ratio (   double) : 50.608
 ```
+### Handle more complicated json
+
+
+If you want to handle more complicated json, you can specify jsonpath to also **path** in columns section like as follows:
+
+```json
+{
+    "result" : "success",
+    "students" : [
+      { "names" : ["John", "Lennon"], "age" : 10 },
+      { "names" : ["Paul", "Maccartney"], "age" : 10 }
+    ]
+}
+```
+
+```yaml
+root: $.students
+columns:
+  - {name: firstName, type: string, path: "names[0]"}
+  - {name: lastName, type: string, path: "names[1]"}
+```
+
+In this case, names[0] will be firstName of schema and names[1] will be lastName.
+
+## Guess
+
+This plugin supports minimal `guess` feature.  You don't have to write `parser:` section in the configuration file.
+After writing `in:` section, you can let embulk guess `parser:` section using this command:
 
 ```
 $ embulk gem install embulk-parser-jsonpath
 $ embulk guess -g jsonpath config.yml -o guessed.yml
 ```
 
+### Example
+
+If you want to `guess` the following JSON file,
+(This JSON data start with array)
+You don't have to need `parser section`.
+
+```json
+[
+  {
+    "name": "Hugh Rutherford",
+    "city": "Mitchellfurt",
+    "street_name": "Ondricka Island",
+    "zip_code": "75232",
+    "registered_at": "2015-09-09 05:28:45",
+    "vegetarian": true,
+    "age": 44,
+    "ratio": 79.092
+  }
+]
+```
+
+```yaml
+in:
+  type: file
+  path_prefix: example/hoge
+out:
+  type: stdout
+```
+
+However, If a JSON data doesn't start with array,
+You have to specify `root` parameter explicitly.
+
+```json
+{
+  "count": 100,
+  "page": 1,
+  "results": [
+    {
+      "name": "Hugh Rutherford",
+      "city": "Mitchellfurt",
+      "street_name": "Ondricka Island",
+      "zip_code": "75232",
+      "registered_at": "2015-09-09 05:28:45",
+      "vegetarian": true,
+      "age": 44,
+      "ratio": 79.092
+    }
+  ]
+}
+```
+
+
+```yaml
+in:
+  type: file
+  path_prefix: example/input
+  parser:
+    type: jsonpath
+    root: "$.results"
+out:
+  type: stdout
+```
+
+
 ## Build
 
 ```

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.3"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -90,10 +90,10 @@ task gemspec {
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
-  spec.authors       = ["Hiroyuki Sato"]
+  spec.authors       = ["Hiroyuki Sato","Takuma kanari"]
   spec.summary       = %[JSON parser with JSONPath plugin for Embulk]
   spec.description   = %[Parses JSON files with JSONPath read by other file input plugins.]
-  spec.email         = ["hiroysato@gmail.com"]
+  spec.email         = ["hiroysato@gmail.com","chemtrails.t@gmail.com"]
   spec.licenses      = ["MIT"]
   spec.homepage      = "https://github.com/hiroyuki-sato/embulk-parser-jsonpath"
 

data/src/main/java/org/embulk/parser/jsonpath/ColumnVisitorImpl.java

@@ -1,29 +1,44 @@
 package org.embulk.parser.jsonpath;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
 import org.embulk.parser.jsonpath.JsonpathParserPlugin.PluginTask;
 import org.embulk.parser.jsonpath.JsonpathParserPlugin.TypecastColumnOption;
-
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParser;
 import org.msgpack.core.MessageTypeException;
-import org.msgpack.value.Value;
 
-public class ColumnVisitorImpl implements ColumnVisitor
+import java.util.List;
+
+import static java.lang.String.format;
+import static org.msgpack.value.ValueFactory.newBoolean;
+import static org.msgpack.value.ValueFactory.newFloat;
+import static org.msgpack.value.ValueFactory.newInteger;
+import static org.msgpack.value.ValueFactory.newString;
+
+public class ColumnVisitorImpl
+        implements ColumnVisitor
 {
+    private static final JsonParser JSON_PARSER = new JsonParser();
+    private static final List<String> BOOL_TRUE_STRINGS = ImmutableList.of("true", "1", "yes", "on", "y", "t");
+    private static final List<String> BOOL_FALSE_STRINGS = ImmutableList.of("false", "0", "no", "off", "n", "f");
+
     protected final PluginTask task;
     protected final Schema schema;
     protected final PageBuilder pageBuilder;
     protected final TimestampParser[] timestampParsers;
     protected final Boolean[] autoTypecasts;
 
-    protected Value value;
+    protected JsonNode value;
 
     public ColumnVisitorImpl(PluginTask task, Schema schema, PageBuilder pageBuilder, TimestampParser[] timestampParsers)
     {
@@ -41,20 +56,20 @@ public class ColumnVisitorImpl implements ColumnVisitor
             this.autoTypecasts[column.getIndex()] = task.getDefaultTypecast();
         }
 
-//        Optional<SchemaConfig> schemaConfig = task.getColumns();
-        SchemaConfig schemaConfig = task.getSchemaConfig();
+        // typecast option supports `columns` only.
+        Optional<SchemaConfig> schemaConfig = task.getSchemaConfig();
 
-//        if (schemaConfig.isPresent()) {
-        for (ColumnConfig columnConfig : schemaConfig.getColumns()) {
-            TypecastColumnOption columnOption = columnConfig.getOption().loadConfig(TypecastColumnOption.class);
-            Boolean autoTypecast = columnOption.getTypecast().or(task.getDefaultTypecast());
-            Column column = schema.lookupColumn(columnConfig.getName());
-            this.autoTypecasts[column.getIndex()] = autoTypecast;
+        if (schemaConfig.isPresent()) {
+            for (ColumnConfig columnConfig : schemaConfig.get().getColumns()) {
+                TypecastColumnOption columnOption = columnConfig.getOption().loadConfig(TypecastColumnOption.class);
+                Boolean autoTypecast = columnOption.getTypecast().or(task.getDefaultTypecast());
+                Column column = schema.lookupColumn(columnConfig.getName());
+                this.autoTypecasts[column.getIndex()] = autoTypecast;
+            }
         }
-//        }
     }
 
-    public void setValue(Value value)
+    public void setValue(JsonNode value)
     {
         this.value = value;
     }
@@ -64,15 +79,32 @@ public class ColumnVisitorImpl implements ColumnVisitor
     {
         if (isNil(value)) {
             pageBuilder.setNull(column);
+            return;
+        }
+
+        final boolean val;
+        if (value.isBoolean()) {
+            val = value.asBoolean();
         }
         else {
-            try {
-                boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(value) : value.asBooleanValue().getBoolean();
-                pageBuilder.setBoolean(column, booleanValue);
+            String stringValue = valueAsString().toLowerCase();
+            if (BOOL_TRUE_STRINGS.contains(stringValue)) {
+                val = true;
             }
-            catch (MessageTypeException e) {
-                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Boolean", value), e);
+            else if (BOOL_FALSE_STRINGS.contains(stringValue)) {
+                val = false;
             }
+            else {
+                throw new JsonRecordValidateException(format("can not convert '%s' to Boolean", value));
+            }
+        }
+
+        try {
+            boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(newBoolean(val)) : val;
+            pageBuilder.setBoolean(column, booleanValue);
+        }
+        catch (MessageTypeException e) {
+            throw new JsonRecordValidateException(format("failed to get \"%s\" as Boolean", value), e);
         }
     }
 
@@ -84,11 +116,11 @@ public class ColumnVisitorImpl implements ColumnVisitor
         }
         else {
             try {
-                long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(value) : value.asIntegerValue().toLong();
+                long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(newInteger(value.asLong())) : value.asLong();
                 pageBuilder.setLong(column, longValue);
             }
             catch (MessageTypeException e) {
-                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Long", value), e);
+                throw new JsonRecordValidateException(format("failed to get \"%s\" as Long", value), e);
             }
         }
     }
@@ -101,11 +133,11 @@ public class ColumnVisitorImpl implements ColumnVisitor
         }
         else {
             try {
-                double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(value) : value.asFloatValue().toDouble();
+                double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(newFloat(value.asDouble())) : value.asDouble();
                 pageBuilder.setDouble(column, doubleValue);
             }
             catch (MessageTypeException e) {
-                throw new JsonRecordValidateException(String.format("failed get \"%s\" as Double", value), e);
+                throw new JsonRecordValidateException(format("failed get \"%s\" as Double", value), e);
             }
         }
     }
@@ -117,12 +149,13 @@ public class ColumnVisitorImpl implements ColumnVisitor
             pageBuilder.setNull(column);
         }
         else {
+            final String stringValue = valueAsString();
             try {
-                String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(value) : value.asStringValue().toString();
+                String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(newString(stringValue)) : stringValue;
                 pageBuilder.setString(column, string);
             }
             catch (MessageTypeException e) {
-                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as String", value), e);
+                throw new JsonRecordValidateException(format("failed to get \"%s\" as String", value), e);
             }
         }
     }
@@ -135,11 +168,11 @@ public class ColumnVisitorImpl implements ColumnVisitor
         }
         else {
             try {
-                Timestamp timestamp = ColumnCaster.asTimestamp(value, timestampParsers[column.getIndex()]);
+                Timestamp timestamp = ColumnCaster.asTimestamp(newString(value.asText()), timestampParsers[column.getIndex()]);
                 pageBuilder.setTimestamp(column, timestamp);
             }
             catch (MessageTypeException e) {
-                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Timestamp", value), e);
+                throw new JsonRecordValidateException(format("failed to get \"%s\" as Timestamp", value), e);
             }
         }
     }
@@ -152,16 +185,21 @@ public class ColumnVisitorImpl implements ColumnVisitor
         }
         else {
             try {
-                pageBuilder.setJson(column, value);
+                pageBuilder.setJson(column, JSON_PARSER.parse(valueAsString()));
             }
-            catch (MessageTypeException e) {
-                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Json", value), e);
+            catch (MessageTypeException | JsonParseException e) {
+                throw new JsonRecordValidateException(format("failed to get \"%s\" as Json", value), e);
             }
         }
     }
 
-    protected boolean isNil(Value v)
+    protected boolean isNil(JsonNode v)
+    {
+        return v == null || v.isNull();
+    }
+
+    private String valueAsString()
     {
-        return v == null || v.isNilValue();
+        return value.isTextual() ? value.asText() : value.toString();
     }
 }

data/src/main/java/org/embulk/parser/jsonpath/JsonpathParserPlugin.java

@@ -1,16 +1,23 @@
 package org.embulk.parser.jsonpath;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeType;
 import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.InvalidJsonException;
 import com.jayway.jsonpath.JsonPath;
 import com.jayway.jsonpath.PathNotFoundException;
+import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider;
+import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInput;
@@ -19,20 +26,16 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.json.JsonParseException;
-import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.util.FileInputInputStream;
 import org.embulk.spi.util.Timestamps;
-import org.msgpack.value.Value;
 import org.slf4j.Logger;
 
-import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
 
+import static java.lang.String.format;
 import static java.util.Locale.ENGLISH;
-import static org.msgpack.value.ValueFactory.newString;
 
 public class JsonpathParserPlugin
         implements ParserPlugin
@@ -40,14 +43,18 @@ public class JsonpathParserPlugin
 
     private static final Logger logger = Exec.getLogger(JsonpathParserPlugin.class);
 
-    private Map<String, Value> columnNameValues;
+    private static final Configuration JSON_PATH_CONFIG = Configuration
+            .builder()
+            .mappingProvider(new JacksonMappingProvider())
+            .jsonProvider(new JacksonJsonNodeJsonProvider())
+            .build();
 
     public interface TypecastColumnOption
             extends Task
     {
         @Config("typecast")
         @ConfigDefault("null")
-        public Optional<Boolean> getTypecast();
+        Optional<Boolean> getTypecast();
     }
 
     public interface PluginTask
@@ -55,10 +62,16 @@ public class JsonpathParserPlugin
     {
         @Config("root")
         @ConfigDefault("\"$\"")
-        public String getRoot();
+        String getRoot();
 
         @Config("columns")
-        SchemaConfig getSchemaConfig();
+        @ConfigDefault("null")
+        Optional<SchemaConfig> getSchemaConfig();
+
+        @Config("schema")
+        @ConfigDefault("null")
+        @Deprecated
+        Optional<SchemaConfig> getOldSchemaConfig();
 
         @Config("default_typecast")
         @ConfigDefault("true")
@@ -69,12 +82,20 @@ public class JsonpathParserPlugin
         boolean getStopOnInvalidRecord();
     }
 
+    public interface JsonpathColumnOption
+            extends Task
+    {
+        @Config("path")
+        @ConfigDefault("null")
+        Optional<String> getPath();
+    }
+
     @Override
     public void transaction(ConfigSource config, ParserPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        Schema schema = task.getSchemaConfig().toSchema();
+        Schema schema = getSchemaConfig(task).toSchema();
 
         control.run(task.dump(), schema);
     }
@@ -86,11 +107,9 @@ public class JsonpathParserPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
         String jsonRoot = task.getRoot();
 
-        setColumnNameValues(schema);
-
         logger.info("JSONPath = " + jsonRoot);
-        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
-        final JsonParser jsonParser = new JsonParser();
+        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, getSchemaConfig(task));
+        final Map<Column, String> jsonPathMap = createJsonPathMap(task, schema);
         final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -98,59 +117,56 @@ public class JsonpathParserPlugin
 
             FileInputInputStream is = new FileInputInputStream(input);
             while (is.nextFile()) {
-                Value value;
+                final JsonNode json;
                 try {
-                    String json;
-                    try {
-                        json = JsonPath.read(is, jsonRoot).toString();
-                    }
-                    catch (IOException e) {
-                        throw Throwables.propagate(e);
-                    }
-                    catch (PathNotFoundException e) {
-                        throw new DataException(String.format(Locale.ENGLISH, "Failed to get json root reason = %s",
-                                e.getMessage()));
-                    }
-
-                    try {
-                        value = jsonParser.parse(json);
-                    }
-                    catch (JsonParseException e) {
-                        throw new DataException(String.format(Locale.ENGLISH, "Parse failed reason = %s, input data = '%s'",
-                                e.getMessage(), json));
-                    }
-
-                    if (!value.isArrayValue()) {
-                        throw new JsonRecordValidateException("Json string is not representing array value.");
-                    }
+                    json = JsonPath.using(JSON_PATH_CONFIG).parse(is).read(jsonRoot, JsonNode.class);
                 }
-                catch (DataException e) {
-                    skipOrThrow(e, stopOnInvalidRecord);
+                catch (PathNotFoundException e) {
+                    skipOrThrow(new DataException(format(Locale.ENGLISH,
+                            "Failed to get root json path='%s'", jsonRoot)), stopOnInvalidRecord);
+                    continue;
+                }
+                catch (InvalidJsonException e) {
+                    skipOrThrow(new DataException(e), stopOnInvalidRecord);
                     continue;
                 }
 
-                for (Value recordValue : value.asArrayValue()) {
-                    if (!recordValue.isMapValue()) {
-                        skipOrThrow(new JsonRecordValidateException("Json string is not representing map value."),
-                                stopOnInvalidRecord);
-                        continue;
-                    }
+                if (!json.isArray()) {
+                    skipOrThrow(new JsonRecordValidateException(format(Locale.ENGLISH,
+                            "Json string is not representing array value json='%s'", json)), stopOnInvalidRecord);
+                    continue;
+                }
 
-                    logger.debug("recordValue = " + recordValue.toString());
-                    final Map<Value, Value> record = recordValue.asMapValue().map();
+                for (JsonNode recordValue : json) {
                     try {
+                        if (recordValue.getNodeType() != JsonNodeType.OBJECT) {
+                            throw new JsonRecordValidateException(format(Locale.ENGLISH,
+                                    "Json string is not representing map value json='%s'", recordValue));
+                        }
+
                         for (Column column : schema.getColumns()) {
-                            Value v = record.get(getColumnNameValue(column));
-                            visitor.setValue(v);
+                            JsonNode value = null;
+                            if (jsonPathMap.containsKey(column)) {
+                                try {
+                                    value = JsonPath.using(JSON_PATH_CONFIG).parse(recordValue).read(jsonPathMap.get(column));
+                                }
+                                catch (PathNotFoundException e) {
+                                    // pass (value is nullable)
+                                }
+                            }
+                            else {
+                                value = recordValue.get(column.getName());
+                            }
+                            visitor.setValue(value);
                             column.visit(visitor);
                         }
+
+                        pageBuilder.addRecord();
                     }
                     catch (DataException e) {
                         skipOrThrow(e, stopOnInvalidRecord);
                         continue;
                     }
-
-                    pageBuilder.addRecord();
                 }
             }
 
@@ -158,19 +174,17 @@ public class JsonpathParserPlugin
         }
     }
 
-    private void setColumnNameValues(Schema schema)
+    private Map<Column, String> createJsonPathMap(PluginTask task, Schema schema)
     {
-        ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
-        for (Column column : schema.getColumns()) {
-            String name = column.getName();
-            builder.put(name, newString(name));
+        ImmutableMap.Builder<Column, String> builder = ImmutableMap.builder();
+        for (int i = 0; i < schema.size(); i++) {
+            ColumnConfig config = getSchemaConfig(task).getColumn(i);
+            JsonpathColumnOption option = config.getOption().loadConfig(JsonpathColumnOption.class);
+            if (option.getPath().isPresent()) {
+                builder.put(schema.getColumn(i), option.getPath().get());
+            }
         }
-        columnNameValues = builder.build();
-    }
-
-    private Value getColumnNameValue(Column column)
-    {
-        return columnNameValues.get(column.getName());
+        return builder.build();
     }
 
     private void skipOrThrow(DataException cause, boolean stopOnInvalidRecord)
@@ -180,4 +194,19 @@ public class JsonpathParserPlugin
         }
         logger.warn(String.format(ENGLISH, "Skipped invalid record (%s)", cause));
     }
+
+    // this method is to keep the backward compatibility of 'schema' option.
+    private SchemaConfig getSchemaConfig(PluginTask task)
+    {
+        if (task.getSchemaConfig().isPresent()) {
+            return task.getSchemaConfig().get();
+        }
+        else if (task.getOldSchemaConfig().isPresent()) {
+            logger.warn("Please use 'columns' option instead of 'schema' because the 'schema' option is deprecated. The next version will stop 'schema' option support.");
+            return task.getOldSchemaConfig().get();
+        }
+        else {
+            throw new ConfigException("Attribute 'columns' is required but not set");
+        }
+    }
 }

data/src/test/java/org/embulk/parser/jsonpath/TestJsonpathParserPlugin.java

@@ -26,7 +26,6 @@ import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
@@ -37,6 +36,7 @@ import static org.embulk.spi.type.Types.LONG;
 import static org.embulk.spi.type.Types.STRING;
 import static org.embulk.spi.type.Types.TIMESTAMP;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -167,6 +167,51 @@ public class TestJsonpathParserPlugin
         }
     }
 
+    @Test
+    public void booleanStrings()
+            throws Exception
+    {
+        SchemaConfig schema = schema(column("_c1", BOOLEAN), column("_c2", BOOLEAN),
+                column("_c3", BOOLEAN), column("_c4", BOOLEAN), column("_c5", BOOLEAN),
+                column("_c6", BOOLEAN), column("_c7", BOOLEAN), column("_c8", BOOLEAN),
+                column("_c9", BOOLEAN), column("_c10", BOOLEAN), column("_c11", BOOLEAN),
+                column("_c12", BOOLEAN));
+        ConfigSource config = this.config.deepCopy().set("columns", schema);
+
+        transaction(config, fileInput("[{\"_c1\" : \"yes\", \"_c2\" : \"true\", \"_c3\" : \"1\",",
+                "\"_c4\" : \"on\", \"_c5\" : \"y\", \"_c6\" : \"t\",",
+                "\"_c7\" : \"no\", \"_c8\" : \"false\", \"_c9\" : \"0\"," ,
+                "\"_c10\" : \"off\", \"_c11\" : \"n\", \"_c12\" : \"f\"}]"));
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+
+        Object[] record = records.get(0);
+        for (int i = 0; i < 5; i++) {
+            assertTrue((boolean) record[i]);
+        }
+        for (int i = 6; i < 11; i++) {
+            assertFalse((boolean) record[i]);
+        }
+    }
+
+    @Test
+    public void invalidBooleanString()
+            throws Exception
+    {
+        SchemaConfig schema = schema(column("_c1", BOOLEAN));
+        ConfigSource config = this.config.deepCopy().set("columns", schema).
+                set("stop_on_invalid_record", true);
+
+        try {
+            transaction(config,
+                    fileInput("[{\"_c1\" : \"INVALID\"}]"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
     @Test
     public void stopOnBrokenColumn()
             throws Exception
@@ -216,9 +261,9 @@ public class TestJsonpathParserPlugin
 
         transaction(config, fileInput(
                 "[",
-                "{}",
-                "{\"_c0\":null,\"_c1\":null,\"_c2\":null}",
-                "{\"_c3\":null,\"_c4\":null,\"_c5\":null}",
+                "{},",
+                "{\"_c0\":null,\"_c1\":null,\"_c2\":null},",
+                "{\"_c3\":null,\"_c4\":null,\"_c5\":null},",
                 "{}",
                 "]"
         ));
@@ -244,8 +289,8 @@ public class TestJsonpathParserPlugin
 
         transaction(config, fileInput(
                 "[",
-                "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}}",
-                "[1, 2, 3]",
+                "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}},",
+                "[1, 2, 3],",
                 "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}",
                 "]"
         ));
@@ -287,8 +332,8 @@ public class TestJsonpathParserPlugin
 
         transaction(config, fileInput(
                 "{\"records\":[",
-                "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}}",
-                "[1, 2, 3]",
+                "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}},",
+                "[1, 2, 3],",
                 "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}",
                 "]}"
         ));
@@ -319,6 +364,81 @@ public class TestJsonpathParserPlugin
         recreatePageOutput();
     }
 
+    @Test
+    public void useJsonPath()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("__c0", BOOLEAN, config().set("path", "$._c0")), column("__c1", LONG, config().set("path", "$._c1")),
+                column("__c2", DOUBLE, config().set("path", "$._c2")), column("__c3", STRING, config().set("path", "$._c3")),
+                column("__c4", TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S %Z").set("path", "$._c4")),
+                column("__c5", JSON, config().set("path", "$._c5")));
+        ConfigSource config = this.config.deepCopy().set("columns", schema);
+
+        transaction(config, fileInput(
+                "[",
+                "{\"_c0\":true,\"_c1\":10,\"_c2\":0.1,\"_c3\":\"embulk\",\"_c4\":\"2016-01-01 00:00:00 UTC\",\"_c5\":{\"k\":\"v\"}},",
+                "[1, 2, 3],",
+                "{\"_c0\":false,\"_c1\":-10,\"_c2\":1.0,\"_c3\":\"エンバルク\",\"_c4\":\"2016-01-01 00:00:00 +0000\",\"_c5\":[\"e0\",\"e1\"]}",
+                "]"
+        ));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(2, records.size());
+
+        Object[] record;
+        {
+            record = records.get(0);
+            assertEquals(true, record[0]);
+            assertEquals(10L, record[1]);
+            assertEquals(0.1, (Double) record[2], 0.0001);
+            assertEquals("embulk", record[3]);
+            assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]);
+            assertEquals(newMap(newString("k"), newString("v")), record[5]);
+        }
+        {
+            record = records.get(1);
+            assertEquals(false, record[0]);
+            assertEquals(-10L, record[1]);
+            assertEquals(1.0, (Double) record[2], 0.0001);
+            assertEquals("エンバルク", record[3]);
+            assertEquals(Timestamp.ofEpochSecond(1451606400L), record[4]);
+            assertEquals(newArray(newString("e0"), newString("e1")), record[5]);
+        }
+
+        recreatePageOutput();
+    }
+
+    @Test
+    public void writeNilsWithJsonPath()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("__c0", BOOLEAN, config().set("path", "$._c0")), column("__c1", LONG, config().set("path", "$._c1")),
+                column("__c2", DOUBLE, config().set("path", "$._c2")), column("__c3", STRING, config().set("path", "$._c3")),
+                column("__c4", TIMESTAMP, config().set("format", "%Y-%m-%d %H:%M:%S %Z").set("path", "$._c4")),
+                column("__c5", JSON, config().set("path", "$._c5")));
+        ConfigSource config = this.config.deepCopy().set("columns", schema);
+
+        transaction(config, fileInput(
+                "[",
+                "{},",
+                "{\"_c0\":null,\"_c1\":null,\"_c2\":null},",
+                "{\"_c3\":null,\"_c4\":null,\"_c5\":null},",
+                "{}",
+                "]"
+        ));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(4, records.size());
+
+        for (Object[] record : records) {
+            for (int i = 0; i < 6; i++) {
+                assertNull(record[i]);
+            }
+        }
+    }
+
     private FileInput fileInput(String... lines)
             throws Exception
     {

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-jsonpath
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Hiroyuki Sato
+- Takuma kanari
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-07 00:00:00.000000000 Z
+date: 2017-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -69,6 +70,7 @@ dependencies:
 description: Parses JSON files with JSONPath read by other file input plugins.
 email:
 - hiroysato@gmail.com
+- chemtrails.t@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -110,7 +112,7 @@ files:
 - src/test/java/org/embulk/parser/jsonpath/cast/TestStringCast.java
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
-- classpath/embulk-parser-jsonpath-0.1.3.jar
+- classpath/embulk-parser-jsonpath-0.2.0.jar
 - classpath/json-path-2.2.0.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/slf4j-api-1.7.16.jar