embulk-parser-jsonpath 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8fa00c28b1cddf31875627709899e97cd22e68c2
-  data.tar.gz: 9dce9b4ee36202ef63065b676221622f420091f5
+  metadata.gz: 183e0da4c58eb9f093c8f2f650b3b405e43cf753
+  data.tar.gz: 5bb2a2cfa9045013efc7b1b4dd2b48f03dc6956d
 SHA512:
-  metadata.gz: ce5b684201f61e8afc284c64b32e9470f468c93ea68b739815507b40c4ad69f703c766bb0eb4a834cabc019826bb519d0bb69f9ff9f0c97aa1afdc5ebd7eb2bc
-  data.tar.gz: 20732822287850a87b98828efd4e272f179237dcd0416960a0b066b20578675c36530869fc7d7aa9bc7ed269a7746c6da908d0337c78034aedbe4f7cfb2b86b9
+  metadata.gz: 858fd75dadb25d362e479669df01c68b2b9b521788824da5c48d3f6c684376a8969176f1fcfb8c8c8efcccb3ad38602b79c14ff503ca2b1c87344ef1298573a1
+  data.tar.gz: 17e5e84ccb3e7441a096fc35d451e0c544364021ae048038f8f7f267fff5da4d8ddc5c9bae0a5e3aa5f476438a371b6d9dff60642cdf8b8d76ac09b1a749c00f

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # ChangeLog
 
+## 0.1.3 (2017-03-07)
+
+* Make enable to skip invalid records/columns(@takumakanari)
+
 ## 0.1.2 (2017-02-28)
 
 * Output error detail when json parse failed.

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/parser/jsonpath/JsonpathParserPlugin.java

@@ -1,8 +1,10 @@
 package org.embulk.parser.jsonpath;
 
 import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableMap;
 import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
@@ -17,6 +19,7 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParseException;
 import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.util.FileInputInputStream;
@@ -93,48 +96,63 @@ public class JsonpathParserPlugin
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
             ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, schema, pageBuilder, timestampParsers);
 
-            try (FileInputInputStream is = new FileInputInputStream(input)) {
-                while (is.nextFile()) {
-                    // TODO more efficient handling.
-                    Value value;
-                    String json = JsonPath.read(is, jsonRoot).toString();
+            FileInputInputStream is = new FileInputInputStream(input);
+            while (is.nextFile()) {
+                Value value;
+                try {
+                    String json;
+                    try {
+                        json = JsonPath.read(is, jsonRoot).toString();
+                    }
+                    catch (IOException e) {
+                        throw Throwables.propagate(e);
+                    }
+                    catch (PathNotFoundException e) {
+                        throw new DataException(String.format(Locale.ENGLISH, "Failed to get json root reason = %s",
+                                e.getMessage()));
+                    }
+
                     try {
                         value = jsonParser.parse(json);
                     }
-                    catch (Exception ex) {
-                        logger.error(String.format(Locale.ENGLISH, "Parse failed input data = '%s'", json));
-                        throw new DataException(String.format(Locale.ENGLISH, "Parse failed reason = %s, input data = '%s'", ex.getMessage(), json));
+                    catch (JsonParseException e) {
+                        throw new DataException(String.format(Locale.ENGLISH, "Parse failed reason = %s, input data = '%s'",
+                                e.getMessage(), json));
                     }
 
                     if (!value.isArrayValue()) {
                         throw new JsonRecordValidateException("Json string is not representing array value.");
                     }
+                }
+                catch (DataException e) {
+                    skipOrThrow(e, stopOnInvalidRecord);
+                    continue;
+                }
 
-                    for (Value recordValue : value.asArrayValue()) {
-                        if (!recordValue.isMapValue()) {
-                            if (stopOnInvalidRecord) {
-                                throw new JsonRecordValidateException("Json string is not representing map value.");
-                            }
-                            logger.warn(String.format(ENGLISH, "Skipped invalid record  %s", recordValue));
-                            continue;
-                        }
+                for (Value recordValue : value.asArrayValue()) {
+                    if (!recordValue.isMapValue()) {
+                        skipOrThrow(new JsonRecordValidateException("Json string is not representing map value."),
+                                stopOnInvalidRecord);
+                        continue;
+                    }
 
-                        logger.debug("recordValue = " + recordValue.toString());
-                        final Map<Value, Value> record = recordValue.asMapValue().map();
+                    logger.debug("recordValue = " + recordValue.toString());
+                    final Map<Value, Value> record = recordValue.asMapValue().map();
+                    try {
                         for (Column column : schema.getColumns()) {
                             Value v = record.get(getColumnNameValue(column));
                             visitor.setValue(v);
                             column.visit(visitor);
                         }
-
-                        pageBuilder.addRecord();
                     }
+                    catch (DataException e) {
+                        skipOrThrow(e, stopOnInvalidRecord);
+                        continue;
+                    }
+
+                    pageBuilder.addRecord();
                 }
             }
-            catch (IOException e) {
-                // TODO more efficient exception handling.
-                throw new DataException("catch IOException " + e);
-            }
 
             pageBuilder.finish();
         }
@@ -154,4 +172,12 @@ public class JsonpathParserPlugin
     {
         return columnNameValues.get(column.getName());
     }
+
+    private void skipOrThrow(DataException cause, boolean stopOnInvalidRecord)
+    {
+        if (stopOnInvalidRecord) {
+            throw cause;
+        }
+        logger.warn(String.format(ENGLISH, "Skipped invalid record (%s)", cause));
+    }
 }

data/src/test/java/org/embulk/parser/jsonpath/TestJsonpathParserPlugin.java

@@ -122,7 +122,7 @@ public class TestJsonpathParserPlugin
     }
 
     @Test
-    public void throwDataException()
+    public void skipBrokenJson()
             throws Exception
     {
         SchemaConfig schema = schema(
@@ -130,6 +130,70 @@ public class TestJsonpathParserPlugin
                 column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
         ConfigSource config = this.config.deepCopy().set("columns", schema);
 
+        transaction(config, fileInput("BROKEN"));
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(0, records.size());
+    }
+
+    @Test
+    public void skipBrokenColumn()
+            throws Exception
+    {
+        SchemaConfig schema = schema(column("_c1", TIMESTAMP));
+        ConfigSource config = this.config.deepCopy().set("columns", schema).
+                set("stop_on_invalid_record", false);
+
+        transaction(config, fileInput("{\"_c1\" : \"INVALID\"}"));
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(0, records.size());
+    }
+
+    @Test
+    public void stopOnBrokenJson()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema).
+                set("stop_on_invalid_record", true);
+
+        try {
+            transaction(config, fileInput("BROKEN"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void stopOnBrokenColumn()
+            throws Exception
+    {
+        SchemaConfig schema = schema(column("_c1", TIMESTAMP));
+        ConfigSource config = this.config.deepCopy().set("columns", schema).
+                set("stop_on_invalid_record", true);
+
+        try {
+            transaction(config, fileInput("{\"_c1\" : \"INVALID\"}"));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void throwDataException()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("_c0", BOOLEAN), column("_c1", LONG), column("_c2", DOUBLE),
+                column("_c3", STRING), column("_c4", TIMESTAMP), column("_c5", JSON));
+        ConfigSource config = this.config.deepCopy().set("columns", schema).
+                set("stop_on_invalid_record", true);
+
         try {
             transaction(config, fileInput(
                     "\"not_map_value\""

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-jsonpath
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Hiroyuki Sato
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-28 00:00:00.000000000 Z
+date: 2017-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -110,7 +110,7 @@ files:
 - src/test/java/org/embulk/parser/jsonpath/cast/TestStringCast.java
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
-- classpath/embulk-parser-jsonpath-0.1.2.jar
+- classpath/embulk-parser-jsonpath-0.1.3.jar
 - classpath/json-path-2.2.0.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/slf4j-api-1.7.16.jar