embulk-filter-expand_json 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4eb62c73f09245613f9c6950d72f840434d6cc16
-  data.tar.gz: b46a7998eeab53f56fe524a6746d9c266ee45a53
+  metadata.gz: 20479395de0c44c53f19f7f0ef50cb51733d4fd8
+  data.tar.gz: d4c932d30cf21a260b92c20b4ea5a0dfcf8e117c
 SHA512:
-  metadata.gz: dd2395c19f4113da5d4c98624ebc8d4c3a2b268bb68a9e0091cc4955ea5fb1696665c1541d9bcf183ba6b8e9b2f76291c340face70e17c6ffbfdd949024beb31
-  data.tar.gz: d1005e506f9f61dcb06e08da8c396983ed04797a25d8d5add72828a14fe19ee1b77ec7c391dfd0518d6f1717a1e1c436769ddedc413ad068e1b343cee2102124
+  metadata.gz: 1d9de6fe928ef8c0109797af93e4372026a67a376fbe6d3b3ca3da4ad9e97af1a887e5c408740238459031c69af5d78c7fe145ef69517190c701607d8efc5f37
+  data.tar.gz: 6667c87842b18e870b5669cd075dad44e5d6b66b9d769554020025ee486f4883485fedbfcd3a6242e1e0f7c9d94d4da952f676f1d75823c7dc7b28c9e722e020

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.1.3 (2016-07-27)
+==================
+- [Enhancement] Improve Exception handling
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/23
+
 0.1.2 (2016-05-31)
 ==================
 - [Enhancement] Add a validation to check column duplication

data/README.md

@@ -69,3 +69,4 @@ $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ## Contributor
 - @Civitaspo
 - @muga
+- @sakama

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -20,7 +20,9 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParseException;
 import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.TimestampParseException;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.type.Types;
 import org.joda.time.DateTimeZone;
@@ -308,22 +310,42 @@ public class FilteredPageOutput
                 pageBuilder.setBoolean(expandedJsonColumn.getColumn(), Boolean.parseBoolean(finalValue));
             }
             else if (Types.DOUBLE.equals(expandedJsonColumn.getColumn().getType())) {
-                pageBuilder.setDouble(expandedJsonColumn.getColumn(), Double.parseDouble(finalValue));
+                try {
+                    pageBuilder.setDouble(expandedJsonColumn.getColumn(), Double.parseDouble(finalValue));
+                }
+                catch (NumberFormatException e) {
+                    throw new JsonValueInvalidException(String.format("Failed to parse '%s' as double", finalValue), e);
+                }
             }
             else if (Types.LONG.equals(expandedJsonColumn.getColumn().getType())) {
-                pageBuilder.setLong(expandedJsonColumn.getColumn(), Long.parseLong(finalValue));
+                try {
+                    pageBuilder.setLong(expandedJsonColumn.getColumn(), Long.parseLong(finalValue));
+                }
+                catch (NumberFormatException e) {
+                    throw new JsonValueInvalidException(String.format("Failed to parse '%s' as long", finalValue), e);
+                }
             }
             else if (Types.TIMESTAMP.equals(expandedJsonColumn.getColumn().getType())) {
                 if (expandedJsonColumn.getTimestampParser().isPresent()) {
                     TimestampParser parser = expandedJsonColumn.getTimestampParser().get();
-                    pageBuilder.setTimestamp(expandedJsonColumn.getColumn(), parser.parse(finalValue));
+                    try {
+                        pageBuilder.setTimestamp(expandedJsonColumn.getColumn(), parser.parse(finalValue));
+                    }
+                    catch (TimestampParseException e) {
+                        throw new JsonValueInvalidException(String.format("Failed to parse '%s' as timestamp", finalValue), e);
+                    }
                 }
                 else {
                     throw new RuntimeException("TimestampParser is absent for column:" + expandedJsonColumn.getKey());
                 }
             }
             else if (Types.JSON.equals(expandedJsonColumn.getColumn().getType())) {
-                pageBuilder.setJson(expandedJsonColumn.getColumn(), jsonParser.parse(finalValue));
+                try {
+                    pageBuilder.setJson(expandedJsonColumn.getColumn(), jsonParser.parse(finalValue));
+                }
+                catch (JsonParseException e) {
+                    throw new JsonValueInvalidException(String.format("Failed to parse '%s' as JSON", finalValue), e);
+                }
             }
         }
     }
@@ -347,4 +369,12 @@ public class FilteredPageOutput
             return String.valueOf(value);
         }
     }
+
+    private class JsonValueInvalidException extends DataException
+    {
+        JsonValueInvalidException(String message, Throwable cause)
+        {
+            super(message, cause);
+        }
+    }
 }

data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java

@@ -500,6 +500,65 @@ public class TestExpandJsonFilterPlugin
         });
     }
 
+    @Test(expected = DataException.class)
+    public void testSetExpandedJsonColumnsSetInvalidDoubleValue()
+    {
+        setExpandedJsonColumnsWithInvalidValue("double", s("abcde"));
+    }
+
+    @Test(expected = DataException.class)
+    public void testSetExpandedJsonColumnsSetInvalidLongValue()
+    {
+        setExpandedJsonColumnsWithInvalidValue("long", s("abcde"));
+    }
+
+    @Test(expected = DataException.class)
+    public void testSetExpandedJsonColumnsSetInvalidTimestampValue()
+    {
+        setExpandedJsonColumnsWithInvalidValue("timestamp", s("abcde"));
+    }
+
+    @Test(expected = DataException.class)
+    public void testSetExpandedJsonColumnsSetInvalidJsonValue()
+    {
+        setExpandedJsonColumnsWithInvalidValue("json", s("abcde"));
+    }
+
+    public void setExpandedJsonColumnsWithInvalidValue(String ValidType, final Value invalidValue)
+    {
+        String configYaml = "" +
+                "type: expand_json\n" +
+                "stop_on_invalid_record: 1\n" +
+                "json_column_name: _c0\n" +
+                "root: $.\n" +
+                "time_zone: Asia/Tokyo\n" +
+                "expanded_columns:\n" +
+                "  - {name: _j0, type: " + ValidType + "}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c0", JSON, "_c1", STRING);
+
+        expandJsonFilterPlugin.transaction(config, schema, new Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                Value data = newMapBuilder()
+                        .put(s("_j0"), invalidValue)
+                        .build();
+
+                try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, data, c1Data)) {
+                        pageOutput.add(page);
+                    }
+
+                    pageOutput.finish();
+                }
+            }
+        });
+    }
+
     @Test
     public void testExpandedJsonValuesWithKeepJsonColumns()
     {

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-expand_json
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-31 00:00:00.000000000 Z
+date: 2016-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Expand Json
 email:
 - civitaspo@gmail.com
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
-- classpath/embulk-filter-expand_json-0.1.2.jar
+- classpath/embulk-filter-expand_json-0.1.3.jar
 - classpath/json-path-2.2.0.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/slf4j-api-1.7.16.jar