embulk-filter-expand_json 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97eabab5d6f09e27e48f707d5c0a791ca6e7e75b
-  data.tar.gz: e69f5076385814eaa587b5ab0c0abdf036e53eaa
+  metadata.gz: 718512990aa52e1733712a39c856c355b57516ed
+  data.tar.gz: 30763a006d651e9a71226d9c09748724e902d067
 SHA512:
-  metadata.gz: 359fe77037b220b4cee280e84c4ecd58374c5f0b22bc6acf868f1969c92168ac6840a2f190ccda81a33ea2afda1f51931ca1a00189a1de9a4b1b9e26c0e9c062
-  data.tar.gz: 190ff72b88236bb85369dafd69b93b80c9c8a08a843057a11cd085ac02e6c89a5f60dd63503cd8a46f9dedb1dbcd8ab5099fc341f86f12136fff8029506c5671
+  metadata.gz: 90bc0426d394ac7d5b56ceab6d5e76a790545322836d6efe1db164df994a831e88845df36de62f351acc5d659daa8759d1bb7344a27bfd11c05404f075262b3e
+  data.tar.gz: afca72527e51e2ce6db90d65d959795a2c14955f95f3d64018a2ffa0855d59e93c8e030eaa6999fda6152daf896838efc4d97efdaf36b99ffb09f09ae4e4c235

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+0.1.0 (2016-04-27)
+==================
+- [Incompatible Change]: Add stop_on_invalid_record option
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/15
+  - https://github.com/civitaspo/embulk-filter-expand_json/issues/14
+
 0.0.6 (2016-03-17)
 ==================
 - [Add] Support JSON type

data/README.md

@@ -17,6 +17,7 @@ expand columns having json into multiple columns
   - **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
   - **type**: type of the column (see below)
   - **format**: format of the timestamp if type is timestamp
+- **stop_on_invalid_record**: Stop bulk load transaction if an invalid record is included (false by default)
 
 ---
 **type of the column**

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.0.6"
+version = "0.1.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java

@@ -41,6 +41,10 @@ public class ExpandJsonFilterPlugin
         @Config("time_zone")
         @ConfigDefault("\"UTC\"")
         public String getTimeZone();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
     }
 
     @Override

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -3,6 +3,7 @@ package org.embulk.filter.expand_json;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Optional;
+import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.jayway.jsonpath.Configuration;
@@ -12,6 +13,7 @@ import com.jayway.jsonpath.ParseContext;
 import com.jayway.jsonpath.ReadContext;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageBuilder;
@@ -105,6 +107,7 @@ public class FilteredPageOutput
 
 
     private final Logger logger = Exec.getLogger(FilteredPageOutput.class);
+    private final boolean stopOnInvalidRecord;
     private final List<UnchangedColumn> unchangedColumns;
     private final List<ExpandedColumn> expandedColumns;
     private final Column jsonColumn;
@@ -187,6 +190,7 @@ public class FilteredPageOutput
 
     FilteredPageOutput(PluginTask task, Schema inputSchema, Schema outputSchema, PageOutput pageOutput)
     {
+        this.stopOnInvalidRecord = task.getStopOnInvalidRecord();
         this.jsonColumn = initializeJsonColumn(task, inputSchema);
         this.unchangedColumns = initializeUnchangedColumns(inputSchema,
                                                            outputSchema,
@@ -202,18 +206,19 @@ public class FilteredPageOutput
     @Override
     public void add(Page page)
     {
-        try {
-            pageReader.setPage(page);
-
-            while (pageReader.nextRecord()) {
+        pageReader.setPage(page);
+        while (pageReader.nextRecord()) {
+            try {
                 setExpandedJsonColumns();
                 setUnchangedColumns();
                 pageBuilder.addRecord();
             }
-        }
-        catch (JsonProcessingException e) {
-            logger.error(e.getMessage());
-            throw Throwables.propagate(e);
+            catch (DataException | JsonProcessingException e) {
+                if (stopOnInvalidRecord) {
+                    throw new DataException(String.format("Found an invalid record"), e);
+                }
+                logger.warn(String.format("Skipped an invalid record (%s)", e.getMessage()));
+            }
         }
     }
 
@@ -266,14 +271,16 @@ public class FilteredPageOutput
         if (pageReader.isNull(jsonColumn)) {
             json = null;
         }
-        else if (jsonColumn.getType() == Types.JSON) {
-            // TODO could use Value object directly and optimize this code
-            String jsonObject = pageReader.getJson(jsonColumn).toJson();
-            json = parseContext.parse(jsonObject);
-        }
-        else { // Types.STRING
-            String jsonObject = pageReader.getString(jsonColumn);
-            json = parseContext.parse(jsonObject);
+        else {
+            String jsonObject;
+            if (jsonColumn.getType().equals(Types.JSON)) {
+                jsonObject = pageReader.getJson(jsonColumn).toJson(); // TODO could use Value object directly and optimize this code
+            }
+            else {
+                jsonObject = pageReader.getString(jsonColumn);
+            }
+
+            json = Strings.isNullOrEmpty(jsonObject) ? null : parseContext.parse(jsonObject);
         }
 
         for (ExpandedColumn expandedJsonColumn: expandedColumns) {

data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java

@@ -12,6 +12,7 @@ import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageOutput;
@@ -21,16 +22,23 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfigException;
 import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
 import org.embulk.spi.type.Type;
+import org.embulk.spi.util.Pages;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.msgpack.value.MapValue;
 import org.msgpack.value.Value;
 
+import java.util.List;
+
 import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.Control;
 import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
 import static org.embulk.spi.type.Types.*;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.msgpack.value.ValueFactory.newArray;
 import static org.msgpack.value.ValueFactory.newBoolean;
 import static org.msgpack.value.ValueFactory.newFloat;
@@ -181,6 +189,73 @@ public class TestExpandJsonFilterPlugin
     Expand Test
      */
 
+    @Test
+    public void testStopOnInvalidRecordOption()
+    {
+        String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c0\n" +
+                "root: $.\n" +
+                "expanded_columns:\n" +
+                "  - {name: _e0, type: json}\n";
+        final ConfigSource conf = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c0", STRING);
+
+        { // stop_on_invalid_record: false
+            ConfigSource config = conf.deepCopy();
+
+            expandJsonFilterPlugin.transaction(config, schema, new Control()
+            {
+                @Override
+                public void run(TaskSource taskSource, Schema outputSchema)
+                {
+                    MockPageOutput mockPageOutput = new MockPageOutput();
+
+                    try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                        for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                                "{\"_e0\":\"\"}", "{\"_e0\":{}}")) {
+                            pageOutput.add(page);
+                        }
+
+                        pageOutput.finish();
+                    }
+
+                    List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+                    assertEquals(1, records.size());
+                    assertEquals(0, ((MapValue) records.get(0)[0]).size()); // {}
+                }
+            });
+        }
+
+        { // stop_on_invalid_record: true
+            ConfigSource config = conf.deepCopy().set("stop_on_invalid_record", true);
+
+            try {
+                expandJsonFilterPlugin.transaction(config, schema, new Control()
+                {
+                    @Override
+                    public void run(TaskSource taskSource, Schema outputSchema)
+                    {
+                        MockPageOutput mockPageOutput = new MockPageOutput();
+
+                        try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                            for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                                    "{\"_e0\":\"\"}", "{\"_e0\":{}}")) {
+                                pageOutput.add(page);
+                            }
+
+                            pageOutput.finish();
+                        }
+                    }
+                });
+                fail();
+            }
+            catch (Throwable t) {
+                assertTrue(t instanceof DataException);
+            }
+        }
+    }
+
     @Test
     public void testExpandJsonKeyToSchema()
     {
@@ -328,6 +403,45 @@ public class TestExpandJsonFilterPlugin
         });
     }
 
+    @Test
+    public void testExpandSpecialJsonValuesFromString()
+    {
+        final String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c1\n" +
+                "root: $.\n" +
+                "expanded_columns:\n" +
+                "  - {name: _e0, type: string}\n" +
+                "  - {name: _e1, type: string}\n"; // the value will be null
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c0", STRING, "_c1", STRING);
+
+        expandJsonFilterPlugin.transaction(config, schema, new Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                MockPageOutput mockPageOutput = new MockPageOutput();
+
+                try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                            "_v0", "")) {
+                        pageOutput.add(page);
+                    }
+
+                    pageOutput.finish();
+                }
+
+                for (Object[] record : Pages.toObjects(outputSchema, mockPageOutput.pages)) {
+                    assertEquals("_v0", record[0]);
+                    assertNull(record[1]);
+                    assertNull(record[2]);
+                }
+            }
+        });
+    }
+
     private static Value s(String value)
     {
         return newString(value);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-expand_json
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-16 00:00:00.000000000 Z
+date: 2016-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
-- classpath/embulk-filter-expand_json-0.0.6.jar
+- classpath/embulk-filter-expand_json-0.1.0.jar
 - classpath/json-path-2.2.0.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/slf4j-api-1.7.16.jar