embulk-filter-expand_json 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07943e1a2bf6a447bccb93cbd9ec0412f00382f0
-  data.tar.gz: 0051fdf7de1702ddd169afb6d9a4fe9ce53e0ded
+  metadata.gz: 97eabab5d6f09e27e48f707d5c0a791ca6e7e75b
+  data.tar.gz: e69f5076385814eaa587b5ab0c0abdf036e53eaa
 SHA512:
-  metadata.gz: 860428fccd8b863e1070ef0e23f657c88e5312031aa222588e51b20c7caa394b2425f26dfc3322fdf69dbc0d3984ff524e3969666cb9a53c68672ead92fd8c25
-  data.tar.gz: d862d2cd98f348242f78a04d5cc2d4117cd4d3923428e181a40b778b509aaf37630abe97158a0f1a8d9351bffcd990aa8675ee5076f522575d86972c8f4bf6d2
+  metadata.gz: 359fe77037b220b4cee280e84c4ecd58374c5f0b22bc6acf868f1969c92168ac6840a2f190ccda81a33ea2afda1f51931ca1a00189a1de9a4b1b9e26c0e9c062
+  data.tar.gz: 190ff72b88236bb85369dafd69b93b80c9c8a08a843057a11cd085ac02e6c89a5f60dd63503cd8a46f9dedb1dbcd8ab5099fc341f86f12136fff8029506c5671

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+0.0.6 (2016-03-17)
+==================
+- [Add] Support JSON type
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/11
+- [Enhancement] Validate json_column_name
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/9
+
 0.0.5 (2016-03-04)
 ==================
 - [Fix] Avoid `NullPointerException` if a column specified as `json_column_name` option doesn't exist in input schema.

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.0.5"
+version = "0.0.6"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.filter.expand_json;
 import com.google.common.collect.ImmutableList;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
@@ -13,6 +14,7 @@ import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.Types;
 import org.slf4j.Logger;
 
 import java.util.List;
@@ -47,8 +49,13 @@ public class ExpandJsonFilterPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        // check if the specified json column exists or not
-        inputSchema.lookupColumn(task.getJsonColumnName());
+        // check if a column specified as json_column_name option exists or not
+        Column jsonColumn = inputSchema.lookupColumn(task.getJsonColumnName());
+        if (jsonColumn.getType() != Types.STRING && jsonColumn.getType() != Types.JSON) {
+            // throws ConfigException if the column is not string or json type.
+            throw new ConfigException(String.format("A column specified as json_column_name option must be string or json type: %s",
+                    new Object[] {jsonColumn.toString()}));
+        }
 
         Schema outputSchema = buildOutputSchema(task, inputSchema);
         control.run(task.dump(), outputSchema);

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -18,6 +18,7 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.type.Types;
 import org.joda.time.DateTimeZone;
@@ -111,6 +112,7 @@ public class FilteredPageOutput
     private final PageBuilder pageBuilder;
     private final ObjectMapper objectMapper = new ObjectMapper();
     private final ParseContext parseContext;
+    private final JsonParser jsonParser = new JsonParser();
 
     private List<ExpandedColumn> initializeExpandedColumns(PluginTask task,
                                                            Schema outputSchema)
@@ -264,7 +266,12 @@ public class FilteredPageOutput
         if (pageReader.isNull(jsonColumn)) {
             json = null;
         }
-        else {
+        else if (jsonColumn.getType() == Types.JSON) {
+            // TODO could use Value object directly and optimize this code
+            String jsonObject = pageReader.getJson(jsonColumn).toJson();
+            json = parseContext.parse(jsonObject);
+        }
+        else { // Types.STRING
             String jsonObject = pageReader.getString(jsonColumn);
             json = parseContext.parse(jsonObject);
         }
@@ -303,6 +310,9 @@ public class FilteredPageOutput
                     throw new RuntimeException("TimestampParser is absent for column:" + expandedJsonColumn.getKey());
                 }
             }
+            else if (Types.JSON.equals(expandedJsonColumn.getColumn().getType())) {
+                pageBuilder.setJson(expandedJsonColumn.getColumn(), jsonParser.parse(finalValue));
+            }
         }
     }
 

data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java

@@ -20,15 +20,24 @@ import org.embulk.spi.PageTestUtils;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfigException;
 import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.type.Type;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.msgpack.value.Value;
 
 import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.Control;
 import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
 import static org.embulk.spi.type.Types.*;
 import static org.junit.Assert.assertEquals;
+import static org.msgpack.value.ValueFactory.newArray;
+import static org.msgpack.value.ValueFactory.newBoolean;
+import static org.msgpack.value.ValueFactory.newFloat;
+import static org.msgpack.value.ValueFactory.newInteger;
+import static org.msgpack.value.ValueFactory.newMap;
+import static org.msgpack.value.ValueFactory.newMapBuilder;
+import static org.msgpack.value.ValueFactory.newString;
 
 public class TestExpandJsonFilterPlugin
 {
@@ -39,16 +48,16 @@ public class TestExpandJsonFilterPlugin
     public ExpectedException exception = ExpectedException.none();
 
 
-    private final Schema schema = Schema.builder()
-            .add("_c0", STRING)
-            .add("_c1", STRING)
-            .build();
     private final String c1Data = "_c1_data";
+    // schema object is recreated per test method. Since each test method might require different schema,
+    // it's better that this field can be overwritten by each method.
+    private Schema schema;
     private ExpandJsonFilterPlugin expandJsonFilterPlugin;
 
     @Before
     public void createResources()
     {
+        schema = schema("_c0", STRING, "_c1", STRING); // default schema
         expandJsonFilterPlugin = new ExpandJsonFilterPlugin();
     }
 
@@ -112,6 +121,27 @@ public class TestExpandJsonFilterPlugin
         });
     }
 
+    @Test
+    public void testThrowExceptionInvalidJsonColumnType()
+    {
+        String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c2\n" +
+                "expanded_columns:\n" +
+                "  - {name: _c1, type: string}";
+        ConfigSource config = getConfigFromYaml(configYaml);
+        schema = schema("_c0", STRING, "_c1", STRING, "_c2", LONG);
+
+        exception.expect(ConfigException.class);
+        expandJsonFilterPlugin.transaction(config, schema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                // do nothing
+            }
+        });
+    }
+
     @Test
     public void testThrowExceptionAbsentExpandedColumns()
     {
@@ -136,7 +166,8 @@ public class TestExpandJsonFilterPlugin
                 "  - {name: _j2, type: long}\n" +
                 "  - {name: _j3, type: timestamp}\n" +
                 "  - {name: _j4, type: double}\n" +
-                "  - {name: _j5, type: string}\n";
+                "  - {name: _j5, type: string}\n" +
+                "  - {name: _j6, type: json}\n";
 
         ConfigSource config = getConfigFromYaml(configYaml);
         PluginTask task = config.loadConfig(PluginTask.class);
@@ -163,6 +194,7 @@ public class TestExpandJsonFilterPlugin
                 "  - {name: _j3, type: timestamp}\n" +
                 "  - {name: _j4, type: double}\n" +
                 "  - {name: _j5, type: string}\n" +
+                "  - {name: _j6, type: json}\n" +
                 "  - {name: _c0, type: string}\n";
 
         ConfigSource config = getConfigFromYaml(configYaml);
@@ -172,15 +204,16 @@ public class TestExpandJsonFilterPlugin
             @Override
             public void run(TaskSource taskSource, Schema outputSchema)
             {
-                assertEquals(7, outputSchema.getColumnCount());
+                assertEquals(8, outputSchema.getColumnCount());
 
                 Column new_j1 = outputSchema.getColumn(0);
                 Column new_j2 = outputSchema.getColumn(1);
                 Column new_j3 = outputSchema.getColumn(2);
                 Column new_j4 = outputSchema.getColumn(3);
                 Column new_j5 = outputSchema.getColumn(4);
-                Column new_c0 = outputSchema.getColumn(5);
-                Column old_c1 = outputSchema.getColumn(6);
+                Column new_j6 = outputSchema.getColumn(5);
+                Column new_c0 = outputSchema.getColumn(6);
+                Column old_c1 = outputSchema.getColumn(7);
 
                 assertEquals("_j1", new_j1.getName());
                 assertEquals(BOOLEAN, new_j1.getType());
@@ -192,17 +225,131 @@ public class TestExpandJsonFilterPlugin
                 assertEquals(DOUBLE, new_j4.getType());
                 assertEquals("_j5", new_j5.getName());
                 assertEquals(STRING, new_j5.getType());
+                assertEquals("_j6", new_j6.getName());
+                assertEquals(JSON, new_j6.getType());
                 assertEquals("_c0", new_c0.getName());
                 assertEquals(STRING, new_c0.getType());
                 assertEquals("_c1", old_c1.getName());
                 assertEquals(STRING, old_c1.getType());
+            }
+        });
+    }
 
+    @Test
+    public void testExpandJsonValuesFromJson()
+    {
+        String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c0\n" +
+                "root: $.\n" +
+                "time_zone: Asia/Tokyo\n" +
+                "expanded_columns:\n" +
+                "  - {name: _j0, type: boolean}\n" +
+                "  - {name: _j1, type: long}\n" +
+                "  - {name: _j2, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
+                "  - {name: _j3, type: double}\n" +
+                "  - {name: _j4, type: string}\n" +
+                "  - {name: _j5, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
+                "  - {name: _j6, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}\n" +
+                // JsonPath: https://github.com/jayway/JsonPath
+                "  - {name: '_j7.store.book[*].author', type: string}\n" +
+                "  - {name: '_j7..book[?(@.price <= $[''_j7''][''expensive''])].author', type: string}\n" +
+                "  - {name: '_j7..book[?(@.isbn)]', type: string}\n" +
+                "  - {name: '_j7..book[?(@.author =~ /.*REES/i)].title', type: string}\n" +
+                "  - {name: '_j7.store.book[2].author', type: string}\n" +
+                "  - {name: _c0, type: string}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c0", JSON, "_c1", STRING);
+
+        expandJsonFilterPlugin.transaction(config, schema, new Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                Value data = newMapBuilder()
+                        .put(s("_j0"), b(true))
+                        .put(s("_j1"), i(2))
+                        .put(s("_j2"), s("2014-10-21 04:44:33 +0900"))
+                        .put(s("_j3"), f(4.4))
+                        .put(s("_j4"), s("v5"))
+                        .put(s("_j5"), s("2014-10-21 04:44:33 +0000"))
+                        .put(s("_j6"), s("2014-10-21 04:44:33"))
+                        .put(s("_j7"), newMapBuilder()
+                                .put(s("store"), newMapBuilder()
+                                        .put(s("book"), newArray(
+                                                newMap(s("author"), s("Nigel Rees"), s("title"), s("Sayings of the Century"), s("price"), f(8.95)),
+                                                newMap(s("author"), s("Evelyn Waugh"), s("title"), s("Sword of Honour"), s("price"), f(12.99)),
+                                                newMap(s("author"), s("Herman Melville"), s("title"), s("Moby Dick"), s("isbn"), s("0-553-21311-3"), s("price"), f(8.99)),
+                                                newMap(s("author"), s("J. R. R. Tolkien"), s("title"), s("The Lord of the Rings"), s("isbn"), s("0-395-19395-8"), s("price"), f(22.99))
+                                        ))
+                                        .put(s("bicycle"), newMap(s("color"), s("red"), s("price"), f(19.95)))
+                                        .build())
+                                .put(s("expensive"), i(10))
+                                .build())
+                        .put(s("_c0"), s("v12"))
+                        .build();
+
+                try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, data, c1Data)) {
+                        pageOutput.add(page);
+                    }
+
+                    pageOutput.finish();
+                }
+
+                PageReader pageReader = new PageReader(outputSchema);
+
+                for (Page page : mockPageOutput.pages) {
+                    pageReader.setPage(page);
+                    assertEquals(true, pageReader.getBoolean(outputSchema.getColumn(0)));
+                    assertEquals(2, pageReader.getLong(outputSchema.getColumn(1)));
+                    assertEquals("2014-10-20 19:44:33 UTC", pageReader.getTimestamp(outputSchema.getColumn(2)).toString());
+                    assertEquals(String.valueOf(4.4), String.valueOf(pageReader.getDouble(outputSchema.getColumn(3))));
+                    assertEquals("v5", pageReader.getString(outputSchema.getColumn(4)));
+                    assertEquals("2014-10-21 04:44:33 UTC", pageReader.getTimestamp(outputSchema.getColumn(5)).toString());
+                    assertEquals("2014-10-20 19:44:33 UTC", pageReader.getTimestamp(outputSchema.getColumn(6)).toString());
+                    assertEquals("[\"Nigel Rees\",\"Evelyn Waugh\",\"Herman Melville\",\"J. R. R. Tolkien\"]",
+                            pageReader.getString(outputSchema.getColumn(7)));
+                    assertEquals("[\"Nigel Rees\",\"Herman Melville\"]", pageReader.getString(outputSchema.getColumn(8)));
+                    assertEquals("" +
+                                    "[" +
+                                    "{\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99}," +
+                                    "{\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}" +
+                                    "]",
+                            pageReader.getString(outputSchema.getColumn(9)));
+                    assertEquals("[\"Sayings of the Century\"]", pageReader.getString(outputSchema.getColumn(10)));
+                    assertEquals("Herman Melville", pageReader.getString(outputSchema.getColumn(11)));
+                    assertEquals("v12", pageReader.getString(outputSchema.getColumn(12)));
+                    assertEquals(c1Data, pageReader.getString(outputSchema.getColumn(13)));
+                }
             }
         });
     }
 
+    private static Value s(String value)
+    {
+        return newString(value);
+    }
+
+    private static Value i(int value)
+    {
+        return newInteger(value);
+    }
+
+    private static Value f(double value)
+    {
+        return newFloat(value);
+    }
+
+    private static Value b(boolean value)
+    {
+        return newBoolean(value);
+    }
+
     @Test
-    public void testExpandJsonValues()
+    public void testExpandJsonValuesFromString()
     {
         String configYaml = "" +
                 "type: expand_json\n" +
@@ -429,4 +576,13 @@ public class TestExpandJsonFilterPlugin
             }
         });
     }
+
+    private static Schema schema(Object... nameAndTypes)
+    {
+        Schema.Builder builder = Schema.builder();
+        for (int i = 0; i < nameAndTypes.length; i += 2) {
+            builder.add((String) nameAndTypes[i], (Type) nameAndTypes[i + 1]);
+        }
+        return builder.build();
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-expand_json
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-04 00:00:00.000000000 Z
+date: 2016-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
-- classpath/embulk-filter-expand_json-0.0.5.jar
+- classpath/embulk-filter-expand_json-0.0.6.jar
 - classpath/json-path-2.2.0.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/slf4j-api-1.7.16.jar