embulk-filter-expand_json 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 718512990aa52e1733712a39c856c355b57516ed
-  data.tar.gz: 30763a006d651e9a71226d9c09748724e902d067
+  metadata.gz: cf8c5e6cae24c44c66c9db8dcef9e9ec14325e05
+  data.tar.gz: 461d263c4d02a2b225ff05199d9a84f7e4ae15ef
 SHA512:
-  metadata.gz: 90bc0426d394ac7d5b56ceab6d5e76a790545322836d6efe1db164df994a831e88845df36de62f351acc5d659daa8759d1bb7344a27bfd11c05404f075262b3e
-  data.tar.gz: afca72527e51e2ce6db90d65d959795a2c14955f95f3d64018a2ffa0855d59e93c8e030eaa6999fda6152daf896838efc4d97efdaf36b99ffb09f09ae4e4c235
+  metadata.gz: f033d39efd8b1532bf305bc338a17686c7195c9d9ec1c5e67f476554ba98326e1f90531794d85f589ad01526cb03a7a499664600832e0091a5b659bddb4a4bbd
+  data.tar.gz: 500294bd0331ca19a6a3c26bf704bbbf0f1694afe5b6a64ff5c54df877a7236e76a26c55b15a2b8cec9081bdb36bcdf95d9f3e25b831db135647f789f741cba0

data/CHANGELOG.md

@@ -1,12 +1,19 @@
+0.1.1 (2016-05-02)
+==================
+- [New Feature] Add keep_expanding_json_column option
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/19
+- [Fix] ix retrieving unchanged json columns
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/17
+
 0.1.0 (2016-04-27)
 ==================
-- [Incompatible Change]: Add stop_on_invalid_record option
+- [Incompatible Change / New Feature]: Add stop_on_invalid_record option
   - https://github.com/civitaspo/embulk-filter-expand_json/pull/15
   - https://github.com/civitaspo/embulk-filter-expand_json/issues/14
 
 0.0.6 (2016-03-17)
 ==================
-- [Add] Support JSON type
+- [New Feature] Support JSON type
   - https://github.com/civitaspo/embulk-filter-expand_json/pull/11
 - [Enhancement] Validate json_column_name
   - https://github.com/civitaspo/embulk-filter-expand_json/pull/9

data/README.md

@@ -17,6 +17,7 @@ expand columns having json into multiple columns
   - **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
   - **type**: type of the column (see below)
   - **format**: format of the timestamp if type is timestamp
+- **keep_expanding_json_column**: Not remove the expanding json column from input schema if it's true (false by default)
 - **stop_on_invalid_record**: Stop bulk load transaction if an invalid record is included (false by default)
 
 ---

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java

@@ -45,6 +45,10 @@ public class ExpandJsonFilterPlugin
         @Config("stop_on_invalid_record")
         @ConfigDefault("false")
         boolean getStopOnInvalidRecord();
+
+        @Config("keep_expanding_json_column")
+        @ConfigDefault("false")
+        public boolean getKeepExpandingJsonColumn();
     }
 
     @Override
@@ -80,10 +84,19 @@ public class ExpandJsonFilterPlugin
         int i = 0; // columns index
         for (Column inputColumn: inputSchema.getColumns()) {
             if (inputColumn.getName().contentEquals(task.getJsonColumnName())) {
-                logger.info("removed column: name: {}, type: {}, index: {}",
+                if (!task.getKeepExpandingJsonColumn()) {
+                    logger.info("removed column: name: {}, type: {}, index: {}",
                             inputColumn.getName(),
                             inputColumn.getType(),
                             inputColumn.getIndex());
+                }
+                else {
+                    logger.info("unchanged expanding column: name: {}, type: {}, index: {}",
+                            inputColumn.getName(),
+                            inputColumn.getType(),
+                            i);
+                    builder.add(new Column(i++, inputColumn.getName(), inputColumn.getType()));
+                }
                 for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
                     logger.info("added column: name: {}, type: {}, options: {}, index: {}",
                                 expandedColumnConfig.getName(),

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -108,6 +108,7 @@ public class FilteredPageOutput
 
     private final Logger logger = Exec.getLogger(FilteredPageOutput.class);
     private final boolean stopOnInvalidRecord;
+    private final boolean keepExpandingJsonColumn;
     private final List<UnchangedColumn> unchangedColumns;
     private final List<ExpandedColumn> expandedColumns;
     private final Column jsonColumn;
@@ -157,7 +158,7 @@ public class FilteredPageOutput
         for (Column outputColumn : outputSchema.getColumns()) {
             for (Column inputColumn : inputSchema.getColumns()) {
                 if (inputColumn.getName().equals(outputColumn.getName()) &&
-                        !excludeColumn.getName().equals(outputColumn.getName())) {
+                        (!excludeColumn.getName().equals(outputColumn.getName()) || keepExpandingJsonColumn)) {
 
                     UnchangedColumn unchangedColumn = new UnchangedColumn(outputColumn.getName(),
                                                                           inputColumn,
@@ -191,6 +192,7 @@ public class FilteredPageOutput
     FilteredPageOutput(PluginTask task, Schema inputSchema, Schema outputSchema, PageOutput pageOutput)
     {
         this.stopOnInvalidRecord = task.getStopOnInvalidRecord();
+        this.keepExpandingJsonColumn = task.getKeepExpandingJsonColumn();
         this.jsonColumn = initializeJsonColumn(task, inputSchema);
         this.unchangedColumns = initializeUnchangedColumns(inputSchema,
                                                            outputSchema,
@@ -261,6 +263,9 @@ public class FilteredPageOutput
             else if (Types.TIMESTAMP.equals(outputColumn.getType())) {
                 pageBuilder.setTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));
             }
+            else { // Json type
+                pageBuilder.setJson(outputColumn, pageReader.getJson(inputColumn));
+            }
         }
     }
 

data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java

@@ -21,6 +21,7 @@ import org.embulk.spi.PageTestUtils;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfigException;
 import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.util.Pages;
 import org.junit.Before;
@@ -183,12 +184,63 @@ public class TestExpandJsonFilterPlugin
         assertEquals("$.", task.getRoot());
         assertEquals("UTC", task.getTimeZone());
         assertEquals("%Y-%m-%d %H:%M:%S.%N %z", task.getDefaultTimestampFormat());
+        assertEquals(false, task.getStopOnInvalidRecord());
+        assertEquals(false, task.getKeepExpandingJsonColumn());
     }
 
     /*
     Expand Test
      */
 
+    @Test
+    public void testUnchangedColumnValues()
+    {
+        String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c6\n" +
+                "root: $.\n" +
+                "expanded_columns:\n" +
+                "  - {name: _e0, type: string}\n";
+        final ConfigSource config = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c0", STRING, "_c1", BOOLEAN, "_c2", DOUBLE,
+                "_c3", LONG, "_c4", TIMESTAMP, "_c5", JSON, "_c6", STRING);
+
+        expandJsonFilterPlugin.transaction(config, schema, new Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                MockPageOutput mockPageOutput = new MockPageOutput();
+
+                try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                            "_v0", // _c0
+                            true,  // _c1
+                            0.2, // _c2
+                            3L,   // _c3
+                            Timestamp.ofEpochSecond(4), // _c4
+                            newMapBuilder().put(s("_e0"), s("_v5")).build(), // _c5
+                            "{\"_e0\":\"_v6\"}")) {
+                        pageOutput.add(page);
+                    }
+
+                    pageOutput.finish();
+                }
+
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+                assertEquals(1, records.size());
+
+                Object[] record = records.get(0);
+                assertEquals("_v0", record[0]);
+                assertEquals(true, record[1]);
+                assertEquals(0.2, (double) record[2], 0.0001);
+                assertEquals(3L, record[3]);
+                assertEquals(Timestamp.ofEpochSecond(4), record[4]);
+                assertEquals(newMapBuilder().put(s("_e0"), s("_v5")).build(), record[5]);
+            }
+        });
+    }
+
     @Test
     public void testStopOnInvalidRecordOption()
     {
@@ -403,6 +455,60 @@ public class TestExpandJsonFilterPlugin
         });
     }
 
+    @Test
+    public void testExpandedJsonValuesWithKeepJsonColumns()
+    {
+        final String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c1\n" +
+                "root: $.\n" +
+                "expanded_columns:\n" +
+                "  - {name: _e0, type: string}\n" +
+                "keep_expanding_json_column: true\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c0", STRING, "_c1", STRING);
+
+        expandJsonFilterPlugin.transaction(config, schema, new Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                MockPageOutput mockPageOutput = new MockPageOutput();
+
+                try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                            "_v0", "{\"_e0\":\"_ev0\"}")) {
+                        pageOutput.add(page);
+                    }
+
+                    pageOutput.finish();
+                }
+
+                assertEquals(3, outputSchema.getColumnCount());
+                Column column;
+                { // 1st column
+                    column = outputSchema.getColumn(0);
+                    assertTrue(column.getName().equals("_c0") && column.getType().equals(STRING));
+                }
+                { // 2nd column
+                    column = outputSchema.getColumn(1);
+                    assertTrue(column.getName().equals("_c1") && column.getType().equals(STRING));
+                }
+                { // 3rd column
+                    column = outputSchema.getColumn(2);
+                    assertTrue(column.getName().equals("_e0") && column.getType().equals(STRING));
+                }
+
+                for (Object[] record : Pages.toObjects(outputSchema, mockPageOutput.pages)) {
+                    assertEquals("_v0", record[0]);
+                    assertEquals("{\"_e0\":\"_ev0\"}", record[1]);
+                    assertEquals("_ev0", record[2]);
+                }
+            }
+        });
+    }
+
     @Test
     public void testExpandSpecialJsonValuesFromString()
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-expand_json
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-27 00:00:00.000000000 Z
+date: 2016-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
-- classpath/embulk-filter-expand_json-0.1.0.jar
+- classpath/embulk-filter-expand_json-0.1.1.jar
 - classpath/json-path-2.2.0.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/slf4j-api-1.7.16.jar