embulk-filter-expand_json 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20479395de0c44c53f19f7f0ef50cb51733d4fd8
-  data.tar.gz: d4c932d30cf21a260b92c20b4ea5a0dfcf8e117c
+  metadata.gz: 1eb2c8defd0d9ac7920f13cdde2f95877f7219e2
+  data.tar.gz: 5cd7424ed6d9863c5572a4b07e3fa83097986ed2
 SHA512:
-  metadata.gz: 1d9de6fe928ef8c0109797af93e4372026a67a376fbe6d3b3ca3da4ad9e97af1a887e5c408740238459031c69af5d78c7fe145ef69517190c701607d8efc5f37
-  data.tar.gz: 6667c87842b18e870b5669cd075dad44e5d6b66b9d769554020025ee486f4883485fedbfcd3a6242e1e0f7c9d94d4da952f676f1d75823c7dc7b28c9e722e020
+  metadata.gz: 46c382f407a962967f65d08a33758380307f2f2403991ad77dded4a68df3e0e88e15a67e0b71b67e7b5c4d76a5a2dc2aa0833ae76dcf078006106b99b6dc0734
+  data.tar.gz: db059276a090fdbddf43ddec272c7cea372aeed6c60cb0b6f00f48084fdc820c370f585bc4abeeedd14662b1a90f1d7c51209510eaa3d246ea21cd484162108e

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+0.2.0 (2017-07-14)
+==================
+- [Incompatible Change]: Remove `time_zone` option, use `default_timezone` instead and column-based timezone.
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/27
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/28
+
+
 0.1.3 (2016-07-27)
 ==================
 - [Enhancement] Improve Exception handling

data/README.md

@@ -18,6 +18,7 @@ expand columns having json into multiple columns
   - **type**: type of the column (see below)
   - **format**: format of the timestamp if type is timestamp
 - **keep_expanding_json_column**: Not remove the expanding json column from input schema if it's true (false by default)
+- **default_timezone**: Time zone of timestamp columns if values don’t include time zone description (`UTC` by default)
 - **stop_on_invalid_record**: Stop bulk load transaction if an invalid record is included (false by default)
 
 ---
@@ -43,7 +44,7 @@ filters:
       - {name: "phone_numbers", type: string}
       - {name: "app_id", type: long}
       - {name: "point", type: double}
-      - {name: "created_at", type: timestamp, format: "%Y-%m-%d"}
+      - {name: "created_at", type: timestamp, format: "%Y-%m-%d", timezone: "UTC"}
       - {name: "profile.anniversary.et", type: string}
       - {name: "profile.anniversary.voluptatem", type: string}
       - {name: "profile.like_words[1]", type: string}

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.1.3"
+version = "0.2.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java

@@ -38,10 +38,7 @@ public class ExpandJsonFilterPlugin
         @Config("expanded_columns")
         public List<ColumnConfig> getExpandedColumns();
 
-        // Time zone of timestamp columns if the value itself doesn’t include time zone description (eg. Asia/Tokyo)
-        @Config("time_zone")
-        @ConfigDefault("\"UTC\"")
-        public String getTimeZone();
+        // default_timezone option from TimestampParser.Task
 
         @Config("stop_on_invalid_record")
         @ConfigDefault("false")
@@ -56,6 +53,11 @@ public class ExpandJsonFilterPlugin
     public void transaction(ConfigSource config, Schema inputSchema,
             FilterPlugin.Control control)
     {
+        // check if deprecated 'time_zone' option is used.
+        if (config.has("time_zone")) {
+            throw new ConfigException("'time_zone' option will be deprecated");
+        }
+
         PluginTask task = config.loadConfig(PluginTask.class);
 
         // check if a column specified as json_column_name option exists or not

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -11,6 +11,8 @@ import com.jayway.jsonpath.JsonPath;
 import com.jayway.jsonpath.Option;
 import com.jayway.jsonpath.ParseContext;
 import com.jayway.jsonpath.ReadContext;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.DataException;
@@ -107,6 +109,19 @@ public class FilteredPageOutput
         }
     }
 
+    private interface TimestampColumnOption
+            extends Task, TimestampParser.TimestampColumnOption
+    {
+    }
+
+    private static TimestampParser createTimestampParser(final PluginTask task,
+                                                         final ColumnConfig columnConfig)
+    {
+        final TimestampColumnOption columnOption = columnConfig.getOption().loadConfig(TimestampColumnOption.class);
+        final String format = columnOption.getFormat().or(task.getDefaultTimestampFormat());
+        final DateTimeZone dateTimeZone = columnOption.getTimeZone().or(task.getDefaultTimeZone());
+        return new TimestampParser(task.getJRuby(), format, dateTimeZone);
+    }
 
     private final Logger logger = Exec.getLogger(FilteredPageOutput.class);
     private final boolean stopOnInvalidRecord;
@@ -130,15 +145,7 @@ public class FilteredPageOutput
 
                     TimestampParser timestampParser = null;
                     if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
-                        String format;
-                        if (expandedColumnConfig.getOption().has("format")) {
-                            format = expandedColumnConfig.getOption().get(String.class, "format");
-                        }
-                        else {
-                            format = task.getDefaultTimestampFormat();
-                        }
-                        DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
-                        timestampParser = new TimestampParser(task.getJRuby(), format, timezone);
+                        timestampParser = createTimestampParser(task, expandedColumnConfig);
                     }
 
                     ExpandedColumn expandedColumn = new ExpandedColumn(outputColumn.getName(),

data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java

@@ -164,6 +164,26 @@ public class TestExpandJsonFilterPlugin
         config.loadConfig(PluginTask.class);
     }
 
+    @Test
+    public void testThrowConfigExceptionIfTimeZoneIsUsed()
+    {
+        String configYaml = "" +
+                "type: expand_json\n" +
+                "time_zone: Asia/Tokyo\n";
+        ConfigSource config = getConfigFromYaml(configYaml);
+        schema = schema("_c0", STRING, "_c1", STRING);
+
+        exception.expect(ConfigException.class);
+        exception.expectMessage("'time_zone' option will be deprecated");
+        expandJsonFilterPlugin.transaction(config, schema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                // do nothing
+            }
+        });
+    }
+
     @Test
     public void testThrowExceptionDuplicatedExpandedColumns()
     {
@@ -227,7 +247,7 @@ public class TestExpandJsonFilterPlugin
         PluginTask task = config.loadConfig(PluginTask.class);
 
         assertEquals("$.", task.getRoot());
-        assertEquals("UTC", task.getTimeZone());
+        assertEquals("UTC", task.getDefaultTimeZone().getID());
         assertEquals("%Y-%m-%d %H:%M:%S.%N %z", task.getDefaultTimestampFormat());
         assertEquals(false, task.getStopOnInvalidRecord());
         assertEquals(false, task.getKeepExpandingJsonColumn());
@@ -407,6 +427,51 @@ public class TestExpandJsonFilterPlugin
         });
     }
 
+    @Test
+    public void testColumnBasedTimezone()
+    {
+        String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c0\n" +
+                "root: $.\n" +
+                "expanded_columns:\n" +
+                "  - {name: _j0, type: timestamp, format: '%Y-%m-%d %H:%M:%S %z'}\n" +
+                "  - {name: _j1, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: 'Asia/Tokyo'}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c0", JSON, "_c1", STRING);
+
+        expandJsonFilterPlugin.transaction(config, schema, new Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                Value data = newMapBuilder()
+                        .put(s("_j0"), s("2014-10-21 04:44:33 +0000"))
+                        .put(s("_j1"), s("2014-10-21 04:44:33"))
+                        .build();
+
+                try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, data, c1Data)) {
+                        pageOutput.add(page);
+                    }
+
+                    pageOutput.finish();
+                }
+
+                PageReader pageReader = new PageReader(outputSchema);
+
+                for (Page page : mockPageOutput.pages) {
+                    pageReader.setPage(page);
+                    assertEquals("2014-10-21 04:44:33 UTC", pageReader.getTimestamp(outputSchema.getColumn(0)).toString());
+                    assertEquals("2014-10-20 19:44:33 UTC", pageReader.getTimestamp(outputSchema.getColumn(1)).toString());
+                    assertEquals(c1Data, pageReader.getString(outputSchema.getColumn(2)));
+                }
+            }
+        });
+    }
+
     @Test
     public void testExpandJsonValuesFromJson()
     {
@@ -414,7 +479,7 @@ public class TestExpandJsonFilterPlugin
                 "type: expand_json\n" +
                 "json_column_name: _c0\n" +
                 "root: $.\n" +
-                "time_zone: Asia/Tokyo\n" +
+                "default_timezone: Asia/Tokyo\n" +
                 "expanded_columns:\n" +
                 "  - {name: _j0, type: boolean}\n" +
                 "  - {name: _j1, type: long}\n" +
@@ -531,7 +596,7 @@ public class TestExpandJsonFilterPlugin
                 "stop_on_invalid_record: 1\n" +
                 "json_column_name: _c0\n" +
                 "root: $.\n" +
-                "time_zone: Asia/Tokyo\n" +
+                "default_timezone: Asia/Tokyo\n" +
                 "expanded_columns:\n" +
                 "  - {name: _j0, type: " + ValidType + "}\n";
 
@@ -679,7 +744,7 @@ public class TestExpandJsonFilterPlugin
                 "type: expand_json\n" +
                 "json_column_name: _c0\n" +
                 "root: $.\n" +
-                "time_zone: Asia/Tokyo\n" +
+                "default_timezone: Asia/Tokyo\n" +
                 "expanded_columns:\n" +
                 "  - {name: _j0, type: boolean}\n" +
                 "  - {name: _j1, type: long}\n" +
@@ -863,7 +928,7 @@ public class TestExpandJsonFilterPlugin
                 "type: expand_json\n" +
                 "json_column_name: _c0\n" +
                 "root: $.\n" +
-                "time_zone: Asia/Tokyo\n" +
+                "default_timezone: Asia/Tokyo\n" +
                 "expanded_columns:\n" +
                 "  - {name: _j0, type: string}\n";
         ConfigSource config = getConfigFromYaml(configYaml);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-expand_json
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-27 00:00:00.000000000 Z
+date: 2017-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,12 +63,12 @@ files:
 - src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java
 - src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java
 - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
-- classpath/accessors-smart-1.1.jar
-- classpath/asm-5.0.3.jar
-- classpath/embulk-filter-expand_json-0.1.3.jar
-- classpath/json-path-2.2.0.jar
-- classpath/json-smart-2.2.1.jar
-- classpath/slf4j-api-1.7.16.jar
+- classpath/accessors-smart-1.2.jar
+- classpath/asm-5.0.4.jar
+- classpath/embulk-filter-expand_json-0.2.0.jar
+- classpath/json-path-2.4.0.jar
+- classpath/json-smart-2.3.jar
+- classpath/slf4j-api-1.7.25.jar
 homepage: https://github.com/civitaspo/embulk-filter-expand_json
 licenses:
 - MIT