embulk-filter-add_time 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47c3533da9505293335ec892ae49e06b462f3b39
-  data.tar.gz: 3332208c1f09e1b0da0d2d9638730083f679159d
+  metadata.gz: 7ebbd2e05d327a034281cd755e19c4a34111aaaf
+  data.tar.gz: 17af5b3075faef634c82c6ca61ad76c294ab457d
 SHA512:
-  metadata.gz: bcd078c499649c2ae4be4e2c527626058cf9cad68511385e269535b25835dde192e2744340014eef1833aaeb1848e31d0c1447c6af8286641d935e7795ec91aa
-  data.tar.gz: 64cfc9c742ff32955508ffe75dd8a83e393102bcb6c80db7b45bf0869a5e6b5863d0d34e8694d596c5e12ff157c60256e1d807e731ef6707fe11b3d755e3adfb
+  metadata.gz: 549eaf5a786b28f306432c75cec775d578d4a2fcf664a74636e6d2a48381e971cd28c5c4cc69772d6aad1738bb2f183481e6b6644c6c3f462d0149a26d43b739
+  data.tar.gz: 52a492c8452c61db9b8076417798555bdf0c25e08d0107b5ceb1b34eff0771b21a3a3ca4c144f99213ba92d6d188ae804b1adf7c96280d451627d7d1374e03e1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.0 - 2016-02-22
+
+* [new feature] Support json_key sub option in from_column option [#7](https://github.com/treasure-data/embulk-filter-add_time/pull/7)
+* [maintenance] Upgrade Embulk v08 [#6](https://github.com/treasure-data/embulk-filter-add_time/pull/6)
+
 ## 0.1.1 - 2016-02-05
 
 * [maintenance] Throw ConfigException if from_column doesn't exist in input schema [#5](https://github.com/treasure-data/embulk-filter-add_time/pull/5)

data/build.gradle

@@ -15,19 +15,19 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.2.0"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.10"
-    provided "org.embulk:embulk-core:0.7.10"
+    compile  "org.embulk:embulk-core:0.8.5"
+    provided "org.embulk:embulk-core:0.8.5"
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-standards:0.7.10"
-    testCompile "org.embulk:embulk-core:0.7.10:tests"
+    testCompile "org.embulk:embulk-standards:0.8.5"
+    testCompile "org.embulk:embulk-core:0.8.5:tests"
     testCompile "org.mockito:mockito-core:1.9.5"
 }
 

data/src/main/java/org/embulk/filter/add_time/AddTimeFilterPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.filter.add_time;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
 import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -69,6 +68,10 @@ public class AddTimeFilterPlugin
         @Config("default_timestamp_format")
         @ConfigDefault("\"%Y-%m-%d %H:%M:%S %z\"") // override default value
         String getDefaultTimestampFormat();
+
+        @Config("json_key")
+        @ConfigDefault("null")
+        Optional<String> getJsonKey();
     }
 
     public interface FromValueConfig

data/src/main/java/org/embulk/filter/add_time/converter/JsonValueCastConverter.java

@@ -0,0 +1,84 @@
+package org.embulk.filter.add_time.converter;
+
+import org.embulk.filter.add_time.AddTimeFilterPlugin.FromColumnConfig;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
+import org.embulk.filter.add_time.AddTimeFilterPlugin.UnixTimestampUnit;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.time.TimestampParser;
+import org.msgpack.core.MessagePackException;
+import org.msgpack.value.Value;
+
+import java.util.Map;
+
+import static org.msgpack.value.ValueFactory.newString;
+
+public class JsonValueCastConverter
+        extends ValueCastConverter
+{
+    private final Value jsonKey;
+    private final TimestampParser fromTimestampParser; // for string value
+    private final UnixTimestampUnit fromUnixTimestampUnit; // for long value
+
+    public JsonValueCastConverter(FromColumnConfig fromColumnConfig, ToColumnConfig toColumnConfig)
+    {
+        super(toColumnConfig);
+        this.jsonKey = newString(fromColumnConfig.getJsonKey().get());
+        this.fromTimestampParser = new TimestampParser(fromColumnConfig, fromColumnConfig);
+        this.fromUnixTimestampUnit = UnixTimestampUnit.of(fromColumnConfig.getUnixTimestampUnit());
+    }
+
+    @Override
+    public void convertValue(final Column column, Value value, final PageBuilder pageBuilder)
+    {
+        try {
+            if (!value.isMapValue()) {
+                throw new InvalidCastException("The value must be map object.");
+            }
+
+            Map<Value, Value> map = value.asMapValue().map();
+            if (!map.containsKey(jsonKey)) {
+                throw new InvalidCastException("This record doesn't have a key specified as json_key.");
+            }
+
+            Value v = map.get(jsonKey);
+            if (v.isStringValue()) {
+                columnVisitor.setValue(stringToTimestamp(v));
+            }
+            else if (v.isIntegerValue()) {
+                columnVisitor.setValue(longToTimestamp(v));
+            }
+            else {
+                throw new InvalidCastException(String.format(
+                        "The value of a key specified as json_key must be long or string type. But it's %s", value.getValueType().name()));
+            }
+
+            columnVisitor.setPageBuilder(pageBuilder);
+            column.visit(columnVisitor);
+        }
+        catch (InvalidCastException | TimestampParseException | MessagePackException e) {
+            log.warn(String.format("Cannot convert (%s): %s", e.getMessage(), value.toJson()));
+            pageBuilder.setNull(column);
+        }
+    }
+
+    private Timestamp stringToTimestamp(Value value)
+    {
+        return fromTimestampParser.parse(value.asStringValue().toString());
+    }
+
+    private Timestamp longToTimestamp(Value value)
+    {
+        return fromUnixTimestampUnit.toTimestamp(value.asIntegerValue().toLong());
+    }
+
+    static class InvalidCastException
+            extends RuntimeException
+    {
+        InvalidCastException(String message) {
+            super(message);
+        }
+    }
+}

data/src/main/java/org/embulk/filter/add_time/converter/SchemaConverter.java

@@ -9,6 +9,7 @@ import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
 import org.embulk.filter.add_time.reader.BooleanColumnReader;
 import org.embulk.filter.add_time.reader.ColumnReader;
 import org.embulk.filter.add_time.reader.DoubleColumnReader;
+import org.embulk.filter.add_time.reader.JsonColumnReader;
 import org.embulk.filter.add_time.reader.LongColumnReader;
 import org.embulk.filter.add_time.reader.StringColumnReader;
 import org.embulk.filter.add_time.reader.TimeValueGenerator;
@@ -21,6 +22,7 @@ import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.type.BooleanType;
 import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.JsonType;
 import org.embulk.spi.type.LongType;
 import org.embulk.spi.type.StringType;
 import org.embulk.spi.type.TimestampType;
@@ -28,6 +30,11 @@ import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
 import org.slf4j.Logger;
 
+import static org.embulk.spi.type.Types.JSON;
+import static org.embulk.spi.type.Types.LONG;
+import static org.embulk.spi.type.Types.STRING;
+import static org.embulk.spi.type.Types.TIMESTAMP;
+
 public class SchemaConverter
 {
     private final Logger log;
@@ -72,9 +79,9 @@ public class SchemaConverter
             }
 
             if (fromColumnConfig.isPresent() && columnName.equals(fromColumnConfig.get().getName())) {
-                if (!columnType.equals(Types.LONG) && !columnType.equals(Types.STRING) && !columnType.equals(Types.TIMESTAMP)) {
+                if (!(columnType.equals(LONG) || columnType.equals(STRING) || columnType.equals(TIMESTAMP) || columnType.equals(JSON))) {
                     throw new ConfigException(String.format(
-                            "The type of the '%s' column specified as from_column must be long, string or timestamp. But it's %s.", columnName, columnType));
+                            "The type of the '%s' column specified as from_column must be long, string, timestamp or json. But it's %s.", columnName, columnType));
                 }
 
                 ColumnReader duplicatee = newColumnReader(columnType, newValueCastConverter(columnType, fromColumnConfig, toColumnConfig));
@@ -152,7 +159,9 @@ public class SchemaConverter
         else if (columnType instanceof TimestampType) {
             return new TimestampColumnReader(valueConverter);
         }
-        // TODO support Json type
+        else if (columnType instanceof JsonType) {
+            return new JsonColumnReader(valueConverter);
+        }
         else {
             throw new ConfigException("Unsupported type: " + columnType); // TODO after json type support, it should be changed to AssertionError.
         }
@@ -166,6 +175,9 @@ public class SchemaConverter
         else if (columnType instanceof StringType) {
             return new StringValueCastConverter(fromColumnConfig.get(), toColumnConfig);
         }
+        else if (columnType instanceof JsonType) {
+            return new JsonValueCastConverter(fromColumnConfig.get(), toColumnConfig);
+        }
         else if (columnType instanceof TimestampType) {
             return new TimestampValueCastConverter(toColumnConfig);
         }
@@ -218,6 +230,12 @@ public class SchemaConverter
                     updateColumn(column, pageReader);
                 }
 
+                @Override
+                public void jsonColumn(Column column)
+                {
+                    updateColumn(column, pageReader);
+                }
+
                 @Override
                 public void timestampColumn(Column column)
                 {

data/src/main/java/org/embulk/filter/add_time/converter/ValueCastConverter.java

@@ -4,16 +4,21 @@ import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
 import org.embulk.filter.add_time.AddTimeFilterPlugin.UnixTimestampUnit;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+import org.slf4j.Logger;
 
 public abstract class ValueCastConverter
         implements ValueConverter
 {
+    protected final Logger log;
     protected final TimestampValueCastVisitor columnVisitor;
 
     public ValueCastConverter(ToColumnConfig toColumnConfig)
     {
+        this.log = Exec.getLogger(ValueCastConverter.class);
         this.columnVisitor = new TimestampValueCastVisitor(UnixTimestampUnit.of(toColumnConfig.getUnixTimestampUnit()));
     }
 
@@ -47,6 +52,12 @@ public abstract class ValueCastConverter
         throw new AssertionError("Should implement in subclass.");
     }
 
+    @Override
+    public void convertValue(final Column column, Value value, final PageBuilder pageBuilder)
+    {
+        throw new AssertionError("Should implement in subclass.");
+    }
+
     @Override
     public void convertValue(Column column, final Timestamp value, final PageBuilder pageBuilder)
     {
@@ -99,6 +110,12 @@ public abstract class ValueCastConverter
             throw new AssertionError("Never call.");
         }
 
+        @Override
+        public void jsonColumn(Column column)
+        {
+            throw new AssertionError("Never call.");
+        }
+
         @Override
         public void timestampColumn(Column column)
         {

data/src/main/java/org/embulk/filter/add_time/converter/ValueConverter.java

@@ -3,6 +3,7 @@ package org.embulk.filter.add_time.converter;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
 
 public interface ValueConverter
 {
@@ -18,5 +19,7 @@ public interface ValueConverter
 
     void convertValue(Column column, String value, PageBuilder pageBuilder);
 
+    void convertValue(Column column, Value value, PageBuilder pageBuilder);
+
     void convertValue(Column column, Timestamp value, PageBuilder pageBuilder);
 }

data/src/main/java/org/embulk/filter/add_time/converter/ValueNoConverter.java

@@ -3,6 +3,7 @@ package org.embulk.filter.add_time.converter;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
 
 public class ValueNoConverter
         implements ValueConverter
@@ -37,6 +38,12 @@ public class ValueNoConverter
         pageBuilder.setString(column, value);
     }
 
+    @Override
+    public void convertValue(Column column, Value value, PageBuilder pageBuilder)
+    {
+        pageBuilder.setJson(column, value);
+    }
+
     @Override
     public void convertValue(Column column, Timestamp value, PageBuilder pageBuilder)
     {

data/src/main/java/org/embulk/filter/add_time/reader/JsonColumnReader.java

@@ -0,0 +1,36 @@
+package org.embulk.filter.add_time.reader;
+
+import org.embulk.filter.add_time.converter.ValueConverter;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.msgpack.value.Value;
+
+public class JsonColumnReader
+        extends AbstractColumnReader<JsonColumnReader>
+{
+    protected Value value;
+
+    public JsonColumnReader(ValueConverter valueConverter)
+    {
+        super(valueConverter);
+    }
+
+    @Override
+    public void readNonNullValue(Column column, PageReader pageReader)
+    {
+        value = pageReader.getJson(column);
+    }
+
+    @Override
+    public void convertNonNullValue(Column column, PageBuilder pageBuilder)
+    {
+        valueConverter.convertValue(column, value, pageBuilder);
+    }
+
+    @Override
+    public void copyValueTo(JsonColumnReader columnReader)
+    {
+        columnReader.value = this.value;
+    }
+}

data/src/test/java/org/embulk/filter/add_time/TestAddTimeFilterPlugin.java

@@ -18,11 +18,16 @@ import org.embulk.spi.util.Pages;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.msgpack.value.Value;
 
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.mockito.Mockito.spy;
+import static org.msgpack.value.ValueFactory.newInteger;
+import static org.msgpack.value.ValueFactory.newMap;
+import static org.msgpack.value.ValueFactory.newString;
 
 public class TestAddTimeFilterPlugin
 {
@@ -39,7 +44,13 @@ public class TestAddTimeFilterPlugin
     {
         plugin = plugin();
         config = runtime.getExec().newConfigSource();
-        inputSchema = schema("c0", Types.BOOLEAN, "c1", Types.LONG, "c2", Types.DOUBLE, "c3", Types.STRING, "c4", Types.TIMESTAMP);
+        inputSchema = schema("c0", Types.BOOLEAN, "c1", Types.LONG, "c2", Types.DOUBLE, "c3",
+                Types.STRING, "c4", Types.TIMESTAMP, "c5", Types.JSON);
+    }
+
+    private Value newSimpleMap()
+    {
+        return newMap(newString("k"), newString("v"));
     }
 
     @Test
@@ -49,7 +60,7 @@ public class TestAddTimeFilterPlugin
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_column", ImmutableMap.of("name", "c1", "unix_timestamp_unit", "sec"));
-            List<Page> pages = newPages(true, 1451646671L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 1451646671L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -62,7 +73,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
 
@@ -70,7 +82,7 @@ public class TestAddTimeFilterPlugin
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_column", ImmutableMap.of("name", "c4"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -83,7 +95,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
 
@@ -91,7 +104,7 @@ public class TestAddTimeFilterPlugin
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_column", ImmutableMap.of("name", "c3"));
-            List<Page> pages = newPages(true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -104,7 +117,45 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("2016-01-01 11:11:11 UTC", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
+            }
+        }
+
+        { // json type
+            ConfigSource conf = this.config.deepCopy()
+                    .set("to_column", ImmutableMap.of("name", "time"))
+                    .set("from_column", ImmutableMap.of(
+                            "name", "c5",
+                            "json_key", "k",
+                            "unix_timestamp_unit", "sec",
+                            "timestamp_format", "%Y-%m-%d %H:%M:%S %z"
+                    ));
+
+            List<Page> pages = newPages(
+                    true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newString("2016-01-01 11:11:11 UTC")),
+                    true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newString("embulk")),
+                    true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newInteger(1451646671))
+            );
+
+            callTansaction(conf, inputSchema, pages);
+
+            assertEquals(3, records.size());
+            Object[] record;
+            {
+                record = records.get(0);
+                assertEquals(newMap(newString("k"), newString("2016-01-01 11:11:11 UTC")), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
+            }
+            {
+                record = records.get(1);
+                assertEquals(newMap(newString("k"), newString("embulk")), record[5]);
+                assertNull(record[6]);
+            }
+            {
+                record = records.get(2);
+                assertEquals(newMap(newString("k"), newInteger(1451646671)), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
     }
@@ -117,7 +168,7 @@ public class TestAddTimeFilterPlugin
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -130,14 +181,15 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
         { // specifies timestamp_format
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11.000 UTC", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -150,14 +202,15 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
         { // specifies unix_timestamp_unit
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", 1451646671, "unix_timestamp_unit", "sec"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -170,7 +223,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
 
@@ -180,7 +234,7 @@ public class TestAddTimeFilterPlugin
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "incremental_time",
                             "from", "2016-01-01 11:11:11 UTC", "to", "2016-01-01 11:11:12 UTC"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -193,7 +247,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
         { // specifies timestamp_format
@@ -201,7 +256,7 @@ public class TestAddTimeFilterPlugin
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "incremental_time",
                             "from", "2016-01-01 11:11:11.000 UTC", "to", "2016-01-01 11:11:12.000 UTC", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -214,7 +269,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
         { // specifies unix_timestamp_unit
@@ -222,7 +278,7 @@ public class TestAddTimeFilterPlugin
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "incremental_time",
                             "from", 1451646671, "to", 1451646672, "unix_timestamp_unit", "sec"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -235,7 +291,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
 
@@ -244,7 +301,7 @@ public class TestAddTimeFilterPlugin
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "upload_time"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -257,7 +314,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(runtime.getExec().getTransactionTime(), record[5]);
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(runtime.getExec().getTransactionTime(), record[6]);
             }
         }
     }
@@ -270,7 +328,7 @@ public class TestAddTimeFilterPlugin
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -283,7 +341,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
             }
         }
 
@@ -292,7 +351,7 @@ public class TestAddTimeFilterPlugin
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "sec"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -305,14 +364,15 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671L, record[5]);
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671L, record[6]);
             }
         }
         { // unix_timestamp: milli
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "milli"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -325,14 +385,15 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671000L, record[5]);
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671000L, record[6]);
             }
         }
         { // unix_timestamp: micro
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "micro"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -345,14 +406,15 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671000000L, record[5]);
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671000000L, record[6]);
             }
         }
         { // unix_timestamp: nano
             ConfigSource conf = this.config.deepCopy()
                     .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "nano"))
                     .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
-            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
+            List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
 
             callTansaction(conf, inputSchema, pages);
 
@@ -365,7 +427,8 @@ public class TestAddTimeFilterPlugin
                 assertEquals(0.1, record[2]);
                 assertEquals("foo", record[3]);
                 assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
-                assertEquals(1451646671000000000L, record[5]);
+                assertEquals(newSimpleMap(), record[5]);
+                assertEquals(1451646671000000000L, record[6]);
             }
         }
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-add_time
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-05 00:00:00.000000000 Z
+date: 2016-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,6 +61,7 @@ files:
 - src/main/java/org/embulk/filter/add_time/AddTimeFilterPlugin.java
 - src/main/java/org/embulk/filter/add_time/converter/ColumnConverter.java
 - src/main/java/org/embulk/filter/add_time/converter/ColumnDuplicator.java
+- src/main/java/org/embulk/filter/add_time/converter/JsonValueCastConverter.java
 - src/main/java/org/embulk/filter/add_time/converter/LongValueCastConverter.java
 - src/main/java/org/embulk/filter/add_time/converter/SchemaConverter.java
 - src/main/java/org/embulk/filter/add_time/converter/SimpleColumnConverter.java
@@ -73,13 +74,14 @@ files:
 - src/main/java/org/embulk/filter/add_time/reader/BooleanColumnReader.java
 - src/main/java/org/embulk/filter/add_time/reader/ColumnReader.java
 - src/main/java/org/embulk/filter/add_time/reader/DoubleColumnReader.java
+- src/main/java/org/embulk/filter/add_time/reader/JsonColumnReader.java
 - src/main/java/org/embulk/filter/add_time/reader/LongColumnReader.java
 - src/main/java/org/embulk/filter/add_time/reader/StringColumnReader.java
 - src/main/java/org/embulk/filter/add_time/reader/TimeValueGenerator.java
 - src/main/java/org/embulk/filter/add_time/reader/TimestampColumnReader.java
 - src/test/java/org/embulk/filter/add_time/TestAddTimeFilterPlugin.java
 - src/test/java/org/embulk/filter/add_time/converter/TestSchemaConverter.java
-- classpath/embulk-filter-add_time-0.1.1.jar
+- classpath/embulk-filter-add_time-0.2.0.jar
 homepage: https://github.com/treasure-data/embulk-filter-add_time
 licenses:
 - Apache 2.0