embulk-filter-add_time 0.1.1 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 47c3533da9505293335ec892ae49e06b462f3b39
4
- data.tar.gz: 3332208c1f09e1b0da0d2d9638730083f679159d
3
+ metadata.gz: 7ebbd2e05d327a034281cd755e19c4a34111aaaf
4
+ data.tar.gz: 17af5b3075faef634c82c6ca61ad76c294ab457d
5
5
  SHA512:
6
- metadata.gz: bcd078c499649c2ae4be4e2c527626058cf9cad68511385e269535b25835dde192e2744340014eef1833aaeb1848e31d0c1447c6af8286641d935e7795ec91aa
7
- data.tar.gz: 64cfc9c742ff32955508ffe75dd8a83e393102bcb6c80db7b45bf0869a5e6b5863d0d34e8694d596c5e12ff157c60256e1d807e731ef6707fe11b3d755e3adfb
6
+ metadata.gz: 549eaf5a786b28f306432c75cec775d578d4a2fcf664a74636e6d2a48381e971cd28c5c4cc69772d6aad1738bb2f183481e6b6644c6c3f462d0149a26d43b739
7
+ data.tar.gz: 52a492c8452c61db9b8076417798555bdf0c25e08d0107b5ceb1b34eff0771b21a3a3ca4c144f99213ba92d6d188ae804b1adf7c96280d451627d7d1374e03e1
@@ -1,3 +1,8 @@
1
+ ## 0.2.0 - 2016-02-22
2
+
3
+ * [new feature] Support json_key sub option in from_column option [#7](https://github.com/treasure-data/embulk-filter-add_time/pull/7)
4
+ * [maintenance] Upgrade Embulk v08 [#6](https://github.com/treasure-data/embulk-filter-add_time/pull/6)
5
+
1
6
  ## 0.1.1 - 2016-02-05
2
7
 
3
8
  * [maintenance] Throw ConfigException if from_column doesn't exist in input schema [#5](https://github.com/treasure-data/embulk-filter-add_time/pull/5)
@@ -15,19 +15,19 @@ configurations {
15
15
  provided
16
16
  }
17
17
 
18
- version = "0.1.1"
18
+ version = "0.2.0"
19
19
 
20
20
  compileJava.options.encoding = 'UTF-8' // source encoding
21
21
  sourceCompatibility = 1.7
22
22
  targetCompatibility = 1.7
23
23
 
24
24
  dependencies {
25
- compile "org.embulk:embulk-core:0.7.10"
26
- provided "org.embulk:embulk-core:0.7.10"
25
+ compile "org.embulk:embulk-core:0.8.5"
26
+ provided "org.embulk:embulk-core:0.8.5"
27
27
 
28
28
  testCompile "junit:junit:4.+"
29
- testCompile "org.embulk:embulk-standards:0.7.10"
30
- testCompile "org.embulk:embulk-core:0.7.10:tests"
29
+ testCompile "org.embulk:embulk-standards:0.8.5"
30
+ testCompile "org.embulk:embulk-core:0.8.5:tests"
31
31
  testCompile "org.mockito:mockito-core:1.9.5"
32
32
  }
33
33
 
@@ -1,6 +1,5 @@
1
1
  package org.embulk.filter.add_time;
2
2
 
3
- import com.fasterxml.jackson.annotation.JsonCreator;
4
3
  import com.google.common.base.Optional;
5
4
  import org.embulk.config.Config;
6
5
  import org.embulk.config.ConfigDefault;
@@ -69,6 +68,10 @@ public class AddTimeFilterPlugin
69
68
  @Config("default_timestamp_format")
70
69
  @ConfigDefault("\"%Y-%m-%d %H:%M:%S %z\"") // override default value
71
70
  String getDefaultTimestampFormat();
71
+
72
+ @Config("json_key")
73
+ @ConfigDefault("null")
74
+ Optional<String> getJsonKey();
72
75
  }
73
76
 
74
77
  public interface FromValueConfig
@@ -0,0 +1,84 @@
1
+ package org.embulk.filter.add_time.converter;
2
+
3
+ import org.embulk.filter.add_time.AddTimeFilterPlugin.FromColumnConfig;
4
+ import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
5
+ import org.embulk.filter.add_time.AddTimeFilterPlugin.UnixTimestampUnit;
6
+ import org.embulk.spi.Column;
7
+ import org.embulk.spi.PageBuilder;
8
+ import org.embulk.spi.time.Timestamp;
9
+ import org.embulk.spi.time.TimestampParseException;
10
+ import org.embulk.spi.time.TimestampParser;
11
+ import org.msgpack.core.MessagePackException;
12
+ import org.msgpack.value.Value;
13
+
14
+ import java.util.Map;
15
+
16
+ import static org.msgpack.value.ValueFactory.newString;
17
+
18
+ public class JsonValueCastConverter
19
+ extends ValueCastConverter
20
+ {
21
+ private final Value jsonKey;
22
+ private final TimestampParser fromTimestampParser; // for string value
23
+ private final UnixTimestampUnit fromUnixTimestampUnit; // for long value
24
+
25
+ public JsonValueCastConverter(FromColumnConfig fromColumnConfig, ToColumnConfig toColumnConfig)
26
+ {
27
+ super(toColumnConfig);
28
+ this.jsonKey = newString(fromColumnConfig.getJsonKey().get());
29
+ this.fromTimestampParser = new TimestampParser(fromColumnConfig, fromColumnConfig);
30
+ this.fromUnixTimestampUnit = UnixTimestampUnit.of(fromColumnConfig.getUnixTimestampUnit());
31
+ }
32
+
33
+ @Override
34
+ public void convertValue(final Column column, Value value, final PageBuilder pageBuilder)
35
+ {
36
+ try {
37
+ if (!value.isMapValue()) {
38
+ throw new InvalidCastException("The value must be map object.");
39
+ }
40
+
41
+ Map<Value, Value> map = value.asMapValue().map();
42
+ if (!map.containsKey(jsonKey)) {
43
+ throw new InvalidCastException("This record doesn't have a key specified as json_key.");
44
+ }
45
+
46
+ Value v = map.get(jsonKey);
47
+ if (v.isStringValue()) {
48
+ columnVisitor.setValue(stringToTimestamp(v));
49
+ }
50
+ else if (v.isIntegerValue()) {
51
+ columnVisitor.setValue(longToTimestamp(v));
52
+ }
53
+ else {
54
+ throw new InvalidCastException(String.format(
55
+ "The value of a key specified as json_key must be long or string type. But it's %s", value.getValueType().name()));
56
+ }
57
+
58
+ columnVisitor.setPageBuilder(pageBuilder);
59
+ column.visit(columnVisitor);
60
+ }
61
+ catch (InvalidCastException | TimestampParseException | MessagePackException e) {
62
+ log.warn(String.format("Cannot convert (%s): %s", e.getMessage(), value.toJson()));
63
+ pageBuilder.setNull(column);
64
+ }
65
+ }
66
+
67
+ private Timestamp stringToTimestamp(Value value)
68
+ {
69
+ return fromTimestampParser.parse(value.asStringValue().toString());
70
+ }
71
+
72
+ private Timestamp longToTimestamp(Value value)
73
+ {
74
+ return fromUnixTimestampUnit.toTimestamp(value.asIntegerValue().toLong());
75
+ }
76
+
77
+ static class InvalidCastException
78
+ extends RuntimeException
79
+ {
80
+ InvalidCastException(String message) {
81
+ super(message);
82
+ }
83
+ }
84
+ }
@@ -9,6 +9,7 @@ import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
9
9
  import org.embulk.filter.add_time.reader.BooleanColumnReader;
10
10
  import org.embulk.filter.add_time.reader.ColumnReader;
11
11
  import org.embulk.filter.add_time.reader.DoubleColumnReader;
12
+ import org.embulk.filter.add_time.reader.JsonColumnReader;
12
13
  import org.embulk.filter.add_time.reader.LongColumnReader;
13
14
  import org.embulk.filter.add_time.reader.StringColumnReader;
14
15
  import org.embulk.filter.add_time.reader.TimeValueGenerator;
@@ -21,6 +22,7 @@ import org.embulk.spi.PageReader;
21
22
  import org.embulk.spi.Schema;
22
23
  import org.embulk.spi.type.BooleanType;
23
24
  import org.embulk.spi.type.DoubleType;
25
+ import org.embulk.spi.type.JsonType;
24
26
  import org.embulk.spi.type.LongType;
25
27
  import org.embulk.spi.type.StringType;
26
28
  import org.embulk.spi.type.TimestampType;
@@ -28,6 +30,11 @@ import org.embulk.spi.type.Type;
28
30
  import org.embulk.spi.type.Types;
29
31
  import org.slf4j.Logger;
30
32
 
33
+ import static org.embulk.spi.type.Types.JSON;
34
+ import static org.embulk.spi.type.Types.LONG;
35
+ import static org.embulk.spi.type.Types.STRING;
36
+ import static org.embulk.spi.type.Types.TIMESTAMP;
37
+
31
38
  public class SchemaConverter
32
39
  {
33
40
  private final Logger log;
@@ -72,9 +79,9 @@ public class SchemaConverter
72
79
  }
73
80
 
74
81
  if (fromColumnConfig.isPresent() && columnName.equals(fromColumnConfig.get().getName())) {
75
- if (!columnType.equals(Types.LONG) && !columnType.equals(Types.STRING) && !columnType.equals(Types.TIMESTAMP)) {
82
+ if (!(columnType.equals(LONG) || columnType.equals(STRING) || columnType.equals(TIMESTAMP) || columnType.equals(JSON))) {
76
83
  throw new ConfigException(String.format(
77
- "The type of the '%s' column specified as from_column must be long, string or timestamp. But it's %s.", columnName, columnType));
84
+ "The type of the '%s' column specified as from_column must be long, string, timestamp or json. But it's %s.", columnName, columnType));
78
85
  }
79
86
 
80
87
  ColumnReader duplicatee = newColumnReader(columnType, newValueCastConverter(columnType, fromColumnConfig, toColumnConfig));
@@ -152,7 +159,9 @@ public class SchemaConverter
152
159
  else if (columnType instanceof TimestampType) {
153
160
  return new TimestampColumnReader(valueConverter);
154
161
  }
155
- // TODO support Json type
162
+ else if (columnType instanceof JsonType) {
163
+ return new JsonColumnReader(valueConverter);
164
+ }
156
165
  else {
157
166
  throw new ConfigException("Unsupported type: " + columnType); // TODO after json type support, it should be changed to AssertionError.
158
167
  }
@@ -166,6 +175,9 @@ public class SchemaConverter
166
175
  else if (columnType instanceof StringType) {
167
176
  return new StringValueCastConverter(fromColumnConfig.get(), toColumnConfig);
168
177
  }
178
+ else if (columnType instanceof JsonType) {
179
+ return new JsonValueCastConverter(fromColumnConfig.get(), toColumnConfig);
180
+ }
169
181
  else if (columnType instanceof TimestampType) {
170
182
  return new TimestampValueCastConverter(toColumnConfig);
171
183
  }
@@ -218,6 +230,12 @@ public class SchemaConverter
218
230
  updateColumn(column, pageReader);
219
231
  }
220
232
 
233
+ @Override
234
+ public void jsonColumn(Column column)
235
+ {
236
+ updateColumn(column, pageReader);
237
+ }
238
+
221
239
  @Override
222
240
  public void timestampColumn(Column column)
223
241
  {
@@ -4,16 +4,21 @@ import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
4
4
  import org.embulk.filter.add_time.AddTimeFilterPlugin.UnixTimestampUnit;
5
5
  import org.embulk.spi.Column;
6
6
  import org.embulk.spi.ColumnVisitor;
7
+ import org.embulk.spi.Exec;
7
8
  import org.embulk.spi.PageBuilder;
8
9
  import org.embulk.spi.time.Timestamp;
10
+ import org.msgpack.value.Value;
11
+ import org.slf4j.Logger;
9
12
 
10
13
  public abstract class ValueCastConverter
11
14
  implements ValueConverter
12
15
  {
16
+ protected final Logger log;
13
17
  protected final TimestampValueCastVisitor columnVisitor;
14
18
 
15
19
  public ValueCastConverter(ToColumnConfig toColumnConfig)
16
20
  {
21
+ this.log = Exec.getLogger(ValueCastConverter.class);
17
22
  this.columnVisitor = new TimestampValueCastVisitor(UnixTimestampUnit.of(toColumnConfig.getUnixTimestampUnit()));
18
23
  }
19
24
 
@@ -47,6 +52,12 @@ public abstract class ValueCastConverter
47
52
  throw new AssertionError("Should implement in subclass.");
48
53
  }
49
54
 
55
+ @Override
56
+ public void convertValue(final Column column, Value value, final PageBuilder pageBuilder)
57
+ {
58
+ throw new AssertionError("Should implement in subclass.");
59
+ }
60
+
50
61
  @Override
51
62
  public void convertValue(Column column, final Timestamp value, final PageBuilder pageBuilder)
52
63
  {
@@ -99,6 +110,12 @@ public abstract class ValueCastConverter
99
110
  throw new AssertionError("Never call.");
100
111
  }
101
112
 
113
+ @Override
114
+ public void jsonColumn(Column column)
115
+ {
116
+ throw new AssertionError("Never call.");
117
+ }
118
+
102
119
  @Override
103
120
  public void timestampColumn(Column column)
104
121
  {
@@ -3,6 +3,7 @@ package org.embulk.filter.add_time.converter;
3
3
  import org.embulk.spi.Column;
4
4
  import org.embulk.spi.PageBuilder;
5
5
  import org.embulk.spi.time.Timestamp;
6
+ import org.msgpack.value.Value;
6
7
 
7
8
  public interface ValueConverter
8
9
  {
@@ -18,5 +19,7 @@ public interface ValueConverter
18
19
 
19
20
  void convertValue(Column column, String value, PageBuilder pageBuilder);
20
21
 
22
+ void convertValue(Column column, Value value, PageBuilder pageBuilder);
23
+
21
24
  void convertValue(Column column, Timestamp value, PageBuilder pageBuilder);
22
25
  }
@@ -3,6 +3,7 @@ package org.embulk.filter.add_time.converter;
3
3
  import org.embulk.spi.Column;
4
4
  import org.embulk.spi.PageBuilder;
5
5
  import org.embulk.spi.time.Timestamp;
6
+ import org.msgpack.value.Value;
6
7
 
7
8
  public class ValueNoConverter
8
9
  implements ValueConverter
@@ -37,6 +38,12 @@ public class ValueNoConverter
37
38
  pageBuilder.setString(column, value);
38
39
  }
39
40
 
41
+ @Override
42
+ public void convertValue(Column column, Value value, PageBuilder pageBuilder)
43
+ {
44
+ pageBuilder.setJson(column, value);
45
+ }
46
+
40
47
  @Override
41
48
  public void convertValue(Column column, Timestamp value, PageBuilder pageBuilder)
42
49
  {
@@ -0,0 +1,36 @@
1
+ package org.embulk.filter.add_time.reader;
2
+
3
+ import org.embulk.filter.add_time.converter.ValueConverter;
4
+ import org.embulk.spi.Column;
5
+ import org.embulk.spi.PageBuilder;
6
+ import org.embulk.spi.PageReader;
7
+ import org.msgpack.value.Value;
8
+
9
+ public class JsonColumnReader
10
+ extends AbstractColumnReader<JsonColumnReader>
11
+ {
12
+ protected Value value;
13
+
14
+ public JsonColumnReader(ValueConverter valueConverter)
15
+ {
16
+ super(valueConverter);
17
+ }
18
+
19
+ @Override
20
+ public void readNonNullValue(Column column, PageReader pageReader)
21
+ {
22
+ value = pageReader.getJson(column);
23
+ }
24
+
25
+ @Override
26
+ public void convertNonNullValue(Column column, PageBuilder pageBuilder)
27
+ {
28
+ valueConverter.convertValue(column, value, pageBuilder);
29
+ }
30
+
31
+ @Override
32
+ public void copyValueTo(JsonColumnReader columnReader)
33
+ {
34
+ columnReader.value = this.value;
35
+ }
36
+ }
@@ -18,11 +18,16 @@ import org.embulk.spi.util.Pages;
18
18
  import org.junit.Before;
19
19
  import org.junit.Rule;
20
20
  import org.junit.Test;
21
+ import org.msgpack.value.Value;
21
22
 
22
23
  import java.util.List;
23
24
 
24
25
  import static org.junit.Assert.assertEquals;
26
+ import static org.junit.Assert.assertNull;
25
27
  import static org.mockito.Mockito.spy;
28
+ import static org.msgpack.value.ValueFactory.newInteger;
29
+ import static org.msgpack.value.ValueFactory.newMap;
30
+ import static org.msgpack.value.ValueFactory.newString;
26
31
 
27
32
  public class TestAddTimeFilterPlugin
28
33
  {
@@ -39,7 +44,13 @@ public class TestAddTimeFilterPlugin
39
44
  {
40
45
  plugin = plugin();
41
46
  config = runtime.getExec().newConfigSource();
42
- inputSchema = schema("c0", Types.BOOLEAN, "c1", Types.LONG, "c2", Types.DOUBLE, "c3", Types.STRING, "c4", Types.TIMESTAMP);
47
+ inputSchema = schema("c0", Types.BOOLEAN, "c1", Types.LONG, "c2", Types.DOUBLE, "c3",
48
+ Types.STRING, "c4", Types.TIMESTAMP, "c5", Types.JSON);
49
+ }
50
+
51
+ private Value newSimpleMap()
52
+ {
53
+ return newMap(newString("k"), newString("v"));
43
54
  }
44
55
 
45
56
  @Test
@@ -49,7 +60,7 @@ public class TestAddTimeFilterPlugin
49
60
  ConfigSource conf = this.config.deepCopy()
50
61
  .set("to_column", ImmutableMap.of("name", "time"))
51
62
  .set("from_column", ImmutableMap.of("name", "c1", "unix_timestamp_unit", "sec"));
52
- List<Page> pages = newPages(true, 1451646671L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
63
+ List<Page> pages = newPages(true, 1451646671L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
53
64
 
54
65
  callTansaction(conf, inputSchema, pages);
55
66
 
@@ -62,7 +73,8 @@ public class TestAddTimeFilterPlugin
62
73
  assertEquals(0.1, record[2]);
63
74
  assertEquals("foo", record[3]);
64
75
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
65
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
76
+ assertEquals(newSimpleMap(), record[5]);
77
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
66
78
  }
67
79
  }
68
80
 
@@ -70,7 +82,7 @@ public class TestAddTimeFilterPlugin
70
82
  ConfigSource conf = this.config.deepCopy()
71
83
  .set("to_column", ImmutableMap.of("name", "time"))
72
84
  .set("from_column", ImmutableMap.of("name", "c4"));
73
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
85
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
74
86
 
75
87
  callTansaction(conf, inputSchema, pages);
76
88
 
@@ -83,7 +95,8 @@ public class TestAddTimeFilterPlugin
83
95
  assertEquals(0.1, record[2]);
84
96
  assertEquals("foo", record[3]);
85
97
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
86
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
98
+ assertEquals(newSimpleMap(), record[5]);
99
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
87
100
  }
88
101
  }
89
102
 
@@ -91,7 +104,7 @@ public class TestAddTimeFilterPlugin
91
104
  ConfigSource conf = this.config.deepCopy()
92
105
  .set("to_column", ImmutableMap.of("name", "time"))
93
106
  .set("from_column", ImmutableMap.of("name", "c3"));
94
- List<Page> pages = newPages(true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671));
107
+ List<Page> pages = newPages(true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
95
108
 
96
109
  callTansaction(conf, inputSchema, pages);
97
110
 
@@ -104,7 +117,45 @@ public class TestAddTimeFilterPlugin
104
117
  assertEquals(0.1, record[2]);
105
118
  assertEquals("2016-01-01 11:11:11 UTC", record[3]);
106
119
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
107
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
120
+ assertEquals(newSimpleMap(), record[5]);
121
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
122
+ }
123
+ }
124
+
125
+ { // json type
126
+ ConfigSource conf = this.config.deepCopy()
127
+ .set("to_column", ImmutableMap.of("name", "time"))
128
+ .set("from_column", ImmutableMap.of(
129
+ "name", "c5",
130
+ "json_key", "k",
131
+ "unix_timestamp_unit", "sec",
132
+ "timestamp_format", "%Y-%m-%d %H:%M:%S %z"
133
+ ));
134
+
135
+ List<Page> pages = newPages(
136
+ true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newString("2016-01-01 11:11:11 UTC")),
137
+ true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newString("embulk")),
138
+ true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newInteger(1451646671))
139
+ );
140
+
141
+ callTansaction(conf, inputSchema, pages);
142
+
143
+ assertEquals(3, records.size());
144
+ Object[] record;
145
+ {
146
+ record = records.get(0);
147
+ assertEquals(newMap(newString("k"), newString("2016-01-01 11:11:11 UTC")), record[5]);
148
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
149
+ }
150
+ {
151
+ record = records.get(1);
152
+ assertEquals(newMap(newString("k"), newString("embulk")), record[5]);
153
+ assertNull(record[6]);
154
+ }
155
+ {
156
+ record = records.get(2);
157
+ assertEquals(newMap(newString("k"), newInteger(1451646671)), record[5]);
158
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
108
159
  }
109
160
  }
110
161
  }
@@ -117,7 +168,7 @@ public class TestAddTimeFilterPlugin
117
168
  ConfigSource conf = this.config.deepCopy()
118
169
  .set("to_column", ImmutableMap.of("name", "time"))
119
170
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
120
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
171
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
121
172
 
122
173
  callTansaction(conf, inputSchema, pages);
123
174
 
@@ -130,14 +181,15 @@ public class TestAddTimeFilterPlugin
130
181
  assertEquals(0.1, record[2]);
131
182
  assertEquals("foo", record[3]);
132
183
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
133
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
184
+ assertEquals(newSimpleMap(), record[5]);
185
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
134
186
  }
135
187
  }
136
188
  { // specifies timestamp_format
137
189
  ConfigSource conf = this.config.deepCopy()
138
190
  .set("to_column", ImmutableMap.of("name", "time"))
139
191
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11.000 UTC", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
140
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
192
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
141
193
 
142
194
  callTansaction(conf, inputSchema, pages);
143
195
 
@@ -150,14 +202,15 @@ public class TestAddTimeFilterPlugin
150
202
  assertEquals(0.1, record[2]);
151
203
  assertEquals("foo", record[3]);
152
204
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
153
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
205
+ assertEquals(newSimpleMap(), record[5]);
206
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
154
207
  }
155
208
  }
156
209
  { // specifies unix_timestamp_unit
157
210
  ConfigSource conf = this.config.deepCopy()
158
211
  .set("to_column", ImmutableMap.of("name", "time"))
159
212
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", 1451646671, "unix_timestamp_unit", "sec"));
160
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
213
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
161
214
 
162
215
  callTansaction(conf, inputSchema, pages);
163
216
 
@@ -170,7 +223,8 @@ public class TestAddTimeFilterPlugin
170
223
  assertEquals(0.1, record[2]);
171
224
  assertEquals("foo", record[3]);
172
225
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
173
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
226
+ assertEquals(newSimpleMap(), record[5]);
227
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
174
228
  }
175
229
  }
176
230
 
@@ -180,7 +234,7 @@ public class TestAddTimeFilterPlugin
180
234
  .set("to_column", ImmutableMap.of("name", "time"))
181
235
  .set("from_value", ImmutableMap.of("mode", "incremental_time",
182
236
  "from", "2016-01-01 11:11:11 UTC", "to", "2016-01-01 11:11:12 UTC"));
183
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
237
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
184
238
 
185
239
  callTansaction(conf, inputSchema, pages);
186
240
 
@@ -193,7 +247,8 @@ public class TestAddTimeFilterPlugin
193
247
  assertEquals(0.1, record[2]);
194
248
  assertEquals("foo", record[3]);
195
249
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
196
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
250
+ assertEquals(newSimpleMap(), record[5]);
251
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
197
252
  }
198
253
  }
199
254
  { // specifies timestamp_format
@@ -201,7 +256,7 @@ public class TestAddTimeFilterPlugin
201
256
  .set("to_column", ImmutableMap.of("name", "time"))
202
257
  .set("from_value", ImmutableMap.of("mode", "incremental_time",
203
258
  "from", "2016-01-01 11:11:11.000 UTC", "to", "2016-01-01 11:11:12.000 UTC", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
204
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
259
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
205
260
 
206
261
  callTansaction(conf, inputSchema, pages);
207
262
 
@@ -214,7 +269,8 @@ public class TestAddTimeFilterPlugin
214
269
  assertEquals(0.1, record[2]);
215
270
  assertEquals("foo", record[3]);
216
271
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
217
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
272
+ assertEquals(newSimpleMap(), record[5]);
273
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
218
274
  }
219
275
  }
220
276
  { // specifies unix_timestamp_unit
@@ -222,7 +278,7 @@ public class TestAddTimeFilterPlugin
222
278
  .set("to_column", ImmutableMap.of("name", "time"))
223
279
  .set("from_value", ImmutableMap.of("mode", "incremental_time",
224
280
  "from", 1451646671, "to", 1451646672, "unix_timestamp_unit", "sec"));
225
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
281
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
226
282
 
227
283
  callTansaction(conf, inputSchema, pages);
228
284
 
@@ -235,7 +291,8 @@ public class TestAddTimeFilterPlugin
235
291
  assertEquals(0.1, record[2]);
236
292
  assertEquals("foo", record[3]);
237
293
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
238
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
294
+ assertEquals(newSimpleMap(), record[5]);
295
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
239
296
  }
240
297
  }
241
298
 
@@ -244,7 +301,7 @@ public class TestAddTimeFilterPlugin
244
301
  ConfigSource conf = this.config.deepCopy()
245
302
  .set("to_column", ImmutableMap.of("name", "time"))
246
303
  .set("from_value", ImmutableMap.of("mode", "upload_time"));
247
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
304
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
248
305
 
249
306
  callTansaction(conf, inputSchema, pages);
250
307
 
@@ -257,7 +314,8 @@ public class TestAddTimeFilterPlugin
257
314
  assertEquals(0.1, record[2]);
258
315
  assertEquals("foo", record[3]);
259
316
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
260
- assertEquals(runtime.getExec().getTransactionTime(), record[5]);
317
+ assertEquals(newSimpleMap(), record[5]);
318
+ assertEquals(runtime.getExec().getTransactionTime(), record[6]);
261
319
  }
262
320
  }
263
321
  }
@@ -270,7 +328,7 @@ public class TestAddTimeFilterPlugin
270
328
  ConfigSource conf = this.config.deepCopy()
271
329
  .set("to_column", ImmutableMap.of("name", "time"))
272
330
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
273
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
331
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
274
332
 
275
333
  callTansaction(conf, inputSchema, pages);
276
334
 
@@ -283,7 +341,8 @@ public class TestAddTimeFilterPlugin
283
341
  assertEquals(0.1, record[2]);
284
342
  assertEquals("foo", record[3]);
285
343
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
286
- assertEquals(1451646671L, ((Timestamp) record[5]).getEpochSecond());
344
+ assertEquals(newSimpleMap(), record[5]);
345
+ assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
287
346
  }
288
347
  }
289
348
 
@@ -292,7 +351,7 @@ public class TestAddTimeFilterPlugin
292
351
  ConfigSource conf = this.config.deepCopy()
293
352
  .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "sec"))
294
353
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
295
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
354
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
296
355
 
297
356
  callTansaction(conf, inputSchema, pages);
298
357
 
@@ -305,14 +364,15 @@ public class TestAddTimeFilterPlugin
305
364
  assertEquals(0.1, record[2]);
306
365
  assertEquals("foo", record[3]);
307
366
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
308
- assertEquals(1451646671L, record[5]);
367
+ assertEquals(newSimpleMap(), record[5]);
368
+ assertEquals(1451646671L, record[6]);
309
369
  }
310
370
  }
311
371
  { // unix_timestamp: milli
312
372
  ConfigSource conf = this.config.deepCopy()
313
373
  .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "milli"))
314
374
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
315
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
375
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
316
376
 
317
377
  callTansaction(conf, inputSchema, pages);
318
378
 
@@ -325,14 +385,15 @@ public class TestAddTimeFilterPlugin
325
385
  assertEquals(0.1, record[2]);
326
386
  assertEquals("foo", record[3]);
327
387
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
328
- assertEquals(1451646671000L, record[5]);
388
+ assertEquals(newSimpleMap(), record[5]);
389
+ assertEquals(1451646671000L, record[6]);
329
390
  }
330
391
  }
331
392
  { // unix_timestamp: micro
332
393
  ConfigSource conf = this.config.deepCopy()
333
394
  .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "micro"))
334
395
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
335
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
396
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
336
397
 
337
398
  callTansaction(conf, inputSchema, pages);
338
399
 
@@ -345,14 +406,15 @@ public class TestAddTimeFilterPlugin
345
406
  assertEquals(0.1, record[2]);
346
407
  assertEquals("foo", record[3]);
347
408
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
348
- assertEquals(1451646671000000L, record[5]);
409
+ assertEquals(newSimpleMap(), record[5]);
410
+ assertEquals(1451646671000000L, record[6]);
349
411
  }
350
412
  }
351
413
  { // unix_timestamp: nano
352
414
  ConfigSource conf = this.config.deepCopy()
353
415
  .set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "nano"))
354
416
  .set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
355
- List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
417
+ List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
356
418
 
357
419
  callTansaction(conf, inputSchema, pages);
358
420
 
@@ -365,7 +427,8 @@ public class TestAddTimeFilterPlugin
365
427
  assertEquals(0.1, record[2]);
366
428
  assertEquals("foo", record[3]);
367
429
  assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
368
- assertEquals(1451646671000000000L, record[5]);
430
+ assertEquals(newSimpleMap(), record[5]);
431
+ assertEquals(1451646671000000000L, record[6]);
369
432
  }
370
433
  }
371
434
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-filter-add_time
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Muga Nishizawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-05 00:00:00.000000000 Z
11
+ date: 2016-02-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -61,6 +61,7 @@ files:
61
61
  - src/main/java/org/embulk/filter/add_time/AddTimeFilterPlugin.java
62
62
  - src/main/java/org/embulk/filter/add_time/converter/ColumnConverter.java
63
63
  - src/main/java/org/embulk/filter/add_time/converter/ColumnDuplicator.java
64
+ - src/main/java/org/embulk/filter/add_time/converter/JsonValueCastConverter.java
64
65
  - src/main/java/org/embulk/filter/add_time/converter/LongValueCastConverter.java
65
66
  - src/main/java/org/embulk/filter/add_time/converter/SchemaConverter.java
66
67
  - src/main/java/org/embulk/filter/add_time/converter/SimpleColumnConverter.java
@@ -73,13 +74,14 @@ files:
73
74
  - src/main/java/org/embulk/filter/add_time/reader/BooleanColumnReader.java
74
75
  - src/main/java/org/embulk/filter/add_time/reader/ColumnReader.java
75
76
  - src/main/java/org/embulk/filter/add_time/reader/DoubleColumnReader.java
77
+ - src/main/java/org/embulk/filter/add_time/reader/JsonColumnReader.java
76
78
  - src/main/java/org/embulk/filter/add_time/reader/LongColumnReader.java
77
79
  - src/main/java/org/embulk/filter/add_time/reader/StringColumnReader.java
78
80
  - src/main/java/org/embulk/filter/add_time/reader/TimeValueGenerator.java
79
81
  - src/main/java/org/embulk/filter/add_time/reader/TimestampColumnReader.java
80
82
  - src/test/java/org/embulk/filter/add_time/TestAddTimeFilterPlugin.java
81
83
  - src/test/java/org/embulk/filter/add_time/converter/TestSchemaConverter.java
82
- - classpath/embulk-filter-add_time-0.1.1.jar
84
+ - classpath/embulk-filter-add_time-0.2.0.jar
83
85
  homepage: https://github.com/treasure-data/embulk-filter-add_time
84
86
  licenses:
85
87
  - Apache 2.0