embulk-filter-add_time 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/build.gradle +5 -5
- data/src/main/java/org/embulk/filter/add_time/AddTimeFilterPlugin.java +4 -1
- data/src/main/java/org/embulk/filter/add_time/converter/JsonValueCastConverter.java +84 -0
- data/src/main/java/org/embulk/filter/add_time/converter/SchemaConverter.java +21 -3
- data/src/main/java/org/embulk/filter/add_time/converter/ValueCastConverter.java +17 -0
- data/src/main/java/org/embulk/filter/add_time/converter/ValueConverter.java +3 -0
- data/src/main/java/org/embulk/filter/add_time/converter/ValueNoConverter.java +7 -0
- data/src/main/java/org/embulk/filter/add_time/reader/JsonColumnReader.java +36 -0
- data/src/test/java/org/embulk/filter/add_time/TestAddTimeFilterPlugin.java +94 -31
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ebbd2e05d327a034281cd755e19c4a34111aaaf
|
4
|
+
data.tar.gz: 17af5b3075faef634c82c6ca61ad76c294ab457d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 549eaf5a786b28f306432c75cec775d578d4a2fcf664a74636e6d2a48381e971cd28c5c4cc69772d6aad1738bb2f183481e6b6644c6c3f462d0149a26d43b739
|
7
|
+
data.tar.gz: 52a492c8452c61db9b8076417798555bdf0c25e08d0107b5ceb1b34eff0771b21a3a3ca4c144f99213ba92d6d188ae804b1adf7c96280d451627d7d1374e03e1
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.2.0 - 2016-02-22
|
2
|
+
|
3
|
+
* [new feature] Support json_key sub option in from_column option [#7](https://github.com/treasure-data/embulk-filter-add_time/pull/7)
|
4
|
+
* [maintenance] Upgrade Embulk v08 [#6](https://github.com/treasure-data/embulk-filter-add_time/pull/6)
|
5
|
+
|
1
6
|
## 0.1.1 - 2016-02-05
|
2
7
|
|
3
8
|
* [maintenance] Throw ConfigException if from_column doesn't exist in input schema [#5](https://github.com/treasure-data/embulk-filter-add_time/pull/5)
|
data/build.gradle
CHANGED
@@ -15,19 +15,19 @@ configurations {
|
|
15
15
|
provided
|
16
16
|
}
|
17
17
|
|
18
|
-
version = "0.
|
18
|
+
version = "0.2.0"
|
19
19
|
|
20
20
|
compileJava.options.encoding = 'UTF-8' // source encoding
|
21
21
|
sourceCompatibility = 1.7
|
22
22
|
targetCompatibility = 1.7
|
23
23
|
|
24
24
|
dependencies {
|
25
|
-
compile "org.embulk:embulk-core:0.
|
26
|
-
provided "org.embulk:embulk-core:0.
|
25
|
+
compile "org.embulk:embulk-core:0.8.5"
|
26
|
+
provided "org.embulk:embulk-core:0.8.5"
|
27
27
|
|
28
28
|
testCompile "junit:junit:4.+"
|
29
|
-
testCompile "org.embulk:embulk-standards:0.
|
30
|
-
testCompile "org.embulk:embulk-core:0.
|
29
|
+
testCompile "org.embulk:embulk-standards:0.8.5"
|
30
|
+
testCompile "org.embulk:embulk-core:0.8.5:tests"
|
31
31
|
testCompile "org.mockito:mockito-core:1.9.5"
|
32
32
|
}
|
33
33
|
|
@@ -1,6 +1,5 @@
|
|
1
1
|
package org.embulk.filter.add_time;
|
2
2
|
|
3
|
-
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
3
|
import com.google.common.base.Optional;
|
5
4
|
import org.embulk.config.Config;
|
6
5
|
import org.embulk.config.ConfigDefault;
|
@@ -69,6 +68,10 @@ public class AddTimeFilterPlugin
|
|
69
68
|
@Config("default_timestamp_format")
|
70
69
|
@ConfigDefault("\"%Y-%m-%d %H:%M:%S %z\"") // override default value
|
71
70
|
String getDefaultTimestampFormat();
|
71
|
+
|
72
|
+
@Config("json_key")
|
73
|
+
@ConfigDefault("null")
|
74
|
+
Optional<String> getJsonKey();
|
72
75
|
}
|
73
76
|
|
74
77
|
public interface FromValueConfig
|
@@ -0,0 +1,84 @@
|
|
1
|
+
package org.embulk.filter.add_time.converter;
|
2
|
+
|
3
|
+
import org.embulk.filter.add_time.AddTimeFilterPlugin.FromColumnConfig;
|
4
|
+
import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
|
5
|
+
import org.embulk.filter.add_time.AddTimeFilterPlugin.UnixTimestampUnit;
|
6
|
+
import org.embulk.spi.Column;
|
7
|
+
import org.embulk.spi.PageBuilder;
|
8
|
+
import org.embulk.spi.time.Timestamp;
|
9
|
+
import org.embulk.spi.time.TimestampParseException;
|
10
|
+
import org.embulk.spi.time.TimestampParser;
|
11
|
+
import org.msgpack.core.MessagePackException;
|
12
|
+
import org.msgpack.value.Value;
|
13
|
+
|
14
|
+
import java.util.Map;
|
15
|
+
|
16
|
+
import static org.msgpack.value.ValueFactory.newString;
|
17
|
+
|
18
|
+
public class JsonValueCastConverter
|
19
|
+
extends ValueCastConverter
|
20
|
+
{
|
21
|
+
private final Value jsonKey;
|
22
|
+
private final TimestampParser fromTimestampParser; // for string value
|
23
|
+
private final UnixTimestampUnit fromUnixTimestampUnit; // for long value
|
24
|
+
|
25
|
+
public JsonValueCastConverter(FromColumnConfig fromColumnConfig, ToColumnConfig toColumnConfig)
|
26
|
+
{
|
27
|
+
super(toColumnConfig);
|
28
|
+
this.jsonKey = newString(fromColumnConfig.getJsonKey().get());
|
29
|
+
this.fromTimestampParser = new TimestampParser(fromColumnConfig, fromColumnConfig);
|
30
|
+
this.fromUnixTimestampUnit = UnixTimestampUnit.of(fromColumnConfig.getUnixTimestampUnit());
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
public void convertValue(final Column column, Value value, final PageBuilder pageBuilder)
|
35
|
+
{
|
36
|
+
try {
|
37
|
+
if (!value.isMapValue()) {
|
38
|
+
throw new InvalidCastException("The value must be map object.");
|
39
|
+
}
|
40
|
+
|
41
|
+
Map<Value, Value> map = value.asMapValue().map();
|
42
|
+
if (!map.containsKey(jsonKey)) {
|
43
|
+
throw new InvalidCastException("This record doesn't have a key specified as json_key.");
|
44
|
+
}
|
45
|
+
|
46
|
+
Value v = map.get(jsonKey);
|
47
|
+
if (v.isStringValue()) {
|
48
|
+
columnVisitor.setValue(stringToTimestamp(v));
|
49
|
+
}
|
50
|
+
else if (v.isIntegerValue()) {
|
51
|
+
columnVisitor.setValue(longToTimestamp(v));
|
52
|
+
}
|
53
|
+
else {
|
54
|
+
throw new InvalidCastException(String.format(
|
55
|
+
"The value of a key specified as json_key must be long or string type. But it's %s", value.getValueType().name()));
|
56
|
+
}
|
57
|
+
|
58
|
+
columnVisitor.setPageBuilder(pageBuilder);
|
59
|
+
column.visit(columnVisitor);
|
60
|
+
}
|
61
|
+
catch (InvalidCastException | TimestampParseException | MessagePackException e) {
|
62
|
+
log.warn(String.format("Cannot convert (%s): %s", e.getMessage(), value.toJson()));
|
63
|
+
pageBuilder.setNull(column);
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
67
|
+
private Timestamp stringToTimestamp(Value value)
|
68
|
+
{
|
69
|
+
return fromTimestampParser.parse(value.asStringValue().toString());
|
70
|
+
}
|
71
|
+
|
72
|
+
private Timestamp longToTimestamp(Value value)
|
73
|
+
{
|
74
|
+
return fromUnixTimestampUnit.toTimestamp(value.asIntegerValue().toLong());
|
75
|
+
}
|
76
|
+
|
77
|
+
static class InvalidCastException
|
78
|
+
extends RuntimeException
|
79
|
+
{
|
80
|
+
InvalidCastException(String message) {
|
81
|
+
super(message);
|
82
|
+
}
|
83
|
+
}
|
84
|
+
}
|
@@ -9,6 +9,7 @@ import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
|
|
9
9
|
import org.embulk.filter.add_time.reader.BooleanColumnReader;
|
10
10
|
import org.embulk.filter.add_time.reader.ColumnReader;
|
11
11
|
import org.embulk.filter.add_time.reader.DoubleColumnReader;
|
12
|
+
import org.embulk.filter.add_time.reader.JsonColumnReader;
|
12
13
|
import org.embulk.filter.add_time.reader.LongColumnReader;
|
13
14
|
import org.embulk.filter.add_time.reader.StringColumnReader;
|
14
15
|
import org.embulk.filter.add_time.reader.TimeValueGenerator;
|
@@ -21,6 +22,7 @@ import org.embulk.spi.PageReader;
|
|
21
22
|
import org.embulk.spi.Schema;
|
22
23
|
import org.embulk.spi.type.BooleanType;
|
23
24
|
import org.embulk.spi.type.DoubleType;
|
25
|
+
import org.embulk.spi.type.JsonType;
|
24
26
|
import org.embulk.spi.type.LongType;
|
25
27
|
import org.embulk.spi.type.StringType;
|
26
28
|
import org.embulk.spi.type.TimestampType;
|
@@ -28,6 +30,11 @@ import org.embulk.spi.type.Type;
|
|
28
30
|
import org.embulk.spi.type.Types;
|
29
31
|
import org.slf4j.Logger;
|
30
32
|
|
33
|
+
import static org.embulk.spi.type.Types.JSON;
|
34
|
+
import static org.embulk.spi.type.Types.LONG;
|
35
|
+
import static org.embulk.spi.type.Types.STRING;
|
36
|
+
import static org.embulk.spi.type.Types.TIMESTAMP;
|
37
|
+
|
31
38
|
public class SchemaConverter
|
32
39
|
{
|
33
40
|
private final Logger log;
|
@@ -72,9 +79,9 @@ public class SchemaConverter
|
|
72
79
|
}
|
73
80
|
|
74
81
|
if (fromColumnConfig.isPresent() && columnName.equals(fromColumnConfig.get().getName())) {
|
75
|
-
if (!columnType.equals(
|
82
|
+
if (!(columnType.equals(LONG) || columnType.equals(STRING) || columnType.equals(TIMESTAMP) || columnType.equals(JSON))) {
|
76
83
|
throw new ConfigException(String.format(
|
77
|
-
"The type of the '%s' column specified as from_column must be long, string or
|
84
|
+
"The type of the '%s' column specified as from_column must be long, string, timestamp or json. But it's %s.", columnName, columnType));
|
78
85
|
}
|
79
86
|
|
80
87
|
ColumnReader duplicatee = newColumnReader(columnType, newValueCastConverter(columnType, fromColumnConfig, toColumnConfig));
|
@@ -152,7 +159,9 @@ public class SchemaConverter
|
|
152
159
|
else if (columnType instanceof TimestampType) {
|
153
160
|
return new TimestampColumnReader(valueConverter);
|
154
161
|
}
|
155
|
-
|
162
|
+
else if (columnType instanceof JsonType) {
|
163
|
+
return new JsonColumnReader(valueConverter);
|
164
|
+
}
|
156
165
|
else {
|
157
166
|
throw new ConfigException("Unsupported type: " + columnType); // TODO after json type support, it should be changed to AssertionError.
|
158
167
|
}
|
@@ -166,6 +175,9 @@ public class SchemaConverter
|
|
166
175
|
else if (columnType instanceof StringType) {
|
167
176
|
return new StringValueCastConverter(fromColumnConfig.get(), toColumnConfig);
|
168
177
|
}
|
178
|
+
else if (columnType instanceof JsonType) {
|
179
|
+
return new JsonValueCastConverter(fromColumnConfig.get(), toColumnConfig);
|
180
|
+
}
|
169
181
|
else if (columnType instanceof TimestampType) {
|
170
182
|
return new TimestampValueCastConverter(toColumnConfig);
|
171
183
|
}
|
@@ -218,6 +230,12 @@ public class SchemaConverter
|
|
218
230
|
updateColumn(column, pageReader);
|
219
231
|
}
|
220
232
|
|
233
|
+
@Override
|
234
|
+
public void jsonColumn(Column column)
|
235
|
+
{
|
236
|
+
updateColumn(column, pageReader);
|
237
|
+
}
|
238
|
+
|
221
239
|
@Override
|
222
240
|
public void timestampColumn(Column column)
|
223
241
|
{
|
@@ -4,16 +4,21 @@ import org.embulk.filter.add_time.AddTimeFilterPlugin.ToColumnConfig;
|
|
4
4
|
import org.embulk.filter.add_time.AddTimeFilterPlugin.UnixTimestampUnit;
|
5
5
|
import org.embulk.spi.Column;
|
6
6
|
import org.embulk.spi.ColumnVisitor;
|
7
|
+
import org.embulk.spi.Exec;
|
7
8
|
import org.embulk.spi.PageBuilder;
|
8
9
|
import org.embulk.spi.time.Timestamp;
|
10
|
+
import org.msgpack.value.Value;
|
11
|
+
import org.slf4j.Logger;
|
9
12
|
|
10
13
|
public abstract class ValueCastConverter
|
11
14
|
implements ValueConverter
|
12
15
|
{
|
16
|
+
protected final Logger log;
|
13
17
|
protected final TimestampValueCastVisitor columnVisitor;
|
14
18
|
|
15
19
|
public ValueCastConverter(ToColumnConfig toColumnConfig)
|
16
20
|
{
|
21
|
+
this.log = Exec.getLogger(ValueCastConverter.class);
|
17
22
|
this.columnVisitor = new TimestampValueCastVisitor(UnixTimestampUnit.of(toColumnConfig.getUnixTimestampUnit()));
|
18
23
|
}
|
19
24
|
|
@@ -47,6 +52,12 @@ public abstract class ValueCastConverter
|
|
47
52
|
throw new AssertionError("Should implement in subclass.");
|
48
53
|
}
|
49
54
|
|
55
|
+
@Override
|
56
|
+
public void convertValue(final Column column, Value value, final PageBuilder pageBuilder)
|
57
|
+
{
|
58
|
+
throw new AssertionError("Should implement in subclass.");
|
59
|
+
}
|
60
|
+
|
50
61
|
@Override
|
51
62
|
public void convertValue(Column column, final Timestamp value, final PageBuilder pageBuilder)
|
52
63
|
{
|
@@ -99,6 +110,12 @@ public abstract class ValueCastConverter
|
|
99
110
|
throw new AssertionError("Never call.");
|
100
111
|
}
|
101
112
|
|
113
|
+
@Override
|
114
|
+
public void jsonColumn(Column column)
|
115
|
+
{
|
116
|
+
throw new AssertionError("Never call.");
|
117
|
+
}
|
118
|
+
|
102
119
|
@Override
|
103
120
|
public void timestampColumn(Column column)
|
104
121
|
{
|
@@ -3,6 +3,7 @@ package org.embulk.filter.add_time.converter;
|
|
3
3
|
import org.embulk.spi.Column;
|
4
4
|
import org.embulk.spi.PageBuilder;
|
5
5
|
import org.embulk.spi.time.Timestamp;
|
6
|
+
import org.msgpack.value.Value;
|
6
7
|
|
7
8
|
public interface ValueConverter
|
8
9
|
{
|
@@ -18,5 +19,7 @@ public interface ValueConverter
|
|
18
19
|
|
19
20
|
void convertValue(Column column, String value, PageBuilder pageBuilder);
|
20
21
|
|
22
|
+
void convertValue(Column column, Value value, PageBuilder pageBuilder);
|
23
|
+
|
21
24
|
void convertValue(Column column, Timestamp value, PageBuilder pageBuilder);
|
22
25
|
}
|
@@ -3,6 +3,7 @@ package org.embulk.filter.add_time.converter;
|
|
3
3
|
import org.embulk.spi.Column;
|
4
4
|
import org.embulk.spi.PageBuilder;
|
5
5
|
import org.embulk.spi.time.Timestamp;
|
6
|
+
import org.msgpack.value.Value;
|
6
7
|
|
7
8
|
public class ValueNoConverter
|
8
9
|
implements ValueConverter
|
@@ -37,6 +38,12 @@ public class ValueNoConverter
|
|
37
38
|
pageBuilder.setString(column, value);
|
38
39
|
}
|
39
40
|
|
41
|
+
@Override
|
42
|
+
public void convertValue(Column column, Value value, PageBuilder pageBuilder)
|
43
|
+
{
|
44
|
+
pageBuilder.setJson(column, value);
|
45
|
+
}
|
46
|
+
|
40
47
|
@Override
|
41
48
|
public void convertValue(Column column, Timestamp value, PageBuilder pageBuilder)
|
42
49
|
{
|
@@ -0,0 +1,36 @@
|
|
1
|
+
package org.embulk.filter.add_time.reader;
|
2
|
+
|
3
|
+
import org.embulk.filter.add_time.converter.ValueConverter;
|
4
|
+
import org.embulk.spi.Column;
|
5
|
+
import org.embulk.spi.PageBuilder;
|
6
|
+
import org.embulk.spi.PageReader;
|
7
|
+
import org.msgpack.value.Value;
|
8
|
+
|
9
|
+
public class JsonColumnReader
|
10
|
+
extends AbstractColumnReader<JsonColumnReader>
|
11
|
+
{
|
12
|
+
protected Value value;
|
13
|
+
|
14
|
+
public JsonColumnReader(ValueConverter valueConverter)
|
15
|
+
{
|
16
|
+
super(valueConverter);
|
17
|
+
}
|
18
|
+
|
19
|
+
@Override
|
20
|
+
public void readNonNullValue(Column column, PageReader pageReader)
|
21
|
+
{
|
22
|
+
value = pageReader.getJson(column);
|
23
|
+
}
|
24
|
+
|
25
|
+
@Override
|
26
|
+
public void convertNonNullValue(Column column, PageBuilder pageBuilder)
|
27
|
+
{
|
28
|
+
valueConverter.convertValue(column, value, pageBuilder);
|
29
|
+
}
|
30
|
+
|
31
|
+
@Override
|
32
|
+
public void copyValueTo(JsonColumnReader columnReader)
|
33
|
+
{
|
34
|
+
columnReader.value = this.value;
|
35
|
+
}
|
36
|
+
}
|
@@ -18,11 +18,16 @@ import org.embulk.spi.util.Pages;
|
|
18
18
|
import org.junit.Before;
|
19
19
|
import org.junit.Rule;
|
20
20
|
import org.junit.Test;
|
21
|
+
import org.msgpack.value.Value;
|
21
22
|
|
22
23
|
import java.util.List;
|
23
24
|
|
24
25
|
import static org.junit.Assert.assertEquals;
|
26
|
+
import static org.junit.Assert.assertNull;
|
25
27
|
import static org.mockito.Mockito.spy;
|
28
|
+
import static org.msgpack.value.ValueFactory.newInteger;
|
29
|
+
import static org.msgpack.value.ValueFactory.newMap;
|
30
|
+
import static org.msgpack.value.ValueFactory.newString;
|
26
31
|
|
27
32
|
public class TestAddTimeFilterPlugin
|
28
33
|
{
|
@@ -39,7 +44,13 @@ public class TestAddTimeFilterPlugin
|
|
39
44
|
{
|
40
45
|
plugin = plugin();
|
41
46
|
config = runtime.getExec().newConfigSource();
|
42
|
-
inputSchema = schema("c0", Types.BOOLEAN, "c1", Types.LONG, "c2", Types.DOUBLE, "c3",
|
47
|
+
inputSchema = schema("c0", Types.BOOLEAN, "c1", Types.LONG, "c2", Types.DOUBLE, "c3",
|
48
|
+
Types.STRING, "c4", Types.TIMESTAMP, "c5", Types.JSON);
|
49
|
+
}
|
50
|
+
|
51
|
+
private Value newSimpleMap()
|
52
|
+
{
|
53
|
+
return newMap(newString("k"), newString("v"));
|
43
54
|
}
|
44
55
|
|
45
56
|
@Test
|
@@ -49,7 +60,7 @@ public class TestAddTimeFilterPlugin
|
|
49
60
|
ConfigSource conf = this.config.deepCopy()
|
50
61
|
.set("to_column", ImmutableMap.of("name", "time"))
|
51
62
|
.set("from_column", ImmutableMap.of("name", "c1", "unix_timestamp_unit", "sec"));
|
52
|
-
List<Page> pages = newPages(true, 1451646671L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
63
|
+
List<Page> pages = newPages(true, 1451646671L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
53
64
|
|
54
65
|
callTansaction(conf, inputSchema, pages);
|
55
66
|
|
@@ -62,7 +73,8 @@ public class TestAddTimeFilterPlugin
|
|
62
73
|
assertEquals(0.1, record[2]);
|
63
74
|
assertEquals("foo", record[3]);
|
64
75
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
65
|
-
assertEquals(
|
76
|
+
assertEquals(newSimpleMap(), record[5]);
|
77
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
66
78
|
}
|
67
79
|
}
|
68
80
|
|
@@ -70,7 +82,7 @@ public class TestAddTimeFilterPlugin
|
|
70
82
|
ConfigSource conf = this.config.deepCopy()
|
71
83
|
.set("to_column", ImmutableMap.of("name", "time"))
|
72
84
|
.set("from_column", ImmutableMap.of("name", "c4"));
|
73
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
85
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
74
86
|
|
75
87
|
callTansaction(conf, inputSchema, pages);
|
76
88
|
|
@@ -83,7 +95,8 @@ public class TestAddTimeFilterPlugin
|
|
83
95
|
assertEquals(0.1, record[2]);
|
84
96
|
assertEquals("foo", record[3]);
|
85
97
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
86
|
-
assertEquals(
|
98
|
+
assertEquals(newSimpleMap(), record[5]);
|
99
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
87
100
|
}
|
88
101
|
}
|
89
102
|
|
@@ -91,7 +104,7 @@ public class TestAddTimeFilterPlugin
|
|
91
104
|
ConfigSource conf = this.config.deepCopy()
|
92
105
|
.set("to_column", ImmutableMap.of("name", "time"))
|
93
106
|
.set("from_column", ImmutableMap.of("name", "c3"));
|
94
|
-
List<Page> pages = newPages(true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671));
|
107
|
+
List<Page> pages = newPages(true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
95
108
|
|
96
109
|
callTansaction(conf, inputSchema, pages);
|
97
110
|
|
@@ -104,7 +117,45 @@ public class TestAddTimeFilterPlugin
|
|
104
117
|
assertEquals(0.1, record[2]);
|
105
118
|
assertEquals("2016-01-01 11:11:11 UTC", record[3]);
|
106
119
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
107
|
-
assertEquals(
|
120
|
+
assertEquals(newSimpleMap(), record[5]);
|
121
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
122
|
+
}
|
123
|
+
}
|
124
|
+
|
125
|
+
{ // json type
|
126
|
+
ConfigSource conf = this.config.deepCopy()
|
127
|
+
.set("to_column", ImmutableMap.of("name", "time"))
|
128
|
+
.set("from_column", ImmutableMap.of(
|
129
|
+
"name", "c5",
|
130
|
+
"json_key", "k",
|
131
|
+
"unix_timestamp_unit", "sec",
|
132
|
+
"timestamp_format", "%Y-%m-%d %H:%M:%S %z"
|
133
|
+
));
|
134
|
+
|
135
|
+
List<Page> pages = newPages(
|
136
|
+
true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newString("2016-01-01 11:11:11 UTC")),
|
137
|
+
true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newString("embulk")),
|
138
|
+
true, 0L, 0.1, "2016-01-01 11:11:11 UTC", Timestamp.ofEpochSecond(1451646671), newMap(newString("k"), newInteger(1451646671))
|
139
|
+
);
|
140
|
+
|
141
|
+
callTansaction(conf, inputSchema, pages);
|
142
|
+
|
143
|
+
assertEquals(3, records.size());
|
144
|
+
Object[] record;
|
145
|
+
{
|
146
|
+
record = records.get(0);
|
147
|
+
assertEquals(newMap(newString("k"), newString("2016-01-01 11:11:11 UTC")), record[5]);
|
148
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
149
|
+
}
|
150
|
+
{
|
151
|
+
record = records.get(1);
|
152
|
+
assertEquals(newMap(newString("k"), newString("embulk")), record[5]);
|
153
|
+
assertNull(record[6]);
|
154
|
+
}
|
155
|
+
{
|
156
|
+
record = records.get(2);
|
157
|
+
assertEquals(newMap(newString("k"), newInteger(1451646671)), record[5]);
|
158
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
108
159
|
}
|
109
160
|
}
|
110
161
|
}
|
@@ -117,7 +168,7 @@ public class TestAddTimeFilterPlugin
|
|
117
168
|
ConfigSource conf = this.config.deepCopy()
|
118
169
|
.set("to_column", ImmutableMap.of("name", "time"))
|
119
170
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
|
120
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
171
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
121
172
|
|
122
173
|
callTansaction(conf, inputSchema, pages);
|
123
174
|
|
@@ -130,14 +181,15 @@ public class TestAddTimeFilterPlugin
|
|
130
181
|
assertEquals(0.1, record[2]);
|
131
182
|
assertEquals("foo", record[3]);
|
132
183
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
133
|
-
assertEquals(
|
184
|
+
assertEquals(newSimpleMap(), record[5]);
|
185
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
134
186
|
}
|
135
187
|
}
|
136
188
|
{ // specifies timestamp_format
|
137
189
|
ConfigSource conf = this.config.deepCopy()
|
138
190
|
.set("to_column", ImmutableMap.of("name", "time"))
|
139
191
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11.000 UTC", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
|
140
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
192
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
141
193
|
|
142
194
|
callTansaction(conf, inputSchema, pages);
|
143
195
|
|
@@ -150,14 +202,15 @@ public class TestAddTimeFilterPlugin
|
|
150
202
|
assertEquals(0.1, record[2]);
|
151
203
|
assertEquals("foo", record[3]);
|
152
204
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
153
|
-
assertEquals(
|
205
|
+
assertEquals(newSimpleMap(), record[5]);
|
206
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
154
207
|
}
|
155
208
|
}
|
156
209
|
{ // specifies unix_timestamp_unit
|
157
210
|
ConfigSource conf = this.config.deepCopy()
|
158
211
|
.set("to_column", ImmutableMap.of("name", "time"))
|
159
212
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", 1451646671, "unix_timestamp_unit", "sec"));
|
160
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
213
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
161
214
|
|
162
215
|
callTansaction(conf, inputSchema, pages);
|
163
216
|
|
@@ -170,7 +223,8 @@ public class TestAddTimeFilterPlugin
|
|
170
223
|
assertEquals(0.1, record[2]);
|
171
224
|
assertEquals("foo", record[3]);
|
172
225
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
173
|
-
assertEquals(
|
226
|
+
assertEquals(newSimpleMap(), record[5]);
|
227
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
174
228
|
}
|
175
229
|
}
|
176
230
|
|
@@ -180,7 +234,7 @@ public class TestAddTimeFilterPlugin
|
|
180
234
|
.set("to_column", ImmutableMap.of("name", "time"))
|
181
235
|
.set("from_value", ImmutableMap.of("mode", "incremental_time",
|
182
236
|
"from", "2016-01-01 11:11:11 UTC", "to", "2016-01-01 11:11:12 UTC"));
|
183
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
237
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
184
238
|
|
185
239
|
callTansaction(conf, inputSchema, pages);
|
186
240
|
|
@@ -193,7 +247,8 @@ public class TestAddTimeFilterPlugin
|
|
193
247
|
assertEquals(0.1, record[2]);
|
194
248
|
assertEquals("foo", record[3]);
|
195
249
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
196
|
-
assertEquals(
|
250
|
+
assertEquals(newSimpleMap(), record[5]);
|
251
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
197
252
|
}
|
198
253
|
}
|
199
254
|
{ // specifies timestamp_format
|
@@ -201,7 +256,7 @@ public class TestAddTimeFilterPlugin
|
|
201
256
|
.set("to_column", ImmutableMap.of("name", "time"))
|
202
257
|
.set("from_value", ImmutableMap.of("mode", "incremental_time",
|
203
258
|
"from", "2016-01-01 11:11:11.000 UTC", "to", "2016-01-01 11:11:12.000 UTC", "timestamp_format", "%Y-%m-%d %H:%M:%S.%N %Z"));
|
204
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
259
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
205
260
|
|
206
261
|
callTansaction(conf, inputSchema, pages);
|
207
262
|
|
@@ -214,7 +269,8 @@ public class TestAddTimeFilterPlugin
|
|
214
269
|
assertEquals(0.1, record[2]);
|
215
270
|
assertEquals("foo", record[3]);
|
216
271
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
217
|
-
assertEquals(
|
272
|
+
assertEquals(newSimpleMap(), record[5]);
|
273
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
218
274
|
}
|
219
275
|
}
|
220
276
|
{ // specifies unix_timestamp_unit
|
@@ -222,7 +278,7 @@ public class TestAddTimeFilterPlugin
|
|
222
278
|
.set("to_column", ImmutableMap.of("name", "time"))
|
223
279
|
.set("from_value", ImmutableMap.of("mode", "incremental_time",
|
224
280
|
"from", 1451646671, "to", 1451646672, "unix_timestamp_unit", "sec"));
|
225
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
281
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
226
282
|
|
227
283
|
callTansaction(conf, inputSchema, pages);
|
228
284
|
|
@@ -235,7 +291,8 @@ public class TestAddTimeFilterPlugin
|
|
235
291
|
assertEquals(0.1, record[2]);
|
236
292
|
assertEquals("foo", record[3]);
|
237
293
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
238
|
-
assertEquals(
|
294
|
+
assertEquals(newSimpleMap(), record[5]);
|
295
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
239
296
|
}
|
240
297
|
}
|
241
298
|
|
@@ -244,7 +301,7 @@ public class TestAddTimeFilterPlugin
|
|
244
301
|
ConfigSource conf = this.config.deepCopy()
|
245
302
|
.set("to_column", ImmutableMap.of("name", "time"))
|
246
303
|
.set("from_value", ImmutableMap.of("mode", "upload_time"));
|
247
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
304
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
248
305
|
|
249
306
|
callTansaction(conf, inputSchema, pages);
|
250
307
|
|
@@ -257,7 +314,8 @@ public class TestAddTimeFilterPlugin
|
|
257
314
|
assertEquals(0.1, record[2]);
|
258
315
|
assertEquals("foo", record[3]);
|
259
316
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
260
|
-
assertEquals(
|
317
|
+
assertEquals(newSimpleMap(), record[5]);
|
318
|
+
assertEquals(runtime.getExec().getTransactionTime(), record[6]);
|
261
319
|
}
|
262
320
|
}
|
263
321
|
}
|
@@ -270,7 +328,7 @@ public class TestAddTimeFilterPlugin
|
|
270
328
|
ConfigSource conf = this.config.deepCopy()
|
271
329
|
.set("to_column", ImmutableMap.of("name", "time"))
|
272
330
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
|
273
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
331
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
274
332
|
|
275
333
|
callTansaction(conf, inputSchema, pages);
|
276
334
|
|
@@ -283,7 +341,8 @@ public class TestAddTimeFilterPlugin
|
|
283
341
|
assertEquals(0.1, record[2]);
|
284
342
|
assertEquals("foo", record[3]);
|
285
343
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
286
|
-
assertEquals(
|
344
|
+
assertEquals(newSimpleMap(), record[5]);
|
345
|
+
assertEquals(1451646671L, ((Timestamp) record[6]).getEpochSecond());
|
287
346
|
}
|
288
347
|
}
|
289
348
|
|
@@ -292,7 +351,7 @@ public class TestAddTimeFilterPlugin
|
|
292
351
|
ConfigSource conf = this.config.deepCopy()
|
293
352
|
.set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "sec"))
|
294
353
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
|
295
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
354
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
296
355
|
|
297
356
|
callTansaction(conf, inputSchema, pages);
|
298
357
|
|
@@ -305,14 +364,15 @@ public class TestAddTimeFilterPlugin
|
|
305
364
|
assertEquals(0.1, record[2]);
|
306
365
|
assertEquals("foo", record[3]);
|
307
366
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
308
|
-
assertEquals(
|
367
|
+
assertEquals(newSimpleMap(), record[5]);
|
368
|
+
assertEquals(1451646671L, record[6]);
|
309
369
|
}
|
310
370
|
}
|
311
371
|
{ // unix_timestamp: milli
|
312
372
|
ConfigSource conf = this.config.deepCopy()
|
313
373
|
.set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "milli"))
|
314
374
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
|
315
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
375
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
316
376
|
|
317
377
|
callTansaction(conf, inputSchema, pages);
|
318
378
|
|
@@ -325,14 +385,15 @@ public class TestAddTimeFilterPlugin
|
|
325
385
|
assertEquals(0.1, record[2]);
|
326
386
|
assertEquals("foo", record[3]);
|
327
387
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
328
|
-
assertEquals(
|
388
|
+
assertEquals(newSimpleMap(), record[5]);
|
389
|
+
assertEquals(1451646671000L, record[6]);
|
329
390
|
}
|
330
391
|
}
|
331
392
|
{ // unix_timestamp: micro
|
332
393
|
ConfigSource conf = this.config.deepCopy()
|
333
394
|
.set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "micro"))
|
334
395
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
|
335
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
396
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
336
397
|
|
337
398
|
callTansaction(conf, inputSchema, pages);
|
338
399
|
|
@@ -345,14 +406,15 @@ public class TestAddTimeFilterPlugin
|
|
345
406
|
assertEquals(0.1, record[2]);
|
346
407
|
assertEquals("foo", record[3]);
|
347
408
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
348
|
-
assertEquals(
|
409
|
+
assertEquals(newSimpleMap(), record[5]);
|
410
|
+
assertEquals(1451646671000000L, record[6]);
|
349
411
|
}
|
350
412
|
}
|
351
413
|
{ // unix_timestamp: nano
|
352
414
|
ConfigSource conf = this.config.deepCopy()
|
353
415
|
.set("to_column", ImmutableMap.of("name", "time", "type", "long", "unix_timestamp_unit", "nano"))
|
354
416
|
.set("from_value", ImmutableMap.of("mode", "fixed_time", "value", "2016-01-01 11:11:11 UTC"));
|
355
|
-
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671));
|
417
|
+
List<Page> pages = newPages(true, 0L, 0.1, "foo", Timestamp.ofEpochSecond(1451646671), newSimpleMap());
|
356
418
|
|
357
419
|
callTansaction(conf, inputSchema, pages);
|
358
420
|
|
@@ -365,7 +427,8 @@ public class TestAddTimeFilterPlugin
|
|
365
427
|
assertEquals(0.1, record[2]);
|
366
428
|
assertEquals("foo", record[3]);
|
367
429
|
assertEquals(1451646671L, ((Timestamp) record[4]).getEpochSecond());
|
368
|
-
assertEquals(
|
430
|
+
assertEquals(newSimpleMap(), record[5]);
|
431
|
+
assertEquals(1451646671000000000L, record[6]);
|
369
432
|
}
|
370
433
|
}
|
371
434
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-add_time
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -61,6 +61,7 @@ files:
|
|
61
61
|
- src/main/java/org/embulk/filter/add_time/AddTimeFilterPlugin.java
|
62
62
|
- src/main/java/org/embulk/filter/add_time/converter/ColumnConverter.java
|
63
63
|
- src/main/java/org/embulk/filter/add_time/converter/ColumnDuplicator.java
|
64
|
+
- src/main/java/org/embulk/filter/add_time/converter/JsonValueCastConverter.java
|
64
65
|
- src/main/java/org/embulk/filter/add_time/converter/LongValueCastConverter.java
|
65
66
|
- src/main/java/org/embulk/filter/add_time/converter/SchemaConverter.java
|
66
67
|
- src/main/java/org/embulk/filter/add_time/converter/SimpleColumnConverter.java
|
@@ -73,13 +74,14 @@ files:
|
|
73
74
|
- src/main/java/org/embulk/filter/add_time/reader/BooleanColumnReader.java
|
74
75
|
- src/main/java/org/embulk/filter/add_time/reader/ColumnReader.java
|
75
76
|
- src/main/java/org/embulk/filter/add_time/reader/DoubleColumnReader.java
|
77
|
+
- src/main/java/org/embulk/filter/add_time/reader/JsonColumnReader.java
|
76
78
|
- src/main/java/org/embulk/filter/add_time/reader/LongColumnReader.java
|
77
79
|
- src/main/java/org/embulk/filter/add_time/reader/StringColumnReader.java
|
78
80
|
- src/main/java/org/embulk/filter/add_time/reader/TimeValueGenerator.java
|
79
81
|
- src/main/java/org/embulk/filter/add_time/reader/TimestampColumnReader.java
|
80
82
|
- src/test/java/org/embulk/filter/add_time/TestAddTimeFilterPlugin.java
|
81
83
|
- src/test/java/org/embulk/filter/add_time/converter/TestSchemaConverter.java
|
82
|
-
- classpath/embulk-filter-add_time-0.
|
84
|
+
- classpath/embulk-filter-add_time-0.2.0.jar
|
83
85
|
homepage: https://github.com/treasure-data/embulk-filter-add_time
|
84
86
|
licenses:
|
85
87
|
- Apache 2.0
|