embulk-filter-timestamp_format 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/CHANGELOG.md +7 -0
- data/README.md +46 -1
- data/bench/config_java.yml +14 -0
- data/bench/config_jruby.yml +14 -0
- data/bench/gen_dummy.rb +5 -0
- data/build.gradle +1 -1
- data/example/double.csv +2 -0
- data/example/double.yml +20 -0
- data/example/{json_example.jsonl → example.jsonl} +0 -0
- data/example/example.yml +4 -12
- data/example/example2.csv +2 -0
- data/example/example2.yml +14 -0
- data/example/json_double.jsonl +1 -0
- data/example/json_double.yml +14 -0
- data/example/json_long.jsonl +1 -0
- data/example/json_long.yml +14 -0
- data/example/json_string.jsonl +2 -0
- data/example/json_string.yml +14 -0
- data/example/long.csv +1 -0
- data/example/long.yml +20 -0
- data/example/string.csv +4 -0
- data/example/{string_example.yml → string.yml} +6 -5
- data/example/string_java.yml +23 -0
- data/example/timestamp.csv +2 -0
- data/example/{timestamp_example.yml → timestamp.yml} +4 -4
- data/src/main/java/org/embulk/filter/timestamp_format/ColumnCaster.java +107 -14
- data/src/main/java/org/embulk/filter/timestamp_format/ColumnVisitorImpl.java +104 -33
- data/src/main/java/org/embulk/filter/timestamp_format/JsonCaster.java +61 -4
- data/src/main/java/org/embulk/filter/timestamp_format/JsonVisitor.java +8 -0
- data/src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java +28 -17
- data/src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java +36 -5
- data/src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java +57 -26
- data/src/main/java/org/embulk/filter/timestamp_format/TimestampUnit.java +112 -0
- data/src/main/java/org/embulk/filter/timestamp_format/TimestampUnitDeserializer.java +54 -0
- data/src/main/java/org/embulk/filter/timestamp_format/cast/DoubleCast.java +32 -0
- data/src/main/java/org/embulk/filter/timestamp_format/cast/LongCast.java +32 -0
- data/src/main/java/org/embulk/filter/timestamp_format/cast/StringCast.java +20 -4
- data/src/main/java/org/embulk/filter/timestamp_format/cast/TimestampCast.java +5 -6
- data/src/test/java/org/embulk/filter/timestamp_format/TestTimestampUnit.java +192 -0
- metadata +29 -8
- data/example/json_example.yml +0 -14
- data/src/test/java/org/embulk/filter/TestTimestampFormatFilterPlugin.java +0 -5
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
package org.embulk.filter.timestamp_format;
|
|
2
|
+
|
|
3
|
+
import org.embulk.spi.time.Timestamp;
|
|
4
|
+
import org.junit.Test;
|
|
5
|
+
|
|
6
|
+
import static org.junit.Assert.assertEquals;
|
|
7
|
+
|
|
8
|
+
public class TestTimestampUnit
|
|
9
|
+
{
|
|
10
|
+
@Test
|
|
11
|
+
public void testLongToTimestamp()
|
|
12
|
+
{
|
|
13
|
+
long epochSecond = 1462087147L;
|
|
14
|
+
long epochNanoSecond = 1462087147100200300L;
|
|
15
|
+
Timestamp timestamp;
|
|
16
|
+
|
|
17
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond / 1000000000, TimestampUnit.Second);
|
|
18
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
19
|
+
assertEquals(0, timestamp.getNano());
|
|
20
|
+
|
|
21
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond / 1000000, TimestampUnit.MilliSecond);
|
|
22
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
23
|
+
assertEquals(100000000, timestamp.getNano());
|
|
24
|
+
|
|
25
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond / 1000, TimestampUnit.MicroSecond);
|
|
26
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
27
|
+
assertEquals(100200000, timestamp.getNano());
|
|
28
|
+
|
|
29
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond, TimestampUnit.NanoSecond);
|
|
30
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
31
|
+
assertEquals(100200300, timestamp.getNano());
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
@Test
|
|
35
|
+
public void testDoubleToTimestamp()
|
|
36
|
+
{
|
|
37
|
+
long epochSecond = 1462087147L;
|
|
38
|
+
double epochNanoSecond = 1462087147100200192.0;
|
|
39
|
+
Timestamp timestamp;
|
|
40
|
+
|
|
41
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond / 1000000000, TimestampUnit.Second);
|
|
42
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
43
|
+
assertEquals(100200192, timestamp.getNano(), 200);
|
|
44
|
+
|
|
45
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond / 1000000, TimestampUnit.MilliSecond);
|
|
46
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
47
|
+
assertEquals(100200192, timestamp.getNano(), 200);
|
|
48
|
+
|
|
49
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond / 1000, TimestampUnit.MicroSecond);
|
|
50
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
51
|
+
assertEquals(100200192, timestamp.getNano(), 200);
|
|
52
|
+
|
|
53
|
+
timestamp = TimestampUnit.toTimestamp(epochNanoSecond, TimestampUnit.NanoSecond);
|
|
54
|
+
assertEquals(epochSecond, timestamp.getEpochSecond());
|
|
55
|
+
assertEquals(100200192, timestamp.getNano());
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
@Test
|
|
59
|
+
public void testTimestampToLong()
|
|
60
|
+
{
|
|
61
|
+
long epochNanoSecond = 1462087147100200300L;
|
|
62
|
+
Timestamp timestamp = Timestamp.ofEpochSecond(0, epochNanoSecond);
|
|
63
|
+
long value;
|
|
64
|
+
|
|
65
|
+
value = TimestampUnit.toLong(timestamp, TimestampUnit.Second);
|
|
66
|
+
assertEquals(epochNanoSecond / 1000000000, value);
|
|
67
|
+
|
|
68
|
+
value = TimestampUnit.toLong(timestamp, TimestampUnit.MilliSecond);
|
|
69
|
+
assertEquals(epochNanoSecond / 1000000, value);
|
|
70
|
+
|
|
71
|
+
value = TimestampUnit.toLong(timestamp, TimestampUnit.MicroSecond);
|
|
72
|
+
assertEquals(epochNanoSecond / 1000, value);
|
|
73
|
+
|
|
74
|
+
value = TimestampUnit.toLong(timestamp, TimestampUnit.NanoSecond);
|
|
75
|
+
assertEquals(epochNanoSecond, value);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
@Test
|
|
79
|
+
public void testTimestampToDouble()
|
|
80
|
+
{
|
|
81
|
+
long epochNanoSecond = 1462087147100200192L;
|
|
82
|
+
Timestamp timestamp = Timestamp.ofEpochSecond(0, epochNanoSecond);
|
|
83
|
+
double value;
|
|
84
|
+
|
|
85
|
+
value = TimestampUnit.toDouble(timestamp, TimestampUnit.Second);
|
|
86
|
+
assertEquals(epochNanoSecond / 1000000000.0, value, 2);
|
|
87
|
+
|
|
88
|
+
value = TimestampUnit.toDouble(timestamp, TimestampUnit.MilliSecond);
|
|
89
|
+
assertEquals(epochNanoSecond / 1000000.0, value, 2);
|
|
90
|
+
|
|
91
|
+
value = TimestampUnit.toDouble(timestamp, TimestampUnit.MicroSecond);
|
|
92
|
+
assertEquals(epochNanoSecond / 1000.0, value, 2);
|
|
93
|
+
|
|
94
|
+
value = TimestampUnit.toDouble(timestamp, TimestampUnit.NanoSecond);
|
|
95
|
+
assertEquals(epochNanoSecond / 1.0, value, 2);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
@Test
|
|
99
|
+
public void testLongChangeUnit()
|
|
100
|
+
{
|
|
101
|
+
long epochNanoSecond = 1462087147100200300L;
|
|
102
|
+
long value;
|
|
103
|
+
|
|
104
|
+
// from second
|
|
105
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.Second);
|
|
106
|
+
assertEquals(epochNanoSecond / 1000000000, value);
|
|
107
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.MilliSecond);
|
|
108
|
+
assertEquals(epochNanoSecond / 1000000000 * 1000, value);
|
|
109
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.MicroSecond);
|
|
110
|
+
assertEquals(epochNanoSecond / 1000000000 * 1000000, value);
|
|
111
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.NanoSecond);
|
|
112
|
+
assertEquals(epochNanoSecond / 1000000000 * 1000000000, value);
|
|
113
|
+
|
|
114
|
+
// from milli second
|
|
115
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.Second);
|
|
116
|
+
assertEquals(epochNanoSecond / 1000000 / 1000, value);
|
|
117
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.MilliSecond);
|
|
118
|
+
assertEquals(epochNanoSecond / 1000000, value);
|
|
119
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.MicroSecond);
|
|
120
|
+
assertEquals(epochNanoSecond / 1000000 * 1000, value);
|
|
121
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.NanoSecond);
|
|
122
|
+
assertEquals(epochNanoSecond / 1000000 * 1000000, value);
|
|
123
|
+
|
|
124
|
+
// from micro second
|
|
125
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.Second);
|
|
126
|
+
assertEquals(epochNanoSecond / 1000 / 1000000, value);
|
|
127
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.MilliSecond);
|
|
128
|
+
assertEquals(epochNanoSecond / 1000 / 1000, value);
|
|
129
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.MicroSecond);
|
|
130
|
+
assertEquals(epochNanoSecond / 1000, value);
|
|
131
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.NanoSecond);
|
|
132
|
+
assertEquals(epochNanoSecond / 1000 * 1000, value);
|
|
133
|
+
|
|
134
|
+
// from nano second
|
|
135
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.Second);
|
|
136
|
+
assertEquals(epochNanoSecond / 1000000000, value);
|
|
137
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.MilliSecond);
|
|
138
|
+
assertEquals(epochNanoSecond / 1000000, value);
|
|
139
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.MicroSecond);
|
|
140
|
+
assertEquals(epochNanoSecond / 1000, value);
|
|
141
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.NanoSecond);
|
|
142
|
+
assertEquals(epochNanoSecond, value);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
@Test
|
|
146
|
+
public void testDoubleChangeUnit()
|
|
147
|
+
{
|
|
148
|
+
double epochNanoSecond = 1462087147100200192L;
|
|
149
|
+
double value;
|
|
150
|
+
|
|
151
|
+
// from second
|
|
152
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.Second);
|
|
153
|
+
assertEquals(epochNanoSecond / 1000000000, value, 2);
|
|
154
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.MilliSecond);
|
|
155
|
+
assertEquals(epochNanoSecond / 1000000000 * 1000, value, 2);
|
|
156
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.MicroSecond);
|
|
157
|
+
assertEquals(epochNanoSecond / 1000000000 * 1000000, value, 2);
|
|
158
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000000, TimestampUnit.Second, TimestampUnit.NanoSecond);
|
|
159
|
+
assertEquals(epochNanoSecond / 1000000000 * 1000000000, value, 2);
|
|
160
|
+
|
|
161
|
+
// from milli second
|
|
162
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.Second);
|
|
163
|
+
assertEquals(epochNanoSecond / 1000000 / 1000, value ,2);
|
|
164
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.MilliSecond);
|
|
165
|
+
assertEquals(epochNanoSecond / 1000000, value, 2);
|
|
166
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.MicroSecond);
|
|
167
|
+
assertEquals(epochNanoSecond / 1000000 * 1000, value, 2);
|
|
168
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000000, TimestampUnit.MilliSecond, TimestampUnit.NanoSecond);
|
|
169
|
+
assertEquals(epochNanoSecond / 1000000 * 1000000, value, 2);
|
|
170
|
+
|
|
171
|
+
// from micro second
|
|
172
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.Second);
|
|
173
|
+
assertEquals(epochNanoSecond / 1000 / 1000000, value, 2);
|
|
174
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.MilliSecond);
|
|
175
|
+
assertEquals(epochNanoSecond / 1000 / 1000, value, 2);
|
|
176
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.MicroSecond);
|
|
177
|
+
assertEquals(epochNanoSecond / 1000, value, 2);
|
|
178
|
+
value = TimestampUnit.changeUnit(epochNanoSecond / 1000, TimestampUnit.MicroSecond, TimestampUnit.NanoSecond);
|
|
179
|
+
assertEquals(epochNanoSecond / 1000 * 1000, value, 2);
|
|
180
|
+
|
|
181
|
+
// from nano second
|
|
182
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.Second);
|
|
183
|
+
assertEquals(epochNanoSecond / 1000000000, value, 2);
|
|
184
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.MilliSecond);
|
|
185
|
+
assertEquals(epochNanoSecond / 1000000, value, 2);
|
|
186
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.MicroSecond);
|
|
187
|
+
assertEquals(epochNanoSecond / 1000, value, 2);
|
|
188
|
+
value = TimestampUnit.changeUnit(epochNanoSecond, TimestampUnit.NanoSecond, TimestampUnit.NanoSecond);
|
|
189
|
+
assertEquals(epochNanoSecond, value, 2);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-timestamp_format
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Naotoshi Seo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-05-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -50,13 +50,30 @@ files:
|
|
|
50
50
|
- CHANGELOG.md
|
|
51
51
|
- LICENSE.txt
|
|
52
52
|
- README.md
|
|
53
|
+
- bench/config_java.yml
|
|
54
|
+
- bench/config_jruby.yml
|
|
55
|
+
- bench/gen_dummy.rb
|
|
53
56
|
- build.gradle
|
|
54
57
|
- config/checkstyle/checkstyle.xml
|
|
58
|
+
- example/double.csv
|
|
59
|
+
- example/double.yml
|
|
60
|
+
- example/example.jsonl
|
|
55
61
|
- example/example.yml
|
|
56
|
-
- example/
|
|
57
|
-
- example/
|
|
58
|
-
- example/
|
|
59
|
-
- example/
|
|
62
|
+
- example/example2.csv
|
|
63
|
+
- example/example2.yml
|
|
64
|
+
- example/json_double.jsonl
|
|
65
|
+
- example/json_double.yml
|
|
66
|
+
- example/json_long.jsonl
|
|
67
|
+
- example/json_long.yml
|
|
68
|
+
- example/json_string.jsonl
|
|
69
|
+
- example/json_string.yml
|
|
70
|
+
- example/long.csv
|
|
71
|
+
- example/long.yml
|
|
72
|
+
- example/string.csv
|
|
73
|
+
- example/string.yml
|
|
74
|
+
- example/string_java.yml
|
|
75
|
+
- example/timestamp.csv
|
|
76
|
+
- example/timestamp.yml
|
|
60
77
|
- gradle/wrapper/gradle-wrapper.jar
|
|
61
78
|
- gradle/wrapper/gradle-wrapper.properties
|
|
62
79
|
- gradlew
|
|
@@ -69,10 +86,14 @@ files:
|
|
|
69
86
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java
|
|
70
87
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java
|
|
71
88
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java
|
|
89
|
+
- src/main/java/org/embulk/filter/timestamp_format/TimestampUnit.java
|
|
90
|
+
- src/main/java/org/embulk/filter/timestamp_format/TimestampUnitDeserializer.java
|
|
91
|
+
- src/main/java/org/embulk/filter/timestamp_format/cast/DoubleCast.java
|
|
92
|
+
- src/main/java/org/embulk/filter/timestamp_format/cast/LongCast.java
|
|
72
93
|
- src/main/java/org/embulk/filter/timestamp_format/cast/StringCast.java
|
|
73
94
|
- src/main/java/org/embulk/filter/timestamp_format/cast/TimestampCast.java
|
|
74
|
-
- src/test/java/org/embulk/filter/
|
|
75
|
-
- classpath/embulk-filter-timestamp_format-0.1.
|
|
95
|
+
- src/test/java/org/embulk/filter/timestamp_format/TestTimestampUnit.java
|
|
96
|
+
- classpath/embulk-filter-timestamp_format-0.1.6.jar
|
|
76
97
|
homepage: https://github.com/sonots/embulk-filter-timestamp_format
|
|
77
98
|
licenses:
|
|
78
99
|
- MIT
|
data/example/json_example.yml
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
in:
|
|
2
|
-
type: file
|
|
3
|
-
path_prefix: example/json_example.jsonl
|
|
4
|
-
parser:
|
|
5
|
-
type: json
|
|
6
|
-
filters:
|
|
7
|
-
- type: timestamp_format
|
|
8
|
-
default_to_timezone: "Asia/Tokyo"
|
|
9
|
-
default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
|
|
10
|
-
columns:
|
|
11
|
-
- {name: "$.record.timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
|
|
12
|
-
- {name: "$.record.nested.nested[0].timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
|
|
13
|
-
out:
|
|
14
|
-
type: "null"
|