embulk-filter-timestamp_format 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/build.gradle +1 -1
- data/example/example.jsonl +2 -2
- data/example/example.yml +1 -0
- data/src/main/java/org/embulk/filter/timestamp_format/ColumnVisitorImpl.java +251 -0
- data/src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java +9 -190
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c06f4d7efebebe0e2abb454b7f6545e6ea843026
|
|
4
|
+
data.tar.gz: d98aceb871fe643ed6afe46501758d15884ee124
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d8aae59c031cd582e3df88466417822e1f952ec9573d1df8fe189109496e909c9ca71d6e448d544ac8ffa64611ec2689651fccb7caa4e5de8559ff3c30bc6213
|
|
7
|
+
data.tar.gz: 938432021acdece8554c884e033f5e4f2bcac15d2c0407fff3134f8757f84f52201e82f14ac46a4d954c900382d3d174e79ee04b94dc76848867a1b4ae4afe9e
|
data/CHANGELOG.md
CHANGED
data/build.gradle
CHANGED
data/example/example.jsonl
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
{"timestamp":"2015-07-12 15:00:00 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00 UTC"}]}}}
|
|
2
|
-
{"timestamp":"2015-07-12 15:00:00.1 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00.1 UTC"}]}}}
|
|
1
|
+
{"timestamp":"2015-07-12 15:00:00 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00 UTC"}]},"ignore_record":{"timestamp":"2015-07-12 15:00:00 UTC"}}
|
|
2
|
+
{"timestamp":"2015-07-12 15:00:00.1 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00.1 UTC"}]},"ignore_record":{"timestamp":"2015-07-12 15:00:00.1 UTC"}}
|
data/example/example.yml
CHANGED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
package org.embulk.filter.timestamp_format;
|
|
2
|
+
|
|
3
|
+
import com.google.common.base.Throwables;
|
|
4
|
+
import org.embulk.spi.PageReader;
|
|
5
|
+
import org.msgpack.value.ArrayValue;
|
|
6
|
+
import org.msgpack.value.MapValue;
|
|
7
|
+
import org.msgpack.value.Value;
|
|
8
|
+
import org.msgpack.value.ValueFactory;
|
|
9
|
+
|
|
10
|
+
import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.ColumnConfig;
|
|
11
|
+
import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.PluginTask;
|
|
12
|
+
|
|
13
|
+
import org.embulk.spi.Column;
|
|
14
|
+
import org.embulk.spi.ColumnVisitor;
|
|
15
|
+
import org.embulk.spi.Exec;
|
|
16
|
+
import org.embulk.spi.PageBuilder;
|
|
17
|
+
import org.embulk.spi.time.Timestamp;
|
|
18
|
+
import org.embulk.spi.time.TimestampParseException;
|
|
19
|
+
import org.joda.time.DateTimeZone;
|
|
20
|
+
import org.slf4j.Logger;
|
|
21
|
+
|
|
22
|
+
import java.util.HashMap;
|
|
23
|
+
import java.util.HashSet;
|
|
24
|
+
import java.util.List;
|
|
25
|
+
import java.util.Map;
|
|
26
|
+
import java.util.Objects;
|
|
27
|
+
|
|
28
|
+
public class ColumnVisitorImpl
|
|
29
|
+
implements ColumnVisitor
|
|
30
|
+
{
|
|
31
|
+
private static final Logger logger = Exec.getLogger(TimestampFormatFilterPlugin.class);
|
|
32
|
+
private final PluginTask task;
|
|
33
|
+
private final PageReader pageReader;
|
|
34
|
+
private final PageBuilder pageBuilder;
|
|
35
|
+
private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<String, TimestampParser>();
|
|
36
|
+
private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<String, TimestampFormatter>();
|
|
37
|
+
private final HashSet<String> shouldVisitRecursivelySet = new HashSet<String>();
|
|
38
|
+
|
|
39
|
+
ColumnVisitorImpl(PluginTask task, PageReader pageReader, PageBuilder pageBuilder)
|
|
40
|
+
{
|
|
41
|
+
this.task = task;
|
|
42
|
+
this.pageReader = pageReader;
|
|
43
|
+
this.pageBuilder = pageBuilder;
|
|
44
|
+
|
|
45
|
+
buildTimestampParserMap();
|
|
46
|
+
buildTimestampFormatterMap();
|
|
47
|
+
buildShouldVisitRecursivelySet();;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
private void buildTimestampParserMap()
|
|
51
|
+
{
|
|
52
|
+
// columnName or jsonPath => TimestampParser
|
|
53
|
+
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
54
|
+
TimestampParser parser = getTimestampParser(columnConfig, task);
|
|
55
|
+
this.timestampParserMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
|
|
60
|
+
{
|
|
61
|
+
DateTimeZone timezone = columnConfig.getFromTimeZone().or(task.getDefaultFromTimeZone());
|
|
62
|
+
List<String> formatList = columnConfig.getFromFormat().or(task.getDefaultFromTimestampFormat());
|
|
63
|
+
return new TimestampParser(task.getJRuby(), formatList, timezone);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
private void buildTimestampFormatterMap()
|
|
67
|
+
{
|
|
68
|
+
// columnName or jsonPath => TimestampFormatter
|
|
69
|
+
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
70
|
+
TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
|
|
71
|
+
this.timestampFormatterMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
|
|
76
|
+
{
|
|
77
|
+
String format = columnConfig.getToFormat().or(task.getDefaultToTimestampFormat());
|
|
78
|
+
DateTimeZone timezone = columnConfig.getToTimeZone().or(task.getDefaultToTimeZone());
|
|
79
|
+
return new TimestampFormatter(task.getJRuby(), format, timezone);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
private void buildShouldVisitRecursivelySet()
|
|
84
|
+
{
|
|
85
|
+
// json partial path => Boolean to avoid unnecessary type: json visit
|
|
86
|
+
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
87
|
+
String name = columnConfig.getName();
|
|
88
|
+
if (!name.startsWith("$.")) {
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
String[] parts = name.split("\\.");
|
|
92
|
+
StringBuilder partialPath = new StringBuilder("$");
|
|
93
|
+
for (int i = 1; i < parts.length; i++) {
|
|
94
|
+
if (parts[i].contains("[")) {
|
|
95
|
+
String[] arrayParts = parts[i].split("\\[");
|
|
96
|
+
partialPath.append(".").append(arrayParts[0]);
|
|
97
|
+
this.shouldVisitRecursivelySet.add(partialPath.toString());
|
|
98
|
+
for (int j = 1; j < arrayParts.length; j++) {
|
|
99
|
+
partialPath.append("[").append(arrayParts[j]);
|
|
100
|
+
this.shouldVisitRecursivelySet.add(partialPath.toString());
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
partialPath.append(".").append(parts[i]);
|
|
105
|
+
this.shouldVisitRecursivelySet.add(partialPath.toString());
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private boolean shouldVisitRecursively(String name)
|
|
112
|
+
{
|
|
113
|
+
return shouldVisitRecursivelySet.contains(name);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
private Value formatTimestampStringRecursively(PluginTask task, String path, Value value)
|
|
117
|
+
throws TimestampParseException
|
|
118
|
+
{
|
|
119
|
+
if (!shouldVisitRecursively(path)) {
|
|
120
|
+
return value;
|
|
121
|
+
}
|
|
122
|
+
if (value.isArrayValue()) {
|
|
123
|
+
ArrayValue arrayValue = value.asArrayValue();
|
|
124
|
+
int size = arrayValue.size();
|
|
125
|
+
Value[] newValue = new Value[size];
|
|
126
|
+
for (int i = 0; i < size; i++) {
|
|
127
|
+
String k = new StringBuilder(path).append("[").append(Integer.toString(i)).append("]").toString();
|
|
128
|
+
Value v = arrayValue.get(i);
|
|
129
|
+
newValue[i] = formatTimestampStringRecursively(task, k, v);
|
|
130
|
+
}
|
|
131
|
+
return ValueFactory.newArray(newValue, true);
|
|
132
|
+
}
|
|
133
|
+
else if (value.isMapValue()) {
|
|
134
|
+
MapValue mapValue = value.asMapValue();
|
|
135
|
+
int size = mapValue.size() * 2;
|
|
136
|
+
Value[] newValue = new Value[size];
|
|
137
|
+
int i = 0;
|
|
138
|
+
for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
|
|
139
|
+
Value k = entry.getKey();
|
|
140
|
+
Value v = entry.getValue();
|
|
141
|
+
String newPath = new StringBuilder(path).append(".").append(k.asStringValue().asString()).toString();
|
|
142
|
+
Value r = formatTimestampStringRecursively(task, newPath, v);
|
|
143
|
+
newValue[i++] = k;
|
|
144
|
+
newValue[i++] = r;
|
|
145
|
+
}
|
|
146
|
+
return ValueFactory.newMap(newValue, true);
|
|
147
|
+
}
|
|
148
|
+
else if (value.isStringValue()) {
|
|
149
|
+
String stringValue = value.asStringValue().asString();
|
|
150
|
+
String newValue = formatTimestampString(task, path, stringValue);
|
|
151
|
+
return (Objects.equals(newValue, stringValue)) ? value : ValueFactory.newString(newValue);
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
return value;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
private String formatTimestampString(PluginTask task, String name, String value)
|
|
159
|
+
throws TimestampParseException
|
|
160
|
+
{
|
|
161
|
+
TimestampParser parser = timestampParserMap.get(name);
|
|
162
|
+
TimestampFormatter formatter = timestampFormatterMap.get(name);
|
|
163
|
+
if (formatter == null || parser == null) {
|
|
164
|
+
return value;
|
|
165
|
+
}
|
|
166
|
+
try {
|
|
167
|
+
Timestamp timestamp = parser.parse(value);
|
|
168
|
+
return formatter.format(timestamp);
|
|
169
|
+
}
|
|
170
|
+
catch (TimestampParseException ex) {
|
|
171
|
+
if (task.getStopOnInvalidRecord()) {
|
|
172
|
+
throw Throwables.propagate(ex);
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
logger.warn("invalid value \"{}\":\"{}\"", name, value);
|
|
176
|
+
return value;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@Override
|
|
183
|
+
public void booleanColumn(Column column)
|
|
184
|
+
{
|
|
185
|
+
if (pageReader.isNull(column)) {
|
|
186
|
+
pageBuilder.setNull(column);
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
pageBuilder.setBoolean(column, pageReader.getBoolean(column));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
@Override
|
|
194
|
+
public void longColumn(Column column)
|
|
195
|
+
{
|
|
196
|
+
if (pageReader.isNull(column)) {
|
|
197
|
+
pageBuilder.setNull(column);
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
pageBuilder.setLong(column, pageReader.getLong(column));
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
@Override
|
|
205
|
+
public void doubleColumn(Column column)
|
|
206
|
+
{
|
|
207
|
+
if (pageReader.isNull(column)) {
|
|
208
|
+
pageBuilder.setNull(column);
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
pageBuilder.setDouble(column, pageReader.getDouble(column));
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
@Override
|
|
216
|
+
public void stringColumn(Column column)
|
|
217
|
+
{
|
|
218
|
+
if (pageReader.isNull(column)) {
|
|
219
|
+
pageBuilder.setNull(column);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
String value = pageReader.getString(column);
|
|
223
|
+
String formatted = formatTimestampString(task, column.getName(), value);
|
|
224
|
+
pageBuilder.setString(column, formatted);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
@Override
|
|
228
|
+
public void jsonColumn(Column column)
|
|
229
|
+
{
|
|
230
|
+
if (pageReader.isNull(column)) {
|
|
231
|
+
pageBuilder.setNull(column);
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
String path = new StringBuilder("$.").append(column.getName()).toString();
|
|
235
|
+
Value value = pageReader.getJson(column);
|
|
236
|
+
Value formatted = formatTimestampStringRecursively(task, path, value);
|
|
237
|
+
pageBuilder.setJson(column, formatted);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
@Override
|
|
242
|
+
public void timestampColumn(Column column)
|
|
243
|
+
{
|
|
244
|
+
if (pageReader.isNull(column)) {
|
|
245
|
+
pageBuilder.setNull(column);
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
package org.embulk.filter.timestamp_format;
|
|
2
2
|
|
|
3
|
-
import com.google.common.base.Throwables;
|
|
4
|
-
|
|
5
3
|
import org.embulk.config.Config;
|
|
6
4
|
import org.embulk.config.ConfigDefault;
|
|
7
|
-
import org.embulk.config.ConfigException;
|
|
8
5
|
import org.embulk.config.ConfigInject;
|
|
9
6
|
import org.embulk.config.ConfigSource;
|
|
10
7
|
import org.embulk.config.Task;
|
|
11
8
|
import org.embulk.config.TaskSource;
|
|
12
9
|
|
|
13
|
-
import org.embulk.spi.Column;
|
|
14
|
-
import org.embulk.spi.ColumnVisitor;
|
|
15
10
|
import org.embulk.spi.Exec;
|
|
16
11
|
import org.embulk.spi.FilterPlugin;
|
|
17
12
|
import org.embulk.spi.Page;
|
|
@@ -19,22 +14,11 @@ import org.embulk.spi.PageBuilder;
|
|
|
19
14
|
import org.embulk.spi.PageOutput;
|
|
20
15
|
import org.embulk.spi.PageReader;
|
|
21
16
|
import org.embulk.spi.Schema;
|
|
22
|
-
import org.embulk.spi.time.Timestamp;
|
|
23
|
-
import org.embulk.spi.time.TimestampParseException;
|
|
24
17
|
|
|
25
|
-
import org.joda.time.DateTimeZone;
|
|
26
18
|
import org.jruby.embed.ScriptingContainer;
|
|
27
|
-
import org.msgpack.value.ArrayValue;
|
|
28
|
-
import org.msgpack.value.MapValue;
|
|
29
|
-
import org.msgpack.value.Value;
|
|
30
|
-
import org.msgpack.value.ValueFactory;
|
|
31
19
|
import org.slf4j.Logger;
|
|
32
20
|
|
|
33
|
-
import java.util.ArrayList;
|
|
34
|
-
import java.util.HashMap;
|
|
35
21
|
import java.util.List;
|
|
36
|
-
import java.util.Map;
|
|
37
|
-
import java.util.Objects;
|
|
38
22
|
|
|
39
23
|
public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
40
24
|
{
|
|
@@ -45,7 +29,7 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
|
45
29
|
}
|
|
46
30
|
|
|
47
31
|
// NOTE: This is not spi.ColumnConfig
|
|
48
|
-
|
|
32
|
+
public interface ColumnConfig extends Task,
|
|
49
33
|
TimestampParser.TimestampColumnOption, TimestampFormatter.TimestampColumnOption
|
|
50
34
|
{
|
|
51
35
|
@Config("name")
|
|
@@ -74,116 +58,31 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
|
74
58
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
75
59
|
|
|
76
60
|
List<ColumnConfig> columns = task.getColumns();
|
|
61
|
+
// throw if column does not exist
|
|
77
62
|
for (ColumnConfig columnConfig : columns) {
|
|
78
63
|
String name = columnConfig.getName();
|
|
79
|
-
if (
|
|
80
|
-
|
|
64
|
+
if (name.startsWith("$.")) {
|
|
65
|
+
String firstName = name.split("\\.", 3)[1];
|
|
66
|
+
inputSchema.lookupColumn(firstName);
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
inputSchema.lookupColumn(name);
|
|
81
70
|
}
|
|
82
71
|
}
|
|
83
72
|
|
|
84
73
|
control.run(task.dump(), inputSchema);
|
|
85
74
|
}
|
|
86
75
|
|
|
87
|
-
private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
|
|
88
|
-
{
|
|
89
|
-
List<TimestampParser> timestampParser = new ArrayList<TimestampParser>();
|
|
90
|
-
DateTimeZone timezone = columnConfig.getFromTimeZone().or(task.getDefaultFromTimeZone());
|
|
91
|
-
List<String> formatList = columnConfig.getFromFormat().or(task.getDefaultFromTimestampFormat());
|
|
92
|
-
return new TimestampParser(task.getJRuby(), formatList, timezone);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
|
|
96
|
-
{
|
|
97
|
-
String format = columnConfig.getToFormat().or(task.getDefaultToTimestampFormat());
|
|
98
|
-
DateTimeZone timezone = columnConfig.getToTimeZone().or(task.getDefaultToTimeZone());
|
|
99
|
-
return new TimestampFormatter(task.getJRuby(), format, timezone);
|
|
100
|
-
}
|
|
101
|
-
|
|
102
76
|
@Override
|
|
103
77
|
public PageOutput open(final TaskSource taskSource, final Schema inputSchema,
|
|
104
78
|
final Schema outputSchema, final PageOutput output)
|
|
105
79
|
{
|
|
106
80
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
107
81
|
|
|
108
|
-
// columnName => TimestampParser
|
|
109
|
-
final HashMap<String, TimestampParser> timestampParserMap = new HashMap<String, TimestampParser>();
|
|
110
|
-
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
111
|
-
TimestampParser parser = getTimestampParser(columnConfig, task);
|
|
112
|
-
timestampParserMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
113
|
-
}
|
|
114
|
-
// columnName => TimestampFormatter
|
|
115
|
-
final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<String, TimestampFormatter>();
|
|
116
|
-
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
117
|
-
TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
|
|
118
|
-
timestampFormatterMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
119
|
-
}
|
|
120
|
-
|
|
121
82
|
return new PageOutput() {
|
|
122
|
-
public Value formatTimestampStringRecursively(PluginTask task, String name, Value value)
|
|
123
|
-
throws TimestampParseException
|
|
124
|
-
{
|
|
125
|
-
if (value.isArrayValue()) {
|
|
126
|
-
ArrayValue arrayValue = value.asArrayValue();
|
|
127
|
-
int size = arrayValue.size();
|
|
128
|
-
Value[] newValue = new Value[size];
|
|
129
|
-
for (int i = 0; i < size; i++) {
|
|
130
|
-
String k = new StringBuilder(name).append("[").append(Integer.toString(i)).append("]").toString();
|
|
131
|
-
Value v = arrayValue.get(i);
|
|
132
|
-
newValue[i] = formatTimestampStringRecursively(task, k, v);
|
|
133
|
-
}
|
|
134
|
-
return ValueFactory.newArray(newValue, true);
|
|
135
|
-
}
|
|
136
|
-
else if (value.isMapValue()) {
|
|
137
|
-
MapValue mapValue = value.asMapValue();
|
|
138
|
-
int size = mapValue.size() * 2;
|
|
139
|
-
Value[] newValue = new Value[size];
|
|
140
|
-
int i = 0;
|
|
141
|
-
for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
|
|
142
|
-
Value k = entry.getKey();
|
|
143
|
-
Value v = entry.getValue();
|
|
144
|
-
String newName = new StringBuilder(name).append(".").append(k.asStringValue().asString()).toString();
|
|
145
|
-
Value r = formatTimestampStringRecursively(task, newName, v);
|
|
146
|
-
newValue[i++] = k;
|
|
147
|
-
newValue[i++] = r;
|
|
148
|
-
}
|
|
149
|
-
return ValueFactory.newMap(newValue, true);
|
|
150
|
-
}
|
|
151
|
-
else if (value.isStringValue()) {
|
|
152
|
-
String stringValue = value.asStringValue().asString();
|
|
153
|
-
String newValue = formatTimestampString(task, name, stringValue);
|
|
154
|
-
return (Objects.equals(newValue, stringValue)) ? value : ValueFactory.newString(newValue);
|
|
155
|
-
}
|
|
156
|
-
else {
|
|
157
|
-
return value;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
public String formatTimestampString(PluginTask task, String name, String value)
|
|
162
|
-
throws TimestampParseException
|
|
163
|
-
{
|
|
164
|
-
TimestampParser parser = timestampParserMap.get(name);
|
|
165
|
-
TimestampFormatter formatter = timestampFormatterMap.get(name);
|
|
166
|
-
if (formatter == null || parser == null) {
|
|
167
|
-
return value;
|
|
168
|
-
}
|
|
169
|
-
try {
|
|
170
|
-
Timestamp timestamp = parser.parse(value);
|
|
171
|
-
return formatter.format(timestamp);
|
|
172
|
-
}
|
|
173
|
-
catch (TimestampParseException ex) {
|
|
174
|
-
if (task.getStopOnInvalidRecord()) {
|
|
175
|
-
throw Throwables.propagate(ex);
|
|
176
|
-
}
|
|
177
|
-
else {
|
|
178
|
-
logger.warn("invalid value \"{}\":\"{}\"", name, value);
|
|
179
|
-
return value;
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
83
|
private PageReader pageReader = new PageReader(inputSchema);
|
|
185
84
|
private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
|
|
186
|
-
private ColumnVisitorImpl visitor = new ColumnVisitorImpl(pageBuilder);
|
|
85
|
+
private ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, pageReader, pageBuilder);
|
|
187
86
|
|
|
188
87
|
@Override
|
|
189
88
|
public void finish()
|
|
@@ -207,86 +106,6 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
|
207
106
|
pageBuilder.addRecord();
|
|
208
107
|
}
|
|
209
108
|
}
|
|
210
|
-
|
|
211
|
-
class ColumnVisitorImpl implements ColumnVisitor
|
|
212
|
-
{
|
|
213
|
-
private final PageBuilder pageBuilder;
|
|
214
|
-
|
|
215
|
-
ColumnVisitorImpl(PageBuilder pageBuilder)
|
|
216
|
-
{
|
|
217
|
-
this.pageBuilder = pageBuilder;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
@Override
|
|
221
|
-
public void booleanColumn(Column column)
|
|
222
|
-
{
|
|
223
|
-
if (pageReader.isNull(column)) {
|
|
224
|
-
pageBuilder.setNull(column);
|
|
225
|
-
}
|
|
226
|
-
else {
|
|
227
|
-
pageBuilder.setBoolean(column, pageReader.getBoolean(column));
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
@Override
|
|
232
|
-
public void longColumn(Column column)
|
|
233
|
-
{
|
|
234
|
-
if (pageReader.isNull(column)) {
|
|
235
|
-
pageBuilder.setNull(column);
|
|
236
|
-
}
|
|
237
|
-
else {
|
|
238
|
-
pageBuilder.setLong(column, pageReader.getLong(column));
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
@Override
|
|
243
|
-
public void doubleColumn(Column column)
|
|
244
|
-
{
|
|
245
|
-
if (pageReader.isNull(column)) {
|
|
246
|
-
pageBuilder.setNull(column);
|
|
247
|
-
}
|
|
248
|
-
else {
|
|
249
|
-
pageBuilder.setDouble(column, pageReader.getDouble(column));
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
@Override
|
|
254
|
-
public void stringColumn(Column column)
|
|
255
|
-
{
|
|
256
|
-
if (pageReader.isNull(column)) {
|
|
257
|
-
pageBuilder.setNull(column);
|
|
258
|
-
return;
|
|
259
|
-
}
|
|
260
|
-
String value = pageReader.getString(column);
|
|
261
|
-
String formatted = formatTimestampString(task, column.getName(), value);
|
|
262
|
-
pageBuilder.setString(column, formatted);
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
@Override
|
|
266
|
-
public void jsonColumn(Column column)
|
|
267
|
-
{
|
|
268
|
-
if (pageReader.isNull(column)) {
|
|
269
|
-
pageBuilder.setNull(column);
|
|
270
|
-
}
|
|
271
|
-
else {
|
|
272
|
-
String name = new StringBuilder("$.").append(column.getName()).toString();
|
|
273
|
-
Value value = pageReader.getJson(column);
|
|
274
|
-
Value formatted = formatTimestampStringRecursively(task, name, value);
|
|
275
|
-
pageBuilder.setJson(column, formatted);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
@Override
|
|
280
|
-
public void timestampColumn(Column column)
|
|
281
|
-
{
|
|
282
|
-
if (pageReader.isNull(column)) {
|
|
283
|
-
pageBuilder.setNull(column);
|
|
284
|
-
}
|
|
285
|
-
else {
|
|
286
|
-
pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
109
|
};
|
|
291
110
|
}
|
|
292
111
|
}
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-timestamp_format
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Naotoshi Seo
|
|
@@ -73,11 +73,12 @@ files:
|
|
|
73
73
|
- gradlew
|
|
74
74
|
- gradlew.bat
|
|
75
75
|
- lib/embulk/filter/timestamp_format.rb
|
|
76
|
+
- src/main/java/org/embulk/filter/timestamp_format/ColumnVisitorImpl.java
|
|
76
77
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java
|
|
77
78
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java
|
|
78
79
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java
|
|
79
80
|
- src/test/java/org/embulk/filter/TestTimestampFormatFilterPlugin.java
|
|
80
|
-
- classpath/embulk-filter-timestamp_format-0.1.
|
|
81
|
+
- classpath/embulk-filter-timestamp_format-0.1.4.jar
|
|
81
82
|
homepage: https://github.com/sonots/embulk-filter-timestamp_format
|
|
82
83
|
licenses:
|
|
83
84
|
- MIT
|