embulk-filter-timestamp_format 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/build.gradle +1 -1
- data/example/example.jsonl +2 -2
- data/example/example.yml +1 -0
- data/src/main/java/org/embulk/filter/timestamp_format/ColumnVisitorImpl.java +251 -0
- data/src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java +9 -190
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c06f4d7efebebe0e2abb454b7f6545e6ea843026
|
|
4
|
+
data.tar.gz: d98aceb871fe643ed6afe46501758d15884ee124
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d8aae59c031cd582e3df88466417822e1f952ec9573d1df8fe189109496e909c9ca71d6e448d544ac8ffa64611ec2689651fccb7caa4e5de8559ff3c30bc6213
|
|
7
|
+
data.tar.gz: 938432021acdece8554c884e033f5e4f2bcac15d2c0407fff3134f8757f84f52201e82f14ac46a4d954c900382d3d174e79ee04b94dc76848867a1b4ae4afe9e
|
data/CHANGELOG.md
CHANGED
data/build.gradle
CHANGED
data/example/example.jsonl
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
{"timestamp":"2015-07-12 15:00:00 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00 UTC"}]}}}
|
|
2
|
-
{"timestamp":"2015-07-12 15:00:00.1 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00.1 UTC"}]}}}
|
|
1
|
+
{"timestamp":"2015-07-12 15:00:00 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00 UTC"}]},"ignore_record":{"timestamp":"2015-07-12 15:00:00 UTC"}}
|
|
2
|
+
{"timestamp":"2015-07-12 15:00:00.1 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00.1 UTC"}]},"ignore_record":{"timestamp":"2015-07-12 15:00:00.1 UTC"}}
|
data/example/example.yml
CHANGED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
package org.embulk.filter.timestamp_format;
|
|
2
|
+
|
|
3
|
+
import com.google.common.base.Throwables;
|
|
4
|
+
import org.embulk.spi.PageReader;
|
|
5
|
+
import org.msgpack.value.ArrayValue;
|
|
6
|
+
import org.msgpack.value.MapValue;
|
|
7
|
+
import org.msgpack.value.Value;
|
|
8
|
+
import org.msgpack.value.ValueFactory;
|
|
9
|
+
|
|
10
|
+
import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.ColumnConfig;
|
|
11
|
+
import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.PluginTask;
|
|
12
|
+
|
|
13
|
+
import org.embulk.spi.Column;
|
|
14
|
+
import org.embulk.spi.ColumnVisitor;
|
|
15
|
+
import org.embulk.spi.Exec;
|
|
16
|
+
import org.embulk.spi.PageBuilder;
|
|
17
|
+
import org.embulk.spi.time.Timestamp;
|
|
18
|
+
import org.embulk.spi.time.TimestampParseException;
|
|
19
|
+
import org.joda.time.DateTimeZone;
|
|
20
|
+
import org.slf4j.Logger;
|
|
21
|
+
|
|
22
|
+
import java.util.HashMap;
|
|
23
|
+
import java.util.HashSet;
|
|
24
|
+
import java.util.List;
|
|
25
|
+
import java.util.Map;
|
|
26
|
+
import java.util.Objects;
|
|
27
|
+
|
|
28
|
+
public class ColumnVisitorImpl
|
|
29
|
+
implements ColumnVisitor
|
|
30
|
+
{
|
|
31
|
+
private static final Logger logger = Exec.getLogger(TimestampFormatFilterPlugin.class);
|
|
32
|
+
private final PluginTask task;
|
|
33
|
+
private final PageReader pageReader;
|
|
34
|
+
private final PageBuilder pageBuilder;
|
|
35
|
+
private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<String, TimestampParser>();
|
|
36
|
+
private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<String, TimestampFormatter>();
|
|
37
|
+
private final HashSet<String> shouldVisitRecursivelySet = new HashSet<String>();
|
|
38
|
+
|
|
39
|
+
ColumnVisitorImpl(PluginTask task, PageReader pageReader, PageBuilder pageBuilder)
|
|
40
|
+
{
|
|
41
|
+
this.task = task;
|
|
42
|
+
this.pageReader = pageReader;
|
|
43
|
+
this.pageBuilder = pageBuilder;
|
|
44
|
+
|
|
45
|
+
buildTimestampParserMap();
|
|
46
|
+
buildTimestampFormatterMap();
|
|
47
|
+
buildShouldVisitRecursivelySet();;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
private void buildTimestampParserMap()
|
|
51
|
+
{
|
|
52
|
+
// columnName or jsonPath => TimestampParser
|
|
53
|
+
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
54
|
+
TimestampParser parser = getTimestampParser(columnConfig, task);
|
|
55
|
+
this.timestampParserMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
|
|
60
|
+
{
|
|
61
|
+
DateTimeZone timezone = columnConfig.getFromTimeZone().or(task.getDefaultFromTimeZone());
|
|
62
|
+
List<String> formatList = columnConfig.getFromFormat().or(task.getDefaultFromTimestampFormat());
|
|
63
|
+
return new TimestampParser(task.getJRuby(), formatList, timezone);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
private void buildTimestampFormatterMap()
|
|
67
|
+
{
|
|
68
|
+
// columnName or jsonPath => TimestampFormatter
|
|
69
|
+
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
70
|
+
TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
|
|
71
|
+
this.timestampFormatterMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
|
|
76
|
+
{
|
|
77
|
+
String format = columnConfig.getToFormat().or(task.getDefaultToTimestampFormat());
|
|
78
|
+
DateTimeZone timezone = columnConfig.getToTimeZone().or(task.getDefaultToTimeZone());
|
|
79
|
+
return new TimestampFormatter(task.getJRuby(), format, timezone);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
private void buildShouldVisitRecursivelySet()
|
|
84
|
+
{
|
|
85
|
+
// json partial path => Boolean to avoid unnecessary type: json visit
|
|
86
|
+
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
87
|
+
String name = columnConfig.getName();
|
|
88
|
+
if (!name.startsWith("$.")) {
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
String[] parts = name.split("\\.");
|
|
92
|
+
StringBuilder partialPath = new StringBuilder("$");
|
|
93
|
+
for (int i = 1; i < parts.length; i++) {
|
|
94
|
+
if (parts[i].contains("[")) {
|
|
95
|
+
String[] arrayParts = parts[i].split("\\[");
|
|
96
|
+
partialPath.append(".").append(arrayParts[0]);
|
|
97
|
+
this.shouldVisitRecursivelySet.add(partialPath.toString());
|
|
98
|
+
for (int j = 1; j < arrayParts.length; j++) {
|
|
99
|
+
partialPath.append("[").append(arrayParts[j]);
|
|
100
|
+
this.shouldVisitRecursivelySet.add(partialPath.toString());
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
partialPath.append(".").append(parts[i]);
|
|
105
|
+
this.shouldVisitRecursivelySet.add(partialPath.toString());
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
private boolean shouldVisitRecursively(String name)
|
|
112
|
+
{
|
|
113
|
+
return shouldVisitRecursivelySet.contains(name);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
private Value formatTimestampStringRecursively(PluginTask task, String path, Value value)
|
|
117
|
+
throws TimestampParseException
|
|
118
|
+
{
|
|
119
|
+
if (!shouldVisitRecursively(path)) {
|
|
120
|
+
return value;
|
|
121
|
+
}
|
|
122
|
+
if (value.isArrayValue()) {
|
|
123
|
+
ArrayValue arrayValue = value.asArrayValue();
|
|
124
|
+
int size = arrayValue.size();
|
|
125
|
+
Value[] newValue = new Value[size];
|
|
126
|
+
for (int i = 0; i < size; i++) {
|
|
127
|
+
String k = new StringBuilder(path).append("[").append(Integer.toString(i)).append("]").toString();
|
|
128
|
+
Value v = arrayValue.get(i);
|
|
129
|
+
newValue[i] = formatTimestampStringRecursively(task, k, v);
|
|
130
|
+
}
|
|
131
|
+
return ValueFactory.newArray(newValue, true);
|
|
132
|
+
}
|
|
133
|
+
else if (value.isMapValue()) {
|
|
134
|
+
MapValue mapValue = value.asMapValue();
|
|
135
|
+
int size = mapValue.size() * 2;
|
|
136
|
+
Value[] newValue = new Value[size];
|
|
137
|
+
int i = 0;
|
|
138
|
+
for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
|
|
139
|
+
Value k = entry.getKey();
|
|
140
|
+
Value v = entry.getValue();
|
|
141
|
+
String newPath = new StringBuilder(path).append(".").append(k.asStringValue().asString()).toString();
|
|
142
|
+
Value r = formatTimestampStringRecursively(task, newPath, v);
|
|
143
|
+
newValue[i++] = k;
|
|
144
|
+
newValue[i++] = r;
|
|
145
|
+
}
|
|
146
|
+
return ValueFactory.newMap(newValue, true);
|
|
147
|
+
}
|
|
148
|
+
else if (value.isStringValue()) {
|
|
149
|
+
String stringValue = value.asStringValue().asString();
|
|
150
|
+
String newValue = formatTimestampString(task, path, stringValue);
|
|
151
|
+
return (Objects.equals(newValue, stringValue)) ? value : ValueFactory.newString(newValue);
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
return value;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
private String formatTimestampString(PluginTask task, String name, String value)
|
|
159
|
+
throws TimestampParseException
|
|
160
|
+
{
|
|
161
|
+
TimestampParser parser = timestampParserMap.get(name);
|
|
162
|
+
TimestampFormatter formatter = timestampFormatterMap.get(name);
|
|
163
|
+
if (formatter == null || parser == null) {
|
|
164
|
+
return value;
|
|
165
|
+
}
|
|
166
|
+
try {
|
|
167
|
+
Timestamp timestamp = parser.parse(value);
|
|
168
|
+
return formatter.format(timestamp);
|
|
169
|
+
}
|
|
170
|
+
catch (TimestampParseException ex) {
|
|
171
|
+
if (task.getStopOnInvalidRecord()) {
|
|
172
|
+
throw Throwables.propagate(ex);
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
logger.warn("invalid value \"{}\":\"{}\"", name, value);
|
|
176
|
+
return value;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@Override
|
|
183
|
+
public void booleanColumn(Column column)
|
|
184
|
+
{
|
|
185
|
+
if (pageReader.isNull(column)) {
|
|
186
|
+
pageBuilder.setNull(column);
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
pageBuilder.setBoolean(column, pageReader.getBoolean(column));
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
@Override
|
|
194
|
+
public void longColumn(Column column)
|
|
195
|
+
{
|
|
196
|
+
if (pageReader.isNull(column)) {
|
|
197
|
+
pageBuilder.setNull(column);
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
pageBuilder.setLong(column, pageReader.getLong(column));
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
@Override
|
|
205
|
+
public void doubleColumn(Column column)
|
|
206
|
+
{
|
|
207
|
+
if (pageReader.isNull(column)) {
|
|
208
|
+
pageBuilder.setNull(column);
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
pageBuilder.setDouble(column, pageReader.getDouble(column));
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
@Override
|
|
216
|
+
public void stringColumn(Column column)
|
|
217
|
+
{
|
|
218
|
+
if (pageReader.isNull(column)) {
|
|
219
|
+
pageBuilder.setNull(column);
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
String value = pageReader.getString(column);
|
|
223
|
+
String formatted = formatTimestampString(task, column.getName(), value);
|
|
224
|
+
pageBuilder.setString(column, formatted);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
@Override
|
|
228
|
+
public void jsonColumn(Column column)
|
|
229
|
+
{
|
|
230
|
+
if (pageReader.isNull(column)) {
|
|
231
|
+
pageBuilder.setNull(column);
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
String path = new StringBuilder("$.").append(column.getName()).toString();
|
|
235
|
+
Value value = pageReader.getJson(column);
|
|
236
|
+
Value formatted = formatTimestampStringRecursively(task, path, value);
|
|
237
|
+
pageBuilder.setJson(column, formatted);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
@Override
|
|
242
|
+
public void timestampColumn(Column column)
|
|
243
|
+
{
|
|
244
|
+
if (pageReader.isNull(column)) {
|
|
245
|
+
pageBuilder.setNull(column);
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
package org.embulk.filter.timestamp_format;
|
|
2
2
|
|
|
3
|
-
import com.google.common.base.Throwables;
|
|
4
|
-
|
|
5
3
|
import org.embulk.config.Config;
|
|
6
4
|
import org.embulk.config.ConfigDefault;
|
|
7
|
-
import org.embulk.config.ConfigException;
|
|
8
5
|
import org.embulk.config.ConfigInject;
|
|
9
6
|
import org.embulk.config.ConfigSource;
|
|
10
7
|
import org.embulk.config.Task;
|
|
11
8
|
import org.embulk.config.TaskSource;
|
|
12
9
|
|
|
13
|
-
import org.embulk.spi.Column;
|
|
14
|
-
import org.embulk.spi.ColumnVisitor;
|
|
15
10
|
import org.embulk.spi.Exec;
|
|
16
11
|
import org.embulk.spi.FilterPlugin;
|
|
17
12
|
import org.embulk.spi.Page;
|
|
@@ -19,22 +14,11 @@ import org.embulk.spi.PageBuilder;
|
|
|
19
14
|
import org.embulk.spi.PageOutput;
|
|
20
15
|
import org.embulk.spi.PageReader;
|
|
21
16
|
import org.embulk.spi.Schema;
|
|
22
|
-
import org.embulk.spi.time.Timestamp;
|
|
23
|
-
import org.embulk.spi.time.TimestampParseException;
|
|
24
17
|
|
|
25
|
-
import org.joda.time.DateTimeZone;
|
|
26
18
|
import org.jruby.embed.ScriptingContainer;
|
|
27
|
-
import org.msgpack.value.ArrayValue;
|
|
28
|
-
import org.msgpack.value.MapValue;
|
|
29
|
-
import org.msgpack.value.Value;
|
|
30
|
-
import org.msgpack.value.ValueFactory;
|
|
31
19
|
import org.slf4j.Logger;
|
|
32
20
|
|
|
33
|
-
import java.util.ArrayList;
|
|
34
|
-
import java.util.HashMap;
|
|
35
21
|
import java.util.List;
|
|
36
|
-
import java.util.Map;
|
|
37
|
-
import java.util.Objects;
|
|
38
22
|
|
|
39
23
|
public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
40
24
|
{
|
|
@@ -45,7 +29,7 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
|
45
29
|
}
|
|
46
30
|
|
|
47
31
|
// NOTE: This is not spi.ColumnConfig
|
|
48
|
-
|
|
32
|
+
public interface ColumnConfig extends Task,
|
|
49
33
|
TimestampParser.TimestampColumnOption, TimestampFormatter.TimestampColumnOption
|
|
50
34
|
{
|
|
51
35
|
@Config("name")
|
|
@@ -74,116 +58,31 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
|
74
58
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
75
59
|
|
|
76
60
|
List<ColumnConfig> columns = task.getColumns();
|
|
61
|
+
// throw if column does not exist
|
|
77
62
|
for (ColumnConfig columnConfig : columns) {
|
|
78
63
|
String name = columnConfig.getName();
|
|
79
|
-
if (
|
|
80
|
-
|
|
64
|
+
if (name.startsWith("$.")) {
|
|
65
|
+
String firstName = name.split("\\.", 3)[1];
|
|
66
|
+
inputSchema.lookupColumn(firstName);
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
inputSchema.lookupColumn(name);
|
|
81
70
|
}
|
|
82
71
|
}
|
|
83
72
|
|
|
84
73
|
control.run(task.dump(), inputSchema);
|
|
85
74
|
}
|
|
86
75
|
|
|
87
|
-
private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
|
|
88
|
-
{
|
|
89
|
-
List<TimestampParser> timestampParser = new ArrayList<TimestampParser>();
|
|
90
|
-
DateTimeZone timezone = columnConfig.getFromTimeZone().or(task.getDefaultFromTimeZone());
|
|
91
|
-
List<String> formatList = columnConfig.getFromFormat().or(task.getDefaultFromTimestampFormat());
|
|
92
|
-
return new TimestampParser(task.getJRuby(), formatList, timezone);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
|
|
96
|
-
{
|
|
97
|
-
String format = columnConfig.getToFormat().or(task.getDefaultToTimestampFormat());
|
|
98
|
-
DateTimeZone timezone = columnConfig.getToTimeZone().or(task.getDefaultToTimeZone());
|
|
99
|
-
return new TimestampFormatter(task.getJRuby(), format, timezone);
|
|
100
|
-
}
|
|
101
|
-
|
|
102
76
|
@Override
|
|
103
77
|
public PageOutput open(final TaskSource taskSource, final Schema inputSchema,
|
|
104
78
|
final Schema outputSchema, final PageOutput output)
|
|
105
79
|
{
|
|
106
80
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
107
81
|
|
|
108
|
-
// columnName => TimestampParser
|
|
109
|
-
final HashMap<String, TimestampParser> timestampParserMap = new HashMap<String, TimestampParser>();
|
|
110
|
-
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
111
|
-
TimestampParser parser = getTimestampParser(columnConfig, task);
|
|
112
|
-
timestampParserMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
113
|
-
}
|
|
114
|
-
// columnName => TimestampFormatter
|
|
115
|
-
final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<String, TimestampFormatter>();
|
|
116
|
-
for (ColumnConfig columnConfig : task.getColumns()) {
|
|
117
|
-
TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
|
|
118
|
-
timestampFormatterMap.put(columnConfig.getName(), parser); // NOTE: value would be null
|
|
119
|
-
}
|
|
120
|
-
|
|
121
82
|
return new PageOutput() {
|
|
122
|
-
public Value formatTimestampStringRecursively(PluginTask task, String name, Value value)
|
|
123
|
-
throws TimestampParseException
|
|
124
|
-
{
|
|
125
|
-
if (value.isArrayValue()) {
|
|
126
|
-
ArrayValue arrayValue = value.asArrayValue();
|
|
127
|
-
int size = arrayValue.size();
|
|
128
|
-
Value[] newValue = new Value[size];
|
|
129
|
-
for (int i = 0; i < size; i++) {
|
|
130
|
-
String k = new StringBuilder(name).append("[").append(Integer.toString(i)).append("]").toString();
|
|
131
|
-
Value v = arrayValue.get(i);
|
|
132
|
-
newValue[i] = formatTimestampStringRecursively(task, k, v);
|
|
133
|
-
}
|
|
134
|
-
return ValueFactory.newArray(newValue, true);
|
|
135
|
-
}
|
|
136
|
-
else if (value.isMapValue()) {
|
|
137
|
-
MapValue mapValue = value.asMapValue();
|
|
138
|
-
int size = mapValue.size() * 2;
|
|
139
|
-
Value[] newValue = new Value[size];
|
|
140
|
-
int i = 0;
|
|
141
|
-
for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
|
|
142
|
-
Value k = entry.getKey();
|
|
143
|
-
Value v = entry.getValue();
|
|
144
|
-
String newName = new StringBuilder(name).append(".").append(k.asStringValue().asString()).toString();
|
|
145
|
-
Value r = formatTimestampStringRecursively(task, newName, v);
|
|
146
|
-
newValue[i++] = k;
|
|
147
|
-
newValue[i++] = r;
|
|
148
|
-
}
|
|
149
|
-
return ValueFactory.newMap(newValue, true);
|
|
150
|
-
}
|
|
151
|
-
else if (value.isStringValue()) {
|
|
152
|
-
String stringValue = value.asStringValue().asString();
|
|
153
|
-
String newValue = formatTimestampString(task, name, stringValue);
|
|
154
|
-
return (Objects.equals(newValue, stringValue)) ? value : ValueFactory.newString(newValue);
|
|
155
|
-
}
|
|
156
|
-
else {
|
|
157
|
-
return value;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
public String formatTimestampString(PluginTask task, String name, String value)
|
|
162
|
-
throws TimestampParseException
|
|
163
|
-
{
|
|
164
|
-
TimestampParser parser = timestampParserMap.get(name);
|
|
165
|
-
TimestampFormatter formatter = timestampFormatterMap.get(name);
|
|
166
|
-
if (formatter == null || parser == null) {
|
|
167
|
-
return value;
|
|
168
|
-
}
|
|
169
|
-
try {
|
|
170
|
-
Timestamp timestamp = parser.parse(value);
|
|
171
|
-
return formatter.format(timestamp);
|
|
172
|
-
}
|
|
173
|
-
catch (TimestampParseException ex) {
|
|
174
|
-
if (task.getStopOnInvalidRecord()) {
|
|
175
|
-
throw Throwables.propagate(ex);
|
|
176
|
-
}
|
|
177
|
-
else {
|
|
178
|
-
logger.warn("invalid value \"{}\":\"{}\"", name, value);
|
|
179
|
-
return value;
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
83
|
private PageReader pageReader = new PageReader(inputSchema);
|
|
185
84
|
private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
|
|
186
|
-
private ColumnVisitorImpl visitor = new ColumnVisitorImpl(pageBuilder);
|
|
85
|
+
private ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, pageReader, pageBuilder);
|
|
187
86
|
|
|
188
87
|
@Override
|
|
189
88
|
public void finish()
|
|
@@ -207,86 +106,6 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
|
|
|
207
106
|
pageBuilder.addRecord();
|
|
208
107
|
}
|
|
209
108
|
}
|
|
210
|
-
|
|
211
|
-
class ColumnVisitorImpl implements ColumnVisitor
|
|
212
|
-
{
|
|
213
|
-
private final PageBuilder pageBuilder;
|
|
214
|
-
|
|
215
|
-
ColumnVisitorImpl(PageBuilder pageBuilder)
|
|
216
|
-
{
|
|
217
|
-
this.pageBuilder = pageBuilder;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
@Override
|
|
221
|
-
public void booleanColumn(Column column)
|
|
222
|
-
{
|
|
223
|
-
if (pageReader.isNull(column)) {
|
|
224
|
-
pageBuilder.setNull(column);
|
|
225
|
-
}
|
|
226
|
-
else {
|
|
227
|
-
pageBuilder.setBoolean(column, pageReader.getBoolean(column));
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
@Override
|
|
232
|
-
public void longColumn(Column column)
|
|
233
|
-
{
|
|
234
|
-
if (pageReader.isNull(column)) {
|
|
235
|
-
pageBuilder.setNull(column);
|
|
236
|
-
}
|
|
237
|
-
else {
|
|
238
|
-
pageBuilder.setLong(column, pageReader.getLong(column));
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
@Override
|
|
243
|
-
public void doubleColumn(Column column)
|
|
244
|
-
{
|
|
245
|
-
if (pageReader.isNull(column)) {
|
|
246
|
-
pageBuilder.setNull(column);
|
|
247
|
-
}
|
|
248
|
-
else {
|
|
249
|
-
pageBuilder.setDouble(column, pageReader.getDouble(column));
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
@Override
|
|
254
|
-
public void stringColumn(Column column)
|
|
255
|
-
{
|
|
256
|
-
if (pageReader.isNull(column)) {
|
|
257
|
-
pageBuilder.setNull(column);
|
|
258
|
-
return;
|
|
259
|
-
}
|
|
260
|
-
String value = pageReader.getString(column);
|
|
261
|
-
String formatted = formatTimestampString(task, column.getName(), value);
|
|
262
|
-
pageBuilder.setString(column, formatted);
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
@Override
|
|
266
|
-
public void jsonColumn(Column column)
|
|
267
|
-
{
|
|
268
|
-
if (pageReader.isNull(column)) {
|
|
269
|
-
pageBuilder.setNull(column);
|
|
270
|
-
}
|
|
271
|
-
else {
|
|
272
|
-
String name = new StringBuilder("$.").append(column.getName()).toString();
|
|
273
|
-
Value value = pageReader.getJson(column);
|
|
274
|
-
Value formatted = formatTimestampStringRecursively(task, name, value);
|
|
275
|
-
pageBuilder.setJson(column, formatted);
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
@Override
|
|
280
|
-
public void timestampColumn(Column column)
|
|
281
|
-
{
|
|
282
|
-
if (pageReader.isNull(column)) {
|
|
283
|
-
pageBuilder.setNull(column);
|
|
284
|
-
}
|
|
285
|
-
else {
|
|
286
|
-
pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
109
|
};
|
|
291
110
|
}
|
|
292
111
|
}
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-timestamp_format
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Naotoshi Seo
|
|
@@ -73,11 +73,12 @@ files:
|
|
|
73
73
|
- gradlew
|
|
74
74
|
- gradlew.bat
|
|
75
75
|
- lib/embulk/filter/timestamp_format.rb
|
|
76
|
+
- src/main/java/org/embulk/filter/timestamp_format/ColumnVisitorImpl.java
|
|
76
77
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java
|
|
77
78
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java
|
|
78
79
|
- src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java
|
|
79
80
|
- src/test/java/org/embulk/filter/TestTimestampFormatFilterPlugin.java
|
|
80
|
-
- classpath/embulk-filter-timestamp_format-0.1.
|
|
81
|
+
- classpath/embulk-filter-timestamp_format-0.1.4.jar
|
|
81
82
|
homepage: https://github.com/sonots/embulk-filter-timestamp_format
|
|
82
83
|
licenses:
|
|
83
84
|
- MIT
|