embulk-output-td 0.3.10 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +19 -0
- data/build.gradle +1 -1
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/RecordWriter.java +0 -5
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +16 -8
- data/src/main/java/org/embulk/output/td/TimeValueGenerator.java +3 -2
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +17 -21
- data/src/main/java/org/embulk/output/td/writer/LongFieldWriter.java +1 -1
- data/src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java +0 -1
- data/src/main/java/org/embulk/output/td/writer/UnixTimestampLongFieldWriter.java +1 -1
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +9 -15
- data/src/test/java/org/embulk/output/td/TestTimeValueGenerator.java +10 -10
- data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +17 -17
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 80ebc07da65f75960a956328d90265a2c5414162
|
4
|
+
data.tar.gz: da6b1c3415ee725a3297381522054e99e531398d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5a5580b4e73b52ff38aadc9d72ffac05f2d7477c6c6580a9925f3f688dcd17a73afe1699ad001c4bc091b04edb0899825b93e3069e7bee4ae7af5e81979ffde0
|
7
|
+
data.tar.gz: 8f4737df24ffb6775e17e6c3b9dcfd330492c988c7bfbd8c71faec993f69923267cd84ddfda0d43aa4946f9bcb5299c320b7e7ea07d0d6a8384a33dd1944a36c
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.3.11 - 2016-11-18
|
2
|
+
|
3
|
+
* [maintenance] Refactor TdOutputPlugin, FieldWriterSet and else [#56](https://github.com/treasure-data/embulk-output-td/pull/56)
|
4
|
+
|
1
5
|
## 0.3.10 - 2016-11-16
|
2
6
|
|
3
7
|
* [maintenance] Setup bintray repo [#55](https://github.com/treasure-data/embulk-output-td/pull/55)
|
data/README.md
CHANGED
@@ -92,3 +92,22 @@ $ ./gradlew gem classpath
|
|
92
92
|
|
93
93
|
### Run on Embulk
|
94
94
|
$ bin/embulk run -I embulk-output-td/lib/ config.yml
|
95
|
+
|
96
|
+
## Release
|
97
|
+
|
98
|
+
### Upload gem to Rubygems.org
|
99
|
+
|
100
|
+
```
|
101
|
+
$ ./gradlew gem # create .gem file under pkg/ directory
|
102
|
+
$ ./gradlew gemPush # create and publish .gem file
|
103
|
+
```
|
104
|
+
|
105
|
+
Repo URL: https://rubygems.org/gems/embulk-output-td
|
106
|
+
|
107
|
+
### Upload jars to Bintray.com
|
108
|
+
|
109
|
+
```
|
110
|
+
$ ./gradlew bintrayUpload
|
111
|
+
```
|
112
|
+
|
113
|
+
Repo URL: https://bintray.com/embulk-output-td/maven/embulk-output-td
|
data/build.gradle
CHANGED
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.3.
|
4
|
+
spec.version = "0.3.11"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
@@ -56,11 +56,6 @@ public class RecordWriter
|
|
56
56
|
this.fileSplitSize = task.getFileSplitSize() * 1024;
|
57
57
|
}
|
58
58
|
|
59
|
-
public static void validateSchema(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
|
60
|
-
{
|
61
|
-
new FieldWriterSet(log, task, schema, false);
|
62
|
-
}
|
63
|
-
|
64
59
|
@VisibleForTesting
|
65
60
|
public void open(final Schema schema)
|
66
61
|
throws IOException
|
@@ -1,6 +1,5 @@
|
|
1
1
|
package org.embulk.output.td;
|
2
2
|
|
3
|
-
import java.io.File;
|
4
3
|
import java.io.IOException;
|
5
4
|
import java.io.InputStream;
|
6
5
|
import java.util.List;
|
@@ -18,13 +17,11 @@ import javax.validation.constraints.Max;
|
|
18
17
|
import com.google.common.annotations.VisibleForTesting;
|
19
18
|
import com.google.common.base.Function;
|
20
19
|
import com.google.common.base.Optional;
|
21
|
-
import com.google.common.base.Preconditions;
|
22
20
|
import com.google.common.base.Predicates;
|
23
21
|
import com.google.common.base.Throwables;
|
22
|
+
import com.google.common.collect.Lists;
|
24
23
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
25
24
|
import com.fasterxml.jackson.annotation.JsonValue;
|
26
|
-
import com.google.common.collect.ImmutableMap;
|
27
|
-
import com.google.common.collect.Lists;
|
28
25
|
import com.treasuredata.client.ProxyConfig;
|
29
26
|
import com.treasuredata.client.TDClient;
|
30
27
|
import com.treasuredata.client.TDClientBuilder;
|
@@ -382,7 +379,7 @@ public class TdOutputPlugin
|
|
382
379
|
}
|
383
380
|
|
384
381
|
// validate FieldWriterSet configuration before transaction is started
|
385
|
-
|
382
|
+
validateFieldWriterSet(log, task, schema);
|
386
383
|
|
387
384
|
return doRun(client, schema, task, control);
|
388
385
|
}
|
@@ -695,7 +692,7 @@ public class TdOutputPlugin
|
|
695
692
|
|
696
693
|
public void longColumn(Column column)
|
697
694
|
{
|
698
|
-
guessedSchema.put(column.getName(), TDColumnType.LONG)
|
695
|
+
guessedSchema.put(column.getName(), TDColumnType.LONG);
|
699
696
|
}
|
700
697
|
|
701
698
|
public void doubleColumn(Column column)
|
@@ -741,7 +738,8 @@ public class TdOutputPlugin
|
|
741
738
|
|
742
739
|
if (!usedNames.containsKey(key)) {
|
743
740
|
usedNames.put(key, 1);
|
744
|
-
}
|
741
|
+
}
|
742
|
+
else {
|
745
743
|
int next = usedNames.get(key);
|
746
744
|
key = key + "_" + next;
|
747
745
|
usedNames.put(key, next + 1);
|
@@ -846,7 +844,7 @@ public class TdOutputPlugin
|
|
846
844
|
|
847
845
|
RecordWriter closeLater = null;
|
848
846
|
try {
|
849
|
-
FieldWriterSet fieldWriters =
|
847
|
+
FieldWriterSet fieldWriters = createFieldWriterSet(log, task, schema);
|
850
848
|
closeLater = new RecordWriter(task, taskIndex, newTDClient(task), fieldWriters);
|
851
849
|
RecordWriter recordWriter = closeLater;
|
852
850
|
recordWriter.open(schema);
|
@@ -869,4 +867,14 @@ public class TdOutputPlugin
|
|
869
867
|
{
|
870
868
|
return System.getProperty("java.io.tmpdir");
|
871
869
|
}
|
870
|
+
|
871
|
+
protected FieldWriterSet createFieldWriterSet(Logger log, PluginTask task, Schema schema)
|
872
|
+
{
|
873
|
+
return FieldWriterSet.createWithValidation(log, task, schema, true);
|
874
|
+
}
|
875
|
+
|
876
|
+
protected void validateFieldWriterSet(Logger log, PluginTask task, Schema schema)
|
877
|
+
{
|
878
|
+
FieldWriterSet.createWithValidation(log, task, schema, false);
|
879
|
+
}
|
872
880
|
}
|
@@ -42,8 +42,9 @@ public abstract class TimeValueGenerator
|
|
42
42
|
|
43
43
|
public IncrementalTimeValueGenerator(final TimeValueConfig config)
|
44
44
|
{
|
45
|
-
|
46
|
-
to = config.getTo().get();
|
45
|
+
this.from = config.getFrom().get();
|
46
|
+
this.to = config.getTo().get();
|
47
|
+
this.current = from;
|
47
48
|
}
|
48
49
|
|
49
50
|
@Override
|
@@ -6,7 +6,6 @@ import com.google.common.annotations.VisibleForTesting;
|
|
6
6
|
import com.google.common.base.Optional;
|
7
7
|
import com.google.common.base.Throwables;
|
8
8
|
import org.embulk.config.ConfigException;
|
9
|
-
import org.embulk.config.ConfigSource;
|
10
9
|
import org.embulk.output.td.TdOutputPlugin;
|
11
10
|
import org.embulk.output.td.TdOutputPlugin.ConvertTimestampType;
|
12
11
|
import org.embulk.output.td.TimeValueConfig;
|
@@ -41,7 +40,14 @@ public class FieldWriterSet
|
|
41
40
|
private final IFieldWriter[] fieldWriters;
|
42
41
|
private final Optional<TimeValueGenerator> staticTimeValue;
|
43
42
|
|
44
|
-
|
43
|
+
protected FieldWriterSet(int fieldCount, IFieldWriter[] fieldWriters, Optional<TimeValueGenerator> staticTimeValue)
|
44
|
+
{
|
45
|
+
this.fieldCount = fieldCount;
|
46
|
+
this.fieldWriters = fieldWriters;
|
47
|
+
this.staticTimeValue = staticTimeValue;
|
48
|
+
}
|
49
|
+
|
50
|
+
public static FieldWriterSet createWithValidation(Logger log, TdOutputPlugin.PluginTask task, Schema schema, boolean runStage)
|
45
51
|
{
|
46
52
|
Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
|
47
53
|
ConvertTimestampType convertTimestampType = task.getConvertTimestampType();
|
@@ -54,7 +60,7 @@ public class FieldWriterSet
|
|
54
60
|
int duplicatePrimaryKeySourceIndex = -1;
|
55
61
|
|
56
62
|
int fc = 0;
|
57
|
-
|
63
|
+
IFieldWriter[] createdFieldWriters = new IFieldWriter[schema.size()];
|
58
64
|
TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
|
59
65
|
|
60
66
|
for (int i = 0; i < schema.size(); i++) {
|
@@ -140,24 +146,18 @@ public class FieldWriterSet
|
|
140
146
|
throw new AssertionError();
|
141
147
|
}
|
142
148
|
|
143
|
-
|
149
|
+
createdFieldWriters[i] = writer;
|
144
150
|
fc += 1;
|
145
151
|
}
|
146
152
|
|
147
153
|
if (foundPrimaryKey) {
|
148
154
|
// appropriate 'time' column is found
|
149
|
-
|
150
|
-
staticTimeValue = Optional.absent();
|
151
|
-
fieldCount = fc;
|
152
|
-
return;
|
155
|
+
return new FieldWriterSet(fc, createdFieldWriters, Optional.<TimeValueGenerator>absent());
|
153
156
|
}
|
154
157
|
|
155
158
|
if (timeValueConfig.isPresent()) {
|
156
159
|
// 'time_value' option is specified
|
157
|
-
|
158
|
-
staticTimeValue = Optional.of(TimeValueGenerator.newGenerator(timeValueConfig.get()));
|
159
|
-
fieldCount = fc + 1;
|
160
|
-
return;
|
160
|
+
return new FieldWriterSet(fc + 1, createdFieldWriters, Optional.of(TimeValueGenerator.newGenerator(timeValueConfig.get())));
|
161
161
|
}
|
162
162
|
|
163
163
|
if (!foundPrimaryKey && duplicatePrimaryKeySourceIndex >= 0) {
|
@@ -189,10 +189,8 @@ public class FieldWriterSet
|
|
189
189
|
}
|
190
190
|
|
191
191
|
// replace existint writer
|
192
|
-
|
193
|
-
|
194
|
-
fieldCount = fc + 1;
|
195
|
-
return;
|
192
|
+
createdFieldWriters[duplicatePrimaryKeySourceIndex] = writer;
|
193
|
+
return new FieldWriterSet(fc + 1, createdFieldWriters, Optional.<TimeValueGenerator>absent());
|
196
194
|
}
|
197
195
|
|
198
196
|
if (!foundPrimaryKey) {
|
@@ -208,9 +206,7 @@ public class FieldWriterSet
|
|
208
206
|
}
|
209
207
|
TimeValueConfig newConfig = Exec.newConfigSource().set("mode", "fixed_time").set("value", uploadTime).loadConfig(TimeValueConfig.class);
|
210
208
|
task.setTimeValue(Optional.of(newConfig));
|
211
|
-
|
212
|
-
fieldCount = fc + 1;
|
213
|
-
return;
|
209
|
+
return new FieldWriterSet(fc + 1, createdFieldWriters, Optional.of(TimeValueGenerator.newGenerator(newConfig)));
|
214
210
|
}
|
215
211
|
|
216
212
|
throw new AssertionError("Cannot select primary key");
|
@@ -236,7 +232,7 @@ public class FieldWriterSet
|
|
236
232
|
return false;
|
237
233
|
}
|
238
234
|
|
239
|
-
|
235
|
+
protected static FieldWriter newSimpleFieldWriter(String columnName, Type columnType, ConvertTimestampType convertTimestampType, TimestampFormatter timestampFormatter)
|
240
236
|
{
|
241
237
|
if (columnType instanceof BooleanType) {
|
242
238
|
return new BooleanFieldWriter(columnName);
|
@@ -261,7 +257,7 @@ public class FieldWriterSet
|
|
261
257
|
}
|
262
258
|
}
|
263
259
|
|
264
|
-
|
260
|
+
protected static FieldWriter newSimpleTimestampFieldWriter(String columnName, Type columnType, ConvertTimestampType convertTimestampType, TimestampFormatter timestampFormatter)
|
265
261
|
{
|
266
262
|
switch (convertTimestampType) {
|
267
263
|
case STRING:
|
@@ -11,7 +11,7 @@ public class UnixTimestampLongFieldWriter
|
|
11
11
|
{
|
12
12
|
private final int fractionUnit;
|
13
13
|
|
14
|
-
UnixTimestampLongFieldWriter(String keyName, int fractionUnit)
|
14
|
+
public UnixTimestampLongFieldWriter(String keyName, int fractionUnit)
|
15
15
|
{
|
16
16
|
super(keyName);
|
17
17
|
this.fractionUnit = fractionUnit;
|
@@ -257,26 +257,22 @@ public class TestTdOutputPlugin
|
|
257
257
|
{
|
258
258
|
{ // no proxy setting
|
259
259
|
PluginTask task = pluginTask(config);
|
260
|
-
|
261
|
-
|
262
|
-
// no error happens
|
260
|
+
TDClient client = plugin.newTDClient(task);
|
261
|
+
// Expect no error happens
|
263
262
|
}
|
264
263
|
|
265
264
|
{ // proxy setting
|
266
265
|
PluginTask task = pluginTask(config.deepCopy()
|
267
266
|
.set("http_proxy", ImmutableMap.of("host", "xxx", "port", "8080", "user", "foo", "password", "PASSWORD")));
|
268
|
-
|
269
|
-
|
270
|
-
// no error happens
|
267
|
+
TDClient client = plugin.newTDClient(task);
|
268
|
+
// Expect no error happens
|
271
269
|
}
|
272
270
|
|
273
|
-
|
274
271
|
{ // proxy setting without user/password
|
275
272
|
PluginTask task = pluginTask(config.deepCopy()
|
276
273
|
.set("http_proxy", ImmutableMap.of("host", "xxx", "port", "8080")));
|
277
|
-
|
278
|
-
|
279
|
-
// no error happens
|
274
|
+
TDClient client = plugin.newTDClient(task);
|
275
|
+
// Expect no error happens
|
280
276
|
}
|
281
277
|
}
|
282
278
|
|
@@ -552,10 +548,8 @@ public class TestTdOutputPlugin
|
|
552
548
|
task.setTempDir(plugin.getEnvironmentTempDirectory());
|
553
549
|
Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
|
554
550
|
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
// no error happens.
|
551
|
+
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
|
552
|
+
// Expect no error happens.
|
559
553
|
}
|
560
554
|
|
561
555
|
public static ConfigSource config()
|
@@ -600,7 +594,7 @@ public class TestTdOutputPlugin
|
|
600
594
|
|
601
595
|
public static FieldWriterSet fieldWriters(Logger log, PluginTask task, Schema schema)
|
602
596
|
{
|
603
|
-
return spy(
|
597
|
+
return spy(FieldWriterSet.createWithValidation(log, task, schema, false));
|
604
598
|
}
|
605
599
|
|
606
600
|
public static RecordWriter recordWriter(PluginTask task, TDClient client, FieldWriterSet fieldWriters)
|
@@ -41,12 +41,12 @@ public class TestTimeValueGenerator
|
|
41
41
|
// incremental_time
|
42
42
|
{ // {from: 0, to: 0} # default incremental_time
|
43
43
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
44
|
-
|
44
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema, false);
|
45
45
|
}
|
46
46
|
{ // {from: 0} # default incremental_time
|
47
47
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
48
48
|
try {
|
49
|
-
|
49
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L))), schema, false);
|
50
50
|
fail();
|
51
51
|
}
|
52
52
|
catch (Throwable t) {
|
@@ -56,7 +56,7 @@ public class TestTimeValueGenerator
|
|
56
56
|
{ // {to: 0} # default incremental_time
|
57
57
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
58
58
|
try {
|
59
|
-
|
59
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L))), schema, false);
|
60
60
|
fail();
|
61
61
|
}
|
62
62
|
catch (Throwable t) {
|
@@ -65,12 +65,12 @@ public class TestTimeValueGenerator
|
|
65
65
|
}
|
66
66
|
{ // {from: 0, to: 0, mode: incremental_time}
|
67
67
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
68
|
-
|
68
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L, "mode", "incremental_time"))), schema, false);
|
69
69
|
}
|
70
70
|
{ // {from: 0, mode: incremental_time}
|
71
71
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
72
72
|
try {
|
73
|
-
|
73
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "mode", "incremental_time"))), schema, false);
|
74
74
|
fail();
|
75
75
|
}
|
76
76
|
catch (Throwable t) {
|
@@ -80,7 +80,7 @@ public class TestTimeValueGenerator
|
|
80
80
|
{ // {to: 0, mode: incremental_time}
|
81
81
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
82
82
|
try {
|
83
|
-
|
83
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L, "mode", "incremental_time"))), schema, false);
|
84
84
|
fail();
|
85
85
|
}
|
86
86
|
catch (Throwable t) {
|
@@ -90,7 +90,7 @@ public class TestTimeValueGenerator
|
|
90
90
|
{ // {mode: incremental_time}
|
91
91
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
92
92
|
try {
|
93
|
-
|
93
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "incremental_time"))), schema, false);
|
94
94
|
fail();
|
95
95
|
}
|
96
96
|
catch (Throwable t) {
|
@@ -101,12 +101,12 @@ public class TestTimeValueGenerator
|
|
101
101
|
// fixed_time
|
102
102
|
{ // {value: 0, mode: fixed_time}
|
103
103
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
104
|
-
|
104
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L, "mode", "fixed_time"))), schema, false);
|
105
105
|
}
|
106
106
|
{ // {mode: fixed_time}
|
107
107
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
108
108
|
try {
|
109
|
-
|
109
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "fixed_time"))), schema, false);
|
110
110
|
}
|
111
111
|
catch (Throwable t) {
|
112
112
|
assertTrue(t instanceof ConfigException);
|
@@ -115,7 +115,7 @@ public class TestTimeValueGenerator
|
|
115
115
|
{ // {value: 0}
|
116
116
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
117
117
|
try {
|
118
|
-
|
118
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L))), schema, false);
|
119
119
|
}
|
120
120
|
catch (Throwable t) {
|
121
121
|
assertTrue(t instanceof ConfigException);
|
@@ -40,7 +40,7 @@ public class TestFieldWriterSet
|
|
40
40
|
{ // if schema doesn't have appropriate time column, it throws ConfigError.
|
41
41
|
schema = schema("_c0", Types.STRING, "time", Types.STRING); // not long or timestamp
|
42
42
|
try {
|
43
|
-
|
43
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config), schema, false);
|
44
44
|
fail();
|
45
45
|
}
|
46
46
|
catch (Throwable t) {
|
@@ -51,7 +51,7 @@ public class TestFieldWriterSet
|
|
51
51
|
{ // if schema doesn't have a column specified as time_column column, it throws ConfigError
|
52
52
|
schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
|
53
53
|
try {
|
54
|
-
|
54
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c2")), schema, false);
|
55
55
|
fail();
|
56
56
|
}
|
57
57
|
catch (Throwable t) {
|
@@ -62,7 +62,7 @@ public class TestFieldWriterSet
|
|
62
62
|
{ // if time_column column is not appropriate column type, it throws ConfigError.
|
63
63
|
schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
|
64
64
|
try {
|
65
|
-
|
65
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c1")), schema, false);
|
66
66
|
fail();
|
67
67
|
}
|
68
68
|
catch (Throwable t) {
|
@@ -73,7 +73,7 @@ public class TestFieldWriterSet
|
|
73
73
|
{ // if both of time_column and time_value are specified, it throws ConfigError.
|
74
74
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
75
75
|
try {
|
76
|
-
|
76
|
+
FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c1").set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema, false);
|
77
77
|
fail();
|
78
78
|
}
|
79
79
|
catch (Throwable t) {
|
@@ -87,14 +87,14 @@ public class TestFieldWriterSet
|
|
87
87
|
{
|
88
88
|
{ // time column (timestamp type) exists
|
89
89
|
Schema schema = schema("time", Types.TIMESTAMP, "_c0", Types.TIMESTAMP);
|
90
|
-
FieldWriterSet writers =
|
90
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config), schema, false);
|
91
91
|
|
92
92
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter);
|
93
93
|
}
|
94
94
|
|
95
95
|
{ // time column (long type) exists
|
96
96
|
Schema schema = schema("time", Types.LONG, "_c0", Types.TIMESTAMP);
|
97
|
-
FieldWriterSet writers =
|
97
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config), schema, false);
|
98
98
|
|
99
99
|
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampLongFieldWriter);
|
100
100
|
|
@@ -106,21 +106,21 @@ public class TestFieldWriterSet
|
|
106
106
|
{
|
107
107
|
{ // time_column option (timestamp type)
|
108
108
|
Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.STRING);
|
109
|
-
FieldWriterSet writers =
|
109
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
110
110
|
|
111
111
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator);
|
112
112
|
}
|
113
113
|
|
114
114
|
{ // time_column option (long type)
|
115
115
|
Schema schema = schema("_c0", Types.LONG, "_c1", Types.STRING);
|
116
|
-
FieldWriterSet writers =
|
116
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
117
117
|
|
118
118
|
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator);
|
119
119
|
}
|
120
120
|
|
121
121
|
{ // time_column option (typestamp type) if time column exists
|
122
122
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
123
|
-
FieldWriterSet writers =
|
123
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
124
124
|
|
125
125
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
126
126
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
|
@@ -128,7 +128,7 @@ public class TestFieldWriterSet
|
|
128
128
|
|
129
129
|
{ // time_column option (long type) if time column exists
|
130
130
|
Schema schema = schema("_c0", Types.LONG, "time", Types.TIMESTAMP);
|
131
|
-
FieldWriterSet writers =
|
131
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
132
132
|
|
133
133
|
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator); // c0
|
134
134
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
|
@@ -140,7 +140,7 @@ public class TestFieldWriterSet
|
|
140
140
|
{
|
141
141
|
{ // if not specify default_timestamp_type_convert_to, use string by default
|
142
142
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
143
|
-
FieldWriterSet writers =
|
143
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy()), schema, false);
|
144
144
|
|
145
145
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
146
146
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
@@ -148,7 +148,7 @@ public class TestFieldWriterSet
|
|
148
148
|
|
149
149
|
{ // and use time_column option
|
150
150
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
151
|
-
FieldWriterSet writers =
|
151
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
152
152
|
|
153
153
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
154
154
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
@@ -156,7 +156,7 @@ public class TestFieldWriterSet
|
|
156
156
|
|
157
157
|
{ // if default_timestamp_type_convert_to is string, use string
|
158
158
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
159
|
-
FieldWriterSet writers =
|
159
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string")), schema, false);
|
160
160
|
|
161
161
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
162
162
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
@@ -164,7 +164,7 @@ public class TestFieldWriterSet
|
|
164
164
|
|
165
165
|
{ // and use time_column option
|
166
166
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
167
|
-
FieldWriterSet writers =
|
167
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string").set("time_column", "_c0")), schema, false);
|
168
168
|
|
169
169
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
170
170
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
@@ -172,7 +172,7 @@ public class TestFieldWriterSet
|
|
172
172
|
|
173
173
|
{ // if default_timestamp_type_conver_to is sec, use long
|
174
174
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
175
|
-
FieldWriterSet writers =
|
175
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec")), schema, false);
|
176
176
|
|
177
177
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter); // c0
|
178
178
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
@@ -180,7 +180,7 @@ public class TestFieldWriterSet
|
|
180
180
|
|
181
181
|
{ // and use time_column option
|
182
182
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
183
|
-
FieldWriterSet writers =
|
183
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec").set("time_column", "_c0")), schema, false);
|
184
184
|
|
185
185
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
186
186
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time renamed
|
@@ -192,7 +192,7 @@ public class TestFieldWriterSet
|
|
192
192
|
throws Exception
|
193
193
|
{
|
194
194
|
Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.LONG);
|
195
|
-
FieldWriterSet writers =
|
195
|
+
FieldWriterSet writers = FieldWriterSet.createWithValidation(log, pluginTask(config), schema, false);
|
196
196
|
|
197
197
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
198
198
|
assertTrue(writers.getFieldWriter(1) instanceof LongFieldWriter); // c1
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-11-
|
11
|
+
date: 2016-11-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,7 +82,7 @@ files:
|
|
82
82
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
83
83
|
- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
|
84
84
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
85
|
-
- classpath/embulk-output-td-0.3.
|
85
|
+
- classpath/embulk-output-td-0.3.11.jar
|
86
86
|
- classpath/hamcrest-core-1.1.jar
|
87
87
|
- classpath/jackson-annotations-2.6.7.jar
|
88
88
|
- classpath/jackson-core-2.6.7.jar
|