embulk-output-td 0.3.5 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/build.gradle +1 -1
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/RecordWriter.java +1 -1
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +39 -5
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +26 -12
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +28 -1
- data/src/test/java/org/embulk/output/td/TestTimeValueGenerator.java +10 -10
- data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +17 -17
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ecdc89638bd881305f9a2103c0f93949487fd7cf
|
4
|
+
data.tar.gz: fbb66bd29c049357f814baf21c649dc0ba6aab1f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ff210b5c00c02e08392c162f5c8ddeb1bebf7ae3432074ce7a17d8a7890692664799ea667262f00548ac9c9e4cc775191138a7f6ec0cd9d835279b615d1594f8
|
7
|
+
data.tar.gz: 2146190b46c8eb9ec89337ff4c03aab16097e9d82bc2c55f36271d0501eef986f8a1d23bcea1dc2db8d7222d90a00501356f4e4f8a9864643a16dd78d264599c
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.3.6 - 2016-07-11
|
2
|
+
|
3
|
+
* [new feature] Http proxy config from system properties [#47](https://github.com/treasure-data/embulk-output-td/pull/47)
|
4
|
+
* [maintenance] Remove redundant logging during run stage [#48](https://github.com/treasure-data/embulk-output-td/pull/48)
|
5
|
+
|
1
6
|
## 0.3.5 - 2016-06-29
|
2
7
|
|
3
8
|
* [new feature] Enable user/password for `http_proxy` option [#46](https://github.com/treasure-data/embulk-output-td/pull/46)
|
data/build.gradle
CHANGED
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.3.
|
4
|
+
spec.version = "0.3.6"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
@@ -8,6 +8,7 @@ import java.util.ArrayList;
|
|
8
8
|
import java.util.Map;
|
9
9
|
import java.util.HashMap;
|
10
10
|
import java.nio.charset.StandardCharsets;
|
11
|
+
import java.util.Properties;
|
11
12
|
import java.util.regex.Pattern;
|
12
13
|
import java.util.zip.GZIPInputStream;
|
13
14
|
|
@@ -17,6 +18,8 @@ import javax.validation.constraints.Max;
|
|
17
18
|
import com.google.common.annotations.VisibleForTesting;
|
18
19
|
import com.google.common.base.Function;
|
19
20
|
import com.google.common.base.Optional;
|
21
|
+
import com.google.common.base.Preconditions;
|
22
|
+
import com.google.common.base.Predicates;
|
20
23
|
import com.google.common.base.Throwables;
|
21
24
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
22
25
|
import com.fasterxml.jackson.annotation.JsonValue;
|
@@ -57,6 +60,9 @@ import org.msgpack.core.MessageUnpacker;
|
|
57
60
|
import org.msgpack.value.Value;
|
58
61
|
import org.slf4j.Logger;
|
59
62
|
|
63
|
+
import static com.google.common.base.Optional.fromNullable;
|
64
|
+
import static java.lang.Integer.parseInt;
|
65
|
+
|
60
66
|
public class TdOutputPlugin
|
61
67
|
implements OutputPlugin
|
62
68
|
{
|
@@ -436,14 +442,42 @@ public class TdOutputPlugin
|
|
436
442
|
builder.setApiKey(task.getApiKey());
|
437
443
|
builder.setEndpoint(task.getEndpoint());
|
438
444
|
builder.setUseSSL(task.getUseSsl());
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
445
|
+
|
446
|
+
Optional<ProxyConfig> proxyConfig = newProxyConfig(task.getHttpProxy());
|
447
|
+
if (proxyConfig.isPresent()) {
|
448
|
+
builder.setProxy(proxyConfig.get());
|
443
449
|
}
|
450
|
+
|
444
451
|
return builder.build();
|
445
452
|
}
|
446
453
|
|
454
|
+
@VisibleForTesting
|
455
|
+
Optional<ProxyConfig> newProxyConfig(Optional<HttpProxyTask> task)
|
456
|
+
{
|
457
|
+
// This plugin searches http proxy settings and configures them to TDClient. The order of proxy setting searching is:
|
458
|
+
// 1. System properties
|
459
|
+
// 2. http_proxy config option provided by this plugin
|
460
|
+
|
461
|
+
Properties props = System.getProperties();
|
462
|
+
if (props.containsKey("http.proxyHost") || props.containsKey("https.proxyHost")) {
|
463
|
+
boolean useSsl = props.containsKey("https.proxyHost");
|
464
|
+
String proto = !useSsl ? "http" : "https";
|
465
|
+
String host = props.getProperty(proto + ".proxyHost");
|
466
|
+
int port = parseInt(props.getProperty(proto + ".proxyPort", !useSsl ? "80" : "443"));
|
467
|
+
Optional<String> user = fromNullable(props.getProperty(proto + ".proxyUser"));
|
468
|
+
Optional<String> password = fromNullable(props.getProperty(proto + ".proxyPassword"));
|
469
|
+
return Optional.of(new ProxyConfig(host, port, useSsl, user, password));
|
470
|
+
}
|
471
|
+
else if (task.isPresent()) {
|
472
|
+
HttpProxyTask proxyTask = task.get();
|
473
|
+
return Optional.of(new ProxyConfig(proxyTask.getHost(), proxyTask.getPort(), proxyTask.getUseSsl(),
|
474
|
+
proxyTask.getUser(), proxyTask.getPassword()));
|
475
|
+
}
|
476
|
+
else {
|
477
|
+
return Optional.absent();
|
478
|
+
}
|
479
|
+
}
|
480
|
+
|
447
481
|
@VisibleForTesting
|
448
482
|
void createTableIfNotExists(TDClient client, String databaseName, String tableName)
|
449
483
|
{
|
@@ -795,7 +829,7 @@ public class TdOutputPlugin
|
|
795
829
|
|
796
830
|
RecordWriter closeLater = null;
|
797
831
|
try {
|
798
|
-
FieldWriterSet fieldWriters = new FieldWriterSet(log, task, schema);
|
832
|
+
FieldWriterSet fieldWriters = new FieldWriterSet(log, task, schema, true);
|
799
833
|
closeLater = new RecordWriter(task, taskIndex, newTDClient(task), fieldWriters);
|
800
834
|
RecordWriter recordWriter = closeLater;
|
801
835
|
recordWriter.open(schema);
|
@@ -41,7 +41,7 @@ public class FieldWriterSet
|
|
41
41
|
private final IFieldWriter[] fieldWriters;
|
42
42
|
private final Optional<TimeValueGenerator> staticTimeValue;
|
43
43
|
|
44
|
-
public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
|
44
|
+
public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema, boolean runStage)
|
45
45
|
{
|
46
46
|
Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
|
47
47
|
ConvertTimestampType convertTimestampType = task.getConvertTimestampType();
|
@@ -79,14 +79,18 @@ public class FieldWriterSet
|
|
79
79
|
if (userDefinedPrimaryKeySourceColumnName.isPresent()) {
|
80
80
|
columnName = newColumnUniqueName(columnName, schema);
|
81
81
|
mode = ColumnWriterMode.SIMPLE_VALUE;
|
82
|
-
|
83
|
-
|
82
|
+
if (!runStage) {
|
83
|
+
log.warn("time_column '{}' is set but 'time' column also exists. The existent 'time' column is renamed to {}",
|
84
|
+
userDefinedPrimaryKeySourceColumnName.get(), columnName);
|
85
|
+
}
|
84
86
|
}
|
85
87
|
else if (timeValueConfig.isPresent()) {
|
86
88
|
columnName = newColumnUniqueName(columnName, schema);
|
87
89
|
mode = ColumnWriterMode.SIMPLE_VALUE;
|
88
|
-
|
89
|
-
|
90
|
+
if (!runStage) {
|
91
|
+
log.warn("time_value is set but 'time' column also exists. The existent 'time' column is renamed to {}",
|
92
|
+
columnName);
|
93
|
+
}
|
90
94
|
}
|
91
95
|
else {
|
92
96
|
mode = ColumnWriterMode.PRIMARY_KEY;
|
@@ -101,10 +105,14 @@ public class FieldWriterSet
|
|
101
105
|
|
102
106
|
switch (mode) {
|
103
107
|
case PRIMARY_KEY:
|
104
|
-
|
108
|
+
if (!runStage) {
|
109
|
+
log.info("Using {}:{} column as the data partitioning key", columnName, columnType);
|
110
|
+
}
|
105
111
|
if (columnType instanceof LongType) {
|
106
112
|
if (task.getUnixTimestampUnit() != TdOutputPlugin.UnixTimestampUnit.SEC) {
|
107
|
-
|
113
|
+
if (!runStage) {
|
114
|
+
log.warn("time column is converted from {} to seconds", task.getUnixTimestampUnit());
|
115
|
+
}
|
108
116
|
}
|
109
117
|
writer = new UnixTimestampLongFieldWriter(columnName, task.getUnixTimestampUnit().getFractionUnit());
|
110
118
|
foundPrimaryKey = true;
|
@@ -160,14 +168,18 @@ public class FieldWriterSet
|
|
160
168
|
|
161
169
|
IFieldWriter writer;
|
162
170
|
if (columnType instanceof LongType) {
|
163
|
-
|
164
|
-
|
171
|
+
if (!runStage) {
|
172
|
+
log.info("Duplicating {}:{} column (unix timestamp {}) to 'time' column as seconds for the data partitioning",
|
173
|
+
columnName, columnType, task.getUnixTimestampUnit());
|
174
|
+
}
|
165
175
|
IFieldWriter fw = new LongFieldWriter(columnName);
|
166
176
|
writer = new UnixTimestampFieldDuplicator(fw, "time", task.getUnixTimestampUnit().getFractionUnit());
|
167
177
|
}
|
168
178
|
else if (columnType instanceof TimestampType) {
|
169
|
-
|
170
|
-
|
179
|
+
if (!runStage) {
|
180
|
+
log.info("Duplicating {}:{} column to 'time' column as seconds for the data partitioning",
|
181
|
+
columnName, columnType);
|
182
|
+
}
|
171
183
|
IFieldWriter fw = newSimpleTimestampFieldWriter(columnName, columnType, convertTimestampType, timestampFormatters[duplicatePrimaryKeySourceIndex]);
|
172
184
|
writer = new TimestampFieldLongDuplicator(fw, "time");
|
173
185
|
}
|
@@ -191,7 +203,9 @@ public class FieldWriterSet
|
|
191
203
|
}
|
192
204
|
|
193
205
|
long uploadTime = System.currentTimeMillis() / 1000;
|
194
|
-
|
206
|
+
if (!runStage) {
|
207
|
+
log.info("'time' column is generated and is set to a unix time {}", uploadTime);
|
208
|
+
}
|
195
209
|
TimeValueConfig newConfig = Exec.newConfigSource().set("mode", "fixed_time").set("value", uploadTime).loadConfig(TimeValueConfig.class);
|
196
210
|
task.setTimeValue(Optional.of(newConfig));
|
197
211
|
staticTimeValue = Optional.of(TimeValueGenerator.newGenerator(newConfig));
|
@@ -1,8 +1,10 @@
|
|
1
1
|
package org.embulk.output.td;
|
2
2
|
|
3
|
+
import com.google.common.base.Optional;
|
3
4
|
import com.google.common.collect.ImmutableList;
|
4
5
|
import com.google.common.collect.ImmutableMap;
|
5
6
|
import com.google.common.collect.Lists;
|
7
|
+
import com.treasuredata.client.ProxyConfig;
|
6
8
|
import com.treasuredata.client.TDClient;
|
7
9
|
import com.treasuredata.client.TDClientHttpConflictException;
|
8
10
|
import com.treasuredata.client.TDClientHttpNotFoundException;
|
@@ -18,6 +20,7 @@ import org.embulk.config.ConfigException;
|
|
18
20
|
import org.embulk.config.ConfigSource;
|
19
21
|
import org.embulk.config.TaskSource;
|
20
22
|
import org.embulk.output.td.TdOutputPlugin.PluginTask;
|
23
|
+
import org.embulk.output.td.TdOutputPlugin.HttpProxyTask;
|
21
24
|
import org.embulk.output.td.TdOutputPlugin.TimestampColumnOption;
|
22
25
|
import org.embulk.output.td.TdOutputPlugin.UnixTimestampUnit;
|
23
26
|
import org.embulk.output.td.writer.FieldWriterSet;
|
@@ -390,6 +393,30 @@ public class TestTdOutputPlugin
|
|
390
393
|
}
|
391
394
|
}
|
392
395
|
|
396
|
+
@Test
|
397
|
+
public void newProxyConfig()
|
398
|
+
{
|
399
|
+
// confirm if proxy system properties override proxy setting by http_proxy config option.
|
400
|
+
|
401
|
+
HttpProxyTask proxyTask = Exec.newConfigSource()
|
402
|
+
.set("host", "option_host")
|
403
|
+
.set("port", 8080)
|
404
|
+
.loadConfig(HttpProxyTask.class);
|
405
|
+
|
406
|
+
String originalProxyHost = System.getProperty("http.proxyHost");
|
407
|
+
try {
|
408
|
+
System.setProperty("http.proxyHost", "property_host");
|
409
|
+
Optional<ProxyConfig> proxyConfig = plugin.newProxyConfig(Optional.of(proxyTask));
|
410
|
+
assertEquals("property_host", proxyConfig.get().getHost());
|
411
|
+
assertEquals(80, proxyConfig.get().getPort());
|
412
|
+
}
|
413
|
+
finally {
|
414
|
+
if (originalProxyHost != null) {
|
415
|
+
System.setProperty("http.proxyHost", originalProxyHost);
|
416
|
+
}
|
417
|
+
}
|
418
|
+
}
|
419
|
+
|
393
420
|
@Test
|
394
421
|
public void completeBulkImportSession()
|
395
422
|
{
|
@@ -533,7 +560,7 @@ public class TestTdOutputPlugin
|
|
533
560
|
|
534
561
|
public static FieldWriterSet fieldWriters(Logger log, PluginTask task, Schema schema)
|
535
562
|
{
|
536
|
-
return spy(new FieldWriterSet(log, task, schema));
|
563
|
+
return spy(new FieldWriterSet(log, task, schema, false));
|
537
564
|
}
|
538
565
|
|
539
566
|
public static RecordWriter recordWriter(PluginTask task, TDClient client, FieldWriterSet fieldWriters)
|
@@ -41,12 +41,12 @@ public class TestTimeValueGenerator
|
|
41
41
|
// incremental_time
|
42
42
|
{ // {from: 0, to: 0} # default incremental_time
|
43
43
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
44
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema);
|
44
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema, false);
|
45
45
|
}
|
46
46
|
{ // {from: 0} # default incremental_time
|
47
47
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
48
48
|
try {
|
49
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L))), schema);
|
49
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L))), schema, false);
|
50
50
|
fail();
|
51
51
|
}
|
52
52
|
catch (Throwable t) {
|
@@ -56,7 +56,7 @@ public class TestTimeValueGenerator
|
|
56
56
|
{ // {to: 0} # default incremental_time
|
57
57
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
58
58
|
try {
|
59
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L))), schema);
|
59
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L))), schema, false);
|
60
60
|
fail();
|
61
61
|
}
|
62
62
|
catch (Throwable t) {
|
@@ -65,12 +65,12 @@ public class TestTimeValueGenerator
|
|
65
65
|
}
|
66
66
|
{ // {from: 0, to: 0, mode: incremental_time}
|
67
67
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
68
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L, "mode", "incremental_time"))), schema);
|
68
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "to", 0L, "mode", "incremental_time"))), schema, false);
|
69
69
|
}
|
70
70
|
{ // {from: 0, mode: incremental_time}
|
71
71
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
72
72
|
try {
|
73
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "mode", "incremental_time"))), schema);
|
73
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("from", 0L, "mode", "incremental_time"))), schema, false);
|
74
74
|
fail();
|
75
75
|
}
|
76
76
|
catch (Throwable t) {
|
@@ -80,7 +80,7 @@ public class TestTimeValueGenerator
|
|
80
80
|
{ // {to: 0, mode: incremental_time}
|
81
81
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
82
82
|
try {
|
83
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L, "mode", "incremental_time"))), schema);
|
83
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("to", 0L, "mode", "incremental_time"))), schema, false);
|
84
84
|
fail();
|
85
85
|
}
|
86
86
|
catch (Throwable t) {
|
@@ -90,7 +90,7 @@ public class TestTimeValueGenerator
|
|
90
90
|
{ // {mode: incremental_time}
|
91
91
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
92
92
|
try {
|
93
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "incremental_time"))), schema);
|
93
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "incremental_time"))), schema, false);
|
94
94
|
fail();
|
95
95
|
}
|
96
96
|
catch (Throwable t) {
|
@@ -101,12 +101,12 @@ public class TestTimeValueGenerator
|
|
101
101
|
// fixed_time
|
102
102
|
{ // {value: 0, mode: fixed_time}
|
103
103
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
104
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L, "mode", "fixed_time"))), schema);
|
104
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L, "mode", "fixed_time"))), schema, false);
|
105
105
|
}
|
106
106
|
{ // {mode: fixed_time}
|
107
107
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
108
108
|
try {
|
109
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "fixed_time"))), schema);
|
109
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("mode", "fixed_time"))), schema, false);
|
110
110
|
}
|
111
111
|
catch (Throwable t) {
|
112
112
|
assertTrue(t instanceof ConfigException);
|
@@ -115,7 +115,7 @@ public class TestTimeValueGenerator
|
|
115
115
|
{ // {value: 0}
|
116
116
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
117
117
|
try {
|
118
|
-
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L))), schema);
|
118
|
+
new FieldWriterSet(log, pluginTask(config.set("time_value", ImmutableMap.of("value", 0L))), schema, false);
|
119
119
|
}
|
120
120
|
catch (Throwable t) {
|
121
121
|
assertTrue(t instanceof ConfigException);
|
@@ -40,7 +40,7 @@ public class TestFieldWriterSet
|
|
40
40
|
{ // if schema doesn't have appropriate time column, it throws ConfigError.
|
41
41
|
schema = schema("_c0", Types.STRING, "time", Types.STRING); // not long or timestamp
|
42
42
|
try {
|
43
|
-
new FieldWriterSet(log, pluginTask(config), schema);
|
43
|
+
new FieldWriterSet(log, pluginTask(config), schema, false);
|
44
44
|
fail();
|
45
45
|
}
|
46
46
|
catch (Throwable t) {
|
@@ -51,7 +51,7 @@ public class TestFieldWriterSet
|
|
51
51
|
{ // if schema doesn't have a column specified as time_column column, it throws ConfigError
|
52
52
|
schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
|
53
53
|
try {
|
54
|
-
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c2")), schema);
|
54
|
+
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c2")), schema, false);
|
55
55
|
fail();
|
56
56
|
}
|
57
57
|
catch (Throwable t) {
|
@@ -62,7 +62,7 @@ public class TestFieldWriterSet
|
|
62
62
|
{ // if time_column column is not appropriate column type, it throws ConfigError.
|
63
63
|
schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
|
64
64
|
try {
|
65
|
-
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c1")), schema);
|
65
|
+
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c1")), schema, false);
|
66
66
|
fail();
|
67
67
|
}
|
68
68
|
catch (Throwable t) {
|
@@ -73,7 +73,7 @@ public class TestFieldWriterSet
|
|
73
73
|
{ // if both of time_column and time_value are specified, it throws ConfigError.
|
74
74
|
schema = schema("_c0", Types.STRING, "_c1", Types.LONG);
|
75
75
|
try {
|
76
|
-
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c1").set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema);
|
76
|
+
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c1").set("time_value", ImmutableMap.of("from", 0L, "to", 0L))), schema, false);
|
77
77
|
fail();
|
78
78
|
}
|
79
79
|
catch (Throwable t) {
|
@@ -87,14 +87,14 @@ public class TestFieldWriterSet
|
|
87
87
|
{
|
88
88
|
{ // time column (timestamp type) exists
|
89
89
|
Schema schema = schema("time", Types.TIMESTAMP, "_c0", Types.TIMESTAMP);
|
90
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
|
90
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema, false);
|
91
91
|
|
92
92
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter);
|
93
93
|
}
|
94
94
|
|
95
95
|
{ // time column (long type) exists
|
96
96
|
Schema schema = schema("time", Types.LONG, "_c0", Types.TIMESTAMP);
|
97
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
|
97
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema, false);
|
98
98
|
|
99
99
|
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampLongFieldWriter);
|
100
100
|
|
@@ -106,21 +106,21 @@ public class TestFieldWriterSet
|
|
106
106
|
{
|
107
107
|
{ // time_column option (timestamp type)
|
108
108
|
Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.STRING);
|
109
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
109
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
110
110
|
|
111
111
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator);
|
112
112
|
}
|
113
113
|
|
114
114
|
{ // time_column option (long type)
|
115
115
|
Schema schema = schema("_c0", Types.LONG, "_c1", Types.STRING);
|
116
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
116
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
117
117
|
|
118
118
|
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator);
|
119
119
|
}
|
120
120
|
|
121
121
|
{ // time_column option (typestamp type) if time column exists
|
122
122
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
123
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
123
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
124
124
|
|
125
125
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
126
126
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
|
@@ -128,7 +128,7 @@ public class TestFieldWriterSet
|
|
128
128
|
|
129
129
|
{ // time_column option (long type) if time column exists
|
130
130
|
Schema schema = schema("_c0", Types.LONG, "time", Types.TIMESTAMP);
|
131
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
131
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
132
132
|
|
133
133
|
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator); // c0
|
134
134
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
|
@@ -140,7 +140,7 @@ public class TestFieldWriterSet
|
|
140
140
|
{
|
141
141
|
{ // if not specify default_timestamp_type_convert_to, use string by default
|
142
142
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
143
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy()), schema);
|
143
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy()), schema, false);
|
144
144
|
|
145
145
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
146
146
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
@@ -148,7 +148,7 @@ public class TestFieldWriterSet
|
|
148
148
|
|
149
149
|
{ // and use time_column option
|
150
150
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
151
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
151
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema, false);
|
152
152
|
|
153
153
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
154
154
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
@@ -156,7 +156,7 @@ public class TestFieldWriterSet
|
|
156
156
|
|
157
157
|
{ // if default_timestamp_type_convert_to is string, use string
|
158
158
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
159
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string")), schema);
|
159
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string")), schema, false);
|
160
160
|
|
161
161
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
162
162
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
@@ -164,7 +164,7 @@ public class TestFieldWriterSet
|
|
164
164
|
|
165
165
|
{ // and use time_column option
|
166
166
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
167
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string").set("time_column", "_c0")), schema);
|
167
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string").set("time_column", "_c0")), schema, false);
|
168
168
|
|
169
169
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
170
170
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
@@ -172,7 +172,7 @@ public class TestFieldWriterSet
|
|
172
172
|
|
173
173
|
{ // if default_timestamp_type_conver_to is sec, use long
|
174
174
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
175
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec")), schema);
|
175
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec")), schema, false);
|
176
176
|
|
177
177
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter); // c0
|
178
178
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
@@ -180,7 +180,7 @@ public class TestFieldWriterSet
|
|
180
180
|
|
181
181
|
{ // and use time_column option
|
182
182
|
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
183
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec").set("time_column", "_c0")), schema);
|
183
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec").set("time_column", "_c0")), schema, false);
|
184
184
|
|
185
185
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
186
186
|
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time renamed
|
@@ -192,7 +192,7 @@ public class TestFieldWriterSet
|
|
192
192
|
throws Exception
|
193
193
|
{
|
194
194
|
Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.LONG);
|
195
|
-
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
|
195
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema, false);
|
196
196
|
|
197
197
|
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
198
198
|
assertTrue(writers.getFieldWriter(1) instanceof LongFieldWriter); // c1
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,7 +82,7 @@ files:
|
|
82
82
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
83
83
|
- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
|
84
84
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
85
|
-
- classpath/embulk-output-td-0.3.
|
85
|
+
- classpath/embulk-output-td-0.3.6.jar
|
86
86
|
- classpath/hamcrest-core-1.1.jar
|
87
87
|
- classpath/jackson-annotations-2.6.2.jar
|
88
88
|
- classpath/jackson-core-2.6.2.jar
|