embulk-output-td 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +2 -1
- data/build.gradle +6 -6
- data/embulk-output-td.gemspec +1 -1
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/td/RecordWriter.java +5 -4
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +51 -2
- data/src/main/java/org/embulk/output/td/writer/FieldWriter.java +1 -0
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +31 -7
- data/src/main/java/org/embulk/output/td/writer/IFieldWriter.java +13 -0
- data/src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java +7 -7
- data/src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java +6 -6
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +1 -1
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +8 -8
- data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +52 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7b42edcec64619a1d24ce6b36967527ab76b4a63
|
4
|
+
data.tar.gz: 402e46fa95353e30cc38668c007893612cc2323c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 013f4c79c83f0d8215fb9d92098dbc75bb18f41b911117b82ae9773b010acba29a4124988fa1ca806abc57934840823059ad8c1ac13a439c40db9f1beea798b8
|
7
|
+
data.tar.gz: ba44f693d010a4967243cf9096019627fadf25d4728c016e9d5727196accb5fd70735225db2b35cee9897aa5440d414603921ead9d07400db4335fb909af74b2
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 0.1.6 - 2015-XX-XX
|
2
|
+
|
3
|
+
* [maintenance] Upgrade Embulk v0.7.10 [#22](https://github.com/treasure-data/embulk-output-td/pull/22)
|
4
|
+
* [maintenance] Upgrade Embulk v0.7.5 [#21](https://github.com/treasure-data/embulk-output-td/pull/21)
|
5
|
+
* [new feature] Add default_timestamp_type_convert_to option [#18](https://github.com/treasure-data/embulk-output-td/pull/18)
|
6
|
+
|
1
7
|
## 0.1.5 - 2015-09-29
|
2
8
|
|
3
9
|
* [new feature] Add replace mode [#17](https://github.com/treasure-data/embulk-output-td/pull/17)
|
data/README.md
CHANGED
@@ -15,7 +15,7 @@ TODO: Write short description here
|
|
15
15
|
- **http_proxy**: http proxy configuration (tuple of host, port and useSsl, default is null)
|
16
16
|
- **use_ssl**: the flag (boolean, default=true)
|
17
17
|
- **auto_create_table**: the flag for creating the database and/or the table if they don't exist (boolean, default=true)
|
18
|
-
- **mode**: two ways to modify data
|
18
|
+
- **mode**: two ways 'append' and 'replace' to modify data (string, default='append')
|
19
19
|
- **database**: database name (string, required)
|
20
20
|
- **table**: table name (string, required)
|
21
21
|
- **session**: bulk_import session name (string, optional)
|
@@ -24,6 +24,7 @@ TODO: Write short description here
|
|
24
24
|
- **tmpdir**: temporal directory
|
25
25
|
- **upload_concurrency**: upload concurrency (int, default=2). max concurrency is 8.
|
26
26
|
- **file_split_size**: split size (long, default=16384 (16MB)).
|
27
|
+
- **default_timestamp_type_convert_to**: configure output type of timestamp columns. Available options are "sec" (convert timestamp to UNIX timestamp in seconds) and "string" (convert timestamp to string). (string, default: `"string"`)
|
27
28
|
- **default_timezone**: default timezone (string, default='UTC')
|
28
29
|
- **default_timestamp_format**: default timestamp format (string, default=`%Y-%m-%d %H:%M:%S.%6N`)
|
29
30
|
- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
|
data/build.gradle
CHANGED
@@ -16,23 +16,23 @@ configurations {
|
|
16
16
|
provided
|
17
17
|
}
|
18
18
|
|
19
|
-
version = "0.1.
|
19
|
+
version = "0.1.6"
|
20
20
|
|
21
21
|
compileJava.options.encoding = 'UTF-8' // source encoding
|
22
22
|
sourceCompatibility = 1.7
|
23
23
|
targetCompatibility = 1.7
|
24
24
|
|
25
25
|
dependencies {
|
26
|
-
compile "org.embulk:embulk-core:0.
|
27
|
-
provided "org.embulk:embulk-core:0.
|
28
|
-
compile "org.embulk:embulk-standards:0.
|
29
|
-
provided "org.embulk:embulk-standards:0.
|
26
|
+
compile "org.embulk:embulk-core:0.7.10"
|
27
|
+
provided "org.embulk:embulk-core:0.7.10"
|
28
|
+
compile "org.embulk:embulk-standards:0.7.10"
|
29
|
+
provided "org.embulk:embulk-standards:0.7.10"
|
30
30
|
compile "org.eclipse.jetty:jetty-client:9.2.2.v20140723"
|
31
31
|
compile "org.msgpack:msgpack:0.6.11"
|
32
32
|
|
33
33
|
testCompile "junit:junit:4.+"
|
34
34
|
testCompile "org.bigtesting:fixd:1.0.0"
|
35
|
-
testCompile "org.embulk:embulk-core:0.
|
35
|
+
testCompile "org.embulk:embulk-core:0.7.10:tests"
|
36
36
|
testCompile "org.mockito:mockito-core:1.9.5"
|
37
37
|
}
|
38
38
|
|
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.1.
|
4
|
+
spec.version = "0.1.6"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Tue Aug 11 00:26:20 PDT 2015
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
|
@@ -4,8 +4,9 @@ import com.google.common.annotations.VisibleForTesting;
|
|
4
4
|
import com.google.common.base.Stopwatch;
|
5
5
|
import com.google.common.base.Throwables;
|
6
6
|
import com.treasuredata.api.TdApiClient;
|
7
|
-
import org.embulk.config.
|
7
|
+
import org.embulk.config.TaskReport;
|
8
8
|
import org.embulk.output.td.writer.FieldWriter;
|
9
|
+
import org.embulk.output.td.writer.IFieldWriter;
|
9
10
|
import org.embulk.output.td.writer.FieldWriterSet;
|
10
11
|
import org.embulk.spi.Column;
|
11
12
|
import org.embulk.spi.ColumnVisitor;
|
@@ -130,7 +131,7 @@ public class RecordWriter
|
|
130
131
|
|
131
132
|
private void write(Column column)
|
132
133
|
{
|
133
|
-
|
134
|
+
IFieldWriter fieldWriter = fieldWriters.getFieldWriter(column.getIndex());
|
134
135
|
try {
|
135
136
|
fieldWriter.writeKeyValue(builder, pageReader, column);
|
136
137
|
}
|
@@ -246,9 +247,9 @@ public class RecordWriter
|
|
246
247
|
}
|
247
248
|
|
248
249
|
@Override
|
249
|
-
public
|
250
|
+
public TaskReport commit()
|
250
251
|
{
|
251
|
-
|
252
|
+
TaskReport report = Exec.newTaskReport();
|
252
253
|
// TODO
|
253
254
|
return report;
|
254
255
|
}
|
@@ -19,7 +19,7 @@ import com.treasuredata.api.TdApiNotFoundException;
|
|
19
19
|
import com.treasuredata.api.model.TDBulkImportSession;
|
20
20
|
import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
|
21
21
|
import com.treasuredata.api.model.TDTable;
|
22
|
-
import org.embulk.config.
|
22
|
+
import org.embulk.config.TaskReport;
|
23
23
|
import org.embulk.config.Config;
|
24
24
|
import org.embulk.config.ConfigDefault;
|
25
25
|
import org.embulk.config.ConfigDiff;
|
@@ -82,6 +82,10 @@ public class TdOutputPlugin
|
|
82
82
|
@ConfigDefault("null")
|
83
83
|
public Optional<String> getSession();
|
84
84
|
|
85
|
+
@Config("default_timestamp_type_convert_to")
|
86
|
+
@ConfigDefault("\"string\"")
|
87
|
+
public ConvertTimestampType getConvertTimestampType();
|
88
|
+
|
85
89
|
@Config("time_column")
|
86
90
|
@ConfigDefault("null")
|
87
91
|
public Optional<String> getTimeColumn();
|
@@ -180,6 +184,51 @@ public class TdOutputPlugin
|
|
180
184
|
public boolean getUseSsl();
|
181
185
|
}
|
182
186
|
|
187
|
+
public static enum ConvertTimestampType
|
188
|
+
{
|
189
|
+
STRING(-1),
|
190
|
+
//SEC_DOUBLE(-1), // TODO
|
191
|
+
SEC(1);
|
192
|
+
//MILLI(1000), // TODO
|
193
|
+
//MICRO(1000000), // TODO
|
194
|
+
//NANO(1000000000); // TODO
|
195
|
+
|
196
|
+
private final int unit;
|
197
|
+
|
198
|
+
private ConvertTimestampType(int unit)
|
199
|
+
{
|
200
|
+
this.unit = unit;
|
201
|
+
}
|
202
|
+
|
203
|
+
public int getFractionUnit()
|
204
|
+
{
|
205
|
+
return unit;
|
206
|
+
}
|
207
|
+
|
208
|
+
@JsonCreator
|
209
|
+
public static ConvertTimestampType of(String s)
|
210
|
+
{
|
211
|
+
switch (s) {
|
212
|
+
case "string": return STRING;
|
213
|
+
//case "sec_double": return SEC_DOUBLE;
|
214
|
+
case "sec": return SEC;
|
215
|
+
//case "milli": return MILLI;
|
216
|
+
//case "micro": return MICRO;
|
217
|
+
//case "nano": return NANO;
|
218
|
+
default:
|
219
|
+
throw new ConfigException(
|
220
|
+
String.format("Unknown convert_timestamp_type '%s'. Supported units are string, sec, milli, micro, nano, and sec_double", s));
|
221
|
+
}
|
222
|
+
}
|
223
|
+
|
224
|
+
@JsonValue
|
225
|
+
@Override
|
226
|
+
public String toString()
|
227
|
+
{
|
228
|
+
return name().toLowerCase();
|
229
|
+
}
|
230
|
+
}
|
231
|
+
|
183
232
|
public static enum UnixTimestampUnit
|
184
233
|
{
|
185
234
|
SEC(1),
|
@@ -304,7 +353,7 @@ public class TdOutputPlugin
|
|
304
353
|
|
305
354
|
public void cleanup(TaskSource taskSource,
|
306
355
|
Schema schema, int processorCount,
|
307
|
-
List<
|
356
|
+
List<TaskReport> successTaskReports)
|
308
357
|
{
|
309
358
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
310
359
|
try (TdApiClient client = newTdApiClient(task)) {
|
@@ -25,17 +25,18 @@ public class FieldWriterSet
|
|
25
25
|
}
|
26
26
|
|
27
27
|
private final int fieldCount;
|
28
|
-
private final
|
28
|
+
private final IFieldWriter[] fieldWriters;
|
29
29
|
|
30
30
|
public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
|
31
31
|
{
|
32
32
|
Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
|
33
|
+
TdOutputPlugin.ConvertTimestampType convertTimestamp = task.getConvertTimestampType();
|
33
34
|
boolean hasPkWriter = false;
|
34
35
|
int duplicatePrimaryKeySourceIndex = -1;
|
35
36
|
int firstTimestampColumnIndex = -1;
|
36
37
|
|
37
38
|
int fc = 0;
|
38
|
-
fieldWriters = new
|
39
|
+
fieldWriters = new IFieldWriter[schema.size()];
|
39
40
|
TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
|
40
41
|
|
41
42
|
for (int i = 0; i < schema.size(); i++) {
|
@@ -109,7 +110,17 @@ public class FieldWriterSet
|
|
109
110
|
writer = new StringFieldWriter(columnName);
|
110
111
|
}
|
111
112
|
else if (columnType instanceof TimestampType) {
|
112
|
-
|
113
|
+
switch (convertTimestamp) {
|
114
|
+
case STRING:
|
115
|
+
writer = new TimestampStringFieldWriter(timestampFormatters[i], columnName);
|
116
|
+
break;
|
117
|
+
case SEC:
|
118
|
+
writer = new TimestampLongFieldWriter(columnName);
|
119
|
+
break;
|
120
|
+
default:
|
121
|
+
// Thread of control doesn't come here but, just in case, it throws ConfigException.
|
122
|
+
throw new ConfigException(String.format("Unknown option {} as convert_timestamp_type", convertTimestamp));
|
123
|
+
}
|
113
124
|
if (firstTimestampColumnIndex < 0) {
|
114
125
|
firstTimestampColumnIndex = i;
|
115
126
|
}
|
@@ -150,16 +161,29 @@ public class FieldWriterSet
|
|
150
161
|
String columnName = schema.getColumnName(duplicatePrimaryKeySourceIndex);
|
151
162
|
Type columnType = schema.getColumnType(duplicatePrimaryKeySourceIndex);
|
152
163
|
|
153
|
-
|
164
|
+
IFieldWriter writer;
|
154
165
|
if (columnType instanceof LongType) {
|
155
166
|
log.info("Duplicating {}:{} column (unix timestamp {}) to 'time' column as seconds for the data partitioning",
|
156
167
|
columnName, columnType, task.getUnixTimestampUnit());
|
157
|
-
|
168
|
+
IFieldWriter fw = new LongFieldWriter(columnName);
|
169
|
+
writer = new UnixTimestampFieldDuplicator(fw, "time", task.getUnixTimestampUnit().getFractionUnit());
|
158
170
|
}
|
159
171
|
else if (columnType instanceof TimestampType) {
|
160
172
|
log.info("Duplicating {}:{} column to 'time' column as seconds for the data partitioning",
|
161
173
|
columnName, columnType);
|
162
|
-
|
174
|
+
IFieldWriter fw;
|
175
|
+
switch (convertTimestamp) {
|
176
|
+
case STRING:
|
177
|
+
fw = new TimestampStringFieldWriter(timestampFormatters[duplicatePrimaryKeySourceIndex], columnName);
|
178
|
+
break;
|
179
|
+
case SEC:
|
180
|
+
fw = new TimestampLongFieldWriter(columnName);
|
181
|
+
break;
|
182
|
+
default:
|
183
|
+
// Thread of control doesn't come here but, just in case, it throws ConfigException.
|
184
|
+
throw new ConfigException(String.format("Unknown option {} as convert_timestamp_type", convertTimestamp));
|
185
|
+
}
|
186
|
+
writer = new TimestampFieldLongDuplicator(fw, "time");
|
163
187
|
}
|
164
188
|
else {
|
165
189
|
throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
|
@@ -194,7 +218,7 @@ public class FieldWriterSet
|
|
194
218
|
return false;
|
195
219
|
}
|
196
220
|
|
197
|
-
public
|
221
|
+
public IFieldWriter getFieldWriter(int index)
|
198
222
|
{
|
199
223
|
return fieldWriters[index];
|
200
224
|
}
|
@@ -0,0 +1,13 @@
|
|
1
|
+
package org.embulk.output.td.writer;
|
2
|
+
|
3
|
+
import org.embulk.output.td.MsgpackGZFileBuilder;
|
4
|
+
import org.embulk.spi.Column;
|
5
|
+
import org.embulk.spi.PageReader;
|
6
|
+
|
7
|
+
import java.io.IOException;
|
8
|
+
|
9
|
+
public interface IFieldWriter
|
10
|
+
{
|
11
|
+
void writeKeyValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
12
|
+
throws IOException;
|
13
|
+
}
|
@@ -8,21 +8,21 @@ import org.embulk.spi.time.TimestampFormatter;
|
|
8
8
|
import java.io.IOException;
|
9
9
|
|
10
10
|
public class TimestampFieldLongDuplicator
|
11
|
-
|
11
|
+
implements IFieldWriter
|
12
12
|
{
|
13
|
+
private final IFieldWriter nextWriter;
|
13
14
|
private final TimestampLongFieldWriter timeFieldWriter;
|
14
15
|
|
15
|
-
public TimestampFieldLongDuplicator(
|
16
|
+
public TimestampFieldLongDuplicator(IFieldWriter nextWriter, String duplicateKeyName)
|
16
17
|
{
|
17
|
-
|
18
|
-
timeFieldWriter = new TimestampLongFieldWriter(
|
18
|
+
this.nextWriter = nextWriter;
|
19
|
+
timeFieldWriter = new TimestampLongFieldWriter(duplicateKeyName);
|
19
20
|
}
|
20
21
|
|
21
|
-
|
22
|
-
public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
22
|
+
public void writeKeyValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
23
23
|
throws IOException
|
24
24
|
{
|
25
|
-
|
25
|
+
nextWriter.writeKeyValue(builder, reader, column);
|
26
26
|
timeFieldWriter.writeKeyValue(builder, reader, column);
|
27
27
|
}
|
28
28
|
}
|
@@ -7,21 +7,21 @@ import org.embulk.spi.PageReader;
|
|
7
7
|
import java.io.IOException;
|
8
8
|
|
9
9
|
public class UnixTimestampFieldDuplicator
|
10
|
-
|
10
|
+
implements IFieldWriter
|
11
11
|
{
|
12
|
+
private final IFieldWriter nextWriter;
|
12
13
|
private final UnixTimestampLongFieldWriter timeFieldWriter;
|
13
14
|
|
14
|
-
public UnixTimestampFieldDuplicator(
|
15
|
+
public UnixTimestampFieldDuplicator(IFieldWriter nextWriter, String duplicateKeyName, int fractionUnit)
|
15
16
|
{
|
16
|
-
|
17
|
+
this.nextWriter = nextWriter;
|
17
18
|
timeFieldWriter = new UnixTimestampLongFieldWriter(duplicateKeyName, fractionUnit);
|
18
19
|
}
|
19
20
|
|
20
|
-
|
21
|
-
public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
21
|
+
public void writeKeyValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
22
22
|
throws IOException
|
23
23
|
{
|
24
|
-
|
24
|
+
nextWriter.writeKeyValue(builder, reader, column);
|
25
25
|
timeFieldWriter.writeKeyValue(builder, reader, column);
|
26
26
|
}
|
27
27
|
}
|
@@ -11,7 +11,7 @@ import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
|
|
11
11
|
import com.treasuredata.api.model.TDTable;
|
12
12
|
import com.treasuredata.api.model.TDTableType;
|
13
13
|
import org.embulk.EmbulkTestRuntime;
|
14
|
-
import org.embulk.config.
|
14
|
+
import org.embulk.config.TaskReport;
|
15
15
|
import org.embulk.config.ConfigDiff;
|
16
16
|
import org.embulk.config.ConfigException;
|
17
17
|
import org.embulk.config.ConfigSource;
|
@@ -121,9 +121,9 @@ public class TestTdOutputPlugin
|
|
121
121
|
assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
122
122
|
{
|
123
123
|
@Override
|
124
|
-
public List<
|
124
|
+
public List<TaskReport> run(TaskSource taskSource)
|
125
125
|
{
|
126
|
-
return Lists.newArrayList(Exec.
|
126
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
127
127
|
}
|
128
128
|
}).get(String.class, "last_session"));
|
129
129
|
}
|
@@ -134,9 +134,9 @@ public class TestTdOutputPlugin
|
|
134
134
|
assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
135
135
|
{
|
136
136
|
@Override
|
137
|
-
public List<
|
137
|
+
public List<TaskReport> run(TaskSource taskSource)
|
138
138
|
{
|
139
|
-
return Lists.newArrayList(Exec.
|
139
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
140
140
|
}
|
141
141
|
}).get(String.class, "last_session"));
|
142
142
|
}
|
@@ -157,9 +157,9 @@ public class TestTdOutputPlugin
|
|
157
157
|
ConfigDiff configDiff = plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
|
158
158
|
{
|
159
159
|
@Override
|
160
|
-
public List<
|
160
|
+
public List<TaskReport> run(TaskSource taskSource)
|
161
161
|
{
|
162
|
-
return Lists.newArrayList(Exec.
|
162
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
163
163
|
}
|
164
164
|
});
|
165
165
|
|
@@ -178,7 +178,7 @@ public class TestTdOutputPlugin
|
|
178
178
|
doReturn(client).when(plugin).newTdApiClient(task);
|
179
179
|
Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
|
180
180
|
|
181
|
-
plugin.cleanup(task.dump(), schema, 0, Lists.newArrayList(Exec.
|
181
|
+
plugin.cleanup(task.dump(), schema, 0, Lists.newArrayList(Exec.newTaskReport()));
|
182
182
|
// no error happens
|
183
183
|
}
|
184
184
|
|
@@ -134,6 +134,58 @@ public class TestFieldWriterSet
|
|
134
134
|
}
|
135
135
|
}
|
136
136
|
|
137
|
+
@Test
|
138
|
+
public void useDefaultTimestampTypeConvertTo()
|
139
|
+
{
|
140
|
+
{ // if not specify default_timestamp_type_convert_to, use string by default
|
141
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
142
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy()), schema);
|
143
|
+
|
144
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
145
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
146
|
+
}
|
147
|
+
|
148
|
+
{ // and use time_column option
|
149
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
150
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
151
|
+
|
152
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
153
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
154
|
+
}
|
155
|
+
|
156
|
+
{ // if default_timestamp_type_convert_to is string, use string
|
157
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
158
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string")), schema);
|
159
|
+
|
160
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
161
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
162
|
+
}
|
163
|
+
|
164
|
+
{ // and use time_column option
|
165
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
166
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string").set("time_column", "_c0")), schema);
|
167
|
+
|
168
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
169
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
170
|
+
}
|
171
|
+
|
172
|
+
{ // if default_timestamp_type_conver_to is sec, use long
|
173
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
174
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec")), schema);
|
175
|
+
|
176
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter); // c0
|
177
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
178
|
+
}
|
179
|
+
|
180
|
+
{ // and use time_column option
|
181
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
182
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec").set("time_column", "_c0")), schema);
|
183
|
+
|
184
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
185
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time renamed
|
186
|
+
}
|
187
|
+
}
|
188
|
+
|
137
189
|
@Test
|
138
190
|
public void useFirstTimestampColumn()
|
139
191
|
throws Exception
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -91,6 +91,7 @@ files:
|
|
91
91
|
- src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java
|
92
92
|
- src/main/java/org/embulk/output/td/writer/FieldWriter.java
|
93
93
|
- src/main/java/org/embulk/output/td/writer/FieldWriterSet.java
|
94
|
+
- src/main/java/org/embulk/output/td/writer/IFieldWriter.java
|
94
95
|
- src/main/java/org/embulk/output/td/writer/LongFieldWriter.java
|
95
96
|
- src/main/java/org/embulk/output/td/writer/StringFieldWriter.java
|
96
97
|
- src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java
|
@@ -102,7 +103,7 @@ files:
|
|
102
103
|
- src/test/java/org/embulk/output/td/TestRecordWriter.java
|
103
104
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
104
105
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
105
|
-
- classpath/embulk-output-td-0.1.
|
106
|
+
- classpath/embulk-output-td-0.1.6.jar
|
106
107
|
- classpath/javassist-3.18.1-GA.jar
|
107
108
|
- classpath/jetty-client-9.2.2.v20140723.jar
|
108
109
|
- classpath/jetty-http-9.2.2.v20140723.jar
|