embulk-output-td 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +2 -1
- data/build.gradle +6 -6
- data/embulk-output-td.gemspec +1 -1
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/td/RecordWriter.java +5 -4
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +51 -2
- data/src/main/java/org/embulk/output/td/writer/FieldWriter.java +1 -0
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +31 -7
- data/src/main/java/org/embulk/output/td/writer/IFieldWriter.java +13 -0
- data/src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java +7 -7
- data/src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java +6 -6
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +1 -1
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +8 -8
- data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +52 -0
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7b42edcec64619a1d24ce6b36967527ab76b4a63
|
|
4
|
+
data.tar.gz: 402e46fa95353e30cc38668c007893612cc2323c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 013f4c79c83f0d8215fb9d92098dbc75bb18f41b911117b82ae9773b010acba29a4124988fa1ca806abc57934840823059ad8c1ac13a439c40db9f1beea798b8
|
|
7
|
+
data.tar.gz: ba44f693d010a4967243cf9096019627fadf25d4728c016e9d5727196accb5fd70735225db2b35cee9897aa5440d414603921ead9d07400db4335fb909af74b2
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## 0.1.6 - 2015-XX-XX
|
|
2
|
+
|
|
3
|
+
* [maintenance] Upgrade Embulk v0.7.10 [#22](https://github.com/treasure-data/embulk-output-td/pull/22)
|
|
4
|
+
* [maintenance] Upgrade Embulk v0.7.5 [#21](https://github.com/treasure-data/embulk-output-td/pull/21)
|
|
5
|
+
* [new feature] Add default_timestamp_type_convert_to option [#18](https://github.com/treasure-data/embulk-output-td/pull/18)
|
|
6
|
+
|
|
1
7
|
## 0.1.5 - 2015-09-29
|
|
2
8
|
|
|
3
9
|
* [new feature] Add replace mode [#17](https://github.com/treasure-data/embulk-output-td/pull/17)
|
data/README.md
CHANGED
|
@@ -15,7 +15,7 @@ TODO: Write short description here
|
|
|
15
15
|
- **http_proxy**: http proxy configuration (tuple of host, port and useSsl, default is null)
|
|
16
16
|
- **use_ssl**: the flag (boolean, default=true)
|
|
17
17
|
- **auto_create_table**: the flag for creating the database and/or the table if they don't exist (boolean, default=true)
|
|
18
|
-
- **mode**: two ways to modify data
|
|
18
|
+
- **mode**: two ways 'append' and 'replace' to modify data (string, default='append')
|
|
19
19
|
- **database**: database name (string, required)
|
|
20
20
|
- **table**: table name (string, required)
|
|
21
21
|
- **session**: bulk_import session name (string, optional)
|
|
@@ -24,6 +24,7 @@ TODO: Write short description here
|
|
|
24
24
|
- **tmpdir**: temporal directory
|
|
25
25
|
- **upload_concurrency**: upload concurrency (int, default=2). max concurrency is 8.
|
|
26
26
|
- **file_split_size**: split size (long, default=16384 (16MB)).
|
|
27
|
+
- **default_timestamp_type_convert_to**: configure output type of timestamp columns. Available options are "sec" (convert timestamp to UNIX timestamp in seconds) and "string" (convert timestamp to string). (string, default: `"string"`)
|
|
27
28
|
- **default_timezone**: default timezone (string, default='UTC')
|
|
28
29
|
- **default_timestamp_format**: default timestamp format (string, default=`%Y-%m-%d %H:%M:%S.%6N`)
|
|
29
30
|
- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
|
data/build.gradle
CHANGED
|
@@ -16,23 +16,23 @@ configurations {
|
|
|
16
16
|
provided
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
version = "0.1.
|
|
19
|
+
version = "0.1.6"
|
|
20
20
|
|
|
21
21
|
compileJava.options.encoding = 'UTF-8' // source encoding
|
|
22
22
|
sourceCompatibility = 1.7
|
|
23
23
|
targetCompatibility = 1.7
|
|
24
24
|
|
|
25
25
|
dependencies {
|
|
26
|
-
compile "org.embulk:embulk-core:0.
|
|
27
|
-
provided "org.embulk:embulk-core:0.
|
|
28
|
-
compile "org.embulk:embulk-standards:0.
|
|
29
|
-
provided "org.embulk:embulk-standards:0.
|
|
26
|
+
compile "org.embulk:embulk-core:0.7.10"
|
|
27
|
+
provided "org.embulk:embulk-core:0.7.10"
|
|
28
|
+
compile "org.embulk:embulk-standards:0.7.10"
|
|
29
|
+
provided "org.embulk:embulk-standards:0.7.10"
|
|
30
30
|
compile "org.eclipse.jetty:jetty-client:9.2.2.v20140723"
|
|
31
31
|
compile "org.msgpack:msgpack:0.6.11"
|
|
32
32
|
|
|
33
33
|
testCompile "junit:junit:4.+"
|
|
34
34
|
testCompile "org.bigtesting:fixd:1.0.0"
|
|
35
|
-
testCompile "org.embulk:embulk-core:0.
|
|
35
|
+
testCompile "org.embulk:embulk-core:0.7.10:tests"
|
|
36
36
|
testCompile "org.mockito:mockito-core:1.9.5"
|
|
37
37
|
}
|
|
38
38
|
|
data/embulk-output-td.gemspec
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
Gem::Specification.new do |spec|
|
|
3
3
|
spec.name = "embulk-output-td"
|
|
4
|
-
spec.version = "0.1.
|
|
4
|
+
spec.version = "0.1.6"
|
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Tue Aug 11 00:26:20 PDT 2015
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
|
|
@@ -4,8 +4,9 @@ import com.google.common.annotations.VisibleForTesting;
|
|
|
4
4
|
import com.google.common.base.Stopwatch;
|
|
5
5
|
import com.google.common.base.Throwables;
|
|
6
6
|
import com.treasuredata.api.TdApiClient;
|
|
7
|
-
import org.embulk.config.
|
|
7
|
+
import org.embulk.config.TaskReport;
|
|
8
8
|
import org.embulk.output.td.writer.FieldWriter;
|
|
9
|
+
import org.embulk.output.td.writer.IFieldWriter;
|
|
9
10
|
import org.embulk.output.td.writer.FieldWriterSet;
|
|
10
11
|
import org.embulk.spi.Column;
|
|
11
12
|
import org.embulk.spi.ColumnVisitor;
|
|
@@ -130,7 +131,7 @@ public class RecordWriter
|
|
|
130
131
|
|
|
131
132
|
private void write(Column column)
|
|
132
133
|
{
|
|
133
|
-
|
|
134
|
+
IFieldWriter fieldWriter = fieldWriters.getFieldWriter(column.getIndex());
|
|
134
135
|
try {
|
|
135
136
|
fieldWriter.writeKeyValue(builder, pageReader, column);
|
|
136
137
|
}
|
|
@@ -246,9 +247,9 @@ public class RecordWriter
|
|
|
246
247
|
}
|
|
247
248
|
|
|
248
249
|
@Override
|
|
249
|
-
public
|
|
250
|
+
public TaskReport commit()
|
|
250
251
|
{
|
|
251
|
-
|
|
252
|
+
TaskReport report = Exec.newTaskReport();
|
|
252
253
|
// TODO
|
|
253
254
|
return report;
|
|
254
255
|
}
|
|
@@ -19,7 +19,7 @@ import com.treasuredata.api.TdApiNotFoundException;
|
|
|
19
19
|
import com.treasuredata.api.model.TDBulkImportSession;
|
|
20
20
|
import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
|
|
21
21
|
import com.treasuredata.api.model.TDTable;
|
|
22
|
-
import org.embulk.config.
|
|
22
|
+
import org.embulk.config.TaskReport;
|
|
23
23
|
import org.embulk.config.Config;
|
|
24
24
|
import org.embulk.config.ConfigDefault;
|
|
25
25
|
import org.embulk.config.ConfigDiff;
|
|
@@ -82,6 +82,10 @@ public class TdOutputPlugin
|
|
|
82
82
|
@ConfigDefault("null")
|
|
83
83
|
public Optional<String> getSession();
|
|
84
84
|
|
|
85
|
+
@Config("default_timestamp_type_convert_to")
|
|
86
|
+
@ConfigDefault("\"string\"")
|
|
87
|
+
public ConvertTimestampType getConvertTimestampType();
|
|
88
|
+
|
|
85
89
|
@Config("time_column")
|
|
86
90
|
@ConfigDefault("null")
|
|
87
91
|
public Optional<String> getTimeColumn();
|
|
@@ -180,6 +184,51 @@ public class TdOutputPlugin
|
|
|
180
184
|
public boolean getUseSsl();
|
|
181
185
|
}
|
|
182
186
|
|
|
187
|
+
public static enum ConvertTimestampType
|
|
188
|
+
{
|
|
189
|
+
STRING(-1),
|
|
190
|
+
//SEC_DOUBLE(-1), // TODO
|
|
191
|
+
SEC(1);
|
|
192
|
+
//MILLI(1000), // TODO
|
|
193
|
+
//MICRO(1000000), // TODO
|
|
194
|
+
//NANO(1000000000); // TODO
|
|
195
|
+
|
|
196
|
+
private final int unit;
|
|
197
|
+
|
|
198
|
+
private ConvertTimestampType(int unit)
|
|
199
|
+
{
|
|
200
|
+
this.unit = unit;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
public int getFractionUnit()
|
|
204
|
+
{
|
|
205
|
+
return unit;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
@JsonCreator
|
|
209
|
+
public static ConvertTimestampType of(String s)
|
|
210
|
+
{
|
|
211
|
+
switch (s) {
|
|
212
|
+
case "string": return STRING;
|
|
213
|
+
//case "sec_double": return SEC_DOUBLE;
|
|
214
|
+
case "sec": return SEC;
|
|
215
|
+
//case "milli": return MILLI;
|
|
216
|
+
//case "micro": return MICRO;
|
|
217
|
+
//case "nano": return NANO;
|
|
218
|
+
default:
|
|
219
|
+
throw new ConfigException(
|
|
220
|
+
String.format("Unknown convert_timestamp_type '%s'. Supported units are string, sec, milli, micro, nano, and sec_double", s));
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
@JsonValue
|
|
225
|
+
@Override
|
|
226
|
+
public String toString()
|
|
227
|
+
{
|
|
228
|
+
return name().toLowerCase();
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
183
232
|
public static enum UnixTimestampUnit
|
|
184
233
|
{
|
|
185
234
|
SEC(1),
|
|
@@ -304,7 +353,7 @@ public class TdOutputPlugin
|
|
|
304
353
|
|
|
305
354
|
public void cleanup(TaskSource taskSource,
|
|
306
355
|
Schema schema, int processorCount,
|
|
307
|
-
List<
|
|
356
|
+
List<TaskReport> successTaskReports)
|
|
308
357
|
{
|
|
309
358
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
310
359
|
try (TdApiClient client = newTdApiClient(task)) {
|
|
@@ -25,17 +25,18 @@ public class FieldWriterSet
|
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
private final int fieldCount;
|
|
28
|
-
private final
|
|
28
|
+
private final IFieldWriter[] fieldWriters;
|
|
29
29
|
|
|
30
30
|
public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
|
|
31
31
|
{
|
|
32
32
|
Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
|
|
33
|
+
TdOutputPlugin.ConvertTimestampType convertTimestamp = task.getConvertTimestampType();
|
|
33
34
|
boolean hasPkWriter = false;
|
|
34
35
|
int duplicatePrimaryKeySourceIndex = -1;
|
|
35
36
|
int firstTimestampColumnIndex = -1;
|
|
36
37
|
|
|
37
38
|
int fc = 0;
|
|
38
|
-
fieldWriters = new
|
|
39
|
+
fieldWriters = new IFieldWriter[schema.size()];
|
|
39
40
|
TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
|
|
40
41
|
|
|
41
42
|
for (int i = 0; i < schema.size(); i++) {
|
|
@@ -109,7 +110,17 @@ public class FieldWriterSet
|
|
|
109
110
|
writer = new StringFieldWriter(columnName);
|
|
110
111
|
}
|
|
111
112
|
else if (columnType instanceof TimestampType) {
|
|
112
|
-
|
|
113
|
+
switch (convertTimestamp) {
|
|
114
|
+
case STRING:
|
|
115
|
+
writer = new TimestampStringFieldWriter(timestampFormatters[i], columnName);
|
|
116
|
+
break;
|
|
117
|
+
case SEC:
|
|
118
|
+
writer = new TimestampLongFieldWriter(columnName);
|
|
119
|
+
break;
|
|
120
|
+
default:
|
|
121
|
+
// Thread of control doesn't come here but, just in case, it throws ConfigException.
|
|
122
|
+
throw new ConfigException(String.format("Unknown option {} as convert_timestamp_type", convertTimestamp));
|
|
123
|
+
}
|
|
113
124
|
if (firstTimestampColumnIndex < 0) {
|
|
114
125
|
firstTimestampColumnIndex = i;
|
|
115
126
|
}
|
|
@@ -150,16 +161,29 @@ public class FieldWriterSet
|
|
|
150
161
|
String columnName = schema.getColumnName(duplicatePrimaryKeySourceIndex);
|
|
151
162
|
Type columnType = schema.getColumnType(duplicatePrimaryKeySourceIndex);
|
|
152
163
|
|
|
153
|
-
|
|
164
|
+
IFieldWriter writer;
|
|
154
165
|
if (columnType instanceof LongType) {
|
|
155
166
|
log.info("Duplicating {}:{} column (unix timestamp {}) to 'time' column as seconds for the data partitioning",
|
|
156
167
|
columnName, columnType, task.getUnixTimestampUnit());
|
|
157
|
-
|
|
168
|
+
IFieldWriter fw = new LongFieldWriter(columnName);
|
|
169
|
+
writer = new UnixTimestampFieldDuplicator(fw, "time", task.getUnixTimestampUnit().getFractionUnit());
|
|
158
170
|
}
|
|
159
171
|
else if (columnType instanceof TimestampType) {
|
|
160
172
|
log.info("Duplicating {}:{} column to 'time' column as seconds for the data partitioning",
|
|
161
173
|
columnName, columnType);
|
|
162
|
-
|
|
174
|
+
IFieldWriter fw;
|
|
175
|
+
switch (convertTimestamp) {
|
|
176
|
+
case STRING:
|
|
177
|
+
fw = new TimestampStringFieldWriter(timestampFormatters[duplicatePrimaryKeySourceIndex], columnName);
|
|
178
|
+
break;
|
|
179
|
+
case SEC:
|
|
180
|
+
fw = new TimestampLongFieldWriter(columnName);
|
|
181
|
+
break;
|
|
182
|
+
default:
|
|
183
|
+
// Thread of control doesn't come here but, just in case, it throws ConfigException.
|
|
184
|
+
throw new ConfigException(String.format("Unknown option {} as convert_timestamp_type", convertTimestamp));
|
|
185
|
+
}
|
|
186
|
+
writer = new TimestampFieldLongDuplicator(fw, "time");
|
|
163
187
|
}
|
|
164
188
|
else {
|
|
165
189
|
throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
|
|
@@ -194,7 +218,7 @@ public class FieldWriterSet
|
|
|
194
218
|
return false;
|
|
195
219
|
}
|
|
196
220
|
|
|
197
|
-
public
|
|
221
|
+
public IFieldWriter getFieldWriter(int index)
|
|
198
222
|
{
|
|
199
223
|
return fieldWriters[index];
|
|
200
224
|
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
package org.embulk.output.td.writer;
|
|
2
|
+
|
|
3
|
+
import org.embulk.output.td.MsgpackGZFileBuilder;
|
|
4
|
+
import org.embulk.spi.Column;
|
|
5
|
+
import org.embulk.spi.PageReader;
|
|
6
|
+
|
|
7
|
+
import java.io.IOException;
|
|
8
|
+
|
|
9
|
+
public interface IFieldWriter
|
|
10
|
+
{
|
|
11
|
+
void writeKeyValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
|
12
|
+
throws IOException;
|
|
13
|
+
}
|
|
@@ -8,21 +8,21 @@ import org.embulk.spi.time.TimestampFormatter;
|
|
|
8
8
|
import java.io.IOException;
|
|
9
9
|
|
|
10
10
|
public class TimestampFieldLongDuplicator
|
|
11
|
-
|
|
11
|
+
implements IFieldWriter
|
|
12
12
|
{
|
|
13
|
+
private final IFieldWriter nextWriter;
|
|
13
14
|
private final TimestampLongFieldWriter timeFieldWriter;
|
|
14
15
|
|
|
15
|
-
public TimestampFieldLongDuplicator(
|
|
16
|
+
public TimestampFieldLongDuplicator(IFieldWriter nextWriter, String duplicateKeyName)
|
|
16
17
|
{
|
|
17
|
-
|
|
18
|
-
timeFieldWriter = new TimestampLongFieldWriter(
|
|
18
|
+
this.nextWriter = nextWriter;
|
|
19
|
+
timeFieldWriter = new TimestampLongFieldWriter(duplicateKeyName);
|
|
19
20
|
}
|
|
20
21
|
|
|
21
|
-
|
|
22
|
-
public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
|
22
|
+
public void writeKeyValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
|
23
23
|
throws IOException
|
|
24
24
|
{
|
|
25
|
-
|
|
25
|
+
nextWriter.writeKeyValue(builder, reader, column);
|
|
26
26
|
timeFieldWriter.writeKeyValue(builder, reader, column);
|
|
27
27
|
}
|
|
28
28
|
}
|
|
@@ -7,21 +7,21 @@ import org.embulk.spi.PageReader;
|
|
|
7
7
|
import java.io.IOException;
|
|
8
8
|
|
|
9
9
|
public class UnixTimestampFieldDuplicator
|
|
10
|
-
|
|
10
|
+
implements IFieldWriter
|
|
11
11
|
{
|
|
12
|
+
private final IFieldWriter nextWriter;
|
|
12
13
|
private final UnixTimestampLongFieldWriter timeFieldWriter;
|
|
13
14
|
|
|
14
|
-
public UnixTimestampFieldDuplicator(
|
|
15
|
+
public UnixTimestampFieldDuplicator(IFieldWriter nextWriter, String duplicateKeyName, int fractionUnit)
|
|
15
16
|
{
|
|
16
|
-
|
|
17
|
+
this.nextWriter = nextWriter;
|
|
17
18
|
timeFieldWriter = new UnixTimestampLongFieldWriter(duplicateKeyName, fractionUnit);
|
|
18
19
|
}
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
|
21
|
+
public void writeKeyValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
|
22
22
|
throws IOException
|
|
23
23
|
{
|
|
24
|
-
|
|
24
|
+
nextWriter.writeKeyValue(builder, reader, column);
|
|
25
25
|
timeFieldWriter.writeKeyValue(builder, reader, column);
|
|
26
26
|
}
|
|
27
27
|
}
|
|
@@ -11,7 +11,7 @@ import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
|
|
|
11
11
|
import com.treasuredata.api.model.TDTable;
|
|
12
12
|
import com.treasuredata.api.model.TDTableType;
|
|
13
13
|
import org.embulk.EmbulkTestRuntime;
|
|
14
|
-
import org.embulk.config.
|
|
14
|
+
import org.embulk.config.TaskReport;
|
|
15
15
|
import org.embulk.config.ConfigDiff;
|
|
16
16
|
import org.embulk.config.ConfigException;
|
|
17
17
|
import org.embulk.config.ConfigSource;
|
|
@@ -121,9 +121,9 @@ public class TestTdOutputPlugin
|
|
|
121
121
|
assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
|
122
122
|
{
|
|
123
123
|
@Override
|
|
124
|
-
public List<
|
|
124
|
+
public List<TaskReport> run(TaskSource taskSource)
|
|
125
125
|
{
|
|
126
|
-
return Lists.newArrayList(Exec.
|
|
126
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
|
127
127
|
}
|
|
128
128
|
}).get(String.class, "last_session"));
|
|
129
129
|
}
|
|
@@ -134,9 +134,9 @@ public class TestTdOutputPlugin
|
|
|
134
134
|
assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
|
135
135
|
{
|
|
136
136
|
@Override
|
|
137
|
-
public List<
|
|
137
|
+
public List<TaskReport> run(TaskSource taskSource)
|
|
138
138
|
{
|
|
139
|
-
return Lists.newArrayList(Exec.
|
|
139
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
|
140
140
|
}
|
|
141
141
|
}).get(String.class, "last_session"));
|
|
142
142
|
}
|
|
@@ -157,9 +157,9 @@ public class TestTdOutputPlugin
|
|
|
157
157
|
ConfigDiff configDiff = plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
|
|
158
158
|
{
|
|
159
159
|
@Override
|
|
160
|
-
public List<
|
|
160
|
+
public List<TaskReport> run(TaskSource taskSource)
|
|
161
161
|
{
|
|
162
|
-
return Lists.newArrayList(Exec.
|
|
162
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
|
163
163
|
}
|
|
164
164
|
});
|
|
165
165
|
|
|
@@ -178,7 +178,7 @@ public class TestTdOutputPlugin
|
|
|
178
178
|
doReturn(client).when(plugin).newTdApiClient(task);
|
|
179
179
|
Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
|
|
180
180
|
|
|
181
|
-
plugin.cleanup(task.dump(), schema, 0, Lists.newArrayList(Exec.
|
|
181
|
+
plugin.cleanup(task.dump(), schema, 0, Lists.newArrayList(Exec.newTaskReport()));
|
|
182
182
|
// no error happens
|
|
183
183
|
}
|
|
184
184
|
|
|
@@ -134,6 +134,58 @@ public class TestFieldWriterSet
|
|
|
134
134
|
}
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
+
@Test
|
|
138
|
+
public void useDefaultTimestampTypeConvertTo()
|
|
139
|
+
{
|
|
140
|
+
{ // if not specify default_timestamp_type_convert_to, use string by default
|
|
141
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
|
142
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy()), schema);
|
|
143
|
+
|
|
144
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
|
145
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
{ // and use time_column option
|
|
149
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
|
150
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
|
151
|
+
|
|
152
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
|
153
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
{ // if default_timestamp_type_convert_to is string, use string
|
|
157
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
|
158
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string")), schema);
|
|
159
|
+
|
|
160
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampStringFieldWriter); // c0
|
|
161
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
{ // and use time_column option
|
|
165
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
|
166
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "string").set("time_column", "_c0")), schema);
|
|
167
|
+
|
|
168
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
|
169
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // time renamed
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
{ // if default_timestamp_type_conver_to is sec, use long
|
|
173
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
|
174
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec")), schema);
|
|
175
|
+
|
|
176
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter); // c0
|
|
177
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
{ // and use time_column option
|
|
181
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
|
182
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("default_timestamp_type_convert_to", "sec").set("time_column", "_c0")), schema);
|
|
183
|
+
|
|
184
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
|
185
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampLongFieldWriter); // time renamed
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
137
189
|
@Test
|
|
138
190
|
public void useFirstTimestampColumn()
|
|
139
191
|
throws Exception
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-td
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Muga Nishizawa
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-
|
|
11
|
+
date: 2015-12-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -91,6 +91,7 @@ files:
|
|
|
91
91
|
- src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java
|
|
92
92
|
- src/main/java/org/embulk/output/td/writer/FieldWriter.java
|
|
93
93
|
- src/main/java/org/embulk/output/td/writer/FieldWriterSet.java
|
|
94
|
+
- src/main/java/org/embulk/output/td/writer/IFieldWriter.java
|
|
94
95
|
- src/main/java/org/embulk/output/td/writer/LongFieldWriter.java
|
|
95
96
|
- src/main/java/org/embulk/output/td/writer/StringFieldWriter.java
|
|
96
97
|
- src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java
|
|
@@ -102,7 +103,7 @@ files:
|
|
|
102
103
|
- src/test/java/org/embulk/output/td/TestRecordWriter.java
|
|
103
104
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
|
104
105
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
|
105
|
-
- classpath/embulk-output-td-0.1.
|
|
106
|
+
- classpath/embulk-output-td-0.1.6.jar
|
|
106
107
|
- classpath/javassist-3.18.1-GA.jar
|
|
107
108
|
- classpath/jetty-client-9.2.2.v20140723.jar
|
|
108
109
|
- classpath/jetty-http-9.2.2.v20140723.jar
|