embulk-output-td 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/build.gradle +9 -8
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +11 -12
- data/src/main/java/org/embulk/output/td/RecordWriter.java +4 -7
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +89 -84
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +9 -0
- data/src/main/java/org/embulk/output/td/writer/JsonFieldWriter.java +23 -0
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +37 -38
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +53 -49
- metadata +9 -30
- data/src/main/java/com/treasuredata/api/TdApiClient.java +0 -506
- data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +0 -79
- data/src/main/java/com/treasuredata/api/TdApiConflictException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiConstants.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiException.java +0 -20
- data/src/main/java/com/treasuredata/api/TdApiExecutionException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +0 -16
- data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +0 -18
- data/src/main/java/com/treasuredata/api/TdApiNotFoundException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiResponseException.java +0 -32
- data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +0 -80
- data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +0 -157
- data/src/main/java/com/treasuredata/api/model/TDColumn.java +0 -129
- data/src/main/java/com/treasuredata/api/model/TDColumnType.java +0 -23
- data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +0 -128
- data/src/main/java/com/treasuredata/api/model/TDDatabase.java +0 -49
- data/src/main/java/com/treasuredata/api/model/TDDatabaseList.java +0 -24
- data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +0 -88
- data/src/main/java/com/treasuredata/api/model/TDPrimitiveColumnType.java +0 -61
- data/src/main/java/com/treasuredata/api/model/TDTable.java +0 -64
- data/src/main/java/com/treasuredata/api/model/TDTableList.java +0 -33
- data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +0 -50
- data/src/main/java/com/treasuredata/api/model/TDTableSchema.java +0 -44
- data/src/main/java/com/treasuredata/api/model/TDTableType.java +0 -37
- data/src/test/java/com/treasuredata/api/TestTdApiClient.java +0 -79
@@ -19,6 +19,7 @@ import org.embulk.spi.Schema;
|
|
19
19
|
import org.embulk.spi.time.TimestampFormatter;
|
20
20
|
import org.embulk.spi.type.BooleanType;
|
21
21
|
import org.embulk.spi.type.DoubleType;
|
22
|
+
import org.embulk.spi.type.JsonType;
|
22
23
|
import org.embulk.spi.type.LongType;
|
23
24
|
import org.embulk.spi.type.StringType;
|
24
25
|
import org.embulk.spi.type.TimestampType;
|
@@ -238,6 +239,9 @@ public class FieldWriterSet
|
|
238
239
|
else if (columnType instanceof TimestampType) {
|
239
240
|
return newSimpleTimestampFieldWriter(columnName, columnType, convertTimestampType, timestampFormatter);
|
240
241
|
}
|
242
|
+
else if (columnType instanceof JsonType) {
|
243
|
+
return new JsonFieldWriter(columnName);
|
244
|
+
}
|
241
245
|
else {
|
242
246
|
throw new ConfigException("Unsupported type: " + columnType);
|
243
247
|
}
|
@@ -300,6 +304,11 @@ public class FieldWriterSet
|
|
300
304
|
addColumn(builder, reader, column);
|
301
305
|
}
|
302
306
|
|
307
|
+
@Override
|
308
|
+
public void jsonColumn(Column column)
|
309
|
+
{
|
310
|
+
addColumn(builder, reader, column);
|
311
|
+
}
|
303
312
|
});
|
304
313
|
|
305
314
|
endRecord(builder);
|
@@ -0,0 +1,23 @@
|
|
1
|
+
package org.embulk.output.td.writer;
|
2
|
+
|
3
|
+
import org.embulk.output.td.MsgpackGZFileBuilder;
|
4
|
+
import org.embulk.spi.Column;
|
5
|
+
import org.embulk.spi.PageReader;
|
6
|
+
|
7
|
+
import java.io.IOException;
|
8
|
+
|
9
|
+
public class JsonFieldWriter
|
10
|
+
extends FieldWriter
|
11
|
+
{
|
12
|
+
public JsonFieldWriter(String keyName)
|
13
|
+
{
|
14
|
+
super(keyName);
|
15
|
+
}
|
16
|
+
|
17
|
+
@Override
|
18
|
+
public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
|
19
|
+
throws IOException
|
20
|
+
{
|
21
|
+
builder.writeString(reader.getJson(column).toJson());
|
22
|
+
}
|
23
|
+
}
|
@@ -1,7 +1,7 @@
|
|
1
1
|
package org.embulk.output.td;
|
2
2
|
|
3
3
|
import com.google.common.collect.ImmutableMap;
|
4
|
-
import com.treasuredata.
|
4
|
+
import com.treasuredata.client.TDClient;
|
5
5
|
import org.embulk.EmbulkTestRuntime;
|
6
6
|
import org.embulk.output.td.TdOutputPlugin.PluginTask;
|
7
7
|
import org.embulk.spi.Page;
|
@@ -12,13 +12,14 @@ import org.embulk.spi.type.Types;
|
|
12
12
|
import org.junit.Before;
|
13
13
|
import org.junit.Rule;
|
14
14
|
import org.junit.Test;
|
15
|
-
import org.msgpack.MessagePack;
|
16
|
-
import org.msgpack.
|
17
|
-
import org.msgpack.
|
15
|
+
import org.msgpack.core.MessagePack;
|
16
|
+
import org.msgpack.core.MessageUnpacker;
|
17
|
+
import org.msgpack.value.Value;
|
18
18
|
import org.slf4j.Logger;
|
19
19
|
|
20
20
|
import java.io.File;
|
21
21
|
import java.io.FileInputStream;
|
22
|
+
import java.util.Map;
|
22
23
|
import java.util.zip.GZIPInputStream;
|
23
24
|
|
24
25
|
import static org.embulk.output.td.TestTdOutputPlugin.config;
|
@@ -27,21 +28,20 @@ import static org.embulk.output.td.TestTdOutputPlugin.plugin;
|
|
27
28
|
import static org.embulk.output.td.TestTdOutputPlugin.pluginTask;
|
28
29
|
import static org.embulk.output.td.TestTdOutputPlugin.schema;
|
29
30
|
import static org.embulk.output.td.TestTdOutputPlugin.recordWriter;
|
30
|
-
import static org.embulk.output.td.TestTdOutputPlugin.
|
31
|
+
import static org.embulk.output.td.TestTdOutputPlugin.tdClient;
|
31
32
|
import static org.junit.Assert.assertEquals;
|
32
33
|
import static org.junit.Assert.assertTrue;
|
33
34
|
import static org.mockito.Matchers.any;
|
34
35
|
import static org.mockito.Matchers.anyString;
|
35
36
|
import static org.mockito.Mockito.doNothing;
|
36
37
|
import static org.mockito.Mockito.spy;
|
37
|
-
import static org.msgpack.
|
38
|
+
import static org.msgpack.value.ValueFactory.newString;
|
38
39
|
|
39
40
|
public class TestRecordWriter
|
40
41
|
{
|
41
42
|
@Rule
|
42
43
|
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
43
44
|
|
44
|
-
private MessagePack msgpack;
|
45
45
|
private Logger log;
|
46
46
|
private Schema schema;
|
47
47
|
private TdOutputPlugin plugin; // mock
|
@@ -51,7 +51,6 @@ public class TestRecordWriter
|
|
51
51
|
@Before
|
52
52
|
public void createResources()
|
53
53
|
{
|
54
|
-
msgpack = new MessagePack();
|
55
54
|
log = runtime.getExec().getLogger(TestRecordWriter.class);
|
56
55
|
|
57
56
|
schema = schema("time", Types.LONG, "_c0", Types.LONG, "_c1", Types.STRING,
|
@@ -65,7 +64,7 @@ public class TestRecordWriter
|
|
65
64
|
public void checkOpenAndClose()
|
66
65
|
throws Exception
|
67
66
|
{
|
68
|
-
recordWriter = recordWriter(task,
|
67
|
+
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
69
68
|
|
70
69
|
// confirm that no error happens
|
71
70
|
try {
|
@@ -80,7 +79,7 @@ public class TestRecordWriter
|
|
80
79
|
public void checkFlushAndFinish()
|
81
80
|
throws Exception
|
82
81
|
{
|
83
|
-
|
82
|
+
TDClient client = spy(plugin.newTDClient(task));
|
84
83
|
recordWriter = recordWriter(task, client, fieldWriters(log, task, schema));
|
85
84
|
|
86
85
|
{ // add no record
|
@@ -116,7 +115,7 @@ public class TestRecordWriter
|
|
116
115
|
public void addNonNullValues()
|
117
116
|
throws Exception
|
118
117
|
{
|
119
|
-
recordWriter = recordWriter(task,
|
118
|
+
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
120
119
|
|
121
120
|
try {
|
122
121
|
recordWriter.open(schema);
|
@@ -133,16 +132,16 @@ public class TestRecordWriter
|
|
133
132
|
// record count 1
|
134
133
|
assertEquals(1, builder.getRecordCount());
|
135
134
|
|
136
|
-
|
137
|
-
|
135
|
+
MessageUnpacker u = MessagePack.newDefaultUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
|
136
|
+
Map<Value, Value> v = u.unpackValue().asMapValue().map();
|
138
137
|
|
139
138
|
// compare actual values
|
140
|
-
assertEquals(1442595600L, v.get(
|
141
|
-
assertEquals(0L, v.get(
|
142
|
-
assertEquals("v", v.get(
|
143
|
-
assertEquals(true, v.get(
|
144
|
-
assertEquals(0.0, v.get(
|
145
|
-
assertEquals("2015-09-18 17:00:00.000", v.get(
|
139
|
+
assertEquals(1442595600L, v.get(newString("time")).asIntegerValue().toLong());
|
140
|
+
assertEquals(0L, v.get(newString("_c0")).asIntegerValue().toLong());
|
141
|
+
assertEquals("v", v.get(newString("_c1")).asStringValue().toString());
|
142
|
+
assertEquals(true, v.get(newString("_c2")).asBooleanValue().getBoolean());
|
143
|
+
assertEquals(0.0, v.get(newString("_c3")).asFloatValue().toFloat(), 0.000001);
|
144
|
+
assertEquals("2015-09-18 17:00:00.000", v.get(newString("_c4")).asStringValue().toString());
|
146
145
|
|
147
146
|
}
|
148
147
|
finally {
|
@@ -154,7 +153,7 @@ public class TestRecordWriter
|
|
154
153
|
public void addNullValues()
|
155
154
|
throws Exception
|
156
155
|
{
|
157
|
-
recordWriter = recordWriter(task,
|
156
|
+
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
158
157
|
|
159
158
|
try {
|
160
159
|
recordWriter.open(schema);
|
@@ -171,15 +170,15 @@ public class TestRecordWriter
|
|
171
170
|
// record count 1
|
172
171
|
assertEquals(1, builder.getRecordCount());
|
173
172
|
|
174
|
-
|
175
|
-
|
173
|
+
MessageUnpacker u = MessagePack.newDefaultUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
|
174
|
+
Map<Value, Value> v = u.unpackValue().asMapValue().map();
|
176
175
|
|
177
176
|
// compare actual values
|
178
|
-
assertTrue(v.get(
|
179
|
-
assertTrue(v.get(
|
180
|
-
assertTrue(v.get(
|
181
|
-
assertTrue(v.get(
|
182
|
-
assertTrue(v.get(
|
177
|
+
assertTrue(v.get(newString("_c0")).isNilValue());
|
178
|
+
assertTrue(v.get(newString("_c1")).isNilValue());
|
179
|
+
assertTrue(v.get(newString("_c2")).isNilValue());
|
180
|
+
assertTrue(v.get(newString("_c3")).isNilValue());
|
181
|
+
assertTrue(v.get(newString("_c4")).isNilValue());
|
183
182
|
|
184
183
|
}
|
185
184
|
finally {
|
@@ -194,7 +193,7 @@ public class TestRecordWriter
|
|
194
193
|
schema = schema("_c0", Types.LONG, "_c1", Types.STRING,
|
195
194
|
"_c2", Types.BOOLEAN, "_c3", Types.DOUBLE, "_c4", Types.TIMESTAMP);
|
196
195
|
task = pluginTask(config().set("session_name", "my_session").set("time_value", ImmutableMap.of("from", 0L, "to", 0L)));
|
197
|
-
recordWriter = recordWriter(task,
|
196
|
+
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
198
197
|
|
199
198
|
try {
|
200
199
|
recordWriter.open(schema);
|
@@ -211,16 +210,16 @@ public class TestRecordWriter
|
|
211
210
|
// record count 1
|
212
211
|
assertEquals(1, builder.getRecordCount());
|
213
212
|
|
214
|
-
|
215
|
-
|
213
|
+
MessageUnpacker u = MessagePack.newDefaultUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
|
214
|
+
Map<Value, Value> v = u.unpackValue().asMapValue().map();
|
216
215
|
|
217
216
|
// compare actual values
|
218
|
-
assertEquals(0L, v.get(
|
219
|
-
assertEquals(0L, v.get(
|
220
|
-
assertEquals("v", v.get(
|
221
|
-
assertEquals(true, v.get(
|
222
|
-
assertEquals(0.0, v.get(
|
223
|
-
assertEquals("2015-09-18 17:00:00.000", v.get(
|
217
|
+
assertEquals(0L, v.get(newString("time")).asIntegerValue().toLong());
|
218
|
+
assertEquals(0L, v.get(newString("_c0")).asIntegerValue().toLong());
|
219
|
+
assertEquals("v", v.get(newString("_c1")).asStringValue().toString());
|
220
|
+
assertEquals(true, v.get(newString("_c2")).asBooleanValue().getBoolean());
|
221
|
+
assertEquals(0.0, v.get(newString("_c3")).asFloatValue().toFloat(), 0.000001);
|
222
|
+
assertEquals("2015-09-18 17:00:00.000", v.get(newString("_c4")).asStringValue().toString());
|
224
223
|
|
225
224
|
}
|
226
225
|
finally {
|
@@ -231,7 +230,7 @@ public class TestRecordWriter
|
|
231
230
|
@Test
|
232
231
|
public void doAbortNorthing()
|
233
232
|
{
|
234
|
-
recordWriter = recordWriter(task,
|
233
|
+
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
235
234
|
recordWriter.abort();
|
236
235
|
// no error happen
|
237
236
|
}
|
@@ -239,7 +238,7 @@ public class TestRecordWriter
|
|
239
238
|
@Test
|
240
239
|
public void checkTaskReport()
|
241
240
|
{
|
242
|
-
recordWriter = recordWriter(task,
|
241
|
+
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
243
242
|
assertTrue(recordWriter.commit().isEmpty());
|
244
243
|
}
|
245
244
|
}
|
@@ -3,21 +3,20 @@ package org.embulk.output.td;
|
|
3
3
|
import com.google.common.collect.ImmutableList;
|
4
4
|
import com.google.common.collect.ImmutableMap;
|
5
5
|
import com.google.common.collect.Lists;
|
6
|
-
import com.treasuredata.
|
7
|
-
import com.treasuredata.
|
8
|
-
import com.treasuredata.
|
9
|
-
import com.treasuredata.
|
10
|
-
import com.treasuredata.
|
11
|
-
import com.treasuredata.
|
12
|
-
import com.treasuredata.
|
13
|
-
import com.treasuredata.
|
6
|
+
import com.treasuredata.client.TDClient;
|
7
|
+
import com.treasuredata.client.TDClientHttpConflictException;
|
8
|
+
import com.treasuredata.client.TDClientHttpNotFoundException;
|
9
|
+
import com.treasuredata.client.model.TDBulkImportSession;
|
10
|
+
import com.treasuredata.client.model.TDBulkImportSession.ImportStatus;
|
11
|
+
import com.treasuredata.client.model.TDColumnType;
|
12
|
+
import com.treasuredata.client.model.TDTable;
|
13
|
+
import com.treasuredata.client.model.TDTableType;
|
14
14
|
import org.embulk.EmbulkTestRuntime;
|
15
15
|
import org.embulk.config.TaskReport;
|
16
16
|
import org.embulk.config.ConfigDiff;
|
17
17
|
import org.embulk.config.ConfigException;
|
18
18
|
import org.embulk.config.ConfigSource;
|
19
19
|
import org.embulk.config.TaskSource;
|
20
|
-
import org.embulk.output.td.TdOutputPlugin.Mode;
|
21
20
|
import org.embulk.output.td.TdOutputPlugin.PluginTask;
|
22
21
|
import org.embulk.output.td.TdOutputPlugin.TimestampColumnOption;
|
23
22
|
import org.embulk.output.td.TdOutputPlugin.UnixTimestampUnit;
|
@@ -38,12 +37,12 @@ import org.slf4j.Logger;
|
|
38
37
|
import java.util.HashMap;
|
39
38
|
import java.util.List;
|
40
39
|
|
41
|
-
import static com.treasuredata.
|
42
|
-
import static com.treasuredata.
|
43
|
-
import static com.treasuredata.
|
44
|
-
import static com.treasuredata.
|
45
|
-
import static com.treasuredata.
|
46
|
-
import static com.treasuredata.
|
40
|
+
import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.COMMITTED;
|
41
|
+
import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.COMMITTING;
|
42
|
+
import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.PERFORMING;
|
43
|
+
import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.READY;
|
44
|
+
import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.UNKNOWN;
|
45
|
+
import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.UPLOADING;
|
47
46
|
import static org.junit.Assert.assertEquals;
|
48
47
|
import static org.junit.Assert.assertTrue;
|
49
48
|
import static org.junit.Assert.fail;
|
@@ -114,12 +113,12 @@ public class TestTdOutputPlugin
|
|
114
113
|
{
|
115
114
|
doReturn("session_name").when(plugin).buildBulkImportSessionName(any(PluginTask.class), any(ExecSession.class));
|
116
115
|
ConfigDiff configDiff = Exec.newConfigDiff().set("last_session", "session_name");
|
117
|
-
doReturn(configDiff).when(plugin).doRun(any(
|
116
|
+
doReturn(configDiff).when(plugin).doRun(any(TDClient.class), any(Schema.class), any(PluginTask.class), any(OutputPlugin.Control.class));
|
118
117
|
Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
|
119
118
|
|
120
119
|
{ // auto_create_table is true
|
121
120
|
ConfigSource config = this.config.deepCopy().set("auto_create_table", "true");
|
122
|
-
doNothing().when(plugin).createTableIfNotExists(any(
|
121
|
+
doNothing().when(plugin).createTableIfNotExists(any(TDClient.class), anyString(), anyString());
|
123
122
|
assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
124
123
|
{
|
125
124
|
@Override
|
@@ -132,7 +131,7 @@ public class TestTdOutputPlugin
|
|
132
131
|
|
133
132
|
{ // auto_create_table is false
|
134
133
|
ConfigSource config = this.config.deepCopy().set("auto_create_table", "false");
|
135
|
-
doNothing().when(plugin).validateTableExists(any(
|
134
|
+
doNothing().when(plugin).validateTableExists(any(TDClient.class), anyString(), anyString());
|
136
135
|
assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
137
136
|
{
|
138
137
|
@Override
|
@@ -152,8 +151,8 @@ public class TestTdOutputPlugin
|
|
152
151
|
task.setSessionName("session_name");
|
153
152
|
task.setLoadTargetTableName("my_table");
|
154
153
|
task.setDoUpload(true);
|
155
|
-
doReturn(true).when(plugin).startBulkImportSession(any(
|
156
|
-
doNothing().when(plugin).completeBulkImportSession(any(
|
154
|
+
doReturn(true).when(plugin).startBulkImportSession(any(TDClient.class), anyString(), anyString(), anyString());
|
155
|
+
doNothing().when(plugin).completeBulkImportSession(any(TDClient.class), any(Schema.class), any(PluginTask.class), anyInt());
|
157
156
|
Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
|
158
157
|
|
159
158
|
ConfigDiff configDiff = plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
|
@@ -175,9 +174,9 @@ public class TestTdOutputPlugin
|
|
175
174
|
task.setSessionName("session_name");
|
176
175
|
task.setLoadTargetTableName("my_table");
|
177
176
|
task.setDoUpload(true);
|
178
|
-
|
177
|
+
TDClient client = spy(plugin.newTDClient(task));
|
179
178
|
doNothing().when(client).deleteBulkImportSession(anyString());
|
180
|
-
doReturn(client).when(plugin).
|
179
|
+
doReturn(client).when(plugin).newTDClient(task);
|
181
180
|
Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
|
182
181
|
|
183
182
|
plugin.cleanup(task.dump(), schema, 0, Lists.newArrayList(Exec.newTaskReport()));
|
@@ -211,11 +210,11 @@ public class TestTdOutputPlugin
|
|
211
210
|
}
|
212
211
|
|
213
212
|
@Test
|
214
|
-
public void
|
213
|
+
public void newTDClient()
|
215
214
|
{
|
216
215
|
{ // no proxy setting
|
217
216
|
PluginTask task = pluginTask(config);
|
218
|
-
try (
|
217
|
+
try (TDClient client = plugin.newTDClient(task)) {
|
219
218
|
}
|
220
219
|
// no error happens
|
221
220
|
}
|
@@ -223,7 +222,7 @@ public class TestTdOutputPlugin
|
|
223
222
|
{ // proxy setting
|
224
223
|
PluginTask task = pluginTask(config.deepCopy()
|
225
224
|
.set("http_proxy", ImmutableMap.of("host", "xxx", "port", "8080")));
|
226
|
-
try (
|
225
|
+
try (TDClient client = plugin.newTDClient(task)) {
|
227
226
|
}
|
228
227
|
// no error happens
|
229
228
|
}
|
@@ -233,10 +232,10 @@ public class TestTdOutputPlugin
|
|
233
232
|
public void createTableIfNotExists()
|
234
233
|
{
|
235
234
|
PluginTask task = pluginTask(config);
|
236
|
-
|
235
|
+
TDClient client = spy(plugin.newTDClient(task));
|
237
236
|
|
238
237
|
{ // database exists but table doesn't exist
|
239
|
-
|
238
|
+
doNothing().when(client).createTable(anyString(), anyString());
|
240
239
|
plugin.createTableIfNotExists(client, "my_db", "my_table");
|
241
240
|
// no error happens
|
242
241
|
}
|
@@ -249,14 +248,14 @@ public class TestTdOutputPlugin
|
|
249
248
|
|
250
249
|
{ // database and table don't exist
|
251
250
|
{ // createTable -> createDB -> createTable
|
252
|
-
doThrow(notFound()).
|
253
|
-
|
251
|
+
doThrow(notFound()).doNothing().when(client).createTable(anyString(), anyString());
|
252
|
+
doNothing().when(client).createDatabase(anyString());
|
254
253
|
plugin.createTableIfNotExists(client, "my_db", "my_table");
|
255
254
|
// no error happens
|
256
255
|
}
|
257
256
|
|
258
257
|
{ // createTable -> createDB -> createTable
|
259
|
-
doThrow(notFound()).
|
258
|
+
doThrow(notFound()).doNothing().when(client).createTable(anyString(), anyString());
|
260
259
|
doThrow(conflict()).when(client).createDatabase(anyString());
|
261
260
|
plugin.createTableIfNotExists(client, "my_db", "my_table");
|
262
261
|
// no error happens
|
@@ -264,7 +263,7 @@ public class TestTdOutputPlugin
|
|
264
263
|
|
265
264
|
{ // createTable -> createDB -> createTable
|
266
265
|
doThrow(notFound()).doThrow(conflict()).when(client).createTable(anyString(), anyString());
|
267
|
-
|
266
|
+
doNothing().when(client).createDatabase(anyString());
|
268
267
|
plugin.createTableIfNotExists(client, "my_db", "my_table");
|
269
268
|
// no error happens
|
270
269
|
}
|
@@ -282,17 +281,17 @@ public class TestTdOutputPlugin
|
|
282
281
|
public void validateTableExists()
|
283
282
|
{
|
284
283
|
PluginTask task = pluginTask(config);
|
285
|
-
|
286
|
-
TDTable table =
|
284
|
+
TDClient client = spy(plugin.newTDClient(task));
|
285
|
+
TDTable table = newTable("my_table", "[]");
|
287
286
|
|
288
287
|
{ // table exists
|
289
|
-
doReturn(ImmutableList.of(table)).when(client).
|
288
|
+
doReturn(ImmutableList.of(table)).when(client).listTables(anyString());
|
290
289
|
plugin.validateTableExists(client, "my_db", "my_table");
|
291
290
|
// no error happens
|
292
291
|
}
|
293
292
|
|
294
293
|
{ // table doesn't exist
|
295
|
-
doReturn(ImmutableList.of()).when(client).
|
294
|
+
doReturn(ImmutableList.of()).when(client).listTables(anyString());
|
296
295
|
try {
|
297
296
|
plugin.validateTableExists(client, "my_db", "my_table");
|
298
297
|
fail();
|
@@ -303,7 +302,7 @@ public class TestTdOutputPlugin
|
|
303
302
|
}
|
304
303
|
|
305
304
|
{ // database doesn't exist
|
306
|
-
doThrow(notFound()).when(client).
|
305
|
+
doThrow(notFound()).when(client).listTables(anyString());
|
307
306
|
try {
|
308
307
|
plugin.validateTableExists(client, "my_db", "my_table");
|
309
308
|
fail();
|
@@ -332,7 +331,7 @@ public class TestTdOutputPlugin
|
|
332
331
|
public void startBulkImportSession()
|
333
332
|
{
|
334
333
|
PluginTask task = pluginTask(config);
|
335
|
-
|
334
|
+
TDClient client = spy(plugin.newTDClient(task));
|
336
335
|
doNothing().when(client).createBulkImportSession(anyString(), anyString(), anyString());
|
337
336
|
|
338
337
|
{ // status is uploading and unfrozen
|
@@ -388,12 +387,12 @@ public class TestTdOutputPlugin
|
|
388
387
|
PluginTask task = pluginTask(config);
|
389
388
|
Schema schema = schema("c0", Types.LONG);
|
390
389
|
|
391
|
-
doReturn(session(UNKNOWN, false)).when(plugin).waitForStatusChange(any(
|
392
|
-
doReturn(new HashMap<String, TDColumnType>()).when(plugin).updateSchema(any(
|
390
|
+
doReturn(session(UNKNOWN, false)).when(plugin).waitForStatusChange(any(TDClient.class), anyString(), any(ImportStatus.class), any(ImportStatus.class), anyString());
|
391
|
+
doReturn(new HashMap<String, TDColumnType>()).when(plugin).updateSchema(any(TDClient.class), any(Schema.class), any(PluginTask.class));
|
393
392
|
|
394
|
-
|
393
|
+
TDClient client = spy(plugin.newTDClient(task));
|
395
394
|
doNothing().when(client).freezeBulkImportSession(anyString());
|
396
|
-
doNothing().when(client).performBulkImportSession(anyString()
|
395
|
+
doNothing().when(client).performBulkImportSession(anyString());
|
397
396
|
doNothing().when(client).commitBulkImportSession(anyString());
|
398
397
|
|
399
398
|
{ // uploading + unfreeze
|
@@ -454,7 +453,7 @@ public class TestTdOutputPlugin
|
|
454
453
|
public void waitForStatusChange()
|
455
454
|
{
|
456
455
|
PluginTask task = pluginTask(config);
|
457
|
-
|
456
|
+
TDClient client = spy(plugin.newTDClient(task));
|
458
457
|
|
459
458
|
{ // performing -> ready
|
460
459
|
doReturn(session(PERFORMING, false)).doReturn(session(READY, false)).when(client).getBulkImportSession("my_session");
|
@@ -512,9 +511,14 @@ public class TestTdOutputPlugin
|
|
512
511
|
return spy(new TdOutputPlugin());
|
513
512
|
}
|
514
513
|
|
515
|
-
public static
|
514
|
+
public static TDClient tdClient(TdOutputPlugin plugin, PluginTask task)
|
516
515
|
{
|
517
|
-
return spy(plugin.
|
516
|
+
return spy(plugin.newTDClient(task));
|
517
|
+
}
|
518
|
+
|
519
|
+
public static TDTable newTable(String name, String schema)
|
520
|
+
{
|
521
|
+
return new TDTable("", name, TDTableType.LOG, schema, 0, 0, "", "", "", "");
|
518
522
|
}
|
519
523
|
|
520
524
|
public static FieldWriterSet fieldWriters(Logger log, PluginTask task, Schema schema)
|
@@ -522,19 +526,19 @@ public class TestTdOutputPlugin
|
|
522
526
|
return spy(new FieldWriterSet(log, task, schema));
|
523
527
|
}
|
524
528
|
|
525
|
-
public static RecordWriter recordWriter(PluginTask task,
|
529
|
+
public static RecordWriter recordWriter(PluginTask task, TDClient client, FieldWriterSet fieldWriters)
|
526
530
|
{
|
527
531
|
return spy(new RecordWriter(task, 0, client, fieldWriters));
|
528
532
|
}
|
529
533
|
|
530
|
-
static
|
534
|
+
static TDClientHttpNotFoundException notFound()
|
531
535
|
{
|
532
|
-
return new
|
536
|
+
return new TDClientHttpNotFoundException("not found");
|
533
537
|
}
|
534
538
|
|
535
|
-
static
|
539
|
+
static TDClientHttpConflictException conflict()
|
536
540
|
{
|
537
|
-
return new
|
541
|
+
return new TDClientHttpConflictException("conflict");
|
538
542
|
}
|
539
543
|
|
540
544
|
private static TDBulkImportSession session(ImportStatus status, boolean uploadFrozen)
|