embulk-output-td 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/build.gradle +9 -8
  4. data/embulk-output-td.gemspec +1 -1
  5. data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +11 -12
  6. data/src/main/java/org/embulk/output/td/RecordWriter.java +4 -7
  7. data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +89 -84
  8. data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +9 -0
  9. data/src/main/java/org/embulk/output/td/writer/JsonFieldWriter.java +23 -0
  10. data/src/test/java/org/embulk/output/td/TestRecordWriter.java +37 -38
  11. data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +53 -49
  12. metadata +9 -30
  13. data/src/main/java/com/treasuredata/api/TdApiClient.java +0 -506
  14. data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +0 -79
  15. data/src/main/java/com/treasuredata/api/TdApiConflictException.java +0 -10
  16. data/src/main/java/com/treasuredata/api/TdApiConstants.java +0 -10
  17. data/src/main/java/com/treasuredata/api/TdApiException.java +0 -20
  18. data/src/main/java/com/treasuredata/api/TdApiExecutionException.java +0 -10
  19. data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +0 -16
  20. data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +0 -18
  21. data/src/main/java/com/treasuredata/api/TdApiNotFoundException.java +0 -10
  22. data/src/main/java/com/treasuredata/api/TdApiResponseException.java +0 -32
  23. data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +0 -80
  24. data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +0 -157
  25. data/src/main/java/com/treasuredata/api/model/TDColumn.java +0 -129
  26. data/src/main/java/com/treasuredata/api/model/TDColumnType.java +0 -23
  27. data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +0 -128
  28. data/src/main/java/com/treasuredata/api/model/TDDatabase.java +0 -49
  29. data/src/main/java/com/treasuredata/api/model/TDDatabaseList.java +0 -24
  30. data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +0 -88
  31. data/src/main/java/com/treasuredata/api/model/TDPrimitiveColumnType.java +0 -61
  32. data/src/main/java/com/treasuredata/api/model/TDTable.java +0 -64
  33. data/src/main/java/com/treasuredata/api/model/TDTableList.java +0 -33
  34. data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +0 -50
  35. data/src/main/java/com/treasuredata/api/model/TDTableSchema.java +0 -44
  36. data/src/main/java/com/treasuredata/api/model/TDTableType.java +0 -37
  37. data/src/test/java/com/treasuredata/api/TestTdApiClient.java +0 -79
@@ -19,6 +19,7 @@ import org.embulk.spi.Schema;
19
19
  import org.embulk.spi.time.TimestampFormatter;
20
20
  import org.embulk.spi.type.BooleanType;
21
21
  import org.embulk.spi.type.DoubleType;
22
+ import org.embulk.spi.type.JsonType;
22
23
  import org.embulk.spi.type.LongType;
23
24
  import org.embulk.spi.type.StringType;
24
25
  import org.embulk.spi.type.TimestampType;
@@ -238,6 +239,9 @@ public class FieldWriterSet
238
239
  else if (columnType instanceof TimestampType) {
239
240
  return newSimpleTimestampFieldWriter(columnName, columnType, convertTimestampType, timestampFormatter);
240
241
  }
242
+ else if (columnType instanceof JsonType) {
243
+ return new JsonFieldWriter(columnName);
244
+ }
241
245
  else {
242
246
  throw new ConfigException("Unsupported type: " + columnType);
243
247
  }
@@ -300,6 +304,11 @@ public class FieldWriterSet
300
304
  addColumn(builder, reader, column);
301
305
  }
302
306
 
307
+ @Override
308
+ public void jsonColumn(Column column)
309
+ {
310
+ addColumn(builder, reader, column);
311
+ }
303
312
  });
304
313
 
305
314
  endRecord(builder);
@@ -0,0 +1,23 @@
1
+ package org.embulk.output.td.writer;
2
+
3
+ import org.embulk.output.td.MsgpackGZFileBuilder;
4
+ import org.embulk.spi.Column;
5
+ import org.embulk.spi.PageReader;
6
+
7
+ import java.io.IOException;
8
+
9
+ public class JsonFieldWriter
10
+ extends FieldWriter
11
+ {
12
+ public JsonFieldWriter(String keyName)
13
+ {
14
+ super(keyName);
15
+ }
16
+
17
+ @Override
18
+ public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
19
+ throws IOException
20
+ {
21
+ builder.writeString(reader.getJson(column).toJson());
22
+ }
23
+ }
@@ -1,7 +1,7 @@
1
1
  package org.embulk.output.td;
2
2
 
3
3
  import com.google.common.collect.ImmutableMap;
4
- import com.treasuredata.api.TdApiClient;
4
+ import com.treasuredata.client.TDClient;
5
5
  import org.embulk.EmbulkTestRuntime;
6
6
  import org.embulk.output.td.TdOutputPlugin.PluginTask;
7
7
  import org.embulk.spi.Page;
@@ -12,13 +12,14 @@ import org.embulk.spi.type.Types;
12
12
  import org.junit.Before;
13
13
  import org.junit.Rule;
14
14
  import org.junit.Test;
15
- import org.msgpack.MessagePack;
16
- import org.msgpack.type.MapValue;
17
- import org.msgpack.unpacker.Unpacker;
15
+ import org.msgpack.core.MessagePack;
16
+ import org.msgpack.core.MessageUnpacker;
17
+ import org.msgpack.value.Value;
18
18
  import org.slf4j.Logger;
19
19
 
20
20
  import java.io.File;
21
21
  import java.io.FileInputStream;
22
+ import java.util.Map;
22
23
  import java.util.zip.GZIPInputStream;
23
24
 
24
25
  import static org.embulk.output.td.TestTdOutputPlugin.config;
@@ -27,21 +28,20 @@ import static org.embulk.output.td.TestTdOutputPlugin.plugin;
27
28
  import static org.embulk.output.td.TestTdOutputPlugin.pluginTask;
28
29
  import static org.embulk.output.td.TestTdOutputPlugin.schema;
29
30
  import static org.embulk.output.td.TestTdOutputPlugin.recordWriter;
30
- import static org.embulk.output.td.TestTdOutputPlugin.tdApiClient;
31
+ import static org.embulk.output.td.TestTdOutputPlugin.tdClient;
31
32
  import static org.junit.Assert.assertEquals;
32
33
  import static org.junit.Assert.assertTrue;
33
34
  import static org.mockito.Matchers.any;
34
35
  import static org.mockito.Matchers.anyString;
35
36
  import static org.mockito.Mockito.doNothing;
36
37
  import static org.mockito.Mockito.spy;
37
- import static org.msgpack.type.ValueFactory.createRawValue;
38
+ import static org.msgpack.value.ValueFactory.newString;
38
39
 
39
40
  public class TestRecordWriter
40
41
  {
41
42
  @Rule
42
43
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
43
44
 
44
- private MessagePack msgpack;
45
45
  private Logger log;
46
46
  private Schema schema;
47
47
  private TdOutputPlugin plugin; // mock
@@ -51,7 +51,6 @@ public class TestRecordWriter
51
51
  @Before
52
52
  public void createResources()
53
53
  {
54
- msgpack = new MessagePack();
55
54
  log = runtime.getExec().getLogger(TestRecordWriter.class);
56
55
 
57
56
  schema = schema("time", Types.LONG, "_c0", Types.LONG, "_c1", Types.STRING,
@@ -65,7 +64,7 @@ public class TestRecordWriter
65
64
  public void checkOpenAndClose()
66
65
  throws Exception
67
66
  {
68
- recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
67
+ recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
69
68
 
70
69
  // confirm that no error happens
71
70
  try {
@@ -80,7 +79,7 @@ public class TestRecordWriter
80
79
  public void checkFlushAndFinish()
81
80
  throws Exception
82
81
  {
83
- TdApiClient client = spy(plugin.newTdApiClient(task));
82
+ TDClient client = spy(plugin.newTDClient(task));
84
83
  recordWriter = recordWriter(task, client, fieldWriters(log, task, schema));
85
84
 
86
85
  { // add no record
@@ -116,7 +115,7 @@ public class TestRecordWriter
116
115
  public void addNonNullValues()
117
116
  throws Exception
118
117
  {
119
- recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
118
+ recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
120
119
 
121
120
  try {
122
121
  recordWriter.open(schema);
@@ -133,16 +132,16 @@ public class TestRecordWriter
133
132
  // record count 1
134
133
  assertEquals(1, builder.getRecordCount());
135
134
 
136
- Unpacker u = msgpack.createUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
137
- MapValue v = u.readValue().asMapValue();
135
+ MessageUnpacker u = MessagePack.newDefaultUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
136
+ Map<Value, Value> v = u.unpackValue().asMapValue().map();
138
137
 
139
138
  // compare actual values
140
- assertEquals(1442595600L, v.get(createRawValue("time")).asIntegerValue().getLong());
141
- assertEquals(0L, v.get(createRawValue("_c0")).asIntegerValue().getLong());
142
- assertEquals("v", v.get(createRawValue("_c1")).asRawValue().getString());
143
- assertEquals(true, v.get(createRawValue("_c2")).asBooleanValue().getBoolean());
144
- assertEquals(0.0, v.get(createRawValue("_c3")).asFloatValue().getDouble(), 0.000001);
145
- assertEquals("2015-09-18 17:00:00.000", v.get(createRawValue("_c4")).asRawValue().getString());
139
+ assertEquals(1442595600L, v.get(newString("time")).asIntegerValue().toLong());
140
+ assertEquals(0L, v.get(newString("_c0")).asIntegerValue().toLong());
141
+ assertEquals("v", v.get(newString("_c1")).asStringValue().toString());
142
+ assertEquals(true, v.get(newString("_c2")).asBooleanValue().getBoolean());
143
+ assertEquals(0.0, v.get(newString("_c3")).asFloatValue().toFloat(), 0.000001);
144
+ assertEquals("2015-09-18 17:00:00.000", v.get(newString("_c4")).asStringValue().toString());
146
145
 
147
146
  }
148
147
  finally {
@@ -154,7 +153,7 @@ public class TestRecordWriter
154
153
  public void addNullValues()
155
154
  throws Exception
156
155
  {
157
- recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
156
+ recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
158
157
 
159
158
  try {
160
159
  recordWriter.open(schema);
@@ -171,15 +170,15 @@ public class TestRecordWriter
171
170
  // record count 1
172
171
  assertEquals(1, builder.getRecordCount());
173
172
 
174
- Unpacker u = msgpack.createUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
175
- MapValue v = u.readValue().asMapValue();
173
+ MessageUnpacker u = MessagePack.newDefaultUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
174
+ Map<Value, Value> v = u.unpackValue().asMapValue().map();
176
175
 
177
176
  // compare actual values
178
- assertTrue(v.get(createRawValue("_c0")).isNilValue());
179
- assertTrue(v.get(createRawValue("_c1")).isNilValue());
180
- assertTrue(v.get(createRawValue("_c2")).isNilValue());
181
- assertTrue(v.get(createRawValue("_c3")).isNilValue());
182
- assertTrue(v.get(createRawValue("_c4")).isNilValue());
177
+ assertTrue(v.get(newString("_c0")).isNilValue());
178
+ assertTrue(v.get(newString("_c1")).isNilValue());
179
+ assertTrue(v.get(newString("_c2")).isNilValue());
180
+ assertTrue(v.get(newString("_c3")).isNilValue());
181
+ assertTrue(v.get(newString("_c4")).isNilValue());
183
182
 
184
183
  }
185
184
  finally {
@@ -194,7 +193,7 @@ public class TestRecordWriter
194
193
  schema = schema("_c0", Types.LONG, "_c1", Types.STRING,
195
194
  "_c2", Types.BOOLEAN, "_c3", Types.DOUBLE, "_c4", Types.TIMESTAMP);
196
195
  task = pluginTask(config().set("session_name", "my_session").set("time_value", ImmutableMap.of("from", 0L, "to", 0L)));
197
- recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
196
+ recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
198
197
 
199
198
  try {
200
199
  recordWriter.open(schema);
@@ -211,16 +210,16 @@ public class TestRecordWriter
211
210
  // record count 1
212
211
  assertEquals(1, builder.getRecordCount());
213
212
 
214
- Unpacker u = msgpack.createUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
215
- MapValue v = u.readValue().asMapValue();
213
+ MessageUnpacker u = MessagePack.newDefaultUnpacker(new GZIPInputStream(new FileInputStream(builder.getFile())));
214
+ Map<Value, Value> v = u.unpackValue().asMapValue().map();
216
215
 
217
216
  // compare actual values
218
- assertEquals(0L, v.get(createRawValue("time")).asIntegerValue().getLong());
219
- assertEquals(0L, v.get(createRawValue("_c0")).asIntegerValue().getLong());
220
- assertEquals("v", v.get(createRawValue("_c1")).asRawValue().getString());
221
- assertEquals(true, v.get(createRawValue("_c2")).asBooleanValue().getBoolean());
222
- assertEquals(0.0, v.get(createRawValue("_c3")).asFloatValue().getDouble(), 0.000001);
223
- assertEquals("2015-09-18 17:00:00.000", v.get(createRawValue("_c4")).asRawValue().getString());
217
+ assertEquals(0L, v.get(newString("time")).asIntegerValue().toLong());
218
+ assertEquals(0L, v.get(newString("_c0")).asIntegerValue().toLong());
219
+ assertEquals("v", v.get(newString("_c1")).asStringValue().toString());
220
+ assertEquals(true, v.get(newString("_c2")).asBooleanValue().getBoolean());
221
+ assertEquals(0.0, v.get(newString("_c3")).asFloatValue().toFloat(), 0.000001);
222
+ assertEquals("2015-09-18 17:00:00.000", v.get(newString("_c4")).asStringValue().toString());
224
223
 
225
224
  }
226
225
  finally {
@@ -231,7 +230,7 @@ public class TestRecordWriter
231
230
  @Test
232
231
  public void doAbortNorthing()
233
232
  {
234
- recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
233
+ recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
235
234
  recordWriter.abort();
236
235
  // no error happen
237
236
  }
@@ -239,7 +238,7 @@ public class TestRecordWriter
239
238
  @Test
240
239
  public void checkTaskReport()
241
240
  {
242
- recordWriter = recordWriter(task, tdApiClient(plugin, task), fieldWriters(log, task, schema));
241
+ recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
243
242
  assertTrue(recordWriter.commit().isEmpty());
244
243
  }
245
244
  }
@@ -3,21 +3,20 @@ package org.embulk.output.td;
3
3
  import com.google.common.collect.ImmutableList;
4
4
  import com.google.common.collect.ImmutableMap;
5
5
  import com.google.common.collect.Lists;
6
- import com.treasuredata.api.TdApiClient;
7
- import com.treasuredata.api.TdApiConflictException;
8
- import com.treasuredata.api.TdApiNotFoundException;
9
- import com.treasuredata.api.model.TDBulkImportSession;
10
- import com.treasuredata.api.model.TDBulkImportSession.ImportStatus;
11
- import com.treasuredata.api.model.TDColumnType;
12
- import com.treasuredata.api.model.TDTable;
13
- import com.treasuredata.api.model.TDTableType;
6
+ import com.treasuredata.client.TDClient;
7
+ import com.treasuredata.client.TDClientHttpConflictException;
8
+ import com.treasuredata.client.TDClientHttpNotFoundException;
9
+ import com.treasuredata.client.model.TDBulkImportSession;
10
+ import com.treasuredata.client.model.TDBulkImportSession.ImportStatus;
11
+ import com.treasuredata.client.model.TDColumnType;
12
+ import com.treasuredata.client.model.TDTable;
13
+ import com.treasuredata.client.model.TDTableType;
14
14
  import org.embulk.EmbulkTestRuntime;
15
15
  import org.embulk.config.TaskReport;
16
16
  import org.embulk.config.ConfigDiff;
17
17
  import org.embulk.config.ConfigException;
18
18
  import org.embulk.config.ConfigSource;
19
19
  import org.embulk.config.TaskSource;
20
- import org.embulk.output.td.TdOutputPlugin.Mode;
21
20
  import org.embulk.output.td.TdOutputPlugin.PluginTask;
22
21
  import org.embulk.output.td.TdOutputPlugin.TimestampColumnOption;
23
22
  import org.embulk.output.td.TdOutputPlugin.UnixTimestampUnit;
@@ -38,12 +37,12 @@ import org.slf4j.Logger;
38
37
  import java.util.HashMap;
39
38
  import java.util.List;
40
39
 
41
- import static com.treasuredata.api.model.TDBulkImportSession.ImportStatus.COMMITTED;
42
- import static com.treasuredata.api.model.TDBulkImportSession.ImportStatus.COMMITTING;
43
- import static com.treasuredata.api.model.TDBulkImportSession.ImportStatus.PERFORMING;
44
- import static com.treasuredata.api.model.TDBulkImportSession.ImportStatus.READY;
45
- import static com.treasuredata.api.model.TDBulkImportSession.ImportStatus.UNKNOWN;
46
- import static com.treasuredata.api.model.TDBulkImportSession.ImportStatus.UPLOADING;
40
+ import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.COMMITTED;
41
+ import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.COMMITTING;
42
+ import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.PERFORMING;
43
+ import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.READY;
44
+ import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.UNKNOWN;
45
+ import static com.treasuredata.client.model.TDBulkImportSession.ImportStatus.UPLOADING;
47
46
  import static org.junit.Assert.assertEquals;
48
47
  import static org.junit.Assert.assertTrue;
49
48
  import static org.junit.Assert.fail;
@@ -114,12 +113,12 @@ public class TestTdOutputPlugin
114
113
  {
115
114
  doReturn("session_name").when(plugin).buildBulkImportSessionName(any(PluginTask.class), any(ExecSession.class));
116
115
  ConfigDiff configDiff = Exec.newConfigDiff().set("last_session", "session_name");
117
- doReturn(configDiff).when(plugin).doRun(any(TdApiClient.class), any(Schema.class), any(PluginTask.class), any(OutputPlugin.Control.class));
116
+ doReturn(configDiff).when(plugin).doRun(any(TDClient.class), any(Schema.class), any(PluginTask.class), any(OutputPlugin.Control.class));
118
117
  Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
119
118
 
120
119
  { // auto_create_table is true
121
120
  ConfigSource config = this.config.deepCopy().set("auto_create_table", "true");
122
- doNothing().when(plugin).createTableIfNotExists(any(TdApiClient.class), anyString(), anyString());
121
+ doNothing().when(plugin).createTableIfNotExists(any(TDClient.class), anyString(), anyString());
123
122
  assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
124
123
  {
125
124
  @Override
@@ -132,7 +131,7 @@ public class TestTdOutputPlugin
132
131
 
133
132
  { // auto_create_table is false
134
133
  ConfigSource config = this.config.deepCopy().set("auto_create_table", "false");
135
- doNothing().when(plugin).validateTableExists(any(TdApiClient.class), anyString(), anyString());
134
+ doNothing().when(plugin).validateTableExists(any(TDClient.class), anyString(), anyString());
136
135
  assertEquals("session_name", plugin.transaction(config, schema, 0, new OutputPlugin.Control()
137
136
  {
138
137
  @Override
@@ -152,8 +151,8 @@ public class TestTdOutputPlugin
152
151
  task.setSessionName("session_name");
153
152
  task.setLoadTargetTableName("my_table");
154
153
  task.setDoUpload(true);
155
- doReturn(true).when(plugin).startBulkImportSession(any(TdApiClient.class), anyString(), anyString(), anyString());
156
- doNothing().when(plugin).completeBulkImportSession(any(TdApiClient.class), any(Schema.class), any(PluginTask.class), anyInt());
154
+ doReturn(true).when(plugin).startBulkImportSession(any(TDClient.class), anyString(), anyString(), anyString());
155
+ doNothing().when(plugin).completeBulkImportSession(any(TDClient.class), any(Schema.class), any(PluginTask.class), anyInt());
157
156
  Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
158
157
 
159
158
  ConfigDiff configDiff = plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
@@ -175,9 +174,9 @@ public class TestTdOutputPlugin
175
174
  task.setSessionName("session_name");
176
175
  task.setLoadTargetTableName("my_table");
177
176
  task.setDoUpload(true);
178
- TdApiClient client = spy(plugin.newTdApiClient(task));
177
+ TDClient client = spy(plugin.newTDClient(task));
179
178
  doNothing().when(client).deleteBulkImportSession(anyString());
180
- doReturn(client).when(plugin).newTdApiClient(task);
179
+ doReturn(client).when(plugin).newTDClient(task);
181
180
  Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
182
181
 
183
182
  plugin.cleanup(task.dump(), schema, 0, Lists.newArrayList(Exec.newTaskReport()));
@@ -211,11 +210,11 @@ public class TestTdOutputPlugin
211
210
  }
212
211
 
213
212
  @Test
214
- public void newTdApiClient()
213
+ public void newTDClient()
215
214
  {
216
215
  { // no proxy setting
217
216
  PluginTask task = pluginTask(config);
218
- try (TdApiClient client = plugin.newTdApiClient(task)) {
217
+ try (TDClient client = plugin.newTDClient(task)) {
219
218
  }
220
219
  // no error happens
221
220
  }
@@ -223,7 +222,7 @@ public class TestTdOutputPlugin
223
222
  { // proxy setting
224
223
  PluginTask task = pluginTask(config.deepCopy()
225
224
  .set("http_proxy", ImmutableMap.of("host", "xxx", "port", "8080")));
226
- try (TdApiClient client = plugin.newTdApiClient(task)) {
225
+ try (TDClient client = plugin.newTDClient(task)) {
227
226
  }
228
227
  // no error happens
229
228
  }
@@ -233,10 +232,10 @@ public class TestTdOutputPlugin
233
232
  public void createTableIfNotExists()
234
233
  {
235
234
  PluginTask task = pluginTask(config);
236
- TdApiClient client = spy(plugin.newTdApiClient(task));
235
+ TDClient client = spy(plugin.newTDClient(task));
237
236
 
238
237
  { // database exists but table doesn't exist
239
- doReturn(null).when(client).createTable(anyString(), anyString());
238
+ doNothing().when(client).createTable(anyString(), anyString());
240
239
  plugin.createTableIfNotExists(client, "my_db", "my_table");
241
240
  // no error happens
242
241
  }
@@ -249,14 +248,14 @@ public class TestTdOutputPlugin
249
248
 
250
249
  { // database and table don't exist
251
250
  { // createTable -> createDB -> createTable
252
- doThrow(notFound()).doReturn(null).when(client).createTable(anyString(), anyString());
253
- doReturn(null).when(client).createDatabase(anyString());
251
+ doThrow(notFound()).doNothing().when(client).createTable(anyString(), anyString());
252
+ doNothing().when(client).createDatabase(anyString());
254
253
  plugin.createTableIfNotExists(client, "my_db", "my_table");
255
254
  // no error happens
256
255
  }
257
256
 
258
257
  { // createTable -> createDB -> createTable
259
- doThrow(notFound()).doReturn(null).when(client).createTable(anyString(), anyString());
258
+ doThrow(notFound()).doNothing().when(client).createTable(anyString(), anyString());
260
259
  doThrow(conflict()).when(client).createDatabase(anyString());
261
260
  plugin.createTableIfNotExists(client, "my_db", "my_table");
262
261
  // no error happens
@@ -264,7 +263,7 @@ public class TestTdOutputPlugin
264
263
 
265
264
  { // createTable -> createDB -> createTable
266
265
  doThrow(notFound()).doThrow(conflict()).when(client).createTable(anyString(), anyString());
267
- doReturn(null).when(client).createDatabase(anyString());
266
+ doNothing().when(client).createDatabase(anyString());
268
267
  plugin.createTableIfNotExists(client, "my_db", "my_table");
269
268
  // no error happens
270
269
  }
@@ -282,17 +281,17 @@ public class TestTdOutputPlugin
282
281
  public void validateTableExists()
283
282
  {
284
283
  PluginTask task = pluginTask(config);
285
- TdApiClient client = spy(plugin.newTdApiClient(task));
286
- TDTable table = new TDTable("my_table", TDTableType.LOG, null);
284
+ TDClient client = spy(plugin.newTDClient(task));
285
+ TDTable table = newTable("my_table", "[]");
287
286
 
288
287
  { // table exists
289
- doReturn(ImmutableList.of(table)).when(client).getTables(anyString());
288
+ doReturn(ImmutableList.of(table)).when(client).listTables(anyString());
290
289
  plugin.validateTableExists(client, "my_db", "my_table");
291
290
  // no error happens
292
291
  }
293
292
 
294
293
  { // table doesn't exist
295
- doReturn(ImmutableList.of()).when(client).getTables(anyString());
294
+ doReturn(ImmutableList.of()).when(client).listTables(anyString());
296
295
  try {
297
296
  plugin.validateTableExists(client, "my_db", "my_table");
298
297
  fail();
@@ -303,7 +302,7 @@ public class TestTdOutputPlugin
303
302
  }
304
303
 
305
304
  { // database doesn't exist
306
- doThrow(notFound()).when(client).getTables(anyString());
305
+ doThrow(notFound()).when(client).listTables(anyString());
307
306
  try {
308
307
  plugin.validateTableExists(client, "my_db", "my_table");
309
308
  fail();
@@ -332,7 +331,7 @@ public class TestTdOutputPlugin
332
331
  public void startBulkImportSession()
333
332
  {
334
333
  PluginTask task = pluginTask(config);
335
- TdApiClient client = spy(plugin.newTdApiClient(task));
334
+ TDClient client = spy(plugin.newTDClient(task));
336
335
  doNothing().when(client).createBulkImportSession(anyString(), anyString(), anyString());
337
336
 
338
337
  { // status is uploading and unfrozen
@@ -388,12 +387,12 @@ public class TestTdOutputPlugin
388
387
  PluginTask task = pluginTask(config);
389
388
  Schema schema = schema("c0", Types.LONG);
390
389
 
391
- doReturn(session(UNKNOWN, false)).when(plugin).waitForStatusChange(any(TdApiClient.class), anyString(), any(ImportStatus.class), any(ImportStatus.class), anyString());
392
- doReturn(new HashMap<String, TDColumnType>()).when(plugin).updateSchema(any(TdApiClient.class), any(Schema.class), any(PluginTask.class));
390
+ doReturn(session(UNKNOWN, false)).when(plugin).waitForStatusChange(any(TDClient.class), anyString(), any(ImportStatus.class), any(ImportStatus.class), anyString());
391
+ doReturn(new HashMap<String, TDColumnType>()).when(plugin).updateSchema(any(TDClient.class), any(Schema.class), any(PluginTask.class));
393
392
 
394
- TdApiClient client = spy(plugin.newTdApiClient(task));
393
+ TDClient client = spy(plugin.newTDClient(task));
395
394
  doNothing().when(client).freezeBulkImportSession(anyString());
396
- doNothing().when(client).performBulkImportSession(anyString(), anyInt());
395
+ doNothing().when(client).performBulkImportSession(anyString());
397
396
  doNothing().when(client).commitBulkImportSession(anyString());
398
397
 
399
398
  { // uploading + unfreeze
@@ -454,7 +453,7 @@ public class TestTdOutputPlugin
454
453
  public void waitForStatusChange()
455
454
  {
456
455
  PluginTask task = pluginTask(config);
457
- TdApiClient client = spy(plugin.newTdApiClient(task));
456
+ TDClient client = spy(plugin.newTDClient(task));
458
457
 
459
458
  { // performing -> ready
460
459
  doReturn(session(PERFORMING, false)).doReturn(session(READY, false)).when(client).getBulkImportSession("my_session");
@@ -512,9 +511,14 @@ public class TestTdOutputPlugin
512
511
  return spy(new TdOutputPlugin());
513
512
  }
514
513
 
515
- public static TdApiClient tdApiClient(TdOutputPlugin plugin, PluginTask task)
514
+ public static TDClient tdClient(TdOutputPlugin plugin, PluginTask task)
516
515
  {
517
- return spy(plugin.newTdApiClient(task));
516
+ return spy(plugin.newTDClient(task));
517
+ }
518
+
519
+ public static TDTable newTable(String name, String schema)
520
+ {
521
+ return new TDTable("", name, TDTableType.LOG, schema, 0, 0, "", "", "", "");
518
522
  }
519
523
 
520
524
  public static FieldWriterSet fieldWriters(Logger log, PluginTask task, Schema schema)
@@ -522,19 +526,19 @@ public class TestTdOutputPlugin
522
526
  return spy(new FieldWriterSet(log, task, schema));
523
527
  }
524
528
 
525
- public static RecordWriter recordWriter(PluginTask task, TdApiClient client, FieldWriterSet fieldWriters)
529
+ public static RecordWriter recordWriter(PluginTask task, TDClient client, FieldWriterSet fieldWriters)
526
530
  {
527
531
  return spy(new RecordWriter(task, 0, client, fieldWriters));
528
532
  }
529
533
 
530
- static TdApiNotFoundException notFound()
534
+ static TDClientHttpNotFoundException notFound()
531
535
  {
532
- return new TdApiNotFoundException(404, "not found".getBytes());
536
+ return new TDClientHttpNotFoundException("not found");
533
537
  }
534
538
 
535
- static TdApiConflictException conflict()
539
+ static TDClientHttpConflictException conflict()
536
540
  {
537
- return new TdApiConflictException(409, "conflict".getBytes());
541
+ return new TDClientHttpConflictException("conflict");
538
542
  }
539
543
 
540
544
  private static TDBulkImportSession session(ImportStatus status, boolean uploadFrozen)