embulk-output-td 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +7 -0
  3. data/CHANGELOG.md +4 -0
  4. data/README.md +1 -0
  5. data/build.gradle +5 -1
  6. data/config/checkstyle/checkstyle.xml +117 -0
  7. data/embulk-output-td.gemspec +1 -1
  8. data/gradle/check.gradle +34 -0
  9. data/src/main/java/com/treasuredata/api/TdApiClient.java +47 -23
  10. data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +3 -3
  11. data/src/main/java/com/treasuredata/api/TdApiConstants.java +6 -2
  12. data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +2 -1
  13. data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +2 -1
  14. data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +1 -1
  15. data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +6 -4
  16. data/src/main/java/com/treasuredata/api/model/TDColumn.java +4 -2
  17. data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +26 -13
  18. data/src/main/java/com/treasuredata/api/model/TDDatabase.java +2 -1
  19. data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +1 -1
  20. data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +4 -2
  21. data/src/main/java/com/treasuredata/api/model/TDTableType.java +2 -1
  22. data/src/main/java/org/embulk/output/td/FinalizableExecutorService.java +35 -17
  23. data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +13 -7
  24. data/src/main/java/org/embulk/output/td/RecordWriter.java +21 -382
  25. data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +175 -40
  26. data/src/main/java/org/embulk/output/td/writer/BooleanFieldWriter.java +23 -0
  27. data/src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java +23 -0
  28. data/src/main/java/org/embulk/output/td/writer/FieldWriter.java +38 -0
  29. data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +206 -0
  30. data/src/main/java/org/embulk/output/td/writer/LongFieldWriter.java +23 -0
  31. data/src/main/java/org/embulk/output/td/writer/StringFieldWriter.java +23 -0
  32. data/src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java +28 -0
  33. data/src/main/java/org/embulk/output/td/writer/TimestampLongFieldWriter.java +23 -0
  34. data/src/main/java/org/embulk/output/td/writer/TimestampStringFieldWriter.java +27 -0
  35. data/src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java +27 -0
  36. data/src/main/java/org/embulk/output/td/writer/UnixTimestampLongFieldWriter.java +26 -0
  37. data/src/test/java/com/treasuredata/api/TestTdApiClient.java +1 -1
  38. data/src/test/java/org/embulk/output/td/TestRecordWriter.java +198 -0
  39. data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +529 -0
  40. data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +146 -0
  41. metadata +29 -14
  42. data/src/test/java/org/embulk/output/td/TestFieldWriter.java +0 -105
@@ -9,7 +9,8 @@ public class TdApiExecutionInterruptedException
9
9
  }
10
10
 
11
11
  @Override
12
- public InterruptedException getCause() {
12
+ public InterruptedException getCause()
13
+ {
13
14
  return (InterruptedException) super.getCause();
14
15
  }
15
16
  }
@@ -11,7 +11,8 @@ public class TdApiExecutionTimeoutException
11
11
  }
12
12
 
13
13
  @Override
14
- public TimeoutException getCause() {
14
+ public TimeoutException getCause()
15
+ {
15
16
  return (TimeoutException) super.getCause();
16
17
  }
17
18
  }
@@ -20,7 +20,7 @@ public class TDArrayColumnType
20
20
  @Override
21
21
  public String toString()
22
22
  {
23
- return "array<"+elementType+">";
23
+ return "array<" + elementType + ">";
24
24
  }
25
25
 
26
26
  @Override
@@ -143,13 +143,15 @@ public class TDBulkImportSession
143
143
 
144
144
  public String getErrorMessage()
145
145
  {
146
- if (validRecords == 0)
146
+ if (validRecords == 0) {
147
147
  return "No record processed";
148
- if (errorRecords > 0)
148
+ }
149
+ if (errorRecords > 0) {
149
150
  return String.format("%d invalid parts", errorParts);
150
- if (errorRecords > 0)
151
+ }
152
+ if (errorRecords > 0) {
151
153
  return String.format("%d invalid records", errorRecords);
152
-
154
+ }
153
155
  return null;
154
156
  }
155
157
  }
@@ -43,13 +43,15 @@ public class TDColumn
43
43
  TDColumnTypeDeserializer.parseColumnType(tuple[1]),
44
44
  tuple[0].getBytes());
45
45
 
46
- } else if (tuple != null && tuple.length == 3) {
46
+ }
47
+ else if (tuple != null && tuple.length == 3) {
47
48
  return new TDColumn(
48
49
  tuple[0],
49
50
  TDColumnTypeDeserializer.parseColumnType(tuple[1]),
50
51
  tuple[2].getBytes());
51
52
 
52
- } else {
53
+ }
54
+ else {
53
55
  throw new RuntimeJsonMappingException("Unexpected string tuple to deserialize TDColumn");
54
56
  }
55
57
  }
@@ -39,19 +39,24 @@ public class TDColumnTypeDeserializer
39
39
  if (p.scan("string")) {
40
40
  return TDPrimitiveColumnType.STRING;
41
41
 
42
- } else if (p.scan("int")) {
42
+ }
43
+ else if (p.scan("int")) {
43
44
  return TDPrimitiveColumnType.INT;
44
45
 
45
- } else if (p.scan("long")) {
46
+ }
47
+ else if (p.scan("long")) {
46
48
  return TDPrimitiveColumnType.LONG;
47
49
 
48
- } else if (p.scan("double")) {
50
+ }
51
+ else if (p.scan("double")) {
49
52
  return TDPrimitiveColumnType.DOUBLE;
50
53
 
51
- } else if (p.scan("float")) {
54
+ }
55
+ else if (p.scan("float")) {
52
56
  return TDPrimitiveColumnType.FLOAT;
53
57
 
54
- } else if (p.scan("array")) {
58
+ }
59
+ else if (p.scan("array")) {
55
60
  if (!p.scan("<")) {
56
61
  throw new IllegalArgumentException("Cannot parse type: expected '<' for array type: " + p.getString());
57
62
  }
@@ -61,7 +66,8 @@ public class TDColumnTypeDeserializer
61
66
  }
62
67
  return new TDArrayColumnType(elementType);
63
68
 
64
- } else if (p.scan("map")) {
69
+ }
70
+ else if (p.scan("map")) {
65
71
  if (!p.scan("<")) {
66
72
  throw new IllegalArgumentException("Cannot parse type: expected '<' for map type: " + p.getString());
67
73
  }
@@ -75,24 +81,29 @@ public class TDColumnTypeDeserializer
75
81
  }
76
82
  return new TDMapColumnType(keyType, valueType);
77
83
 
78
- } else {
84
+ }
85
+ else {
79
86
  throw new IllegalArgumentException("Cannot parse type: " + p.getString());
80
87
  }
81
88
  }
82
89
 
83
- private static class Parser {
90
+ private static class Parser
91
+ {
84
92
  private final String string;
85
93
  private int offset;
86
94
 
87
- public Parser(String string) {
95
+ public Parser(String string)
96
+ {
88
97
  this.string = string;
89
98
  }
90
99
 
91
- public String getString() {
100
+ public String getString()
101
+ {
92
102
  return string;
93
103
  }
94
104
 
95
- public boolean scan(String s) {
105
+ public boolean scan(String s)
106
+ {
96
107
  skipSpaces();
97
108
  if (string.startsWith(s, offset)) {
98
109
  offset += s.length();
@@ -101,12 +112,14 @@ public class TDColumnTypeDeserializer
101
112
  return false;
102
113
  }
103
114
 
104
- public boolean eof() {
115
+ public boolean eof()
116
+ {
105
117
  skipSpaces();
106
118
  return string.length() <= offset;
107
119
  }
108
120
 
109
- private void skipSpaces() {
121
+ private void skipSpaces()
122
+ {
110
123
  while (string.startsWith(" ", offset)) {
111
124
  offset++;
112
125
  }
@@ -22,7 +22,8 @@ public class TDDatabase
22
22
  return name;
23
23
  }
24
24
 
25
- public boolean isWritable() {
25
+ public boolean isWritable()
26
+ {
26
27
  // TODO not implemented yet
27
28
  return true;
28
29
  }
@@ -27,7 +27,7 @@ public class TDMapColumnType
27
27
  @Override
28
28
  public String toString()
29
29
  {
30
- return "map<"+keyType+","+valueType+">";
30
+ return "map<" + keyType + "," + valueType + ">";
31
31
  }
32
32
 
33
33
  @Override
@@ -17,12 +17,14 @@ public class TDTablePermission
17
17
  }
18
18
 
19
19
  @JsonProperty("importable")
20
- public boolean isImportable() {
20
+ public boolean isImportable()
21
+ {
21
22
  return importable;
22
23
  }
23
24
 
24
25
  @JsonProperty("queryable")
25
- public boolean isQueryable() {
26
+ public boolean isQueryable()
27
+ {
26
28
  return queryable;
27
29
  }
28
30
 
@@ -21,7 +21,8 @@ public enum TDTableType
21
21
  {
22
22
  if ("log".equals(name)) {
23
23
  return LOG;
24
- } else if ("item".equals(name)) {
24
+ }
25
+ else if ("item".equals(name)) {
25
26
  return ITEM;
26
27
  }
27
28
  throw new RuntimeJsonMappingException("Unexpected string tuple to deserialize TDTableType");
@@ -17,7 +17,8 @@ public class FinalizableExecutorService
17
17
  {
18
18
  @Override
19
19
  public void close()
20
- throws IOException {
20
+ throws IOException
21
+ {
21
22
  // ignore
22
23
  }
23
24
  }
@@ -25,58 +26,75 @@ public class FinalizableExecutorService
25
26
  protected ExecutorService threads;
26
27
  protected Queue<RunningTask> runningTasks;
27
28
 
28
- public FinalizableExecutorService() {
29
+ public FinalizableExecutorService()
30
+ {
29
31
  this.threads = Executors.newCachedThreadPool();
30
32
  this.runningTasks = new LinkedList<>();
31
33
  }
32
34
 
33
- private static class RunningTask {
35
+ private static class RunningTask
36
+ {
34
37
  private Future<Void> future;
35
38
  private Closeable finalizer;
36
39
 
37
- RunningTask(Future<Void> future, Closeable finalizer) {
40
+ RunningTask(Future<Void> future, Closeable finalizer)
41
+ {
38
42
  this.future = future;
39
43
  this.finalizer = finalizer;
40
44
  }
41
45
 
42
- public void join() throws IOException {
46
+ public void join()
47
+ throws IOException
48
+ {
43
49
  try {
44
50
  future.get();
45
- } catch (InterruptedException ex) {
46
- throw new IOException(ex);
47
- } catch (ExecutionException ex) {
48
- throw new IOException(ex.getCause());
51
+ }
52
+ catch (InterruptedException e) {
53
+ throw new IOException(e);
54
+ }
55
+ catch (ExecutionException e) {
56
+ throw new IOException(e.getCause());
49
57
  }
50
58
  finalizer.close();
51
59
  }
52
60
 
53
- public void abort() throws IOException {
61
+ public void abort()
62
+ throws IOException
63
+ {
54
64
  finalizer.close();
55
65
  }
56
66
  }
57
67
 
58
- public void submit(Callable<Void> task, Closeable finalizer) {
68
+ public void submit(Callable<Void> task, Closeable finalizer)
69
+ {
59
70
  Future<Void> future = threads.submit(task);
60
71
  runningTasks.add(new RunningTask(future, finalizer));
61
72
  }
62
73
 
63
- public void joinPartial(long upto) throws IOException {
64
- while(runningTasks.size() > upto) {
74
+ public void joinPartial(long upto)
75
+ throws IOException
76
+ {
77
+ while (runningTasks.size() > upto) {
65
78
  runningTasks.peek().join();
66
79
  runningTasks.remove();
67
80
  }
68
81
  }
69
82
 
70
- public void joinAll() throws IOException {
83
+ public void joinAll()
84
+ throws IOException
85
+ {
71
86
  joinPartial(0);
72
87
  }
73
88
 
74
- public void shutdown() throws IOException {
89
+ public void shutdown()
90
+ throws IOException
91
+ {
75
92
  try {
76
93
  joinAll();
77
- } finally {
94
+ }
95
+ finally {
78
96
  threads.shutdown();
79
- for(RunningTask task : runningTasks) {
97
+ for (RunningTask task : runningTasks) {
80
98
  task.abort();
81
99
  }
82
100
  }
@@ -1,6 +1,5 @@
1
1
  package org.embulk.output.td;
2
2
 
3
- import org.embulk.spi.Exec;
4
3
  import org.msgpack.MessagePack;
5
4
  import org.msgpack.packer.Packer;
6
5
 
@@ -18,29 +17,35 @@ import static com.google.common.base.Preconditions.checkNotNull;
18
17
  public class MsgpackGZFileBuilder
19
18
  implements Closeable
20
19
  {
21
- static class DataSizeFilter extends FilterOutputStream {
20
+ static class DataSizeFilter
21
+ extends FilterOutputStream
22
+ {
22
23
  private long size = 0;
23
24
 
24
- public DataSizeFilter(OutputStream out) {
25
+ public DataSizeFilter(OutputStream out)
26
+ {
25
27
  super(out);
26
28
  }
27
29
 
28
30
  @Override
29
- public void write(int b) throws IOException
31
+ public void write(int b)
32
+ throws IOException
30
33
  {
31
34
  size += 1;
32
35
  super.write(b);
33
36
  }
34
37
 
35
38
  @Override
36
- public void write(byte[] b, int off, int len) throws IOException
39
+ public void write(byte[] b, int off, int len)
40
+ throws IOException
37
41
  {
38
42
  size += len;
39
43
  super.write(b, off, len);
40
44
  }
41
45
 
42
46
  @Override
43
- public void close() throws IOException
47
+ public void close()
48
+ throws IOException
44
49
  {
45
50
  super.close();
46
51
  }
@@ -94,7 +99,8 @@ public class MsgpackGZFileBuilder
94
99
  {
95
100
  try {
96
101
  packer.flush();
97
- } finally {
102
+ }
103
+ finally {
98
104
  close();
99
105
  }
100
106
  }
@@ -1,11 +1,12 @@
1
1
  package org.embulk.output.td;
2
2
 
3
- import com.google.common.base.Optional;
3
+ import com.google.common.annotations.VisibleForTesting;
4
4
  import com.google.common.base.Stopwatch;
5
5
  import com.google.common.base.Throwables;
6
6
  import com.treasuredata.api.TdApiClient;
7
7
  import org.embulk.config.CommitReport;
8
- import org.embulk.config.ConfigException;
8
+ import org.embulk.output.td.writer.FieldWriter;
9
+ import org.embulk.output.td.writer.FieldWriterSet;
9
10
  import org.embulk.spi.Column;
10
11
  import org.embulk.spi.ColumnVisitor;
11
12
  import org.embulk.spi.Exec;
@@ -13,16 +14,6 @@ import org.embulk.spi.Page;
13
14
  import org.embulk.spi.PageReader;
14
15
  import org.embulk.spi.Schema;
15
16
  import org.embulk.spi.TransactionalPageOutput;
16
- import org.embulk.spi.time.TimestampFormatter;
17
- import org.embulk.spi.type.BooleanType;
18
- import org.embulk.spi.type.DoubleType;
19
- import org.embulk.spi.type.LongType;
20
- import org.embulk.spi.type.StringType;
21
- import org.embulk.spi.type.TimestampType;
22
- import org.embulk.spi.type.Type;
23
- import org.embulk.spi.util.Timestamps;
24
- import org.joda.time.DateTimeZone;
25
- import org.jruby.embed.ScriptingContainer;
26
17
  import org.msgpack.MessagePack;
27
18
  import org.slf4j.Logger;
28
19
 
@@ -91,6 +82,12 @@ public class RecordWriter
91
82
  this.builder = new MsgpackGZFileBuilder(msgpack, tempFile);
92
83
  }
93
84
 
85
+ @VisibleForTesting
86
+ MsgpackGZFileBuilder getBuilder()
87
+ {
88
+ return builder;
89
+ }
90
+
94
91
  @Override
95
92
  public void add(final Page page)
96
93
  {
@@ -136,7 +133,8 @@ public class RecordWriter
136
133
  FieldWriter fieldWriter = fieldWriters.getFieldWriter(column.getIndex());
137
134
  try {
138
135
  fieldWriter.writeKeyValue(builder, pageReader, column);
139
- } catch (IOException e) {
136
+ }
137
+ catch (IOException e) {
140
138
  throw Throwables.propagate(e);
141
139
  }
142
140
  }
@@ -150,7 +148,8 @@ public class RecordWriter
150
148
  }
151
149
  }
152
150
 
153
- } catch (IOException e) {
151
+ }
152
+ catch (IOException e) {
154
153
  throw Throwables.propagate(e);
155
154
  }
156
155
  }
@@ -206,9 +205,11 @@ public class RecordWriter
206
205
  {
207
206
  try {
208
207
  flush();
209
- } catch (IOException e) {
208
+ }
209
+ catch (IOException e) {
210
210
  throw Throwables.propagate(e);
211
- } finally {
211
+ }
212
+ finally {
212
213
  close();
213
214
  }
214
215
  }
@@ -220,7 +221,8 @@ public class RecordWriter
220
221
  try {
221
222
  executor.joinAll();
222
223
  executor.shutdown(); // shutdown calls joinAll
223
- } finally {
224
+ }
225
+ finally {
224
226
  if (builder != null) {
225
227
  builder.close();
226
228
  builder.delete();
@@ -231,7 +233,8 @@ public class RecordWriter
231
233
  client.close();
232
234
  }
233
235
  }
234
- } catch (IOException e) {
236
+ }
237
+ catch (IOException e) {
235
238
  throw Throwables.propagate(e);
236
239
  }
237
240
  }
@@ -249,368 +252,4 @@ public class RecordWriter
249
252
  // TODO
250
253
  return report;
251
254
  }
252
-
253
- static class FieldWriterSet
254
- {
255
- private enum ColumnWriterMode
256
- {
257
- PRIMARY_KEY,
258
- SIMPLE_VALUE,
259
- DUPLICATE_PRIMARY_KEY;
260
- }
261
-
262
- private final int fieldCount;
263
- private final FieldWriter[] fieldWriters;
264
-
265
- public FieldWriterSet(Logger log, TdOutputPlugin.PluginTask task, Schema schema)
266
- {
267
- Optional<String> userDefinedPrimaryKeySourceColumnName = task.getTimeColumn();
268
- boolean hasPkWriter = false;
269
- int duplicatePrimaryKeySourceIndex = -1;
270
- int firstTimestampColumnIndex = -1;
271
-
272
- int fc = 0;
273
- fieldWriters = new FieldWriter[schema.size()];
274
- TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
275
-
276
- for (int i = 0; i < schema.size(); i++) {
277
- String columnName = schema.getColumnName(i);
278
- Type columnType = schema.getColumnType(i);
279
-
280
- // choose the mode
281
- final ColumnWriterMode mode;
282
-
283
- if (userDefinedPrimaryKeySourceColumnName.isPresent() &&
284
- columnName.equals(userDefinedPrimaryKeySourceColumnName.get())) {
285
- // found time_column
286
- if ("time".equals(userDefinedPrimaryKeySourceColumnName.get())) {
287
- mode = ColumnWriterMode.PRIMARY_KEY;
288
- } else {
289
- mode = ColumnWriterMode.DUPLICATE_PRIMARY_KEY;
290
- }
291
-
292
- } else if ("time".equals(columnName)) {
293
- // the column name is same with the primary key name.
294
- if (userDefinedPrimaryKeySourceColumnName.isPresent()) {
295
- columnName = newColumnUniqueName(columnName, schema);
296
- mode = ColumnWriterMode.SIMPLE_VALUE;
297
- log.warn("time_column '{}' is set but 'time' column also exists. The existent 'time' column is renamed to {}",
298
- userDefinedPrimaryKeySourceColumnName.get(), "time", "time", columnName);
299
- } else {
300
- mode = ColumnWriterMode.PRIMARY_KEY;
301
- }
302
-
303
- } else {
304
- mode = ColumnWriterMode.SIMPLE_VALUE;
305
- }
306
-
307
- // create the fieldWriters writer depending on the mode
308
- final FieldWriter writer;
309
-
310
- switch (mode) {
311
- case PRIMARY_KEY:
312
- log.info("Using {}:{} column as the data partitioning key", columnName, columnType);
313
- if (columnType instanceof LongType) {
314
- if (task.getUnixTimestampUnit() != TdOutputPlugin.UnixTimestampUnit.SEC) {
315
- log.warn("time column is converted from {} to seconds", task.getUnixTimestampUnit());
316
- }
317
- writer = new UnixTimestampLongFieldWriter(columnName, task.getUnixTimestampUnit().getFractionUnit());
318
- hasPkWriter = true;
319
- } else if (columnType instanceof TimestampType) {
320
- writer = new TimestampLongFieldWriter(columnName);
321
-
322
- hasPkWriter = true;
323
- } else {
324
- throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
325
- columnName, columnType));
326
- }
327
- break;
328
-
329
- case SIMPLE_VALUE:
330
- if (columnType instanceof BooleanType) {
331
- writer = new BooleanFieldWriter(columnName);
332
- } else if (columnType instanceof LongType) {
333
- writer = new LongFieldWriter(columnName);
334
- } else if (columnType instanceof DoubleType) {
335
- writer = new DoubleFieldWriter(columnName);
336
- } else if (columnType instanceof StringType) {
337
- writer = new StringFieldWriter(columnName);
338
- } else if (columnType instanceof TimestampType) {
339
- writer = new TimestampStringFieldWriter(timestampFormatters[i], columnName);
340
- if (firstTimestampColumnIndex < 0) {
341
- firstTimestampColumnIndex = i;
342
- }
343
- } else {
344
- throw new ConfigException("Unsupported type: " + columnType);
345
- }
346
- break;
347
-
348
- case DUPLICATE_PRIMARY_KEY:
349
- duplicatePrimaryKeySourceIndex = i;
350
- writer = null; // handle later
351
- break;
352
-
353
- default:
354
- throw new AssertionError();
355
- }
356
-
357
- fieldWriters[i] = writer;
358
- fc += 1;
359
- }
360
-
361
- if (!hasPkWriter) {
362
- // PRIMARY_KEY was not found.
363
- if (duplicatePrimaryKeySourceIndex < 0) {
364
- if (userDefinedPrimaryKeySourceColumnName.isPresent()) {
365
- throw new ConfigException(String.format("time_column '%s' does not exist", userDefinedPrimaryKeySourceColumnName.get()));
366
- } else if (firstTimestampColumnIndex >= 0) {
367
- // if time is not found, use the first timestamp column
368
- duplicatePrimaryKeySourceIndex = firstTimestampColumnIndex;
369
- } else {
370
- throw new ConfigException(String.format("TD output plugin requires at least one timestamp column, or a long column named 'time'"));
371
- }
372
- }
373
-
374
- String columnName = schema.getColumnName(duplicatePrimaryKeySourceIndex);
375
- Type columnType = schema.getColumnType(duplicatePrimaryKeySourceIndex);
376
-
377
- FieldWriter writer;
378
- if (columnType instanceof LongType) {
379
- log.info("Duplicating {}:{} column (unix timestamp {}) to 'time' column as seconds for the data partitioning",
380
- columnName, columnType, task.getUnixTimestampUnit());
381
- writer = new UnixTimestampFieldDuplicator(columnName, "time", task.getUnixTimestampUnit().getFractionUnit());
382
- } else if (columnType instanceof TimestampType) {
383
- log.info("Duplicating {}:{} column to 'time' column as seconds for the data partitioning",
384
- columnName, columnType);
385
- writer = new TimestampFieldLongDuplicator(timestampFormatters[duplicatePrimaryKeySourceIndex], columnName, "time");
386
- } else {
387
- throw new ConfigException(String.format("Type of '%s' column must be long or timestamp but got %s",
388
- columnName, columnType));
389
- }
390
-
391
- // replace existint writer
392
- fieldWriters[duplicatePrimaryKeySourceIndex] = writer;
393
- fc += 1;
394
- }
395
-
396
- fieldCount = fc;
397
- }
398
-
399
- private static String newColumnUniqueName(String originalName, Schema schema)
400
- {
401
- String name = originalName;
402
- do {
403
- name += "_";
404
- } while (containsColumnName(schema, name));
405
- return name;
406
- }
407
-
408
- private static boolean containsColumnName(Schema schema, String name)
409
- {
410
- for (Column c : schema.getColumns()) {
411
- if (c.getName().equals(name)) {
412
- return true;
413
- }
414
- }
415
- return false;
416
- }
417
-
418
- public FieldWriter getFieldWriter(int index)
419
- {
420
- return fieldWriters[index];
421
- }
422
-
423
- public int getFieldCount()
424
- {
425
- return fieldCount;
426
- }
427
- }
428
-
429
- static abstract class FieldWriter
430
- {
431
- private final String keyName;
432
-
433
- protected FieldWriter(String keyName)
434
- {
435
- this.keyName = keyName;
436
- }
437
-
438
- public void writeKeyValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
439
- throws IOException
440
- {
441
- writeKey(builder);
442
- if (reader.isNull(column)) {
443
- builder.writeNil();
444
- } else {
445
- writeValue(builder, reader, column);
446
- }
447
- }
448
-
449
- private void writeKey(MsgpackGZFileBuilder builder)
450
- throws IOException
451
- {
452
- builder.writeString(keyName);
453
- }
454
-
455
- protected abstract void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
456
- throws IOException;
457
- }
458
-
459
- static class DoubleFieldWriter
460
- extends FieldWriter
461
- {
462
- public DoubleFieldWriter(String keyName)
463
- {
464
- super(keyName);
465
- }
466
-
467
- @Override
468
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
469
- throws IOException
470
- {
471
- builder.writeDouble(reader.getDouble(column));
472
- }
473
- }
474
-
475
- static class BooleanFieldWriter
476
- extends FieldWriter
477
- {
478
- public BooleanFieldWriter(String keyName)
479
- {
480
- super(keyName);
481
- }
482
-
483
- @Override
484
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
485
- throws IOException
486
- {
487
- builder.writeBoolean(reader.getBoolean(column));
488
- }
489
- }
490
-
491
- static class LongFieldWriter
492
- extends FieldWriter
493
- {
494
- LongFieldWriter(String keyName)
495
- {
496
- super(keyName);
497
- }
498
-
499
- @Override
500
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
501
- throws IOException
502
- {
503
- builder.writeLong(reader.getLong(column));
504
- }
505
- }
506
-
507
- static class UnixTimestampLongFieldWriter
508
- extends FieldWriter
509
- {
510
- private final int fractionUnit;
511
-
512
- UnixTimestampLongFieldWriter(String keyName, int fractionUnit)
513
- {
514
- super(keyName);
515
- this.fractionUnit = fractionUnit;
516
- }
517
-
518
- @Override
519
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
520
- throws IOException
521
- {
522
- builder.writeLong(reader.getLong(column) / fractionUnit);
523
- }
524
- }
525
-
526
- static class StringFieldWriter
527
- extends FieldWriter
528
- {
529
- public StringFieldWriter(String keyName)
530
- {
531
- super(keyName);
532
- }
533
-
534
- @Override
535
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
536
- throws IOException
537
- {
538
- builder.writeString(reader.getString(column));
539
- }
540
- }
541
-
542
- static class TimestampStringFieldWriter
543
- extends FieldWriter
544
- {
545
- private final TimestampFormatter formatter;
546
-
547
- public TimestampStringFieldWriter(TimestampFormatter formatter, String keyName)
548
- {
549
- super(keyName);
550
- this.formatter = formatter;
551
- }
552
-
553
- @Override
554
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
555
- throws IOException
556
- {
557
- builder.writeString(formatter.format(reader.getTimestamp(column)));
558
- }
559
- }
560
-
561
- static class TimestampLongFieldWriter
562
- extends FieldWriter
563
- {
564
- public TimestampLongFieldWriter(String keyName)
565
- {
566
- super(keyName);
567
- }
568
-
569
- @Override
570
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
571
- throws IOException
572
- {
573
- builder.writeLong(reader.getTimestamp(column).getEpochSecond());
574
- }
575
- }
576
-
577
- static class UnixTimestampFieldDuplicator
578
- extends LongFieldWriter
579
- {
580
- private final UnixTimestampLongFieldWriter timeFieldWriter;
581
-
582
- public UnixTimestampFieldDuplicator(String keyName, String duplicateKeyName, int fractionUnit)
583
- {
584
- super(keyName);
585
- timeFieldWriter = new UnixTimestampLongFieldWriter(duplicateKeyName, fractionUnit);
586
- }
587
-
588
- @Override
589
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
590
- throws IOException
591
- {
592
- super.writeValue(builder, reader, column);
593
- timeFieldWriter.writeKeyValue(builder, reader, column);
594
- }
595
- }
596
-
597
- static class TimestampFieldLongDuplicator
598
- extends TimestampStringFieldWriter
599
- {
600
- private final TimestampLongFieldWriter timeFieldWriter;
601
-
602
- public TimestampFieldLongDuplicator(TimestampFormatter formatter, String keyName, String longDuplicateKeyName)
603
- {
604
- super(formatter, keyName);
605
- timeFieldWriter = new TimestampLongFieldWriter(longDuplicateKeyName);
606
- }
607
-
608
- @Override
609
- public void writeValue(MsgpackGZFileBuilder builder, PageReader reader, Column column)
610
- throws IOException
611
- {
612
- super.writeValue(builder, reader, column);
613
- timeFieldWriter.writeKeyValue(builder, reader, column);
614
- }
615
- }
616
255
  }