embulk-output-td 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +7 -0
  3. data/CHANGELOG.md +4 -0
  4. data/README.md +1 -0
  5. data/build.gradle +5 -1
  6. data/config/checkstyle/checkstyle.xml +117 -0
  7. data/embulk-output-td.gemspec +1 -1
  8. data/gradle/check.gradle +34 -0
  9. data/src/main/java/com/treasuredata/api/TdApiClient.java +47 -23
  10. data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +3 -3
  11. data/src/main/java/com/treasuredata/api/TdApiConstants.java +6 -2
  12. data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +2 -1
  13. data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +2 -1
  14. data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +1 -1
  15. data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +6 -4
  16. data/src/main/java/com/treasuredata/api/model/TDColumn.java +4 -2
  17. data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +26 -13
  18. data/src/main/java/com/treasuredata/api/model/TDDatabase.java +2 -1
  19. data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +1 -1
  20. data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +4 -2
  21. data/src/main/java/com/treasuredata/api/model/TDTableType.java +2 -1
  22. data/src/main/java/org/embulk/output/td/FinalizableExecutorService.java +35 -17
  23. data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +13 -7
  24. data/src/main/java/org/embulk/output/td/RecordWriter.java +21 -382
  25. data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +175 -40
  26. data/src/main/java/org/embulk/output/td/writer/BooleanFieldWriter.java +23 -0
  27. data/src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java +23 -0
  28. data/src/main/java/org/embulk/output/td/writer/FieldWriter.java +38 -0
  29. data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +206 -0
  30. data/src/main/java/org/embulk/output/td/writer/LongFieldWriter.java +23 -0
  31. data/src/main/java/org/embulk/output/td/writer/StringFieldWriter.java +23 -0
  32. data/src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java +28 -0
  33. data/src/main/java/org/embulk/output/td/writer/TimestampLongFieldWriter.java +23 -0
  34. data/src/main/java/org/embulk/output/td/writer/TimestampStringFieldWriter.java +27 -0
  35. data/src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java +27 -0
  36. data/src/main/java/org/embulk/output/td/writer/UnixTimestampLongFieldWriter.java +26 -0
  37. data/src/test/java/com/treasuredata/api/TestTdApiClient.java +1 -1
  38. data/src/test/java/org/embulk/output/td/TestRecordWriter.java +198 -0
  39. data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +529 -0
  40. data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +146 -0
  41. metadata +29 -14
  42. data/src/test/java/org/embulk/output/td/TestFieldWriter.java +0 -105
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 666d33ca9f399f704557254a12fe57a64cea15e1
4
- data.tar.gz: 39f9653363277e171d68efea0c522f8f1df7b823
3
+ metadata.gz: 8cc6a923fccf05a7db159155ae97c69a09726a56
4
+ data.tar.gz: 15c9772dc65e9b6d84d94b1ad3a1f3feb6c8fb14
5
5
  SHA512:
6
- metadata.gz: 2f369e3e6b25df31d0275f17006a5670eab7b73ed495a4dd2240253d13b58bed2481cfe1f3d03e2926bf1c18aefc6477b39320f120cadc1c484239fa85563892
7
- data.tar.gz: d426f8baff389eaa8fb81e1f1b9692d7587a45bb0d39bfe8dbf05e21c827d71b3800d075d1ec4a11bb224c94858ed49d9d3b8e6f2a2ca3d9d04a4314cb6da539
6
+ metadata.gz: df93720dd047f60469594ee432d1cd48f2b8039fcad6921e065e996637c5bab535fbb6aa5bdd01895f5db2852a042683b85b82bc77ded8aca3ff85a49d296a54
7
+ data.tar.gz: 9a222a60dba6bb17121ef3911a0b552b5a3f744452dd56a71867f64f4ae54e0e2182260e51b8ffc3accebbd2b77125694b1c007be4218fe7f961397eb9dc4033
data/.travis.yml ADDED
@@ -0,0 +1,7 @@
1
+ language: java
2
+ jdk: oraclejdk7
3
+
4
+ script: ./gradlew clean test
5
+
6
+ after_success:
7
+ - ./gradlew check jacocoRootReport
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 0.1.5 - 2015-09-29
2
+
3
+ * [new feature] Add replace mode [#17](https://github.com/treasure-data/embulk-output-td/pull/17)
4
+
1
5
  ## 0.1.4 - 2015-08-17
2
6
 
3
7
  * [maintenance] Upgrade Embulk v0.6.25
data/README.md CHANGED
@@ -15,6 +15,7 @@ TODO: Write short description here
15
15
  - **http_proxy**: http proxy configuration (tuple of host, port and useSsl, default is null)
16
16
  - **use_ssl**: the flag (boolean, default=true)
17
17
  - **auto_create_table**: the flag for creating the database and/or the table if they don't exist (boolean, default=true)
18
+ - **mode**: two ways to modify data [append, replace] (string, default='append')
18
19
  - **database**: database name (string, required)
19
20
  - **table**: table name (string, required)
20
21
  - **session**: bulk_import session name (string, optional)
data/build.gradle CHANGED
@@ -3,6 +3,9 @@ plugins {
3
3
  id "com.github.jruby-gradle.base" version "0.1.5"
4
4
  id "java"
5
5
  }
6
+
7
+ apply from: 'gradle/check.gradle'
8
+
6
9
  import com.github.jrubygradle.JRubyExec
7
10
  repositories {
8
11
  mavenCentral()
@@ -13,7 +16,7 @@ configurations {
13
16
  provided
14
17
  }
15
18
 
16
- version = "0.1.4"
19
+ version = "0.1.5"
17
20
 
18
21
  compileJava.options.encoding = 'UTF-8' // source encoding
19
22
  sourceCompatibility = 1.7
@@ -30,6 +33,7 @@ dependencies {
30
33
  testCompile "junit:junit:4.+"
31
34
  testCompile "org.bigtesting:fixd:1.0.0"
32
35
  testCompile "org.embulk:embulk-core:0.6.25:tests"
36
+ testCompile "org.mockito:mockito-core:1.9.5"
33
37
  }
34
38
 
35
39
  task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -0,0 +1,117 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <module name="Checker">
6
+ <module name="FileTabCharacter"/>
7
+ <module name="NewlineAtEndOfFile">
8
+ <property name="lineSeparator" value="lf"/>
9
+ </module>
10
+ <module name="RegexpMultiline">
11
+ <property name="format" value="\r"/>
12
+ <property name="message" value="Line contains carriage return"/>
13
+ </module>
14
+ <module name="RegexpMultiline">
15
+ <property name="format" value=" \n"/>
16
+ <property name="message" value="Line has trailing whitespace"/>
17
+ </module>
18
+ <module name="RegexpMultiline">
19
+ <property name="format" value="\{\n\n"/>
20
+ <property name="message" value="Blank line after opening brace"/>
21
+ </module>
22
+ <module name="RegexpMultiline">
23
+ <property name="format" value="\n\n\}"/>
24
+ <property name="message" value="Blank line before closing brace"/>
25
+ </module>
26
+ <module name="RegexpMultiline">
27
+ <property name="format" value="\n\n\n"/>
28
+ <property name="message" value="Multiple consecutive blank lines"/>
29
+ </module>
30
+ <module name="RegexpMultiline">
31
+ <property name="format" value="\n\n\Z"/>
32
+ <property name="message" value="Blank line before end of file"/>
33
+ </module>
34
+
35
+ <module name="TreeWalker">
36
+ <module name="EmptyBlock">
37
+ <property name="option" value="text"/>
38
+ <property name="tokens" value="
39
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
40
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
41
+ </module>
42
+ <module name="EmptyStatement"/>
43
+ <module name="EmptyForInitializerPad"/>
44
+ <module name="EmptyForIteratorPad">
45
+ <property name="option" value="space"/>
46
+ </module>
47
+ <module name="MethodParamPad">
48
+ <property name="allowLineBreaks" value="true"/>
49
+ <property name="option" value="nospace"/>
50
+ </module>
51
+ <module name="ParenPad"/>
52
+ <module name="TypecastParenPad"/>
53
+ <module name="NeedBraces"/>
54
+ <module name="LeftCurly">
55
+ <property name="option" value="nl"/>
56
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
57
+ </module>
58
+ <module name="LeftCurly">
59
+ <property name="option" value="eol"/>
60
+ <property name="tokens" value="
61
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
62
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
63
+ </module>
64
+ <module name="RightCurly">
65
+ <property name="option" value="alone"/>
66
+ </module>
67
+ <module name="GenericWhitespace"/>
68
+ <module name="WhitespaceAfter"/>
69
+ <module name="NoWhitespaceBefore"/>
70
+
71
+ <module name="UpperEll"/>
72
+ <module name="DefaultComesLast"/>
73
+ <module name="ArrayTypeStyle"/>
74
+ <module name="MultipleVariableDeclarations"/>
75
+ <module name="ModifierOrder"/>
76
+ <module name="OneStatementPerLine"/>
77
+ <module name="StringLiteralEquality"/>
78
+ <module name="MutableException"/>
79
+ <module name="EqualsHashCode"/>
80
+ <module name="InnerAssignment"/>
81
+ <module name="InterfaceIsType"/>
82
+ <module name="HideUtilityClassConstructor"/>
83
+
84
+ <module name="MemberName"/>
85
+ <module name="LocalVariableName"/>
86
+ <module name="LocalFinalVariableName"/>
87
+ <module name="TypeName"/>
88
+ <module name="PackageName"/>
89
+ <module name="ParameterName"/>
90
+ <module name="StaticVariableName"/>
91
+ <module name="ClassTypeParameterName">
92
+ <property name="format" value="^[A-Z][0-9]?$"/>
93
+ </module>
94
+ <module name="MethodTypeParameterName">
95
+ <property name="format" value="^[A-Z][0-9]?$"/>
96
+ </module>
97
+
98
+ <module name="AvoidStarImport"/>
99
+ <module name="RedundantImport"/>
100
+ <module name="UnusedImports"/>
101
+
102
+ <module name="WhitespaceAround">
103
+ <property name="allowEmptyConstructors" value="true"/>
104
+ <property name="allowEmptyMethods" value="true"/>
105
+ <property name="ignoreEnhancedForColon" value="false"/>
106
+ <property name="tokens" value="
107
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
108
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
109
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
110
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
111
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
112
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
113
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
114
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
115
+ </module>
116
+ </module>
117
+ </module>
@@ -1,7 +1,7 @@
1
1
 
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "embulk-output-td"
4
- spec.version = "0.1.4"
4
+ spec.version = "0.1.5"
5
5
  spec.authors = ["Muga Nishizawa"]
6
6
  spec.summary = %[TreasureData output plugin for Embulk]
7
7
  spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
@@ -0,0 +1,34 @@
1
+ // Checkstyle
2
+ // @see https://docs.gradle.org/2.5/userguide/checkstyle_plugin.html
3
+ apply plugin: 'checkstyle'
4
+ checkstyle {
5
+ ignoreFailures = true
6
+ // @see https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml
7
+ //configFile = rootProject.file('./checkstyle.xml') // default {project.projectDir}/config/checkstyle/checkstyle.xml
8
+ }
9
+
10
+ // FindBugs
11
+ // @see https://docs.gradle.org/2.5/userguide/findbugs_plugin.html
12
+ apply plugin: 'findbugs'
13
+ findbugs {
14
+ ignoreFailures = true
15
+ }
16
+
17
+ // PMD
18
+ // @see https://docs.gradle.org/2.5/userguide/pmd_plugin.html
19
+ apply plugin: 'pmd'
20
+ tasks.withType(Pmd) {
21
+ ignoreFailures = true
22
+ reports.html.enabled true
23
+ }
24
+
25
+ // JaCoCo
26
+ // @see https://docs.gradle.org/2.5/userguide/jacoco_plugin.html
27
+ apply plugin: 'jacoco'
28
+ jacocoTestReport { // will use td-client v0.6.x
29
+ afterEvaluate {
30
+ classDirectories = files(classDirectories.files.collect {
31
+ fileTree(dir: it, exclude: 'com/treasuredata/api/**')
32
+ })
33
+ }
34
+ }
@@ -91,7 +91,8 @@ public class TdApiClient
91
91
  {
92
92
  try {
93
93
  http.start();
94
- } catch (Exception e) {
94
+ }
95
+ catch (Exception e) {
95
96
  throw new IOException("Failed to start http client", e);
96
97
  }
97
98
  }
@@ -101,7 +102,8 @@ public class TdApiClient
101
102
  {
102
103
  try {
103
104
  http.stop();
104
- } catch (Exception e) {
105
+ }
106
+ catch (Exception e) {
105
107
  throw new RuntimeException("Failed to stop http client", e);
106
108
  }
107
109
  }
@@ -138,7 +140,8 @@ public class TdApiClient
138
140
  return createTable(databaseName, tableName, TDTableType.LOG);
139
141
  }
140
142
 
141
- public TDTable createTable(String databaseName, String tableName, TDTableType tableType) {
143
+ public TDTable createTable(String databaseName, String tableName, TDTableType tableType)
144
+ {
142
145
  Request request = prepareExchange(HttpMethod.POST,
143
146
  buildUrl("/v3/table/create", databaseName, tableName, tableType.name().toLowerCase()));
144
147
  ContentResponse response = executeExchange(request);
@@ -154,6 +157,13 @@ public class TdApiClient
154
157
  ContentResponse response = executeExchange(request);
155
158
  }
156
159
 
160
+ public void renameTable(String databaseName, String oldName, String newName)
161
+ {
162
+ Request request = prepareExchange(HttpMethod.POST,
163
+ buildUrl("/v3/table/rename", databaseName, oldName, newName));
164
+ ContentResponse response = executeExchange(request);
165
+ }
166
+
157
167
  public void createBulkImportSession(String sessionName, String databaseName, String tableName)
158
168
  {
159
169
  Request request = prepareExchange(HttpMethod.POST,
@@ -223,8 +233,8 @@ public class TdApiClient
223
233
 
224
234
  private Request prepareExchange(HttpMethod method, String url)
225
235
  {
226
- return prepareExchange(method, url, Collections.<String,String>emptyMap(),
227
- Collections.<String,String>emptyMap());
236
+ return prepareExchange(method, url, Collections.<String, String>emptyMap(),
237
+ Collections.<String, String>emptyMap());
228
238
  }
229
239
 
230
240
  private Request prepareExchange(HttpMethod method,
@@ -248,7 +258,7 @@ public class TdApiClient
248
258
  request.agent(config.getAgentName());
249
259
  request.header("Authorization", "TD1 " + apiKey);
250
260
  //request.timeout(60, TimeUnit.SECONDS);
251
- for (Map.Entry<String, String> entry: headers.entrySet()) {
261
+ for (Map.Entry<String, String> entry : headers.entrySet()) {
252
262
  request.header(entry.getKey(), entry.getValue());
253
263
  }
254
264
  String dateHeader = setDateHeader(request);
@@ -264,7 +274,8 @@ public class TdApiClient
264
274
  {
265
275
  try {
266
276
  return URLEncoder.encode(s, "UTF-8");
267
- } catch (UnsupportedEncodingException e) {
277
+ }
278
+ catch (UnsupportedEncodingException e) {
268
279
  throw new AssertionError(e);
269
280
  }
270
281
  }
@@ -272,7 +283,8 @@ public class TdApiClient
272
283
  private static final ThreadLocal<SimpleDateFormat> RFC2822_FORMAT =
273
284
  new ThreadLocal<SimpleDateFormat>() {
274
285
  @Override
275
- protected SimpleDateFormat initialValue() {
286
+ protected SimpleDateFormat initialValue()
287
+ {
276
288
  return new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss Z", Locale.ENGLISH);
277
289
  }
278
290
  };
@@ -288,10 +300,12 @@ public class TdApiClient
288
300
  private static final ThreadLocal<MessageDigest> SHA1 =
289
301
  new ThreadLocal<MessageDigest>() {
290
302
  @Override
291
- protected MessageDigest initialValue() {
303
+ protected MessageDigest initialValue()
304
+ {
292
305
  try {
293
306
  return MessageDigest.getInstance("SHA-1");
294
- } catch (NoSuchAlgorithmException e) {
307
+ }
308
+ catch (NoSuchAlgorithmException e) {
295
309
  throw new RuntimeException("SHA-1 digest algorithm must be available but not found", e);
296
310
  }
297
311
  }
@@ -316,8 +330,8 @@ public class TdApiClient
316
330
  char[] array = new char[bytes.length * 2];
317
331
  for (int i = 0; i < bytes.length; i++) {
318
332
  int b = (int) bytes[i];
319
- array[i*2] = hexChars[(b & 0xf0) >> 4];
320
- array[i*2+1] = hexChars[b & 0x0f];
333
+ array[i * 2] = hexChars[(b & 0xf0) >> 4];
334
+ array[i * 2 + 1] = hexChars[b & 0x0f];
321
335
  }
322
336
  return new String(array);
323
337
  }
@@ -333,7 +347,8 @@ public class TdApiClient
333
347
  sb.append("/");
334
348
  sb.append(URLEncoder.encode(param, "UTF-8"));
335
349
  }
336
- } catch (UnsupportedEncodingException ex) {
350
+ }
351
+ catch (UnsupportedEncodingException ex) {
337
352
  throw new AssertionError(ex);
338
353
  }
339
354
  return sb.toString();
@@ -370,10 +385,12 @@ public class TdApiClient
370
385
  // retry on 50x and other errors
371
386
  exception = new TdApiResponseException(status, response.getContent());
372
387
 
373
- } catch (TdApiException e) {
388
+ }
389
+ catch (TdApiException e) {
374
390
  throw e;
375
391
 
376
- } catch (Exception e) {
392
+ }
393
+ catch (Exception e) {
377
394
  // retry on RuntimeException
378
395
  exception = e;
379
396
  }
@@ -393,7 +410,8 @@ public class TdApiClient
393
410
  Thread.sleep(retryWait);
394
411
  retryWait *= 2;
395
412
  }
396
- } catch (InterruptedException e) {
413
+ }
414
+ catch (InterruptedException e) {
397
415
  throw new TdApiExecutionInterruptedException(e);
398
416
  }
399
417
  }
@@ -404,14 +422,18 @@ public class TdApiClient
404
422
  try {
405
423
  objectMapper.writeValue(bo, obj);
406
424
  return new String(bo.toByteArray(), "UTF-8");
407
- } catch (UnsupportedEncodingException ex) {
425
+ }
426
+ catch (UnsupportedEncodingException ex) {
408
427
  throw new AssertionError(ex);
409
- } catch (IOException ex) {
428
+ }
429
+ catch (IOException ex) {
410
430
  throw new RuntimeException(ex);
411
- } finally {
431
+ }
432
+ finally {
412
433
  try {
413
434
  bo.close();
414
- } catch (IOException ex) {
435
+ }
436
+ catch (IOException ex) {
415
437
  throw new RuntimeException(ex);
416
438
  }
417
439
  }
@@ -421,7 +443,7 @@ public class TdApiClient
421
443
  {
422
444
  try {
423
445
  StringBuilder sb = new StringBuilder();
424
- for(int i=0; i < kvs.length; i+=2) {
446
+ for (int i = 0; i < kvs.length; i += 2) {
425
447
  if (i > 0) {
426
448
  sb.append("&");
427
449
  }
@@ -430,7 +452,8 @@ public class TdApiClient
430
452
  .append(encode(kvs[i + 1]));
431
453
  }
432
454
  return sb.toString().getBytes("UTF-8");
433
- } catch (UnsupportedEncodingException ex) {
455
+ }
456
+ catch (UnsupportedEncodingException ex) {
434
457
  throw new AssertionError(ex);
435
458
  }
436
459
  }
@@ -439,7 +462,8 @@ public class TdApiClient
439
462
  {
440
463
  try {
441
464
  return objectMapper.readValue(content, valueType);
442
- } catch (IOException ex) {
465
+ }
466
+ catch (IOException ex) {
443
467
  throw new RuntimeException(ex);
444
468
  }
445
469
  }
@@ -3,7 +3,6 @@ package com.treasuredata.api;
3
3
  import com.google.common.base.Optional;
4
4
 
5
5
  public class TdApiClientConfig
6
- implements TdApiConstants
7
6
  {
8
7
  public static class HttpProxyConfig
9
8
  {
@@ -73,7 +72,8 @@ public class TdApiClientConfig
73
72
  return httpProxyConfig;
74
73
  }
75
74
 
76
- public String getAgentName() {
77
- return AGENT_NAME;
75
+ public String getAgentName()
76
+ {
77
+ return TdApiConstants.AGENT_NAME;
78
78
  }
79
79
  }
@@ -1,6 +1,10 @@
1
1
  package com.treasuredata.api;
2
2
 
3
- public interface TdApiConstants
3
+ public final class TdApiConstants
4
4
  {
5
- String AGENT_NAME = "TdApiClient v0.6";
5
+ public static final String AGENT_NAME = "TdApiClient v0.6";
6
+
7
+ private TdApiConstants()
8
+ {
9
+ }
6
10
  }