embulk-output-td 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 666d33ca9f399f704557254a12fe57a64cea15e1
-  data.tar.gz: 39f9653363277e171d68efea0c522f8f1df7b823
+  metadata.gz: 8cc6a923fccf05a7db159155ae97c69a09726a56
+  data.tar.gz: 15c9772dc65e9b6d84d94b1ad3a1f3feb6c8fb14
 SHA512:
-  metadata.gz: 2f369e3e6b25df31d0275f17006a5670eab7b73ed495a4dd2240253d13b58bed2481cfe1f3d03e2926bf1c18aefc6477b39320f120cadc1c484239fa85563892
-  data.tar.gz: d426f8baff389eaa8fb81e1f1b9692d7587a45bb0d39bfe8dbf05e21c827d71b3800d075d1ec4a11bb224c94858ed49d9d3b8e6f2a2ca3d9d04a4314cb6da539
+  metadata.gz: df93720dd047f60469594ee432d1cd48f2b8039fcad6921e065e996637c5bab535fbb6aa5bdd01895f5db2852a042683b85b82bc77ded8aca3ff85a49d296a54
+  data.tar.gz: 9a222a60dba6bb17121ef3911a0b552b5a3f744452dd56a71867f64f4ae54e0e2182260e51b8ffc3accebbd2b77125694b1c007be4218fe7f961397eb9dc4033

data/.travis.yml

@@ -0,0 +1,7 @@
+language: java
+jdk: oraclejdk7
+
+script: ./gradlew clean test
+
+after_success:
+- ./gradlew check jacocoRootReport

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.5 - 2015-09-29
+
+* [new feature] Add replace mode [#17](https://github.com/treasure-data/embulk-output-td/pull/17)
+
 ## 0.1.4 - 2015-08-17
 
 * [maintenance] Upgrade Embulk v0.6.25

data/README.md

@@ -15,6 +15,7 @@ TODO: Write short description here
 - **http_proxy**: http proxy configuration (tuple of host, port and useSsl, default is null)
 - **use_ssl**: the flag (boolean, default=true)
 - **auto_create_table**: the flag for creating the database and/or the table if they don't exist (boolean, default=true)
+- **mode**: two ways to modify data [append, replace] (string, default='append')
 - **database**: database name (string, required)
 - **table**: table name (string, required)
 - **session**: bulk_import session name (string, optional)

data/build.gradle

@@ -3,6 +3,9 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
 }
+
+apply from: 'gradle/check.gradle'
+
 import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
@@ -13,7 +16,7 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.5"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7
@@ -30,6 +33,7 @@ dependencies {
     testCompile "junit:junit:4.+"
     testCompile "org.bigtesting:fixd:1.0.0"
     testCompile  "org.embulk:embulk-core:0.6.25:tests"
+    testCompile  "org.mockito:mockito-core:1.9.5"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/config/checkstyle/checkstyle.xml

@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.1.4"
+  spec.version       = "0.1.5"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/gradle/check.gradle

@@ -0,0 +1,34 @@
+// Checkstyle
+// @see https://docs.gradle.org/2.5/userguide/checkstyle_plugin.html
+apply plugin: 'checkstyle'
+checkstyle {
+    ignoreFailures = true
+    // @see https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml
+    //configFile = rootProject.file('./checkstyle.xml') // default {project.projectDir}/config/checkstyle/checkstyle.xml
+}
+
+// FindBugs
+// @see https://docs.gradle.org/2.5/userguide/findbugs_plugin.html
+apply plugin: 'findbugs'
+findbugs {
+    ignoreFailures = true
+}
+
+// PMD
+// @see https://docs.gradle.org/2.5/userguide/pmd_plugin.html
+apply plugin: 'pmd'
+tasks.withType(Pmd) {
+    ignoreFailures = true
+    reports.html.enabled true
+}
+
+// JaCoCo
+// @see https://docs.gradle.org/2.5/userguide/jacoco_plugin.html
+apply plugin: 'jacoco'
+jacocoTestReport { // will use td-client v0.6.x
+    afterEvaluate {
+        classDirectories = files(classDirectories.files.collect {
+            fileTree(dir: it, exclude: 'com/treasuredata/api/**')
+        })
+    }
+}

data/src/main/java/com/treasuredata/api/TdApiClient.java

@@ -91,7 +91,8 @@ public class TdApiClient
     {
         try {
             http.start();
-        } catch (Exception e) {
+        }
+        catch (Exception e) {
             throw new IOException("Failed to start http client", e);
         }
     }
@@ -101,7 +102,8 @@ public class TdApiClient
     {
         try {
             http.stop();
-        } catch (Exception e) {
+        }
+        catch (Exception e) {
             throw new RuntimeException("Failed to stop http client", e);
         }
     }
@@ -138,7 +140,8 @@ public class TdApiClient
         return createTable(databaseName, tableName, TDTableType.LOG);
     }
 
-    public TDTable createTable(String databaseName, String tableName, TDTableType tableType) {
+    public TDTable createTable(String databaseName, String tableName, TDTableType tableType)
+    {
         Request request = prepareExchange(HttpMethod.POST,
                 buildUrl("/v3/table/create", databaseName, tableName, tableType.name().toLowerCase()));
         ContentResponse response = executeExchange(request);
@@ -154,6 +157,13 @@ public class TdApiClient
         ContentResponse response = executeExchange(request);
     }
 
+    public void renameTable(String databaseName, String oldName, String newName)
+    {
+        Request request = prepareExchange(HttpMethod.POST,
+                buildUrl("/v3/table/rename", databaseName, oldName, newName));
+        ContentResponse response = executeExchange(request);
+    }
+
     public void createBulkImportSession(String sessionName, String databaseName, String tableName)
     {
         Request request = prepareExchange(HttpMethod.POST,
@@ -223,8 +233,8 @@ public class TdApiClient
 
     private Request prepareExchange(HttpMethod method, String url)
     {
-        return prepareExchange(method, url, Collections.<String,String>emptyMap(),
-                Collections.<String,String>emptyMap());
+        return prepareExchange(method, url, Collections.<String, String>emptyMap(),
+                Collections.<String, String>emptyMap());
     }
 
     private Request prepareExchange(HttpMethod method,
@@ -248,7 +258,7 @@ public class TdApiClient
         request.agent(config.getAgentName());
         request.header("Authorization", "TD1 " + apiKey);
         //request.timeout(60, TimeUnit.SECONDS);
-        for (Map.Entry<String, String> entry: headers.entrySet()) {
+        for (Map.Entry<String, String> entry : headers.entrySet()) {
             request.header(entry.getKey(), entry.getValue());
         }
         String dateHeader = setDateHeader(request);
@@ -264,7 +274,8 @@ public class TdApiClient
     {
         try {
             return URLEncoder.encode(s, "UTF-8");
-        } catch (UnsupportedEncodingException e) {
+        }
+        catch (UnsupportedEncodingException e) {
             throw new AssertionError(e);
         }
     }
@@ -272,7 +283,8 @@ public class TdApiClient
     private static final ThreadLocal<SimpleDateFormat> RFC2822_FORMAT =
         new ThreadLocal<SimpleDateFormat>() {
             @Override
-            protected SimpleDateFormat initialValue() {
+            protected SimpleDateFormat initialValue()
+            {
                 return new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss Z", Locale.ENGLISH);
             }
         };
@@ -288,10 +300,12 @@ public class TdApiClient
     private static final ThreadLocal<MessageDigest> SHA1 =
         new ThreadLocal<MessageDigest>() {
             @Override
-            protected MessageDigest initialValue() {
+            protected MessageDigest initialValue()
+            {
                 try {
                     return MessageDigest.getInstance("SHA-1");
-                } catch (NoSuchAlgorithmException e) {
+                }
+                catch (NoSuchAlgorithmException e) {
                     throw new RuntimeException("SHA-1 digest algorithm must be available but not found", e);
                 }
             }
@@ -316,8 +330,8 @@ public class TdApiClient
         char[] array = new char[bytes.length * 2];
         for (int i = 0; i < bytes.length; i++) {
             int b = (int) bytes[i];
-            array[i*2] = hexChars[(b & 0xf0) >> 4];
-            array[i*2+1] = hexChars[b & 0x0f];
+            array[i * 2] = hexChars[(b & 0xf0) >> 4];
+            array[i * 2 + 1] = hexChars[b & 0x0f];
         }
         return new String(array);
     }
@@ -333,7 +347,8 @@ public class TdApiClient
                 sb.append("/");
                 sb.append(URLEncoder.encode(param, "UTF-8"));
             }
-        } catch (UnsupportedEncodingException ex) {
+        }
+        catch (UnsupportedEncodingException ex) {
             throw new AssertionError(ex);
         }
         return sb.toString();
@@ -370,10 +385,12 @@ public class TdApiClient
                     // retry on 50x and other errors
                     exception = new TdApiResponseException(status, response.getContent());
 
-                } catch (TdApiException e) {
+                }
+                catch (TdApiException e) {
                     throw e;
 
-                } catch (Exception e) {
+                }
+                catch (Exception e) {
                     // retry on RuntimeException
                     exception = e;
                 }
@@ -393,7 +410,8 @@ public class TdApiClient
                 Thread.sleep(retryWait);
                 retryWait *= 2;
             }
-        } catch (InterruptedException e) {
+        }
+        catch (InterruptedException e) {
             throw new TdApiExecutionInterruptedException(e);
         }
     }
@@ -404,14 +422,18 @@ public class TdApiClient
         try {
             objectMapper.writeValue(bo, obj);
             return new String(bo.toByteArray(), "UTF-8");
-        } catch (UnsupportedEncodingException ex) {
+        }
+        catch (UnsupportedEncodingException ex) {
             throw new AssertionError(ex);
-        } catch (IOException ex) {
+        }
+        catch (IOException ex) {
             throw new RuntimeException(ex);
-        } finally {
+        }
+        finally {
             try {
                 bo.close();
-            } catch (IOException ex) {
+            }
+            catch (IOException ex) {
                 throw new RuntimeException(ex);
             }
         }
@@ -421,7 +443,7 @@ public class TdApiClient
     {
         try {
             StringBuilder sb = new StringBuilder();
-            for(int i=0; i < kvs.length; i+=2) {
+            for (int i = 0; i < kvs.length; i += 2) {
                 if (i > 0) {
                     sb.append("&");
                 }
@@ -430,7 +452,8 @@ public class TdApiClient
                   .append(encode(kvs[i + 1]));
             }
             return sb.toString().getBytes("UTF-8");
-        } catch (UnsupportedEncodingException ex) {
+        }
+        catch (UnsupportedEncodingException ex) {
             throw new AssertionError(ex);
         }
     }
@@ -439,7 +462,8 @@ public class TdApiClient
     {
         try {
             return objectMapper.readValue(content, valueType);
-        } catch (IOException ex) {
+        }
+        catch (IOException ex) {
             throw new RuntimeException(ex);
         }
     }

data/src/main/java/com/treasuredata/api/TdApiClientConfig.java

@@ -3,7 +3,6 @@ package com.treasuredata.api;
 import com.google.common.base.Optional;
 
 public class TdApiClientConfig
-        implements TdApiConstants
 {
     public static class HttpProxyConfig
     {
@@ -73,7 +72,8 @@ public class TdApiClientConfig
         return httpProxyConfig;
     }
 
-    public String getAgentName() {
-        return AGENT_NAME;
+    public String getAgentName()
+    {
+        return TdApiConstants.AGENT_NAME;
     }
 }

data/src/main/java/com/treasuredata/api/TdApiConstants.java

@@ -1,6 +1,10 @@
 package com.treasuredata.api;
 
-public interface TdApiConstants
+public final class TdApiConstants
 {
-    String AGENT_NAME = "TdApiClient v0.6";
+    public static final String AGENT_NAME = "TdApiClient v0.6";
+
+    private TdApiConstants()
+    {
+    }
 }