embulk-output-td 0.3.15 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d84da88a0b812745d299e5cc9f79c0da3de7c935
-  data.tar.gz: 124b87a0282e8ef587a8426fafff5eb161d7c6ac
+  metadata.gz: 11b5c3e330412f6124619e5a42b1a2752c2fe163
+  data.tar.gz: cc39b754466f6bd06fa4f2148235d29991d7c899
 SHA512:
-  metadata.gz: c8dddfbb22ffe88bddc057fff3027b1e2ad8b7e3ce13a50d735de9731119c2c2510f9714d5464b81e15d94f776a2ca724d684b4b44a53ed2e0a83a57af0f5adc
-  data.tar.gz: 10c5381c7b06618cdd3c931684f8debbc7ef579b74aab8f72f5ae0e346cd9af71e86ef7adfd9ba08d96755d169f2663a8657e62d3b4a1f8899d8eaccea816ea7
+  metadata.gz: c00286be234967a6693ee52c3c06042dc7e49be1ccfe8ecb4f4f4a0df4d7fe265394f57a4ef27c0628798d4ca7d0a3a0d343b63c065a38490d3e64beb2a3bbfd
+  data.tar.gz: e6e5f805e8a1630d1a5cc768e018df977496ca605042a783a2294111d241d080691f79a183cee37ae06133bf3535f2947e3ef78c24e17ca71b468f8ca5b8a150

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.0 - 2017-06-26
+
+* [maintenance] Use TDClient#showTable method to to get table schema [#64](https://github.com/treasure-data/embulk-output-td/pull/64)
+* [maintenance] Use TDClient#appendTableSchema method to update table schema [#65](https://github.com/treasure-data/embulk-output-td/pull/65)
+
 ## 0.3.15 - 2017-05-30
 
 * [fix bug] Use underscores for sessions names to fix #70 [#71](https://github.com/treasure-data/embulk-output-td/pull/71)

data/README.md

@@ -1,6 +1,8 @@
 # TD output plugin for Embulk
 
-TODO: Write short description here
+[Treasure Data Service](https://www.treasuredata.com/) output plugin for [Embulk](https://github.com/embulk/embulk)
+
+**NOTICE**: embulk-output-td v0.4.0+ only supports **Embulk v0.8.22+**.
 
 ## Overview
 
@@ -110,4 +112,4 @@ Repo URL: https://rubygems.org/gems/embulk-output-td
 $ ./gradlew bintrayUpload
 ```
 
-Repo URL: https://bintray.com/embulk-output-td/maven/embulk-output-td
+Repo URL: https://bintray.com/embulk-output-td/maven/embulk-output-td

data/build.gradle

@@ -19,7 +19,7 @@ configurations {
 }
 
 group = "org.embulk.output.td"
-version = "0.3.15"
+version = "0.4.0"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7
@@ -32,7 +32,7 @@ dependencies {
     provided    "org.embulk:embulk-standards:0.8.+"
     compile     "org.msgpack:msgpack-core:0.8.+"
     provided    "org.msgpack:msgpack-core:0.8.+"
-    compile     "com.treasuredata.client:td-client:0.7.29"
+    compile    "com.treasuredata.client:td-client:0.7.38"
 
     testCompile "junit:junit:4.+"
     testCompile "org.bigtesting:fixd:1.0.0"

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.3.15"
+  spec.version       = "0.4.0"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/org/embulk/output/td/TdOutputPlugin.java

@@ -33,6 +33,7 @@ import com.treasuredata.client.model.TDBulkImportSession.ImportStatus;
 import com.treasuredata.client.model.TDColumn;
 import com.treasuredata.client.model.TDColumnType;
 import com.treasuredata.client.model.TDTable;
+import org.embulk.EmbulkVersion;
 import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -356,6 +357,8 @@ public class TdOutputPlugin
             task.setTempDir(Optional.of(getEnvironmentTempDirectory()));
         }
 
+        checkEmbulkVersion();
+
         try (TDClient client = newTDClient(task)) {
             String databaseName = task.getDatabase();
             String tableName = task.getTable();
@@ -555,15 +558,10 @@ public class TdOutputPlugin
     void validateTableExists(TDClient client, String databaseName, String tableName)
     {
         try {
-            for (TDTable table : client.listTables(databaseName)) {
-                if (table.getName().equals(tableName)) {
-                    return;
-                }
-            }
-            throw new ConfigException(String.format("Table \"%s\".\"%s\" doesn't exist", databaseName, tableName));
+            client.showTable(databaseName, tableName);
         }
         catch (TDClientHttpNotFoundException ex) {
-            throw new ConfigException(String.format("Database \"%s\" doesn't exist", databaseName), ex);
+            throw new ConfigException(String.format("Database \"%s\" or table \"%s\" doesn't exist", databaseName, tableName), ex);
         }
     }
 
@@ -754,7 +752,7 @@ public class TdOutputPlugin
             newSchema.add(new TDColumn(pair.getKey(), pair.getValue(), key.getBytes(StandardCharsets.UTF_8)));
         }
 
-        client.updateTableSchema(databaseName, task.getLoadTargetTableName(), newSchema);
+        client.appendTableSchema(databaseName, task.getLoadTargetTableName(), newSchema);
         return guessedSchema;
     }
 
@@ -770,12 +768,12 @@ public class TdOutputPlugin
 
     private static TDTable findTable(TDClient client, String databaseName, String tableName)
     {
-        for (TDTable table : client.listTables(databaseName)) {
-            if (table.getName().equals(tableName)) {
-                return table;
-            }
+        try {
+            return client.showTable(databaseName, tableName);
+        }
+        catch(TDClientHttpNotFoundException e) {
+            return null;
         }
-        return null;
     }
 
     private static final Pattern COLUMN_NAME_PATTERN = Pattern.compile("\\A[a-z_][a-z0-9_]*\\z");
@@ -817,6 +815,21 @@ public class TdOutputPlugin
         });
     }
 
+    private void checkEmbulkVersion()
+    {
+        // Embulk v0.8.21 and lower version uses old Jackson and doesn't works with this plugin
+        // https://github.com/embulk/embulk/pull/615
+        String[] versionNumber = EmbulkVersion.VERSION.split("-"); // to split e.g. "0.8.26-SNAPSHOT" or "0.8.30-ALPHA1"
+        if (versionNumber[0] != null) {
+            String[] versions = versionNumber[0].split("\\.");
+            if (versions.length == 3) {
+                if (Integer.valueOf(versions[1]) <= 8 && Integer.valueOf(versions[2]) < 22) {
+                    throw new ConfigException("embulk-output-td v0.4.x+ only supports Embulk v0.8.22 or higher versions");
+                }
+            }
+        }
+    }
+
     @VisibleForTesting
     TDBulkImportSession waitForStatusChange(TDClient client, String sessionName,
             ImportStatus current, ImportStatus expecting, String operation)

data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java

@@ -1,7 +1,6 @@
 package org.embulk.output.td;
 
 import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.treasuredata.client.ProxyConfig;
@@ -333,24 +332,13 @@ public class TestTdOutputPlugin
         TDTable table = newTable("my_table", "[]");
 
         { // table exists
-            doReturn(ImmutableList.of(table)).when(client).listTables(anyString());
+            doReturn(table).when(client).showTable(anyString(), anyString());
             plugin.validateTableExists(client, "my_db", "my_table");
             // no error happens
         }
 
-        { // table doesn't exist
-            doReturn(ImmutableList.of()).when(client).listTables(anyString());
-            try {
-                plugin.validateTableExists(client, "my_db", "my_table");
-                fail();
-            }
-            catch (Throwable t) {
-                assertTrue(t instanceof ConfigException);
-            }
-        }
-
-        { // database doesn't exist
-            doThrow(notFound()).when(client).listTables(anyString());
+        { // database or table doesn't exist
+            doThrow(notFound()).when(client).showTable(anyString(), anyString());
             try {
                 plugin.validateTableExists(client, "my_db", "my_table");
                 fail();

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.3.15
+  version: 0.4.0
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-30 00:00:00.000000000 Z
+date: 2017-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.
 email:
 - muga.nishizawa@gmail.com
@@ -82,7 +82,7 @@ files:
 - src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
 - src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
 - src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
-- classpath/embulk-output-td-0.3.15.jar
+- classpath/embulk-output-td-0.4.0.jar
 - classpath/hamcrest-core-1.1.jar
 - classpath/jackson-datatype-json-org-2.6.7.jar
 - classpath/jetty-client-9.2.2.v20140723.jar
@@ -92,7 +92,7 @@ files:
 - classpath/json-20090211_1.jar
 - classpath/json-simple-1.1.1.jar
 - classpath/junit-4.10.jar
-- classpath/td-client-0.7.29.jar
+- classpath/td-client-0.7.38.jar
 homepage: https://github.com/treasure-data/embulk-output-td
 licenses:
 - Apache 2.0