embulk-output-td 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fab6ca938cab003937a19a42137eff48f2f54ce4
-  data.tar.gz: 29bf2c6ffd58a63b49939cff46f6c017f3e39566
+  metadata.gz: e783c1c2be4214064efd3f5ad141710fc886bb9f
+  data.tar.gz: f341aed71e1d97e92f9deec129291c44223a380c
 SHA512:
-  metadata.gz: 45f6a5ffffa32cbbe729aa99d203e1cdca7a25efdbf67fe700ce693597f8423c767ef003f340c253a82e8a7f9e81b6691470e04c81e4d0bbe8410e65cc44423f
-  data.tar.gz: 09e42035b8e00f4c8d77fa6c172ab82f5c4bfa24347649eef6aecd184ba77b5a745b48aafc547c9a8cd756f53f2d5da8e3853c1ec8434592e0a51f8dc7171d0a
+  metadata.gz: 8f4ddb9be5f86ad7dc067d646d67cc96018c362d77f0409d8a3df8f8092efcbf66c928fb91ec4391f4d3852ed87e992246161170c10c3bb7f2f9bdba479617e4
+  data.tar.gz: bd6d393b4cf7c980d423daba0d79e8b4d0e487dec9e30c942be975f08899cb4c90ea49e6928c444dec966f591ef9ee6ea67d1c15c2e714616f0106d22ed7e594

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.2 - 2016-02-29
+
+* [maintenance] Update a table schema by Embulk's guessed columns if the # of the uploaded record is zero [#36](https://github.com/treasure-data/embulk-output-td/pull/36)
+* [maintenance] replace and truncate modes create a new table if the table doesn't exists [#35](https://github.com/treasure-data/embulk-output-td/pull/35)
+
 ## 0.2.1 - 2016-01-28
 
 * [new feature] Add truncate mode [#33](https://github.com/treasure-data/embulk-output-td/pull/33)

data/README.md

@@ -39,11 +39,11 @@ TODO: Write short description here
 * **replace**:
   - Creates new temp table and uploads data to the temp table first.
   - After uploading finished, the table specified as 'table' option is replaced with the temp table.
-  - Schema in existing table is added to the replaced table.
+  - Schema in existing table is not migrated to the replaced table.
 * **truncate**:
   - Creates new temp table and uploads data to the temp table first.
   - After uploading finished, the table specified as 'table' option is replaced with the temp table.
-  - Schema in existing table is not migrated to the replaced table.
+  - Schema in existing table is added to the replaced table.
 
 ## Example
 Here is sample configuration for TD output plugin.

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
     provided
 }
 
-version = "0.2.1"
+version = "0.2.2"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.2.1"
+  spec.version       = "0.2.2"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/org/embulk/output/td/TdOutputPlugin.java

@@ -19,6 +19,7 @@ import com.google.common.base.Throwables;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import com.treasuredata.api.TdApiClient;
 import com.treasuredata.api.TdApiClientConfig;
 import com.treasuredata.api.TdApiClientConfig.HttpProxyConfig;
@@ -330,7 +331,7 @@ public class TdOutputPlugin
             switch (task.getMode()) {
             case APPEND:
                 if (task.getAutoCreateTable()) {
-                    // auto_create_table is valid only with append mode (replace mode always creates a new table)
+                    // auto_create_table is valid only with append mode
                     createTableIfNotExists(client, databaseName, tableName);
                 }
                 else {
@@ -342,8 +343,9 @@ public class TdOutputPlugin
 
             case REPLACE:
             case TRUNCATE:
-                task.setLoadTargetTableName(
-                        createTemporaryTableWithPrefix(client, databaseName, makeTablePrefix(task)));
+                // replace and truncate modes always create a new table if the table doesn't exist
+                createTableIfNotExists(client, databaseName, tableName);
+                task.setLoadTargetTableName(createTemporaryTableWithPrefix(client, databaseName, makeTablePrefix(task)));
                 break;
             }
 
@@ -629,16 +631,8 @@ public class TdOutputPlugin
 
     Map<String, TDColumnType> updateSchema(TdApiClient client, Schema inputSchema, PluginTask task)
     {
-        if (task.getMode() == Mode.REPLACE) { // replace mode doesn't update the table schema.
-            return ImmutableMap.of();
-        }
-
         String databaseName = task.getDatabase();
-
         TDTable table = findTable(client, databaseName, task.getTable());
-        if (table == null) {
-            return new HashMap<>();
-        }
 
         final Map<String, TDColumnType> guessedSchema = new HashMap<>();
         inputSchema.visitColumns(new ColumnVisitor() {
@@ -669,13 +663,22 @@ public class TdOutputPlugin
         });
 
         Map<String, Integer> usedNames = new HashMap<>();
-        for (TDColumn existent : table.getColumns()) {
-            usedNames.put(new String(existent.getKey()), 1);
-            guessedSchema.remove(existent.getName()); // don't change type of existent columns
+        if (task.getMode() != Mode.REPLACE) {
+            for (TDColumn existent : table.getColumns()) {
+                usedNames.put(new String(existent.getKey()), 1);
+                guessedSchema.remove(existent.getName()); // don't change type of existent columns
+            }
         }
         guessedSchema.remove("time"); // don't change type of 'time' column
 
-        List<TDColumn> newSchema = new ArrayList<>(table.getColumns());
+        List<TDColumn> newSchema;
+        if (task.getMode() != Mode.REPLACE) {
+            newSchema = new ArrayList<>(table.getColumns());
+        }
+        else {
+            newSchema = Lists.newArrayList();
+        }
+
         for (Map.Entry<String, TDColumnType> pair : guessedSchema.entrySet()) {
             String key = renameColumn(pair.getKey());
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-29 00:00:00.000000000 Z
+date: 2016-02-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ files:
 - src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
 - src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
 - src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
-- classpath/embulk-output-td-0.2.1.jar
+- classpath/embulk-output-td-0.2.2.jar
 - classpath/javassist-3.18.1-GA.jar
 - classpath/jetty-client-9.2.2.v20140723.jar
 - classpath/jetty-http-9.2.2.v20140723.jar