embulk-output-td 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b8a00ad99b615b2fe506d8a7e39dc95e68d7aa2
-  data.tar.gz: 4aafa0cf29d14a0555173c5e20daa6fe072e555a
+  metadata.gz: ba5bf97263f5de22e4250ceb1688deafc4eb293b
+  data.tar.gz: 0f84076ad2661788165625f0096c53345b88c342
 SHA512:
-  metadata.gz: b4c4a49e9e4368df40c92afa5026158d08f226b86254d9a8727aa0154e7f22c7ea84f63145249e0e7152afd9967ad8761bca096afdd3fffa1a70310ab2f5ad16
-  data.tar.gz: 848eaba5d024f0b70133bbc39c273dfb2d24317377c9ad2fe8801991df76c8858b6b7cf55d7c2448bd31da89f3832a6f26f8958b0cb2573fa0082d2a2df67800
+  metadata.gz: 9cab468073debd71cb2eb6177f995ab750049ef6e12e405d0ee63645cdf17635ddfa968b11575e79187be79266a6387e93a8ab7ad52264a3c7a4bcd09c7aee10
+  data.tar.gz: c1a93fd57f6aeb903bdd7cd03a3be788c6d548aaf0012c58809b93bbf3420c90766b01593c5bec0f414ee57568efd7d70e3d162c752034fba6ea6b4fdf131385

data/CHANGELOG.md

@@ -1,5 +1,10 @@
+## 0.3.13 - 2017-03-03
+
+* [maintenance] Not send perform and commit requests if no parts [#63](https://github.com/treasure-data/embulk-output-td/pull/63)
+
 ## 0.3.12 - 2016-11-30
 
+* [maintenance] Upgrade td-client v0.7.29 [#61](https://github.com/treasure-data/embulk-output-td/pull/61)
 * [maintenance] Change variable type of PluginTask.setTempDir() [#60](https://github.com/treasure-data/embulk-output-td/pull/60)
 
 ## 0.3.11 - 2016-11-18

data/build.gradle

@@ -19,7 +19,7 @@ configurations {
 }
 
 group = "org.embulk.output.td"
-version = "0.3.12"
+version = "0.3.13"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/embulk-output-td.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-td"
-  spec.version       = "0.3.12"
+  spec.version       = "0.3.13"
   spec.authors       = ["Muga Nishizawa"]
   spec.summary       = %[TreasureData output plugin for Embulk]
   spec.description   = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]

data/src/main/java/org/embulk/output/td/RecordWriter.java

@@ -193,8 +193,8 @@ public class RecordWriter
     @Override
     public TaskReport commit()
     {
-        TaskReport report = Exec.newTaskReport();
-        //  TODO
+        TaskReport report = Exec.newTaskReport()
+                .set(TdOutputPlugin.TASK_REPORT_UPLOADED_PART_NUMBER, partSeqId);
         return report;
     }
 }

data/src/main/java/org/embulk/output/td/TdOutputPlugin.java

@@ -331,6 +331,8 @@ public class TdOutputPlugin
         }
     }
 
+    static final String TASK_REPORT_UPLOADED_PART_NUMBER = "uploaded_part_number";
+
     private final Logger log;
 
     public TdOutputPlugin()
@@ -400,8 +402,16 @@ public class TdOutputPlugin
     {
         boolean doUpload = startBulkImportSession(client, task.getSessionName(), task.getDatabase(), task.getLoadTargetTableName());
         task.setDoUpload(doUpload);
-        control.run(task.dump());
-        completeBulkImportSession(client, schema, task, 0);  // TODO perform job priority
+        List<TaskReport> taskReports = control.run(task.dump());
+        if (!isNoUploadedParts(taskReports)) {
+            completeBulkImportSession(client, schema, task, 0);  // TODO perform job priority
+        }
+        else {
+            // if no parts, it skips submitting requests for perform and commit.
+            log.info("Skip performing and committing bulk import session '{}' since no parts are uploaded.", task.getSessionName());
+            Map<String, TDColumnType> newColumns = updateSchema(client, schema, task);
+            printNewAddedColumns(newColumns);
+        }
 
         // commit
         switch (task.getMode()) {
@@ -645,12 +655,7 @@ public class TdOutputPlugin
             log.info("    error records: {}", session.getErrorRecords());
             log.info("    valid parts: {}", session.getValidParts());
             log.info("    error parts: {}", session.getErrorParts());
-            if (!newColumns.isEmpty()) {
-                log.info("    new columns:");
-            }
-            for (Map.Entry<String, TDColumnType> pair : newColumns.entrySet()) {
-                log.info("      - {}: {}", pair.getKey(), pair.getValue());
-            }
+            printNewAddedColumns(newColumns);
 
             if (session.getErrorRecords() > 0L) {
                 showBulkImportErrorRecords(client, sessionName, (int) Math.min(session.getErrorRecords(), task.getDisplayedErrorRecordsCountLimit()));
@@ -752,6 +757,16 @@ public class TdOutputPlugin
         return guessedSchema;
     }
 
+    void printNewAddedColumns(Map<String, TDColumnType> newColumns)
+    {
+        if (!newColumns.isEmpty()) {
+            log.info("    new columns:");
+        }
+        for (Map.Entry<String, TDColumnType> pair : newColumns.entrySet()) {
+            log.info("      - {}: {}", pair.getKey(), pair.getValue());
+        }
+    }
+
     private static TDTable findTable(TDClient client, String databaseName, String tableName)
     {
         for (TDTable table : client.listTables(databaseName)) {
@@ -830,6 +845,18 @@ public class TdOutputPlugin
         }
     }
 
+    boolean isNoUploadedParts(List<TaskReport> taskReports)
+    {
+        int partNumber = 0;
+        for (TaskReport taskReport : taskReports) {
+            if (!taskReport.has(TASK_REPORT_UPLOADED_PART_NUMBER)) {
+                return false;
+            }
+            partNumber += taskReport.get(int.class, TASK_REPORT_UPLOADED_PART_NUMBER);
+        }
+        return partNumber == 0;
+    }
+
     @VisibleForTesting
     void renameTable(TDClient client, String databaseName, String oldName, String newName)
     {

data/src/test/java/org/embulk/output/td/TestRecordWriter.java

@@ -245,6 +245,6 @@ public class TestRecordWriter
     public void checkTaskReport()
     {
         recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
-        assertTrue(recordWriter.commit().isEmpty());
+        assertTrue(recordWriter.commit().has(TdOutputPlugin.TASK_REPORT_UPLOADED_PART_NUMBER));
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-td
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2017-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -82,7 +82,7 @@ files:
 - src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
 - src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
 - src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
-- classpath/embulk-output-td-0.3.12.jar
+- classpath/embulk-output-td-0.3.13.jar
 - classpath/hamcrest-core-1.1.jar
 - classpath/jackson-annotations-2.6.7.jar
 - classpath/jackson-core-2.6.7.jar