embulk-output-td 0.3.12 → 0.3.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/build.gradle +1 -1
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/RecordWriter.java +2 -2
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +35 -8
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ba5bf97263f5de22e4250ceb1688deafc4eb293b
|
4
|
+
data.tar.gz: 0f84076ad2661788165625f0096c53345b88c342
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9cab468073debd71cb2eb6177f995ab750049ef6e12e405d0ee63645cdf17635ddfa968b11575e79187be79266a6387e93a8ab7ad52264a3c7a4bcd09c7aee10
|
7
|
+
data.tar.gz: c1a93fd57f6aeb903bdd7cd03a3be788c6d548aaf0012c58809b93bbf3420c90766b01593c5bec0f414ee57568efd7d70e3d162c752034fba6ea6b4fdf131385
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,10 @@
|
|
1
|
+
## 0.3.13 - 2017-03-03
|
2
|
+
|
3
|
+
* [maintenance] Not send perform and commit requests if no parts [#63](https://github.com/treasure-data/embulk-output-td/pull/63)
|
4
|
+
|
1
5
|
## 0.3.12 - 2016-11-30
|
2
6
|
|
7
|
+
* [maintenance] Upgrade td-client v0.7.29 [#61](https://github.com/treasure-data/embulk-output-td/pull/61)
|
3
8
|
* [maintenance] Change variable type of PluginTask.setTempDir() [#60](https://github.com/treasure-data/embulk-output-td/pull/60)
|
4
9
|
|
5
10
|
## 0.3.11 - 2016-11-18
|
data/build.gradle
CHANGED
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.3.
|
4
|
+
spec.version = "0.3.13"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
@@ -193,8 +193,8 @@ public class RecordWriter
|
|
193
193
|
@Override
|
194
194
|
public TaskReport commit()
|
195
195
|
{
|
196
|
-
TaskReport report = Exec.newTaskReport()
|
197
|
-
|
196
|
+
TaskReport report = Exec.newTaskReport()
|
197
|
+
.set(TdOutputPlugin.TASK_REPORT_UPLOADED_PART_NUMBER, partSeqId);
|
198
198
|
return report;
|
199
199
|
}
|
200
200
|
}
|
@@ -331,6 +331,8 @@ public class TdOutputPlugin
|
|
331
331
|
}
|
332
332
|
}
|
333
333
|
|
334
|
+
static final String TASK_REPORT_UPLOADED_PART_NUMBER = "uploaded_part_number";
|
335
|
+
|
334
336
|
private final Logger log;
|
335
337
|
|
336
338
|
public TdOutputPlugin()
|
@@ -400,8 +402,16 @@ public class TdOutputPlugin
|
|
400
402
|
{
|
401
403
|
boolean doUpload = startBulkImportSession(client, task.getSessionName(), task.getDatabase(), task.getLoadTargetTableName());
|
402
404
|
task.setDoUpload(doUpload);
|
403
|
-
control.run(task.dump());
|
404
|
-
|
405
|
+
List<TaskReport> taskReports = control.run(task.dump());
|
406
|
+
if (!isNoUploadedParts(taskReports)) {
|
407
|
+
completeBulkImportSession(client, schema, task, 0); // TODO perform job priority
|
408
|
+
}
|
409
|
+
else {
|
410
|
+
// if no parts, it skips submitting requests for perform and commit.
|
411
|
+
log.info("Skip performing and committing bulk import session '{}' since no parts are uploaded.", task.getSessionName());
|
412
|
+
Map<String, TDColumnType> newColumns = updateSchema(client, schema, task);
|
413
|
+
printNewAddedColumns(newColumns);
|
414
|
+
}
|
405
415
|
|
406
416
|
// commit
|
407
417
|
switch (task.getMode()) {
|
@@ -645,12 +655,7 @@ public class TdOutputPlugin
|
|
645
655
|
log.info(" error records: {}", session.getErrorRecords());
|
646
656
|
log.info(" valid parts: {}", session.getValidParts());
|
647
657
|
log.info(" error parts: {}", session.getErrorParts());
|
648
|
-
|
649
|
-
log.info(" new columns:");
|
650
|
-
}
|
651
|
-
for (Map.Entry<String, TDColumnType> pair : newColumns.entrySet()) {
|
652
|
-
log.info(" - {}: {}", pair.getKey(), pair.getValue());
|
653
|
-
}
|
658
|
+
printNewAddedColumns(newColumns);
|
654
659
|
|
655
660
|
if (session.getErrorRecords() > 0L) {
|
656
661
|
showBulkImportErrorRecords(client, sessionName, (int) Math.min(session.getErrorRecords(), task.getDisplayedErrorRecordsCountLimit()));
|
@@ -752,6 +757,16 @@ public class TdOutputPlugin
|
|
752
757
|
return guessedSchema;
|
753
758
|
}
|
754
759
|
|
760
|
+
void printNewAddedColumns(Map<String, TDColumnType> newColumns)
|
761
|
+
{
|
762
|
+
if (!newColumns.isEmpty()) {
|
763
|
+
log.info(" new columns:");
|
764
|
+
}
|
765
|
+
for (Map.Entry<String, TDColumnType> pair : newColumns.entrySet()) {
|
766
|
+
log.info(" - {}: {}", pair.getKey(), pair.getValue());
|
767
|
+
}
|
768
|
+
}
|
769
|
+
|
755
770
|
private static TDTable findTable(TDClient client, String databaseName, String tableName)
|
756
771
|
{
|
757
772
|
for (TDTable table : client.listTables(databaseName)) {
|
@@ -830,6 +845,18 @@ public class TdOutputPlugin
|
|
830
845
|
}
|
831
846
|
}
|
832
847
|
|
848
|
+
boolean isNoUploadedParts(List<TaskReport> taskReports)
|
849
|
+
{
|
850
|
+
int partNumber = 0;
|
851
|
+
for (TaskReport taskReport : taskReports) {
|
852
|
+
if (!taskReport.has(TASK_REPORT_UPLOADED_PART_NUMBER)) {
|
853
|
+
return false;
|
854
|
+
}
|
855
|
+
partNumber += taskReport.get(int.class, TASK_REPORT_UPLOADED_PART_NUMBER);
|
856
|
+
}
|
857
|
+
return partNumber == 0;
|
858
|
+
}
|
859
|
+
|
833
860
|
@VisibleForTesting
|
834
861
|
void renameTable(TDClient client, String databaseName, String oldName, String newName)
|
835
862
|
{
|
@@ -245,6 +245,6 @@ public class TestRecordWriter
|
|
245
245
|
public void checkTaskReport()
|
246
246
|
{
|
247
247
|
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
248
|
-
assertTrue(recordWriter.commit().
|
248
|
+
assertTrue(recordWriter.commit().has(TdOutputPlugin.TASK_REPORT_UPLOADED_PART_NUMBER));
|
249
249
|
}
|
250
250
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,7 +82,7 @@ files:
|
|
82
82
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
83
83
|
- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
|
84
84
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
85
|
-
- classpath/embulk-output-td-0.3.
|
85
|
+
- classpath/embulk-output-td-0.3.13.jar
|
86
86
|
- classpath/hamcrest-core-1.1.jar
|
87
87
|
- classpath/jackson-annotations-2.6.7.jar
|
88
88
|
- classpath/jackson-core-2.6.7.jar
|