embulk-output-td 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +2 -2
- data/build.gradle +1 -1
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +18 -15
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e783c1c2be4214064efd3f5ad141710fc886bb9f
|
4
|
+
data.tar.gz: f341aed71e1d97e92f9deec129291c44223a380c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f4ddb9be5f86ad7dc067d646d67cc96018c362d77f0409d8a3df8f8092efcbf66c928fb91ec4391f4d3852ed87e992246161170c10c3bb7f2f9bdba479617e4
|
7
|
+
data.tar.gz: bd6d393b4cf7c980d423daba0d79e8b4d0e487dec9e30c942be975f08899cb4c90ea49e6928c444dec966f591ef9ee6ea67d1c15c2e714616f0106d22ed7e594
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.2.2 - 2016-02-29
|
2
|
+
|
3
|
+
* [maintenance] Update a table schema by Embulk's guessed columns if the # of the uploaded record is zero [#36](https://github.com/treasure-data/embulk-output-td/pull/36)
|
4
|
+
* [maintenance] replace and truncate modes create a new table if the table doesn't exists [#35](https://github.com/treasure-data/embulk-output-td/pull/35)
|
5
|
+
|
1
6
|
## 0.2.1 - 2016-01-28
|
2
7
|
|
3
8
|
* [new feature] Add truncate mode [#33](https://github.com/treasure-data/embulk-output-td/pull/33)
|
data/README.md
CHANGED
@@ -39,11 +39,11 @@ TODO: Write short description here
|
|
39
39
|
* **replace**:
|
40
40
|
- Creates new temp table and uploads data to the temp table first.
|
41
41
|
- After uploading finished, the table specified as 'table' option is replaced with the temp table.
|
42
|
-
- Schema in existing table is
|
42
|
+
- Schema in existing table is not migrated to the replaced table.
|
43
43
|
* **truncate**:
|
44
44
|
- Creates new temp table and uploads data to the temp table first.
|
45
45
|
- After uploading finished, the table specified as 'table' option is replaced with the temp table.
|
46
|
-
- Schema in existing table is
|
46
|
+
- Schema in existing table is added to the replaced table.
|
47
47
|
|
48
48
|
## Example
|
49
49
|
Here is sample configuration for TD output plugin.
|
data/build.gradle
CHANGED
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.2.
|
4
|
+
spec.version = "0.2.2"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
@@ -19,6 +19,7 @@ import com.google.common.base.Throwables;
|
|
19
19
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
20
20
|
import com.fasterxml.jackson.annotation.JsonValue;
|
21
21
|
import com.google.common.collect.ImmutableMap;
|
22
|
+
import com.google.common.collect.Lists;
|
22
23
|
import com.treasuredata.api.TdApiClient;
|
23
24
|
import com.treasuredata.api.TdApiClientConfig;
|
24
25
|
import com.treasuredata.api.TdApiClientConfig.HttpProxyConfig;
|
@@ -330,7 +331,7 @@ public class TdOutputPlugin
|
|
330
331
|
switch (task.getMode()) {
|
331
332
|
case APPEND:
|
332
333
|
if (task.getAutoCreateTable()) {
|
333
|
-
// auto_create_table is valid only with append mode
|
334
|
+
// auto_create_table is valid only with append mode
|
334
335
|
createTableIfNotExists(client, databaseName, tableName);
|
335
336
|
}
|
336
337
|
else {
|
@@ -342,8 +343,9 @@ public class TdOutputPlugin
|
|
342
343
|
|
343
344
|
case REPLACE:
|
344
345
|
case TRUNCATE:
|
345
|
-
|
346
|
-
|
346
|
+
// replace and truncate modes always create a new table if the table doesn't exist
|
347
|
+
createTableIfNotExists(client, databaseName, tableName);
|
348
|
+
task.setLoadTargetTableName(createTemporaryTableWithPrefix(client, databaseName, makeTablePrefix(task)));
|
347
349
|
break;
|
348
350
|
}
|
349
351
|
|
@@ -629,16 +631,8 @@ public class TdOutputPlugin
|
|
629
631
|
|
630
632
|
Map<String, TDColumnType> updateSchema(TdApiClient client, Schema inputSchema, PluginTask task)
|
631
633
|
{
|
632
|
-
if (task.getMode() == Mode.REPLACE) { // replace mode doesn't update the table schema.
|
633
|
-
return ImmutableMap.of();
|
634
|
-
}
|
635
|
-
|
636
634
|
String databaseName = task.getDatabase();
|
637
|
-
|
638
635
|
TDTable table = findTable(client, databaseName, task.getTable());
|
639
|
-
if (table == null) {
|
640
|
-
return new HashMap<>();
|
641
|
-
}
|
642
636
|
|
643
637
|
final Map<String, TDColumnType> guessedSchema = new HashMap<>();
|
644
638
|
inputSchema.visitColumns(new ColumnVisitor() {
|
@@ -669,13 +663,22 @@ public class TdOutputPlugin
|
|
669
663
|
});
|
670
664
|
|
671
665
|
Map<String, Integer> usedNames = new HashMap<>();
|
672
|
-
|
673
|
-
|
674
|
-
|
666
|
+
if (task.getMode() != Mode.REPLACE) {
|
667
|
+
for (TDColumn existent : table.getColumns()) {
|
668
|
+
usedNames.put(new String(existent.getKey()), 1);
|
669
|
+
guessedSchema.remove(existent.getName()); // don't change type of existent columns
|
670
|
+
}
|
675
671
|
}
|
676
672
|
guessedSchema.remove("time"); // don't change type of 'time' column
|
677
673
|
|
678
|
-
List<TDColumn> newSchema
|
674
|
+
List<TDColumn> newSchema;
|
675
|
+
if (task.getMode() != Mode.REPLACE) {
|
676
|
+
newSchema = new ArrayList<>(table.getColumns());
|
677
|
+
}
|
678
|
+
else {
|
679
|
+
newSchema = Lists.newArrayList();
|
680
|
+
}
|
681
|
+
|
679
682
|
for (Map.Entry<String, TDColumnType> pair : guessedSchema.entrySet()) {
|
680
683
|
String key = renameColumn(pair.getKey());
|
681
684
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-02-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,7 +106,7 @@ files:
|
|
106
106
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
107
107
|
- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
|
108
108
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
109
|
-
- classpath/embulk-output-td-0.2.
|
109
|
+
- classpath/embulk-output-td-0.2.2.jar
|
110
110
|
- classpath/javassist-3.18.1-GA.jar
|
111
111
|
- classpath/jetty-client-9.2.2.v20140723.jar
|
112
112
|
- classpath/jetty-http-9.2.2.v20140723.jar
|