embulk-output-td 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +2 -2
- data/build.gradle +1 -1
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +18 -15
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e783c1c2be4214064efd3f5ad141710fc886bb9f
|
4
|
+
data.tar.gz: f341aed71e1d97e92f9deec129291c44223a380c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f4ddb9be5f86ad7dc067d646d67cc96018c362d77f0409d8a3df8f8092efcbf66c928fb91ec4391f4d3852ed87e992246161170c10c3bb7f2f9bdba479617e4
|
7
|
+
data.tar.gz: bd6d393b4cf7c980d423daba0d79e8b4d0e487dec9e30c942be975f08899cb4c90ea49e6928c444dec966f591ef9ee6ea67d1c15c2e714616f0106d22ed7e594
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.2.2 - 2016-02-29
|
2
|
+
|
3
|
+
* [maintenance] Update a table schema by Embulk's guessed columns if the # of the uploaded record is zero [#36](https://github.com/treasure-data/embulk-output-td/pull/36)
|
4
|
+
* [maintenance] replace and truncate modes create a new table if the table doesn't exists [#35](https://github.com/treasure-data/embulk-output-td/pull/35)
|
5
|
+
|
1
6
|
## 0.2.1 - 2016-01-28
|
2
7
|
|
3
8
|
* [new feature] Add truncate mode [#33](https://github.com/treasure-data/embulk-output-td/pull/33)
|
data/README.md
CHANGED
@@ -39,11 +39,11 @@ TODO: Write short description here
|
|
39
39
|
* **replace**:
|
40
40
|
- Creates new temp table and uploads data to the temp table first.
|
41
41
|
- After uploading finished, the table specified as 'table' option is replaced with the temp table.
|
42
|
-
- Schema in existing table is
|
42
|
+
- Schema in existing table is not migrated to the replaced table.
|
43
43
|
* **truncate**:
|
44
44
|
- Creates new temp table and uploads data to the temp table first.
|
45
45
|
- After uploading finished, the table specified as 'table' option is replaced with the temp table.
|
46
|
-
- Schema in existing table is
|
46
|
+
- Schema in existing table is added to the replaced table.
|
47
47
|
|
48
48
|
## Example
|
49
49
|
Here is sample configuration for TD output plugin.
|
data/build.gradle
CHANGED
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.2.
|
4
|
+
spec.version = "0.2.2"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
@@ -19,6 +19,7 @@ import com.google.common.base.Throwables;
|
|
19
19
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
20
20
|
import com.fasterxml.jackson.annotation.JsonValue;
|
21
21
|
import com.google.common.collect.ImmutableMap;
|
22
|
+
import com.google.common.collect.Lists;
|
22
23
|
import com.treasuredata.api.TdApiClient;
|
23
24
|
import com.treasuredata.api.TdApiClientConfig;
|
24
25
|
import com.treasuredata.api.TdApiClientConfig.HttpProxyConfig;
|
@@ -330,7 +331,7 @@ public class TdOutputPlugin
|
|
330
331
|
switch (task.getMode()) {
|
331
332
|
case APPEND:
|
332
333
|
if (task.getAutoCreateTable()) {
|
333
|
-
// auto_create_table is valid only with append mode
|
334
|
+
// auto_create_table is valid only with append mode
|
334
335
|
createTableIfNotExists(client, databaseName, tableName);
|
335
336
|
}
|
336
337
|
else {
|
@@ -342,8 +343,9 @@ public class TdOutputPlugin
|
|
342
343
|
|
343
344
|
case REPLACE:
|
344
345
|
case TRUNCATE:
|
345
|
-
|
346
|
-
|
346
|
+
// replace and truncate modes always create a new table if the table doesn't exist
|
347
|
+
createTableIfNotExists(client, databaseName, tableName);
|
348
|
+
task.setLoadTargetTableName(createTemporaryTableWithPrefix(client, databaseName, makeTablePrefix(task)));
|
347
349
|
break;
|
348
350
|
}
|
349
351
|
|
@@ -629,16 +631,8 @@ public class TdOutputPlugin
|
|
629
631
|
|
630
632
|
Map<String, TDColumnType> updateSchema(TdApiClient client, Schema inputSchema, PluginTask task)
|
631
633
|
{
|
632
|
-
if (task.getMode() == Mode.REPLACE) { // replace mode doesn't update the table schema.
|
633
|
-
return ImmutableMap.of();
|
634
|
-
}
|
635
|
-
|
636
634
|
String databaseName = task.getDatabase();
|
637
|
-
|
638
635
|
TDTable table = findTable(client, databaseName, task.getTable());
|
639
|
-
if (table == null) {
|
640
|
-
return new HashMap<>();
|
641
|
-
}
|
642
636
|
|
643
637
|
final Map<String, TDColumnType> guessedSchema = new HashMap<>();
|
644
638
|
inputSchema.visitColumns(new ColumnVisitor() {
|
@@ -669,13 +663,22 @@ public class TdOutputPlugin
|
|
669
663
|
});
|
670
664
|
|
671
665
|
Map<String, Integer> usedNames = new HashMap<>();
|
672
|
-
|
673
|
-
|
674
|
-
|
666
|
+
if (task.getMode() != Mode.REPLACE) {
|
667
|
+
for (TDColumn existent : table.getColumns()) {
|
668
|
+
usedNames.put(new String(existent.getKey()), 1);
|
669
|
+
guessedSchema.remove(existent.getName()); // don't change type of existent columns
|
670
|
+
}
|
675
671
|
}
|
676
672
|
guessedSchema.remove("time"); // don't change type of 'time' column
|
677
673
|
|
678
|
-
List<TDColumn> newSchema
|
674
|
+
List<TDColumn> newSchema;
|
675
|
+
if (task.getMode() != Mode.REPLACE) {
|
676
|
+
newSchema = new ArrayList<>(table.getColumns());
|
677
|
+
}
|
678
|
+
else {
|
679
|
+
newSchema = Lists.newArrayList();
|
680
|
+
}
|
681
|
+
|
679
682
|
for (Map.Entry<String, TDColumnType> pair : guessedSchema.entrySet()) {
|
680
683
|
String key = renameColumn(pair.getKey());
|
681
684
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-02-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,7 +106,7 @@ files:
|
|
106
106
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
107
107
|
- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
|
108
108
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
109
|
-
- classpath/embulk-output-td-0.2.
|
109
|
+
- classpath/embulk-output-td-0.2.2.jar
|
110
110
|
- classpath/javassist-3.18.1-GA.jar
|
111
111
|
- classpath/jetty-client-9.2.2.v20140723.jar
|
112
112
|
- classpath/jetty-http-9.2.2.v20140723.jar
|