embulk-output-td 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +13 -1
- data/build.gradle +1 -1
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +13 -2
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fab6ca938cab003937a19a42137eff48f2f54ce4
|
4
|
+
data.tar.gz: 29bf2c6ffd58a63b49939cff46f6c017f3e39566
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 45f6a5ffffa32cbbe729aa99d203e1cdca7a25efdbf67fe700ce693597f8423c767ef003f340c253a82e8a7f9e81b6691470e04c81e4d0bbe8410e65cc44423f
|
7
|
+
data.tar.gz: 09e42035b8e00f4c8d77fa6c172ab82f5c4bfa24347649eef6aecd184ba77b5a745b48aafc547c9a8cd756f53f2d5da8e3853c1ec8434592e0a51f8dc7171d0a
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.2.1 - 2016-01-28
|
2
|
+
|
3
|
+
* [new feature] Add truncate mode [#33](https://github.com/treasure-data/embulk-output-td/pull/33)
|
4
|
+
|
1
5
|
## 0.2.0 - 2016-01-12
|
2
6
|
|
3
7
|
* [new feature] Not use first timestamp column as primary key [#32](https://github.com/treasure-data/embulk-output-td/pull/32)
|
data/README.md
CHANGED
@@ -15,7 +15,7 @@ TODO: Write short description here
|
|
15
15
|
- **http_proxy**: http proxy configuration (tuple of host, port and useSsl, default is null)
|
16
16
|
- **use_ssl**: the flag (boolean, default=true)
|
17
17
|
- **auto_create_table**: the flag for creating the database and/or the table if they don't exist (boolean, default=true)
|
18
|
-
- **mode**:
|
18
|
+
- **mode**: 'append', 'replace' and 'truncate' (string, default='append')
|
19
19
|
- **database**: database name (string, required)
|
20
20
|
- **table**: table name (string, required)
|
21
21
|
- **session**: bulk_import session name (string, optional)
|
@@ -33,6 +33,18 @@ TODO: Write short description here
|
|
33
33
|
- **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
|
34
34
|
- **format**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, value of default_timestamp_format option is used by default)
|
35
35
|
|
36
|
+
## Modes
|
37
|
+
* **append**:
|
38
|
+
- Uploads data to existing table directly.
|
39
|
+
* **replace**:
|
40
|
+
- Creates new temp table and uploads data to the temp table first.
|
41
|
+
- After uploading finished, the table specified as 'table' option is replaced with the temp table.
|
42
|
+
- Schema in existing table is added to the replaced table.
|
43
|
+
* **truncate**:
|
44
|
+
- Creates new temp table and uploads data to the temp table first.
|
45
|
+
- After uploading finished, the table specified as 'table' option is replaced with the temp table.
|
46
|
+
- Schema in existing table is not migrated to the replaced table.
|
47
|
+
|
36
48
|
## Example
|
37
49
|
Here is sample configuration for TD output plugin.
|
38
50
|
```yaml
|
data/build.gradle
CHANGED
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.2.
|
4
|
+
spec.version = "0.2.1"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
@@ -18,6 +18,7 @@ import com.google.common.base.Optional;
|
|
18
18
|
import com.google.common.base.Throwables;
|
19
19
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
20
20
|
import com.fasterxml.jackson.annotation.JsonValue;
|
21
|
+
import com.google.common.collect.ImmutableMap;
|
21
22
|
import com.treasuredata.api.TdApiClient;
|
22
23
|
import com.treasuredata.api.TdApiClientConfig;
|
23
24
|
import com.treasuredata.api.TdApiClientConfig.HttpProxyConfig;
|
@@ -171,7 +172,7 @@ public class TdOutputPlugin
|
|
171
172
|
|
172
173
|
public enum Mode
|
173
174
|
{
|
174
|
-
APPEND, REPLACE;
|
175
|
+
APPEND, REPLACE, TRUNCATE;
|
175
176
|
|
176
177
|
@JsonCreator
|
177
178
|
public static Mode fromConfig(String value)
|
@@ -181,8 +182,10 @@ public class TdOutputPlugin
|
|
181
182
|
return APPEND;
|
182
183
|
case "replace":
|
183
184
|
return REPLACE;
|
185
|
+
case "truncate":
|
186
|
+
return TRUNCATE;
|
184
187
|
default:
|
185
|
-
throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are [append, replace]", value));
|
188
|
+
throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are [append, replace, truncate]", value));
|
186
189
|
}
|
187
190
|
}
|
188
191
|
|
@@ -194,6 +197,8 @@ public class TdOutputPlugin
|
|
194
197
|
return "append";
|
195
198
|
case REPLACE:
|
196
199
|
return "replace";
|
200
|
+
case TRUNCATE:
|
201
|
+
return "truncate";
|
197
202
|
default:
|
198
203
|
throw new IllegalStateException();
|
199
204
|
}
|
@@ -336,6 +341,7 @@ public class TdOutputPlugin
|
|
336
341
|
break;
|
337
342
|
|
338
343
|
case REPLACE:
|
344
|
+
case TRUNCATE:
|
339
345
|
task.setLoadTargetTableName(
|
340
346
|
createTemporaryTableWithPrefix(client, databaseName, makeTablePrefix(task)));
|
341
347
|
break;
|
@@ -372,6 +378,7 @@ public class TdOutputPlugin
|
|
372
378
|
// already done
|
373
379
|
break;
|
374
380
|
case REPLACE:
|
381
|
+
case TRUNCATE:
|
375
382
|
// rename table
|
376
383
|
renameTable(client, task.getDatabase(), task.getLoadTargetTableName(), task.getTable());
|
377
384
|
}
|
@@ -622,6 +629,10 @@ public class TdOutputPlugin
|
|
622
629
|
|
623
630
|
Map<String, TDColumnType> updateSchema(TdApiClient client, Schema inputSchema, PluginTask task)
|
624
631
|
{
|
632
|
+
if (task.getMode() == Mode.REPLACE) { // replace mode doesn't update the table schema.
|
633
|
+
return ImmutableMap.of();
|
634
|
+
}
|
635
|
+
|
625
636
|
String databaseName = task.getDatabase();
|
626
637
|
|
627
638
|
TDTable table = findTable(client, databaseName, task.getTable());
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,7 +106,7 @@ files:
|
|
106
106
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
107
107
|
- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
|
108
108
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
109
|
-
- classpath/embulk-output-td-0.2.
|
109
|
+
- classpath/embulk-output-td-0.2.1.jar
|
110
110
|
- classpath/javassist-3.18.1-GA.jar
|
111
111
|
- classpath/jetty-client-9.2.2.v20140723.jar
|
112
112
|
- classpath/jetty-http-9.2.2.v20140723.jar
|