embulk-output-td 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -1
- data/README.md +1 -1
- data/build.gradle +1 -1
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/RecordWriter.java +1 -1
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +14 -2
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +6 -2
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +1 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 80f63ad74ba0c51fdcd64103df283845f7b9d2df
|
4
|
+
data.tar.gz: 8f96d9061f19f649f61fde281145bf303f323011
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bc0220f507c1d133f3b96a5a9efa6bf6766abbb16e07c1d47b2e93951d27f6292560d48ea3e1e2a8dc8361bd862ef5a0ab7334ca9585ebcebab6392ecf6121b8
|
7
|
+
data.tar.gz: dd9f4fc167f89423b8467f352f522a853107e2c6a9d3e8c1e0d3c8e279207b2b3d6a57e65825f1544f91b1d0641aa62e0bf1da47efccbe48a607afa609da4a59
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,10 @@
|
|
1
|
+
## 0.3.2 - 2016-05-10
|
2
|
+
|
3
|
+
* [maintenance] Fix bug Plugin couldn't create temporal directory and job failed at Windows 10 [#39](https://github.com/treasure-data/embulk-output-td/pull/39)
|
4
|
+
|
1
5
|
## 0.3.1 - 2016-04-25
|
2
6
|
|
3
|
-
* [maintenance] Fix bug TransactionalPageOutput.flush method throws NullPointerException
|
7
|
+
* [maintenance] Fix bug TransactionalPageOutput.flush method throws NullPointerException [#38](https://github.com/treasure-data/embulk-output-td/pull/38)
|
4
8
|
|
5
9
|
## 0.3.0 - 2016-03-03
|
6
10
|
|
data/README.md
CHANGED
@@ -21,7 +21,7 @@ TODO: Write short description here
|
|
21
21
|
- **session**: bulk_import session name (string, optional)
|
22
22
|
- **time_column**: user-defined time column (string, optional)
|
23
23
|
- **unix_timestamp_unit**: if type of "time" or **time_column** is long, it's considered unix timestamp. This option specify its unit in sec, milli, micro or nano (enum, default: `sec`)
|
24
|
-
- **tmpdir**: temporal directory
|
24
|
+
- **tmpdir**: temporal directory (string, optional) if set to null, plugin will use directory that could get from System.property
|
25
25
|
- **upload_concurrency**: upload concurrency (int, default=2). max concurrency is 8.
|
26
26
|
- **file_split_size**: split size (long, default=16384 (16MB)).
|
27
27
|
- **stop_on_invalid_record**: stop bulk load transaction if a file includes invalid record (such as invalid timestamp) (boolean, default=false).
|
data/build.gradle
CHANGED
data/embulk-output-td.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-td"
|
4
|
-
spec.version = "0.3.
|
4
|
+
spec.version = "0.3.2"
|
5
5
|
spec.authors = ["Muga Nishizawa"]
|
6
6
|
spec.summary = %[TreasureData output plugin for Embulk]
|
7
7
|
spec.description = %[TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.]
|
@@ -50,7 +50,7 @@ public class RecordWriter
|
|
50
50
|
this.taskIndex = taskIndex;
|
51
51
|
|
52
52
|
this.fieldWriters = fieldWriters;
|
53
|
-
this.tempDir = new File(task.getTempDir());
|
53
|
+
this.tempDir = new File(task.getTempDir().get());
|
54
54
|
this.executor = new FinalizableExecutorService();
|
55
55
|
this.uploadConcurrency = task.getUploadConcurrency();
|
56
56
|
this.fileSplitSize = task.getFileSplitSize() * 1024;
|
@@ -1,5 +1,6 @@
|
|
1
1
|
package org.embulk.output.td;
|
2
2
|
|
3
|
+
import java.io.File;
|
3
4
|
import java.io.IOException;
|
4
5
|
import java.io.InputStream;
|
5
6
|
import java.util.List;
|
@@ -118,8 +119,9 @@ public class TdOutputPlugin
|
|
118
119
|
public UnixTimestampUnit getUnixTimestampUnit();
|
119
120
|
|
120
121
|
@Config("tmpdir")
|
121
|
-
@ConfigDefault("
|
122
|
-
public String getTempDir();
|
122
|
+
@ConfigDefault("null")
|
123
|
+
public Optional<String> getTempDir();
|
124
|
+
public void setTempDir(String dir);
|
123
125
|
|
124
126
|
@Config("upload_concurrency")
|
125
127
|
@ConfigDefault("2")
|
@@ -324,6 +326,10 @@ public class TdOutputPlugin
|
|
324
326
|
// generate session name
|
325
327
|
task.setSessionName(buildBulkImportSessionName(task, Exec.session()));
|
326
328
|
|
329
|
+
if (!task.getTempDir().isPresent()) {
|
330
|
+
task.setTempDir(getEnvironmentTempDirectory());
|
331
|
+
}
|
332
|
+
|
327
333
|
try (TDClient client = newTDClient(task)) {
|
328
334
|
String databaseName = task.getDatabase();
|
329
335
|
String tableName = task.getTable();
|
@@ -798,4 +804,10 @@ public class TdOutputPlugin
|
|
798
804
|
}
|
799
805
|
}
|
800
806
|
}
|
807
|
+
|
808
|
+
@VisibleForTesting
|
809
|
+
String getEnvironmentTempDirectory()
|
810
|
+
{
|
811
|
+
return System.getProperty("java.io.tmpdir");
|
812
|
+
}
|
801
813
|
}
|
@@ -57,7 +57,7 @@ public class TestRecordWriter
|
|
57
57
|
"_c2", Types.BOOLEAN, "_c3", Types.DOUBLE, "_c4", Types.TIMESTAMP);
|
58
58
|
|
59
59
|
plugin = plugin();
|
60
|
-
task = pluginTask(config().set("session_name", "my_session"));
|
60
|
+
task = pluginTask(config().set("session_name", "my_session").set("tmpdir", plugin.getEnvironmentTempDirectory()));
|
61
61
|
}
|
62
62
|
|
63
63
|
@Test
|
@@ -192,7 +192,11 @@ public class TestRecordWriter
|
|
192
192
|
{
|
193
193
|
schema = schema("_c0", Types.LONG, "_c1", Types.STRING,
|
194
194
|
"_c2", Types.BOOLEAN, "_c3", Types.DOUBLE, "_c4", Types.TIMESTAMP);
|
195
|
-
task = pluginTask(config()
|
195
|
+
task = pluginTask(config()
|
196
|
+
.set("session_name", "my_session")
|
197
|
+
.set("time_value", ImmutableMap.of("from", 0L, "to", 0L))
|
198
|
+
.set("tmpdir", plugin.getEnvironmentTempDirectory())
|
199
|
+
);
|
196
200
|
recordWriter = recordWriter(task, tdClient(plugin, task), fieldWriters(log, task, schema));
|
197
201
|
|
198
202
|
try {
|
@@ -473,6 +473,7 @@ public class TestTdOutputPlugin
|
|
473
473
|
task.setSessionName("session_name");
|
474
474
|
task.setLoadTargetTableName("my_table");
|
475
475
|
task.setDoUpload(true);
|
476
|
+
task.setTempDir(plugin.getEnvironmentTempDirectory());
|
476
477
|
Schema schema = schema("time", Types.LONG, "c0", Types.STRING, "c1", Types.STRING);
|
477
478
|
|
478
479
|
try (TransactionalPageOutput output = plugin.open(task.dump(), schema, 0)) {
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,7 +82,7 @@ files:
|
|
82
82
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
83
83
|
- src/test/java/org/embulk/output/td/TestTimeValueGenerator.java
|
84
84
|
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
85
|
-
- classpath/embulk-output-td-0.3.
|
85
|
+
- classpath/embulk-output-td-0.3.2.jar
|
86
86
|
- classpath/hamcrest-core-1.1.jar
|
87
87
|
- classpath/jackson-datatype-json-org-2.4.4.jar
|
88
88
|
- classpath/jetty-client-9.2.2.v20140723.jar
|