embulk-output-td 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +7 -0
- data/CHANGELOG.md +4 -0
- data/README.md +1 -0
- data/build.gradle +5 -1
- data/config/checkstyle/checkstyle.xml +117 -0
- data/embulk-output-td.gemspec +1 -1
- data/gradle/check.gradle +34 -0
- data/src/main/java/com/treasuredata/api/TdApiClient.java +47 -23
- data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +3 -3
- data/src/main/java/com/treasuredata/api/TdApiConstants.java +6 -2
- data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +2 -1
- data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +2 -1
- data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +1 -1
- data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +6 -4
- data/src/main/java/com/treasuredata/api/model/TDColumn.java +4 -2
- data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +26 -13
- data/src/main/java/com/treasuredata/api/model/TDDatabase.java +2 -1
- data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +1 -1
- data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +4 -2
- data/src/main/java/com/treasuredata/api/model/TDTableType.java +2 -1
- data/src/main/java/org/embulk/output/td/FinalizableExecutorService.java +35 -17
- data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +13 -7
- data/src/main/java/org/embulk/output/td/RecordWriter.java +21 -382
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +175 -40
- data/src/main/java/org/embulk/output/td/writer/BooleanFieldWriter.java +23 -0
- data/src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java +23 -0
- data/src/main/java/org/embulk/output/td/writer/FieldWriter.java +38 -0
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +206 -0
- data/src/main/java/org/embulk/output/td/writer/LongFieldWriter.java +23 -0
- data/src/main/java/org/embulk/output/td/writer/StringFieldWriter.java +23 -0
- data/src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java +28 -0
- data/src/main/java/org/embulk/output/td/writer/TimestampLongFieldWriter.java +23 -0
- data/src/main/java/org/embulk/output/td/writer/TimestampStringFieldWriter.java +27 -0
- data/src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java +27 -0
- data/src/main/java/org/embulk/output/td/writer/UnixTimestampLongFieldWriter.java +26 -0
- data/src/test/java/com/treasuredata/api/TestTdApiClient.java +1 -1
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +198 -0
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +529 -0
- data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +146 -0
- metadata +29 -14
- data/src/test/java/org/embulk/output/td/TestFieldWriter.java +0 -105
@@ -0,0 +1,146 @@
|
|
1
|
+
package org.embulk.output.td.writer;
|
2
|
+
|
3
|
+
import org.embulk.EmbulkTestRuntime;
|
4
|
+
import org.embulk.config.ConfigException;
|
5
|
+
import org.embulk.config.ConfigSource;
|
6
|
+
import org.embulk.spi.Exec;
|
7
|
+
import org.embulk.spi.Schema;
|
8
|
+
import org.embulk.spi.type.Types;
|
9
|
+
import org.junit.Before;
|
10
|
+
import org.junit.Rule;
|
11
|
+
import org.junit.Test;
|
12
|
+
import org.slf4j.Logger;
|
13
|
+
|
14
|
+
import static org.embulk.output.td.TestTdOutputPlugin.config;
|
15
|
+
import static org.embulk.output.td.TestTdOutputPlugin.pluginTask;
|
16
|
+
import static org.embulk.output.td.TestTdOutputPlugin.schema;
|
17
|
+
import static org.junit.Assert.assertTrue;
|
18
|
+
import static org.junit.Assert.fail;
|
19
|
+
|
20
|
+
public class TestFieldWriterSet
|
21
|
+
{
|
22
|
+
@Rule
|
23
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
24
|
+
|
25
|
+
private Logger log;
|
26
|
+
private ConfigSource config;
|
27
|
+
private Schema schema;
|
28
|
+
|
29
|
+
@Before
|
30
|
+
public void createResources()
|
31
|
+
{
|
32
|
+
log = Exec.getLogger(TestFieldWriterSet.class);
|
33
|
+
config = config();
|
34
|
+
}
|
35
|
+
|
36
|
+
@Test
|
37
|
+
public void validateFieldWriterSet()
|
38
|
+
{
|
39
|
+
{ // if schema doesn't have appropriate time column, it throws ConfigError.
|
40
|
+
schema = schema("_c0", Types.STRING, "time", Types.STRING); // not long or timestamp
|
41
|
+
try {
|
42
|
+
new FieldWriterSet(log, pluginTask(config), schema);
|
43
|
+
fail();
|
44
|
+
}
|
45
|
+
catch (Throwable t) {
|
46
|
+
assertTrue(t instanceof ConfigException);
|
47
|
+
}
|
48
|
+
}
|
49
|
+
|
50
|
+
{ // if schema doesn't have time column and the user doesn't specify time_column option, it throws ConfigError.
|
51
|
+
schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
|
52
|
+
try {
|
53
|
+
new FieldWriterSet(log, pluginTask(config), schema);
|
54
|
+
fail();
|
55
|
+
}
|
56
|
+
catch (Throwable t) {
|
57
|
+
assertTrue(t instanceof ConfigException);
|
58
|
+
}
|
59
|
+
}
|
60
|
+
|
61
|
+
{ // if schema doesn't have a column specified as time_column column, it throws ConfigError
|
62
|
+
schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
|
63
|
+
try {
|
64
|
+
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c2")), schema);
|
65
|
+
fail();
|
66
|
+
}
|
67
|
+
catch (Throwable t) {
|
68
|
+
assertTrue(t instanceof ConfigException);
|
69
|
+
}
|
70
|
+
}
|
71
|
+
|
72
|
+
{ // if time_column column is not appropriate column type, it throws ConfigError.
|
73
|
+
schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
|
74
|
+
try {
|
75
|
+
new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c1")), schema);
|
76
|
+
fail();
|
77
|
+
}
|
78
|
+
catch (Throwable t) {
|
79
|
+
assertTrue(t instanceof ConfigException);
|
80
|
+
}
|
81
|
+
}
|
82
|
+
}
|
83
|
+
|
84
|
+
@Test
|
85
|
+
public void hasTimeColumn()
|
86
|
+
{
|
87
|
+
{ // time column (timestamp type) exists
|
88
|
+
Schema schema = schema("time", Types.TIMESTAMP, "_c0", Types.TIMESTAMP);
|
89
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
|
90
|
+
|
91
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter);
|
92
|
+
}
|
93
|
+
|
94
|
+
{ // time column (long type) exists
|
95
|
+
Schema schema = schema("time", Types.LONG, "_c0", Types.TIMESTAMP);
|
96
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
|
97
|
+
|
98
|
+
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampLongFieldWriter);
|
99
|
+
|
100
|
+
}
|
101
|
+
}
|
102
|
+
|
103
|
+
@Test
|
104
|
+
public void specifiedTimeColumnOption()
|
105
|
+
{
|
106
|
+
{ // time_column option (timestamp type)
|
107
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.STRING);
|
108
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
109
|
+
|
110
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator);
|
111
|
+
}
|
112
|
+
|
113
|
+
{ // time_column option (long type)
|
114
|
+
Schema schema = schema("_c0", Types.LONG, "_c1", Types.STRING);
|
115
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
116
|
+
|
117
|
+
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator);
|
118
|
+
}
|
119
|
+
|
120
|
+
{ // time_column option (typestamp type) if time column exists
|
121
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
|
122
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
123
|
+
|
124
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
|
125
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
|
126
|
+
}
|
127
|
+
|
128
|
+
{ // time_column option (long type) if time column exists
|
129
|
+
Schema schema = schema("_c0", Types.LONG, "time", Types.TIMESTAMP);
|
130
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
|
131
|
+
|
132
|
+
assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator); // c0
|
133
|
+
assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
|
134
|
+
}
|
135
|
+
}
|
136
|
+
|
137
|
+
@Test
|
138
|
+
public void useFirstTimestampColumn()
|
139
|
+
throws Exception
|
140
|
+
{
|
141
|
+
Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.LONG);
|
142
|
+
FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
|
143
|
+
|
144
|
+
assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator);
|
145
|
+
}
|
146
|
+
}
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.0'
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
16
|
- - ~>
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: '1.0'
|
19
|
+
name: bundler
|
25
20
|
prerelease: false
|
26
21
|
type: :development
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
|
-
- -
|
24
|
+
- - ~>
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
30
|
- - '>='
|
37
31
|
- !ruby/object:Gem::Version
|
38
32
|
version: '10.0'
|
33
|
+
name: rake
|
39
34
|
prerelease: false
|
40
35
|
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
41
|
description: TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.
|
42
42
|
email:
|
43
43
|
- muga.nishizawa@gmail.com
|
@@ -46,10 +46,13 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- CHANGELOG.md
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
53
|
+
- config/checkstyle/checkstyle.xml
|
52
54
|
- embulk-output-td.gemspec
|
55
|
+
- gradle/check.gradle
|
53
56
|
- gradle/wrapper/gradle-wrapper.jar
|
54
57
|
- gradle/wrapper/gradle-wrapper.properties
|
55
58
|
- gradlew
|
@@ -84,10 +87,22 @@ files:
|
|
84
87
|
- src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java
|
85
88
|
- src/main/java/org/embulk/output/td/RecordWriter.java
|
86
89
|
- src/main/java/org/embulk/output/td/TdOutputPlugin.java
|
90
|
+
- src/main/java/org/embulk/output/td/writer/BooleanFieldWriter.java
|
91
|
+
- src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java
|
92
|
+
- src/main/java/org/embulk/output/td/writer/FieldWriter.java
|
93
|
+
- src/main/java/org/embulk/output/td/writer/FieldWriterSet.java
|
94
|
+
- src/main/java/org/embulk/output/td/writer/LongFieldWriter.java
|
95
|
+
- src/main/java/org/embulk/output/td/writer/StringFieldWriter.java
|
96
|
+
- src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java
|
97
|
+
- src/main/java/org/embulk/output/td/writer/TimestampLongFieldWriter.java
|
98
|
+
- src/main/java/org/embulk/output/td/writer/TimestampStringFieldWriter.java
|
99
|
+
- src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java
|
100
|
+
- src/main/java/org/embulk/output/td/writer/UnixTimestampLongFieldWriter.java
|
87
101
|
- src/test/java/com/treasuredata/api/TestTdApiClient.java
|
88
|
-
- src/test/java/org/embulk/output/td/
|
102
|
+
- src/test/java/org/embulk/output/td/TestRecordWriter.java
|
89
103
|
- src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
|
90
|
-
-
|
104
|
+
- src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
|
105
|
+
- classpath/embulk-output-td-0.1.5.jar
|
91
106
|
- classpath/javassist-3.18.1-GA.jar
|
92
107
|
- classpath/jetty-client-9.2.2.v20140723.jar
|
93
108
|
- classpath/jetty-http-9.2.2.v20140723.jar
|
@@ -1,105 +0,0 @@
|
|
1
|
-
package org.embulk.output.td;
|
2
|
-
|
3
|
-
import com.google.common.collect.ImmutableList;
|
4
|
-
import org.embulk.EmbulkTestRuntime;
|
5
|
-
import org.embulk.config.ConfigLoader;
|
6
|
-
import org.embulk.config.ConfigSource;
|
7
|
-
import org.embulk.output.td.RecordWriter.FieldWriterSet;
|
8
|
-
import org.embulk.spi.Column;
|
9
|
-
import org.embulk.spi.ColumnConfig;
|
10
|
-
import org.embulk.spi.Exec;
|
11
|
-
import org.embulk.spi.Schema;
|
12
|
-
import org.embulk.spi.SchemaConfig;
|
13
|
-
import org.embulk.spi.type.Type;
|
14
|
-
import org.embulk.spi.type.Types;
|
15
|
-
import org.junit.Before;
|
16
|
-
import org.junit.Rule;
|
17
|
-
import org.junit.Test;
|
18
|
-
import org.slf4j.Logger;
|
19
|
-
|
20
|
-
import static org.junit.Assert.assertEquals;
|
21
|
-
import static org.junit.Assert.assertTrue;
|
22
|
-
|
23
|
-
public class TestFieldWriter
|
24
|
-
{
|
25
|
-
@Rule
|
26
|
-
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
27
|
-
|
28
|
-
private Logger log;
|
29
|
-
|
30
|
-
@Before
|
31
|
-
public void createLogger()
|
32
|
-
{
|
33
|
-
log = Exec.getLogger(TestFieldWriter.class);
|
34
|
-
}
|
35
|
-
|
36
|
-
private ConfigSource config()
|
37
|
-
{
|
38
|
-
return new ConfigLoader(runtime.getModelManager()).newConfigSource();
|
39
|
-
}
|
40
|
-
|
41
|
-
private Schema schema(Column... columns)
|
42
|
-
{
|
43
|
-
ImmutableList.Builder<Column> builder = new ImmutableList.Builder<Column>();
|
44
|
-
for (Column col : columns) {
|
45
|
-
builder.add(col);
|
46
|
-
}
|
47
|
-
return new Schema(builder.build());
|
48
|
-
}
|
49
|
-
|
50
|
-
private Column column(int index, String name, Type type)
|
51
|
-
{
|
52
|
-
return new Column(index, name, type);
|
53
|
-
}
|
54
|
-
|
55
|
-
private TdOutputPlugin.PluginTask task(ConfigSource outConfig)
|
56
|
-
{
|
57
|
-
return outConfig.loadConfig(TdOutputPlugin.PluginTask.class);
|
58
|
-
}
|
59
|
-
|
60
|
-
@Test
|
61
|
-
public void test()
|
62
|
-
{
|
63
|
-
{ // time column exists
|
64
|
-
// out: config
|
65
|
-
ConfigSource outConfig = config()
|
66
|
-
.set("apikey", "xxx")
|
67
|
-
.set("database", "mydb")
|
68
|
-
.set("table", "mytbl");
|
69
|
-
|
70
|
-
// schema
|
71
|
-
Schema schema = schema(
|
72
|
-
column(0, "time", Types.TIMESTAMP),
|
73
|
-
column(1, "c1", Types.TIMESTAMP));
|
74
|
-
|
75
|
-
// create field writers
|
76
|
-
FieldWriterSet writers = new FieldWriterSet(log, task(outConfig), schema);
|
77
|
-
|
78
|
-
assertEquals(schema.getColumnCount(), writers.getFieldCount());
|
79
|
-
assertTrue(writers.getFieldWriter(0) instanceof RecordWriter.TimestampLongFieldWriter);
|
80
|
-
assertTrue(writers.getFieldWriter(1) instanceof RecordWriter.TimestampStringFieldWriter);
|
81
|
-
}
|
82
|
-
|
83
|
-
{ // time column doesn't exists. users need to specify another column as time column
|
84
|
-
// out: config
|
85
|
-
ConfigSource outConfig = config()
|
86
|
-
.set("apikey", "xxx")
|
87
|
-
.set("database", "mydb")
|
88
|
-
.set("table", "mytbl")
|
89
|
-
.set("time_column", "c1");
|
90
|
-
TdOutputPlugin.PluginTask task = outConfig.loadConfig(TdOutputPlugin.PluginTask.class);
|
91
|
-
|
92
|
-
// schema
|
93
|
-
Schema schema = schema(
|
94
|
-
column(0, "c0", Types.TIMESTAMP),
|
95
|
-
column(1, "c1", Types.TIMESTAMP));
|
96
|
-
|
97
|
-
// create field writers
|
98
|
-
FieldWriterSet writers = new FieldWriterSet(log, task(outConfig), schema);
|
99
|
-
|
100
|
-
assertEquals(schema.getColumnCount() + 1, writers.getFieldCount());
|
101
|
-
assertTrue(writers.getFieldWriter(0) instanceof RecordWriter.TimestampStringFieldWriter);
|
102
|
-
assertTrue(writers.getFieldWriter(1) instanceof RecordWriter.TimestampFieldLongDuplicator);
|
103
|
-
}
|
104
|
-
}
|
105
|
-
}
|