embulk-output-td 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +7 -0
  3. data/CHANGELOG.md +4 -0
  4. data/README.md +1 -0
  5. data/build.gradle +5 -1
  6. data/config/checkstyle/checkstyle.xml +117 -0
  7. data/embulk-output-td.gemspec +1 -1
  8. data/gradle/check.gradle +34 -0
  9. data/src/main/java/com/treasuredata/api/TdApiClient.java +47 -23
  10. data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +3 -3
  11. data/src/main/java/com/treasuredata/api/TdApiConstants.java +6 -2
  12. data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +2 -1
  13. data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +2 -1
  14. data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +1 -1
  15. data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +6 -4
  16. data/src/main/java/com/treasuredata/api/model/TDColumn.java +4 -2
  17. data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +26 -13
  18. data/src/main/java/com/treasuredata/api/model/TDDatabase.java +2 -1
  19. data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +1 -1
  20. data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +4 -2
  21. data/src/main/java/com/treasuredata/api/model/TDTableType.java +2 -1
  22. data/src/main/java/org/embulk/output/td/FinalizableExecutorService.java +35 -17
  23. data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +13 -7
  24. data/src/main/java/org/embulk/output/td/RecordWriter.java +21 -382
  25. data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +175 -40
  26. data/src/main/java/org/embulk/output/td/writer/BooleanFieldWriter.java +23 -0
  27. data/src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java +23 -0
  28. data/src/main/java/org/embulk/output/td/writer/FieldWriter.java +38 -0
  29. data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +206 -0
  30. data/src/main/java/org/embulk/output/td/writer/LongFieldWriter.java +23 -0
  31. data/src/main/java/org/embulk/output/td/writer/StringFieldWriter.java +23 -0
  32. data/src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java +28 -0
  33. data/src/main/java/org/embulk/output/td/writer/TimestampLongFieldWriter.java +23 -0
  34. data/src/main/java/org/embulk/output/td/writer/TimestampStringFieldWriter.java +27 -0
  35. data/src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java +27 -0
  36. data/src/main/java/org/embulk/output/td/writer/UnixTimestampLongFieldWriter.java +26 -0
  37. data/src/test/java/com/treasuredata/api/TestTdApiClient.java +1 -1
  38. data/src/test/java/org/embulk/output/td/TestRecordWriter.java +198 -0
  39. data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +529 -0
  40. data/src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java +146 -0
  41. metadata +29 -14
  42. data/src/test/java/org/embulk/output/td/TestFieldWriter.java +0 -105
@@ -0,0 +1,146 @@
1
+ package org.embulk.output.td.writer;
2
+
3
+ import org.embulk.EmbulkTestRuntime;
4
+ import org.embulk.config.ConfigException;
5
+ import org.embulk.config.ConfigSource;
6
+ import org.embulk.spi.Exec;
7
+ import org.embulk.spi.Schema;
8
+ import org.embulk.spi.type.Types;
9
+ import org.junit.Before;
10
+ import org.junit.Rule;
11
+ import org.junit.Test;
12
+ import org.slf4j.Logger;
13
+
14
+ import static org.embulk.output.td.TestTdOutputPlugin.config;
15
+ import static org.embulk.output.td.TestTdOutputPlugin.pluginTask;
16
+ import static org.embulk.output.td.TestTdOutputPlugin.schema;
17
+ import static org.junit.Assert.assertTrue;
18
+ import static org.junit.Assert.fail;
19
+
20
+ public class TestFieldWriterSet
21
+ {
22
+ @Rule
23
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
24
+
25
+ private Logger log;
26
+ private ConfigSource config;
27
+ private Schema schema;
28
+
29
+ @Before
30
+ public void createResources()
31
+ {
32
+ log = Exec.getLogger(TestFieldWriterSet.class);
33
+ config = config();
34
+ }
35
+
36
+ @Test
37
+ public void validateFieldWriterSet()
38
+ {
39
+ { // if schema doesn't have appropriate time column, it throws ConfigError.
40
+ schema = schema("_c0", Types.STRING, "time", Types.STRING); // not long or timestamp
41
+ try {
42
+ new FieldWriterSet(log, pluginTask(config), schema);
43
+ fail();
44
+ }
45
+ catch (Throwable t) {
46
+ assertTrue(t instanceof ConfigException);
47
+ }
48
+ }
49
+
50
+ { // if schema doesn't have time column and the user doesn't specify time_column option, it throws ConfigError.
51
+ schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
52
+ try {
53
+ new FieldWriterSet(log, pluginTask(config), schema);
54
+ fail();
55
+ }
56
+ catch (Throwable t) {
57
+ assertTrue(t instanceof ConfigException);
58
+ }
59
+ }
60
+
61
+ { // if schema doesn't have a column specified as time_column column, it throws ConfigError
62
+ schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
63
+ try {
64
+ new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c2")), schema);
65
+ fail();
66
+ }
67
+ catch (Throwable t) {
68
+ assertTrue(t instanceof ConfigException);
69
+ }
70
+ }
71
+
72
+ { // if time_column column is not appropriate column type, it throws ConfigError.
73
+ schema = schema("_c0", Types.STRING, "_c1", Types.STRING);
74
+ try {
75
+ new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c1")), schema);
76
+ fail();
77
+ }
78
+ catch (Throwable t) {
79
+ assertTrue(t instanceof ConfigException);
80
+ }
81
+ }
82
+ }
83
+
84
+ @Test
85
+ public void hasTimeColumn()
86
+ {
87
+ { // time column (timestamp type) exists
88
+ Schema schema = schema("time", Types.TIMESTAMP, "_c0", Types.TIMESTAMP);
89
+ FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
90
+
91
+ assertTrue(writers.getFieldWriter(0) instanceof TimestampLongFieldWriter);
92
+ }
93
+
94
+ { // time column (long type) exists
95
+ Schema schema = schema("time", Types.LONG, "_c0", Types.TIMESTAMP);
96
+ FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
97
+
98
+ assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampLongFieldWriter);
99
+
100
+ }
101
+ }
102
+
103
+ @Test
104
+ public void specifiedTimeColumnOption()
105
+ {
106
+ { // time_column option (timestamp type)
107
+ Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.STRING);
108
+ FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
109
+
110
+ assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator);
111
+ }
112
+
113
+ { // time_column option (long type)
114
+ Schema schema = schema("_c0", Types.LONG, "_c1", Types.STRING);
115
+ FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
116
+
117
+ assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator);
118
+ }
119
+
120
+ { // time_column option (typestamp type) if time column exists
121
+ Schema schema = schema("_c0", Types.TIMESTAMP, "time", Types.TIMESTAMP);
122
+ FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
123
+
124
+ assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator); // c0
125
+ assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
126
+ }
127
+
128
+ { // time_column option (long type) if time column exists
129
+ Schema schema = schema("_c0", Types.LONG, "time", Types.TIMESTAMP);
130
+ FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config.deepCopy().set("time_column", "_c0")), schema);
131
+
132
+ assertTrue(writers.getFieldWriter(0) instanceof UnixTimestampFieldDuplicator); // c0
133
+ assertTrue(writers.getFieldWriter(1) instanceof TimestampStringFieldWriter); // renamed column
134
+ }
135
+ }
136
+
137
+ @Test
138
+ public void useFirstTimestampColumn()
139
+ throws Exception
140
+ {
141
+ Schema schema = schema("_c0", Types.TIMESTAMP, "_c1", Types.LONG);
142
+ FieldWriterSet writers = new FieldWriterSet(log, pluginTask(config), schema);
143
+
144
+ assertTrue(writers.getFieldWriter(0) instanceof TimestampFieldLongDuplicator);
145
+ }
146
+ }
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-td
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Muga Nishizawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-17 00:00:00.000000000 Z
11
+ date: 2015-09-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - ~>
23
17
  - !ruby/object:Gem::Version
24
18
  version: '1.0'
19
+ name: bundler
25
20
  prerelease: false
26
21
  type: :development
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
- - - '>='
24
+ - - ~>
32
25
  - !ruby/object:Gem::Version
33
- version: '10.0'
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - '>='
37
31
  - !ruby/object:Gem::Version
38
32
  version: '10.0'
33
+ name: rake
39
34
  prerelease: false
40
35
  type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
41
  description: TreasureData output plugin is an Embulk plugin that loads records to TreasureData read by any input plugins. Search the input plugins by 'embulk-output' keyword.
42
42
  email:
43
43
  - muga.nishizawa@gmail.com
@@ -46,10 +46,13 @@ extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
48
  - .gitignore
49
+ - .travis.yml
49
50
  - CHANGELOG.md
50
51
  - README.md
51
52
  - build.gradle
53
+ - config/checkstyle/checkstyle.xml
52
54
  - embulk-output-td.gemspec
55
+ - gradle/check.gradle
53
56
  - gradle/wrapper/gradle-wrapper.jar
54
57
  - gradle/wrapper/gradle-wrapper.properties
55
58
  - gradlew
@@ -84,10 +87,22 @@ files:
84
87
  - src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java
85
88
  - src/main/java/org/embulk/output/td/RecordWriter.java
86
89
  - src/main/java/org/embulk/output/td/TdOutputPlugin.java
90
+ - src/main/java/org/embulk/output/td/writer/BooleanFieldWriter.java
91
+ - src/main/java/org/embulk/output/td/writer/DoubleFieldWriter.java
92
+ - src/main/java/org/embulk/output/td/writer/FieldWriter.java
93
+ - src/main/java/org/embulk/output/td/writer/FieldWriterSet.java
94
+ - src/main/java/org/embulk/output/td/writer/LongFieldWriter.java
95
+ - src/main/java/org/embulk/output/td/writer/StringFieldWriter.java
96
+ - src/main/java/org/embulk/output/td/writer/TimestampFieldLongDuplicator.java
97
+ - src/main/java/org/embulk/output/td/writer/TimestampLongFieldWriter.java
98
+ - src/main/java/org/embulk/output/td/writer/TimestampStringFieldWriter.java
99
+ - src/main/java/org/embulk/output/td/writer/UnixTimestampFieldDuplicator.java
100
+ - src/main/java/org/embulk/output/td/writer/UnixTimestampLongFieldWriter.java
87
101
  - src/test/java/com/treasuredata/api/TestTdApiClient.java
88
- - src/test/java/org/embulk/output/td/TestFieldWriter.java
102
+ - src/test/java/org/embulk/output/td/TestRecordWriter.java
89
103
  - src/test/java/org/embulk/output/td/TestTdOutputPlugin.java
90
- - classpath/embulk-output-td-0.1.4.jar
104
+ - src/test/java/org/embulk/output/td/writer/TestFieldWriterSet.java
105
+ - classpath/embulk-output-td-0.1.5.jar
91
106
  - classpath/javassist-3.18.1-GA.jar
92
107
  - classpath/jetty-client-9.2.2.v20140723.jar
93
108
  - classpath/jetty-http-9.2.2.v20140723.jar
@@ -1,105 +0,0 @@
1
- package org.embulk.output.td;
2
-
3
- import com.google.common.collect.ImmutableList;
4
- import org.embulk.EmbulkTestRuntime;
5
- import org.embulk.config.ConfigLoader;
6
- import org.embulk.config.ConfigSource;
7
- import org.embulk.output.td.RecordWriter.FieldWriterSet;
8
- import org.embulk.spi.Column;
9
- import org.embulk.spi.ColumnConfig;
10
- import org.embulk.spi.Exec;
11
- import org.embulk.spi.Schema;
12
- import org.embulk.spi.SchemaConfig;
13
- import org.embulk.spi.type.Type;
14
- import org.embulk.spi.type.Types;
15
- import org.junit.Before;
16
- import org.junit.Rule;
17
- import org.junit.Test;
18
- import org.slf4j.Logger;
19
-
20
- import static org.junit.Assert.assertEquals;
21
- import static org.junit.Assert.assertTrue;
22
-
23
- public class TestFieldWriter
24
- {
25
- @Rule
26
- public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
27
-
28
- private Logger log;
29
-
30
- @Before
31
- public void createLogger()
32
- {
33
- log = Exec.getLogger(TestFieldWriter.class);
34
- }
35
-
36
- private ConfigSource config()
37
- {
38
- return new ConfigLoader(runtime.getModelManager()).newConfigSource();
39
- }
40
-
41
- private Schema schema(Column... columns)
42
- {
43
- ImmutableList.Builder<Column> builder = new ImmutableList.Builder<Column>();
44
- for (Column col : columns) {
45
- builder.add(col);
46
- }
47
- return new Schema(builder.build());
48
- }
49
-
50
- private Column column(int index, String name, Type type)
51
- {
52
- return new Column(index, name, type);
53
- }
54
-
55
- private TdOutputPlugin.PluginTask task(ConfigSource outConfig)
56
- {
57
- return outConfig.loadConfig(TdOutputPlugin.PluginTask.class);
58
- }
59
-
60
- @Test
61
- public void test()
62
- {
63
- { // time column exists
64
- // out: config
65
- ConfigSource outConfig = config()
66
- .set("apikey", "xxx")
67
- .set("database", "mydb")
68
- .set("table", "mytbl");
69
-
70
- // schema
71
- Schema schema = schema(
72
- column(0, "time", Types.TIMESTAMP),
73
- column(1, "c1", Types.TIMESTAMP));
74
-
75
- // create field writers
76
- FieldWriterSet writers = new FieldWriterSet(log, task(outConfig), schema);
77
-
78
- assertEquals(schema.getColumnCount(), writers.getFieldCount());
79
- assertTrue(writers.getFieldWriter(0) instanceof RecordWriter.TimestampLongFieldWriter);
80
- assertTrue(writers.getFieldWriter(1) instanceof RecordWriter.TimestampStringFieldWriter);
81
- }
82
-
83
- { // time column doesn't exists. users need to specify another column as time column
84
- // out: config
85
- ConfigSource outConfig = config()
86
- .set("apikey", "xxx")
87
- .set("database", "mydb")
88
- .set("table", "mytbl")
89
- .set("time_column", "c1");
90
- TdOutputPlugin.PluginTask task = outConfig.loadConfig(TdOutputPlugin.PluginTask.class);
91
-
92
- // schema
93
- Schema schema = schema(
94
- column(0, "c0", Types.TIMESTAMP),
95
- column(1, "c1", Types.TIMESTAMP));
96
-
97
- // create field writers
98
- FieldWriterSet writers = new FieldWriterSet(log, task(outConfig), schema);
99
-
100
- assertEquals(schema.getColumnCount() + 1, writers.getFieldCount());
101
- assertTrue(writers.getFieldWriter(0) instanceof RecordWriter.TimestampStringFieldWriter);
102
- assertTrue(writers.getFieldWriter(1) instanceof RecordWriter.TimestampFieldLongDuplicator);
103
- }
104
- }
105
- }