embulk-filter-column 0.5.2 → 0.5.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +18 -4
- data/build.gradle +2 -1
- data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java +0 -4
- data/src/main/java/org/embulk/filter/column/JsonColumn.java +0 -1
- data/src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java +182 -0
- data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java +4 -6
- data/src/test/java/org/embulk/filter/column/TestJsonColumn.java +5 -5
- data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java +17 -13
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b597e430acd8f39ad24000c423fc00ad0396b84c
|
4
|
+
data.tar.gz: 1ab18bcd72e1e49d9aaf65f7343a11b9421e61ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 53d313c43ce6557672c2e4c78782a7089132fb87c9942210d4dd18844f143e3a046f4cb0b8561101100406e0a5df6351652b4afa18ff14ebde30cf818495cad6
|
7
|
+
data.tar.gz: ba2fb51e24bf1925ce078e545bf143345324b6ad332c96fa0184bc0a98b639d4df8d911ea3d3f84fa61c27c8458952a8f7c77f9813b850af4b3205548c6cecc0
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -121,9 +121,9 @@ $.payload.array[*]
|
|
121
121
|
|
122
122
|
EXAMPLE:
|
123
123
|
|
124
|
-
* [example/
|
125
|
-
* [example/
|
126
|
-
* [example/
|
124
|
+
* [example/columns.yml](example/columns.yml)
|
125
|
+
* [example/add_columns.yml](example/add_columns.yml)
|
126
|
+
* [example/drop_columns.yml](example/drop_columns.yml)
|
127
127
|
|
128
128
|
NOTE:
|
129
129
|
|
@@ -159,12 +159,26 @@ $ ./gradlew test jacocoTestReport
|
|
159
159
|
|
160
160
|
open build/reports/jacoco/test/html/index.html
|
161
161
|
|
162
|
-
Run checkstyle:
|
162
|
+
Run checkstyle and findbugs:
|
163
163
|
|
164
164
|
```
|
165
165
|
$ ./gradlew check
|
166
166
|
```
|
167
167
|
|
168
|
+
Run only checkstyle:
|
169
|
+
|
170
|
+
```
|
171
|
+
$ ./gradlew checkstyleMain
|
172
|
+
$ ./gradlew checkstyleTest
|
173
|
+
```
|
174
|
+
|
175
|
+
Run only findbugs:
|
176
|
+
|
177
|
+
```
|
178
|
+
$ ./gradlew findbugsMain
|
179
|
+
$ ./gradlew findbugsTest
|
180
|
+
```
|
181
|
+
|
168
182
|
Release gem:
|
169
183
|
|
170
184
|
```
|
data/build.gradle
CHANGED
@@ -4,6 +4,7 @@ plugins {
|
|
4
4
|
id "java"
|
5
5
|
id "checkstyle"
|
6
6
|
id "jacoco"
|
7
|
+
id "findbugs"
|
7
8
|
}
|
8
9
|
import com.github.jrubygradle.JRubyExec
|
9
10
|
repositories {
|
@@ -14,7 +15,7 @@ configurations {
|
|
14
15
|
provided
|
15
16
|
}
|
16
17
|
|
17
|
-
version = "0.5.
|
18
|
+
version = "0.5.3"
|
18
19
|
sourceCompatibility = 1.7
|
19
20
|
targetCompatibility = 1.7
|
20
21
|
|
@@ -97,10 +97,6 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
97
97
|
List<ColumnConfig> addColumns = task.getAddColumns();
|
98
98
|
List<ColumnConfig> dropColumns = task.getDropColumns();
|
99
99
|
|
100
|
-
if (columns.size() == 0 && addColumns.size() == 0 && dropColumns.size() == 0) {
|
101
|
-
throw new ConfigException("One of \"columns\", \"add_columns\", \"drop_columns\" must be specified.");
|
102
|
-
}
|
103
|
-
|
104
100
|
if (columns.size() > 0 && dropColumns.size() > 0) {
|
105
101
|
throw new ConfigException("Either of \"columns\", \"drop_columns\" can be specified.");
|
106
102
|
}
|
@@ -2,7 +2,6 @@ package org.embulk.filter.column;
|
|
2
2
|
|
3
3
|
import org.embulk.config.ConfigException;
|
4
4
|
import org.embulk.spi.type.Type;
|
5
|
-
import org.msgpack.value.IntegerValue;
|
6
5
|
import org.msgpack.value.StringValue;
|
7
6
|
import org.msgpack.value.Value;
|
8
7
|
import org.msgpack.value.ValueFactory;
|
@@ -0,0 +1,182 @@
|
|
1
|
+
package org.embulk.filter.column;
|
2
|
+
|
3
|
+
import com.google.common.collect.Lists;
|
4
|
+
|
5
|
+
import org.embulk.EmbulkTestRuntime;
|
6
|
+
import org.embulk.config.ConfigException;
|
7
|
+
import org.embulk.config.ConfigLoader;
|
8
|
+
import org.embulk.config.ConfigSource;
|
9
|
+
import org.embulk.config.TaskSource;
|
10
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
11
|
+
import org.embulk.spi.Column;
|
12
|
+
import org.embulk.spi.Exec;
|
13
|
+
import org.embulk.spi.FilterPlugin;
|
14
|
+
import org.embulk.spi.Schema;
|
15
|
+
import org.junit.Before;
|
16
|
+
import org.junit.Rule;
|
17
|
+
import org.junit.Test;
|
18
|
+
|
19
|
+
import static org.embulk.spi.type.Types.BOOLEAN;
|
20
|
+
import static org.embulk.spi.type.Types.DOUBLE;
|
21
|
+
import static org.embulk.spi.type.Types.JSON;
|
22
|
+
import static org.embulk.spi.type.Types.LONG;
|
23
|
+
import static org.embulk.spi.type.Types.STRING;
|
24
|
+
import static org.embulk.spi.type.Types.TIMESTAMP;
|
25
|
+
import static org.junit.Assert.assertEquals;
|
26
|
+
|
27
|
+
public class TestColumnFilterPlugin
|
28
|
+
{
|
29
|
+
@Rule
|
30
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
31
|
+
|
32
|
+
private ColumnFilterPlugin plugin;
|
33
|
+
|
34
|
+
@Before
|
35
|
+
public void createReasource()
|
36
|
+
{
|
37
|
+
plugin = new ColumnFilterPlugin();
|
38
|
+
}
|
39
|
+
|
40
|
+
private Schema schema(Column... columns)
|
41
|
+
{
|
42
|
+
return new Schema(Lists.newArrayList(columns));
|
43
|
+
}
|
44
|
+
|
45
|
+
private ConfigSource configFromYamlString(String... lines)
|
46
|
+
{
|
47
|
+
StringBuilder builder = new StringBuilder();
|
48
|
+
for (String line : lines) {
|
49
|
+
builder.append(line).append("\n");
|
50
|
+
}
|
51
|
+
String yamlString = builder.toString();
|
52
|
+
|
53
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
54
|
+
return loader.fromYamlString(yamlString);
|
55
|
+
}
|
56
|
+
|
57
|
+
private PluginTask taskFromYamlString(String... lines)
|
58
|
+
{
|
59
|
+
ConfigSource config = configFromYamlString(lines);
|
60
|
+
return config.loadConfig(PluginTask.class);
|
61
|
+
}
|
62
|
+
|
63
|
+
private void transaction(ConfigSource config, Schema inputSchema)
|
64
|
+
{
|
65
|
+
plugin.transaction(config, inputSchema, new FilterPlugin.Control() {
|
66
|
+
@Override
|
67
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
68
|
+
{
|
69
|
+
}
|
70
|
+
});
|
71
|
+
}
|
72
|
+
|
73
|
+
@Test
|
74
|
+
public void buildOutputSchema_Columns()
|
75
|
+
{
|
76
|
+
PluginTask task = taskFromYamlString(
|
77
|
+
"type: column",
|
78
|
+
"columns:",
|
79
|
+
" - {name: timestamp}",
|
80
|
+
" - {name: string}",
|
81
|
+
" - {name: boolean}",
|
82
|
+
" - {name: long}",
|
83
|
+
" - {name: double}",
|
84
|
+
" - {name: json}");
|
85
|
+
Schema inputSchema = Schema.builder()
|
86
|
+
.add("timestamp", TIMESTAMP)
|
87
|
+
.add("string", STRING)
|
88
|
+
.add("boolean", BOOLEAN)
|
89
|
+
.add("long", LONG)
|
90
|
+
.add("double", DOUBLE)
|
91
|
+
.add("json", JSON)
|
92
|
+
.add("remove_me", STRING)
|
93
|
+
.build();
|
94
|
+
|
95
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
96
|
+
assertEquals(6, outputSchema.size());
|
97
|
+
|
98
|
+
Column column;
|
99
|
+
{
|
100
|
+
column = outputSchema.getColumn(0);
|
101
|
+
assertEquals("timestamp", column.getName());
|
102
|
+
}
|
103
|
+
}
|
104
|
+
|
105
|
+
@Test
|
106
|
+
public void buildOutputSchema_DropColumns()
|
107
|
+
{
|
108
|
+
PluginTask task = taskFromYamlString(
|
109
|
+
"type: column",
|
110
|
+
"drop_columns:",
|
111
|
+
" - {name: timestamp}",
|
112
|
+
" - {name: string}",
|
113
|
+
" - {name: boolean}",
|
114
|
+
" - {name: long}",
|
115
|
+
" - {name: double}",
|
116
|
+
" - {name: json}");
|
117
|
+
Schema inputSchema = Schema.builder()
|
118
|
+
.add("timestamp", TIMESTAMP)
|
119
|
+
.add("string", STRING)
|
120
|
+
.add("boolean", BOOLEAN)
|
121
|
+
.add("long", LONG)
|
122
|
+
.add("double", DOUBLE)
|
123
|
+
.add("json", JSON)
|
124
|
+
.add("keep_me", STRING)
|
125
|
+
.build();
|
126
|
+
|
127
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
128
|
+
assertEquals(1, outputSchema.size());
|
129
|
+
|
130
|
+
Column column;
|
131
|
+
{
|
132
|
+
column = outputSchema.getColumn(0);
|
133
|
+
assertEquals("keep_me", column.getName());
|
134
|
+
}
|
135
|
+
}
|
136
|
+
|
137
|
+
@Test
|
138
|
+
public void buildOutputSchema_AddColumns()
|
139
|
+
{
|
140
|
+
PluginTask task = taskFromYamlString(
|
141
|
+
"type: column",
|
142
|
+
"add_columns:",
|
143
|
+
" - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
|
144
|
+
" - {name: string, type: string, default: string}",
|
145
|
+
" - {name: boolean, type: boolean, default: true}",
|
146
|
+
" - {name: long, type: long, default: 0}",
|
147
|
+
" - {name: double, type: double, default: 0.5}",
|
148
|
+
" - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
|
149
|
+
Schema inputSchema = Schema.builder()
|
150
|
+
.add("keep_me", STRING)
|
151
|
+
.build();
|
152
|
+
|
153
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
154
|
+
assertEquals(7, outputSchema.size());
|
155
|
+
|
156
|
+
Column column;
|
157
|
+
{
|
158
|
+
column = outputSchema.getColumn(0);
|
159
|
+
assertEquals("keep_me", column.getName());
|
160
|
+
}
|
161
|
+
{
|
162
|
+
column = outputSchema.getColumn(1);
|
163
|
+
assertEquals("timestamp", column.getName());
|
164
|
+
}
|
165
|
+
}
|
166
|
+
|
167
|
+
@Test(expected = ConfigException.class)
|
168
|
+
public void configure_EitherOfColumnsOrDropColumnsCanBeSpecified()
|
169
|
+
{
|
170
|
+
ConfigSource config = configFromYamlString(
|
171
|
+
"type: column",
|
172
|
+
"columns:",
|
173
|
+
"- {name: a}",
|
174
|
+
"drop_columns:",
|
175
|
+
"- {name: a}");
|
176
|
+
Schema inputSchema = schema(
|
177
|
+
new Column(0, "a", STRING),
|
178
|
+
new Column(1, "b", STRING));
|
179
|
+
|
180
|
+
transaction(config, inputSchema);
|
181
|
+
}
|
182
|
+
}
|
@@ -1,10 +1,9 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
-
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
4
|
-
|
5
3
|
import org.embulk.EmbulkTestRuntime;
|
6
4
|
import org.embulk.config.ConfigLoader;
|
7
5
|
import org.embulk.config.ConfigSource;
|
6
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
8
7
|
import org.embulk.spi.Exec;
|
9
8
|
import org.embulk.spi.Page;
|
10
9
|
import org.embulk.spi.PageBuilder;
|
@@ -23,14 +22,14 @@ import static org.embulk.spi.type.Types.BOOLEAN;
|
|
23
22
|
import static org.embulk.spi.type.Types.DOUBLE;
|
24
23
|
import static org.embulk.spi.type.Types.JSON;
|
25
24
|
import static org.embulk.spi.type.Types.LONG;
|
25
|
+
import static org.embulk.spi.type.Types.STRING;
|
26
26
|
import static org.embulk.spi.type.Types.TIMESTAMP;
|
27
27
|
import static org.junit.Assert.assertEquals;
|
28
28
|
|
29
|
-
import static org.embulk.spi.type.Types.STRING;
|
30
|
-
|
31
29
|
import java.util.List;
|
32
30
|
|
33
|
-
public class TestColumnVisitorImpl
|
31
|
+
public class TestColumnVisitorImpl
|
32
|
+
{
|
34
33
|
@Rule
|
35
34
|
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
36
35
|
|
@@ -290,5 +289,4 @@ public class TestColumnVisitorImpl {
|
|
290
289
|
assertEquals("src", record[1]);
|
291
290
|
}
|
292
291
|
}
|
293
|
-
|
294
292
|
}
|
@@ -1,15 +1,15 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
+
import org.embulk.spi.type.Types;
|
3
4
|
import org.junit.Test;
|
5
|
+
import org.msgpack.value.Value;
|
6
|
+
import org.msgpack.value.ValueFactory;
|
4
7
|
|
5
8
|
import static org.junit.Assert.assertEquals;
|
6
9
|
import static org.junit.Assert.fail;
|
7
10
|
|
8
|
-
|
9
|
-
|
10
|
-
import org.msgpack.value.ValueFactory;
|
11
|
-
|
12
|
-
public class TestJsonColumn {
|
11
|
+
public class TestJsonColumn
|
12
|
+
{
|
13
13
|
@Test
|
14
14
|
public void initialize()
|
15
15
|
{
|
@@ -1,18 +1,14 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
-
import com.google.common.collect.Lists;
|
4
|
-
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
5
|
-
|
6
3
|
import org.embulk.EmbulkTestRuntime;
|
7
4
|
import org.embulk.config.ConfigLoader;
|
8
5
|
import org.embulk.config.ConfigSource;
|
9
|
-
import org.embulk.
|
6
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
10
7
|
import org.embulk.spi.Exec;
|
11
8
|
import org.embulk.spi.Schema;
|
12
9
|
import org.junit.Before;
|
13
10
|
import org.junit.Rule;
|
14
11
|
import org.junit.Test;
|
15
|
-
|
16
12
|
import org.msgpack.value.MapValue;
|
17
13
|
import org.msgpack.value.Value;
|
18
14
|
import org.msgpack.value.ValueFactory;
|
@@ -25,7 +21,8 @@ import static org.junit.Assert.assertTrue;
|
|
25
21
|
import java.util.HashMap;
|
26
22
|
import java.util.HashSet;
|
27
23
|
|
28
|
-
public class TestJsonVisitor
|
24
|
+
public class TestJsonVisitor
|
25
|
+
{
|
29
26
|
@Rule
|
30
27
|
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
31
28
|
|
@@ -204,7 +201,8 @@ public class TestJsonVisitor {
|
|
204
201
|
}
|
205
202
|
|
206
203
|
@Test
|
207
|
-
public void buildJsonSchema_Mix()
|
204
|
+
public void buildJsonSchema_Mix()
|
205
|
+
{
|
208
206
|
PluginTask task = taskFromYamlString(
|
209
207
|
"type: column",
|
210
208
|
"drop_columns:",
|
@@ -225,7 +223,8 @@ public class TestJsonVisitor {
|
|
225
223
|
}
|
226
224
|
|
227
225
|
@Test
|
228
|
-
public void visitMap_DropColumns()
|
226
|
+
public void visitMap_DropColumns()
|
227
|
+
{
|
229
228
|
PluginTask task = taskFromYamlString(
|
230
229
|
"type: column",
|
231
230
|
"drop_columns:",
|
@@ -250,7 +249,8 @@ public class TestJsonVisitor {
|
|
250
249
|
}
|
251
250
|
|
252
251
|
@Test
|
253
|
-
public void visitMap_AddColumns()
|
252
|
+
public void visitMap_AddColumns()
|
253
|
+
{
|
254
254
|
PluginTask task = taskFromYamlString(
|
255
255
|
"type: column",
|
256
256
|
"add_columns:",
|
@@ -276,7 +276,8 @@ public class TestJsonVisitor {
|
|
276
276
|
}
|
277
277
|
|
278
278
|
@Test
|
279
|
-
public void visitMap_Columns()
|
279
|
+
public void visitMap_Columns()
|
280
|
+
{
|
280
281
|
PluginTask task = taskFromYamlString(
|
281
282
|
"type: column",
|
282
283
|
"columns:",
|
@@ -304,7 +305,8 @@ public class TestJsonVisitor {
|
|
304
305
|
}
|
305
306
|
|
306
307
|
@Test
|
307
|
-
public void visitArray_DropColumns()
|
308
|
+
public void visitArray_DropColumns()
|
309
|
+
{
|
308
310
|
PluginTask task = taskFromYamlString(
|
309
311
|
"type: column",
|
310
312
|
"drop_columns:",
|
@@ -329,7 +331,8 @@ public class TestJsonVisitor {
|
|
329
331
|
}
|
330
332
|
|
331
333
|
@Test
|
332
|
-
public void visitArray_AddColumns()
|
334
|
+
public void visitArray_AddColumns()
|
335
|
+
{
|
333
336
|
PluginTask task = taskFromYamlString(
|
334
337
|
"type: column",
|
335
338
|
"add_columns:",
|
@@ -356,7 +359,8 @@ public class TestJsonVisitor {
|
|
356
359
|
}
|
357
360
|
|
358
361
|
@Test
|
359
|
-
public void visitArray_Columns()
|
362
|
+
public void visitArray_Columns()
|
363
|
+
{
|
360
364
|
PluginTask task = taskFromYamlString(
|
361
365
|
"type: column",
|
362
366
|
"columns:",
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-column
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naotoshi Seo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -67,10 +67,11 @@ files:
|
|
67
67
|
- src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
|
68
68
|
- src/main/java/org/embulk/filter/column/JsonColumn.java
|
69
69
|
- src/main/java/org/embulk/filter/column/JsonVisitor.java
|
70
|
+
- src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java
|
70
71
|
- src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java
|
71
72
|
- src/test/java/org/embulk/filter/column/TestJsonColumn.java
|
72
73
|
- src/test/java/org/embulk/filter/column/TestJsonVisitor.java
|
73
|
-
- classpath/embulk-filter-column-0.5.
|
74
|
+
- classpath/embulk-filter-column-0.5.3.jar
|
74
75
|
homepage: https://github.com/sonots/embulk-filter-column
|
75
76
|
licenses:
|
76
77
|
- MIT
|