embulk-filter-column 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +18 -4
- data/build.gradle +2 -1
- data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java +0 -4
- data/src/main/java/org/embulk/filter/column/JsonColumn.java +0 -1
- data/src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java +182 -0
- data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java +4 -6
- data/src/test/java/org/embulk/filter/column/TestJsonColumn.java +5 -5
- data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java +17 -13
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b597e430acd8f39ad24000c423fc00ad0396b84c
|
4
|
+
data.tar.gz: 1ab18bcd72e1e49d9aaf65f7343a11b9421e61ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 53d313c43ce6557672c2e4c78782a7089132fb87c9942210d4dd18844f143e3a046f4cb0b8561101100406e0a5df6351652b4afa18ff14ebde30cf818495cad6
|
7
|
+
data.tar.gz: ba2fb51e24bf1925ce078e545bf143345324b6ad332c96fa0184bc0a98b639d4df8d911ea3d3f84fa61c27c8458952a8f7c77f9813b850af4b3205548c6cecc0
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -121,9 +121,9 @@ $.payload.array[*]
|
|
121
121
|
|
122
122
|
EXAMPLE:
|
123
123
|
|
124
|
-
* [example/
|
125
|
-
* [example/
|
126
|
-
* [example/
|
124
|
+
* [example/columns.yml](example/columns.yml)
|
125
|
+
* [example/add_columns.yml](example/add_columns.yml)
|
126
|
+
* [example/drop_columns.yml](example/drop_columns.yml)
|
127
127
|
|
128
128
|
NOTE:
|
129
129
|
|
@@ -159,12 +159,26 @@ $ ./gradlew test jacocoTestReport
|
|
159
159
|
|
160
160
|
open build/reports/jacoco/test/html/index.html
|
161
161
|
|
162
|
-
Run checkstyle:
|
162
|
+
Run checkstyle and findbugs:
|
163
163
|
|
164
164
|
```
|
165
165
|
$ ./gradlew check
|
166
166
|
```
|
167
167
|
|
168
|
+
Run only checkstyle:
|
169
|
+
|
170
|
+
```
|
171
|
+
$ ./gradlew checkstyleMain
|
172
|
+
$ ./gradlew checkstyleTest
|
173
|
+
```
|
174
|
+
|
175
|
+
Run only findbugs:
|
176
|
+
|
177
|
+
```
|
178
|
+
$ ./gradlew findbugsMain
|
179
|
+
$ ./gradlew findbugsTest
|
180
|
+
```
|
181
|
+
|
168
182
|
Release gem:
|
169
183
|
|
170
184
|
```
|
data/build.gradle
CHANGED
@@ -4,6 +4,7 @@ plugins {
|
|
4
4
|
id "java"
|
5
5
|
id "checkstyle"
|
6
6
|
id "jacoco"
|
7
|
+
id "findbugs"
|
7
8
|
}
|
8
9
|
import com.github.jrubygradle.JRubyExec
|
9
10
|
repositories {
|
@@ -14,7 +15,7 @@ configurations {
|
|
14
15
|
provided
|
15
16
|
}
|
16
17
|
|
17
|
-
version = "0.5.
|
18
|
+
version = "0.5.3"
|
18
19
|
sourceCompatibility = 1.7
|
19
20
|
targetCompatibility = 1.7
|
20
21
|
|
@@ -97,10 +97,6 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
97
97
|
List<ColumnConfig> addColumns = task.getAddColumns();
|
98
98
|
List<ColumnConfig> dropColumns = task.getDropColumns();
|
99
99
|
|
100
|
-
if (columns.size() == 0 && addColumns.size() == 0 && dropColumns.size() == 0) {
|
101
|
-
throw new ConfigException("One of \"columns\", \"add_columns\", \"drop_columns\" must be specified.");
|
102
|
-
}
|
103
|
-
|
104
100
|
if (columns.size() > 0 && dropColumns.size() > 0) {
|
105
101
|
throw new ConfigException("Either of \"columns\", \"drop_columns\" can be specified.");
|
106
102
|
}
|
@@ -2,7 +2,6 @@ package org.embulk.filter.column;
|
|
2
2
|
|
3
3
|
import org.embulk.config.ConfigException;
|
4
4
|
import org.embulk.spi.type.Type;
|
5
|
-
import org.msgpack.value.IntegerValue;
|
6
5
|
import org.msgpack.value.StringValue;
|
7
6
|
import org.msgpack.value.Value;
|
8
7
|
import org.msgpack.value.ValueFactory;
|
@@ -0,0 +1,182 @@
|
|
1
|
+
package org.embulk.filter.column;
|
2
|
+
|
3
|
+
import com.google.common.collect.Lists;
|
4
|
+
|
5
|
+
import org.embulk.EmbulkTestRuntime;
|
6
|
+
import org.embulk.config.ConfigException;
|
7
|
+
import org.embulk.config.ConfigLoader;
|
8
|
+
import org.embulk.config.ConfigSource;
|
9
|
+
import org.embulk.config.TaskSource;
|
10
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
11
|
+
import org.embulk.spi.Column;
|
12
|
+
import org.embulk.spi.Exec;
|
13
|
+
import org.embulk.spi.FilterPlugin;
|
14
|
+
import org.embulk.spi.Schema;
|
15
|
+
import org.junit.Before;
|
16
|
+
import org.junit.Rule;
|
17
|
+
import org.junit.Test;
|
18
|
+
|
19
|
+
import static org.embulk.spi.type.Types.BOOLEAN;
|
20
|
+
import static org.embulk.spi.type.Types.DOUBLE;
|
21
|
+
import static org.embulk.spi.type.Types.JSON;
|
22
|
+
import static org.embulk.spi.type.Types.LONG;
|
23
|
+
import static org.embulk.spi.type.Types.STRING;
|
24
|
+
import static org.embulk.spi.type.Types.TIMESTAMP;
|
25
|
+
import static org.junit.Assert.assertEquals;
|
26
|
+
|
27
|
+
public class TestColumnFilterPlugin
|
28
|
+
{
|
29
|
+
@Rule
|
30
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
31
|
+
|
32
|
+
private ColumnFilterPlugin plugin;
|
33
|
+
|
34
|
+
@Before
|
35
|
+
public void createReasource()
|
36
|
+
{
|
37
|
+
plugin = new ColumnFilterPlugin();
|
38
|
+
}
|
39
|
+
|
40
|
+
private Schema schema(Column... columns)
|
41
|
+
{
|
42
|
+
return new Schema(Lists.newArrayList(columns));
|
43
|
+
}
|
44
|
+
|
45
|
+
private ConfigSource configFromYamlString(String... lines)
|
46
|
+
{
|
47
|
+
StringBuilder builder = new StringBuilder();
|
48
|
+
for (String line : lines) {
|
49
|
+
builder.append(line).append("\n");
|
50
|
+
}
|
51
|
+
String yamlString = builder.toString();
|
52
|
+
|
53
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
54
|
+
return loader.fromYamlString(yamlString);
|
55
|
+
}
|
56
|
+
|
57
|
+
private PluginTask taskFromYamlString(String... lines)
|
58
|
+
{
|
59
|
+
ConfigSource config = configFromYamlString(lines);
|
60
|
+
return config.loadConfig(PluginTask.class);
|
61
|
+
}
|
62
|
+
|
63
|
+
private void transaction(ConfigSource config, Schema inputSchema)
|
64
|
+
{
|
65
|
+
plugin.transaction(config, inputSchema, new FilterPlugin.Control() {
|
66
|
+
@Override
|
67
|
+
public void run(TaskSource taskSource, Schema outputSchema)
|
68
|
+
{
|
69
|
+
}
|
70
|
+
});
|
71
|
+
}
|
72
|
+
|
73
|
+
@Test
|
74
|
+
public void buildOutputSchema_Columns()
|
75
|
+
{
|
76
|
+
PluginTask task = taskFromYamlString(
|
77
|
+
"type: column",
|
78
|
+
"columns:",
|
79
|
+
" - {name: timestamp}",
|
80
|
+
" - {name: string}",
|
81
|
+
" - {name: boolean}",
|
82
|
+
" - {name: long}",
|
83
|
+
" - {name: double}",
|
84
|
+
" - {name: json}");
|
85
|
+
Schema inputSchema = Schema.builder()
|
86
|
+
.add("timestamp", TIMESTAMP)
|
87
|
+
.add("string", STRING)
|
88
|
+
.add("boolean", BOOLEAN)
|
89
|
+
.add("long", LONG)
|
90
|
+
.add("double", DOUBLE)
|
91
|
+
.add("json", JSON)
|
92
|
+
.add("remove_me", STRING)
|
93
|
+
.build();
|
94
|
+
|
95
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
96
|
+
assertEquals(6, outputSchema.size());
|
97
|
+
|
98
|
+
Column column;
|
99
|
+
{
|
100
|
+
column = outputSchema.getColumn(0);
|
101
|
+
assertEquals("timestamp", column.getName());
|
102
|
+
}
|
103
|
+
}
|
104
|
+
|
105
|
+
@Test
|
106
|
+
public void buildOutputSchema_DropColumns()
|
107
|
+
{
|
108
|
+
PluginTask task = taskFromYamlString(
|
109
|
+
"type: column",
|
110
|
+
"drop_columns:",
|
111
|
+
" - {name: timestamp}",
|
112
|
+
" - {name: string}",
|
113
|
+
" - {name: boolean}",
|
114
|
+
" - {name: long}",
|
115
|
+
" - {name: double}",
|
116
|
+
" - {name: json}");
|
117
|
+
Schema inputSchema = Schema.builder()
|
118
|
+
.add("timestamp", TIMESTAMP)
|
119
|
+
.add("string", STRING)
|
120
|
+
.add("boolean", BOOLEAN)
|
121
|
+
.add("long", LONG)
|
122
|
+
.add("double", DOUBLE)
|
123
|
+
.add("json", JSON)
|
124
|
+
.add("keep_me", STRING)
|
125
|
+
.build();
|
126
|
+
|
127
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
128
|
+
assertEquals(1, outputSchema.size());
|
129
|
+
|
130
|
+
Column column;
|
131
|
+
{
|
132
|
+
column = outputSchema.getColumn(0);
|
133
|
+
assertEquals("keep_me", column.getName());
|
134
|
+
}
|
135
|
+
}
|
136
|
+
|
137
|
+
@Test
|
138
|
+
public void buildOutputSchema_AddColumns()
|
139
|
+
{
|
140
|
+
PluginTask task = taskFromYamlString(
|
141
|
+
"type: column",
|
142
|
+
"add_columns:",
|
143
|
+
" - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
|
144
|
+
" - {name: string, type: string, default: string}",
|
145
|
+
" - {name: boolean, type: boolean, default: true}",
|
146
|
+
" - {name: long, type: long, default: 0}",
|
147
|
+
" - {name: double, type: double, default: 0.5}",
|
148
|
+
" - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
|
149
|
+
Schema inputSchema = Schema.builder()
|
150
|
+
.add("keep_me", STRING)
|
151
|
+
.build();
|
152
|
+
|
153
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
154
|
+
assertEquals(7, outputSchema.size());
|
155
|
+
|
156
|
+
Column column;
|
157
|
+
{
|
158
|
+
column = outputSchema.getColumn(0);
|
159
|
+
assertEquals("keep_me", column.getName());
|
160
|
+
}
|
161
|
+
{
|
162
|
+
column = outputSchema.getColumn(1);
|
163
|
+
assertEquals("timestamp", column.getName());
|
164
|
+
}
|
165
|
+
}
|
166
|
+
|
167
|
+
@Test(expected = ConfigException.class)
|
168
|
+
public void configure_EitherOfColumnsOrDropColumnsCanBeSpecified()
|
169
|
+
{
|
170
|
+
ConfigSource config = configFromYamlString(
|
171
|
+
"type: column",
|
172
|
+
"columns:",
|
173
|
+
"- {name: a}",
|
174
|
+
"drop_columns:",
|
175
|
+
"- {name: a}");
|
176
|
+
Schema inputSchema = schema(
|
177
|
+
new Column(0, "a", STRING),
|
178
|
+
new Column(1, "b", STRING));
|
179
|
+
|
180
|
+
transaction(config, inputSchema);
|
181
|
+
}
|
182
|
+
}
|
@@ -1,10 +1,9 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
-
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
4
|
-
|
5
3
|
import org.embulk.EmbulkTestRuntime;
|
6
4
|
import org.embulk.config.ConfigLoader;
|
7
5
|
import org.embulk.config.ConfigSource;
|
6
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
8
7
|
import org.embulk.spi.Exec;
|
9
8
|
import org.embulk.spi.Page;
|
10
9
|
import org.embulk.spi.PageBuilder;
|
@@ -23,14 +22,14 @@ import static org.embulk.spi.type.Types.BOOLEAN;
|
|
23
22
|
import static org.embulk.spi.type.Types.DOUBLE;
|
24
23
|
import static org.embulk.spi.type.Types.JSON;
|
25
24
|
import static org.embulk.spi.type.Types.LONG;
|
25
|
+
import static org.embulk.spi.type.Types.STRING;
|
26
26
|
import static org.embulk.spi.type.Types.TIMESTAMP;
|
27
27
|
import static org.junit.Assert.assertEquals;
|
28
28
|
|
29
|
-
import static org.embulk.spi.type.Types.STRING;
|
30
|
-
|
31
29
|
import java.util.List;
|
32
30
|
|
33
|
-
public class TestColumnVisitorImpl
|
31
|
+
public class TestColumnVisitorImpl
|
32
|
+
{
|
34
33
|
@Rule
|
35
34
|
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
36
35
|
|
@@ -290,5 +289,4 @@ public class TestColumnVisitorImpl {
|
|
290
289
|
assertEquals("src", record[1]);
|
291
290
|
}
|
292
291
|
}
|
293
|
-
|
294
292
|
}
|
@@ -1,15 +1,15 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
+
import org.embulk.spi.type.Types;
|
3
4
|
import org.junit.Test;
|
5
|
+
import org.msgpack.value.Value;
|
6
|
+
import org.msgpack.value.ValueFactory;
|
4
7
|
|
5
8
|
import static org.junit.Assert.assertEquals;
|
6
9
|
import static org.junit.Assert.fail;
|
7
10
|
|
8
|
-
|
9
|
-
|
10
|
-
import org.msgpack.value.ValueFactory;
|
11
|
-
|
12
|
-
public class TestJsonColumn {
|
11
|
+
public class TestJsonColumn
|
12
|
+
{
|
13
13
|
@Test
|
14
14
|
public void initialize()
|
15
15
|
{
|
@@ -1,18 +1,14 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
-
import com.google.common.collect.Lists;
|
4
|
-
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
5
|
-
|
6
3
|
import org.embulk.EmbulkTestRuntime;
|
7
4
|
import org.embulk.config.ConfigLoader;
|
8
5
|
import org.embulk.config.ConfigSource;
|
9
|
-
import org.embulk.
|
6
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
10
7
|
import org.embulk.spi.Exec;
|
11
8
|
import org.embulk.spi.Schema;
|
12
9
|
import org.junit.Before;
|
13
10
|
import org.junit.Rule;
|
14
11
|
import org.junit.Test;
|
15
|
-
|
16
12
|
import org.msgpack.value.MapValue;
|
17
13
|
import org.msgpack.value.Value;
|
18
14
|
import org.msgpack.value.ValueFactory;
|
@@ -25,7 +21,8 @@ import static org.junit.Assert.assertTrue;
|
|
25
21
|
import java.util.HashMap;
|
26
22
|
import java.util.HashSet;
|
27
23
|
|
28
|
-
public class TestJsonVisitor
|
24
|
+
public class TestJsonVisitor
|
25
|
+
{
|
29
26
|
@Rule
|
30
27
|
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
31
28
|
|
@@ -204,7 +201,8 @@ public class TestJsonVisitor {
|
|
204
201
|
}
|
205
202
|
|
206
203
|
@Test
|
207
|
-
public void buildJsonSchema_Mix()
|
204
|
+
public void buildJsonSchema_Mix()
|
205
|
+
{
|
208
206
|
PluginTask task = taskFromYamlString(
|
209
207
|
"type: column",
|
210
208
|
"drop_columns:",
|
@@ -225,7 +223,8 @@ public class TestJsonVisitor {
|
|
225
223
|
}
|
226
224
|
|
227
225
|
@Test
|
228
|
-
public void visitMap_DropColumns()
|
226
|
+
public void visitMap_DropColumns()
|
227
|
+
{
|
229
228
|
PluginTask task = taskFromYamlString(
|
230
229
|
"type: column",
|
231
230
|
"drop_columns:",
|
@@ -250,7 +249,8 @@ public class TestJsonVisitor {
|
|
250
249
|
}
|
251
250
|
|
252
251
|
@Test
|
253
|
-
public void visitMap_AddColumns()
|
252
|
+
public void visitMap_AddColumns()
|
253
|
+
{
|
254
254
|
PluginTask task = taskFromYamlString(
|
255
255
|
"type: column",
|
256
256
|
"add_columns:",
|
@@ -276,7 +276,8 @@ public class TestJsonVisitor {
|
|
276
276
|
}
|
277
277
|
|
278
278
|
@Test
|
279
|
-
public void visitMap_Columns()
|
279
|
+
public void visitMap_Columns()
|
280
|
+
{
|
280
281
|
PluginTask task = taskFromYamlString(
|
281
282
|
"type: column",
|
282
283
|
"columns:",
|
@@ -304,7 +305,8 @@ public class TestJsonVisitor {
|
|
304
305
|
}
|
305
306
|
|
306
307
|
@Test
|
307
|
-
public void visitArray_DropColumns()
|
308
|
+
public void visitArray_DropColumns()
|
309
|
+
{
|
308
310
|
PluginTask task = taskFromYamlString(
|
309
311
|
"type: column",
|
310
312
|
"drop_columns:",
|
@@ -329,7 +331,8 @@ public class TestJsonVisitor {
|
|
329
331
|
}
|
330
332
|
|
331
333
|
@Test
|
332
|
-
public void visitArray_AddColumns()
|
334
|
+
public void visitArray_AddColumns()
|
335
|
+
{
|
333
336
|
PluginTask task = taskFromYamlString(
|
334
337
|
"type: column",
|
335
338
|
"add_columns:",
|
@@ -356,7 +359,8 @@ public class TestJsonVisitor {
|
|
356
359
|
}
|
357
360
|
|
358
361
|
@Test
|
359
|
-
public void visitArray_Columns()
|
362
|
+
public void visitArray_Columns()
|
363
|
+
{
|
360
364
|
PluginTask task = taskFromYamlString(
|
361
365
|
"type: column",
|
362
366
|
"columns:",
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-column
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Naotoshi Seo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -67,10 +67,11 @@ files:
|
|
67
67
|
- src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
|
68
68
|
- src/main/java/org/embulk/filter/column/JsonColumn.java
|
69
69
|
- src/main/java/org/embulk/filter/column/JsonVisitor.java
|
70
|
+
- src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java
|
70
71
|
- src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java
|
71
72
|
- src/test/java/org/embulk/filter/column/TestJsonColumn.java
|
72
73
|
- src/test/java/org/embulk/filter/column/TestJsonVisitor.java
|
73
|
-
- classpath/embulk-filter-column-0.5.
|
74
|
+
- classpath/embulk-filter-column-0.5.3.jar
|
74
75
|
homepage: https://github.com/sonots/embulk-filter-column
|
75
76
|
licenses:
|
76
77
|
- MIT
|