embulk-filter-column 0.5.2 → 0.5.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 827fdada3d29078b86a59877f66c9e2ebf1f8793
4
- data.tar.gz: 268ce04ccace8ef31c24aa989bc0ebebc99f3015
3
+ metadata.gz: b597e430acd8f39ad24000c423fc00ad0396b84c
4
+ data.tar.gz: 1ab18bcd72e1e49d9aaf65f7343a11b9421e61ae
5
5
  SHA512:
6
- metadata.gz: 1f59e24aac1088d4d420a1fdb2f02614049bbeaa7afe262689f09860d80d6139a38eabc26cbcacab770a0ee1ecfcd4f9d98fcbe73a64cc344aba3463c50322b4
7
- data.tar.gz: 81eaf28c207fcdd325cfc62de40acea4dfc1b79555f764d936e420a9bbb185f4a0b908844c875f3c07e441c02bccef66f1a3d8b26bb5310c8b384e0c3d81307f
6
+ metadata.gz: 53d313c43ce6557672c2e4c78782a7089132fb87c9942210d4dd18844f143e3a046f4cb0b8561101100406e0a5df6351652b4afa18ff14ebde30cf818495cad6
7
+ data.tar.gz: ba2fb51e24bf1925ce078e545bf143345324b6ad332c96fa0184bc0a98b639d4df8d911ea3d3f84fa61c27c8458952a8f7c77f9813b850af4b3205548c6cecc0
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ # 0.5.3 (2016-06-21)
2
+
3
+ Changes:
4
+
5
+ * Remove parameter requirements that one of columns or add_columns or drop_columns are required
6
+
1
7
  # 0.5.2 (2016-06-07)
2
8
 
3
9
  Fixes:
data/README.md CHANGED
@@ -121,9 +121,9 @@ $.payload.array[*]
121
121
 
122
122
  EXAMPLE:
123
123
 
124
- * [example/json_columns.yml](example/json_columns.yml)
125
- * [example/json_add_columns.yml](example/json_add_columns.yml)
126
- * [example/json_drop_columns.yml](example/json_drop_columns.yml)
124
+ * [example/columns.yml](example/columns.yml)
125
+ * [example/add_columns.yml](example/add_columns.yml)
126
+ * [example/drop_columns.yml](example/drop_columns.yml)
127
127
 
128
128
  NOTE:
129
129
 
@@ -159,12 +159,26 @@ $ ./gradlew test jacocoTestReport
159
159
 
160
160
  open build/reports/jacoco/test/html/index.html
161
161
 
162
- Run checkstyle:
162
+ Run checkstyle and findbugs:
163
163
 
164
164
  ```
165
165
  $ ./gradlew check
166
166
  ```
167
167
 
168
+ Run only checkstyle:
169
+
170
+ ```
171
+ $ ./gradlew checkstyleMain
172
+ $ ./gradlew checkstyleTest
173
+ ```
174
+
175
+ Run only findbugs:
176
+
177
+ ```
178
+ $ ./gradlew findbugsMain
179
+ $ ./gradlew findbugsTest
180
+ ```
181
+
168
182
  Release gem:
169
183
 
170
184
  ```
data/build.gradle CHANGED
@@ -4,6 +4,7 @@ plugins {
4
4
  id "java"
5
5
  id "checkstyle"
6
6
  id "jacoco"
7
+ id "findbugs"
7
8
  }
8
9
  import com.github.jrubygradle.JRubyExec
9
10
  repositories {
@@ -14,7 +15,7 @@ configurations {
14
15
  provided
15
16
  }
16
17
 
17
- version = "0.5.2"
18
+ version = "0.5.3"
18
19
  sourceCompatibility = 1.7
19
20
  targetCompatibility = 1.7
20
21
 
@@ -97,10 +97,6 @@ public class ColumnFilterPlugin implements FilterPlugin
97
97
  List<ColumnConfig> addColumns = task.getAddColumns();
98
98
  List<ColumnConfig> dropColumns = task.getDropColumns();
99
99
 
100
- if (columns.size() == 0 && addColumns.size() == 0 && dropColumns.size() == 0) {
101
- throw new ConfigException("One of \"columns\", \"add_columns\", \"drop_columns\" must be specified.");
102
- }
103
-
104
100
  if (columns.size() > 0 && dropColumns.size() > 0) {
105
101
  throw new ConfigException("Either of \"columns\", \"drop_columns\" can be specified.");
106
102
  }
@@ -2,7 +2,6 @@ package org.embulk.filter.column;
2
2
 
3
3
  import org.embulk.config.ConfigException;
4
4
  import org.embulk.spi.type.Type;
5
- import org.msgpack.value.IntegerValue;
6
5
  import org.msgpack.value.StringValue;
7
6
  import org.msgpack.value.Value;
8
7
  import org.msgpack.value.ValueFactory;
@@ -0,0 +1,182 @@
1
+ package org.embulk.filter.column;
2
+
3
+ import com.google.common.collect.Lists;
4
+
5
+ import org.embulk.EmbulkTestRuntime;
6
+ import org.embulk.config.ConfigException;
7
+ import org.embulk.config.ConfigLoader;
8
+ import org.embulk.config.ConfigSource;
9
+ import org.embulk.config.TaskSource;
10
+ import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
11
+ import org.embulk.spi.Column;
12
+ import org.embulk.spi.Exec;
13
+ import org.embulk.spi.FilterPlugin;
14
+ import org.embulk.spi.Schema;
15
+ import org.junit.Before;
16
+ import org.junit.Rule;
17
+ import org.junit.Test;
18
+
19
+ import static org.embulk.spi.type.Types.BOOLEAN;
20
+ import static org.embulk.spi.type.Types.DOUBLE;
21
+ import static org.embulk.spi.type.Types.JSON;
22
+ import static org.embulk.spi.type.Types.LONG;
23
+ import static org.embulk.spi.type.Types.STRING;
24
+ import static org.embulk.spi.type.Types.TIMESTAMP;
25
+ import static org.junit.Assert.assertEquals;
26
+
27
+ public class TestColumnFilterPlugin
28
+ {
29
+ @Rule
30
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
31
+
32
+ private ColumnFilterPlugin plugin;
33
+
34
+ @Before
35
+ public void createReasource()
36
+ {
37
+ plugin = new ColumnFilterPlugin();
38
+ }
39
+
40
+ private Schema schema(Column... columns)
41
+ {
42
+ return new Schema(Lists.newArrayList(columns));
43
+ }
44
+
45
+ private ConfigSource configFromYamlString(String... lines)
46
+ {
47
+ StringBuilder builder = new StringBuilder();
48
+ for (String line : lines) {
49
+ builder.append(line).append("\n");
50
+ }
51
+ String yamlString = builder.toString();
52
+
53
+ ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
54
+ return loader.fromYamlString(yamlString);
55
+ }
56
+
57
+ private PluginTask taskFromYamlString(String... lines)
58
+ {
59
+ ConfigSource config = configFromYamlString(lines);
60
+ return config.loadConfig(PluginTask.class);
61
+ }
62
+
63
+ private void transaction(ConfigSource config, Schema inputSchema)
64
+ {
65
+ plugin.transaction(config, inputSchema, new FilterPlugin.Control() {
66
+ @Override
67
+ public void run(TaskSource taskSource, Schema outputSchema)
68
+ {
69
+ }
70
+ });
71
+ }
72
+
73
+ @Test
74
+ public void buildOutputSchema_Columns()
75
+ {
76
+ PluginTask task = taskFromYamlString(
77
+ "type: column",
78
+ "columns:",
79
+ " - {name: timestamp}",
80
+ " - {name: string}",
81
+ " - {name: boolean}",
82
+ " - {name: long}",
83
+ " - {name: double}",
84
+ " - {name: json}");
85
+ Schema inputSchema = Schema.builder()
86
+ .add("timestamp", TIMESTAMP)
87
+ .add("string", STRING)
88
+ .add("boolean", BOOLEAN)
89
+ .add("long", LONG)
90
+ .add("double", DOUBLE)
91
+ .add("json", JSON)
92
+ .add("remove_me", STRING)
93
+ .build();
94
+
95
+ Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
96
+ assertEquals(6, outputSchema.size());
97
+
98
+ Column column;
99
+ {
100
+ column = outputSchema.getColumn(0);
101
+ assertEquals("timestamp", column.getName());
102
+ }
103
+ }
104
+
105
+ @Test
106
+ public void buildOutputSchema_DropColumns()
107
+ {
108
+ PluginTask task = taskFromYamlString(
109
+ "type: column",
110
+ "drop_columns:",
111
+ " - {name: timestamp}",
112
+ " - {name: string}",
113
+ " - {name: boolean}",
114
+ " - {name: long}",
115
+ " - {name: double}",
116
+ " - {name: json}");
117
+ Schema inputSchema = Schema.builder()
118
+ .add("timestamp", TIMESTAMP)
119
+ .add("string", STRING)
120
+ .add("boolean", BOOLEAN)
121
+ .add("long", LONG)
122
+ .add("double", DOUBLE)
123
+ .add("json", JSON)
124
+ .add("keep_me", STRING)
125
+ .build();
126
+
127
+ Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
128
+ assertEquals(1, outputSchema.size());
129
+
130
+ Column column;
131
+ {
132
+ column = outputSchema.getColumn(0);
133
+ assertEquals("keep_me", column.getName());
134
+ }
135
+ }
136
+
137
+ @Test
138
+ public void buildOutputSchema_AddColumns()
139
+ {
140
+ PluginTask task = taskFromYamlString(
141
+ "type: column",
142
+ "add_columns:",
143
+ " - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
144
+ " - {name: string, type: string, default: string}",
145
+ " - {name: boolean, type: boolean, default: true}",
146
+ " - {name: long, type: long, default: 0}",
147
+ " - {name: double, type: double, default: 0.5}",
148
+ " - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
149
+ Schema inputSchema = Schema.builder()
150
+ .add("keep_me", STRING)
151
+ .build();
152
+
153
+ Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
154
+ assertEquals(7, outputSchema.size());
155
+
156
+ Column column;
157
+ {
158
+ column = outputSchema.getColumn(0);
159
+ assertEquals("keep_me", column.getName());
160
+ }
161
+ {
162
+ column = outputSchema.getColumn(1);
163
+ assertEquals("timestamp", column.getName());
164
+ }
165
+ }
166
+
167
+ @Test(expected = ConfigException.class)
168
+ public void configure_EitherOfColumnsOrDropColumnsCanBeSpecified()
169
+ {
170
+ ConfigSource config = configFromYamlString(
171
+ "type: column",
172
+ "columns:",
173
+ "- {name: a}",
174
+ "drop_columns:",
175
+ "- {name: a}");
176
+ Schema inputSchema = schema(
177
+ new Column(0, "a", STRING),
178
+ new Column(1, "b", STRING));
179
+
180
+ transaction(config, inputSchema);
181
+ }
182
+ }
@@ -1,10 +1,9 @@
1
1
  package org.embulk.filter.column;
2
2
 
3
- import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
4
-
5
3
  import org.embulk.EmbulkTestRuntime;
6
4
  import org.embulk.config.ConfigLoader;
7
5
  import org.embulk.config.ConfigSource;
6
+ import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
8
7
  import org.embulk.spi.Exec;
9
8
  import org.embulk.spi.Page;
10
9
  import org.embulk.spi.PageBuilder;
@@ -23,14 +22,14 @@ import static org.embulk.spi.type.Types.BOOLEAN;
23
22
  import static org.embulk.spi.type.Types.DOUBLE;
24
23
  import static org.embulk.spi.type.Types.JSON;
25
24
  import static org.embulk.spi.type.Types.LONG;
25
+ import static org.embulk.spi.type.Types.STRING;
26
26
  import static org.embulk.spi.type.Types.TIMESTAMP;
27
27
  import static org.junit.Assert.assertEquals;
28
28
 
29
- import static org.embulk.spi.type.Types.STRING;
30
-
31
29
  import java.util.List;
32
30
 
33
- public class TestColumnVisitorImpl {
31
+ public class TestColumnVisitorImpl
32
+ {
34
33
  @Rule
35
34
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
36
35
 
@@ -290,5 +289,4 @@ public class TestColumnVisitorImpl {
290
289
  assertEquals("src", record[1]);
291
290
  }
292
291
  }
293
-
294
292
  }
@@ -1,15 +1,15 @@
1
1
  package org.embulk.filter.column;
2
2
 
3
+ import org.embulk.spi.type.Types;
3
4
  import org.junit.Test;
5
+ import org.msgpack.value.Value;
6
+ import org.msgpack.value.ValueFactory;
4
7
 
5
8
  import static org.junit.Assert.assertEquals;
6
9
  import static org.junit.Assert.fail;
7
10
 
8
- import org.embulk.spi.type.Types;
9
- import org.msgpack.value.Value;
10
- import org.msgpack.value.ValueFactory;
11
-
12
- public class TestJsonColumn {
11
+ public class TestJsonColumn
12
+ {
13
13
  @Test
14
14
  public void initialize()
15
15
  {
@@ -1,18 +1,14 @@
1
1
  package org.embulk.filter.column;
2
2
 
3
- import com.google.common.collect.Lists;
4
- import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
5
-
6
3
  import org.embulk.EmbulkTestRuntime;
7
4
  import org.embulk.config.ConfigLoader;
8
5
  import org.embulk.config.ConfigSource;
9
- import org.embulk.spi.Column;
6
+ import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
10
7
  import org.embulk.spi.Exec;
11
8
  import org.embulk.spi.Schema;
12
9
  import org.junit.Before;
13
10
  import org.junit.Rule;
14
11
  import org.junit.Test;
15
-
16
12
  import org.msgpack.value.MapValue;
17
13
  import org.msgpack.value.Value;
18
14
  import org.msgpack.value.ValueFactory;
@@ -25,7 +21,8 @@ import static org.junit.Assert.assertTrue;
25
21
  import java.util.HashMap;
26
22
  import java.util.HashSet;
27
23
 
28
- public class TestJsonVisitor {
24
+ public class TestJsonVisitor
25
+ {
29
26
  @Rule
30
27
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
31
28
 
@@ -204,7 +201,8 @@ public class TestJsonVisitor {
204
201
  }
205
202
 
206
203
  @Test
207
- public void buildJsonSchema_Mix() {
204
+ public void buildJsonSchema_Mix()
205
+ {
208
206
  PluginTask task = taskFromYamlString(
209
207
  "type: column",
210
208
  "drop_columns:",
@@ -225,7 +223,8 @@ public class TestJsonVisitor {
225
223
  }
226
224
 
227
225
  @Test
228
- public void visitMap_DropColumns() {
226
+ public void visitMap_DropColumns()
227
+ {
229
228
  PluginTask task = taskFromYamlString(
230
229
  "type: column",
231
230
  "drop_columns:",
@@ -250,7 +249,8 @@ public class TestJsonVisitor {
250
249
  }
251
250
 
252
251
  @Test
253
- public void visitMap_AddColumns() {
252
+ public void visitMap_AddColumns()
253
+ {
254
254
  PluginTask task = taskFromYamlString(
255
255
  "type: column",
256
256
  "add_columns:",
@@ -276,7 +276,8 @@ public class TestJsonVisitor {
276
276
  }
277
277
 
278
278
  @Test
279
- public void visitMap_Columns() {
279
+ public void visitMap_Columns()
280
+ {
280
281
  PluginTask task = taskFromYamlString(
281
282
  "type: column",
282
283
  "columns:",
@@ -304,7 +305,8 @@ public class TestJsonVisitor {
304
305
  }
305
306
 
306
307
  @Test
307
- public void visitArray_DropColumns() {
308
+ public void visitArray_DropColumns()
309
+ {
308
310
  PluginTask task = taskFromYamlString(
309
311
  "type: column",
310
312
  "drop_columns:",
@@ -329,7 +331,8 @@ public class TestJsonVisitor {
329
331
  }
330
332
 
331
333
  @Test
332
- public void visitArray_AddColumns() {
334
+ public void visitArray_AddColumns()
335
+ {
333
336
  PluginTask task = taskFromYamlString(
334
337
  "type: column",
335
338
  "add_columns:",
@@ -356,7 +359,8 @@ public class TestJsonVisitor {
356
359
  }
357
360
 
358
361
  @Test
359
- public void visitArray_Columns() {
362
+ public void visitArray_Columns()
363
+ {
360
364
  PluginTask task = taskFromYamlString(
361
365
  "type: column",
362
366
  "columns:",
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-filter-column
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
4
+ version: 0.5.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-06 00:00:00.000000000 Z
11
+ date: 2016-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -67,10 +67,11 @@ files:
67
67
  - src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
68
68
  - src/main/java/org/embulk/filter/column/JsonColumn.java
69
69
  - src/main/java/org/embulk/filter/column/JsonVisitor.java
70
+ - src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java
70
71
  - src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java
71
72
  - src/test/java/org/embulk/filter/column/TestJsonColumn.java
72
73
  - src/test/java/org/embulk/filter/column/TestJsonVisitor.java
73
- - classpath/embulk-filter-column-0.5.2.jar
74
+ - classpath/embulk-filter-column-0.5.3.jar
74
75
  homepage: https://github.com/sonots/embulk-filter-column
75
76
  licenses:
76
77
  - MIT