embulk-filter-column 0.5.2 → 0.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 827fdada3d29078b86a59877f66c9e2ebf1f8793
4
- data.tar.gz: 268ce04ccace8ef31c24aa989bc0ebebc99f3015
3
+ metadata.gz: b597e430acd8f39ad24000c423fc00ad0396b84c
4
+ data.tar.gz: 1ab18bcd72e1e49d9aaf65f7343a11b9421e61ae
5
5
  SHA512:
6
- metadata.gz: 1f59e24aac1088d4d420a1fdb2f02614049bbeaa7afe262689f09860d80d6139a38eabc26cbcacab770a0ee1ecfcd4f9d98fcbe73a64cc344aba3463c50322b4
7
- data.tar.gz: 81eaf28c207fcdd325cfc62de40acea4dfc1b79555f764d936e420a9bbb185f4a0b908844c875f3c07e441c02bccef66f1a3d8b26bb5310c8b384e0c3d81307f
6
+ metadata.gz: 53d313c43ce6557672c2e4c78782a7089132fb87c9942210d4dd18844f143e3a046f4cb0b8561101100406e0a5df6351652b4afa18ff14ebde30cf818495cad6
7
+ data.tar.gz: ba2fb51e24bf1925ce078e545bf143345324b6ad332c96fa0184bc0a98b639d4df8d911ea3d3f84fa61c27c8458952a8f7c77f9813b850af4b3205548c6cecc0
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ # 0.5.3 (2016-06-21)
2
+
3
+ Changes:
4
+
5
+ * Remove parameter requirements that one of columns or add_columns or drop_columns are required
6
+
1
7
  # 0.5.2 (2016-06-07)
2
8
 
3
9
  Fixes:
data/README.md CHANGED
@@ -121,9 +121,9 @@ $.payload.array[*]
121
121
 
122
122
  EXAMPLE:
123
123
 
124
- * [example/json_columns.yml](example/json_columns.yml)
125
- * [example/json_add_columns.yml](example/json_add_columns.yml)
126
- * [example/json_drop_columns.yml](example/json_drop_columns.yml)
124
+ * [example/columns.yml](example/columns.yml)
125
+ * [example/add_columns.yml](example/add_columns.yml)
126
+ * [example/drop_columns.yml](example/drop_columns.yml)
127
127
 
128
128
  NOTE:
129
129
 
@@ -159,12 +159,26 @@ $ ./gradlew test jacocoTestReport
159
159
 
160
160
  open build/reports/jacoco/test/html/index.html
161
161
 
162
- Run checkstyle:
162
+ Run checkstyle and findbugs:
163
163
 
164
164
  ```
165
165
  $ ./gradlew check
166
166
  ```
167
167
 
168
+ Run only checkstyle:
169
+
170
+ ```
171
+ $ ./gradlew checkstyleMain
172
+ $ ./gradlew checkstyleTest
173
+ ```
174
+
175
+ Run only findbugs:
176
+
177
+ ```
178
+ $ ./gradlew findbugsMain
179
+ $ ./gradlew findbugsTest
180
+ ```
181
+
168
182
  Release gem:
169
183
 
170
184
  ```
data/build.gradle CHANGED
@@ -4,6 +4,7 @@ plugins {
4
4
  id "java"
5
5
  id "checkstyle"
6
6
  id "jacoco"
7
+ id "findbugs"
7
8
  }
8
9
  import com.github.jrubygradle.JRubyExec
9
10
  repositories {
@@ -14,7 +15,7 @@ configurations {
14
15
  provided
15
16
  }
16
17
 
17
- version = "0.5.2"
18
+ version = "0.5.3"
18
19
  sourceCompatibility = 1.7
19
20
  targetCompatibility = 1.7
20
21
 
@@ -97,10 +97,6 @@ public class ColumnFilterPlugin implements FilterPlugin
97
97
  List<ColumnConfig> addColumns = task.getAddColumns();
98
98
  List<ColumnConfig> dropColumns = task.getDropColumns();
99
99
 
100
- if (columns.size() == 0 && addColumns.size() == 0 && dropColumns.size() == 0) {
101
- throw new ConfigException("One of \"columns\", \"add_columns\", \"drop_columns\" must be specified.");
102
- }
103
-
104
100
  if (columns.size() > 0 && dropColumns.size() > 0) {
105
101
  throw new ConfigException("Either of \"columns\", \"drop_columns\" can be specified.");
106
102
  }
@@ -2,7 +2,6 @@ package org.embulk.filter.column;
2
2
 
3
3
  import org.embulk.config.ConfigException;
4
4
  import org.embulk.spi.type.Type;
5
- import org.msgpack.value.IntegerValue;
6
5
  import org.msgpack.value.StringValue;
7
6
  import org.msgpack.value.Value;
8
7
  import org.msgpack.value.ValueFactory;
@@ -0,0 +1,182 @@
1
+ package org.embulk.filter.column;
2
+
3
+ import com.google.common.collect.Lists;
4
+
5
+ import org.embulk.EmbulkTestRuntime;
6
+ import org.embulk.config.ConfigException;
7
+ import org.embulk.config.ConfigLoader;
8
+ import org.embulk.config.ConfigSource;
9
+ import org.embulk.config.TaskSource;
10
+ import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
11
+ import org.embulk.spi.Column;
12
+ import org.embulk.spi.Exec;
13
+ import org.embulk.spi.FilterPlugin;
14
+ import org.embulk.spi.Schema;
15
+ import org.junit.Before;
16
+ import org.junit.Rule;
17
+ import org.junit.Test;
18
+
19
+ import static org.embulk.spi.type.Types.BOOLEAN;
20
+ import static org.embulk.spi.type.Types.DOUBLE;
21
+ import static org.embulk.spi.type.Types.JSON;
22
+ import static org.embulk.spi.type.Types.LONG;
23
+ import static org.embulk.spi.type.Types.STRING;
24
+ import static org.embulk.spi.type.Types.TIMESTAMP;
25
+ import static org.junit.Assert.assertEquals;
26
+
27
+ public class TestColumnFilterPlugin
28
+ {
29
+ @Rule
30
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
31
+
32
+ private ColumnFilterPlugin plugin;
33
+
34
+ @Before
35
+ public void createReasource()
36
+ {
37
+ plugin = new ColumnFilterPlugin();
38
+ }
39
+
40
+ private Schema schema(Column... columns)
41
+ {
42
+ return new Schema(Lists.newArrayList(columns));
43
+ }
44
+
45
+ private ConfigSource configFromYamlString(String... lines)
46
+ {
47
+ StringBuilder builder = new StringBuilder();
48
+ for (String line : lines) {
49
+ builder.append(line).append("\n");
50
+ }
51
+ String yamlString = builder.toString();
52
+
53
+ ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
54
+ return loader.fromYamlString(yamlString);
55
+ }
56
+
57
+ private PluginTask taskFromYamlString(String... lines)
58
+ {
59
+ ConfigSource config = configFromYamlString(lines);
60
+ return config.loadConfig(PluginTask.class);
61
+ }
62
+
63
+ private void transaction(ConfigSource config, Schema inputSchema)
64
+ {
65
+ plugin.transaction(config, inputSchema, new FilterPlugin.Control() {
66
+ @Override
67
+ public void run(TaskSource taskSource, Schema outputSchema)
68
+ {
69
+ }
70
+ });
71
+ }
72
+
73
+ @Test
74
+ public void buildOutputSchema_Columns()
75
+ {
76
+ PluginTask task = taskFromYamlString(
77
+ "type: column",
78
+ "columns:",
79
+ " - {name: timestamp}",
80
+ " - {name: string}",
81
+ " - {name: boolean}",
82
+ " - {name: long}",
83
+ " - {name: double}",
84
+ " - {name: json}");
85
+ Schema inputSchema = Schema.builder()
86
+ .add("timestamp", TIMESTAMP)
87
+ .add("string", STRING)
88
+ .add("boolean", BOOLEAN)
89
+ .add("long", LONG)
90
+ .add("double", DOUBLE)
91
+ .add("json", JSON)
92
+ .add("remove_me", STRING)
93
+ .build();
94
+
95
+ Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
96
+ assertEquals(6, outputSchema.size());
97
+
98
+ Column column;
99
+ {
100
+ column = outputSchema.getColumn(0);
101
+ assertEquals("timestamp", column.getName());
102
+ }
103
+ }
104
+
105
+ @Test
106
+ public void buildOutputSchema_DropColumns()
107
+ {
108
+ PluginTask task = taskFromYamlString(
109
+ "type: column",
110
+ "drop_columns:",
111
+ " - {name: timestamp}",
112
+ " - {name: string}",
113
+ " - {name: boolean}",
114
+ " - {name: long}",
115
+ " - {name: double}",
116
+ " - {name: json}");
117
+ Schema inputSchema = Schema.builder()
118
+ .add("timestamp", TIMESTAMP)
119
+ .add("string", STRING)
120
+ .add("boolean", BOOLEAN)
121
+ .add("long", LONG)
122
+ .add("double", DOUBLE)
123
+ .add("json", JSON)
124
+ .add("keep_me", STRING)
125
+ .build();
126
+
127
+ Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
128
+ assertEquals(1, outputSchema.size());
129
+
130
+ Column column;
131
+ {
132
+ column = outputSchema.getColumn(0);
133
+ assertEquals("keep_me", column.getName());
134
+ }
135
+ }
136
+
137
+ @Test
138
+ public void buildOutputSchema_AddColumns()
139
+ {
140
+ PluginTask task = taskFromYamlString(
141
+ "type: column",
142
+ "add_columns:",
143
+ " - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
144
+ " - {name: string, type: string, default: string}",
145
+ " - {name: boolean, type: boolean, default: true}",
146
+ " - {name: long, type: long, default: 0}",
147
+ " - {name: double, type: double, default: 0.5}",
148
+ " - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
149
+ Schema inputSchema = Schema.builder()
150
+ .add("keep_me", STRING)
151
+ .build();
152
+
153
+ Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
154
+ assertEquals(7, outputSchema.size());
155
+
156
+ Column column;
157
+ {
158
+ column = outputSchema.getColumn(0);
159
+ assertEquals("keep_me", column.getName());
160
+ }
161
+ {
162
+ column = outputSchema.getColumn(1);
163
+ assertEquals("timestamp", column.getName());
164
+ }
165
+ }
166
+
167
+ @Test(expected = ConfigException.class)
168
+ public void configure_EitherOfColumnsOrDropColumnsCanBeSpecified()
169
+ {
170
+ ConfigSource config = configFromYamlString(
171
+ "type: column",
172
+ "columns:",
173
+ "- {name: a}",
174
+ "drop_columns:",
175
+ "- {name: a}");
176
+ Schema inputSchema = schema(
177
+ new Column(0, "a", STRING),
178
+ new Column(1, "b", STRING));
179
+
180
+ transaction(config, inputSchema);
181
+ }
182
+ }
@@ -1,10 +1,9 @@
1
1
  package org.embulk.filter.column;
2
2
 
3
- import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
4
-
5
3
  import org.embulk.EmbulkTestRuntime;
6
4
  import org.embulk.config.ConfigLoader;
7
5
  import org.embulk.config.ConfigSource;
6
+ import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
8
7
  import org.embulk.spi.Exec;
9
8
  import org.embulk.spi.Page;
10
9
  import org.embulk.spi.PageBuilder;
@@ -23,14 +22,14 @@ import static org.embulk.spi.type.Types.BOOLEAN;
23
22
  import static org.embulk.spi.type.Types.DOUBLE;
24
23
  import static org.embulk.spi.type.Types.JSON;
25
24
  import static org.embulk.spi.type.Types.LONG;
25
+ import static org.embulk.spi.type.Types.STRING;
26
26
  import static org.embulk.spi.type.Types.TIMESTAMP;
27
27
  import static org.junit.Assert.assertEquals;
28
28
 
29
- import static org.embulk.spi.type.Types.STRING;
30
-
31
29
  import java.util.List;
32
30
 
33
- public class TestColumnVisitorImpl {
31
+ public class TestColumnVisitorImpl
32
+ {
34
33
  @Rule
35
34
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
36
35
 
@@ -290,5 +289,4 @@ public class TestColumnVisitorImpl {
290
289
  assertEquals("src", record[1]);
291
290
  }
292
291
  }
293
-
294
292
  }
@@ -1,15 +1,15 @@
1
1
  package org.embulk.filter.column;
2
2
 
3
+ import org.embulk.spi.type.Types;
3
4
  import org.junit.Test;
5
+ import org.msgpack.value.Value;
6
+ import org.msgpack.value.ValueFactory;
4
7
 
5
8
  import static org.junit.Assert.assertEquals;
6
9
  import static org.junit.Assert.fail;
7
10
 
8
- import org.embulk.spi.type.Types;
9
- import org.msgpack.value.Value;
10
- import org.msgpack.value.ValueFactory;
11
-
12
- public class TestJsonColumn {
11
+ public class TestJsonColumn
12
+ {
13
13
  @Test
14
14
  public void initialize()
15
15
  {
@@ -1,18 +1,14 @@
1
1
  package org.embulk.filter.column;
2
2
 
3
- import com.google.common.collect.Lists;
4
- import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
5
-
6
3
  import org.embulk.EmbulkTestRuntime;
7
4
  import org.embulk.config.ConfigLoader;
8
5
  import org.embulk.config.ConfigSource;
9
- import org.embulk.spi.Column;
6
+ import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
10
7
  import org.embulk.spi.Exec;
11
8
  import org.embulk.spi.Schema;
12
9
  import org.junit.Before;
13
10
  import org.junit.Rule;
14
11
  import org.junit.Test;
15
-
16
12
  import org.msgpack.value.MapValue;
17
13
  import org.msgpack.value.Value;
18
14
  import org.msgpack.value.ValueFactory;
@@ -25,7 +21,8 @@ import static org.junit.Assert.assertTrue;
25
21
  import java.util.HashMap;
26
22
  import java.util.HashSet;
27
23
 
28
- public class TestJsonVisitor {
24
+ public class TestJsonVisitor
25
+ {
29
26
  @Rule
30
27
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
31
28
 
@@ -204,7 +201,8 @@ public class TestJsonVisitor {
204
201
  }
205
202
 
206
203
  @Test
207
- public void buildJsonSchema_Mix() {
204
+ public void buildJsonSchema_Mix()
205
+ {
208
206
  PluginTask task = taskFromYamlString(
209
207
  "type: column",
210
208
  "drop_columns:",
@@ -225,7 +223,8 @@ public class TestJsonVisitor {
225
223
  }
226
224
 
227
225
  @Test
228
- public void visitMap_DropColumns() {
226
+ public void visitMap_DropColumns()
227
+ {
229
228
  PluginTask task = taskFromYamlString(
230
229
  "type: column",
231
230
  "drop_columns:",
@@ -250,7 +249,8 @@ public class TestJsonVisitor {
250
249
  }
251
250
 
252
251
  @Test
253
- public void visitMap_AddColumns() {
252
+ public void visitMap_AddColumns()
253
+ {
254
254
  PluginTask task = taskFromYamlString(
255
255
  "type: column",
256
256
  "add_columns:",
@@ -276,7 +276,8 @@ public class TestJsonVisitor {
276
276
  }
277
277
 
278
278
  @Test
279
- public void visitMap_Columns() {
279
+ public void visitMap_Columns()
280
+ {
280
281
  PluginTask task = taskFromYamlString(
281
282
  "type: column",
282
283
  "columns:",
@@ -304,7 +305,8 @@ public class TestJsonVisitor {
304
305
  }
305
306
 
306
307
  @Test
307
- public void visitArray_DropColumns() {
308
+ public void visitArray_DropColumns()
309
+ {
308
310
  PluginTask task = taskFromYamlString(
309
311
  "type: column",
310
312
  "drop_columns:",
@@ -329,7 +331,8 @@ public class TestJsonVisitor {
329
331
  }
330
332
 
331
333
  @Test
332
- public void visitArray_AddColumns() {
334
+ public void visitArray_AddColumns()
335
+ {
333
336
  PluginTask task = taskFromYamlString(
334
337
  "type: column",
335
338
  "add_columns:",
@@ -356,7 +359,8 @@ public class TestJsonVisitor {
356
359
  }
357
360
 
358
361
  @Test
359
- public void visitArray_Columns() {
362
+ public void visitArray_Columns()
363
+ {
360
364
  PluginTask task = taskFromYamlString(
361
365
  "type: column",
362
366
  "columns:",
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-filter-column
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
4
+ version: 0.5.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-06 00:00:00.000000000 Z
11
+ date: 2016-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -67,10 +67,11 @@ files:
67
67
  - src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
68
68
  - src/main/java/org/embulk/filter/column/JsonColumn.java
69
69
  - src/main/java/org/embulk/filter/column/JsonVisitor.java
70
+ - src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java
70
71
  - src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java
71
72
  - src/test/java/org/embulk/filter/column/TestJsonColumn.java
72
73
  - src/test/java/org/embulk/filter/column/TestJsonVisitor.java
73
- - classpath/embulk-filter-column-0.5.2.jar
74
+ - classpath/embulk-filter-column-0.5.3.jar
74
75
  homepage: https://github.com/sonots/embulk-filter-column
75
76
  licenses:
76
77
  - MIT