embulk-filter-column 0.5.1 → 0.5.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java +2 -2
- data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java +294 -0
- data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java +47 -69
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 827fdada3d29078b86a59877f66c9e2ebf1f8793
|
|
4
|
+
data.tar.gz: 268ce04ccace8ef31c24aa989bc0ebebc99f3015
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1f59e24aac1088d4d420a1fdb2f02614049bbeaa7afe262689f09860d80d6139a38eabc26cbcacab770a0ee1ecfcd4f9d98fcbe73a64cc344aba3463c50322b4
|
|
7
|
+
data.tar.gz: 81eaf28c207fcdd325cfc62de40acea4dfc1b79555f764d936e420a9bbb185f4a0b908844c875f3c07e441c02bccef66f1a3d8b26bb5310c8b384e0c3d81307f
|
data/CHANGELOG.md
CHANGED
data/build.gradle
CHANGED
|
@@ -244,13 +244,13 @@ public class ColumnVisitorImpl implements ColumnVisitor
|
|
|
244
244
|
pageBuilder.setNull(outputColumn);
|
|
245
245
|
}
|
|
246
246
|
else {
|
|
247
|
-
String jsonPath = new StringBuilder("$.").append(
|
|
247
|
+
String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
|
|
248
248
|
pageBuilder.setJson(outputColumn, jsonVisitor.visit(jsonPath, defaultValue));
|
|
249
249
|
}
|
|
250
250
|
}
|
|
251
251
|
else {
|
|
252
252
|
Value value = pageReader.getJson(inputColumn);
|
|
253
|
-
String jsonPath = new StringBuilder("$.").append(
|
|
253
|
+
String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
|
|
254
254
|
pageBuilder.setJson(outputColumn, jsonVisitor.visit(jsonPath, value));
|
|
255
255
|
}
|
|
256
256
|
}
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
package org.embulk.filter.column;
|
|
2
|
+
|
|
3
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
|
4
|
+
|
|
5
|
+
import org.embulk.EmbulkTestRuntime;
|
|
6
|
+
import org.embulk.config.ConfigLoader;
|
|
7
|
+
import org.embulk.config.ConfigSource;
|
|
8
|
+
import org.embulk.spi.Exec;
|
|
9
|
+
import org.embulk.spi.Page;
|
|
10
|
+
import org.embulk.spi.PageBuilder;
|
|
11
|
+
import org.embulk.spi.PageReader;
|
|
12
|
+
import org.embulk.spi.PageTestUtils;
|
|
13
|
+
import org.embulk.spi.Schema;
|
|
14
|
+
import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
|
15
|
+
import org.embulk.spi.time.Timestamp;
|
|
16
|
+
import org.embulk.spi.util.Pages;
|
|
17
|
+
import org.junit.Before;
|
|
18
|
+
import org.junit.Rule;
|
|
19
|
+
import org.junit.Test;
|
|
20
|
+
import org.msgpack.value.ValueFactory;
|
|
21
|
+
|
|
22
|
+
import static org.embulk.spi.type.Types.BOOLEAN;
|
|
23
|
+
import static org.embulk.spi.type.Types.DOUBLE;
|
|
24
|
+
import static org.embulk.spi.type.Types.JSON;
|
|
25
|
+
import static org.embulk.spi.type.Types.LONG;
|
|
26
|
+
import static org.embulk.spi.type.Types.TIMESTAMP;
|
|
27
|
+
import static org.junit.Assert.assertEquals;
|
|
28
|
+
|
|
29
|
+
import static org.embulk.spi.type.Types.STRING;
|
|
30
|
+
|
|
31
|
+
import java.util.List;
|
|
32
|
+
|
|
33
|
+
public class TestColumnVisitorImpl {
|
|
34
|
+
@Rule
|
|
35
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
|
36
|
+
|
|
37
|
+
@Before
|
|
38
|
+
public void createReasource()
|
|
39
|
+
{
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
private ConfigSource config()
|
|
43
|
+
{
|
|
44
|
+
return runtime.getExec().newConfigSource();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private PluginTask taskFromYamlString(String... lines)
|
|
48
|
+
{
|
|
49
|
+
StringBuilder builder = new StringBuilder();
|
|
50
|
+
for (String line : lines) {
|
|
51
|
+
builder.append(line).append("\n");
|
|
52
|
+
}
|
|
53
|
+
String yamlString = builder.toString();
|
|
54
|
+
|
|
55
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
|
56
|
+
ConfigSource config = loader.fromYamlString(yamlString);
|
|
57
|
+
return config.loadConfig(PluginTask.class);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private List<Object[]> filter(PluginTask task, Schema inputSchema, Object ... objects)
|
|
61
|
+
{
|
|
62
|
+
MockPageOutput output = new MockPageOutput();
|
|
63
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
|
64
|
+
PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
|
|
65
|
+
PageReader pageReader = new PageReader(inputSchema);
|
|
66
|
+
ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);
|
|
67
|
+
|
|
68
|
+
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
|
|
69
|
+
for (Page page : pages) {
|
|
70
|
+
pageReader.setPage(page);
|
|
71
|
+
|
|
72
|
+
while (pageReader.nextRecord()) {
|
|
73
|
+
outputSchema.visitColumns(visitor);
|
|
74
|
+
pageBuilder.addRecord();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
pageBuilder.finish();
|
|
78
|
+
pageBuilder.close();
|
|
79
|
+
return Pages.toObjects(outputSchema, output.pages);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
@Test
|
|
83
|
+
public void visit_Columns_WithDrop()
|
|
84
|
+
{
|
|
85
|
+
PluginTask task = taskFromYamlString(
|
|
86
|
+
"type: column",
|
|
87
|
+
"columns:",
|
|
88
|
+
" - {name: timestamp}",
|
|
89
|
+
" - {name: string}",
|
|
90
|
+
" - {name: boolean}",
|
|
91
|
+
" - {name: long}",
|
|
92
|
+
" - {name: double}",
|
|
93
|
+
" - {name: json}");
|
|
94
|
+
Schema inputSchema = Schema.builder()
|
|
95
|
+
.add("timestamp", TIMESTAMP)
|
|
96
|
+
.add("string", STRING)
|
|
97
|
+
.add("boolean", BOOLEAN)
|
|
98
|
+
.add("long", LONG)
|
|
99
|
+
.add("double", DOUBLE)
|
|
100
|
+
.add("json", JSON)
|
|
101
|
+
.add("remove_me", STRING)
|
|
102
|
+
.build();
|
|
103
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
104
|
+
Timestamp.ofEpochSecond(0), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me",
|
|
105
|
+
Timestamp.ofEpochSecond(0), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me");
|
|
106
|
+
|
|
107
|
+
assertEquals(2, records.size());
|
|
108
|
+
|
|
109
|
+
Object[] record;
|
|
110
|
+
{
|
|
111
|
+
record = records.get(0);
|
|
112
|
+
assertEquals(6, record.length);
|
|
113
|
+
assertEquals(Timestamp.ofEpochSecond(0), record[0]);
|
|
114
|
+
assertEquals("string", record[1]);
|
|
115
|
+
assertEquals(new Boolean(true), record[2]);
|
|
116
|
+
assertEquals(new Long(0), record[3]);
|
|
117
|
+
assertEquals(new Double(0.5), record[4]);
|
|
118
|
+
assertEquals(ValueFactory.newString("json"), record[5]);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
@Test
|
|
123
|
+
public void visit_Columns_WithDefault()
|
|
124
|
+
{
|
|
125
|
+
PluginTask task = taskFromYamlString(
|
|
126
|
+
"type: column",
|
|
127
|
+
"columns:",
|
|
128
|
+
" - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
|
|
129
|
+
" - {name: string, type: string, default: string}",
|
|
130
|
+
" - {name: boolean, type: boolean, default: true}",
|
|
131
|
+
" - {name: long, type: long, default: 0}",
|
|
132
|
+
" - {name: double, type: double, default: 0.5}",
|
|
133
|
+
" - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
|
|
134
|
+
Schema inputSchema = Schema.builder()
|
|
135
|
+
.add("timestamp", TIMESTAMP)
|
|
136
|
+
.add("string", STRING)
|
|
137
|
+
.add("boolean", BOOLEAN)
|
|
138
|
+
.add("long", LONG)
|
|
139
|
+
.add("double", DOUBLE)
|
|
140
|
+
.add("json", JSON)
|
|
141
|
+
.add("remove_me", STRING)
|
|
142
|
+
.build();
|
|
143
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
144
|
+
Timestamp.ofEpochSecond(1436745600), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me",
|
|
145
|
+
null, null, null, null, null, null, "remove_me");
|
|
146
|
+
|
|
147
|
+
assertEquals(2, records.size());
|
|
148
|
+
|
|
149
|
+
Object[] record;
|
|
150
|
+
{
|
|
151
|
+
record = records.get(0);
|
|
152
|
+
assertEquals(6, record.length);
|
|
153
|
+
assertEquals(Timestamp.ofEpochSecond(1436745600), record[0]);
|
|
154
|
+
assertEquals("string", record[1]);
|
|
155
|
+
assertEquals(new Boolean(true), record[2]);
|
|
156
|
+
assertEquals(new Long(0), record[3]);
|
|
157
|
+
assertEquals(new Double(0.5), record[4]);
|
|
158
|
+
assertEquals(ValueFactory.newString("json"), record[5]);
|
|
159
|
+
}
|
|
160
|
+
{
|
|
161
|
+
record = records.get(1);
|
|
162
|
+
assertEquals(6, record.length);
|
|
163
|
+
assertEquals(Timestamp.ofEpochSecond(1436745600), record[0]);
|
|
164
|
+
assertEquals("string", record[1]);
|
|
165
|
+
assertEquals(new Boolean(true), record[2]);
|
|
166
|
+
assertEquals(new Long(0), record[3]);
|
|
167
|
+
assertEquals(new Double(0.5), record[4]);
|
|
168
|
+
assertEquals("{\"foo\":\"bar\"}", record[5].toString());
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
@Test
|
|
173
|
+
public void visit_Columns_WithSrc()
|
|
174
|
+
{
|
|
175
|
+
PluginTask task = taskFromYamlString(
|
|
176
|
+
"type: column",
|
|
177
|
+
"columns:",
|
|
178
|
+
" - {name: copy, src: src}");
|
|
179
|
+
Schema inputSchema = Schema.builder()
|
|
180
|
+
.add("src", STRING)
|
|
181
|
+
.build();
|
|
182
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
183
|
+
"src");
|
|
184
|
+
|
|
185
|
+
assertEquals(1, records.size());
|
|
186
|
+
|
|
187
|
+
Object[] record;
|
|
188
|
+
{
|
|
189
|
+
record = records.get(0);
|
|
190
|
+
assertEquals(1, record.length);
|
|
191
|
+
assertEquals("src", record[0]);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
@Test
|
|
196
|
+
public void visit_DropColumns()
|
|
197
|
+
{
|
|
198
|
+
PluginTask task = taskFromYamlString(
|
|
199
|
+
"type: column",
|
|
200
|
+
"drop_columns:",
|
|
201
|
+
" - {name: timestamp}",
|
|
202
|
+
" - {name: string}",
|
|
203
|
+
" - {name: boolean}",
|
|
204
|
+
" - {name: long}",
|
|
205
|
+
" - {name: double}",
|
|
206
|
+
" - {name: json}");
|
|
207
|
+
Schema inputSchema = Schema.builder()
|
|
208
|
+
.add("timestamp", TIMESTAMP)
|
|
209
|
+
.add("string", STRING)
|
|
210
|
+
.add("boolean", BOOLEAN)
|
|
211
|
+
.add("long", LONG)
|
|
212
|
+
.add("double", DOUBLE)
|
|
213
|
+
.add("json", JSON)
|
|
214
|
+
.add("keep_me", STRING)
|
|
215
|
+
.build();
|
|
216
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
217
|
+
Timestamp.ofEpochSecond(1436745600), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "keep_me",
|
|
218
|
+
null, null, null, null, null, null, "keep_me");
|
|
219
|
+
|
|
220
|
+
assertEquals(2, records.size());
|
|
221
|
+
|
|
222
|
+
Object[] record;
|
|
223
|
+
{
|
|
224
|
+
record = records.get(0);
|
|
225
|
+
assertEquals(1, record.length);
|
|
226
|
+
assertEquals("keep_me", record[0]);
|
|
227
|
+
}
|
|
228
|
+
{
|
|
229
|
+
record = records.get(1);
|
|
230
|
+
assertEquals(1, record.length);
|
|
231
|
+
assertEquals("keep_me", record[0]);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
@Test
|
|
236
|
+
public void visit_AddColumns_WithDefault()
|
|
237
|
+
{
|
|
238
|
+
PluginTask task = taskFromYamlString(
|
|
239
|
+
"type: column",
|
|
240
|
+
"add_columns:",
|
|
241
|
+
" - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
|
|
242
|
+
" - {name: string, type: string, default: string}",
|
|
243
|
+
" - {name: boolean, type: boolean, default: true}",
|
|
244
|
+
" - {name: long, type: long, default: 0}",
|
|
245
|
+
" - {name: double, type: double, default: 0.5}",
|
|
246
|
+
" - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
|
|
247
|
+
Schema inputSchema = Schema.builder()
|
|
248
|
+
.add("keep_me", STRING)
|
|
249
|
+
.build();
|
|
250
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
251
|
+
"keep_me",
|
|
252
|
+
"keep_me");
|
|
253
|
+
|
|
254
|
+
assertEquals(2, records.size());
|
|
255
|
+
|
|
256
|
+
Object[] record;
|
|
257
|
+
{
|
|
258
|
+
record = records.get(0);
|
|
259
|
+
assertEquals(7, record.length);
|
|
260
|
+
assertEquals("keep_me", record[0]);
|
|
261
|
+
assertEquals(Timestamp.ofEpochSecond(1436745600), record[1]);
|
|
262
|
+
assertEquals("string", record[2]);
|
|
263
|
+
assertEquals(new Boolean(true), record[3]);
|
|
264
|
+
assertEquals(new Long(0), record[4]);
|
|
265
|
+
assertEquals(new Double(0.5), record[5]);
|
|
266
|
+
assertEquals("{\"foo\":\"bar\"}", record[6].toString());
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
@Test
|
|
271
|
+
public void visit_AddColumns_WithSrc()
|
|
272
|
+
{
|
|
273
|
+
PluginTask task = taskFromYamlString(
|
|
274
|
+
"type: column",
|
|
275
|
+
"add_columns:",
|
|
276
|
+
" - {name: copy, src: src}");
|
|
277
|
+
Schema inputSchema = Schema.builder()
|
|
278
|
+
.add("src", STRING)
|
|
279
|
+
.build();
|
|
280
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
281
|
+
"src");
|
|
282
|
+
|
|
283
|
+
assertEquals(1, records.size());
|
|
284
|
+
|
|
285
|
+
Object[] record;
|
|
286
|
+
{
|
|
287
|
+
record = records.get(0);
|
|
288
|
+
assertEquals(2, record.length);
|
|
289
|
+
assertEquals("src", record[0]);
|
|
290
|
+
assertEquals("src", record[1]);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
}
|
|
@@ -1,51 +1,29 @@
|
|
|
1
1
|
package org.embulk.filter.column;
|
|
2
2
|
|
|
3
|
-
import com.google.common.base.Optional;
|
|
4
3
|
import com.google.common.collect.Lists;
|
|
5
|
-
import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
|
|
6
4
|
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
|
7
5
|
|
|
8
6
|
import org.embulk.EmbulkTestRuntime;
|
|
9
7
|
import org.embulk.config.ConfigLoader;
|
|
10
8
|
import org.embulk.config.ConfigSource;
|
|
11
|
-
import org.embulk.config.TaskSource;
|
|
12
9
|
import org.embulk.spi.Column;
|
|
13
10
|
import org.embulk.spi.Exec;
|
|
14
|
-
import org.embulk.spi.FileInput;
|
|
15
|
-
import org.embulk.spi.ParserPlugin;
|
|
16
11
|
import org.embulk.spi.Schema;
|
|
17
|
-
import org.embulk.spi.SchemaConfig;
|
|
18
|
-
import org.embulk.spi.type.Type;
|
|
19
|
-
import org.joda.time.DateTimeZone;
|
|
20
12
|
import org.junit.Before;
|
|
21
13
|
import org.junit.Rule;
|
|
22
14
|
import org.junit.Test;
|
|
23
15
|
|
|
24
|
-
import static junit.framework.TestCase.assertFalse;
|
|
25
|
-
import static org.junit.Assert.assertEquals;
|
|
26
|
-
import static org.junit.Assert.fail;
|
|
27
|
-
|
|
28
16
|
import org.msgpack.value.MapValue;
|
|
29
17
|
import org.msgpack.value.Value;
|
|
30
18
|
import org.msgpack.value.ValueFactory;
|
|
31
19
|
|
|
32
|
-
import static org.embulk.spi.type.Types.BOOLEAN;
|
|
33
|
-
import static org.embulk.spi.type.Types.DOUBLE;
|
|
34
20
|
import static org.embulk.spi.type.Types.JSON;
|
|
35
|
-
import static org.embulk.spi.type.Types.LONG;
|
|
36
|
-
import static org.embulk.spi.type.Types.STRING;
|
|
37
|
-
import static org.embulk.spi.type.Types.TIMESTAMP;
|
|
38
21
|
import static org.junit.Assert.assertEquals;
|
|
39
|
-
import static org.junit.Assert.
|
|
22
|
+
import static org.junit.Assert.assertFalse;
|
|
40
23
|
import static org.junit.Assert.assertTrue;
|
|
41
|
-
import static org.junit.Assert.fail;
|
|
42
24
|
|
|
43
|
-
import java.io.File;
|
|
44
|
-
import java.io.IOException;
|
|
45
|
-
import java.util.ArrayList;
|
|
46
25
|
import java.util.HashMap;
|
|
47
26
|
import java.util.HashSet;
|
|
48
|
-
import java.util.List;
|
|
49
27
|
|
|
50
28
|
public class TestJsonVisitor {
|
|
51
29
|
@Rule
|
|
@@ -54,7 +32,6 @@ public class TestJsonVisitor {
|
|
|
54
32
|
@Before
|
|
55
33
|
public void createReasource()
|
|
56
34
|
{
|
|
57
|
-
// config = config().set("type", "column");
|
|
58
35
|
}
|
|
59
36
|
|
|
60
37
|
private ConfigSource config()
|
|
@@ -62,12 +39,7 @@ public class TestJsonVisitor {
|
|
|
62
39
|
return runtime.getExec().newConfigSource();
|
|
63
40
|
}
|
|
64
41
|
|
|
65
|
-
private
|
|
66
|
-
{
|
|
67
|
-
return new Schema(Lists.newArrayList(columns));
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
private ConfigSource configFromYamlString(String... lines)
|
|
42
|
+
private PluginTask taskFromYamlString(String... lines)
|
|
71
43
|
{
|
|
72
44
|
StringBuilder builder = new StringBuilder();
|
|
73
45
|
for (String line : lines) {
|
|
@@ -76,12 +48,7 @@ public class TestJsonVisitor {
|
|
|
76
48
|
String yamlString = builder.toString();
|
|
77
49
|
|
|
78
50
|
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
|
79
|
-
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
private PluginTask taskFromYamlString(String... lines)
|
|
83
|
-
{
|
|
84
|
-
ConfigSource config = configFromYamlString(lines);
|
|
51
|
+
ConfigSource config = loader.fromYamlString(yamlString);
|
|
85
52
|
return config.loadConfig(PluginTask.class);
|
|
86
53
|
}
|
|
87
54
|
|
|
@@ -102,9 +69,10 @@ public class TestJsonVisitor {
|
|
|
102
69
|
" - {name: \"$.json1.b.b[1].b\", type: string, default: foo}",
|
|
103
70
|
"drop_columns:",
|
|
104
71
|
" - {name: \"$.json1.c.c[*].c\"}");
|
|
105
|
-
Schema inputSchema =
|
|
106
|
-
|
|
107
|
-
|
|
72
|
+
Schema inputSchema = Schema.builder()
|
|
73
|
+
.add("json1", JSON)
|
|
74
|
+
.add("json2", JSON)
|
|
75
|
+
.build();
|
|
108
76
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
109
77
|
|
|
110
78
|
assertTrue(subject.shouldVisit("$.json1.a.a.a"));
|
|
@@ -131,9 +99,10 @@ public class TestJsonVisitor {
|
|
|
131
99
|
" - {name: $.json1.a.default}",
|
|
132
100
|
" - {name: $.json1.a.copy}",
|
|
133
101
|
" - {name: \"$.json1.a.copy_array[1]\"}");
|
|
134
|
-
Schema inputSchema =
|
|
135
|
-
|
|
136
|
-
|
|
102
|
+
Schema inputSchema = Schema.builder()
|
|
103
|
+
.add("json1", JSON)
|
|
104
|
+
.add("json2", JSON)
|
|
105
|
+
.build();
|
|
137
106
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
138
107
|
|
|
139
108
|
assertFalse(subject.jsonDropColumns.containsKey("$.json1"));
|
|
@@ -163,9 +132,10 @@ public class TestJsonVisitor {
|
|
|
163
132
|
" - {name: $.json1.a.default, type: string, default: foo}",
|
|
164
133
|
" - {name: $.json1.a.copy, src: $.json1.a.src}",
|
|
165
134
|
" - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
|
|
166
|
-
Schema inputSchema =
|
|
167
|
-
|
|
168
|
-
|
|
135
|
+
Schema inputSchema = Schema.builder()
|
|
136
|
+
.add("json1", JSON)
|
|
137
|
+
.add("json2", JSON)
|
|
138
|
+
.build();
|
|
169
139
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
170
140
|
|
|
171
141
|
assertFalse(subject.jsonAddColumns.containsKey("$.json1"));
|
|
@@ -202,9 +172,10 @@ public class TestJsonVisitor {
|
|
|
202
172
|
" - {name: $.json1.a.default, type: string, default: foo}",
|
|
203
173
|
" - {name: $.json1.a.copy, src: $.json1.a.src}",
|
|
204
174
|
" - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
|
|
205
|
-
Schema inputSchema =
|
|
206
|
-
|
|
207
|
-
|
|
175
|
+
Schema inputSchema = Schema.builder()
|
|
176
|
+
.add("json1", JSON)
|
|
177
|
+
.add("json2", JSON)
|
|
178
|
+
.build();
|
|
208
179
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
209
180
|
|
|
210
181
|
assertFalse(subject.jsonColumns.containsKey("$.json1"));
|
|
@@ -242,9 +213,10 @@ public class TestJsonVisitor {
|
|
|
242
213
|
" - {name: $.json1.a.copy, src: $.json1.a.src}",
|
|
243
214
|
"columns:",
|
|
244
215
|
" - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
|
|
245
|
-
Schema inputSchema =
|
|
246
|
-
|
|
247
|
-
|
|
216
|
+
Schema inputSchema = Schema.builder()
|
|
217
|
+
.add("json1", JSON)
|
|
218
|
+
.add("json2", JSON)
|
|
219
|
+
.build();
|
|
248
220
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
249
221
|
|
|
250
222
|
assertFalse(subject.jsonDropColumns.isEmpty());
|
|
@@ -259,9 +231,10 @@ public class TestJsonVisitor {
|
|
|
259
231
|
"drop_columns:",
|
|
260
232
|
" - {name: $.json1.k1.k1}",
|
|
261
233
|
" - {name: $.json1.k2}");
|
|
262
|
-
Schema inputSchema =
|
|
263
|
-
|
|
264
|
-
|
|
234
|
+
Schema inputSchema = Schema.builder()
|
|
235
|
+
.add("json1", JSON)
|
|
236
|
+
.add("json2", JSON)
|
|
237
|
+
.build();
|
|
265
238
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
266
239
|
|
|
267
240
|
// {"k1":{"k1":"v"},"k2":{"k2":"v"}}
|
|
@@ -284,9 +257,10 @@ public class TestJsonVisitor {
|
|
|
284
257
|
" - {name: $.json1.k3, type: json, default: \"{}\"}",
|
|
285
258
|
" - {name: $.json1.k3.k3, type: string, default: v}",
|
|
286
259
|
" - {name: $.json1.k4, src: $.json1.k2}");
|
|
287
|
-
Schema inputSchema =
|
|
288
|
-
|
|
289
|
-
|
|
260
|
+
Schema inputSchema = Schema.builder()
|
|
261
|
+
.add("json1", JSON)
|
|
262
|
+
.add("json2", JSON)
|
|
263
|
+
.build();
|
|
290
264
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
291
265
|
|
|
292
266
|
// {"k1":{"k1":"v"},"k2":{"k2":"v"}}
|
|
@@ -311,9 +285,10 @@ public class TestJsonVisitor {
|
|
|
311
285
|
" - {name: $.json1.k3, type: json, default: \"{}\"}",
|
|
312
286
|
" - {name: $.json1.k3.k3, type: string, default: v}",
|
|
313
287
|
" - {name: $.json1.k4, src: $.json1.k2}");
|
|
314
|
-
Schema inputSchema =
|
|
315
|
-
|
|
316
|
-
|
|
288
|
+
Schema inputSchema = Schema.builder()
|
|
289
|
+
.add("json1", JSON)
|
|
290
|
+
.add("json2", JSON)
|
|
291
|
+
.build();
|
|
317
292
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
318
293
|
|
|
319
294
|
// {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
|
|
@@ -335,9 +310,10 @@ public class TestJsonVisitor {
|
|
|
335
310
|
"drop_columns:",
|
|
336
311
|
" - {name: \"$.json1.k1[0].k1\"}",
|
|
337
312
|
" - {name: \"$.json1.k2[*]\"}");
|
|
338
|
-
Schema inputSchema =
|
|
339
|
-
|
|
340
|
-
|
|
313
|
+
Schema inputSchema = Schema.builder()
|
|
314
|
+
.add("json1", JSON)
|
|
315
|
+
.add("json2", JSON)
|
|
316
|
+
.build();
|
|
341
317
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
342
318
|
|
|
343
319
|
// {"k1":[{"k1":"v"}[,"k2":["v","v"]}
|
|
@@ -361,9 +337,10 @@ public class TestJsonVisitor {
|
|
|
361
337
|
" - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
|
|
362
338
|
" - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
|
|
363
339
|
" - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
|
|
364
|
-
Schema inputSchema =
|
|
365
|
-
|
|
366
|
-
|
|
340
|
+
Schema inputSchema = Schema.builder()
|
|
341
|
+
.add("json1", JSON)
|
|
342
|
+
.add("json2", JSON)
|
|
343
|
+
.build();
|
|
367
344
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
368
345
|
|
|
369
346
|
// {"k1":[{"k1":"v"}],"k2":["v","v"]}
|
|
@@ -389,9 +366,10 @@ public class TestJsonVisitor {
|
|
|
389
366
|
" - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
|
|
390
367
|
" - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
|
|
391
368
|
" - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
|
|
392
|
-
Schema inputSchema =
|
|
393
|
-
|
|
394
|
-
|
|
369
|
+
Schema inputSchema = Schema.builder()
|
|
370
|
+
.add("json1", JSON)
|
|
371
|
+
.add("json2", JSON)
|
|
372
|
+
.build();
|
|
395
373
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
396
374
|
|
|
397
375
|
// {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-column
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Naotoshi Seo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-06-
|
|
11
|
+
date: 2016-06-06 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -67,9 +67,10 @@ files:
|
|
|
67
67
|
- src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
|
|
68
68
|
- src/main/java/org/embulk/filter/column/JsonColumn.java
|
|
69
69
|
- src/main/java/org/embulk/filter/column/JsonVisitor.java
|
|
70
|
+
- src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java
|
|
70
71
|
- src/test/java/org/embulk/filter/column/TestJsonColumn.java
|
|
71
72
|
- src/test/java/org/embulk/filter/column/TestJsonVisitor.java
|
|
72
|
-
- classpath/embulk-filter-column-0.5.
|
|
73
|
+
- classpath/embulk-filter-column-0.5.2.jar
|
|
73
74
|
homepage: https://github.com/sonots/embulk-filter-column
|
|
74
75
|
licenses:
|
|
75
76
|
- MIT
|