embulk-filter-column 0.5.4 → 0.6.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/CHANGELOG.md +6 -0
- data/README.md +1 -0
- data/build.gradle +2 -1
- data/example/bracket_notations.yml +40 -0
- data/example/edgecase.tsv +4 -0
- data/example/edgecase.yml +22 -0
- data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java +7 -5
- data/src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java +3 -2
- data/src/main/java/org/embulk/filter/column/JsonColumn.java +80 -66
- data/src/main/java/org/embulk/filter/column/JsonVisitor.java +33 -34
- data/src/main/java/org/embulk/filter/column/PathTokenUtil.java +39 -0
- data/src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java +1 -1
- data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java +1 -1
- data/src/test/java/org/embulk/filter/column/TestJsonColumn.java +38 -16
- data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java +439 -51
- metadata +15 -5
@@ -0,0 +1,39 @@
|
|
1
|
+
package org.embulk.filter.column;
|
2
|
+
|
3
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayIndexOperation;
|
4
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayPathToken;
|
5
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.FunctionPathToken;
|
6
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
|
7
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PredicatePathToken;
|
8
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.ScanPathToken;
|
9
|
+
import org.embulk.config.ConfigException;
|
10
|
+
|
11
|
+
public class PathTokenUtil
|
12
|
+
{
|
13
|
+
public static void assertSupportedPathToken(PathToken pathToken, String path)
|
14
|
+
{
|
15
|
+
if (pathToken instanceof ArrayPathToken) {
|
16
|
+
ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) pathToken).getArrayIndexOperation();
|
17
|
+
assertSupportedArrayPathToken(arrayIndexOperation, path);
|
18
|
+
}
|
19
|
+
else if (pathToken instanceof ScanPathToken) {
|
20
|
+
throw new ConfigException(String.format("scan path token is not supported \"%s\"", path));
|
21
|
+
}
|
22
|
+
else if (pathToken instanceof FunctionPathToken) {
|
23
|
+
throw new ConfigException(String.format("function path token is not supported \"%s\"", path));
|
24
|
+
}
|
25
|
+
else if (pathToken instanceof PredicatePathToken) {
|
26
|
+
throw new ConfigException(String.format("predicate path token is not supported \"%s\"", path));
|
27
|
+
}
|
28
|
+
}
|
29
|
+
|
30
|
+
public static void assertSupportedArrayPathToken(ArrayIndexOperation arrayIndexOperation, String path)
|
31
|
+
{
|
32
|
+
if (arrayIndexOperation == null) {
|
33
|
+
throw new ConfigException(String.format("Array Slice Operation is not supported \"%s\"", path));
|
34
|
+
}
|
35
|
+
else if (!arrayIndexOperation.isSingleIndexOperation()) {
|
36
|
+
throw new ConfigException(String.format("Multi Array Indexes is not supported \"%s\"", path));
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
@@ -15,7 +15,7 @@ public class TestJsonColumn
|
|
15
15
|
{
|
16
16
|
try {
|
17
17
|
JsonColumn column = new JsonColumn("$.foo.bar", Types.BOOLEAN);
|
18
|
-
assertEquals("
|
18
|
+
assertEquals("$['foo']['bar']", column.getSrc());
|
19
19
|
assertEquals(ValueFactory.newNil(), column.getDefaultValue());
|
20
20
|
}
|
21
21
|
catch (Exception e) {
|
@@ -24,8 +24,8 @@ public class TestJsonColumn
|
|
24
24
|
|
25
25
|
try {
|
26
26
|
Value defaultValue = ValueFactory.newBoolean(true);
|
27
|
-
JsonColumn column = new JsonColumn("
|
28
|
-
assertEquals("
|
27
|
+
JsonColumn column = new JsonColumn("$['foo']['bar']", Types.BOOLEAN, defaultValue);
|
28
|
+
assertEquals("$['foo']['bar']", column.getSrc());
|
29
29
|
assertEquals(defaultValue, column.getDefaultValue());
|
30
30
|
}
|
31
31
|
catch (Exception e) {
|
@@ -36,22 +36,44 @@ public class TestJsonColumn
|
|
36
36
|
@Test
|
37
37
|
public void parentPath()
|
38
38
|
{
|
39
|
-
assertEquals("
|
40
|
-
assertEquals("
|
41
|
-
assertEquals("$", JsonColumn.parentPath("
|
42
|
-
assertEquals("
|
43
|
-
assertEquals("
|
39
|
+
assertEquals("$['foo']['bar']", JsonColumn.parentPath("$.foo.bar.baz"));
|
40
|
+
assertEquals("$['foo']", JsonColumn.parentPath("$.foo.bar"));
|
41
|
+
assertEquals("$", JsonColumn.parentPath("$['foo']"));
|
42
|
+
assertEquals("$['foo'][0]", JsonColumn.parentPath("$.foo[0][1]"));
|
43
|
+
assertEquals("$['foo']", JsonColumn.parentPath("$.foo[0]"));
|
44
44
|
assertEquals("$", JsonColumn.parentPath("$[0]"));
|
45
45
|
}
|
46
46
|
|
47
47
|
@Test
|
48
|
-
public void
|
48
|
+
public void tailName()
|
49
49
|
{
|
50
|
-
assertEquals("baz", JsonColumn.
|
51
|
-
assertEquals("bar", JsonColumn.
|
52
|
-
assertEquals("foo", JsonColumn.
|
53
|
-
assertEquals("[1]", JsonColumn.
|
54
|
-
assertEquals("[0]", JsonColumn.
|
55
|
-
assertEquals("[0]", JsonColumn.
|
50
|
+
assertEquals("['baz']", JsonColumn.tailName("$['foo'].bar.baz"));
|
51
|
+
assertEquals("['bar']", JsonColumn.tailName("$.foo.bar"));
|
52
|
+
assertEquals("['foo']", JsonColumn.tailName("$.foo"));
|
53
|
+
assertEquals("[1]", JsonColumn.tailName("$.foo[0][1]"));
|
54
|
+
assertEquals("[0]", JsonColumn.tailName("$.foo[0]"));
|
55
|
+
assertEquals("[0]", JsonColumn.tailName("$[0]"));
|
56
56
|
}
|
57
|
-
|
57
|
+
|
58
|
+
@Test
|
59
|
+
public void getTailNameValue()
|
60
|
+
{
|
61
|
+
assertEquals(ValueFactory.newString("baz"), new JsonColumn("$['foo'].bar.baz", Types.BOOLEAN).getTailNameValue());
|
62
|
+
assertEquals(ValueFactory.newString("bar"), new JsonColumn("$.foo.bar", Types.BOOLEAN).getTailNameValue());
|
63
|
+
assertEquals(ValueFactory.newString("foo"), new JsonColumn("$.foo", Types.BOOLEAN).getTailNameValue());
|
64
|
+
assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0][1]", Types.BOOLEAN).getTailNameValue());
|
65
|
+
assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0]", Types.BOOLEAN).getTailNameValue());
|
66
|
+
assertEquals(ValueFactory.newNil(), new JsonColumn("$[0]", Types.BOOLEAN).getTailNameValue());
|
67
|
+
}
|
68
|
+
|
69
|
+
@Test
|
70
|
+
public void tailIndex()
|
71
|
+
{
|
72
|
+
assertEquals(null, JsonColumn.tailIndex("$['foo'].bar.baz"));
|
73
|
+
assertEquals(null, JsonColumn.tailIndex("$.foo.bar"));
|
74
|
+
assertEquals(null, JsonColumn.tailIndex("$.foo"));
|
75
|
+
assertEquals(new Long(1), JsonColumn.tailIndex("$.foo[0][1]"));
|
76
|
+
assertEquals(new Long(0), JsonColumn.tailIndex("$.foo[0]"));
|
77
|
+
assertEquals(new Long(0), JsonColumn.tailIndex("$[0]"));
|
78
|
+
}
|
79
|
+
}
|
@@ -13,6 +13,7 @@ import org.junit.Test;
|
|
13
13
|
import org.msgpack.value.MapValue;
|
14
14
|
import org.msgpack.value.Value;
|
15
15
|
import org.msgpack.value.ValueFactory;
|
16
|
+
import com.dena.analytics.jsonpathcompiler.InvalidPathException;
|
16
17
|
|
17
18
|
import static org.embulk.spi.type.Types.JSON;
|
18
19
|
import static org.junit.Assert.assertEquals;
|
@@ -28,7 +29,7 @@ public class TestJsonVisitor
|
|
28
29
|
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
29
30
|
|
30
31
|
@Before
|
31
|
-
public void
|
32
|
+
public void createResource()
|
32
33
|
{
|
33
34
|
}
|
34
35
|
|
@@ -85,19 +86,19 @@ public class TestJsonVisitor
|
|
85
86
|
.build();
|
86
87
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
87
88
|
|
88
|
-
assertTrue(subject.shouldVisit("
|
89
|
-
assertTrue(subject.shouldVisit("
|
90
|
-
assertTrue(subject.shouldVisit("
|
91
|
-
assertTrue(subject.shouldVisit("
|
92
|
-
assertTrue(subject.shouldVisit("
|
93
|
-
assertTrue(subject.shouldVisit("
|
94
|
-
assertTrue(subject.shouldVisit("
|
95
|
-
assertTrue(subject.shouldVisit("
|
96
|
-
assertTrue(subject.shouldVisit("
|
97
|
-
assertTrue(subject.shouldVisit("
|
98
|
-
assertTrue(subject.shouldVisit("
|
99
|
-
assertTrue(subject.shouldVisit("
|
100
|
-
assertFalse(subject.shouldVisit("
|
89
|
+
assertTrue(subject.shouldVisit("$['json1']['a']['a']['a']"));
|
90
|
+
assertTrue(subject.shouldVisit("$['json1']['a']['a']"));
|
91
|
+
assertTrue(subject.shouldVisit("$['json1']['a']"));
|
92
|
+
assertTrue(subject.shouldVisit("$['json1']['b']['b'][1]['b']"));
|
93
|
+
assertTrue(subject.shouldVisit("$['json1']['b']['b'][1]"));
|
94
|
+
assertTrue(subject.shouldVisit("$['json1']['b']['b']"));
|
95
|
+
assertTrue(subject.shouldVisit("$['json1']['b']"));
|
96
|
+
assertTrue(subject.shouldVisit("$['json1']['c']['c'][*]['c']"));
|
97
|
+
assertTrue(subject.shouldVisit("$['json1']['c']['c'][*]"));
|
98
|
+
assertTrue(subject.shouldVisit("$['json1']['c']['c']"));
|
99
|
+
assertTrue(subject.shouldVisit("$['json1']['c']"));
|
100
|
+
assertTrue(subject.shouldVisit("$['json1']"));
|
101
|
+
assertFalse(subject.shouldVisit("$['json2']"));
|
101
102
|
}
|
102
103
|
|
103
104
|
@Test
|
@@ -115,21 +116,21 @@ public class TestJsonVisitor
|
|
115
116
|
.build();
|
116
117
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
117
118
|
|
118
|
-
assertFalse(subject.jsonDropColumns.containsKey("
|
119
|
-
assertTrue(subject.jsonDropColumns.containsKey("
|
120
|
-
assertTrue(subject.jsonDropColumns.containsKey("
|
119
|
+
assertFalse(subject.jsonDropColumns.containsKey("$['json1']"));
|
120
|
+
assertTrue(subject.jsonDropColumns.containsKey("$['json1']['a']"));
|
121
|
+
assertTrue(subject.jsonDropColumns.containsKey("$['json1']['a']['copy_array']"));
|
121
122
|
|
122
123
|
{
|
123
|
-
HashSet<String> jsonColumns = subject.jsonDropColumns.get("
|
124
|
+
HashSet<String> jsonColumns = subject.jsonDropColumns.get("$['json1']['a']");
|
124
125
|
assertEquals(2, jsonColumns.size());
|
125
|
-
assertTrue(jsonColumns.contains("
|
126
|
-
assertTrue(jsonColumns.contains("
|
126
|
+
assertTrue(jsonColumns.contains("$['json1']['a']['default']"));
|
127
|
+
assertTrue(jsonColumns.contains("$['json1']['a']['copy']"));
|
127
128
|
}
|
128
129
|
|
129
130
|
{
|
130
|
-
HashSet<String> jsonColumns = subject.jsonDropColumns.get("
|
131
|
+
HashSet<String> jsonColumns = subject.jsonDropColumns.get("$['json1']['a']['copy_array']");
|
131
132
|
assertEquals(1, jsonColumns.size());
|
132
|
-
assertTrue(jsonColumns.contains("
|
133
|
+
assertTrue(jsonColumns.contains("$['json1']['a']['copy_array'][1]"));
|
133
134
|
}
|
134
135
|
}
|
135
136
|
|
@@ -148,28 +149,28 @@ public class TestJsonVisitor
|
|
148
149
|
.build();
|
149
150
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
150
151
|
|
151
|
-
assertFalse(subject.jsonAddColumns.containsKey("
|
152
|
-
assertTrue(subject.jsonAddColumns.containsKey("
|
153
|
-
assertTrue(subject.jsonAddColumns.containsKey("
|
152
|
+
assertFalse(subject.jsonAddColumns.containsKey("$['json1']"));
|
153
|
+
assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']"));
|
154
|
+
assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']['copy_array']"));
|
154
155
|
|
155
156
|
{
|
156
|
-
HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("
|
157
|
+
HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$['json1']['a']");
|
157
158
|
assertEquals(2, jsonColumns.size());
|
158
159
|
String[] keys = jsonColumns.keySet().toArray(new String[0]);
|
159
160
|
JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
|
160
|
-
assertEquals("
|
161
|
-
assertEquals("
|
162
|
-
assertEquals("
|
163
|
-
assertEquals("
|
161
|
+
assertEquals("$['json1']['a']['default']", keys[0]);
|
162
|
+
assertEquals("$['json1']['a']['default']", values[0].getPath());
|
163
|
+
assertEquals("$['json1']['a']['copy']", keys[1]);
|
164
|
+
assertEquals("$['json1']['a']['copy']", values[1].getPath());
|
164
165
|
}
|
165
166
|
|
166
167
|
{
|
167
|
-
HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("
|
168
|
+
HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$['json1']['a']['copy_array']");
|
168
169
|
assertEquals(1, jsonColumns.size());
|
169
170
|
String[] keys = jsonColumns.keySet().toArray(new String[0]);
|
170
171
|
JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
|
171
|
-
assertEquals("
|
172
|
-
assertEquals("
|
172
|
+
assertEquals("$['json1']['a']['copy_array'][1]", keys[0]);
|
173
|
+
assertEquals("$['json1']['a']['copy_array'][1]", values[0].getPath());
|
173
174
|
}
|
174
175
|
}
|
175
176
|
|
@@ -188,28 +189,28 @@ public class TestJsonVisitor
|
|
188
189
|
.build();
|
189
190
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
190
191
|
|
191
|
-
assertFalse(subject.jsonColumns.containsKey("
|
192
|
-
assertTrue(subject.jsonColumns.containsKey("
|
193
|
-
assertTrue(subject.jsonColumns.containsKey("
|
192
|
+
assertFalse(subject.jsonColumns.containsKey("$['json1']"));
|
193
|
+
assertTrue(subject.jsonColumns.containsKey("$['json1']['a']"));
|
194
|
+
assertTrue(subject.jsonColumns.containsKey("$['json1']['a']['copy_array']"));
|
194
195
|
|
195
196
|
{
|
196
|
-
HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("
|
197
|
+
HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$['json1']['a']");
|
197
198
|
assertEquals(2, jsonColumns.size());
|
198
199
|
String[] keys = jsonColumns.keySet().toArray(new String[0]);
|
199
200
|
JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
|
200
|
-
assertEquals("
|
201
|
-
assertEquals("
|
202
|
-
assertEquals("
|
203
|
-
assertEquals("
|
201
|
+
assertEquals("$['json1']['a']['default']", keys[0]);
|
202
|
+
assertEquals("$['json1']['a']['default']", values[0].getPath());
|
203
|
+
assertEquals("$['json1']['a']['copy']", keys[1]);
|
204
|
+
assertEquals("$['json1']['a']['copy']", values[1].getPath());
|
204
205
|
}
|
205
206
|
|
206
207
|
{
|
207
|
-
HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("
|
208
|
+
HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$['json1']['a']['copy_array']");
|
208
209
|
assertEquals(1, jsonColumns.size());
|
209
210
|
String[] keys = jsonColumns.keySet().toArray(new String[0]);
|
210
211
|
JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
|
211
|
-
assertEquals("
|
212
|
-
assertEquals("
|
212
|
+
assertEquals("$['json1']['a']['copy_array'][1]", keys[0]);
|
213
|
+
assertEquals("$['json1']['a']['copy_array'][1]", values[0].getPath());
|
213
214
|
}
|
214
215
|
}
|
215
216
|
|
@@ -257,7 +258,7 @@ public class TestJsonVisitor
|
|
257
258
|
k1, ValueFactory.newMap(k1, v),
|
258
259
|
k2, ValueFactory.newMap(k2, v));
|
259
260
|
|
260
|
-
MapValue visited = subject.visit("
|
261
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
261
262
|
assertEquals("{\"k1\":{}}", visited.toString());
|
262
263
|
}
|
263
264
|
|
@@ -284,7 +285,7 @@ public class TestJsonVisitor
|
|
284
285
|
k1, ValueFactory.newMap(k1, v),
|
285
286
|
k2, ValueFactory.newMap(k2, v));
|
286
287
|
|
287
|
-
MapValue visited = subject.visit("
|
288
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
288
289
|
assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
|
289
290
|
}
|
290
291
|
|
@@ -313,7 +314,7 @@ public class TestJsonVisitor
|
|
313
314
|
k1, ValueFactory.newMap(k1, v),
|
314
315
|
k2, ValueFactory.newMap(k2, v));
|
315
316
|
|
316
|
-
MapValue visited = subject.visit("
|
317
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
317
318
|
assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
|
318
319
|
}
|
319
320
|
|
@@ -339,7 +340,7 @@ public class TestJsonVisitor
|
|
339
340
|
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
|
340
341
|
k2, ValueFactory.newArray(v, v));
|
341
342
|
|
342
|
-
MapValue visited = subject.visit("
|
343
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
343
344
|
assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
|
344
345
|
}
|
345
346
|
|
@@ -367,7 +368,7 @@ public class TestJsonVisitor
|
|
367
368
|
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
|
368
369
|
k2, ValueFactory.newArray(v, v));
|
369
370
|
|
370
|
-
MapValue visited = subject.visit("
|
371
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
371
372
|
assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
|
372
373
|
}
|
373
374
|
|
@@ -397,7 +398,394 @@ public class TestJsonVisitor
|
|
397
398
|
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
|
398
399
|
k2, ValueFactory.newArray(v, v));
|
399
400
|
|
400
|
-
MapValue visited = subject.visit("
|
401
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
401
402
|
assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
|
402
403
|
}
|
403
|
-
|
404
|
+
|
405
|
+
@Test
|
406
|
+
public void visitMap_dropColumnsUsingBracketNotation()
|
407
|
+
{
|
408
|
+
PluginTask task = taskFromYamlString(
|
409
|
+
"type: column",
|
410
|
+
"drop_columns:",
|
411
|
+
" - {name: \"$['json1']['k1']['k1']\"}",
|
412
|
+
" - {name: \"$['json1']['k2']\"}");
|
413
|
+
Schema inputSchema = Schema.builder()
|
414
|
+
.add("json1", JSON)
|
415
|
+
.add("json2", JSON)
|
416
|
+
.build();
|
417
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
418
|
+
|
419
|
+
// {"k1":{"k1":"v"},"k2":{"k2":"v"}}
|
420
|
+
Value k1 = ValueFactory.newString("k1");
|
421
|
+
Value k2 = ValueFactory.newString("k2");
|
422
|
+
Value v = ValueFactory.newString("v");
|
423
|
+
Value map = ValueFactory.newMap(
|
424
|
+
k1, ValueFactory.newMap(k1, v),
|
425
|
+
k2, ValueFactory.newMap(k2, v));
|
426
|
+
|
427
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
428
|
+
assertEquals("{\"k1\":{}}", visited.toString());
|
429
|
+
}
|
430
|
+
|
431
|
+
@Test
|
432
|
+
public void visitMap_addColumnsUsingBracketNotation()
|
433
|
+
{
|
434
|
+
PluginTask task = taskFromYamlString(
|
435
|
+
"type: column",
|
436
|
+
"add_columns:",
|
437
|
+
" - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
|
438
|
+
" - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
|
439
|
+
" - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
|
440
|
+
Schema inputSchema = Schema.builder()
|
441
|
+
.add("json1", JSON)
|
442
|
+
.add("json2", JSON)
|
443
|
+
.build();
|
444
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
445
|
+
|
446
|
+
// {"k1":{"k1":"v"},"k2":{"k2":"v"}}
|
447
|
+
Value k1 = ValueFactory.newString("k1");
|
448
|
+
Value k2 = ValueFactory.newString("k2");
|
449
|
+
Value v = ValueFactory.newString("v");
|
450
|
+
Value map = ValueFactory.newMap(
|
451
|
+
k1, ValueFactory.newMap(k1, v),
|
452
|
+
k2, ValueFactory.newMap(k2, v));
|
453
|
+
|
454
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
455
|
+
assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
|
456
|
+
}
|
457
|
+
|
458
|
+
@Test
|
459
|
+
public void visitMap_columnsUsingBracketNotation()
|
460
|
+
{
|
461
|
+
PluginTask task = taskFromYamlString(
|
462
|
+
"type: column",
|
463
|
+
"columns:",
|
464
|
+
" - {name: \"$['json1']['k1']\"}",
|
465
|
+
" - {name: \"$['json1']['k2']['k2']\"}",
|
466
|
+
" - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
|
467
|
+
" - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
|
468
|
+
" - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
|
469
|
+
Schema inputSchema = Schema.builder()
|
470
|
+
.add("json1", JSON)
|
471
|
+
.build();
|
472
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
473
|
+
|
474
|
+
// {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
|
475
|
+
Value k1 = ValueFactory.newString("k1");
|
476
|
+
Value k2 = ValueFactory.newString("k2");
|
477
|
+
Value v = ValueFactory.newString("v");
|
478
|
+
Value map = ValueFactory.newMap(
|
479
|
+
k1, ValueFactory.newMap(k1, v),
|
480
|
+
k2, ValueFactory.newMap(k2, v));
|
481
|
+
|
482
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
483
|
+
assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
|
484
|
+
}
|
485
|
+
|
486
|
+
@Test
|
487
|
+
public void visitArray_dropColumnsUsingBracketNotation()
|
488
|
+
{
|
489
|
+
PluginTask task = taskFromYamlString(
|
490
|
+
"type: column",
|
491
|
+
"drop_columns:",
|
492
|
+
" - {name: \"$['json1']['k1'][0]['k1']\"}",
|
493
|
+
" - {name: \"$['json1']['k2'][*]\"}"); // ending with [*] is allowed for drop_columns, but not for others
|
494
|
+
Schema inputSchema = Schema.builder()
|
495
|
+
.add("json1", JSON)
|
496
|
+
.add("json2", JSON)
|
497
|
+
.build();
|
498
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
499
|
+
|
500
|
+
// {"k1":[{"k1":"v"}[,"k2":["v","v"]}
|
501
|
+
Value k1 = ValueFactory.newString("k1");
|
502
|
+
Value k2 = ValueFactory.newString("k2");
|
503
|
+
Value v = ValueFactory.newString("v");
|
504
|
+
Value map = ValueFactory.newMap(
|
505
|
+
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
|
506
|
+
k2, ValueFactory.newArray(v, v));
|
507
|
+
|
508
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
509
|
+
assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
|
510
|
+
}
|
511
|
+
|
512
|
+
@Test
|
513
|
+
public void visitArray_addColumnsUsingBracketNotation()
|
514
|
+
{
|
515
|
+
PluginTask task = taskFromYamlString(
|
516
|
+
"type: column",
|
517
|
+
"add_columns:",
|
518
|
+
" - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
|
519
|
+
" - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
|
520
|
+
" - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
|
521
|
+
" - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
|
522
|
+
Schema inputSchema = Schema.builder()
|
523
|
+
.add("json1", JSON)
|
524
|
+
.add("json2", JSON)
|
525
|
+
.build();
|
526
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
527
|
+
|
528
|
+
// {"k1":[{"k1":"v"}],"k2":["v","v"]}
|
529
|
+
Value k1 = ValueFactory.newString("k1");
|
530
|
+
Value k2 = ValueFactory.newString("k2");
|
531
|
+
Value v = ValueFactory.newString("v");
|
532
|
+
Value map = ValueFactory.newMap(
|
533
|
+
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
|
534
|
+
k2, ValueFactory.newArray(v, v));
|
535
|
+
|
536
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
537
|
+
assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
|
538
|
+
}
|
539
|
+
|
540
|
+
@Test
|
541
|
+
public void visitArray_columnsUsingBracketNotation()
|
542
|
+
{
|
543
|
+
PluginTask task = taskFromYamlString(
|
544
|
+
"type: column",
|
545
|
+
"columns:",
|
546
|
+
" - {name: \"$['json1']['k1']\"}",
|
547
|
+
" - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
|
548
|
+
" - {name: \"$['json1']['k2'][0]\"}",
|
549
|
+
" - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
|
550
|
+
" - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
|
551
|
+
" - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
|
552
|
+
Schema inputSchema = Schema.builder()
|
553
|
+
.add("json1", JSON)
|
554
|
+
.build();
|
555
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
556
|
+
|
557
|
+
// {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
|
558
|
+
Value k1 = ValueFactory.newString("k1");
|
559
|
+
Value k2 = ValueFactory.newString("k2");
|
560
|
+
Value v = ValueFactory.newString("v");
|
561
|
+
Value map = ValueFactory.newMap(
|
562
|
+
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
|
563
|
+
k2, ValueFactory.newArray(v, v));
|
564
|
+
|
565
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
566
|
+
assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
|
567
|
+
}
|
568
|
+
|
569
|
+
// Because the dot notation is converted to single quotes by default,
|
570
|
+
// it can be mixed with the bracket notation of single quotes
|
571
|
+
@Test
|
572
|
+
public void visit_withDotAndBracket()
|
573
|
+
{
|
574
|
+
PluginTask task = taskFromYamlString(
|
575
|
+
"type: column",
|
576
|
+
"columns:",
|
577
|
+
" - {name: \"$.json1['k_1']\"}",
|
578
|
+
" - {name: \"$.json1['k_1'][0]['k_1']\"}",
|
579
|
+
" - {name: \"$['json1']['k_2']\"}",
|
580
|
+
" - {name: \"$['json1']['k_2']['k_2']\"}");
|
581
|
+
Schema inputSchema = Schema.builder()
|
582
|
+
.add("json1", JSON)
|
583
|
+
.build();
|
584
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
585
|
+
|
586
|
+
// {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
|
587
|
+
Value k1 = ValueFactory.newString("k_1");
|
588
|
+
Value k2 = ValueFactory.newString("k_2");
|
589
|
+
Value v = ValueFactory.newString("v");
|
590
|
+
Value map = ValueFactory.newMap(
|
591
|
+
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
|
592
|
+
k2, ValueFactory.newMap(k2, v));
|
593
|
+
|
594
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
595
|
+
assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
|
596
|
+
}
|
597
|
+
|
598
|
+
// Because the bracket notation of double quotes converted to single quotes internally
|
599
|
+
// it can be mixed with the bracket notation of single quotes
|
600
|
+
@Test
|
601
|
+
public void visit_withSingleQuotesAndDoubleQuotes()
|
602
|
+
{
|
603
|
+
PluginTask task = taskFromYamlString(
|
604
|
+
"type: column",
|
605
|
+
"columns:",
|
606
|
+
" - {name: \"$['json1']['k_1']\", src: \"$['json1']['k.1']\"}",
|
607
|
+
" - {name: '$[\"json1\"][\"k_1\"][0][\"k_1\"]', src: '$[\"json1\"][\"k_1\"][0][\"k.1\"]'}",
|
608
|
+
" - {name: '$[\"json1\"][\"k_2\"]', src: '$[\"json1\"][\"k.2\"]'}",
|
609
|
+
" - {name: '$[\"json1\"][\"k_2\"][\"k_2\"]', src: '$[\"json1\"][\"k_2\"][\"k.2\"]'}");
|
610
|
+
Schema inputSchema = Schema.builder()
|
611
|
+
.add("json1", JSON)
|
612
|
+
.build();
|
613
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
614
|
+
|
615
|
+
// {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
|
616
|
+
Value k1 = ValueFactory.newString("k.1");
|
617
|
+
Value k2 = ValueFactory.newString("k.2");
|
618
|
+
Value v = ValueFactory.newString("v");
|
619
|
+
Value map = ValueFactory.newMap(
|
620
|
+
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
|
621
|
+
k2, ValueFactory.newMap(k2, v));
|
622
|
+
|
623
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
624
|
+
assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
|
625
|
+
}
|
626
|
+
|
627
|
+
@Test
|
628
|
+
public void visit_withComplexRename()
|
629
|
+
{
|
630
|
+
PluginTask task = taskFromYamlString(
|
631
|
+
"type: column",
|
632
|
+
"columns:",
|
633
|
+
" - {name: \"$.json1['k____1']\", src: \"$.json1['k.-=+1']\"}",
|
634
|
+
" - {name: \"$.json1['k____1'][0]['k____1']\", src: \"$.json1['k____1'][0]['k.-=+1']\"}",
|
635
|
+
" - {name: \"$['json1']['k_2']\", src: \"$['json1']['k.2']\"}",
|
636
|
+
" - {name: \"$['json1']['k_2']['k_2']\", src: \"$['json1']['k_2']['k.2']\"}");
|
637
|
+
Schema inputSchema = Schema.builder()
|
638
|
+
.add("json1", JSON)
|
639
|
+
.build();
|
640
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
641
|
+
|
642
|
+
// {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
|
643
|
+
Value k1 = ValueFactory.newString("k.-=+1");
|
644
|
+
Value k2 = ValueFactory.newString("k.2");
|
645
|
+
Value v = ValueFactory.newString("v");
|
646
|
+
Value map = ValueFactory.newMap(
|
647
|
+
k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
|
648
|
+
k2, ValueFactory.newMap(k2, v));
|
649
|
+
|
650
|
+
MapValue visited = subject.visit("$['json1']", map).asMapValue();
|
651
|
+
assertEquals("{\"k____1\":[{\"k____1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
|
652
|
+
}
|
653
|
+
|
654
|
+
@Test
|
655
|
+
public void visit_withColumnNameIncludingSingleQuotes()
|
656
|
+
{
|
657
|
+
PluginTask task = taskFromYamlString(
|
658
|
+
"type: column",
|
659
|
+
"columns:",
|
660
|
+
" - {name: \"$[\\\"'json1\\\"]['k1']\"}");
|
661
|
+
Schema inputSchema = Schema.builder()
|
662
|
+
.add("'json1", JSON)
|
663
|
+
.build();
|
664
|
+
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
665
|
+
|
666
|
+
// {"k1":"v"}
|
667
|
+
Value k1 = ValueFactory.newString("k1");
|
668
|
+
Value v = ValueFactory.newString("v");
|
669
|
+
Value map = ValueFactory.newMap(k1, v);
|
670
|
+
|
671
|
+
MapValue visited = subject.visit("$['\\'json1']", map).asMapValue();
|
672
|
+
assertEquals("{\"k1\":\"v\"}", visited.toString());
|
673
|
+
}
|
674
|
+
|
675
|
+
@Test(expected = ConfigException.class)
|
676
|
+
public void constructor_mustBeRaisedConfigExceptionWithMultiProperties() {
|
677
|
+
PluginTask task = taskFromYamlString(
|
678
|
+
"type: column",
|
679
|
+
"columns:",
|
680
|
+
"- name: \"$['json1','k1']\"");
|
681
|
+
Schema inputSchema = Schema.builder()
|
682
|
+
.add("json1", JSON)
|
683
|
+
.build();
|
684
|
+
jsonVisitor(task, inputSchema);
|
685
|
+
}
|
686
|
+
|
687
|
+
// It is recognized multi properties if the square brackets does not close properly
|
688
|
+
@Test(expected = InvalidPathException.class)
|
689
|
+
public void constructor_mustBeRaisedInvalidPathExceptionWithPropertyIsNotSeparatedByCommas()
|
690
|
+
{
|
691
|
+
PluginTask task = taskFromYamlString(
|
692
|
+
"type: column",
|
693
|
+
"columns:",
|
694
|
+
" - name: \"$['json1'}['k1']\"");
|
695
|
+
Schema inputSchema = Schema.builder()
|
696
|
+
.add("json1", JSON)
|
697
|
+
.build();
|
698
|
+
jsonVisitor(task, inputSchema);
|
699
|
+
}
|
700
|
+
|
701
|
+
@Test(expected = ConfigException.class)
|
702
|
+
public void constructor_mustBeRaisedConfigExceptionWithFunctionPathToken()
|
703
|
+
{
|
704
|
+
PluginTask task = taskFromYamlString(
|
705
|
+
"type: column",
|
706
|
+
"columns:",
|
707
|
+
"- name: \"$['json1'].length()\"");
|
708
|
+
Schema inputSchema = Schema.builder()
|
709
|
+
.add("json1", JSON)
|
710
|
+
.build();
|
711
|
+
jsonVisitor(task, inputSchema);
|
712
|
+
}
|
713
|
+
|
714
|
+
@Test(expected = ConfigException.class)
|
715
|
+
public void constructor_mustBeRaisedConfigExceptionWithPredicatePathToken()
|
716
|
+
{
|
717
|
+
PluginTask task = taskFromYamlString(
|
718
|
+
"type: column",
|
719
|
+
"columns:",
|
720
|
+
"- name: \"$.store.book[?(@.price < 10)]\"");
|
721
|
+
Schema inputSchema = Schema.builder()
|
722
|
+
.add("store", JSON)
|
723
|
+
.build();
|
724
|
+
jsonVisitor(task, inputSchema);
|
725
|
+
}
|
726
|
+
|
727
|
+
@Test(expected = ConfigException.class)
|
728
|
+
public void constructor_mustBeRaisedConfigExceptionWithScanPathToken()
|
729
|
+
{
|
730
|
+
PluginTask task = taskFromYamlString(
|
731
|
+
"type: column",
|
732
|
+
"columns:",
|
733
|
+
"- name: \"$.json1..key1\"");
|
734
|
+
Schema inputSchema = Schema.builder()
|
735
|
+
.add("json1", JSON)
|
736
|
+
.build();
|
737
|
+
jsonVisitor(task, inputSchema);
|
738
|
+
}
|
739
|
+
|
740
|
+
@Test(expected = ConfigException.class)
|
741
|
+
public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperation()
|
742
|
+
{
|
743
|
+
PluginTask task = taskFromYamlString(
|
744
|
+
"type: column",
|
745
|
+
"columns:",
|
746
|
+
"- name: \"$.json1[0,1]\"");
|
747
|
+
Schema inputSchema = Schema.builder()
|
748
|
+
.add("json1", JSON)
|
749
|
+
.build();
|
750
|
+
jsonVisitor(task, inputSchema);
|
751
|
+
}
|
752
|
+
|
753
|
+
@Test(expected = ConfigException.class)
|
754
|
+
public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperationAtMiddlePosition()
|
755
|
+
{
|
756
|
+
PluginTask task = taskFromYamlString(
|
757
|
+
"type: column",
|
758
|
+
"columns:",
|
759
|
+
"- name: \"$.json1[0,1].key1\"");
|
760
|
+
Schema inputSchema = Schema.builder()
|
761
|
+
.add("json1", JSON)
|
762
|
+
.build();
|
763
|
+
jsonVisitor(task, inputSchema);
|
764
|
+
}
|
765
|
+
|
766
|
+
@Test(expected = ConfigException.class)
|
767
|
+
public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperation()
|
768
|
+
{
|
769
|
+
PluginTask task = taskFromYamlString(
|
770
|
+
"type: column",
|
771
|
+
"columns:",
|
772
|
+
"- name: \"$.json1[1:2]\"");
|
773
|
+
Schema inputSchema = Schema.builder()
|
774
|
+
.add("json1", JSON)
|
775
|
+
.build();
|
776
|
+
jsonVisitor(task, inputSchema);
|
777
|
+
}
|
778
|
+
|
779
|
+
@Test(expected = ConfigException.class)
|
780
|
+
public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperationAtMiddlePosition()
|
781
|
+
{
|
782
|
+
PluginTask task = taskFromYamlString(
|
783
|
+
"type: column",
|
784
|
+
"columns:",
|
785
|
+
"- name: \"$.json1[1:2].key1\"");
|
786
|
+
Schema inputSchema = Schema.builder()
|
787
|
+
.add("json1", JSON)
|
788
|
+
.build();
|
789
|
+
jsonVisitor(task, inputSchema);
|
790
|
+
}
|
791
|
+
}
|