embulk-filter-column 0.5.4 → 0.6.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,39 @@
1
+ package org.embulk.filter.column;
2
+
3
+ import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayIndexOperation;
4
+ import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayPathToken;
5
+ import com.dena.analytics.jsonpathcompiler.expressions.path.FunctionPathToken;
6
+ import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
7
+ import com.dena.analytics.jsonpathcompiler.expressions.path.PredicatePathToken;
8
+ import com.dena.analytics.jsonpathcompiler.expressions.path.ScanPathToken;
9
+ import org.embulk.config.ConfigException;
10
+
11
+ public class PathTokenUtil
12
+ {
13
+ public static void assertSupportedPathToken(PathToken pathToken, String path)
14
+ {
15
+ if (pathToken instanceof ArrayPathToken) {
16
+ ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) pathToken).getArrayIndexOperation();
17
+ assertSupportedArrayPathToken(arrayIndexOperation, path);
18
+ }
19
+ else if (pathToken instanceof ScanPathToken) {
20
+ throw new ConfigException(String.format("scan path token is not supported \"%s\"", path));
21
+ }
22
+ else if (pathToken instanceof FunctionPathToken) {
23
+ throw new ConfigException(String.format("function path token is not supported \"%s\"", path));
24
+ }
25
+ else if (pathToken instanceof PredicatePathToken) {
26
+ throw new ConfigException(String.format("predicate path token is not supported \"%s\"", path));
27
+ }
28
+ }
29
+
30
+ public static void assertSupportedArrayPathToken(ArrayIndexOperation arrayIndexOperation, String path)
31
+ {
32
+ if (arrayIndexOperation == null) {
33
+ throw new ConfigException(String.format("Array Slice Operation is not supported \"%s\"", path));
34
+ }
35
+ else if (!arrayIndexOperation.isSingleIndexOperation()) {
36
+ throw new ConfigException(String.format("Multi Array Indexes is not supported \"%s\"", path));
37
+ }
38
+ }
39
+ }
@@ -32,7 +32,7 @@ public class TestColumnFilterPlugin
32
32
  private ColumnFilterPlugin plugin;
33
33
 
34
34
  @Before
35
- public void createReasource()
35
+ public void createResource()
36
36
  {
37
37
  plugin = new ColumnFilterPlugin();
38
38
  }
@@ -34,7 +34,7 @@ public class TestColumnVisitorImpl
34
34
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
35
35
 
36
36
  @Before
37
- public void createReasource()
37
+ public void createResource()
38
38
  {
39
39
  }
40
40
 
@@ -15,7 +15,7 @@ public class TestJsonColumn
15
15
  {
16
16
  try {
17
17
  JsonColumn column = new JsonColumn("$.foo.bar", Types.BOOLEAN);
18
- assertEquals("$.foo.bar", column.getSrc());
18
+ assertEquals("$['foo']['bar']", column.getSrc());
19
19
  assertEquals(ValueFactory.newNil(), column.getDefaultValue());
20
20
  }
21
21
  catch (Exception e) {
@@ -24,8 +24,8 @@ public class TestJsonColumn
24
24
 
25
25
  try {
26
26
  Value defaultValue = ValueFactory.newBoolean(true);
27
- JsonColumn column = new JsonColumn("$.foo.bar", Types.BOOLEAN, defaultValue);
28
- assertEquals("$.foo.bar", column.getSrc());
27
+ JsonColumn column = new JsonColumn("$['foo']['bar']", Types.BOOLEAN, defaultValue);
28
+ assertEquals("$['foo']['bar']", column.getSrc());
29
29
  assertEquals(defaultValue, column.getDefaultValue());
30
30
  }
31
31
  catch (Exception e) {
@@ -36,22 +36,44 @@ public class TestJsonColumn
36
36
  @Test
37
37
  public void parentPath()
38
38
  {
39
- assertEquals("$.foo.bar", JsonColumn.parentPath("$.foo.bar.baz"));
40
- assertEquals("$.foo", JsonColumn.parentPath("$.foo.bar"));
41
- assertEquals("$", JsonColumn.parentPath("$.foo"));
42
- assertEquals("$.foo[0]", JsonColumn.parentPath("$.foo[0][1]"));
43
- assertEquals("$.foo", JsonColumn.parentPath("$.foo[0]"));
39
+ assertEquals("$['foo']['bar']", JsonColumn.parentPath("$.foo.bar.baz"));
40
+ assertEquals("$['foo']", JsonColumn.parentPath("$.foo.bar"));
41
+ assertEquals("$", JsonColumn.parentPath("$['foo']"));
42
+ assertEquals("$['foo'][0]", JsonColumn.parentPath("$.foo[0][1]"));
43
+ assertEquals("$['foo']", JsonColumn.parentPath("$.foo[0]"));
44
44
  assertEquals("$", JsonColumn.parentPath("$[0]"));
45
45
  }
46
46
 
47
47
  @Test
48
- public void baseName()
48
+ public void tailName()
49
49
  {
50
- assertEquals("baz", JsonColumn.baseName("$.foo.bar.baz"));
51
- assertEquals("bar", JsonColumn.baseName("$.foo.bar"));
52
- assertEquals("foo", JsonColumn.baseName("$.foo"));
53
- assertEquals("[1]", JsonColumn.baseName("$.foo[0][1]"));
54
- assertEquals("[0]", JsonColumn.baseName("$.foo[0]"));
55
- assertEquals("[0]", JsonColumn.baseName("$[0]"));
50
+ assertEquals("['baz']", JsonColumn.tailName("$['foo'].bar.baz"));
51
+ assertEquals("['bar']", JsonColumn.tailName("$.foo.bar"));
52
+ assertEquals("['foo']", JsonColumn.tailName("$.foo"));
53
+ assertEquals("[1]", JsonColumn.tailName("$.foo[0][1]"));
54
+ assertEquals("[0]", JsonColumn.tailName("$.foo[0]"));
55
+ assertEquals("[0]", JsonColumn.tailName("$[0]"));
56
56
  }
57
- }
57
+
58
+ @Test
59
+ public void getTailNameValue()
60
+ {
61
+ assertEquals(ValueFactory.newString("baz"), new JsonColumn("$['foo'].bar.baz", Types.BOOLEAN).getTailNameValue());
62
+ assertEquals(ValueFactory.newString("bar"), new JsonColumn("$.foo.bar", Types.BOOLEAN).getTailNameValue());
63
+ assertEquals(ValueFactory.newString("foo"), new JsonColumn("$.foo", Types.BOOLEAN).getTailNameValue());
64
+ assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0][1]", Types.BOOLEAN).getTailNameValue());
65
+ assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0]", Types.BOOLEAN).getTailNameValue());
66
+ assertEquals(ValueFactory.newNil(), new JsonColumn("$[0]", Types.BOOLEAN).getTailNameValue());
67
+ }
68
+
69
+ @Test
70
+ public void tailIndex()
71
+ {
72
+ assertEquals(null, JsonColumn.tailIndex("$['foo'].bar.baz"));
73
+ assertEquals(null, JsonColumn.tailIndex("$.foo.bar"));
74
+ assertEquals(null, JsonColumn.tailIndex("$.foo"));
75
+ assertEquals(new Long(1), JsonColumn.tailIndex("$.foo[0][1]"));
76
+ assertEquals(new Long(0), JsonColumn.tailIndex("$.foo[0]"));
77
+ assertEquals(new Long(0), JsonColumn.tailIndex("$[0]"));
78
+ }
79
+ }
@@ -13,6 +13,7 @@ import org.junit.Test;
13
13
  import org.msgpack.value.MapValue;
14
14
  import org.msgpack.value.Value;
15
15
  import org.msgpack.value.ValueFactory;
16
+ import com.dena.analytics.jsonpathcompiler.InvalidPathException;
16
17
 
17
18
  import static org.embulk.spi.type.Types.JSON;
18
19
  import static org.junit.Assert.assertEquals;
@@ -28,7 +29,7 @@ public class TestJsonVisitor
28
29
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
29
30
 
30
31
  @Before
31
- public void createReasource()
32
+ public void createResource()
32
33
  {
33
34
  }
34
35
 
@@ -85,19 +86,19 @@ public class TestJsonVisitor
85
86
  .build();
86
87
  JsonVisitor subject = jsonVisitor(task, inputSchema);
87
88
 
88
- assertTrue(subject.shouldVisit("$.json1.a.a.a"));
89
- assertTrue(subject.shouldVisit("$.json1.a.a"));
90
- assertTrue(subject.shouldVisit("$.json1.a"));
91
- assertTrue(subject.shouldVisit("$.json1.b.b[1].b"));
92
- assertTrue(subject.shouldVisit("$.json1.b.b[1]"));
93
- assertTrue(subject.shouldVisit("$.json1.b.b"));
94
- assertTrue(subject.shouldVisit("$.json1.b"));
95
- assertTrue(subject.shouldVisit("$.json1.c.c[*].c"));
96
- assertTrue(subject.shouldVisit("$.json1.c.c[*]"));
97
- assertTrue(subject.shouldVisit("$.json1.c.c"));
98
- assertTrue(subject.shouldVisit("$.json1.c"));
99
- assertTrue(subject.shouldVisit("$.json1"));
100
- assertFalse(subject.shouldVisit("$.json2"));
89
+ assertTrue(subject.shouldVisit("$['json1']['a']['a']['a']"));
90
+ assertTrue(subject.shouldVisit("$['json1']['a']['a']"));
91
+ assertTrue(subject.shouldVisit("$['json1']['a']"));
92
+ assertTrue(subject.shouldVisit("$['json1']['b']['b'][1]['b']"));
93
+ assertTrue(subject.shouldVisit("$['json1']['b']['b'][1]"));
94
+ assertTrue(subject.shouldVisit("$['json1']['b']['b']"));
95
+ assertTrue(subject.shouldVisit("$['json1']['b']"));
96
+ assertTrue(subject.shouldVisit("$['json1']['c']['c'][*]['c']"));
97
+ assertTrue(subject.shouldVisit("$['json1']['c']['c'][*]"));
98
+ assertTrue(subject.shouldVisit("$['json1']['c']['c']"));
99
+ assertTrue(subject.shouldVisit("$['json1']['c']"));
100
+ assertTrue(subject.shouldVisit("$['json1']"));
101
+ assertFalse(subject.shouldVisit("$['json2']"));
101
102
  }
102
103
 
103
104
  @Test
@@ -115,21 +116,21 @@ public class TestJsonVisitor
115
116
  .build();
116
117
  JsonVisitor subject = jsonVisitor(task, inputSchema);
117
118
 
118
- assertFalse(subject.jsonDropColumns.containsKey("$.json1"));
119
- assertTrue(subject.jsonDropColumns.containsKey("$.json1.a"));
120
- assertTrue(subject.jsonDropColumns.containsKey("$.json1.a.copy_array"));
119
+ assertFalse(subject.jsonDropColumns.containsKey("$['json1']"));
120
+ assertTrue(subject.jsonDropColumns.containsKey("$['json1']['a']"));
121
+ assertTrue(subject.jsonDropColumns.containsKey("$['json1']['a']['copy_array']"));
121
122
 
122
123
  {
123
- HashSet<String> jsonColumns = subject.jsonDropColumns.get("$.json1.a");
124
+ HashSet<String> jsonColumns = subject.jsonDropColumns.get("$['json1']['a']");
124
125
  assertEquals(2, jsonColumns.size());
125
- assertTrue(jsonColumns.contains("$.json1.a.default"));
126
- assertTrue(jsonColumns.contains("$.json1.a.copy"));
126
+ assertTrue(jsonColumns.contains("$['json1']['a']['default']"));
127
+ assertTrue(jsonColumns.contains("$['json1']['a']['copy']"));
127
128
  }
128
129
 
129
130
  {
130
- HashSet<String> jsonColumns = subject.jsonDropColumns.get("$.json1.a.copy_array");
131
+ HashSet<String> jsonColumns = subject.jsonDropColumns.get("$['json1']['a']['copy_array']");
131
132
  assertEquals(1, jsonColumns.size());
132
- assertTrue(jsonColumns.contains("$.json1.a.copy_array[1]"));
133
+ assertTrue(jsonColumns.contains("$['json1']['a']['copy_array'][1]"));
133
134
  }
134
135
  }
135
136
 
@@ -148,28 +149,28 @@ public class TestJsonVisitor
148
149
  .build();
149
150
  JsonVisitor subject = jsonVisitor(task, inputSchema);
150
151
 
151
- assertFalse(subject.jsonAddColumns.containsKey("$.json1"));
152
- assertTrue(subject.jsonAddColumns.containsKey("$.json1.a"));
153
- assertTrue(subject.jsonAddColumns.containsKey("$.json1.a.copy_array"));
152
+ assertFalse(subject.jsonAddColumns.containsKey("$['json1']"));
153
+ assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']"));
154
+ assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']['copy_array']"));
154
155
 
155
156
  {
156
- HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$.json1.a");
157
+ HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$['json1']['a']");
157
158
  assertEquals(2, jsonColumns.size());
158
159
  String[] keys = jsonColumns.keySet().toArray(new String[0]);
159
160
  JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
160
- assertEquals("$.json1.a.default", keys[0]);
161
- assertEquals("$.json1.a.default", values[0].getPath());
162
- assertEquals("$.json1.a.copy", keys[1]);
163
- assertEquals("$.json1.a.copy", values[1].getPath());
161
+ assertEquals("$['json1']['a']['default']", keys[0]);
162
+ assertEquals("$['json1']['a']['default']", values[0].getPath());
163
+ assertEquals("$['json1']['a']['copy']", keys[1]);
164
+ assertEquals("$['json1']['a']['copy']", values[1].getPath());
164
165
  }
165
166
 
166
167
  {
167
- HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$.json1.a.copy_array");
168
+ HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$['json1']['a']['copy_array']");
168
169
  assertEquals(1, jsonColumns.size());
169
170
  String[] keys = jsonColumns.keySet().toArray(new String[0]);
170
171
  JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
171
- assertEquals("$.json1.a.copy_array[1]", keys[0]);
172
- assertEquals("$.json1.a.copy_array[1]", values[0].getPath());
172
+ assertEquals("$['json1']['a']['copy_array'][1]", keys[0]);
173
+ assertEquals("$['json1']['a']['copy_array'][1]", values[0].getPath());
173
174
  }
174
175
  }
175
176
 
@@ -188,28 +189,28 @@ public class TestJsonVisitor
188
189
  .build();
189
190
  JsonVisitor subject = jsonVisitor(task, inputSchema);
190
191
 
191
- assertFalse(subject.jsonColumns.containsKey("$.json1"));
192
- assertTrue(subject.jsonColumns.containsKey("$.json1.a"));
193
- assertTrue(subject.jsonColumns.containsKey("$.json1.a.copy_array"));
192
+ assertFalse(subject.jsonColumns.containsKey("$['json1']"));
193
+ assertTrue(subject.jsonColumns.containsKey("$['json1']['a']"));
194
+ assertTrue(subject.jsonColumns.containsKey("$['json1']['a']['copy_array']"));
194
195
 
195
196
  {
196
- HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$.json1.a");
197
+ HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$['json1']['a']");
197
198
  assertEquals(2, jsonColumns.size());
198
199
  String[] keys = jsonColumns.keySet().toArray(new String[0]);
199
200
  JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
200
- assertEquals("$.json1.a.default", keys[0]);
201
- assertEquals("$.json1.a.default", values[0].getPath());
202
- assertEquals("$.json1.a.copy", keys[1]);
203
- assertEquals("$.json1.a.copy", values[1].getPath());
201
+ assertEquals("$['json1']['a']['default']", keys[0]);
202
+ assertEquals("$['json1']['a']['default']", values[0].getPath());
203
+ assertEquals("$['json1']['a']['copy']", keys[1]);
204
+ assertEquals("$['json1']['a']['copy']", values[1].getPath());
204
205
  }
205
206
 
206
207
  {
207
- HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$.json1.a.copy_array");
208
+ HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$['json1']['a']['copy_array']");
208
209
  assertEquals(1, jsonColumns.size());
209
210
  String[] keys = jsonColumns.keySet().toArray(new String[0]);
210
211
  JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
211
- assertEquals("$.json1.a.copy_array[1]", keys[0]);
212
- assertEquals("$.json1.a.copy_array[1]", values[0].getPath());
212
+ assertEquals("$['json1']['a']['copy_array'][1]", keys[0]);
213
+ assertEquals("$['json1']['a']['copy_array'][1]", values[0].getPath());
213
214
  }
214
215
  }
215
216
 
@@ -257,7 +258,7 @@ public class TestJsonVisitor
257
258
  k1, ValueFactory.newMap(k1, v),
258
259
  k2, ValueFactory.newMap(k2, v));
259
260
 
260
- MapValue visited = subject.visit("$.json1", map).asMapValue();
261
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
261
262
  assertEquals("{\"k1\":{}}", visited.toString());
262
263
  }
263
264
 
@@ -284,7 +285,7 @@ public class TestJsonVisitor
284
285
  k1, ValueFactory.newMap(k1, v),
285
286
  k2, ValueFactory.newMap(k2, v));
286
287
 
287
- MapValue visited = subject.visit("$.json1", map).asMapValue();
288
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
288
289
  assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
289
290
  }
290
291
 
@@ -313,7 +314,7 @@ public class TestJsonVisitor
313
314
  k1, ValueFactory.newMap(k1, v),
314
315
  k2, ValueFactory.newMap(k2, v));
315
316
 
316
- MapValue visited = subject.visit("$.json1", map).asMapValue();
317
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
317
318
  assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
318
319
  }
319
320
 
@@ -339,7 +340,7 @@ public class TestJsonVisitor
339
340
  k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
340
341
  k2, ValueFactory.newArray(v, v));
341
342
 
342
- MapValue visited = subject.visit("$.json1", map).asMapValue();
343
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
343
344
  assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
344
345
  }
345
346
 
@@ -367,7 +368,7 @@ public class TestJsonVisitor
367
368
  k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
368
369
  k2, ValueFactory.newArray(v, v));
369
370
 
370
- MapValue visited = subject.visit("$.json1", map).asMapValue();
371
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
371
372
  assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
372
373
  }
373
374
 
@@ -397,7 +398,394 @@ public class TestJsonVisitor
397
398
  k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
398
399
  k2, ValueFactory.newArray(v, v));
399
400
 
400
- MapValue visited = subject.visit("$.json1", map).asMapValue();
401
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
401
402
  assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
402
403
  }
403
- }
404
+
405
+ @Test
406
+ public void visitMap_dropColumnsUsingBracketNotation()
407
+ {
408
+ PluginTask task = taskFromYamlString(
409
+ "type: column",
410
+ "drop_columns:",
411
+ " - {name: \"$['json1']['k1']['k1']\"}",
412
+ " - {name: \"$['json1']['k2']\"}");
413
+ Schema inputSchema = Schema.builder()
414
+ .add("json1", JSON)
415
+ .add("json2", JSON)
416
+ .build();
417
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
418
+
419
+ // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
420
+ Value k1 = ValueFactory.newString("k1");
421
+ Value k2 = ValueFactory.newString("k2");
422
+ Value v = ValueFactory.newString("v");
423
+ Value map = ValueFactory.newMap(
424
+ k1, ValueFactory.newMap(k1, v),
425
+ k2, ValueFactory.newMap(k2, v));
426
+
427
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
428
+ assertEquals("{\"k1\":{}}", visited.toString());
429
+ }
430
+
431
+ @Test
432
+ public void visitMap_addColumnsUsingBracketNotation()
433
+ {
434
+ PluginTask task = taskFromYamlString(
435
+ "type: column",
436
+ "add_columns:",
437
+ " - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
438
+ " - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
439
+ " - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
440
+ Schema inputSchema = Schema.builder()
441
+ .add("json1", JSON)
442
+ .add("json2", JSON)
443
+ .build();
444
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
445
+
446
+ // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
447
+ Value k1 = ValueFactory.newString("k1");
448
+ Value k2 = ValueFactory.newString("k2");
449
+ Value v = ValueFactory.newString("v");
450
+ Value map = ValueFactory.newMap(
451
+ k1, ValueFactory.newMap(k1, v),
452
+ k2, ValueFactory.newMap(k2, v));
453
+
454
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
455
+ assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
456
+ }
457
+
458
+ @Test
459
+ public void visitMap_columnsUsingBracketNotation()
460
+ {
461
+ PluginTask task = taskFromYamlString(
462
+ "type: column",
463
+ "columns:",
464
+ " - {name: \"$['json1']['k1']\"}",
465
+ " - {name: \"$['json1']['k2']['k2']\"}",
466
+ " - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
467
+ " - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
468
+ " - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
469
+ Schema inputSchema = Schema.builder()
470
+ .add("json1", JSON)
471
+ .build();
472
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
473
+
474
+ // {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
475
+ Value k1 = ValueFactory.newString("k1");
476
+ Value k2 = ValueFactory.newString("k2");
477
+ Value v = ValueFactory.newString("v");
478
+ Value map = ValueFactory.newMap(
479
+ k1, ValueFactory.newMap(k1, v),
480
+ k2, ValueFactory.newMap(k2, v));
481
+
482
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
483
+ assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
484
+ }
485
+
486
+ @Test
487
+ public void visitArray_dropColumnsUsingBracketNotation()
488
+ {
489
+ PluginTask task = taskFromYamlString(
490
+ "type: column",
491
+ "drop_columns:",
492
+ " - {name: \"$['json1']['k1'][0]['k1']\"}",
493
+ " - {name: \"$['json1']['k2'][*]\"}"); // ending with [*] is allowed for drop_columns, but not for others
494
+ Schema inputSchema = Schema.builder()
495
+ .add("json1", JSON)
496
+ .add("json2", JSON)
497
+ .build();
498
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
499
+
500
+ // {"k1":[{"k1":"v"}[,"k2":["v","v"]}
501
+ Value k1 = ValueFactory.newString("k1");
502
+ Value k2 = ValueFactory.newString("k2");
503
+ Value v = ValueFactory.newString("v");
504
+ Value map = ValueFactory.newMap(
505
+ k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
506
+ k2, ValueFactory.newArray(v, v));
507
+
508
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
509
+ assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
510
+ }
511
+
512
+ @Test
513
+ public void visitArray_addColumnsUsingBracketNotation()
514
+ {
515
+ PluginTask task = taskFromYamlString(
516
+ "type: column",
517
+ "add_columns:",
518
+ " - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
519
+ " - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
520
+ " - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
521
+ " - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
522
+ Schema inputSchema = Schema.builder()
523
+ .add("json1", JSON)
524
+ .add("json2", JSON)
525
+ .build();
526
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
527
+
528
+ // {"k1":[{"k1":"v"}],"k2":["v","v"]}
529
+ Value k1 = ValueFactory.newString("k1");
530
+ Value k2 = ValueFactory.newString("k2");
531
+ Value v = ValueFactory.newString("v");
532
+ Value map = ValueFactory.newMap(
533
+ k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
534
+ k2, ValueFactory.newArray(v, v));
535
+
536
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
537
+ assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
538
+ }
539
+
540
+ @Test
541
+ public void visitArray_columnsUsingBracketNotation()
542
+ {
543
+ PluginTask task = taskFromYamlString(
544
+ "type: column",
545
+ "columns:",
546
+ " - {name: \"$['json1']['k1']\"}",
547
+ " - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
548
+ " - {name: \"$['json1']['k2'][0]\"}",
549
+ " - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
550
+ " - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
551
+ " - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
552
+ Schema inputSchema = Schema.builder()
553
+ .add("json1", JSON)
554
+ .build();
555
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
556
+
557
+ // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
558
+ Value k1 = ValueFactory.newString("k1");
559
+ Value k2 = ValueFactory.newString("k2");
560
+ Value v = ValueFactory.newString("v");
561
+ Value map = ValueFactory.newMap(
562
+ k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
563
+ k2, ValueFactory.newArray(v, v));
564
+
565
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
566
+ assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
567
+ }
568
+
569
+ // Because the dot notation is converted to single quotes by default,
570
+ // it can be mixed with the bracket notation of single quotes
571
+ @Test
572
+ public void visit_withDotAndBracket()
573
+ {
574
+ PluginTask task = taskFromYamlString(
575
+ "type: column",
576
+ "columns:",
577
+ " - {name: \"$.json1['k_1']\"}",
578
+ " - {name: \"$.json1['k_1'][0]['k_1']\"}",
579
+ " - {name: \"$['json1']['k_2']\"}",
580
+ " - {name: \"$['json1']['k_2']['k_2']\"}");
581
+ Schema inputSchema = Schema.builder()
582
+ .add("json1", JSON)
583
+ .build();
584
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
585
+
586
+ // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
587
+ Value k1 = ValueFactory.newString("k_1");
588
+ Value k2 = ValueFactory.newString("k_2");
589
+ Value v = ValueFactory.newString("v");
590
+ Value map = ValueFactory.newMap(
591
+ k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
592
+ k2, ValueFactory.newMap(k2, v));
593
+
594
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
595
+ assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
596
+ }
597
+
598
+ // Because the bracket notation of double quotes converted to single quotes internally
599
+ // it can be mixed with the bracket notation of single quotes
600
+ @Test
601
+ public void visit_withSingleQuotesAndDoubleQuotes()
602
+ {
603
+ PluginTask task = taskFromYamlString(
604
+ "type: column",
605
+ "columns:",
606
+ " - {name: \"$['json1']['k_1']\", src: \"$['json1']['k.1']\"}",
607
+ " - {name: '$[\"json1\"][\"k_1\"][0][\"k_1\"]', src: '$[\"json1\"][\"k_1\"][0][\"k.1\"]'}",
608
+ " - {name: '$[\"json1\"][\"k_2\"]', src: '$[\"json1\"][\"k.2\"]'}",
609
+ " - {name: '$[\"json1\"][\"k_2\"][\"k_2\"]', src: '$[\"json1\"][\"k_2\"][\"k.2\"]'}");
610
+ Schema inputSchema = Schema.builder()
611
+ .add("json1", JSON)
612
+ .build();
613
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
614
+
615
+ // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
616
+ Value k1 = ValueFactory.newString("k.1");
617
+ Value k2 = ValueFactory.newString("k.2");
618
+ Value v = ValueFactory.newString("v");
619
+ Value map = ValueFactory.newMap(
620
+ k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
621
+ k2, ValueFactory.newMap(k2, v));
622
+
623
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
624
+ assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
625
+ }
626
+
627
+ @Test
628
+ public void visit_withComplexRename()
629
+ {
630
+ PluginTask task = taskFromYamlString(
631
+ "type: column",
632
+ "columns:",
633
+ " - {name: \"$.json1['k____1']\", src: \"$.json1['k.-=+1']\"}",
634
+ " - {name: \"$.json1['k____1'][0]['k____1']\", src: \"$.json1['k____1'][0]['k.-=+1']\"}",
635
+ " - {name: \"$['json1']['k_2']\", src: \"$['json1']['k.2']\"}",
636
+ " - {name: \"$['json1']['k_2']['k_2']\", src: \"$['json1']['k_2']['k.2']\"}");
637
+ Schema inputSchema = Schema.builder()
638
+ .add("json1", JSON)
639
+ .build();
640
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
641
+
642
+ // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
643
+ Value k1 = ValueFactory.newString("k.-=+1");
644
+ Value k2 = ValueFactory.newString("k.2");
645
+ Value v = ValueFactory.newString("v");
646
+ Value map = ValueFactory.newMap(
647
+ k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
648
+ k2, ValueFactory.newMap(k2, v));
649
+
650
+ MapValue visited = subject.visit("$['json1']", map).asMapValue();
651
+ assertEquals("{\"k____1\":[{\"k____1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
652
+ }
653
+
654
+ @Test
655
+ public void visit_withColumnNameIncludingSingleQuotes()
656
+ {
657
+ PluginTask task = taskFromYamlString(
658
+ "type: column",
659
+ "columns:",
660
+ " - {name: \"$[\\\"'json1\\\"]['k1']\"}");
661
+ Schema inputSchema = Schema.builder()
662
+ .add("'json1", JSON)
663
+ .build();
664
+ JsonVisitor subject = jsonVisitor(task, inputSchema);
665
+
666
+ // {"k1":"v"}
667
+ Value k1 = ValueFactory.newString("k1");
668
+ Value v = ValueFactory.newString("v");
669
+ Value map = ValueFactory.newMap(k1, v);
670
+
671
+ MapValue visited = subject.visit("$['\\'json1']", map).asMapValue();
672
+ assertEquals("{\"k1\":\"v\"}", visited.toString());
673
+ }
674
+
675
+ @Test(expected = ConfigException.class)
676
+ public void constructor_mustBeRaisedConfigExceptionWithMultiProperties() {
677
+ PluginTask task = taskFromYamlString(
678
+ "type: column",
679
+ "columns:",
680
+ "- name: \"$['json1','k1']\"");
681
+ Schema inputSchema = Schema.builder()
682
+ .add("json1", JSON)
683
+ .build();
684
+ jsonVisitor(task, inputSchema);
685
+ }
686
+
687
+ // It is recognized multi properties if the square brackets does not close properly
688
+ @Test(expected = InvalidPathException.class)
689
+ public void constructor_mustBeRaisedInvalidPathExceptionWithPropertyIsNotSeparatedByCommas()
690
+ {
691
+ PluginTask task = taskFromYamlString(
692
+ "type: column",
693
+ "columns:",
694
+ " - name: \"$['json1'}['k1']\"");
695
+ Schema inputSchema = Schema.builder()
696
+ .add("json1", JSON)
697
+ .build();
698
+ jsonVisitor(task, inputSchema);
699
+ }
700
+
701
+ @Test(expected = ConfigException.class)
702
+ public void constructor_mustBeRaisedConfigExceptionWithFunctionPathToken()
703
+ {
704
+ PluginTask task = taskFromYamlString(
705
+ "type: column",
706
+ "columns:",
707
+ "- name: \"$['json1'].length()\"");
708
+ Schema inputSchema = Schema.builder()
709
+ .add("json1", JSON)
710
+ .build();
711
+ jsonVisitor(task, inputSchema);
712
+ }
713
+
714
+ @Test(expected = ConfigException.class)
715
+ public void constructor_mustBeRaisedConfigExceptionWithPredicatePathToken()
716
+ {
717
+ PluginTask task = taskFromYamlString(
718
+ "type: column",
719
+ "columns:",
720
+ "- name: \"$.store.book[?(@.price < 10)]\"");
721
+ Schema inputSchema = Schema.builder()
722
+ .add("store", JSON)
723
+ .build();
724
+ jsonVisitor(task, inputSchema);
725
+ }
726
+
727
+ @Test(expected = ConfigException.class)
728
+ public void constructor_mustBeRaisedConfigExceptionWithScanPathToken()
729
+ {
730
+ PluginTask task = taskFromYamlString(
731
+ "type: column",
732
+ "columns:",
733
+ "- name: \"$.json1..key1\"");
734
+ Schema inputSchema = Schema.builder()
735
+ .add("json1", JSON)
736
+ .build();
737
+ jsonVisitor(task, inputSchema);
738
+ }
739
+
740
+ @Test(expected = ConfigException.class)
741
+ public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperation()
742
+ {
743
+ PluginTask task = taskFromYamlString(
744
+ "type: column",
745
+ "columns:",
746
+ "- name: \"$.json1[0,1]\"");
747
+ Schema inputSchema = Schema.builder()
748
+ .add("json1", JSON)
749
+ .build();
750
+ jsonVisitor(task, inputSchema);
751
+ }
752
+
753
+ @Test(expected = ConfigException.class)
754
+ public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperationAtMiddlePosition()
755
+ {
756
+ PluginTask task = taskFromYamlString(
757
+ "type: column",
758
+ "columns:",
759
+ "- name: \"$.json1[0,1].key1\"");
760
+ Schema inputSchema = Schema.builder()
761
+ .add("json1", JSON)
762
+ .build();
763
+ jsonVisitor(task, inputSchema);
764
+ }
765
+
766
+ @Test(expected = ConfigException.class)
767
+ public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperation()
768
+ {
769
+ PluginTask task = taskFromYamlString(
770
+ "type: column",
771
+ "columns:",
772
+ "- name: \"$.json1[1:2]\"");
773
+ Schema inputSchema = Schema.builder()
774
+ .add("json1", JSON)
775
+ .build();
776
+ jsonVisitor(task, inputSchema);
777
+ }
778
+
779
+ @Test(expected = ConfigException.class)
780
+ public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperationAtMiddlePosition()
781
+ {
782
+ PluginTask task = taskFromYamlString(
783
+ "type: column",
784
+ "columns:",
785
+ "- name: \"$.json1[1:2].key1\"");
786
+ Schema inputSchema = Schema.builder()
787
+ .add("json1", JSON)
788
+ .build();
789
+ jsonVisitor(task, inputSchema);
790
+ }
791
+ }