embulk-filter-column 0.6.0.pre4 → 0.6.0.pre5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/build.gradle +1 -1
  3. data/src/main/java/org/embulk/filter/column/JsonColumn.java +6 -0
  4. data/src/main/java/org/embulk/filter/column/JsonVisitor.java +22 -10
  5. data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java +23 -12
  6. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cdfb9c7f62fa8351885416d57e2d517025af0111
4
- data.tar.gz: e09a8286941062cbf5d47bf73e4fd75b011b062b
3
+ metadata.gz: 123d41cdcecc136bceb265c9d77f40719717955e
4
+ data.tar.gz: 250eef3285efb754089b9b6de2fe181721d960a5
5
5
  SHA512:
6
- metadata.gz: 798f800dffdd91ba5333e96f8d9efaf22c448220c56e34f16029dd75b88035447b7acf60bb3312c2e30f1910292e0572db3b11e4b3247fb117e162d6bb417305
7
- data.tar.gz: b6e459926af443198e02a716b20bc866ecfefa109b7771a70a2a238dd51a6d0b168745a9612c944bc6588c86caff8515bff61bda21e824ae00ec29838b0686ec
6
+ metadata.gz: 5e7a8e33c7c8bbe68701174b3533dd4d568324280ee74c35a1187aadcf4eff77ce0f74c790e5c4295d972cd59bea08063e8444d39336e86698e4f927b575371e
7
+ data.tar.gz: 5bd87c09fa8528e632171e67594a2326e2a00514e7083b57eb9d72a4c6f6ae23628ff14c01f877b9d6819c27d0f08f524690a4534dc2039a33f75379ae947347
data/build.gradle CHANGED
@@ -15,7 +15,7 @@ configurations {
15
15
  provided
16
16
  }
17
17
 
18
- version = "0.6.0.pre4"
18
+ version = "0.6.0.pre5"
19
19
  sourceCompatibility = 1.7
20
20
  targetCompatibility = 1.7
21
21
 
data/src/main/java/org/embulk/filter/column/JsonColumn.java CHANGED
@@ -7,6 +7,7 @@ import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
7
7
  import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
8
8
  import io.github.medjed.jsonpathcompiler.expressions.path.RootPathToken;
9
9
  import io.github.medjed.jsonpathcompiler.expressions.path.PropertyPathToken;
10
+ import io.github.medjed.jsonpathcompiler.expressions.path.WildcardPathToken;
10
11
  import org.embulk.config.ConfigException;
11
12
  import org.embulk.spi.type.Type;
12
13
  import org.msgpack.value.StringValue;
@@ -32,6 +33,8 @@ public class JsonColumn
32
33
  private StringValue srcParentPathValue = null;
33
34
  private Value srcTailNameValue = null;
34
35
 
36
+ public static int WILDCARD_INDEX = -1;
37
+
35
38
  public JsonColumn(String path, Type type)
36
39
  {
37
40
  this(path, type, null, null);
@@ -97,6 +100,9 @@ public class JsonColumn
97
100
  JsonPathUtil.assertSupportedArrayPathToken(arrayIndexOperation, path);
98
101
  return arrayIndexOperation.indexes().get(0).longValue();
99
102
  }
103
+ else if (tail instanceof WildcardPathToken) {
104
+ return new Long(WILDCARD_INDEX);
105
+ }
100
106
  else {
101
107
  return null;
102
108
  }
data/src/main/java/org/embulk/filter/column/JsonVisitor.java CHANGED
@@ -5,6 +5,7 @@ import io.github.medjed.jsonpathcompiler.expressions.Path;
5
5
  import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
6
6
  import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
7
7
  import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
8
+ import io.github.medjed.jsonpathcompiler.expressions.path.WildcardPathToken;
8
9
  import org.embulk.config.ConfigException;
9
10
  import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
10
11
  import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
@@ -286,7 +287,7 @@ public class JsonVisitor
286
287
  partialPath.append(parts.getPathFragment());
287
288
  PathToken next = parts.next();
288
289
  JsonColumn jsonColumn;
289
- if (next instanceof ArrayPathToken) {
290
+ if (next instanceof ArrayPathToken || next instanceof WildcardPathToken) {
290
291
  jsonColumn = new JsonColumn(partialPath.toString(), Types.JSON, ValueFactory.newArray(new Value[0], false));
291
292
  }
292
293
  else {
@@ -337,15 +338,26 @@ public class JsonVisitor
337
338
  else if (this.jsonColumns.containsKey(rootPath)) {
338
339
  for (JsonColumn jsonColumn : this.jsonColumns.get(rootPath).values()) {
339
340
  int src = jsonColumn.getSrcTailIndex().intValue();
340
- Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
341
- if (v == null) {
342
- v = jsonColumn.getDefaultValue();
341
+ if (src == JsonColumn.WILDCARD_INDEX) {
342
+ for (int i = 0; i < size; i++) {
343
+ Value v = arrayValue.get(i);
344
+ if (v == null) {
345
+ v = jsonColumn.getDefaultValue();
346
+ }
347
+ String newPath = jsonColumn.getPath();
348
+ Value visited = visit(newPath, v);
349
+ newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
350
+ }
351
+ }
352
+ else {
353
+ Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
354
+ if (v == null) {
355
+ v = jsonColumn.getDefaultValue();
356
+ }
357
+ String newPath = jsonColumn.getPath();
358
+ Value visited = visit(newPath, v);
359
+ newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
343
360
  }
344
- String newPath = jsonColumn.getPath();
345
- Value visited = visit(newPath, v);
346
- // int i = jsonColumn.getTailIndex().intValue();
347
- // index is shifted, so j++ is used.
348
- newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
349
361
  }
350
362
  }
351
363
  else {
@@ -358,7 +370,7 @@ public class JsonVisitor
358
370
  if (this.jsonAddColumns.containsKey(rootPath)) {
359
371
  for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
360
372
  int i = jsonColumn.getTailIndex().intValue();
361
- if (i < size) {
373
+ if (i == JsonColumn.WILDCARD_INDEX || i < size) {
362
374
  // index for add_columns must be larger than size
363
375
  // just skip because we can not raise ConfigException beforehand for flexible JSON
364
376
  continue;
data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java CHANGED
@@ -384,23 +384,26 @@ public class TestJsonVisitor
384
384
  "type: column",
385
385
  "drop_columns:",
386
386
  " - {name: \"$.json1.k1[0].k1\"}",
387
- " - {name: \"$.json1.k2[*]\"}"); // ending with [*] is allowed for drop_columns, but not for others
387
+ " - {name: \"$.json1.k2[*]\"}", // ending with [*] is allowed for drop_columns, but not for others
388
+ " - {name: \"$.json1.k3[*].k1\"}");
388
389
  Schema inputSchema = Schema.builder()
389
390
  .add("json1", JSON)
390
391
  .add("json2", JSON)
391
392
  .build();
392
393
  JsonVisitor subject = jsonVisitor(task, inputSchema);
393
394
 
394
- // {"k1":[{"k1":"v"}[,"k2":["v","v"]}
395
+ // {"k1":[{"k1":"v"}],"k2":["v","v"],"k3":[{"k3":"v"}]}
395
396
  Value k1 = ValueFactory.newString("k1");
396
397
  Value k2 = ValueFactory.newString("k2");
398
+ Value k3 = ValueFactory.newString("k3");
397
399
  Value v = ValueFactory.newString("v");
398
400
  Value map = ValueFactory.newMap(
399
401
  k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
400
- k2, ValueFactory.newArray(v, v));
402
+ k2, ValueFactory.newArray(v, v),
403
+ k3, ValueFactory.newArray(ValueFactory.newMap(k1, v)));
401
404
 
402
405
  MapValue visited = subject.visit("$['json1']", map).asMapValue();
403
- assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
406
+ assertEquals("{\"k1\":[{}],\"k2\":[],\"k3\":[{}]}", visited.toString());
404
407
  }
405
408
 
406
409
  @Test
@@ -410,23 +413,27 @@ public class TestJsonVisitor
410
413
  "type: column",
411
414
  "add_columns:",
412
415
  " - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
413
- " - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
416
+ " - {name: \"$.json1.k3[*].k2\", type: string, default: v}",
417
+ " - {name: \"$.json1.k4[*].k1\", type: string, default: v}",
418
+ " - {name: \"$.json1.k5[0].k1\", type: string, default: v}");
414
419
  Schema inputSchema = Schema.builder()
415
420
  .add("json1", JSON)
416
421
  .add("json2", JSON)
417
422
  .build();
418
423
  JsonVisitor subject = jsonVisitor(task, inputSchema);
419
424
 
420
- // {"k1":[{"k1":"v"}],"k2":["v","v"]}
425
+ // {"k1":[{"k1":"v"}],"k2":["v","v"],"k3":[{"k1":"v"}]}
421
426
  Value k1 = ValueFactory.newString("k1");
422
427
  Value k2 = ValueFactory.newString("k2");
428
+ Value k3 = ValueFactory.newString("k3");
423
429
  Value v = ValueFactory.newString("v");
424
430
  Value map = ValueFactory.newMap(
425
431
  k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
426
- k2, ValueFactory.newArray(v, v));
432
+ k2, ValueFactory.newArray(v, v),
433
+ k3, ValueFactory.newArray(ValueFactory.newMap(k1, v)));
427
434
 
428
435
  MapValue visited = subject.visit("$['json1']", map).asMapValue();
429
- assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
436
+ assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k1\":\"v\",\"k2\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString());
430
437
  }
431
438
 
432
439
  @Test
@@ -437,23 +444,27 @@ public class TestJsonVisitor
437
444
  "columns:",
438
445
  " - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
439
446
  " - {name: \"$.json1.k2[0]\"}",
440
- " - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
447
+ " - {name: \"$.json1.k3[*].k1\"}",
448
+ " - {name: \"$.json1.k4[*].k1\", type: string, default: v}",
449
+ " - {name: \"$.json1.k5[0].k1\", type: string, default: v}");
441
450
  Schema inputSchema = Schema.builder()
442
451
  .add("json1", JSON)
443
452
  .add("json2", JSON)
444
453
  .build();
445
454
  JsonVisitor subject = jsonVisitor(task, inputSchema);
446
455
 
447
- // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
456
+ // {"k1":[{"k1":"v"},"v"],"k2":["v","v"],"k3":[{"k1":"v","k2":"v"}]}
448
457
  Value k1 = ValueFactory.newString("k1");
449
458
  Value k2 = ValueFactory.newString("k2");
459
+ Value k3 = ValueFactory.newString("k3");
450
460
  Value v = ValueFactory.newString("v");
451
461
  Value map = ValueFactory.newMap(
452
462
  k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
453
- k2, ValueFactory.newArray(v, v));
463
+ k2, ValueFactory.newArray(v, v),
464
+ k3, ValueFactory.newArray(ValueFactory.newMap(k1, v, k2, v)));
454
465
 
455
466
  MapValue visited = subject.visit("$['json1']", map).asMapValue();
456
- assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
467
+ assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k1\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString());
457
468
  }
458
469
 
459
470
  @Test
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-filter-column
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0.pre4
4
+ version: 0.6.0.pre5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Naotoshi Seo
@@ -82,7 +82,7 @@ files:
82
82
  - classpath/accessors-smart-1.1.jar
83
83
  - classpath/asm-5.0.3.jar
84
84
  - classpath/commons-lang3-3.4.jar
85
- - classpath/embulk-filter-column-0.6.0.pre4.jar
85
+ - classpath/embulk-filter-column-0.6.0.pre5.jar
86
86
  - classpath/json-smart-2.2.1.jar
87
87
  - classpath/JsonPathCompiler-0.0.12.jar
88
88
  - classpath/slf4j-api-1.7.21.jar