embulk-filter-column 0.5.4 → 0.6.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/CHANGELOG.md +6 -0
- data/README.md +1 -0
- data/build.gradle +2 -1
- data/example/bracket_notations.yml +40 -0
- data/example/edgecase.tsv +4 -0
- data/example/edgecase.yml +22 -0
- data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java +7 -5
- data/src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java +3 -2
- data/src/main/java/org/embulk/filter/column/JsonColumn.java +80 -66
- data/src/main/java/org/embulk/filter/column/JsonVisitor.java +33 -34
- data/src/main/java/org/embulk/filter/column/PathTokenUtil.java +39 -0
- data/src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java +1 -1
- data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java +1 -1
- data/src/test/java/org/embulk/filter/column/TestJsonColumn.java +38 -16
- data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java +439 -51
- metadata +15 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ca161d763f10dc37da758c3bac6d0384a3b4edf
|
4
|
+
data.tar.gz: dfe8251c7268898b6ba42c4aa0da2298b2242ba7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c03b1816ca0d300c3b9260dea666d3a361ce9413e685e4ac419923ad27ca925428031969888c7a9d8f25cd511ae127acb987194b761b608044319fee96653ea8
|
7
|
+
data.tar.gz: 37d963fc19b1c78028ce043810b3c88badba92b8a72d116e2b30e208dcacdd85a88d2d55a6379cfd9fffff7d234fbd50b306c40d9a3adc74316e7cc522b295bb
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
data/build.gradle
CHANGED
@@ -15,13 +15,14 @@ configurations {
|
|
15
15
|
provided
|
16
16
|
}
|
17
17
|
|
18
|
-
version = "0.
|
18
|
+
version = "0.6.0.pre1"
|
19
19
|
sourceCompatibility = 1.7
|
20
20
|
targetCompatibility = 1.7
|
21
21
|
|
22
22
|
dependencies {
|
23
23
|
compile "org.embulk:embulk-core:0.8.+"
|
24
24
|
provided "org.embulk:embulk-core:0.8.+"
|
25
|
+
compile 'com.dena.analytics:JsonPathCompiler:0.0.6'
|
25
26
|
|
26
27
|
testCompile "junit:junit:4.+"
|
27
28
|
testCompile "org.embulk:embulk-core:0.8.+:tests"
|
@@ -0,0 +1,40 @@
|
|
1
|
+
in:
|
2
|
+
type: file
|
3
|
+
path_prefix: example/example.csv
|
4
|
+
parser:
|
5
|
+
type: csv
|
6
|
+
charset: UTF-8
|
7
|
+
newline: CRLF
|
8
|
+
null_string: 'NULL'
|
9
|
+
skip_header_lines: 1
|
10
|
+
comment_line_marker: '#'
|
11
|
+
columns:
|
12
|
+
- {name: time, type: timestamp, format: "%Y-%m-%d"}
|
13
|
+
- {name: id, type: long}
|
14
|
+
- {name: name, type: string}
|
15
|
+
- {name: score, type: double}
|
16
|
+
- {name: json, type: json}
|
17
|
+
filters:
|
18
|
+
- type: column
|
19
|
+
add_columns:
|
20
|
+
- {name: "$['json']['array.a']", type: json, default: "[{\"a\":\"a0\"},{\"a\":\"a1\"}]"}
|
21
|
+
- {name: "$['json']['array_b']", type: json, default: "[{\"b\":\"b\"},{\"b\":\"b\"}]"}
|
22
|
+
- type: column
|
23
|
+
columns:
|
24
|
+
- {name: time, default: "2015-07-13", format: "%Y-%m-%d"}
|
25
|
+
- {name: name, default: "foo"}
|
26
|
+
- {name: foo, default: 1, type: long}
|
27
|
+
- {name: id}
|
28
|
+
- {name: copy_score, src: score}
|
29
|
+
- {name: json, default: "{\"foo\":\"FOO\",\"array.a\":[{\"a\":\"default0\"},{\"a\":\"default1\"}],\"array_b\":[{\"b\":\"default\"},{\"b\":\"default\"}]}"}
|
30
|
+
- {name: "$['json']['foo']"}
|
31
|
+
- {name: "$['json']['copy_foo']", src: "$['json']['foo']"}
|
32
|
+
- {name: "$['json']['drop_foo']", src: "$['json']['foo']"}
|
33
|
+
- {name: "$['json']['array_a']", src: "$['json']['array.a']"}
|
34
|
+
- {name: "$['json']['array_a'][0]"}
|
35
|
+
- {name: "$['json']['array_b']"}
|
36
|
+
- type: column
|
37
|
+
drop_columns:
|
38
|
+
- {name: "$['json']['drop_foo']"}
|
39
|
+
out:
|
40
|
+
type: stdout
|
@@ -0,0 +1,22 @@
|
|
1
|
+
in:
|
2
|
+
type: file
|
3
|
+
path_prefix: example/edgecase.tsv
|
4
|
+
parser:
|
5
|
+
type: csv
|
6
|
+
delimiter: \t
|
7
|
+
charset: UTF-8
|
8
|
+
newline: CRLF
|
9
|
+
null_string: 'NULL'
|
10
|
+
skip_header_lines: 1
|
11
|
+
comment_line_marker: '#'
|
12
|
+
columns:
|
13
|
+
- {name: json, type: json}
|
14
|
+
filters:
|
15
|
+
- type: column
|
16
|
+
columns:
|
17
|
+
- {name: json, default: "{\"phone']Num\\\"]ber\":\"FOO\"}"}
|
18
|
+
- {name: "$.json[\"phone']Num\\\"]ber\"]"}
|
19
|
+
- {name: $.json.copy_foo, src: "$.json[\"phone']Num\\\"]ber\"]"}
|
20
|
+
- {name: "$.json['foo\\']\"]foo']"}
|
21
|
+
out:
|
22
|
+
type: stdout
|
@@ -27,6 +27,8 @@ import org.slf4j.Logger;
|
|
27
27
|
|
28
28
|
import java.util.List;
|
29
29
|
|
30
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PathCompiler;
|
31
|
+
|
30
32
|
public class ColumnFilterPlugin implements FilterPlugin
|
31
33
|
{
|
32
34
|
private static final Logger logger = Exec.getLogger(ColumnFilterPlugin.class);
|
@@ -117,7 +119,7 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
117
119
|
boolean matched = false;
|
118
120
|
for (ColumnConfig dropColumn : dropColumns) {
|
119
121
|
// skip json path notation to build outputSchema
|
120
|
-
if (dropColumn.getName()
|
122
|
+
if (PathCompiler.isProbablyJsonPath(dropColumn.getName())) {
|
121
123
|
continue;
|
122
124
|
}
|
123
125
|
if (dropColumn.getName().equals(name)) {
|
@@ -134,10 +136,10 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
134
136
|
else if (columns.size() > 0) {
|
135
137
|
for (ColumnConfig column : columns) {
|
136
138
|
// skip json path notation to build output schema
|
137
|
-
if (column.getName()
|
139
|
+
if (PathCompiler.isProbablyJsonPath(column.getName())) {
|
138
140
|
continue;
|
139
141
|
}
|
140
|
-
if (column.getSrc().isPresent() && column.getSrc().get()
|
142
|
+
if (column.getSrc().isPresent() && PathCompiler.isProbablyJsonPath(column.getSrc().get())) {
|
141
143
|
continue;
|
142
144
|
}
|
143
145
|
|
@@ -178,10 +180,10 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
178
180
|
if (addColumns.size() > 0) {
|
179
181
|
for (ColumnConfig column : addColumns) {
|
180
182
|
// skip json path notation to build output schema
|
181
|
-
if (column.getName()
|
183
|
+
if (PathCompiler.isProbablyJsonPath(column.getName())) {
|
182
184
|
continue;
|
183
185
|
}
|
184
|
-
if (column.getSrc().isPresent() && column.getSrc().get()
|
186
|
+
if (column.getSrc().isPresent() && PathCompiler.isProbablyJsonPath(column.getSrc().get())) {
|
185
187
|
continue;
|
186
188
|
}
|
187
189
|
|
@@ -1,5 +1,6 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
+
import com.dena.analytics.jsonpathcompiler.expressions.Utils;
|
3
4
|
import com.google.common.base.Throwables;
|
4
5
|
|
5
6
|
import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
|
@@ -244,13 +245,13 @@ public class ColumnVisitorImpl implements ColumnVisitor
|
|
244
245
|
pageBuilder.setNull(outputColumn);
|
245
246
|
}
|
246
247
|
else {
|
247
|
-
String jsonPath = new StringBuilder("
|
248
|
+
String jsonPath = new StringBuilder("$['").append(Utils.escape(outputColumn.getName(), true)).append("']").toString();
|
248
249
|
pageBuilder.setJson(outputColumn, jsonVisitor.visit(jsonPath, defaultValue));
|
249
250
|
}
|
250
251
|
}
|
251
252
|
else {
|
252
253
|
Value value = pageReader.getJson(inputColumn);
|
253
|
-
String jsonPath = new StringBuilder("
|
254
|
+
String jsonPath = new StringBuilder("$['").append(Utils.escape(outputColumn.getName(), true)).append("']").toString();
|
254
255
|
pageBuilder.setJson(outputColumn, jsonVisitor.visit(jsonPath, value));
|
255
256
|
}
|
256
257
|
}
|
@@ -1,5 +1,12 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
+
import com.dena.analytics.jsonpathcompiler.expressions.Path;
|
4
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayIndexOperation;
|
5
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayPathToken;
|
6
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PathCompiler;
|
7
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
|
8
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.RootPathToken;
|
9
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PropertyPathToken;
|
3
10
|
import org.embulk.config.ConfigException;
|
4
11
|
import org.embulk.spi.type.Type;
|
5
12
|
import org.msgpack.value.StringValue;
|
@@ -15,17 +22,15 @@ public class JsonColumn
|
|
15
22
|
|
16
23
|
private StringValue pathValue = null;
|
17
24
|
private String parentPath = null;
|
18
|
-
private
|
19
|
-
private Long baseIndex = null;
|
25
|
+
private Long tailIndex = null;
|
20
26
|
private StringValue parentPathValue = null;
|
21
|
-
private
|
27
|
+
private Value tailNameValue = null;
|
22
28
|
|
23
29
|
private StringValue srcValue = null;
|
24
30
|
private String srcParentPath = null;
|
25
|
-
private
|
26
|
-
private Long srcBaseIndex = null;
|
31
|
+
private Long srcTailIndex = null;
|
27
32
|
private StringValue srcParentPathValue = null;
|
28
|
-
private
|
33
|
+
private Value srcTailNameValue = null;
|
29
34
|
|
30
35
|
public JsonColumn(String path, Type type)
|
31
36
|
{
|
@@ -39,33 +44,67 @@ public class JsonColumn
|
|
39
44
|
|
40
45
|
public JsonColumn(String path, Type type, Value defaultValue, String src)
|
41
46
|
{
|
42
|
-
|
47
|
+
Path compiledPath = PathCompiler.compile(path);
|
48
|
+
Path compiledSrc = src == null ? compiledPath : PathCompiler.compile(src);
|
49
|
+
RootPathToken compiledRoot = (RootPathToken) compiledPath.getRoot();
|
50
|
+
RootPathToken compiledSrcRoot = (RootPathToken) compiledSrc.getRoot();
|
51
|
+
this.path = compiledPath.toString();
|
43
52
|
this.type = type;
|
44
53
|
this.defaultValue = (defaultValue == null ? ValueFactory.newNil() : defaultValue);
|
45
|
-
this.src = (
|
54
|
+
this.src = compiledSrc.toString();
|
46
55
|
|
47
56
|
this.pathValue = ValueFactory.newString(path);
|
48
|
-
this.parentPath =
|
49
|
-
|
50
|
-
if (
|
57
|
+
this.parentPath = compiledPath.getParentPath();
|
58
|
+
|
59
|
+
if (compiledRoot.getTailPath().equals("[*]")) {
|
51
60
|
throw new ConfigException(String.format("%s wrongly ends with [*], perhaps you can remove the [*]", path));
|
52
61
|
}
|
53
|
-
this.
|
62
|
+
this.tailIndex = tailIndex(compiledRoot);
|
54
63
|
this.parentPathValue = ValueFactory.newString(parentPath);
|
55
|
-
|
64
|
+
String tailName = getTailName(compiledRoot);
|
65
|
+
this.tailNameValue = tailName == null ? ValueFactory.newNil() : ValueFactory.newString(tailName);
|
56
66
|
|
57
67
|
this.srcValue = ValueFactory.newString(this.src);
|
58
|
-
this.srcParentPath =
|
59
|
-
this.
|
60
|
-
this.srcBaseIndex = baseIndex(this.src);
|
68
|
+
this.srcParentPath = compiledSrc.getParentPath();
|
69
|
+
this.srcTailIndex = tailIndex(compiledSrcRoot);
|
61
70
|
this.srcParentPathValue = ValueFactory.newString(this.srcParentPath);
|
62
|
-
|
71
|
+
String srcTailName = getTailName(compiledSrcRoot);
|
72
|
+
this.srcTailNameValue = srcTailName == null ? ValueFactory.newNil() : ValueFactory.newString(srcTailName);
|
63
73
|
|
64
|
-
if (!
|
74
|
+
if (!srcParentPath.equals(parentPath)) {
|
65
75
|
throw new ConfigException(String.format("The branch (parent path) of src \"%s\" must be same with of name \"%s\" yet", src, path));
|
66
76
|
}
|
67
77
|
}
|
68
78
|
|
79
|
+
// $['foo'] or $.foo => foo
|
80
|
+
// $['foo'][0] or $.foo[0] or $['foo'][*] or $.foo[*] => null
|
81
|
+
private String getTailName(RootPathToken root)
|
82
|
+
{
|
83
|
+
PathToken pathToken = root.getTail();
|
84
|
+
if (pathToken instanceof PropertyPathToken) {
|
85
|
+
if (!((PropertyPathToken) pathToken).singlePropertyCase()) {
|
86
|
+
throw new ConfigException(String.format("Multiple property is not supported \"%s\"", root.toString()));
|
87
|
+
}
|
88
|
+
return ((PropertyPathToken) pathToken).getProperties().get(0);
|
89
|
+
}
|
90
|
+
else {
|
91
|
+
return null;
|
92
|
+
}
|
93
|
+
}
|
94
|
+
|
95
|
+
private Long tailIndex(RootPathToken root)
|
96
|
+
{
|
97
|
+
PathToken tail = root.getTail();
|
98
|
+
if (tail instanceof ArrayPathToken) {
|
99
|
+
ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) tail).getArrayIndexOperation();
|
100
|
+
PathTokenUtil.assertSupportedArrayPathToken(arrayIndexOperation, path);
|
101
|
+
return arrayIndexOperation.indexes().get(0).longValue();
|
102
|
+
}
|
103
|
+
else {
|
104
|
+
return null;
|
105
|
+
}
|
106
|
+
}
|
107
|
+
|
69
108
|
public String getPath()
|
70
109
|
{
|
71
110
|
return path;
|
@@ -96,14 +135,9 @@ public class JsonColumn
|
|
96
135
|
return parentPath;
|
97
136
|
}
|
98
137
|
|
99
|
-
public
|
100
|
-
{
|
101
|
-
return baseName;
|
102
|
-
}
|
103
|
-
|
104
|
-
public Long getBaseIndex()
|
138
|
+
public Long tailIndex()
|
105
139
|
{
|
106
|
-
return
|
140
|
+
return tailIndex;
|
107
141
|
}
|
108
142
|
|
109
143
|
public StringValue getParentPathValue()
|
@@ -111,9 +145,9 @@ public class JsonColumn
|
|
111
145
|
return parentPathValue;
|
112
146
|
}
|
113
147
|
|
114
|
-
public
|
148
|
+
public Value getTailNameValue()
|
115
149
|
{
|
116
|
-
return
|
150
|
+
return tailNameValue;
|
117
151
|
}
|
118
152
|
|
119
153
|
public StringValue getSrcValue()
|
@@ -126,14 +160,9 @@ public class JsonColumn
|
|
126
160
|
return srcParentPath;
|
127
161
|
}
|
128
162
|
|
129
|
-
public
|
130
|
-
{
|
131
|
-
return srcBaseName;
|
132
|
-
}
|
133
|
-
|
134
|
-
public Long getSrcBaseIndex()
|
163
|
+
public Long getSrcTailIndex()
|
135
164
|
{
|
136
|
-
return
|
165
|
+
return srcTailIndex;
|
137
166
|
}
|
138
167
|
|
139
168
|
public StringValue getSrcParentPathValue()
|
@@ -141,51 +170,36 @@ public class JsonColumn
|
|
141
170
|
return srcParentPathValue;
|
142
171
|
}
|
143
172
|
|
144
|
-
public
|
173
|
+
public Value getSrcTailNameValue()
|
145
174
|
{
|
146
|
-
return
|
175
|
+
return srcTailNameValue;
|
147
176
|
}
|
148
177
|
|
149
178
|
// like File.dirname
|
150
179
|
public static String parentPath(String path)
|
151
180
|
{
|
152
|
-
|
153
|
-
StringBuilder builder = new StringBuilder();
|
154
|
-
for (int i = 0; i < parts.length - 1; i++) {
|
155
|
-
builder.append(".").append(parts[i]);
|
156
|
-
}
|
157
|
-
if (parts[parts.length - 1].contains("[")) {
|
158
|
-
String[] arrayParts = parts[parts.length - 1].split("\\[");
|
159
|
-
builder.append(".").append(arrayParts[0]);
|
160
|
-
for (int j = 1; j < arrayParts.length - 1; j++) {
|
161
|
-
builder.append("[").append(arrayParts[j]);
|
162
|
-
}
|
163
|
-
}
|
164
|
-
return builder.deleteCharAt(0).toString();
|
181
|
+
return PathCompiler.compile(path).getParentPath();
|
165
182
|
}
|
166
183
|
|
167
|
-
public static String
|
184
|
+
public static String tailName(String path)
|
168
185
|
{
|
169
|
-
|
170
|
-
String[] arrayParts = parts[parts.length - 1].split("\\[");
|
171
|
-
if (arrayParts.length == 1) { // no [i]
|
172
|
-
return arrayParts[arrayParts.length - 1];
|
173
|
-
}
|
174
|
-
else {
|
175
|
-
return "[" + arrayParts[arrayParts.length - 1];
|
176
|
-
}
|
186
|
+
return ((RootPathToken) PathCompiler.compile(path).getRoot()).getTailPath();
|
177
187
|
}
|
178
188
|
|
179
|
-
public static Long
|
189
|
+
public static Long tailIndex(String path)
|
180
190
|
{
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
191
|
+
Path compiledPath = PathCompiler.compile(path);
|
192
|
+
PathToken tail = ((RootPathToken) compiledPath.getRoot()).getTail();
|
193
|
+
if (tail instanceof ArrayPathToken) {
|
194
|
+
ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) tail).getArrayIndexOperation();
|
195
|
+
if (arrayIndexOperation == null) {
|
196
|
+
throw new ConfigException(String.format("Array Slice Operation is not supported \"%s\"", path));
|
197
|
+
}
|
198
|
+
if (arrayIndexOperation.isSingleIndexOperation()) {
|
199
|
+
return arrayIndexOperation.indexes().get(0).longValue();
|
186
200
|
}
|
187
|
-
|
188
|
-
|
201
|
+
else {
|
202
|
+
throw new ConfigException(String.format("Multi Array Indexes is not supported \"%s\"", path));
|
189
203
|
}
|
190
204
|
}
|
191
205
|
else {
|
@@ -1,5 +1,8 @@
|
|
1
1
|
package org.embulk.filter.column;
|
2
2
|
|
3
|
+
import com.dena.analytics.jsonpathcompiler.expressions.Path;
|
4
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PathCompiler;
|
5
|
+
import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
|
3
6
|
import org.embulk.config.ConfigException;
|
4
7
|
import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
|
5
8
|
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
@@ -80,29 +83,32 @@ public class JsonVisitor
|
|
80
83
|
|
81
84
|
private void jsonColumnsPut(String path, JsonColumn value)
|
82
85
|
{
|
83
|
-
|
86
|
+
Path compiledPath = PathCompiler.compile(path);
|
87
|
+
String parentPath = compiledPath.getParentPath();
|
84
88
|
if (! jsonColumns.containsKey(parentPath)) {
|
85
89
|
jsonColumns.put(parentPath, new LinkedHashMap<String, JsonColumn>());
|
86
90
|
}
|
87
|
-
jsonColumns.get(parentPath).put(
|
91
|
+
jsonColumns.get(parentPath).put(compiledPath.toString(), value);
|
88
92
|
}
|
89
93
|
|
90
94
|
private void jsonAddColumnsPut(String path, JsonColumn value)
|
91
95
|
{
|
92
|
-
|
96
|
+
Path compiledPath = PathCompiler.compile(path);
|
97
|
+
String parentPath = compiledPath.getParentPath();
|
93
98
|
if (! jsonAddColumns.containsKey(parentPath)) {
|
94
99
|
jsonAddColumns.put(parentPath, new LinkedHashMap<String, JsonColumn>());
|
95
100
|
}
|
96
|
-
jsonAddColumns.get(parentPath).put(
|
101
|
+
jsonAddColumns.get(parentPath).put(compiledPath.toString(), value);
|
97
102
|
}
|
98
103
|
|
99
104
|
private void jsonDropColumnsPut(String path)
|
100
105
|
{
|
101
|
-
|
106
|
+
Path compiledPath = PathCompiler.compile(path);
|
107
|
+
String parentPath = compiledPath.getParentPath();
|
102
108
|
if (! jsonDropColumns.containsKey(parentPath)) {
|
103
109
|
jsonDropColumns.put(parentPath, new HashSet<String>());
|
104
110
|
}
|
105
|
-
jsonDropColumns.get(parentPath).add(
|
111
|
+
jsonDropColumns.get(parentPath).add(compiledPath.toString());
|
106
112
|
}
|
107
113
|
|
108
114
|
// build jsonColumns, jsonAddColumns, and jsonDropColumns
|
@@ -117,7 +123,7 @@ public class JsonVisitor
|
|
117
123
|
for (ColumnConfig dropColumn : dropColumns) {
|
118
124
|
String name = dropColumn.getName();
|
119
125
|
// skip NON json path notation to build output schema
|
120
|
-
if (!
|
126
|
+
if (! PathCompiler.isProbablyJsonPath(name)) {
|
121
127
|
continue;
|
122
128
|
}
|
123
129
|
jsonDropColumnsPut(name);
|
@@ -127,7 +133,7 @@ public class JsonVisitor
|
|
127
133
|
for (ColumnConfig column : columns) {
|
128
134
|
String name = column.getName();
|
129
135
|
// skip NON json path notation to build output schema
|
130
|
-
if (!
|
136
|
+
if (! PathCompiler.isProbablyJsonPath(name)) {
|
131
137
|
continue;
|
132
138
|
}
|
133
139
|
if (column.getSrc().isPresent()) {
|
@@ -151,7 +157,7 @@ public class JsonVisitor
|
|
151
157
|
for (ColumnConfig column : addColumns) {
|
152
158
|
String name = column.getName();
|
153
159
|
// skip NON json path notation to build output schema
|
154
|
-
if (!
|
160
|
+
if (! PathCompiler.isProbablyJsonPath(name)) {
|
155
161
|
continue;
|
156
162
|
}
|
157
163
|
if (column.getSrc().isPresent()) {
|
@@ -179,26 +185,19 @@ public class JsonVisitor
|
|
179
185
|
|
180
186
|
for (ColumnConfig columnConfig : columnConfigs) {
|
181
187
|
String name = columnConfig.getName();
|
182
|
-
if (!
|
188
|
+
if (!PathCompiler.isProbablyJsonPath(name)) {
|
183
189
|
continue;
|
184
190
|
}
|
185
|
-
|
191
|
+
Path path = PathCompiler.compile(name);
|
192
|
+
PathToken parts = path.getRoot();
|
193
|
+
int count = parts.getTokenCount();
|
186
194
|
StringBuilder partialPath = new StringBuilder("$");
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
// Simply add [0] or [*] here
|
194
|
-
partialPath.append("[").append(arrayParts[j]);
|
195
|
-
this.shouldVisitSet.add(partialPath.toString());
|
196
|
-
}
|
197
|
-
}
|
198
|
-
else {
|
199
|
-
partialPath.append(".").append(parts[i]);
|
200
|
-
this.shouldVisitSet.add(partialPath.toString());
|
201
|
-
}
|
195
|
+
// skip "$"
|
196
|
+
for (int i = 1; i < count; i++) {
|
197
|
+
parts = parts.next();
|
198
|
+
PathTokenUtil.assertSupportedPathToken(parts, name);
|
199
|
+
partialPath.append(parts.getPathFragment().toString());
|
200
|
+
this.shouldVisitSet.add(partialPath.toString());
|
202
201
|
}
|
203
202
|
}
|
204
203
|
}
|
@@ -220,7 +219,7 @@ public class JsonVisitor
|
|
220
219
|
String newMapJsonPath(String rootPath, Value elementPathValue)
|
221
220
|
{
|
222
221
|
String elementPath = elementPathValue.asStringValue().asString();
|
223
|
-
String newPath = new StringBuilder(rootPath).append("
|
222
|
+
String newPath = new StringBuilder(rootPath).append("['").append(elementPath).append("']").toString();
|
224
223
|
return newPath;
|
225
224
|
}
|
226
225
|
|
@@ -241,14 +240,14 @@ public class JsonVisitor
|
|
241
240
|
}
|
242
241
|
else if (this.jsonColumns.containsKey(rootPath)) {
|
243
242
|
for (JsonColumn jsonColumn : this.jsonColumns.get(rootPath).values()) {
|
244
|
-
int src = jsonColumn.
|
243
|
+
int src = jsonColumn.getSrcTailIndex().intValue();
|
245
244
|
Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
|
246
245
|
if (v == null) {
|
247
246
|
v = jsonColumn.getDefaultValue();
|
248
247
|
}
|
249
248
|
String newPath = jsonColumn.getPath();
|
250
249
|
Value visited = visit(newPath, v);
|
251
|
-
// int i = jsonColumn.
|
250
|
+
// int i = jsonColumn.tailIndex().intValue();
|
252
251
|
// index is shifted, so j++ is used.
|
253
252
|
newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
|
254
253
|
}
|
@@ -262,7 +261,7 @@ public class JsonVisitor
|
|
262
261
|
}
|
263
262
|
if (this.jsonAddColumns.containsKey(rootPath)) {
|
264
263
|
for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
|
265
|
-
int src = jsonColumn.
|
264
|
+
int src = jsonColumn.getSrcTailIndex().intValue();
|
266
265
|
Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
|
267
266
|
if (v == null) {
|
268
267
|
v = jsonColumn.getDefaultValue();
|
@@ -297,14 +296,14 @@ public class JsonVisitor
|
|
297
296
|
else if (this.jsonColumns.containsKey(rootPath)) {
|
298
297
|
Map<Value, Value> map = mapValue.map();
|
299
298
|
for (JsonColumn jsonColumn : this.jsonColumns.get(rootPath).values()) {
|
300
|
-
Value src = jsonColumn.
|
299
|
+
Value src = jsonColumn.getSrcTailNameValue();
|
301
300
|
Value v = map.get(src);
|
302
301
|
if (v == null) {
|
303
302
|
v = jsonColumn.getDefaultValue();
|
304
303
|
}
|
305
304
|
String newPath = jsonColumn.getPath();
|
306
305
|
Value visited = visit(newPath, v);
|
307
|
-
newValue.add(i++, jsonColumn.
|
306
|
+
newValue.add(i++, jsonColumn.getTailNameValue());
|
308
307
|
newValue.add(i++, visited == null ? ValueFactory.newNil() : visited);
|
309
308
|
}
|
310
309
|
}
|
@@ -321,14 +320,14 @@ public class JsonVisitor
|
|
321
320
|
if (this.jsonAddColumns.containsKey(rootPath)) {
|
322
321
|
Map<Value, Value> map = mapValue.map();
|
323
322
|
for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
|
324
|
-
Value src = jsonColumn.
|
323
|
+
Value src = jsonColumn.getSrcTailNameValue();
|
325
324
|
Value v = map.get(src);
|
326
325
|
if (v == null) {
|
327
326
|
v = jsonColumn.getDefaultValue();
|
328
327
|
}
|
329
328
|
String newPath = jsonColumn.getPath();
|
330
329
|
Value visited = visit(newPath, v);
|
331
|
-
newValue.add(i++, jsonColumn.
|
330
|
+
newValue.add(i++, jsonColumn.getTailNameValue());
|
332
331
|
newValue.add(i++, visited == null ? ValueFactory.newNil() : visited);
|
333
332
|
}
|
334
333
|
}
|