embulk-filter-expand_json 0.0.3 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +4 -0
- data/build.gradle +22 -4
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java +14 -4
- data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java +191 -94
- data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java +44 -7
- metadata +11 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 07943e1a2bf6a447bccb93cbd9ec0412f00382f0
|
|
4
|
+
data.tar.gz: 0051fdf7de1702ddd169afb6d9a4fe9ce53e0ded
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 860428fccd8b863e1070ef0e23f657c88e5312031aa222588e51b20c7caa394b2425f26dfc3322fdf69dbc0d3984ff524e3969666cb9a53c68672ead92fd8c25
|
|
7
|
+
data.tar.gz: d862d2cd98f348242f78a04d5cc2d4117cd4d3923428e181a40b778b509aaf37630abe97158a0f1a8d9351bffcd990aa8675ee5076f522575d86972c8f4bf6d2
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
0.0.5 (2016-03-04)
|
|
2
|
+
==================
|
|
3
|
+
- [Fix] Avoid `NullPointerException` if a column specified as `json_column_name` option doesn't exist in input schema.
|
|
4
|
+
- https://github.com/civitaspo/embulk-filter-expand_json/pull/6 from @muga
|
|
5
|
+
- [Fix] Migrate for Embulk v0.8.x
|
|
6
|
+
- https://github.com/civitaspo/embulk-filter-expand_json/pull/7
|
data/README.md
CHANGED
data/build.gradle
CHANGED
|
@@ -4,6 +4,7 @@ plugins {
|
|
|
4
4
|
id "com.github.kt3k.coveralls" version "2.4.0"
|
|
5
5
|
id "jacoco"
|
|
6
6
|
id "java"
|
|
7
|
+
id "checkstyle"
|
|
7
8
|
}
|
|
8
9
|
import com.github.jrubygradle.JRubyExec
|
|
9
10
|
repositories {
|
|
@@ -14,16 +15,16 @@ configurations {
|
|
|
14
15
|
provided
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
version = "0.0.
|
|
18
|
+
version = "0.0.5"
|
|
18
19
|
sourceCompatibility = 1.7
|
|
19
20
|
targetCompatibility = 1.7
|
|
20
21
|
|
|
21
22
|
dependencies {
|
|
22
|
-
compile "org.embulk:embulk-core:0.
|
|
23
|
-
provided "org.embulk:embulk-core:0.
|
|
23
|
+
compile "org.embulk:embulk-core:0.8.6"
|
|
24
|
+
provided "org.embulk:embulk-core:0.8.6"
|
|
24
25
|
compile "com.jayway.jsonpath:json-path:2.+"
|
|
25
26
|
testCompile "junit:junit:4.+"
|
|
26
|
-
testCompile "org.embulk:embulk-core:0.
|
|
27
|
+
testCompile "org.embulk:embulk-core:0.8.6:tests"
|
|
27
28
|
}
|
|
28
29
|
|
|
29
30
|
jacocoTestReport {
|
|
@@ -40,6 +41,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
40
41
|
}
|
|
41
42
|
clean { delete "classpath" }
|
|
42
43
|
|
|
44
|
+
checkstyle {
|
|
45
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
46
|
+
toolVersion = '6.14.1'
|
|
47
|
+
}
|
|
48
|
+
checkstyleMain {
|
|
49
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
50
|
+
ignoreFailures = true
|
|
51
|
+
}
|
|
52
|
+
checkstyleTest {
|
|
53
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
54
|
+
ignoreFailures = true
|
|
55
|
+
}
|
|
56
|
+
task checkstyle(type: Checkstyle) {
|
|
57
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
58
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
59
|
+
}
|
|
60
|
+
|
|
43
61
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
44
62
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
45
63
|
script "${project.name}.gemspec"
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName"/>
|
|
96
|
+
<module name="ClassTypeParameterName">
|
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
98
|
+
</module>
|
|
99
|
+
<module name="MethodTypeParameterName">
|
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
101
|
+
</module>
|
|
102
|
+
|
|
103
|
+
<module name="AvoidStarImport"/>
|
|
104
|
+
<module name="RedundantImport"/>
|
|
105
|
+
<module name="UnusedImports"/>
|
|
106
|
+
<module name="ImportOrder">
|
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
|
108
|
+
<property name="separated" value="true"/>
|
|
109
|
+
<property name="option" value="bottom"/>
|
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
111
|
+
</module>
|
|
112
|
+
|
|
113
|
+
<module name="WhitespaceAround">
|
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
117
|
+
<property name="tokens" value="
|
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
|
128
|
+
</module>
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName"/>
|
|
86
|
+
<module name="ClassTypeParameterName">
|
|
87
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
88
|
+
</module>
|
|
89
|
+
<module name="MethodTypeParameterName">
|
|
90
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
91
|
+
</module>
|
|
92
|
+
|
|
93
|
+
<module name="WhitespaceAround">
|
|
94
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
95
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
96
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
97
|
+
<property name="tokens" value="
|
|
98
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
99
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
100
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
101
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
102
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
103
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
104
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
105
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
106
|
+
</module>
|
|
107
|
+
</module>
|
|
108
|
+
</module>
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -46,6 +46,10 @@ public class ExpandJsonFilterPlugin
|
|
|
46
46
|
FilterPlugin.Control control)
|
|
47
47
|
{
|
|
48
48
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
49
|
+
|
|
50
|
+
// check if the specified json column exists or not
|
|
51
|
+
inputSchema.lookupColumn(task.getJsonColumnName());
|
|
52
|
+
|
|
49
53
|
Schema outputSchema = buildOutputSchema(task, inputSchema);
|
|
50
54
|
control.run(task.dump(), outputSchema);
|
|
51
55
|
}
|
|
@@ -65,14 +69,16 @@ public class ExpandJsonFilterPlugin
|
|
|
65
69
|
int i = 0; // columns index
|
|
66
70
|
for (Column inputColumn: inputSchema.getColumns()) {
|
|
67
71
|
if (inputColumn.getName().contentEquals(task.getJsonColumnName())) {
|
|
68
|
-
logger.info("removed column: name: {}, type: {}",
|
|
72
|
+
logger.info("removed column: name: {}, type: {}, index: {}",
|
|
69
73
|
inputColumn.getName(),
|
|
70
|
-
inputColumn.getType()
|
|
74
|
+
inputColumn.getType(),
|
|
75
|
+
inputColumn.getIndex());
|
|
71
76
|
for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
|
|
72
|
-
logger.info("added column: name: {}, type: {}, options: {}",
|
|
77
|
+
logger.info("added column: name: {}, type: {}, options: {}, index: {}",
|
|
73
78
|
expandedColumnConfig.getName(),
|
|
74
79
|
expandedColumnConfig.getType(),
|
|
75
|
-
expandedColumnConfig.getOption()
|
|
80
|
+
expandedColumnConfig.getOption(),
|
|
81
|
+
i);
|
|
76
82
|
Column outputColumn = new Column(i++,
|
|
77
83
|
expandedColumnConfig.getName(),
|
|
78
84
|
expandedColumnConfig.getType());
|
|
@@ -80,6 +86,10 @@ public class ExpandJsonFilterPlugin
|
|
|
80
86
|
}
|
|
81
87
|
}
|
|
82
88
|
else {
|
|
89
|
+
logger.info("unchanged column: name: {}, type: {}, index: {}",
|
|
90
|
+
inputColumn.getName(),
|
|
91
|
+
inputColumn.getType(),
|
|
92
|
+
i);
|
|
83
93
|
Column outputColumn = new Column(i++,
|
|
84
94
|
inputColumn.getName(),
|
|
85
95
|
inputColumn.getType());
|
|
@@ -2,12 +2,13 @@ package org.embulk.filter.expand_json;
|
|
|
2
2
|
|
|
3
3
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
4
4
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
5
|
+
import com.google.common.base.Optional;
|
|
5
6
|
import com.google.common.base.Throwables;
|
|
6
7
|
import com.google.common.collect.ImmutableList;
|
|
7
|
-
import com.google.common.collect.Maps;
|
|
8
8
|
import com.jayway.jsonpath.Configuration;
|
|
9
9
|
import com.jayway.jsonpath.JsonPath;
|
|
10
10
|
import com.jayway.jsonpath.Option;
|
|
11
|
+
import com.jayway.jsonpath.ParseContext;
|
|
11
12
|
import com.jayway.jsonpath.ReadContext;
|
|
12
13
|
import org.embulk.spi.Column;
|
|
13
14
|
import org.embulk.spi.ColumnConfig;
|
|
@@ -22,7 +23,6 @@ import org.embulk.spi.type.Types;
|
|
|
22
23
|
import org.joda.time.DateTimeZone;
|
|
23
24
|
import org.slf4j.Logger;
|
|
24
25
|
|
|
25
|
-
import java.util.HashMap;
|
|
26
26
|
import java.util.List;
|
|
27
27
|
import java.util.Map;
|
|
28
28
|
|
|
@@ -34,49 +34,167 @@ import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
|
|
|
34
34
|
public class FilteredPageOutput
|
|
35
35
|
implements PageOutput
|
|
36
36
|
{
|
|
37
|
+
private class ExpandedColumn
|
|
38
|
+
{
|
|
39
|
+
private final String key;
|
|
40
|
+
private final Column column;
|
|
41
|
+
private final String jsonPath;
|
|
42
|
+
private final Optional<TimestampParser> timestampParser;
|
|
43
|
+
|
|
44
|
+
ExpandedColumn(String key,
|
|
45
|
+
Column column,
|
|
46
|
+
String jsonPath,
|
|
47
|
+
Optional<TimestampParser> timestampParser)
|
|
48
|
+
{
|
|
49
|
+
this.key = key;
|
|
50
|
+
this.column = column;
|
|
51
|
+
this.jsonPath = jsonPath;
|
|
52
|
+
this.timestampParser = timestampParser;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
public String getKey()
|
|
56
|
+
{
|
|
57
|
+
return key;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
public Column getColumn()
|
|
61
|
+
{
|
|
62
|
+
return column;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
public String getJsonPath()
|
|
66
|
+
{
|
|
67
|
+
return jsonPath;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
public Optional<TimestampParser> getTimestampParser()
|
|
71
|
+
{
|
|
72
|
+
return timestampParser;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
private class UnchangedColumn
|
|
77
|
+
{
|
|
78
|
+
private final String key;
|
|
79
|
+
private final Column inputColumn;
|
|
80
|
+
private final Column outputColumn;
|
|
81
|
+
|
|
82
|
+
UnchangedColumn(String key, Column inputColumn, Column outputColumn)
|
|
83
|
+
{
|
|
84
|
+
this.key = key;
|
|
85
|
+
this.inputColumn = inputColumn;
|
|
86
|
+
this.outputColumn = outputColumn;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
public String getKey()
|
|
90
|
+
{
|
|
91
|
+
return key;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
public Column getInputColumn()
|
|
95
|
+
{
|
|
96
|
+
return inputColumn;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
public Column getOutputColumn()
|
|
100
|
+
{
|
|
101
|
+
return outputColumn;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
37
106
|
private final Logger logger = Exec.getLogger(FilteredPageOutput.class);
|
|
38
|
-
private final
|
|
39
|
-
private final List<
|
|
40
|
-
private final List<Column> expandedJsonColumns;
|
|
41
|
-
private final HashMap<String, TimestampParser> timestampParserHashMap;
|
|
107
|
+
private final List<UnchangedColumn> unchangedColumns;
|
|
108
|
+
private final List<ExpandedColumn> expandedColumns;
|
|
42
109
|
private final Column jsonColumn;
|
|
43
110
|
private final PageReader pageReader;
|
|
44
|
-
private final Schema inputSchema;
|
|
45
|
-
private final Schema outputSchema;
|
|
46
111
|
private final PageBuilder pageBuilder;
|
|
47
|
-
private final
|
|
112
|
+
private final ObjectMapper objectMapper = new ObjectMapper();
|
|
113
|
+
private final ParseContext parseContext;
|
|
48
114
|
|
|
49
|
-
|
|
115
|
+
private List<ExpandedColumn> initializeExpandedColumns(PluginTask task,
|
|
116
|
+
Schema outputSchema)
|
|
50
117
|
{
|
|
51
|
-
|
|
118
|
+
ImmutableList.Builder<ExpandedColumn> expandedJsonColumnsBuilder = ImmutableList.builder();
|
|
119
|
+
for (Column outputColumn : outputSchema.getColumns()) {
|
|
120
|
+
for (ColumnConfig expandedColumnConfig : task.getExpandedColumns()) {
|
|
121
|
+
if (outputColumn.getName().equals(expandedColumnConfig.getName())) {
|
|
52
122
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
123
|
+
TimestampParser timestampParser = null;
|
|
124
|
+
if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
|
|
125
|
+
String format;
|
|
126
|
+
if (expandedColumnConfig.getOption().has("format")) {
|
|
127
|
+
format = expandedColumnConfig.getOption().get(String.class, "format");
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
format = task.getDefaultTimestampFormat();
|
|
131
|
+
}
|
|
132
|
+
DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
|
|
133
|
+
timestampParser = new TimestampParser(task.getJRuby(), format, timezone);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
ExpandedColumn expandedColumn = new ExpandedColumn(outputColumn.getName(),
|
|
137
|
+
outputColumn,
|
|
138
|
+
task.getRoot() + outputColumn.getName(),
|
|
139
|
+
Optional.fromNullable(timestampParser));
|
|
140
|
+
expandedJsonColumnsBuilder.add(expandedColumn);
|
|
141
|
+
}
|
|
58
142
|
}
|
|
59
|
-
|
|
60
|
-
|
|
143
|
+
}
|
|
144
|
+
return expandedJsonColumnsBuilder.build();
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
private List<UnchangedColumn> initializeUnchangedColumns(Schema inputSchema,
|
|
148
|
+
Schema outputSchema,
|
|
149
|
+
Column excludeColumn)
|
|
150
|
+
{
|
|
151
|
+
ImmutableList.Builder<UnchangedColumn> unchangedColumnsBuilder = ImmutableList.builder();
|
|
152
|
+
for (Column outputColumn : outputSchema.getColumns()) {
|
|
153
|
+
for (Column inputColumn : inputSchema.getColumns()) {
|
|
154
|
+
if (inputColumn.getName().equals(outputColumn.getName()) &&
|
|
155
|
+
!excludeColumn.getName().equals(outputColumn.getName())) {
|
|
156
|
+
|
|
157
|
+
UnchangedColumn unchangedColumn = new UnchangedColumn(outputColumn.getName(),
|
|
158
|
+
inputColumn,
|
|
159
|
+
outputColumn);
|
|
160
|
+
unchangedColumnsBuilder.add(unchangedColumn);
|
|
161
|
+
}
|
|
61
162
|
}
|
|
62
163
|
}
|
|
63
|
-
|
|
64
|
-
|
|
164
|
+
return unchangedColumnsBuilder.build();
|
|
165
|
+
}
|
|
65
166
|
|
|
66
|
-
|
|
167
|
+
private Column initializeJsonColumn(PluginTask task, Schema inputSchema)
|
|
168
|
+
{
|
|
169
|
+
Column jsonColumn = null;
|
|
67
170
|
for (Column column: inputSchema.getColumns()) {
|
|
68
171
|
if (column.getName().contentEquals(task.getJsonColumnName())) {
|
|
69
|
-
|
|
172
|
+
jsonColumn = column;
|
|
70
173
|
}
|
|
71
174
|
}
|
|
72
|
-
|
|
175
|
+
return jsonColumn;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
private ParseContext initializeParseContext()
|
|
179
|
+
{
|
|
180
|
+
Configuration conf = Configuration.defaultConfiguration();
|
|
181
|
+
conf = conf.addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL);
|
|
182
|
+
conf = conf.addOptions(Option.SUPPRESS_EXCEPTIONS);
|
|
183
|
+
return JsonPath.using(conf);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
FilteredPageOutput(PluginTask task, Schema inputSchema, Schema outputSchema, PageOutput pageOutput)
|
|
187
|
+
{
|
|
188
|
+
this.jsonColumn = initializeJsonColumn(task, inputSchema);
|
|
189
|
+
this.unchangedColumns = initializeUnchangedColumns(inputSchema,
|
|
190
|
+
outputSchema,
|
|
191
|
+
jsonColumn);
|
|
192
|
+
this.expandedColumns = initializeExpandedColumns(task,
|
|
193
|
+
outputSchema);
|
|
73
194
|
|
|
74
|
-
this.timestampParserHashMap = buildTimestampParserHashMap(task);
|
|
75
195
|
this.pageReader = new PageReader(inputSchema);
|
|
76
|
-
this.inputSchema = inputSchema;
|
|
77
|
-
this.outputSchema = outputSchema;
|
|
78
|
-
this.pageOutput = pageOutput;
|
|
79
196
|
this.pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, pageOutput);
|
|
197
|
+
this.parseContext = initializeParseContext();
|
|
80
198
|
}
|
|
81
199
|
|
|
82
200
|
@Override
|
|
@@ -86,8 +204,8 @@ public class FilteredPageOutput
|
|
|
86
204
|
pageReader.setPage(page);
|
|
87
205
|
|
|
88
206
|
while (pageReader.nextRecord()) {
|
|
89
|
-
|
|
90
|
-
|
|
207
|
+
setExpandedJsonColumns();
|
|
208
|
+
setUnchangedColumns();
|
|
91
209
|
pageBuilder.addRecord();
|
|
92
210
|
}
|
|
93
211
|
}
|
|
@@ -101,7 +219,6 @@ public class FilteredPageOutput
|
|
|
101
219
|
public void finish()
|
|
102
220
|
{
|
|
103
221
|
pageBuilder.finish();
|
|
104
|
-
pageOutput.finish();
|
|
105
222
|
}
|
|
106
223
|
|
|
107
224
|
@Override
|
|
@@ -109,116 +226,97 @@ public class FilteredPageOutput
|
|
|
109
226
|
{
|
|
110
227
|
pageReader.close();
|
|
111
228
|
pageBuilder.close();
|
|
112
|
-
pageOutput.close();
|
|
113
229
|
}
|
|
114
230
|
|
|
115
|
-
private HashMap<String, TimestampParser> buildTimestampParserHashMap(PluginTask task)
|
|
116
|
-
{
|
|
117
|
-
final HashMap<String, TimestampParser> timestampParserHashMap = Maps.newHashMap();
|
|
118
|
-
for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
|
|
119
|
-
if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
|
|
120
|
-
String format;
|
|
121
|
-
if (expandedColumnConfig.getOption().has("format")) {
|
|
122
|
-
format = expandedColumnConfig.getOption().get(String.class, "format");
|
|
123
|
-
}
|
|
124
|
-
else {
|
|
125
|
-
format = task.getDefaultTimestampFormat();
|
|
126
|
-
}
|
|
127
|
-
DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
|
|
128
|
-
TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
|
|
129
|
-
|
|
130
|
-
String columnName = expandedColumnConfig.getName();
|
|
131
|
-
|
|
132
|
-
timestampParserHashMap.put(columnName, parser);
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
return timestampParserHashMap;
|
|
137
|
-
}
|
|
138
231
|
|
|
139
|
-
private void
|
|
140
|
-
for (
|
|
232
|
+
private void setUnchangedColumns() {
|
|
233
|
+
for (UnchangedColumn unchangedColumn : unchangedColumns) {
|
|
234
|
+
Column inputColumn = unchangedColumn.getInputColumn();
|
|
235
|
+
Column outputColumn = unchangedColumn.getOutputColumn();
|
|
236
|
+
|
|
141
237
|
if (pageReader.isNull(inputColumn)) {
|
|
142
|
-
pageBuilder.setNull(
|
|
238
|
+
pageBuilder.setNull(outputColumn);
|
|
143
239
|
continue;
|
|
144
240
|
}
|
|
145
241
|
|
|
146
|
-
if (Types.STRING.equals(
|
|
147
|
-
pageBuilder.setString(
|
|
242
|
+
if (Types.STRING.equals(outputColumn.getType())) {
|
|
243
|
+
pageBuilder.setString(outputColumn, pageReader.getString(inputColumn));
|
|
148
244
|
}
|
|
149
|
-
else if (Types.BOOLEAN.equals(
|
|
150
|
-
pageBuilder.setBoolean(
|
|
245
|
+
else if (Types.BOOLEAN.equals(outputColumn.getType())) {
|
|
246
|
+
pageBuilder.setBoolean(outputColumn, pageReader.getBoolean(inputColumn));
|
|
151
247
|
}
|
|
152
|
-
else if (Types.DOUBLE.equals(
|
|
153
|
-
pageBuilder.setDouble(
|
|
248
|
+
else if (Types.DOUBLE.equals(outputColumn.getType())) {
|
|
249
|
+
pageBuilder.setDouble(outputColumn, pageReader.getDouble(inputColumn));
|
|
154
250
|
}
|
|
155
|
-
else if (Types.LONG.equals(
|
|
156
|
-
pageBuilder.setLong(
|
|
251
|
+
else if (Types.LONG.equals(outputColumn.getType())) {
|
|
252
|
+
pageBuilder.setLong(outputColumn, pageReader.getLong(inputColumn));
|
|
157
253
|
}
|
|
158
|
-
else if (Types.TIMESTAMP.equals(
|
|
159
|
-
pageBuilder.setTimestamp(
|
|
254
|
+
else if (Types.TIMESTAMP.equals(outputColumn.getType())) {
|
|
255
|
+
pageBuilder.setTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));
|
|
160
256
|
}
|
|
161
257
|
}
|
|
162
258
|
}
|
|
163
259
|
|
|
164
|
-
private void setExpandedJsonColumns(
|
|
260
|
+
private void setExpandedJsonColumns()
|
|
165
261
|
throws JsonProcessingException
|
|
166
262
|
{
|
|
167
263
|
final ReadContext json;
|
|
168
|
-
if (pageReader.isNull(
|
|
264
|
+
if (pageReader.isNull(jsonColumn)) {
|
|
169
265
|
json = null;
|
|
170
266
|
}
|
|
171
267
|
else {
|
|
172
|
-
String jsonObject = pageReader.getString(
|
|
173
|
-
|
|
174
|
-
conf = conf.addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL);
|
|
175
|
-
conf = conf.addOptions(Option.SUPPRESS_EXCEPTIONS);
|
|
176
|
-
json = JsonPath.using(conf).parse(jsonObject);
|
|
268
|
+
String jsonObject = pageReader.getString(jsonColumn);
|
|
269
|
+
json = parseContext.parse(jsonObject);
|
|
177
270
|
}
|
|
178
271
|
|
|
179
|
-
for (
|
|
272
|
+
for (ExpandedColumn expandedJsonColumn: expandedColumns) {
|
|
180
273
|
if (json == null) {
|
|
181
|
-
pageBuilder.setNull(expandedJsonColumn);
|
|
274
|
+
pageBuilder.setNull(expandedJsonColumn.getColumn());
|
|
182
275
|
continue;
|
|
183
276
|
}
|
|
184
277
|
|
|
185
|
-
Object value = json.read(
|
|
186
|
-
final String finalValue =
|
|
278
|
+
Object value = json.read(expandedJsonColumn.getJsonPath());
|
|
279
|
+
final String finalValue = convertJsonNodeAsString(value);
|
|
187
280
|
if (finalValue == null) {
|
|
188
|
-
pageBuilder.setNull(expandedJsonColumn);
|
|
281
|
+
pageBuilder.setNull(expandedJsonColumn.getColumn());
|
|
189
282
|
continue;
|
|
190
283
|
}
|
|
191
284
|
|
|
192
|
-
if (Types.STRING.equals(expandedJsonColumn.getType())) {
|
|
193
|
-
pageBuilder.setString(expandedJsonColumn, finalValue);
|
|
285
|
+
if (Types.STRING.equals(expandedJsonColumn.getColumn().getType())) {
|
|
286
|
+
pageBuilder.setString(expandedJsonColumn.getColumn(), finalValue);
|
|
194
287
|
}
|
|
195
|
-
else if (Types.BOOLEAN.equals(expandedJsonColumn.getType())) {
|
|
196
|
-
pageBuilder.setBoolean(expandedJsonColumn, Boolean.parseBoolean(finalValue));
|
|
288
|
+
else if (Types.BOOLEAN.equals(expandedJsonColumn.getColumn().getType())) {
|
|
289
|
+
pageBuilder.setBoolean(expandedJsonColumn.getColumn(), Boolean.parseBoolean(finalValue));
|
|
197
290
|
}
|
|
198
|
-
else if (Types.DOUBLE.equals(expandedJsonColumn.getType())) {
|
|
199
|
-
pageBuilder.setDouble(expandedJsonColumn, Double.parseDouble(finalValue));
|
|
291
|
+
else if (Types.DOUBLE.equals(expandedJsonColumn.getColumn().getType())) {
|
|
292
|
+
pageBuilder.setDouble(expandedJsonColumn.getColumn(), Double.parseDouble(finalValue));
|
|
200
293
|
}
|
|
201
|
-
else if (Types.LONG.equals(expandedJsonColumn.getType())) {
|
|
202
|
-
pageBuilder.setLong(expandedJsonColumn, Long.parseLong(finalValue));
|
|
294
|
+
else if (Types.LONG.equals(expandedJsonColumn.getColumn().getType())) {
|
|
295
|
+
pageBuilder.setLong(expandedJsonColumn.getColumn(), Long.parseLong(finalValue));
|
|
203
296
|
}
|
|
204
|
-
else if (Types.TIMESTAMP.equals(expandedJsonColumn.getType())) {
|
|
205
|
-
|
|
206
|
-
|
|
297
|
+
else if (Types.TIMESTAMP.equals(expandedJsonColumn.getColumn().getType())) {
|
|
298
|
+
if (expandedJsonColumn.getTimestampParser().isPresent()) {
|
|
299
|
+
TimestampParser parser = expandedJsonColumn.getTimestampParser().get();
|
|
300
|
+
pageBuilder.setTimestamp(expandedJsonColumn.getColumn(), parser.parse(finalValue));
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
throw new RuntimeException("TimestampParser is absent for column:" + expandedJsonColumn.getKey());
|
|
304
|
+
}
|
|
207
305
|
}
|
|
208
306
|
}
|
|
209
307
|
}
|
|
210
308
|
|
|
211
|
-
private String
|
|
309
|
+
private String convertJsonNodeAsString(Object value)
|
|
212
310
|
throws JsonProcessingException
|
|
213
311
|
{
|
|
214
312
|
if (value == null) {
|
|
215
313
|
return null;
|
|
216
314
|
}
|
|
217
315
|
else if (value instanceof List) {
|
|
218
|
-
return
|
|
316
|
+
return objectMapper.writeValueAsString(value);
|
|
219
317
|
}
|
|
220
318
|
else if (value instanceof Map) {
|
|
221
|
-
return
|
|
319
|
+
return objectMapper.writeValueAsString(value);
|
|
222
320
|
}
|
|
223
321
|
else if (value instanceof String) {
|
|
224
322
|
return (String) value;
|
|
@@ -227,5 +325,4 @@ public class FilteredPageOutput
|
|
|
227
325
|
return String.valueOf(value);
|
|
228
326
|
}
|
|
229
327
|
}
|
|
230
|
-
|
|
231
328
|
}
|
|
@@ -18,6 +18,7 @@ import org.embulk.spi.PageOutput;
|
|
|
18
18
|
import org.embulk.spi.PageReader;
|
|
19
19
|
import org.embulk.spi.PageTestUtils;
|
|
20
20
|
import org.embulk.spi.Schema;
|
|
21
|
+
import org.embulk.spi.SchemaConfigException;
|
|
21
22
|
import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
|
22
23
|
import org.junit.Before;
|
|
23
24
|
import org.junit.Rule;
|
|
@@ -40,7 +41,9 @@ public class TestExpandJsonFilterPlugin
|
|
|
40
41
|
|
|
41
42
|
private final Schema schema = Schema.builder()
|
|
42
43
|
.add("_c0", STRING)
|
|
44
|
+
.add("_c1", STRING)
|
|
43
45
|
.build();
|
|
46
|
+
private final String c1Data = "_c1_data";
|
|
44
47
|
private ExpandJsonFilterPlugin expandJsonFilterPlugin;
|
|
45
48
|
|
|
46
49
|
@Before
|
|
@@ -89,6 +92,26 @@ public class TestExpandJsonFilterPlugin
|
|
|
89
92
|
config.loadConfig(PluginTask.class);
|
|
90
93
|
}
|
|
91
94
|
|
|
95
|
+
@Test
|
|
96
|
+
public void testThrowExceptionInvalidJsonColumnName()
|
|
97
|
+
{
|
|
98
|
+
String configYaml = "" +
|
|
99
|
+
"type: expand_json\n" +
|
|
100
|
+
"json_column_name: not_exist\n" +
|
|
101
|
+
"expanded_columns:\n" +
|
|
102
|
+
" - {name: _c1, type: string}";
|
|
103
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
104
|
+
|
|
105
|
+
exception.expect(SchemaConfigException.class);
|
|
106
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control() {
|
|
107
|
+
@Override
|
|
108
|
+
public void run(TaskSource taskSource, Schema schema)
|
|
109
|
+
{
|
|
110
|
+
// do nothing
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
|
|
92
115
|
@Test
|
|
93
116
|
public void testThrowExceptionAbsentExpandedColumns()
|
|
94
117
|
{
|
|
@@ -139,7 +162,8 @@ public class TestExpandJsonFilterPlugin
|
|
|
139
162
|
" - {name: _j2, type: long}\n" +
|
|
140
163
|
" - {name: _j3, type: timestamp}\n" +
|
|
141
164
|
" - {name: _j4, type: double}\n" +
|
|
142
|
-
" - {name: _j5, type: string}\n"
|
|
165
|
+
" - {name: _j5, type: string}\n" +
|
|
166
|
+
" - {name: _c0, type: string}\n";
|
|
143
167
|
|
|
144
168
|
ConfigSource config = getConfigFromYaml(configYaml);
|
|
145
169
|
|
|
@@ -148,13 +172,15 @@ public class TestExpandJsonFilterPlugin
|
|
|
148
172
|
@Override
|
|
149
173
|
public void run(TaskSource taskSource, Schema outputSchema)
|
|
150
174
|
{
|
|
151
|
-
assertEquals(
|
|
175
|
+
assertEquals(7, outputSchema.getColumnCount());
|
|
152
176
|
|
|
153
177
|
Column new_j1 = outputSchema.getColumn(0);
|
|
154
178
|
Column new_j2 = outputSchema.getColumn(1);
|
|
155
179
|
Column new_j3 = outputSchema.getColumn(2);
|
|
156
180
|
Column new_j4 = outputSchema.getColumn(3);
|
|
157
181
|
Column new_j5 = outputSchema.getColumn(4);
|
|
182
|
+
Column new_c0 = outputSchema.getColumn(5);
|
|
183
|
+
Column old_c1 = outputSchema.getColumn(6);
|
|
158
184
|
|
|
159
185
|
assertEquals("_j1", new_j1.getName());
|
|
160
186
|
assertEquals(BOOLEAN, new_j1.getType());
|
|
@@ -166,6 +192,11 @@ public class TestExpandJsonFilterPlugin
|
|
|
166
192
|
assertEquals(DOUBLE, new_j4.getType());
|
|
167
193
|
assertEquals("_j5", new_j5.getName());
|
|
168
194
|
assertEquals(STRING, new_j5.getType());
|
|
195
|
+
assertEquals("_c0", new_c0.getName());
|
|
196
|
+
assertEquals(STRING, new_c0.getType());
|
|
197
|
+
assertEquals("_c1", old_c1.getName());
|
|
198
|
+
assertEquals(STRING, old_c1.getType());
|
|
199
|
+
|
|
169
200
|
}
|
|
170
201
|
});
|
|
171
202
|
}
|
|
@@ -191,7 +222,8 @@ public class TestExpandJsonFilterPlugin
|
|
|
191
222
|
" - {name: '_j7..book[?(@.price <= $[''_j7''][''expensive''])].author', type: string}\n" +
|
|
192
223
|
" - {name: '_j7..book[?(@.isbn)]', type: string}\n" +
|
|
193
224
|
" - {name: '_j7..book[?(@.author =~ /.*REES/i)].title', type: string}\n" +
|
|
194
|
-
" - {name: '_j7.store.book[2].author', type: string}\n"
|
|
225
|
+
" - {name: '_j7.store.book[2].author', type: string}\n" +
|
|
226
|
+
" - {name: _c0, type: string}\n";
|
|
195
227
|
|
|
196
228
|
ConfigSource config = getConfigFromYaml(configYaml);
|
|
197
229
|
|
|
@@ -292,12 +324,13 @@ public class TestExpandJsonFilterPlugin
|
|
|
292
324
|
}
|
|
293
325
|
*/
|
|
294
326
|
);
|
|
327
|
+
builder.put("_c0", "v12");
|
|
295
328
|
|
|
296
329
|
String data = convertToJsonString(builder.build());
|
|
297
330
|
|
|
298
331
|
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
|
|
299
332
|
schema,
|
|
300
|
-
data)) {
|
|
333
|
+
data, c1Data)) {
|
|
301
334
|
pageOutput.add(page);
|
|
302
335
|
}
|
|
303
336
|
|
|
@@ -323,7 +356,8 @@ public class TestExpandJsonFilterPlugin
|
|
|
323
356
|
pageReader.getString(outputSchema.getColumn(7)));
|
|
324
357
|
assertEquals("[\"Nigel Rees\",\"Herman Melville\"]",
|
|
325
358
|
pageReader.getString(outputSchema.getColumn(8)));
|
|
326
|
-
assertEquals("
|
|
359
|
+
assertEquals("" +
|
|
360
|
+
"[" +
|
|
327
361
|
"{" +
|
|
328
362
|
"\"author\":\"Herman Melville\"," +
|
|
329
363
|
"\"title\":\"Moby Dick\"," +
|
|
@@ -342,6 +376,10 @@ public class TestExpandJsonFilterPlugin
|
|
|
342
376
|
pageReader.getString(outputSchema.getColumn(10)));
|
|
343
377
|
assertEquals("Herman Melville",
|
|
344
378
|
pageReader.getString(outputSchema.getColumn(11)));
|
|
379
|
+
assertEquals("v12",
|
|
380
|
+
pageReader.getString(outputSchema.getColumn(12)));
|
|
381
|
+
assertEquals(c1Data,
|
|
382
|
+
pageReader.getString(outputSchema.getColumn(13)));
|
|
345
383
|
}
|
|
346
384
|
}
|
|
347
385
|
});
|
|
@@ -373,7 +411,7 @@ public class TestExpandJsonFilterPlugin
|
|
|
373
411
|
String data = getBrokenJsonString();
|
|
374
412
|
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(),
|
|
375
413
|
schema,
|
|
376
|
-
data)) {
|
|
414
|
+
data, c1Data)) {
|
|
377
415
|
exception.expect(InvalidJsonException.class);
|
|
378
416
|
exception.expectMessage("Unexpected End Of File position 12: null");
|
|
379
417
|
pageOutput.add(page);
|
|
@@ -391,5 +429,4 @@ public class TestExpandJsonFilterPlugin
|
|
|
391
429
|
}
|
|
392
430
|
});
|
|
393
431
|
}
|
|
394
|
-
|
|
395
432
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-expand_json
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Civitaspo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2016-03-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -47,9 +47,12 @@ extra_rdoc_files: []
|
|
|
47
47
|
files:
|
|
48
48
|
- .gitignore
|
|
49
49
|
- .travis.yml
|
|
50
|
+
- CHANGELOG.md
|
|
50
51
|
- LICENSE.txt
|
|
51
52
|
- README.md
|
|
52
53
|
- build.gradle
|
|
54
|
+
- config/checkstyle/checkstyle.xml
|
|
55
|
+
- config/checkstyle/default.xml
|
|
53
56
|
- example/config.yml
|
|
54
57
|
- example/data.tsv
|
|
55
58
|
- gradle/wrapper/gradle-wrapper.jar
|
|
@@ -60,11 +63,12 @@ files:
|
|
|
60
63
|
- src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java
|
|
61
64
|
- src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java
|
|
62
65
|
- src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
|
|
63
|
-
- classpath/
|
|
64
|
-
- classpath/asm-
|
|
65
|
-
- classpath/embulk-filter-expand_json-0.0.
|
|
66
|
-
- classpath/json-path-2.
|
|
67
|
-
- classpath/json-smart-2.
|
|
66
|
+
- classpath/accessors-smart-1.1.jar
|
|
67
|
+
- classpath/asm-5.0.3.jar
|
|
68
|
+
- classpath/embulk-filter-expand_json-0.0.5.jar
|
|
69
|
+
- classpath/json-path-2.2.0.jar
|
|
70
|
+
- classpath/json-smart-2.2.1.jar
|
|
71
|
+
- classpath/slf4j-api-1.7.16.jar
|
|
68
72
|
homepage: https://github.com/civitaspo/embulk-filter-expand_json
|
|
69
73
|
licenses:
|
|
70
74
|
- MIT
|