embulk-parser-jsonpath 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.travis.yml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +107 -0
- data/build.gradle +111 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/example/conf.yml +18 -0
- data/example/dummy.rb +27 -0
- data/example/input.json +1006 -0
- data/example/input2.json +1006 -0
- data/example/seed.yml +8 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +160 -0
- data/gradlew.bat +90 -0
- data/lib/embulk/guess/jsonpath.rb +61 -0
- data/lib/embulk/parser/jsonpath.rb +3 -0
- data/src/main/java/org/embulk/parser/jsonpath/ColumnCaster.java +97 -0
- data/src/main/java/org/embulk/parser/jsonpath/ColumnVisitorImpl.java +167 -0
- data/src/main/java/org/embulk/parser/jsonpath/JsonRecordValidateException.java +22 -0
- data/src/main/java/org/embulk/parser/jsonpath/JsonpathParserPlugin.java +148 -0
- data/src/main/java/org/embulk/parser/jsonpath/cast/BooleanCast.java +39 -0
- data/src/main/java/org/embulk/parser/jsonpath/cast/DoubleCast.java +41 -0
- data/src/main/java/org/embulk/parser/jsonpath/cast/JsonCast.java +40 -0
- data/src/main/java/org/embulk/parser/jsonpath/cast/LongCast.java +47 -0
- data/src/main/java/org/embulk/parser/jsonpath/cast/StringCast.java +82 -0
- data/src/test/java/org/embulk/parser/jsonpath/TestColumnCaster.java +256 -0
- data/src/test/java/org/embulk/parser/jsonpath/TestJsonpathParserPlugin.java +292 -0
- data/src/test/java/org/embulk/parser/jsonpath/cast/TestBooleanCast.java +56 -0
- data/src/test/java/org/embulk/parser/jsonpath/cast/TestDoubleCast.java +49 -0
- data/src/test/java/org/embulk/parser/jsonpath/cast/TestJsonCast.java +79 -0
- data/src/test/java/org/embulk/parser/jsonpath/cast/TestLongCast.java +41 -0
- data/src/test/java/org/embulk/parser/jsonpath/cast/TestStringCast.java +103 -0
- metadata +113 -0
data/example/seed.yml
ADDED
Binary file
|
data/gradlew
ADDED
@@ -0,0 +1,160 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
##############################################################################
|
4
|
+
##
|
5
|
+
## Gradle start up script for UN*X
|
6
|
+
##
|
7
|
+
##############################################################################
|
8
|
+
|
9
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
10
|
+
DEFAULT_JVM_OPTS=""
|
11
|
+
|
12
|
+
APP_NAME="Gradle"
|
13
|
+
APP_BASE_NAME=`basename "$0"`
|
14
|
+
|
15
|
+
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
16
|
+
MAX_FD="maximum"
|
17
|
+
|
18
|
+
warn ( ) {
|
19
|
+
echo "$*"
|
20
|
+
}
|
21
|
+
|
22
|
+
die ( ) {
|
23
|
+
echo
|
24
|
+
echo "$*"
|
25
|
+
echo
|
26
|
+
exit 1
|
27
|
+
}
|
28
|
+
|
29
|
+
# OS specific support (must be 'true' or 'false').
|
30
|
+
cygwin=false
|
31
|
+
msys=false
|
32
|
+
darwin=false
|
33
|
+
case "`uname`" in
|
34
|
+
CYGWIN* )
|
35
|
+
cygwin=true
|
36
|
+
;;
|
37
|
+
Darwin* )
|
38
|
+
darwin=true
|
39
|
+
;;
|
40
|
+
MINGW* )
|
41
|
+
msys=true
|
42
|
+
;;
|
43
|
+
esac
|
44
|
+
|
45
|
+
# Attempt to set APP_HOME
|
46
|
+
# Resolve links: $0 may be a link
|
47
|
+
PRG="$0"
|
48
|
+
# Need this for relative symlinks.
|
49
|
+
while [ -h "$PRG" ] ; do
|
50
|
+
ls=`ls -ld "$PRG"`
|
51
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
52
|
+
if expr "$link" : '/.*' > /dev/null; then
|
53
|
+
PRG="$link"
|
54
|
+
else
|
55
|
+
PRG=`dirname "$PRG"`"/$link"
|
56
|
+
fi
|
57
|
+
done
|
58
|
+
SAVED="`pwd`"
|
59
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
60
|
+
APP_HOME="`pwd -P`"
|
61
|
+
cd "$SAVED" >/dev/null
|
62
|
+
|
63
|
+
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
64
|
+
|
65
|
+
# Determine the Java command to use to start the JVM.
|
66
|
+
if [ -n "$JAVA_HOME" ] ; then
|
67
|
+
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
68
|
+
# IBM's JDK on AIX uses strange locations for the executables
|
69
|
+
JAVACMD="$JAVA_HOME/jre/sh/java"
|
70
|
+
else
|
71
|
+
JAVACMD="$JAVA_HOME/bin/java"
|
72
|
+
fi
|
73
|
+
if [ ! -x "$JAVACMD" ] ; then
|
74
|
+
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
75
|
+
|
76
|
+
Please set the JAVA_HOME variable in your environment to match the
|
77
|
+
location of your Java installation."
|
78
|
+
fi
|
79
|
+
else
|
80
|
+
JAVACMD="java"
|
81
|
+
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
82
|
+
|
83
|
+
Please set the JAVA_HOME variable in your environment to match the
|
84
|
+
location of your Java installation."
|
85
|
+
fi
|
86
|
+
|
87
|
+
# Increase the maximum file descriptors if we can.
|
88
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
89
|
+
MAX_FD_LIMIT=`ulimit -H -n`
|
90
|
+
if [ $? -eq 0 ] ; then
|
91
|
+
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
92
|
+
MAX_FD="$MAX_FD_LIMIT"
|
93
|
+
fi
|
94
|
+
ulimit -n $MAX_FD
|
95
|
+
if [ $? -ne 0 ] ; then
|
96
|
+
warn "Could not set maximum file descriptor limit: $MAX_FD"
|
97
|
+
fi
|
98
|
+
else
|
99
|
+
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
|
100
|
+
fi
|
101
|
+
fi
|
102
|
+
|
103
|
+
# For Darwin, add options to specify how the application appears in the dock
|
104
|
+
if $darwin; then
|
105
|
+
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
|
106
|
+
fi
|
107
|
+
|
108
|
+
# For Cygwin, switch paths to Windows format before running java
|
109
|
+
if $cygwin ; then
|
110
|
+
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
111
|
+
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
112
|
+
JAVACMD=`cygpath --unix "$JAVACMD"`
|
113
|
+
|
114
|
+
# We build the pattern for arguments to be converted via cygpath
|
115
|
+
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
116
|
+
SEP=""
|
117
|
+
for dir in $ROOTDIRSRAW ; do
|
118
|
+
ROOTDIRS="$ROOTDIRS$SEP$dir"
|
119
|
+
SEP="|"
|
120
|
+
done
|
121
|
+
OURCYGPATTERN="(^($ROOTDIRS))"
|
122
|
+
# Add a user-defined pattern to the cygpath arguments
|
123
|
+
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
|
124
|
+
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
|
125
|
+
fi
|
126
|
+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
127
|
+
i=0
|
128
|
+
for arg in "$@" ; do
|
129
|
+
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
|
130
|
+
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
|
131
|
+
|
132
|
+
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
|
133
|
+
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
|
134
|
+
else
|
135
|
+
eval `echo args$i`="\"$arg\""
|
136
|
+
fi
|
137
|
+
i=$((i+1))
|
138
|
+
done
|
139
|
+
case $i in
|
140
|
+
(0) set -- ;;
|
141
|
+
(1) set -- "$args0" ;;
|
142
|
+
(2) set -- "$args0" "$args1" ;;
|
143
|
+
(3) set -- "$args0" "$args1" "$args2" ;;
|
144
|
+
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
|
145
|
+
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
|
146
|
+
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
|
147
|
+
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
|
148
|
+
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
|
149
|
+
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
|
150
|
+
esac
|
151
|
+
fi
|
152
|
+
|
153
|
+
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
|
154
|
+
function splitJvmOpts() {
|
155
|
+
JVM_OPTS=("$@")
|
156
|
+
}
|
157
|
+
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
|
158
|
+
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
|
159
|
+
|
160
|
+
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
|
data/gradlew.bat
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
12
|
+
set DEFAULT_JVM_OPTS=
|
13
|
+
|
14
|
+
set DIRNAME=%~dp0
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
16
|
+
set APP_BASE_NAME=%~n0
|
17
|
+
set APP_HOME=%DIRNAME%
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
+
|
54
|
+
:win9xME_args
|
55
|
+
@rem Slurp the command line arguments.
|
56
|
+
set CMD_LINE_ARGS=
|
57
|
+
set _SKIP=2
|
58
|
+
|
59
|
+
:win9xME_args_slurp
|
60
|
+
if "x%~1" == "x" goto execute
|
61
|
+
|
62
|
+
set CMD_LINE_ARGS=%*
|
63
|
+
goto execute
|
64
|
+
|
65
|
+
:4NT_args
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
+
set CMD_LINE_ARGS=%$
|
68
|
+
|
69
|
+
:execute
|
70
|
+
@rem Setup the command line
|
71
|
+
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
+
|
74
|
+
@rem Execute Gradle
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
+
|
77
|
+
:end
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
+
|
81
|
+
:fail
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
+
rem the _cmd.exe /c_ return code!
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
+
exit /b 1
|
86
|
+
|
87
|
+
:mainEnd
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
89
|
+
|
90
|
+
:omega
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module Embulk
|
2
|
+
module Guess
|
3
|
+
|
4
|
+
# TODO implement guess plugin to make this command work:
|
5
|
+
# $ embulk guess -g "jsonpath" partial-config.yml
|
6
|
+
#
|
7
|
+
# Depending on the file format the plugin uses, you can use choose
|
8
|
+
# one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
|
9
|
+
# or line guess (LineGuessPlugin).
|
10
|
+
|
11
|
+
#class Jsonpath < GuessPlugin
|
12
|
+
# Plugin.register_guess("jsonpath", self)
|
13
|
+
#
|
14
|
+
# def guess(config, sample_buffer)
|
15
|
+
# if sample_buffer[0,2] == GZIP_HEADER
|
16
|
+
# guessed = {}
|
17
|
+
# guessed["type"] = "jsonpath"
|
18
|
+
# guessed["property1"] = "guessed-value"
|
19
|
+
# return {"parser" => guessed}
|
20
|
+
# else
|
21
|
+
# return {}
|
22
|
+
# end
|
23
|
+
# end
|
24
|
+
#end
|
25
|
+
|
26
|
+
#class Jsonpath < TextGuessPlugin
|
27
|
+
# Plugin.register_guess("jsonpath", self)
|
28
|
+
#
|
29
|
+
# def guess_text(config, sample_text)
|
30
|
+
# js = JSON.parse(sample_text) rescue nil
|
31
|
+
# if js && js["mykeyword"] == "keyword"
|
32
|
+
# guessed = {}
|
33
|
+
# guessed["type"] = "jsonpath"
|
34
|
+
# guessed["property1"] = "guessed-value"
|
35
|
+
# return {"parser" => guessed}
|
36
|
+
# else
|
37
|
+
# return {}
|
38
|
+
# end
|
39
|
+
# end
|
40
|
+
#end
|
41
|
+
|
42
|
+
#class Jsonpath < LineGuessPlugin
|
43
|
+
# Plugin.register_guess("jsonpath", self)
|
44
|
+
#
|
45
|
+
# def guess_lines(config, sample_lines)
|
46
|
+
# all_line_matched = sample_lines.all? do |line|
|
47
|
+
# line =~ /mypattern/
|
48
|
+
# end
|
49
|
+
# if all_line_matched
|
50
|
+
# guessed = {}
|
51
|
+
# guessed["type"] = "jsonpath"
|
52
|
+
# guessed["property1"] = "guessed-value"
|
53
|
+
# return {"parser" => guessed}
|
54
|
+
# else
|
55
|
+
# return {}
|
56
|
+
# end
|
57
|
+
# end
|
58
|
+
#end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
package org.embulk.parser.jsonpath;
|
2
|
+
|
3
|
+
import org.embulk.parser.jsonpath.cast.BooleanCast;
|
4
|
+
import org.embulk.parser.jsonpath.cast.DoubleCast;
|
5
|
+
import org.embulk.parser.jsonpath.cast.JsonCast;
|
6
|
+
import org.embulk.parser.jsonpath.cast.LongCast;
|
7
|
+
import org.embulk.parser.jsonpath.cast.StringCast;
|
8
|
+
import org.embulk.spi.DataException;
|
9
|
+
import org.embulk.spi.time.Timestamp;
|
10
|
+
import org.embulk.spi.time.TimestampParser;
|
11
|
+
import org.msgpack.value.Value;
|
12
|
+
|
13
|
+
class ColumnCaster
|
14
|
+
{
|
15
|
+
private ColumnCaster() {}
|
16
|
+
|
17
|
+
public static boolean asBoolean(Value value) throws DataException
|
18
|
+
{
|
19
|
+
if (value.isBooleanValue()) {
|
20
|
+
return value.asBooleanValue().getBoolean();
|
21
|
+
}
|
22
|
+
else if (value.isIntegerValue()) {
|
23
|
+
return LongCast.asBoolean(value.asIntegerValue().asLong());
|
24
|
+
}
|
25
|
+
else if (value.isFloatValue()) {
|
26
|
+
return DoubleCast.asBoolean(value.asFloatValue().toDouble());
|
27
|
+
}
|
28
|
+
else if (value.isStringValue()) {
|
29
|
+
return StringCast.asBoolean(value.asStringValue().asString());
|
30
|
+
}
|
31
|
+
else {
|
32
|
+
return JsonCast.asBoolean(value);
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
36
|
+
public static long asLong(Value value) throws DataException
|
37
|
+
{
|
38
|
+
if (value.isBooleanValue()) {
|
39
|
+
return BooleanCast.asLong(value.asBooleanValue().getBoolean());
|
40
|
+
}
|
41
|
+
else if (value.isIntegerValue()) {
|
42
|
+
return value.asIntegerValue().asLong();
|
43
|
+
}
|
44
|
+
else if (value.isFloatValue()) {
|
45
|
+
return DoubleCast.asLong(value.asFloatValue().toDouble());
|
46
|
+
}
|
47
|
+
else if (value.isStringValue()) {
|
48
|
+
return StringCast.asLong(value.asStringValue().asString());
|
49
|
+
}
|
50
|
+
else {
|
51
|
+
return JsonCast.asLong(value);
|
52
|
+
}
|
53
|
+
}
|
54
|
+
|
55
|
+
public static double asDouble(Value value) throws DataException
|
56
|
+
{
|
57
|
+
if (value.isBooleanValue()) {
|
58
|
+
return BooleanCast.asDouble(value.asBooleanValue().getBoolean());
|
59
|
+
}
|
60
|
+
else if (value.isIntegerValue()) {
|
61
|
+
return LongCast.asDouble(value.asIntegerValue().asLong());
|
62
|
+
}
|
63
|
+
else if (value.isFloatValue()) {
|
64
|
+
return value.asFloatValue().toDouble();
|
65
|
+
}
|
66
|
+
else if (value.isStringValue()) {
|
67
|
+
return StringCast.asDouble(value.asStringValue().asString());
|
68
|
+
}
|
69
|
+
else {
|
70
|
+
return JsonCast.asDouble(value);
|
71
|
+
}
|
72
|
+
}
|
73
|
+
|
74
|
+
public static String asString(Value value) throws DataException
|
75
|
+
{
|
76
|
+
return value.toString();
|
77
|
+
}
|
78
|
+
|
79
|
+
public static Timestamp asTimestamp(Value value, TimestampParser parser) throws DataException
|
80
|
+
{
|
81
|
+
if (value.isBooleanValue()) {
|
82
|
+
return BooleanCast.asTimestamp(value.asBooleanValue().getBoolean());
|
83
|
+
}
|
84
|
+
else if (value.isIntegerValue()) {
|
85
|
+
return LongCast.asTimestamp(value.asIntegerValue().asLong());
|
86
|
+
}
|
87
|
+
else if (value.isFloatValue()) {
|
88
|
+
return DoubleCast.asTimestamp(value.asFloatValue().toDouble());
|
89
|
+
}
|
90
|
+
else if (value.isStringValue()) {
|
91
|
+
return StringCast.asTimestamp(value.asStringValue().asString(), parser);
|
92
|
+
}
|
93
|
+
else {
|
94
|
+
return JsonCast.asTimestamp(value);
|
95
|
+
}
|
96
|
+
}
|
97
|
+
}
|
@@ -0,0 +1,167 @@
|
|
1
|
+
package org.embulk.parser.jsonpath;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.embulk.parser.jsonpath.JsonpathParserPlugin.PluginTask;
|
5
|
+
import org.embulk.parser.jsonpath.JsonpathParserPlugin.TypecastColumnOption;
|
6
|
+
|
7
|
+
import org.embulk.spi.Column;
|
8
|
+
import org.embulk.spi.ColumnConfig;
|
9
|
+
import org.embulk.spi.ColumnVisitor;
|
10
|
+
import org.embulk.spi.PageBuilder;
|
11
|
+
import org.embulk.spi.Schema;
|
12
|
+
import org.embulk.spi.SchemaConfig;
|
13
|
+
import org.embulk.spi.time.Timestamp;
|
14
|
+
import org.embulk.spi.time.TimestampParser;
|
15
|
+
import org.msgpack.core.MessageTypeException;
|
16
|
+
import org.msgpack.value.Value;
|
17
|
+
|
18
|
+
public class ColumnVisitorImpl implements ColumnVisitor
|
19
|
+
{
|
20
|
+
protected final PluginTask task;
|
21
|
+
protected final Schema schema;
|
22
|
+
protected final PageBuilder pageBuilder;
|
23
|
+
protected final TimestampParser[] timestampParsers;
|
24
|
+
protected final Boolean[] autoTypecasts;
|
25
|
+
|
26
|
+
protected Value value;
|
27
|
+
|
28
|
+
public ColumnVisitorImpl(PluginTask task, Schema schema, PageBuilder pageBuilder, TimestampParser[] timestampParsers)
|
29
|
+
{
|
30
|
+
this.task = task;
|
31
|
+
this.schema = schema;
|
32
|
+
this.pageBuilder = pageBuilder;
|
33
|
+
this.timestampParsers = timestampParsers.clone();
|
34
|
+
this.autoTypecasts = new Boolean[schema.size()];
|
35
|
+
buildAutoTypecasts();
|
36
|
+
}
|
37
|
+
|
38
|
+
private void buildAutoTypecasts()
|
39
|
+
{
|
40
|
+
for (Column column : schema.getColumns()) {
|
41
|
+
this.autoTypecasts[column.getIndex()] = task.getDefaultTypecast();
|
42
|
+
}
|
43
|
+
|
44
|
+
// Optional<SchemaConfig> schemaConfig = task.getColumns();
|
45
|
+
SchemaConfig schemaConfig = task.getSchemaConfig();
|
46
|
+
|
47
|
+
// if (schemaConfig.isPresent()) {
|
48
|
+
for (ColumnConfig columnConfig : schemaConfig.getColumns()) {
|
49
|
+
TypecastColumnOption columnOption = columnConfig.getOption().loadConfig(TypecastColumnOption.class);
|
50
|
+
Boolean autoTypecast = columnOption.getTypecast().or(task.getDefaultTypecast());
|
51
|
+
Column column = schema.lookupColumn(columnConfig.getName());
|
52
|
+
this.autoTypecasts[column.getIndex()] = autoTypecast;
|
53
|
+
}
|
54
|
+
// }
|
55
|
+
}
|
56
|
+
|
57
|
+
public void setValue(Value value)
|
58
|
+
{
|
59
|
+
this.value = value;
|
60
|
+
}
|
61
|
+
|
62
|
+
@Override
|
63
|
+
public void booleanColumn(Column column)
|
64
|
+
{
|
65
|
+
if (isNil(value)) {
|
66
|
+
pageBuilder.setNull(column);
|
67
|
+
}
|
68
|
+
else {
|
69
|
+
try {
|
70
|
+
boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(value) : value.asBooleanValue().getBoolean();
|
71
|
+
pageBuilder.setBoolean(column, booleanValue);
|
72
|
+
}
|
73
|
+
catch (MessageTypeException e) {
|
74
|
+
throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Boolean", value), e);
|
75
|
+
}
|
76
|
+
}
|
77
|
+
}
|
78
|
+
|
79
|
+
@Override
|
80
|
+
public void longColumn(Column column)
|
81
|
+
{
|
82
|
+
if (isNil(value)) {
|
83
|
+
pageBuilder.setNull(column);
|
84
|
+
}
|
85
|
+
else {
|
86
|
+
try {
|
87
|
+
long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(value) : value.asIntegerValue().toLong();
|
88
|
+
pageBuilder.setLong(column, longValue);
|
89
|
+
}
|
90
|
+
catch (MessageTypeException e) {
|
91
|
+
throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Long", value), e);
|
92
|
+
}
|
93
|
+
}
|
94
|
+
}
|
95
|
+
|
96
|
+
@Override
|
97
|
+
public void doubleColumn(Column column)
|
98
|
+
{
|
99
|
+
if (isNil(value)) {
|
100
|
+
pageBuilder.setNull(column);
|
101
|
+
}
|
102
|
+
else {
|
103
|
+
try {
|
104
|
+
double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(value) : value.asFloatValue().toDouble();
|
105
|
+
pageBuilder.setDouble(column, doubleValue);
|
106
|
+
}
|
107
|
+
catch (MessageTypeException e) {
|
108
|
+
throw new JsonRecordValidateException(String.format("failed get \"%s\" as Double", value), e);
|
109
|
+
}
|
110
|
+
}
|
111
|
+
}
|
112
|
+
|
113
|
+
@Override
|
114
|
+
public void stringColumn(Column column)
|
115
|
+
{
|
116
|
+
if (isNil(value)) {
|
117
|
+
pageBuilder.setNull(column);
|
118
|
+
}
|
119
|
+
else {
|
120
|
+
try {
|
121
|
+
String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(value) : value.asStringValue().toString();
|
122
|
+
pageBuilder.setString(column, string);
|
123
|
+
}
|
124
|
+
catch (MessageTypeException e) {
|
125
|
+
throw new JsonRecordValidateException(String.format("failed to get \"%s\" as String", value), e);
|
126
|
+
}
|
127
|
+
}
|
128
|
+
}
|
129
|
+
|
130
|
+
@Override
|
131
|
+
public void timestampColumn(Column column)
|
132
|
+
{
|
133
|
+
if (isNil(value)) {
|
134
|
+
pageBuilder.setNull(column);
|
135
|
+
}
|
136
|
+
else {
|
137
|
+
try {
|
138
|
+
Timestamp timestamp = ColumnCaster.asTimestamp(value, timestampParsers[column.getIndex()]);
|
139
|
+
pageBuilder.setTimestamp(column, timestamp);
|
140
|
+
}
|
141
|
+
catch (MessageTypeException e) {
|
142
|
+
throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Timestamp", value), e);
|
143
|
+
}
|
144
|
+
}
|
145
|
+
}
|
146
|
+
|
147
|
+
@Override
|
148
|
+
public void jsonColumn(Column column)
|
149
|
+
{
|
150
|
+
if (isNil(value)) {
|
151
|
+
pageBuilder.setNull(column);
|
152
|
+
}
|
153
|
+
else {
|
154
|
+
try {
|
155
|
+
pageBuilder.setJson(column, value);
|
156
|
+
}
|
157
|
+
catch (MessageTypeException e) {
|
158
|
+
throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Json", value), e);
|
159
|
+
}
|
160
|
+
}
|
161
|
+
}
|
162
|
+
|
163
|
+
protected boolean isNil(Value v)
|
164
|
+
{
|
165
|
+
return v == null || v.isNilValue();
|
166
|
+
}
|
167
|
+
}
|