embulk-output-aster 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.DS_Store +0 -0
- data/.gitignore +12 -0
- data/LICENSE.txt +21 -0
- data/README.md +84 -0
- data/build.gradle +98 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/embulk-output-aster.iml +12 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +160 -0
- data/gradlew.bat +90 -0
- data/lib/embulk-output-jdbc-0.7.11.jar +0 -0
- data/lib/embulk/output/aster.rb +3 -0
- data/lib/noarch-aster-jdbc-driver.jar +0 -0
- data/sample/config.yml +25 -0
- data/src/main/java/org/embulk/output/aster/AsterOutputPlugin.java +95 -0
- data/src/main/java/org/embulk/output/aster/jdbc/AsterBatchInsert.java +37 -0
- data/src/main/java/org/embulk/output/aster/jdbc/AsterOutputConnection.java +315 -0
- data/src/main/java/org/embulk/output/aster/jdbc/AsterOutputConnector.java +60 -0
- data/src/test/java/org/embulk/output/aster/TestAsterOutputPlugin.java +5 -0
- metadata +23 -2
- data/classpath/embulk-output-aster-0.0.1.jar +0 -0
@@ -0,0 +1,12 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<module external.linked.project.id="embulk-output-teradata" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.1.3" type="JAVA_MODULE" version="4">
|
3
|
+
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
4
|
+
<exclude-output />
|
5
|
+
<content url="file://$MODULE_DIR$">
|
6
|
+
<excludeFolder url="file://$MODULE_DIR$/.gradle" />
|
7
|
+
<excludeFolder url="file://$MODULE_DIR$/build" />
|
8
|
+
</content>
|
9
|
+
<orderEntry type="inheritedJdk" />
|
10
|
+
<orderEntry type="sourceFolder" forTests="false" />
|
11
|
+
</component>
|
12
|
+
</module>
|
Binary file
|
data/gradlew
ADDED
@@ -0,0 +1,160 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
##############################################################################
|
4
|
+
##
|
5
|
+
## Gradle start up script for UN*X
|
6
|
+
##
|
7
|
+
##############################################################################
|
8
|
+
|
9
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
10
|
+
DEFAULT_JVM_OPTS=""
|
11
|
+
|
12
|
+
APP_NAME="Gradle"
|
13
|
+
APP_BASE_NAME=`basename "$0"`
|
14
|
+
|
15
|
+
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
16
|
+
MAX_FD="maximum"
|
17
|
+
|
18
|
+
warn ( ) {
|
19
|
+
echo "$*"
|
20
|
+
}
|
21
|
+
|
22
|
+
die ( ) {
|
23
|
+
echo
|
24
|
+
echo "$*"
|
25
|
+
echo
|
26
|
+
exit 1
|
27
|
+
}
|
28
|
+
|
29
|
+
# OS specific support (must be 'true' or 'false').
|
30
|
+
cygwin=false
|
31
|
+
msys=false
|
32
|
+
darwin=false
|
33
|
+
case "`uname`" in
|
34
|
+
CYGWIN* )
|
35
|
+
cygwin=true
|
36
|
+
;;
|
37
|
+
Darwin* )
|
38
|
+
darwin=true
|
39
|
+
;;
|
40
|
+
MINGW* )
|
41
|
+
msys=true
|
42
|
+
;;
|
43
|
+
esac
|
44
|
+
|
45
|
+
# Attempt to set APP_HOME
|
46
|
+
# Resolve links: $0 may be a link
|
47
|
+
PRG="$0"
|
48
|
+
# Need this for relative symlinks.
|
49
|
+
while [ -h "$PRG" ] ; do
|
50
|
+
ls=`ls -ld "$PRG"`
|
51
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
52
|
+
if expr "$link" : '/.*' > /dev/null; then
|
53
|
+
PRG="$link"
|
54
|
+
else
|
55
|
+
PRG=`dirname "$PRG"`"/$link"
|
56
|
+
fi
|
57
|
+
done
|
58
|
+
SAVED="`pwd`"
|
59
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
60
|
+
APP_HOME="`pwd -P`"
|
61
|
+
cd "$SAVED" >/dev/null
|
62
|
+
|
63
|
+
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
64
|
+
|
65
|
+
# Determine the Java command to use to start the JVM.
|
66
|
+
if [ -n "$JAVA_HOME" ] ; then
|
67
|
+
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
68
|
+
# IBM's JDK on AIX uses strange locations for the executables
|
69
|
+
JAVACMD="$JAVA_HOME/jre/sh/java"
|
70
|
+
else
|
71
|
+
JAVACMD="$JAVA_HOME/bin/java"
|
72
|
+
fi
|
73
|
+
if [ ! -x "$JAVACMD" ] ; then
|
74
|
+
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
75
|
+
|
76
|
+
Please set the JAVA_HOME variable in your environment to match the
|
77
|
+
location of your Java installation."
|
78
|
+
fi
|
79
|
+
else
|
80
|
+
JAVACMD="java"
|
81
|
+
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
82
|
+
|
83
|
+
Please set the JAVA_HOME variable in your environment to match the
|
84
|
+
location of your Java installation."
|
85
|
+
fi
|
86
|
+
|
87
|
+
# Increase the maximum file descriptors if we can.
|
88
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
89
|
+
MAX_FD_LIMIT=`ulimit -H -n`
|
90
|
+
if [ $? -eq 0 ] ; then
|
91
|
+
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
92
|
+
MAX_FD="$MAX_FD_LIMIT"
|
93
|
+
fi
|
94
|
+
ulimit -n $MAX_FD
|
95
|
+
if [ $? -ne 0 ] ; then
|
96
|
+
warn "Could not set maximum file descriptor limit: $MAX_FD"
|
97
|
+
fi
|
98
|
+
else
|
99
|
+
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
|
100
|
+
fi
|
101
|
+
fi
|
102
|
+
|
103
|
+
# For Darwin, add options to specify how the application appears in the dock
|
104
|
+
if $darwin; then
|
105
|
+
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
|
106
|
+
fi
|
107
|
+
|
108
|
+
# For Cygwin, switch paths to Windows format before running java
|
109
|
+
if $cygwin ; then
|
110
|
+
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
111
|
+
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
112
|
+
JAVACMD=`cygpath --unix "$JAVACMD"`
|
113
|
+
|
114
|
+
# We build the pattern for arguments to be converted via cygpath
|
115
|
+
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
116
|
+
SEP=""
|
117
|
+
for dir in $ROOTDIRSRAW ; do
|
118
|
+
ROOTDIRS="$ROOTDIRS$SEP$dir"
|
119
|
+
SEP="|"
|
120
|
+
done
|
121
|
+
OURCYGPATTERN="(^($ROOTDIRS))"
|
122
|
+
# Add a user-defined pattern to the cygpath arguments
|
123
|
+
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
|
124
|
+
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
|
125
|
+
fi
|
126
|
+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
127
|
+
i=0
|
128
|
+
for arg in "$@" ; do
|
129
|
+
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
|
130
|
+
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
|
131
|
+
|
132
|
+
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
|
133
|
+
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
|
134
|
+
else
|
135
|
+
eval `echo args$i`="\"$arg\""
|
136
|
+
fi
|
137
|
+
i=$((i+1))
|
138
|
+
done
|
139
|
+
case $i in
|
140
|
+
(0) set -- ;;
|
141
|
+
(1) set -- "$args0" ;;
|
142
|
+
(2) set -- "$args0" "$args1" ;;
|
143
|
+
(3) set -- "$args0" "$args1" "$args2" ;;
|
144
|
+
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
|
145
|
+
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
|
146
|
+
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
|
147
|
+
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
|
148
|
+
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
|
149
|
+
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
|
150
|
+
esac
|
151
|
+
fi
|
152
|
+
|
153
|
+
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
|
154
|
+
function splitJvmOpts() {
|
155
|
+
JVM_OPTS=("$@")
|
156
|
+
}
|
157
|
+
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
|
158
|
+
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
|
159
|
+
|
160
|
+
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
|
data/gradlew.bat
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
12
|
+
set DEFAULT_JVM_OPTS=
|
13
|
+
|
14
|
+
set DIRNAME=%~dp0
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
16
|
+
set APP_BASE_NAME=%~n0
|
17
|
+
set APP_HOME=%DIRNAME%
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
+
|
54
|
+
:win9xME_args
|
55
|
+
@rem Slurp the command line arguments.
|
56
|
+
set CMD_LINE_ARGS=
|
57
|
+
set _SKIP=2
|
58
|
+
|
59
|
+
:win9xME_args_slurp
|
60
|
+
if "x%~1" == "x" goto execute
|
61
|
+
|
62
|
+
set CMD_LINE_ARGS=%*
|
63
|
+
goto execute
|
64
|
+
|
65
|
+
:4NT_args
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
+
set CMD_LINE_ARGS=%$
|
68
|
+
|
69
|
+
:execute
|
70
|
+
@rem Setup the command line
|
71
|
+
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
+
|
74
|
+
@rem Execute Gradle
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
+
|
77
|
+
:end
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
+
|
81
|
+
:fail
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
+
rem the _cmd.exe /c_ return code!
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
+
exit /b 1
|
86
|
+
|
87
|
+
:mainEnd
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
89
|
+
|
90
|
+
:omega
|
Binary file
|
Binary file
|
data/sample/config.yml
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
in:
|
2
|
+
type: randomj
|
3
|
+
rows: 16
|
4
|
+
threads: 1
|
5
|
+
primary_key: myid
|
6
|
+
schema:
|
7
|
+
- {name: myid, type: long}
|
8
|
+
- {name: named, type: string}
|
9
|
+
- {name: x_flag, type: boolean}
|
10
|
+
- {name: pit_rate, type: double}
|
11
|
+
- {name: score, type: long}
|
12
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
13
|
+
- {name: purchase, type: timestamp, format: '%Y/%m/%d'}
|
14
|
+
|
15
|
+
out:
|
16
|
+
type: aster
|
17
|
+
url: jdbc:ncluster://192.168.100.100
|
18
|
+
user: beehive
|
19
|
+
password: beehive
|
20
|
+
database: beehive
|
21
|
+
schema: staging
|
22
|
+
table: t1
|
23
|
+
distribute_key: myid
|
24
|
+
mode: replace
|
25
|
+
after_load: "ANALYZE staging.t1"
|
@@ -0,0 +1,95 @@
|
|
1
|
+
package org.embulk.output.aster;
|
2
|
+
|
3
|
+
import java.util.Properties;
|
4
|
+
import java.io.IOException;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
|
7
|
+
import com.google.common.base.Optional;
|
8
|
+
import com.google.common.collect.ImmutableSet;
|
9
|
+
|
10
|
+
import org.embulk.config.Config;
|
11
|
+
import org.embulk.config.ConfigDefault;
|
12
|
+
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
13
|
+
import org.embulk.output.jdbc.BatchInsert;
|
14
|
+
import org.embulk.output.jdbc.MergeConfig;
|
15
|
+
import org.embulk.output.aster.jdbc.AsterBatchInsert;
|
16
|
+
import org.embulk.output.aster.jdbc.AsterOutputConnector;
|
17
|
+
|
18
|
+
public class AsterOutputPlugin
|
19
|
+
extends AbstractJdbcOutputPlugin
|
20
|
+
{
|
21
|
+
|
22
|
+
public interface AsterPluginTask
|
23
|
+
extends PluginTask
|
24
|
+
{
|
25
|
+
@Config("url")
|
26
|
+
public String getUrl();
|
27
|
+
|
28
|
+
@Config("user")
|
29
|
+
@ConfigDefault("beehive")
|
30
|
+
public String getUser();
|
31
|
+
|
32
|
+
@Config("password")
|
33
|
+
@ConfigDefault("beehive")
|
34
|
+
public String getPassword();
|
35
|
+
|
36
|
+
@Config("database")
|
37
|
+
@ConfigDefault("beehive")
|
38
|
+
public String getDatabase();
|
39
|
+
|
40
|
+
@Config("schema")
|
41
|
+
@ConfigDefault("null")
|
42
|
+
public Optional<String> getSchema();
|
43
|
+
|
44
|
+
@Config("table")
|
45
|
+
@ConfigDefault("null")
|
46
|
+
public String getTable();
|
47
|
+
|
48
|
+
@Config("distribute_key")
|
49
|
+
@ConfigDefault("null")
|
50
|
+
public Optional<String> getDistributeKey();
|
51
|
+
|
52
|
+
@Config("max_table_name_length")
|
53
|
+
@ConfigDefault("30")
|
54
|
+
public int getMaxTableNameLength();
|
55
|
+
}
|
56
|
+
|
57
|
+
@Override
|
58
|
+
protected Class<? extends PluginTask> getTaskClass()
|
59
|
+
{
|
60
|
+
return AsterPluginTask.class;
|
61
|
+
}
|
62
|
+
|
63
|
+
@Override
|
64
|
+
protected Features getFeatures(PluginTask task)
|
65
|
+
{
|
66
|
+
AsterPluginTask t = (AsterPluginTask) task;
|
67
|
+
return new Features()
|
68
|
+
.setMaxTableNameLength(t.getMaxTableNameLength())
|
69
|
+
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE));
|
70
|
+
}
|
71
|
+
|
72
|
+
@Override
|
73
|
+
protected AsterOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
74
|
+
{
|
75
|
+
AsterPluginTask t = (AsterPluginTask) task;
|
76
|
+
|
77
|
+
Properties props = new Properties();
|
78
|
+
|
79
|
+
props.putAll(t.getOptions());
|
80
|
+
props.setProperty("database", t.getDatabase());
|
81
|
+
props.setProperty("user", t.getUser());
|
82
|
+
props.setProperty("password", t.getPassword());
|
83
|
+
|
84
|
+
logger.info("Connecting to {} options {}", t.getUrl(), props);
|
85
|
+
props.setProperty("password", t.getPassword());
|
86
|
+
|
87
|
+
return new AsterOutputConnector(t.getUrl(), t.getSchema(), t.getDistributeKey(), props);
|
88
|
+
}
|
89
|
+
|
90
|
+
@Override
|
91
|
+
protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
|
92
|
+
{
|
93
|
+
return new AsterBatchInsert(getConnector(task, true), mergeConfig);
|
94
|
+
}
|
95
|
+
}
|
@@ -0,0 +1,37 @@
|
|
1
|
+
package org.embulk.output.aster.jdbc;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.sql.Types;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import com.google.common.base.Optional;
|
7
|
+
import org.embulk.output.jdbc.MergeConfig;
|
8
|
+
import org.embulk.output.jdbc.StandardBatchInsert;
|
9
|
+
|
10
|
+
public class AsterBatchInsert
|
11
|
+
extends StandardBatchInsert
|
12
|
+
{
|
13
|
+
public AsterBatchInsert(AsterOutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
|
14
|
+
{
|
15
|
+
super(connector, mergeConfig);
|
16
|
+
}
|
17
|
+
|
18
|
+
@Override
|
19
|
+
public void setFloat(float v) throws IOException, SQLException
|
20
|
+
{
|
21
|
+
if (Float.isNaN(v) || Float.isInfinite(v)) {
|
22
|
+
setNull(Types.REAL); // TODO get through argument
|
23
|
+
} else {
|
24
|
+
super.setFloat(v);
|
25
|
+
}
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
public void setDouble(double v) throws IOException, SQLException
|
30
|
+
{
|
31
|
+
if (Double.isNaN(v) || Double.isInfinite(v)) {
|
32
|
+
setNull(Types.DOUBLE); // TODO get through argument
|
33
|
+
} else {
|
34
|
+
super.setDouble(v);
|
35
|
+
}
|
36
|
+
}
|
37
|
+
}
|
@@ -0,0 +1,315 @@
|
|
1
|
+
package org.embulk.output.aster.jdbc;
|
2
|
+
|
3
|
+
import java.nio.charset.Charset;
|
4
|
+
import java.nio.charset.StandardCharsets;
|
5
|
+
import java.sql.Connection;
|
6
|
+
import java.sql.DatabaseMetaData;
|
7
|
+
import java.sql.SQLException;
|
8
|
+
import java.sql.Statement;
|
9
|
+
|
10
|
+
import com.google.common.base.Optional;
|
11
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
12
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
13
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
14
|
+
import org.embulk.output.jdbc.TableIdentifier;
|
15
|
+
import org.slf4j.Logger;
|
16
|
+
import org.embulk.spi.Exec;
|
17
|
+
|
18
|
+
import static java.lang.String.format;
|
19
|
+
|
20
|
+
public class AsterOutputConnection
|
21
|
+
extends JdbcOutputConnection
|
22
|
+
{
|
23
|
+
private final Logger logger = Exec.getLogger(AsterOutputConnection.class);
|
24
|
+
protected final Connection connection;
|
25
|
+
protected final DatabaseMetaData databaseMetaData;
|
26
|
+
protected String identifierQuoteString;
|
27
|
+
protected Optional<String> schema;
|
28
|
+
protected Optional<String> distributeKey;
|
29
|
+
|
30
|
+
public AsterOutputConnection(Connection connection, Optional<String> schema, Optional<String> distributeKey)
|
31
|
+
throws SQLException
|
32
|
+
{
|
33
|
+
super(connection, null);
|
34
|
+
this.connection = connection;
|
35
|
+
this.databaseMetaData = connection.getMetaData();
|
36
|
+
this.schema = schema;
|
37
|
+
this.distributeKey = distributeKey;
|
38
|
+
this.identifierQuoteString = databaseMetaData.getIdentifierQuoteString();
|
39
|
+
if (schema.isPresent()) {
|
40
|
+
setSearchPath(schema.get());
|
41
|
+
}
|
42
|
+
}
|
43
|
+
|
44
|
+
@Override
|
45
|
+
public void close() throws SQLException
|
46
|
+
{
|
47
|
+
if (!connection.isClosed()) {
|
48
|
+
executeQuery("END;");
|
49
|
+
|
50
|
+
connection.close();
|
51
|
+
}
|
52
|
+
}
|
53
|
+
|
54
|
+
public String getSchemaName()
|
55
|
+
{
|
56
|
+
return schemaName;
|
57
|
+
}
|
58
|
+
|
59
|
+
public DatabaseMetaData getMetaData() throws SQLException
|
60
|
+
{
|
61
|
+
return databaseMetaData;
|
62
|
+
}
|
63
|
+
|
64
|
+
public Charset getTableNameCharset() throws SQLException
|
65
|
+
{
|
66
|
+
return StandardCharsets.UTF_8;
|
67
|
+
}
|
68
|
+
|
69
|
+
protected void setSearchPath(String schema) throws SQLException
|
70
|
+
{
|
71
|
+
String sql = "SET search_path TO " + quoteIdentifierString(schema);
|
72
|
+
executeQuery(sql);
|
73
|
+
}
|
74
|
+
|
75
|
+
@Override
|
76
|
+
protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema) {
|
77
|
+
StringBuilder sb = new StringBuilder();
|
78
|
+
sb.append("CREATE ANALYTIC TABLE IF NOT EXISTS ");
|
79
|
+
this.quoteTableIdentifier(sb, table);
|
80
|
+
sb.append(this.buildCreateTableSchemaSql(schema));
|
81
|
+
return sb.toString();
|
82
|
+
}
|
83
|
+
|
84
|
+
@Override
|
85
|
+
protected void dropTableIfExists(Statement stmt, TableIdentifier table) throws SQLException
|
86
|
+
{
|
87
|
+
String sql = format("DROP TABLE IF EXISTS %s", this.quoteTableIdentifier(table));
|
88
|
+
this.executeUpdate(stmt, sql);
|
89
|
+
}
|
90
|
+
|
91
|
+
@Override
|
92
|
+
protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema) {
|
93
|
+
StringBuilder sb = new StringBuilder();
|
94
|
+
sb.append("CREATE ANALYTIC TABLE ");
|
95
|
+
this.quoteTableIdentifier(sb, table);
|
96
|
+
sb.append(this.buildCreateTableSchemaSql(schema));
|
97
|
+
return sb.toString();
|
98
|
+
}
|
99
|
+
|
100
|
+
|
101
|
+
protected String buildCreateTableSchemaSql(JdbcSchema schema)
|
102
|
+
{
|
103
|
+
StringBuilder sb = new StringBuilder();
|
104
|
+
|
105
|
+
sb.append(" (");
|
106
|
+
for (int i=0; i < schema.getCount(); i++) {
|
107
|
+
if (i != 0) { sb.append(", "); }
|
108
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
109
|
+
sb.append(" ");
|
110
|
+
String typeName = getCreateTableTypeName(schema.getColumn(i));
|
111
|
+
sb.append(typeName);
|
112
|
+
}
|
113
|
+
sb.append(")");
|
114
|
+
|
115
|
+
if (distributeKey.isPresent()) {
|
116
|
+
sb.append(format("DISTRIBUTE BY HASH(%s)", distributeKey.get()));
|
117
|
+
} else {
|
118
|
+
sb.append("DISTRIBUTE BY REPLICATION");
|
119
|
+
}
|
120
|
+
|
121
|
+
return sb.toString();
|
122
|
+
}
|
123
|
+
|
124
|
+
protected String getCreateTableTypeName(JdbcColumn c)
|
125
|
+
{
|
126
|
+
if (c.getDeclaredType().isPresent()) {
|
127
|
+
return c.getDeclaredType().get();
|
128
|
+
} else {
|
129
|
+
return buildColumnTypeName(c);
|
130
|
+
}
|
131
|
+
}
|
132
|
+
|
133
|
+
protected String buildColumnTypeName(JdbcColumn c)
|
134
|
+
{
|
135
|
+
String simpleTypeName = c.getSimpleTypeName();
|
136
|
+
switch (getColumnDeclareType(simpleTypeName, c)) {
|
137
|
+
case SIZE:
|
138
|
+
return format("%s(%d)", simpleTypeName, c.getSizeTypeParameter());
|
139
|
+
case SIZE_AND_SCALE:
|
140
|
+
if (c.getScaleTypeParameter() < 0) {
|
141
|
+
return format("%s(%d,0)", simpleTypeName, c.getSizeTypeParameter());
|
142
|
+
} else {
|
143
|
+
return format("%s(%d,%d)", simpleTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
|
144
|
+
}
|
145
|
+
case SIZE_AND_OPTIONAL_SCALE:
|
146
|
+
if (c.getScaleTypeParameter() < 0) {
|
147
|
+
return format("%s(%d)", simpleTypeName, c.getSizeTypeParameter());
|
148
|
+
} else {
|
149
|
+
return format("%s(%d,%d)", simpleTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
|
150
|
+
}
|
151
|
+
default: // SIMPLE
|
152
|
+
if (simpleTypeName.equals("CLOB"))
|
153
|
+
{
|
154
|
+
return "CHARACTER VARYING";
|
155
|
+
}
|
156
|
+
return simpleTypeName;
|
157
|
+
}
|
158
|
+
}
|
159
|
+
|
160
|
+
protected void executeSql(String sql) throws SQLException
|
161
|
+
{
|
162
|
+
Statement stmt = connection.createStatement();
|
163
|
+
try {
|
164
|
+
executeUpdate(stmt, sql);
|
165
|
+
commitIfNecessary(connection);
|
166
|
+
} catch (SQLException ex) {
|
167
|
+
throw safeRollback(connection, ex);
|
168
|
+
} finally {
|
169
|
+
stmt.close();
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
@Override
|
174
|
+
protected String buildRenameTableSql(TableIdentifier fromTable, TableIdentifier toTable) {
|
175
|
+
StringBuilder sb = new StringBuilder();
|
176
|
+
sb.append("ALTER TABLE ");
|
177
|
+
this.quoteTableIdentifier(sb, fromTable);
|
178
|
+
sb.append(" RENAME TO ");
|
179
|
+
this.quoteTableIdentifier(sb, toTable);
|
180
|
+
return sb.toString();
|
181
|
+
}
|
182
|
+
|
183
|
+
|
184
|
+
protected void quoteIdentifierString(StringBuilder sb, String str)
|
185
|
+
{
|
186
|
+
sb.append(quoteIdentifierString(str, identifierQuoteString));
|
187
|
+
}
|
188
|
+
|
189
|
+
protected String quoteIdentifierString(String str)
|
190
|
+
{
|
191
|
+
return quoteIdentifierString(str, identifierQuoteString);
|
192
|
+
}
|
193
|
+
|
194
|
+
protected String quoteIdentifierString(String str, String quoteString)
|
195
|
+
{
|
196
|
+
// TODO if identifierQuoteString.equals(" ") && str.contains([^a-zA-Z0-9_connection.getMetaData().getExtraNameCharacters()])
|
197
|
+
// TODO if str.contains(identifierQuoteString);
|
198
|
+
return quoteString + str + quoteString;
|
199
|
+
}
|
200
|
+
|
201
|
+
public boolean isValidConnection(int timeout) throws SQLException
|
202
|
+
{
|
203
|
+
Statement stmt = connection.createStatement();
|
204
|
+
try {
|
205
|
+
stmt.executeQuery("SELECT 1").close();
|
206
|
+
return true;
|
207
|
+
} catch (SQLException ex) {
|
208
|
+
return false;
|
209
|
+
} finally {
|
210
|
+
stmt.close();
|
211
|
+
}
|
212
|
+
}
|
213
|
+
|
214
|
+
protected String[] getDeterministicSqlStates()
|
215
|
+
{
|
216
|
+
return new String[0];
|
217
|
+
}
|
218
|
+
|
219
|
+
protected int[] getDeterministicErrorCodes()
|
220
|
+
{
|
221
|
+
return new int[0];
|
222
|
+
}
|
223
|
+
|
224
|
+
protected Class[] getDeterministicRootCauses()
|
225
|
+
{
|
226
|
+
return new Class[] {
|
227
|
+
// Don't retry on UnknownHostException.
|
228
|
+
java.net.UnknownHostException.class,
|
229
|
+
|
230
|
+
//// we should not retry on connect() error?
|
231
|
+
//java.net.ConnectException.class,
|
232
|
+
};
|
233
|
+
}
|
234
|
+
|
235
|
+
public boolean isRetryableException(SQLException exception)
|
236
|
+
{
|
237
|
+
String sqlState = exception.getSQLState();
|
238
|
+
for (String deterministic : getDeterministicSqlStates()) {
|
239
|
+
if (sqlState.equals(deterministic)) {
|
240
|
+
return false;
|
241
|
+
}
|
242
|
+
}
|
243
|
+
|
244
|
+
int errorCode = exception.getErrorCode();
|
245
|
+
for (int deterministic : getDeterministicErrorCodes()) {
|
246
|
+
if (errorCode == deterministic) {
|
247
|
+
return false;
|
248
|
+
}
|
249
|
+
}
|
250
|
+
|
251
|
+
Throwable rootCause = getRootCause(exception);
|
252
|
+
for (Class deterministic : getDeterministicRootCauses()) {
|
253
|
+
if (deterministic.equals(rootCause.getClass())) {
|
254
|
+
return false;
|
255
|
+
}
|
256
|
+
}
|
257
|
+
|
258
|
+
return true;
|
259
|
+
}
|
260
|
+
|
261
|
+
private Throwable getRootCause(Throwable e) {
|
262
|
+
while (e.getCause() != null) {
|
263
|
+
e = e.getCause();
|
264
|
+
}
|
265
|
+
return e;
|
266
|
+
}
|
267
|
+
|
268
|
+
protected void executeQuery(String sql) throws SQLException
|
269
|
+
{
|
270
|
+
Statement stmt = connection.createStatement();
|
271
|
+
try {
|
272
|
+
executeUpdate(stmt, sql);
|
273
|
+
commitIfNecessary(connection);
|
274
|
+
} finally {
|
275
|
+
stmt.close();
|
276
|
+
}
|
277
|
+
}
|
278
|
+
|
279
|
+
protected int executeUpdate(Statement stmt, String sql) throws SQLException
|
280
|
+
{
|
281
|
+
logger.info("SQL: " + sql);
|
282
|
+
long startTime = System.currentTimeMillis();
|
283
|
+
int count = stmt.executeUpdate(sql);
|
284
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
285
|
+
if (count == 0) {
|
286
|
+
logger.info(format("> %.2f seconds", seconds));
|
287
|
+
} else {
|
288
|
+
logger.info(format("> %.2f seconds (%,d rows)", seconds, count));
|
289
|
+
}
|
290
|
+
return count;
|
291
|
+
}
|
292
|
+
|
293
|
+
protected void commitIfNecessary(Connection con) throws SQLException
|
294
|
+
{
|
295
|
+
if (!con.getAutoCommit()) {
|
296
|
+
con.commit();
|
297
|
+
}
|
298
|
+
}
|
299
|
+
|
300
|
+
protected SQLException safeRollback(Connection con, SQLException cause)
|
301
|
+
{
|
302
|
+
try {
|
303
|
+
if (!con.getAutoCommit()) {
|
304
|
+
con.rollback();
|
305
|
+
}
|
306
|
+
return cause;
|
307
|
+
} catch (SQLException ex) {
|
308
|
+
if (cause != null) {
|
309
|
+
cause.addSuppressed(ex);
|
310
|
+
return cause;
|
311
|
+
}
|
312
|
+
return ex;
|
313
|
+
}
|
314
|
+
}
|
315
|
+
}
|