embulk-output-postgres-udf 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +17 -0
- data/build.gradle +11 -5
- data/ci/config.yml +37 -0
- data/ci/csv/sample_01.csv.gz +0 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew.bat +90 -90
- data/src/main/java/org/embulk/output/PostgresUDFConnector.java +48 -19
- data/src/main/java/org/embulk/output/PostgresUDFOutputPlugin.java +3 -3
- metadata +16 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 37c4908db6315bd68cc4dd25ba11342740de1a23
|
4
|
+
data.tar.gz: ce34918cfbcb005d90100fac17bc71852b0910df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cfc2a0da351d5d809717d278bfcd47412caf4659ac647e0a25c982795a2ccf2bce8c238c5bc4e743918d4f0be31e0815a0944687499a1372517c141ef0ed9562
|
7
|
+
data.tar.gz: 63b18e4c1848b01a3bc60c6da9095cc669d94d997d97a2232fd239f60c212cafe82a9061ef2b55b21b9f118192b0335538f87c74dad2aa3d5dfdb6f2096c8856
|
data/.travis.yml
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
language: java
|
2
|
+
addons:
|
3
|
+
postgresql: 9.4
|
4
|
+
jdk:
|
5
|
+
- oraclejdk7
|
6
|
+
- oraclejdk8
|
7
|
+
env:
|
8
|
+
- version=0.7.4
|
9
|
+
before_install:
|
10
|
+
- curl --create-dirs -o .embulk/bin/embulk -L "https://bintray.com/artifact/download/embulk/maven/embulk-${version}.jar"
|
11
|
+
- chmod +x .embulk/bin/embulk
|
12
|
+
- export PATH=".embulk/bin:$PATH"
|
13
|
+
before_script:
|
14
|
+
- ./gradlew clean gem
|
15
|
+
- psql -c 'create database travis_ci_test;' -U postgres
|
16
|
+
script:
|
17
|
+
- ls ci/*.yml | xargs -I% embulk run -I lib %
|
data/build.gradle
CHANGED
@@ -12,15 +12,21 @@ configurations {
|
|
12
12
|
provided
|
13
13
|
}
|
14
14
|
|
15
|
-
|
16
|
-
|
15
|
+
version = "0.2.0"
|
16
|
+
|
17
|
+
ext {
|
18
|
+
javaVersion = '1.7'
|
17
19
|
}
|
18
20
|
|
19
|
-
|
21
|
+
compileJava {
|
22
|
+
sourceCompatibility = javaVersion
|
23
|
+
targetCompatibility = javaVersion
|
24
|
+
options.compilerArgs += ['-source', javaVersion, '-target', javaVersion]
|
25
|
+
}
|
20
26
|
|
21
27
|
dependencies {
|
22
|
-
compile "org.embulk:embulk-core:0.
|
23
|
-
provided "org.embulk:embulk-core:0.
|
28
|
+
compile "org.embulk:embulk-core:0.7.4"
|
29
|
+
provided "org.embulk:embulk-core:0.7.4"
|
24
30
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
25
31
|
compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
|
26
32
|
testCompile "junit:junit:4.+"
|
data/ci/config.yml
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
in:
|
2
|
+
type: file
|
3
|
+
path_prefix: ./ci/csv/sample_
|
4
|
+
parser:
|
5
|
+
charset: UTF-8
|
6
|
+
newline: CRLF
|
7
|
+
type: csv
|
8
|
+
delimiter: ','
|
9
|
+
quote: '"'
|
10
|
+
escape: ''
|
11
|
+
skip_header_lines: 1
|
12
|
+
columns:
|
13
|
+
- {name: id, type: long}
|
14
|
+
- {name: account, type: long}
|
15
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
16
|
+
- {name: purchase, type: timestamp, format: '%Y%m%d'}
|
17
|
+
- {name: comment, type: string}
|
18
|
+
out:
|
19
|
+
type: postgres_udf
|
20
|
+
host: localhost
|
21
|
+
user: postgres
|
22
|
+
database: travis_ci_test
|
23
|
+
function: |
|
24
|
+
begin
|
25
|
+
begin
|
26
|
+
create table if not exists sample (
|
27
|
+
id integer,
|
28
|
+
account integer,
|
29
|
+
time timestamp,
|
30
|
+
purchase timestamp,
|
31
|
+
comment text
|
32
|
+
);
|
33
|
+
exception
|
34
|
+
when unique_violation then -- do nothing
|
35
|
+
end;
|
36
|
+
insert into sample values(id, account, time, purchase, comment);
|
37
|
+
end;
|
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Tue Aug 11 00:26:20 PDT 2015
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
|
data/gradlew.bat
CHANGED
@@ -1,90 +1,90 @@
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
2
|
-
@rem ##########################################################################
|
3
|
-
@rem
|
4
|
-
@rem Gradle startup script for Windows
|
5
|
-
@rem
|
6
|
-
@rem ##########################################################################
|
7
|
-
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
10
|
-
|
11
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
12
|
-
set DEFAULT_JVM_OPTS=
|
13
|
-
|
14
|
-
set DIRNAME=%~dp0
|
15
|
-
if "%DIRNAME%" == "" set DIRNAME=.
|
16
|
-
set APP_BASE_NAME=%~n0
|
17
|
-
set APP_HOME=%DIRNAME%
|
18
|
-
|
19
|
-
@rem Find java.exe
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
-
|
22
|
-
set JAVA_EXE=java.exe
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
25
|
-
|
26
|
-
echo.
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
-
echo.
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
-
echo location of your Java installation.
|
31
|
-
|
32
|
-
goto fail
|
33
|
-
|
34
|
-
:findJavaFromJavaHome
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
-
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
39
|
-
|
40
|
-
echo.
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
-
echo.
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
-
echo location of your Java installation.
|
45
|
-
|
46
|
-
goto fail
|
47
|
-
|
48
|
-
:init
|
49
|
-
@rem Get command-line arguments, handling Windowz variants
|
50
|
-
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
-
|
54
|
-
:win9xME_args
|
55
|
-
@rem Slurp the command line arguments.
|
56
|
-
set CMD_LINE_ARGS=
|
57
|
-
set _SKIP=2
|
58
|
-
|
59
|
-
:win9xME_args_slurp
|
60
|
-
if "x%~1" == "x" goto execute
|
61
|
-
|
62
|
-
set CMD_LINE_ARGS=%*
|
63
|
-
goto execute
|
64
|
-
|
65
|
-
:4NT_args
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
-
set CMD_LINE_ARGS=%$
|
68
|
-
|
69
|
-
:execute
|
70
|
-
@rem Setup the command line
|
71
|
-
|
72
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
-
|
74
|
-
@rem Execute Gradle
|
75
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
-
|
77
|
-
:end
|
78
|
-
@rem End local scope for the variables with windows NT shell
|
79
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
-
|
81
|
-
:fail
|
82
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
-
rem the _cmd.exe /c_ return code!
|
84
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
-
exit /b 1
|
86
|
-
|
87
|
-
:mainEnd
|
88
|
-
if "%OS%"=="Windows_NT" endlocal
|
89
|
-
|
90
|
-
:omega
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
12
|
+
set DEFAULT_JVM_OPTS=
|
13
|
+
|
14
|
+
set DIRNAME=%~dp0
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
16
|
+
set APP_BASE_NAME=%~n0
|
17
|
+
set APP_HOME=%DIRNAME%
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
+
|
54
|
+
:win9xME_args
|
55
|
+
@rem Slurp the command line arguments.
|
56
|
+
set CMD_LINE_ARGS=
|
57
|
+
set _SKIP=2
|
58
|
+
|
59
|
+
:win9xME_args_slurp
|
60
|
+
if "x%~1" == "x" goto execute
|
61
|
+
|
62
|
+
set CMD_LINE_ARGS=%*
|
63
|
+
goto execute
|
64
|
+
|
65
|
+
:4NT_args
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
+
set CMD_LINE_ARGS=%$
|
68
|
+
|
69
|
+
:execute
|
70
|
+
@rem Setup the command line
|
71
|
+
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
+
|
74
|
+
@rem Execute Gradle
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
+
|
77
|
+
:end
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
+
|
81
|
+
:fail
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
+
rem the _cmd.exe /c_ return code!
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
+
exit /b 1
|
86
|
+
|
87
|
+
:mainEnd
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
89
|
+
|
90
|
+
:omega
|
@@ -1,8 +1,13 @@
|
|
1
1
|
package org.embulk.output;
|
2
2
|
|
3
3
|
import com.google.common.base.Throwables;
|
4
|
+
import org.embulk.spi.Exec;
|
5
|
+
import org.embulk.spi.util.RetryExecutor;
|
6
|
+
import org.slf4j.Logger;
|
4
7
|
|
5
|
-
import java.sql
|
8
|
+
import java.sql.Connection;
|
9
|
+
import java.sql.Driver;
|
10
|
+
import java.sql.SQLException;
|
6
11
|
import java.util.Properties;
|
7
12
|
|
8
13
|
/**
|
@@ -10,6 +15,8 @@ import java.util.Properties;
|
|
10
15
|
*/
|
11
16
|
public class PostgresUDFConnector {
|
12
17
|
private static final Driver driver = new org.postgresql.Driver();
|
18
|
+
private final Logger log = Exec.getLogger(PostgresUDFOutputPlugin.class);
|
19
|
+
|
13
20
|
|
14
21
|
private final String url;
|
15
22
|
private final Properties properties;
|
@@ -35,25 +42,47 @@ public class PostgresUDFConnector {
|
|
35
42
|
}
|
36
43
|
|
37
44
|
private Connection createConnection() throws SQLException {
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
45
|
+
try {
|
46
|
+
return RetryExecutor.retryExecutor()
|
47
|
+
.withRetryLimit(12)
|
48
|
+
.withInitialRetryWait(1000)
|
49
|
+
.withMaxRetryWait(30 * 60 * 1000)
|
50
|
+
.runInterruptible(new RetryExecutor.Retryable<Connection>() {
|
51
|
+
@Override
|
52
|
+
public Connection call() throws Exception {
|
53
|
+
return driver.connect(url, properties);
|
54
|
+
}
|
42
55
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
+
@Override
|
57
|
+
public boolean isRetryableException(Exception exception) {
|
58
|
+
/*
|
59
|
+
if (exception instanceof SQLRecoverableException) return true;
|
60
|
+
if (exception instanceof SQLTimeoutException) return true;
|
61
|
+
if (exception instanceof ConnectException) return true;
|
62
|
+
*/
|
63
|
+
return true;
|
64
|
+
}
|
65
|
+
|
66
|
+
@Override
|
67
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException {
|
68
|
+
String message = String.format("connection failed. Retrying %d/%d after %d seconds. Message: %s",
|
69
|
+
retryCount, retryLimit, retryWait/1000, exception.getMessage());
|
70
|
+
if (retryCount % 3 == 0) {
|
71
|
+
log.warn(message, exception);
|
72
|
+
} else {
|
73
|
+
log.warn(message);
|
74
|
+
}
|
75
|
+
}
|
76
|
+
|
77
|
+
@Override
|
78
|
+
public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException {
|
79
|
+
}
|
80
|
+
});
|
81
|
+
} catch (InterruptedException e) {
|
82
|
+
throw new RuntimeException(e);
|
83
|
+
} catch (RetryExecutor.RetryGiveupException e) {
|
84
|
+
Throwables.propagateIfInstanceOf(e.getCause(), SQLException.class);
|
85
|
+
throw Throwables.propagate(e.getCause());
|
56
86
|
}
|
57
|
-
throw firstException;
|
58
87
|
}
|
59
88
|
}
|
@@ -76,7 +76,7 @@ public class PostgresUDFOutputPlugin implements OutputPlugin {
|
|
76
76
|
}
|
77
77
|
|
78
78
|
@Override
|
79
|
-
public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<
|
79
|
+
public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports) {
|
80
80
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
81
81
|
try {
|
82
82
|
ConnectionWrapper con = getConnector(task).connect(true);
|
@@ -197,14 +197,14 @@ public class PostgresUDFOutputPlugin implements OutputPlugin {
|
|
197
197
|
}
|
198
198
|
|
199
199
|
@Override
|
200
|
-
public
|
200
|
+
public TaskReport commit() {
|
201
201
|
try {
|
202
202
|
connection.commit();
|
203
203
|
}
|
204
204
|
catch (Exception e) {
|
205
205
|
throw Throwables.propagate(e);
|
206
206
|
}
|
207
|
-
return Exec.
|
207
|
+
return Exec.newTaskReport();
|
208
208
|
}
|
209
209
|
|
210
210
|
}
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-postgres-udf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- kakusuke
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.0'
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
16
|
- - ~>
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: '1.0'
|
19
|
+
name: bundler
|
25
20
|
prerelease: false
|
26
21
|
type: :development
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
|
-
- -
|
24
|
+
- - ~>
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
30
|
- - '>='
|
37
31
|
- !ruby/object:Gem::Version
|
38
32
|
version: '10.0'
|
33
|
+
name: rake
|
39
34
|
prerelease: false
|
40
35
|
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
41
|
description: Dumps records to PostgreSQL via user-defined function.
|
42
42
|
email:
|
43
43
|
- bananafishmonger+github@gmail.com
|
@@ -46,9 +46,12 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- LICENSE.txt
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
53
|
+
- ci/config.yml
|
54
|
+
- ci/csv/sample_01.csv.gz
|
52
55
|
- gradle/wrapper/gradle-wrapper.jar
|
53
56
|
- gradle/wrapper/gradle-wrapper.properties
|
54
57
|
- gradlew
|
@@ -58,7 +61,7 @@ files:
|
|
58
61
|
- src/main/java/org/embulk/output/PostgresUDFConnector.java
|
59
62
|
- src/main/java/org/embulk/output/PostgresUDFOutputPlugin.java
|
60
63
|
- src/test/java/org/embulk/output/TestPostgresUDFOutputPlugin.java
|
61
|
-
- classpath/embulk-output-postgres-udf-0.
|
64
|
+
- classpath/embulk-output-postgres-udf-0.2.0.jar
|
62
65
|
- classpath/jna-4.1.0.jar
|
63
66
|
- classpath/jna-platform-4.1.0.jar
|
64
67
|
- classpath/postgresql-9.4-1200-jdbc41.jar
|