embulk-output-s3_per_record 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/build.gradle +1 -1
- data/gradlew.bat +90 -90
- data/src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java +7 -1
- metadata +19 -19
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bc9e8ea415e7bb44a3c3c5395fcfdd40605a4885
|
|
4
|
+
data.tar.gz: ed4fa3983ab263f06c3c1293da4f818d894e4bd7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 603e7b62bddb09d2246c1873b71da7e2d73819442822680337adda5453a373292f6b413ddcf9eef527e66d1931e1c049a4d9e3cc9d2297f1d4876b31cc506beb
|
|
7
|
+
data.tar.gz: 99d2923ecbf1638a21fd571b5d8883683168101abe9080d9debe8e6188cf07b8f8d0c19ea9884acae73754216b20fb69995f3cc8a9819ed5570be0c0c8cf024b
|
data/README.md
CHANGED
|
@@ -17,6 +17,7 @@ S3 object key can be composed of another column.
|
|
|
17
17
|
- **aws_access_key_id**: (optional) AWS access key id. If not given, [DefaultAWSCredentialsProviderChain](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) is used to get credentials.
|
|
18
18
|
- **aws_secret_access_key**: (optional) AWS secret access key. Required if `aws_access_key_id` is given.
|
|
19
19
|
- **base64**: (default false) If true, decode the value as Base64 before uploading.
|
|
20
|
+
- **retry_limit**: (default 2) On connection errors,this plugin automatically retry up to this times.
|
|
20
21
|
|
|
21
22
|
## Example
|
|
22
23
|
|
data/build.gradle
CHANGED
data/gradlew.bat
CHANGED
|
@@ -1,90 +1,90 @@
|
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
|
2
|
-
@rem ##########################################################################
|
|
3
|
-
@rem
|
|
4
|
-
@rem Gradle startup script for Windows
|
|
5
|
-
@rem
|
|
6
|
-
@rem ##########################################################################
|
|
7
|
-
|
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
-
|
|
11
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
-
set DEFAULT_JVM_OPTS=
|
|
13
|
-
|
|
14
|
-
set DIRNAME=%~dp0
|
|
15
|
-
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
-
set APP_BASE_NAME=%~n0
|
|
17
|
-
set APP_HOME=%DIRNAME%
|
|
18
|
-
|
|
19
|
-
@rem Find java.exe
|
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
-
|
|
22
|
-
set JAVA_EXE=java.exe
|
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
-
|
|
26
|
-
echo.
|
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
-
echo.
|
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
-
echo location of your Java installation.
|
|
31
|
-
|
|
32
|
-
goto fail
|
|
33
|
-
|
|
34
|
-
:findJavaFromJavaHome
|
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
-
|
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
|
39
|
-
|
|
40
|
-
echo.
|
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
-
echo.
|
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
-
echo location of your Java installation.
|
|
45
|
-
|
|
46
|
-
goto fail
|
|
47
|
-
|
|
48
|
-
:init
|
|
49
|
-
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
-
|
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
-
|
|
54
|
-
:win9xME_args
|
|
55
|
-
@rem Slurp the command line arguments.
|
|
56
|
-
set CMD_LINE_ARGS=
|
|
57
|
-
set _SKIP=2
|
|
58
|
-
|
|
59
|
-
:win9xME_args_slurp
|
|
60
|
-
if "x%~1" == "x" goto execute
|
|
61
|
-
|
|
62
|
-
set CMD_LINE_ARGS=%*
|
|
63
|
-
goto execute
|
|
64
|
-
|
|
65
|
-
:4NT_args
|
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
-
set CMD_LINE_ARGS=%$
|
|
68
|
-
|
|
69
|
-
:execute
|
|
70
|
-
@rem Setup the command line
|
|
71
|
-
|
|
72
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
-
|
|
74
|
-
@rem Execute Gradle
|
|
75
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
-
|
|
77
|
-
:end
|
|
78
|
-
@rem End local scope for the variables with windows NT shell
|
|
79
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
-
|
|
81
|
-
:fail
|
|
82
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
-
rem the _cmd.exe /c_ return code!
|
|
84
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
-
exit /b 1
|
|
86
|
-
|
|
87
|
-
:mainEnd
|
|
88
|
-
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
-
|
|
90
|
-
:omega
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
|
2
|
+
@rem ##########################################################################
|
|
3
|
+
@rem
|
|
4
|
+
@rem Gradle startup script for Windows
|
|
5
|
+
@rem
|
|
6
|
+
@rem ##########################################################################
|
|
7
|
+
|
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
+
|
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
+
set DEFAULT_JVM_OPTS=
|
|
13
|
+
|
|
14
|
+
set DIRNAME=%~dp0
|
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
+
set APP_BASE_NAME=%~n0
|
|
17
|
+
set APP_HOME=%DIRNAME%
|
|
18
|
+
|
|
19
|
+
@rem Find java.exe
|
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
+
|
|
22
|
+
set JAVA_EXE=java.exe
|
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
+
|
|
26
|
+
echo.
|
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
+
echo.
|
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
+
echo location of your Java installation.
|
|
31
|
+
|
|
32
|
+
goto fail
|
|
33
|
+
|
|
34
|
+
:findJavaFromJavaHome
|
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
+
|
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
|
39
|
+
|
|
40
|
+
echo.
|
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
+
echo.
|
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
+
echo location of your Java installation.
|
|
45
|
+
|
|
46
|
+
goto fail
|
|
47
|
+
|
|
48
|
+
:init
|
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
+
|
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
+
|
|
54
|
+
:win9xME_args
|
|
55
|
+
@rem Slurp the command line arguments.
|
|
56
|
+
set CMD_LINE_ARGS=
|
|
57
|
+
set _SKIP=2
|
|
58
|
+
|
|
59
|
+
:win9xME_args_slurp
|
|
60
|
+
if "x%~1" == "x" goto execute
|
|
61
|
+
|
|
62
|
+
set CMD_LINE_ARGS=%*
|
|
63
|
+
goto execute
|
|
64
|
+
|
|
65
|
+
:4NT_args
|
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
+
set CMD_LINE_ARGS=%$
|
|
68
|
+
|
|
69
|
+
:execute
|
|
70
|
+
@rem Setup the command line
|
|
71
|
+
|
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
+
|
|
74
|
+
@rem Execute Gradle
|
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
+
|
|
77
|
+
:end
|
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
+
|
|
81
|
+
:fail
|
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
+
rem the _cmd.exe /c_ return code!
|
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
+
exit /b 1
|
|
86
|
+
|
|
87
|
+
:mainEnd
|
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
+
|
|
90
|
+
:omega
|
|
@@ -78,6 +78,11 @@ public class S3PerRecordOutputPlugin
|
|
|
78
78
|
@Config("base64")
|
|
79
79
|
@ConfigDefault("false")
|
|
80
80
|
boolean getBase64();
|
|
81
|
+
|
|
82
|
+
// Set retry limit. Default is 2.
|
|
83
|
+
@Config("retry_limit")
|
|
84
|
+
@ConfigDefault("2")
|
|
85
|
+
Integer getRetryLimit();
|
|
81
86
|
}
|
|
82
87
|
|
|
83
88
|
@Override
|
|
@@ -122,7 +127,7 @@ public class S3PerRecordOutputPlugin
|
|
|
122
127
|
private final Column dataColumn;
|
|
123
128
|
private final Schema schema;
|
|
124
129
|
private final boolean decodeBase64;
|
|
125
|
-
private final int retryLimit
|
|
130
|
+
private final int retryLimit;
|
|
126
131
|
|
|
127
132
|
public S3PerRecordPageOutput(PluginTask task, Schema schema) {
|
|
128
133
|
this.schema = schema;
|
|
@@ -130,6 +135,7 @@ public class S3PerRecordOutputPlugin
|
|
|
130
135
|
keyPattern = makeKeyPattern(task.getKey());
|
|
131
136
|
dataColumn = schema.lookupColumn(task.getDataColumn());
|
|
132
137
|
decodeBase64 = task.getBase64();
|
|
138
|
+
retryLimit = task.getRetryLimit();
|
|
133
139
|
|
|
134
140
|
AWSCredentials credentials;
|
|
135
141
|
if (task.getAwsAccessKeyId().isPresent() && task.getAwsSecretAccessKey().isPresent()) {
|
metadata
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-s3_per_record
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- tomykaira
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-08-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
name: bundler
|
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
16
|
-
requirements:
|
|
17
|
-
- - ~>
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: '1.0'
|
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
|
21
15
|
requirements:
|
|
22
16
|
- - ~>
|
|
23
17
|
- !ruby/object:Gem::Version
|
|
24
18
|
version: '1.0'
|
|
19
|
+
name: bundler
|
|
25
20
|
prerelease: false
|
|
26
21
|
type: :development
|
|
27
|
-
- !ruby/object:Gem::Dependency
|
|
28
|
-
name: rake
|
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
30
23
|
requirements:
|
|
31
|
-
- -
|
|
24
|
+
- - ~>
|
|
32
25
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
26
|
+
version: '1.0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
|
35
29
|
requirements:
|
|
36
30
|
- - '>='
|
|
37
31
|
- !ruby/object:Gem::Version
|
|
38
32
|
version: '10.0'
|
|
33
|
+
name: rake
|
|
39
34
|
prerelease: false
|
|
40
35
|
type: :development
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - '>='
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '10.0'
|
|
41
41
|
description: Upload a column's value to S3 as one S3 object per row.
|
|
42
42
|
email:
|
|
43
43
|
- tomykaira@gmail.com
|
|
@@ -58,15 +58,15 @@ files:
|
|
|
58
58
|
- lib/embulk/output/s3_per_record.rb
|
|
59
59
|
- src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java
|
|
60
60
|
- src/test/java/org/embulk/output/s3_per_record/TestS3PerRecordOutputPlugin.java
|
|
61
|
-
- classpath/
|
|
62
|
-
- classpath/aws-java-sdk-kms-1.10.71.jar
|
|
63
|
-
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
64
|
-
- classpath/commons-codec-1.6.jar
|
|
65
|
-
- classpath/commons-logging-1.1.3.jar
|
|
66
|
-
- classpath/embulk-output-s3_per_record-0.3.0.jar
|
|
61
|
+
- classpath/jackson-dataformat-cbor-2.5.3.jar
|
|
67
62
|
- classpath/httpclient-4.3.6.jar
|
|
63
|
+
- classpath/commons-logging-1.1.3.jar
|
|
64
|
+
- classpath/commons-codec-1.6.jar
|
|
65
|
+
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
66
|
+
- classpath/aws-java-sdk-core-1.10.71.jar
|
|
67
|
+
- classpath/embulk-output-s3_per_record-0.3.1.jar
|
|
68
68
|
- classpath/httpcore-4.3.3.jar
|
|
69
|
-
- classpath/
|
|
69
|
+
- classpath/aws-java-sdk-kms-1.10.71.jar
|
|
70
70
|
homepage: https://github.com/tomykaira/embulk-output-s3_per_record
|
|
71
71
|
licenses:
|
|
72
72
|
- MIT
|