embulk-output-s3_per_record 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/build.gradle +1 -1
- data/gradlew.bat +90 -90
- data/src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java +7 -1
- metadata +19 -19
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bc9e8ea415e7bb44a3c3c5395fcfdd40605a4885
|
|
4
|
+
data.tar.gz: ed4fa3983ab263f06c3c1293da4f818d894e4bd7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 603e7b62bddb09d2246c1873b71da7e2d73819442822680337adda5453a373292f6b413ddcf9eef527e66d1931e1c049a4d9e3cc9d2297f1d4876b31cc506beb
|
|
7
|
+
data.tar.gz: 99d2923ecbf1638a21fd571b5d8883683168101abe9080d9debe8e6188cf07b8f8d0c19ea9884acae73754216b20fb69995f3cc8a9819ed5570be0c0c8cf024b
|
data/README.md
CHANGED
|
@@ -17,6 +17,7 @@ S3 object key can be composed of another column.
|
|
|
17
17
|
- **aws_access_key_id**: (optional) AWS access key id. If not given, [DefaultAWSCredentialsProviderChain](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) is used to get credentials.
|
|
18
18
|
- **aws_secret_access_key**: (optional) AWS secret access key. Required if `aws_access_key_id` is given.
|
|
19
19
|
- **base64**: (default false) If true, decode the value as Base64 before uploading.
|
|
20
|
+
- **retry_limit**: (default 2) On connection errors,this plugin automatically retry up to this times.
|
|
20
21
|
|
|
21
22
|
## Example
|
|
22
23
|
|
data/build.gradle
CHANGED
data/gradlew.bat
CHANGED
|
@@ -1,90 +1,90 @@
|
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
|
2
|
-
@rem ##########################################################################
|
|
3
|
-
@rem
|
|
4
|
-
@rem Gradle startup script for Windows
|
|
5
|
-
@rem
|
|
6
|
-
@rem ##########################################################################
|
|
7
|
-
|
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
-
|
|
11
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
-
set DEFAULT_JVM_OPTS=
|
|
13
|
-
|
|
14
|
-
set DIRNAME=%~dp0
|
|
15
|
-
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
-
set APP_BASE_NAME=%~n0
|
|
17
|
-
set APP_HOME=%DIRNAME%
|
|
18
|
-
|
|
19
|
-
@rem Find java.exe
|
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
-
|
|
22
|
-
set JAVA_EXE=java.exe
|
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
-
|
|
26
|
-
echo.
|
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
-
echo.
|
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
-
echo location of your Java installation.
|
|
31
|
-
|
|
32
|
-
goto fail
|
|
33
|
-
|
|
34
|
-
:findJavaFromJavaHome
|
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
-
|
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
|
39
|
-
|
|
40
|
-
echo.
|
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
-
echo.
|
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
-
echo location of your Java installation.
|
|
45
|
-
|
|
46
|
-
goto fail
|
|
47
|
-
|
|
48
|
-
:init
|
|
49
|
-
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
-
|
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
-
|
|
54
|
-
:win9xME_args
|
|
55
|
-
@rem Slurp the command line arguments.
|
|
56
|
-
set CMD_LINE_ARGS=
|
|
57
|
-
set _SKIP=2
|
|
58
|
-
|
|
59
|
-
:win9xME_args_slurp
|
|
60
|
-
if "x%~1" == "x" goto execute
|
|
61
|
-
|
|
62
|
-
set CMD_LINE_ARGS=%*
|
|
63
|
-
goto execute
|
|
64
|
-
|
|
65
|
-
:4NT_args
|
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
-
set CMD_LINE_ARGS=%$
|
|
68
|
-
|
|
69
|
-
:execute
|
|
70
|
-
@rem Setup the command line
|
|
71
|
-
|
|
72
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
-
|
|
74
|
-
@rem Execute Gradle
|
|
75
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
-
|
|
77
|
-
:end
|
|
78
|
-
@rem End local scope for the variables with windows NT shell
|
|
79
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
-
|
|
81
|
-
:fail
|
|
82
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
-
rem the _cmd.exe /c_ return code!
|
|
84
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
-
exit /b 1
|
|
86
|
-
|
|
87
|
-
:mainEnd
|
|
88
|
-
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
-
|
|
90
|
-
:omega
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
|
2
|
+
@rem ##########################################################################
|
|
3
|
+
@rem
|
|
4
|
+
@rem Gradle startup script for Windows
|
|
5
|
+
@rem
|
|
6
|
+
@rem ##########################################################################
|
|
7
|
+
|
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
+
|
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
+
set DEFAULT_JVM_OPTS=
|
|
13
|
+
|
|
14
|
+
set DIRNAME=%~dp0
|
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
+
set APP_BASE_NAME=%~n0
|
|
17
|
+
set APP_HOME=%DIRNAME%
|
|
18
|
+
|
|
19
|
+
@rem Find java.exe
|
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
+
|
|
22
|
+
set JAVA_EXE=java.exe
|
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
+
|
|
26
|
+
echo.
|
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
+
echo.
|
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
+
echo location of your Java installation.
|
|
31
|
+
|
|
32
|
+
goto fail
|
|
33
|
+
|
|
34
|
+
:findJavaFromJavaHome
|
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
+
|
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
|
39
|
+
|
|
40
|
+
echo.
|
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
+
echo.
|
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
+
echo location of your Java installation.
|
|
45
|
+
|
|
46
|
+
goto fail
|
|
47
|
+
|
|
48
|
+
:init
|
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
+
|
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
+
|
|
54
|
+
:win9xME_args
|
|
55
|
+
@rem Slurp the command line arguments.
|
|
56
|
+
set CMD_LINE_ARGS=
|
|
57
|
+
set _SKIP=2
|
|
58
|
+
|
|
59
|
+
:win9xME_args_slurp
|
|
60
|
+
if "x%~1" == "x" goto execute
|
|
61
|
+
|
|
62
|
+
set CMD_LINE_ARGS=%*
|
|
63
|
+
goto execute
|
|
64
|
+
|
|
65
|
+
:4NT_args
|
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
+
set CMD_LINE_ARGS=%$
|
|
68
|
+
|
|
69
|
+
:execute
|
|
70
|
+
@rem Setup the command line
|
|
71
|
+
|
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
+
|
|
74
|
+
@rem Execute Gradle
|
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
+
|
|
77
|
+
:end
|
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
+
|
|
81
|
+
:fail
|
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
+
rem the _cmd.exe /c_ return code!
|
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
+
exit /b 1
|
|
86
|
+
|
|
87
|
+
:mainEnd
|
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
+
|
|
90
|
+
:omega
|
|
@@ -78,6 +78,11 @@ public class S3PerRecordOutputPlugin
|
|
|
78
78
|
@Config("base64")
|
|
79
79
|
@ConfigDefault("false")
|
|
80
80
|
boolean getBase64();
|
|
81
|
+
|
|
82
|
+
// Set retry limit. Default is 2.
|
|
83
|
+
@Config("retry_limit")
|
|
84
|
+
@ConfigDefault("2")
|
|
85
|
+
Integer getRetryLimit();
|
|
81
86
|
}
|
|
82
87
|
|
|
83
88
|
@Override
|
|
@@ -122,7 +127,7 @@ public class S3PerRecordOutputPlugin
|
|
|
122
127
|
private final Column dataColumn;
|
|
123
128
|
private final Schema schema;
|
|
124
129
|
private final boolean decodeBase64;
|
|
125
|
-
private final int retryLimit
|
|
130
|
+
private final int retryLimit;
|
|
126
131
|
|
|
127
132
|
public S3PerRecordPageOutput(PluginTask task, Schema schema) {
|
|
128
133
|
this.schema = schema;
|
|
@@ -130,6 +135,7 @@ public class S3PerRecordOutputPlugin
|
|
|
130
135
|
keyPattern = makeKeyPattern(task.getKey());
|
|
131
136
|
dataColumn = schema.lookupColumn(task.getDataColumn());
|
|
132
137
|
decodeBase64 = task.getBase64();
|
|
138
|
+
retryLimit = task.getRetryLimit();
|
|
133
139
|
|
|
134
140
|
AWSCredentials credentials;
|
|
135
141
|
if (task.getAwsAccessKeyId().isPresent() && task.getAwsSecretAccessKey().isPresent()) {
|
metadata
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-s3_per_record
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- tomykaira
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-08-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
name: bundler
|
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
16
|
-
requirements:
|
|
17
|
-
- - ~>
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: '1.0'
|
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
|
21
15
|
requirements:
|
|
22
16
|
- - ~>
|
|
23
17
|
- !ruby/object:Gem::Version
|
|
24
18
|
version: '1.0'
|
|
19
|
+
name: bundler
|
|
25
20
|
prerelease: false
|
|
26
21
|
type: :development
|
|
27
|
-
- !ruby/object:Gem::Dependency
|
|
28
|
-
name: rake
|
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
30
23
|
requirements:
|
|
31
|
-
- -
|
|
24
|
+
- - ~>
|
|
32
25
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
26
|
+
version: '1.0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
|
35
29
|
requirements:
|
|
36
30
|
- - '>='
|
|
37
31
|
- !ruby/object:Gem::Version
|
|
38
32
|
version: '10.0'
|
|
33
|
+
name: rake
|
|
39
34
|
prerelease: false
|
|
40
35
|
type: :development
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - '>='
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '10.0'
|
|
41
41
|
description: Upload a column's value to S3 as one S3 object per row.
|
|
42
42
|
email:
|
|
43
43
|
- tomykaira@gmail.com
|
|
@@ -58,15 +58,15 @@ files:
|
|
|
58
58
|
- lib/embulk/output/s3_per_record.rb
|
|
59
59
|
- src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java
|
|
60
60
|
- src/test/java/org/embulk/output/s3_per_record/TestS3PerRecordOutputPlugin.java
|
|
61
|
-
- classpath/
|
|
62
|
-
- classpath/aws-java-sdk-kms-1.10.71.jar
|
|
63
|
-
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
64
|
-
- classpath/commons-codec-1.6.jar
|
|
65
|
-
- classpath/commons-logging-1.1.3.jar
|
|
66
|
-
- classpath/embulk-output-s3_per_record-0.3.0.jar
|
|
61
|
+
- classpath/jackson-dataformat-cbor-2.5.3.jar
|
|
67
62
|
- classpath/httpclient-4.3.6.jar
|
|
63
|
+
- classpath/commons-logging-1.1.3.jar
|
|
64
|
+
- classpath/commons-codec-1.6.jar
|
|
65
|
+
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
66
|
+
- classpath/aws-java-sdk-core-1.10.71.jar
|
|
67
|
+
- classpath/embulk-output-s3_per_record-0.3.1.jar
|
|
68
68
|
- classpath/httpcore-4.3.3.jar
|
|
69
|
-
- classpath/
|
|
69
|
+
- classpath/aws-java-sdk-kms-1.10.71.jar
|
|
70
70
|
homepage: https://github.com/tomykaira/embulk-output-s3_per_record
|
|
71
71
|
licenses:
|
|
72
72
|
- MIT
|