embulk-output-s3_per_record 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/gradlew.bat +90 -90
- data/src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java +32 -10
- metadata +19 -19
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 322b2a912a69a61b5f4a4b977148f68fe26e194b
|
|
4
|
+
data.tar.gz: 4fc391c4a8f67d88f5fa0880f10c3a833b569847
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6059553638e649414fed1cce92a0cb097e3a432ef53f472d014e917459e491384415424e7a52b7dae5e8a5e3c8fc89afcba1d33a886ee6f470c068a0d52140f2
|
|
7
|
+
data.tar.gz: bd60f4ca7f1b9e815eb90174552449e9510fdca4b28717cc53ba6d4de7e489bc90358bc67c909f5b0554a57c4b5fc255e2cbdd872b754d3388957961e54d6d44
|
data/build.gradle
CHANGED
data/gradlew.bat
CHANGED
|
@@ -1,90 +1,90 @@
|
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
|
2
|
-
@rem ##########################################################################
|
|
3
|
-
@rem
|
|
4
|
-
@rem Gradle startup script for Windows
|
|
5
|
-
@rem
|
|
6
|
-
@rem ##########################################################################
|
|
7
|
-
|
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
-
|
|
11
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
-
set DEFAULT_JVM_OPTS=
|
|
13
|
-
|
|
14
|
-
set DIRNAME=%~dp0
|
|
15
|
-
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
-
set APP_BASE_NAME=%~n0
|
|
17
|
-
set APP_HOME=%DIRNAME%
|
|
18
|
-
|
|
19
|
-
@rem Find java.exe
|
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
-
|
|
22
|
-
set JAVA_EXE=java.exe
|
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
-
|
|
26
|
-
echo.
|
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
-
echo.
|
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
-
echo location of your Java installation.
|
|
31
|
-
|
|
32
|
-
goto fail
|
|
33
|
-
|
|
34
|
-
:findJavaFromJavaHome
|
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
-
|
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
|
39
|
-
|
|
40
|
-
echo.
|
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
-
echo.
|
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
-
echo location of your Java installation.
|
|
45
|
-
|
|
46
|
-
goto fail
|
|
47
|
-
|
|
48
|
-
:init
|
|
49
|
-
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
-
|
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
-
|
|
54
|
-
:win9xME_args
|
|
55
|
-
@rem Slurp the command line arguments.
|
|
56
|
-
set CMD_LINE_ARGS=
|
|
57
|
-
set _SKIP=2
|
|
58
|
-
|
|
59
|
-
:win9xME_args_slurp
|
|
60
|
-
if "x%~1" == "x" goto execute
|
|
61
|
-
|
|
62
|
-
set CMD_LINE_ARGS=%*
|
|
63
|
-
goto execute
|
|
64
|
-
|
|
65
|
-
:4NT_args
|
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
-
set CMD_LINE_ARGS=%$
|
|
68
|
-
|
|
69
|
-
:execute
|
|
70
|
-
@rem Setup the command line
|
|
71
|
-
|
|
72
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
-
|
|
74
|
-
@rem Execute Gradle
|
|
75
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
-
|
|
77
|
-
:end
|
|
78
|
-
@rem End local scope for the variables with windows NT shell
|
|
79
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
-
|
|
81
|
-
:fail
|
|
82
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
-
rem the _cmd.exe /c_ return code!
|
|
84
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
-
exit /b 1
|
|
86
|
-
|
|
87
|
-
:mainEnd
|
|
88
|
-
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
-
|
|
90
|
-
:omega
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
|
2
|
+
@rem ##########################################################################
|
|
3
|
+
@rem
|
|
4
|
+
@rem Gradle startup script for Windows
|
|
5
|
+
@rem
|
|
6
|
+
@rem ##########################################################################
|
|
7
|
+
|
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
+
|
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
+
set DEFAULT_JVM_OPTS=
|
|
13
|
+
|
|
14
|
+
set DIRNAME=%~dp0
|
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
+
set APP_BASE_NAME=%~n0
|
|
17
|
+
set APP_HOME=%DIRNAME%
|
|
18
|
+
|
|
19
|
+
@rem Find java.exe
|
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
+
|
|
22
|
+
set JAVA_EXE=java.exe
|
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
+
|
|
26
|
+
echo.
|
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
+
echo.
|
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
+
echo location of your Java installation.
|
|
31
|
+
|
|
32
|
+
goto fail
|
|
33
|
+
|
|
34
|
+
:findJavaFromJavaHome
|
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
+
|
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
|
39
|
+
|
|
40
|
+
echo.
|
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
+
echo.
|
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
+
echo location of your Java installation.
|
|
45
|
+
|
|
46
|
+
goto fail
|
|
47
|
+
|
|
48
|
+
:init
|
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
+
|
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
+
|
|
54
|
+
:win9xME_args
|
|
55
|
+
@rem Slurp the command line arguments.
|
|
56
|
+
set CMD_LINE_ARGS=
|
|
57
|
+
set _SKIP=2
|
|
58
|
+
|
|
59
|
+
:win9xME_args_slurp
|
|
60
|
+
if "x%~1" == "x" goto execute
|
|
61
|
+
|
|
62
|
+
set CMD_LINE_ARGS=%*
|
|
63
|
+
goto execute
|
|
64
|
+
|
|
65
|
+
:4NT_args
|
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
+
set CMD_LINE_ARGS=%$
|
|
68
|
+
|
|
69
|
+
:execute
|
|
70
|
+
@rem Setup the command line
|
|
71
|
+
|
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
+
|
|
74
|
+
@rem Execute Gradle
|
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
+
|
|
77
|
+
:end
|
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
+
|
|
81
|
+
:fail
|
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
+
rem the _cmd.exe /c_ return code!
|
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
+
exit /b 1
|
|
86
|
+
|
|
87
|
+
:mainEnd
|
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
+
|
|
90
|
+
:omega
|
|
@@ -10,6 +10,7 @@ import java.util.concurrent.atomic.AtomicLong;
|
|
|
10
10
|
|
|
11
11
|
import javax.validation.constraints.NotNull;
|
|
12
12
|
|
|
13
|
+
import com.amazonaws.services.s3.model.AmazonS3Exception;
|
|
13
14
|
import org.embulk.config.Config;
|
|
14
15
|
import org.embulk.config.ConfigDefault;
|
|
15
16
|
import org.embulk.config.ConfigDiff;
|
|
@@ -121,6 +122,7 @@ public class S3PerRecordOutputPlugin
|
|
|
121
122
|
private final Column dataColumn;
|
|
122
123
|
private final Schema schema;
|
|
123
124
|
private final boolean decodeBase64;
|
|
125
|
+
private final int retryLimit = 2;
|
|
124
126
|
|
|
125
127
|
public S3PerRecordPageOutput(PluginTask task, Schema schema) {
|
|
126
128
|
this.schema = schema;
|
|
@@ -180,17 +182,37 @@ public class S3PerRecordOutputPlugin
|
|
|
180
182
|
ObjectMetadata metadata = new ObjectMetadata();
|
|
181
183
|
metadata.setContentLength(payloadBytes.length);
|
|
182
184
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
nextLoggingRowCount
|
|
185
|
+
int retryCount = 0;
|
|
186
|
+
int retryWait = 1000; // ms
|
|
187
|
+
while (true) {
|
|
188
|
+
try (InputStream is = new ByteArrayInputStream(payloadBytes)) {
|
|
189
|
+
Upload upload = transferManager.upload(bucket, key, is, metadata);
|
|
190
|
+
upload.waitForUploadResult();
|
|
191
|
+
long rows = processedRows.incrementAndGet();
|
|
192
|
+
if (rows == nextLoggingRowCount) {
|
|
193
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
|
194
|
+
logger.info(String.format("> Uploaded %,d rows in %.2f seconds", rows, seconds));
|
|
195
|
+
nextLoggingRowCount *= 2;
|
|
196
|
+
}
|
|
197
|
+
break;
|
|
198
|
+
} catch (AmazonS3Exception e) {
|
|
199
|
+
if (!e.isRetryable())
|
|
200
|
+
throw e;
|
|
201
|
+
|
|
202
|
+
if (retryCount > retryLimit)
|
|
203
|
+
throw e;
|
|
204
|
+
|
|
205
|
+
retryCount++;
|
|
206
|
+
logger.warn(String.format("> Upload failed by %s, Retry Uploading in after %d ms (%d of %d)", e.getMessage(), retryWait, retryCount, retryLimit));
|
|
207
|
+
try {
|
|
208
|
+
Thread.sleep(retryWait);
|
|
209
|
+
retryWait = retryWait * 2;
|
|
210
|
+
} catch (InterruptedException e1) {
|
|
211
|
+
Thread.currentThread().interrupt();
|
|
212
|
+
}
|
|
213
|
+
} catch (InterruptedException | IOException e) {
|
|
214
|
+
throw new RuntimeException(e);
|
|
191
215
|
}
|
|
192
|
-
} catch (InterruptedException | IOException e) {
|
|
193
|
-
throw new RuntimeException(e);
|
|
194
216
|
}
|
|
195
217
|
}
|
|
196
218
|
}
|
metadata
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-s3_per_record
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- tomykaira
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-06-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
|
|
14
|
+
name: bundler
|
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
15
16
|
requirements:
|
|
16
17
|
- - ~>
|
|
17
18
|
- !ruby/object:Gem::Version
|
|
18
19
|
version: '1.0'
|
|
19
|
-
|
|
20
|
-
prerelease: false
|
|
21
|
-
type: :development
|
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
20
|
+
requirement: !ruby/object:Gem::Requirement
|
|
23
21
|
requirements:
|
|
24
22
|
- - ~>
|
|
25
23
|
- !ruby/object:Gem::Version
|
|
26
24
|
version: '1.0'
|
|
25
|
+
prerelease: false
|
|
26
|
+
type: :development
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
|
-
|
|
28
|
+
name: rake
|
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
29
30
|
requirements:
|
|
30
31
|
- - '>='
|
|
31
32
|
- !ruby/object:Gem::Version
|
|
32
33
|
version: '10.0'
|
|
33
|
-
|
|
34
|
-
prerelease: false
|
|
35
|
-
type: :development
|
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
37
35
|
requirements:
|
|
38
36
|
- - '>='
|
|
39
37
|
- !ruby/object:Gem::Version
|
|
40
38
|
version: '10.0'
|
|
39
|
+
prerelease: false
|
|
40
|
+
type: :development
|
|
41
41
|
description: Upload a column's value to S3 as one S3 object per row.
|
|
42
42
|
email:
|
|
43
43
|
- tomykaira@gmail.com
|
|
@@ -58,15 +58,15 @@ files:
|
|
|
58
58
|
- lib/embulk/output/s3_per_record.rb
|
|
59
59
|
- src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java
|
|
60
60
|
- src/test/java/org/embulk/output/s3_per_record/TestS3PerRecordOutputPlugin.java
|
|
61
|
-
- classpath/jackson-dataformat-cbor-2.5.3.jar
|
|
62
|
-
- classpath/httpclient-4.3.6.jar
|
|
63
|
-
- classpath/commons-logging-1.1.3.jar
|
|
64
|
-
- classpath/commons-codec-1.6.jar
|
|
65
|
-
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
66
61
|
- classpath/aws-java-sdk-core-1.10.71.jar
|
|
67
|
-
- classpath/embulk-output-s3_per_record-0.2.0.jar
|
|
68
|
-
- classpath/httpcore-4.3.3.jar
|
|
69
62
|
- classpath/aws-java-sdk-kms-1.10.71.jar
|
|
63
|
+
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
64
|
+
- classpath/commons-codec-1.6.jar
|
|
65
|
+
- classpath/commons-logging-1.1.3.jar
|
|
66
|
+
- classpath/embulk-output-s3_per_record-0.3.0.jar
|
|
67
|
+
- classpath/httpclient-4.3.6.jar
|
|
68
|
+
- classpath/httpcore-4.3.3.jar
|
|
69
|
+
- classpath/jackson-dataformat-cbor-2.5.3.jar
|
|
70
70
|
homepage: https://github.com/tomykaira/embulk-output-s3_per_record
|
|
71
71
|
licenses:
|
|
72
72
|
- MIT
|