embulk-output-s3_per_record 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/gradlew.bat +90 -90
- data/src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java +32 -10
- metadata +19 -19
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 322b2a912a69a61b5f4a4b977148f68fe26e194b
|
|
4
|
+
data.tar.gz: 4fc391c4a8f67d88f5fa0880f10c3a833b569847
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6059553638e649414fed1cce92a0cb097e3a432ef53f472d014e917459e491384415424e7a52b7dae5e8a5e3c8fc89afcba1d33a886ee6f470c068a0d52140f2
|
|
7
|
+
data.tar.gz: bd60f4ca7f1b9e815eb90174552449e9510fdca4b28717cc53ba6d4de7e489bc90358bc67c909f5b0554a57c4b5fc255e2cbdd872b754d3388957961e54d6d44
|
data/build.gradle
CHANGED
data/gradlew.bat
CHANGED
|
@@ -1,90 +1,90 @@
|
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
|
2
|
-
@rem ##########################################################################
|
|
3
|
-
@rem
|
|
4
|
-
@rem Gradle startup script for Windows
|
|
5
|
-
@rem
|
|
6
|
-
@rem ##########################################################################
|
|
7
|
-
|
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
-
|
|
11
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
-
set DEFAULT_JVM_OPTS=
|
|
13
|
-
|
|
14
|
-
set DIRNAME=%~dp0
|
|
15
|
-
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
-
set APP_BASE_NAME=%~n0
|
|
17
|
-
set APP_HOME=%DIRNAME%
|
|
18
|
-
|
|
19
|
-
@rem Find java.exe
|
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
-
|
|
22
|
-
set JAVA_EXE=java.exe
|
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
-
|
|
26
|
-
echo.
|
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
-
echo.
|
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
-
echo location of your Java installation.
|
|
31
|
-
|
|
32
|
-
goto fail
|
|
33
|
-
|
|
34
|
-
:findJavaFromJavaHome
|
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
-
|
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
|
39
|
-
|
|
40
|
-
echo.
|
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
-
echo.
|
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
-
echo location of your Java installation.
|
|
45
|
-
|
|
46
|
-
goto fail
|
|
47
|
-
|
|
48
|
-
:init
|
|
49
|
-
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
-
|
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
-
|
|
54
|
-
:win9xME_args
|
|
55
|
-
@rem Slurp the command line arguments.
|
|
56
|
-
set CMD_LINE_ARGS=
|
|
57
|
-
set _SKIP=2
|
|
58
|
-
|
|
59
|
-
:win9xME_args_slurp
|
|
60
|
-
if "x%~1" == "x" goto execute
|
|
61
|
-
|
|
62
|
-
set CMD_LINE_ARGS=%*
|
|
63
|
-
goto execute
|
|
64
|
-
|
|
65
|
-
:4NT_args
|
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
-
set CMD_LINE_ARGS=%$
|
|
68
|
-
|
|
69
|
-
:execute
|
|
70
|
-
@rem Setup the command line
|
|
71
|
-
|
|
72
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
-
|
|
74
|
-
@rem Execute Gradle
|
|
75
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
-
|
|
77
|
-
:end
|
|
78
|
-
@rem End local scope for the variables with windows NT shell
|
|
79
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
-
|
|
81
|
-
:fail
|
|
82
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
-
rem the _cmd.exe /c_ return code!
|
|
84
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
-
exit /b 1
|
|
86
|
-
|
|
87
|
-
:mainEnd
|
|
88
|
-
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
-
|
|
90
|
-
:omega
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
|
2
|
+
@rem ##########################################################################
|
|
3
|
+
@rem
|
|
4
|
+
@rem Gradle startup script for Windows
|
|
5
|
+
@rem
|
|
6
|
+
@rem ##########################################################################
|
|
7
|
+
|
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
|
10
|
+
|
|
11
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
|
12
|
+
set DEFAULT_JVM_OPTS=
|
|
13
|
+
|
|
14
|
+
set DIRNAME=%~dp0
|
|
15
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
|
16
|
+
set APP_BASE_NAME=%~n0
|
|
17
|
+
set APP_HOME=%DIRNAME%
|
|
18
|
+
|
|
19
|
+
@rem Find java.exe
|
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
|
21
|
+
|
|
22
|
+
set JAVA_EXE=java.exe
|
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
|
25
|
+
|
|
26
|
+
echo.
|
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
|
28
|
+
echo.
|
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
30
|
+
echo location of your Java installation.
|
|
31
|
+
|
|
32
|
+
goto fail
|
|
33
|
+
|
|
34
|
+
:findJavaFromJavaHome
|
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
|
37
|
+
|
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
|
39
|
+
|
|
40
|
+
echo.
|
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
|
42
|
+
echo.
|
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
|
44
|
+
echo location of your Java installation.
|
|
45
|
+
|
|
46
|
+
goto fail
|
|
47
|
+
|
|
48
|
+
:init
|
|
49
|
+
@rem Get command-line arguments, handling Windowz variants
|
|
50
|
+
|
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
|
53
|
+
|
|
54
|
+
:win9xME_args
|
|
55
|
+
@rem Slurp the command line arguments.
|
|
56
|
+
set CMD_LINE_ARGS=
|
|
57
|
+
set _SKIP=2
|
|
58
|
+
|
|
59
|
+
:win9xME_args_slurp
|
|
60
|
+
if "x%~1" == "x" goto execute
|
|
61
|
+
|
|
62
|
+
set CMD_LINE_ARGS=%*
|
|
63
|
+
goto execute
|
|
64
|
+
|
|
65
|
+
:4NT_args
|
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
|
67
|
+
set CMD_LINE_ARGS=%$
|
|
68
|
+
|
|
69
|
+
:execute
|
|
70
|
+
@rem Setup the command line
|
|
71
|
+
|
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
|
73
|
+
|
|
74
|
+
@rem Execute Gradle
|
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
|
76
|
+
|
|
77
|
+
:end
|
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
|
80
|
+
|
|
81
|
+
:fail
|
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
|
83
|
+
rem the _cmd.exe /c_ return code!
|
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
|
85
|
+
exit /b 1
|
|
86
|
+
|
|
87
|
+
:mainEnd
|
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
|
89
|
+
|
|
90
|
+
:omega
|
|
@@ -10,6 +10,7 @@ import java.util.concurrent.atomic.AtomicLong;
|
|
|
10
10
|
|
|
11
11
|
import javax.validation.constraints.NotNull;
|
|
12
12
|
|
|
13
|
+
import com.amazonaws.services.s3.model.AmazonS3Exception;
|
|
13
14
|
import org.embulk.config.Config;
|
|
14
15
|
import org.embulk.config.ConfigDefault;
|
|
15
16
|
import org.embulk.config.ConfigDiff;
|
|
@@ -121,6 +122,7 @@ public class S3PerRecordOutputPlugin
|
|
|
121
122
|
private final Column dataColumn;
|
|
122
123
|
private final Schema schema;
|
|
123
124
|
private final boolean decodeBase64;
|
|
125
|
+
private final int retryLimit = 2;
|
|
124
126
|
|
|
125
127
|
public S3PerRecordPageOutput(PluginTask task, Schema schema) {
|
|
126
128
|
this.schema = schema;
|
|
@@ -180,17 +182,37 @@ public class S3PerRecordOutputPlugin
|
|
|
180
182
|
ObjectMetadata metadata = new ObjectMetadata();
|
|
181
183
|
metadata.setContentLength(payloadBytes.length);
|
|
182
184
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
nextLoggingRowCount
|
|
185
|
+
int retryCount = 0;
|
|
186
|
+
int retryWait = 1000; // ms
|
|
187
|
+
while (true) {
|
|
188
|
+
try (InputStream is = new ByteArrayInputStream(payloadBytes)) {
|
|
189
|
+
Upload upload = transferManager.upload(bucket, key, is, metadata);
|
|
190
|
+
upload.waitForUploadResult();
|
|
191
|
+
long rows = processedRows.incrementAndGet();
|
|
192
|
+
if (rows == nextLoggingRowCount) {
|
|
193
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
|
194
|
+
logger.info(String.format("> Uploaded %,d rows in %.2f seconds", rows, seconds));
|
|
195
|
+
nextLoggingRowCount *= 2;
|
|
196
|
+
}
|
|
197
|
+
break;
|
|
198
|
+
} catch (AmazonS3Exception e) {
|
|
199
|
+
if (!e.isRetryable())
|
|
200
|
+
throw e;
|
|
201
|
+
|
|
202
|
+
if (retryCount > retryLimit)
|
|
203
|
+
throw e;
|
|
204
|
+
|
|
205
|
+
retryCount++;
|
|
206
|
+
logger.warn(String.format("> Upload failed by %s, Retry Uploading in after %d ms (%d of %d)", e.getMessage(), retryWait, retryCount, retryLimit));
|
|
207
|
+
try {
|
|
208
|
+
Thread.sleep(retryWait);
|
|
209
|
+
retryWait = retryWait * 2;
|
|
210
|
+
} catch (InterruptedException e1) {
|
|
211
|
+
Thread.currentThread().interrupt();
|
|
212
|
+
}
|
|
213
|
+
} catch (InterruptedException | IOException e) {
|
|
214
|
+
throw new RuntimeException(e);
|
|
191
215
|
}
|
|
192
|
-
} catch (InterruptedException | IOException e) {
|
|
193
|
-
throw new RuntimeException(e);
|
|
194
216
|
}
|
|
195
217
|
}
|
|
196
218
|
}
|
metadata
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-s3_per_record
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- tomykaira
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-06-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
|
|
14
|
+
name: bundler
|
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
15
16
|
requirements:
|
|
16
17
|
- - ~>
|
|
17
18
|
- !ruby/object:Gem::Version
|
|
18
19
|
version: '1.0'
|
|
19
|
-
|
|
20
|
-
prerelease: false
|
|
21
|
-
type: :development
|
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
20
|
+
requirement: !ruby/object:Gem::Requirement
|
|
23
21
|
requirements:
|
|
24
22
|
- - ~>
|
|
25
23
|
- !ruby/object:Gem::Version
|
|
26
24
|
version: '1.0'
|
|
25
|
+
prerelease: false
|
|
26
|
+
type: :development
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
|
-
|
|
28
|
+
name: rake
|
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
29
30
|
requirements:
|
|
30
31
|
- - '>='
|
|
31
32
|
- !ruby/object:Gem::Version
|
|
32
33
|
version: '10.0'
|
|
33
|
-
|
|
34
|
-
prerelease: false
|
|
35
|
-
type: :development
|
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
37
35
|
requirements:
|
|
38
36
|
- - '>='
|
|
39
37
|
- !ruby/object:Gem::Version
|
|
40
38
|
version: '10.0'
|
|
39
|
+
prerelease: false
|
|
40
|
+
type: :development
|
|
41
41
|
description: Upload a column's value to S3 as one S3 object per row.
|
|
42
42
|
email:
|
|
43
43
|
- tomykaira@gmail.com
|
|
@@ -58,15 +58,15 @@ files:
|
|
|
58
58
|
- lib/embulk/output/s3_per_record.rb
|
|
59
59
|
- src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java
|
|
60
60
|
- src/test/java/org/embulk/output/s3_per_record/TestS3PerRecordOutputPlugin.java
|
|
61
|
-
- classpath/jackson-dataformat-cbor-2.5.3.jar
|
|
62
|
-
- classpath/httpclient-4.3.6.jar
|
|
63
|
-
- classpath/commons-logging-1.1.3.jar
|
|
64
|
-
- classpath/commons-codec-1.6.jar
|
|
65
|
-
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
66
61
|
- classpath/aws-java-sdk-core-1.10.71.jar
|
|
67
|
-
- classpath/embulk-output-s3_per_record-0.2.0.jar
|
|
68
|
-
- classpath/httpcore-4.3.3.jar
|
|
69
62
|
- classpath/aws-java-sdk-kms-1.10.71.jar
|
|
63
|
+
- classpath/aws-java-sdk-s3-1.10.71.jar
|
|
64
|
+
- classpath/commons-codec-1.6.jar
|
|
65
|
+
- classpath/commons-logging-1.1.3.jar
|
|
66
|
+
- classpath/embulk-output-s3_per_record-0.3.0.jar
|
|
67
|
+
- classpath/httpclient-4.3.6.jar
|
|
68
|
+
- classpath/httpcore-4.3.3.jar
|
|
69
|
+
- classpath/jackson-dataformat-cbor-2.5.3.jar
|
|
70
70
|
homepage: https://github.com/tomykaira/embulk-output-s3_per_record
|
|
71
71
|
licenses:
|
|
72
72
|
- MIT
|