embulk-output-s3_per_record 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 59291fd824e6db4a6f56e4e3b107f1be43a71dfa
4
- data.tar.gz: 0cb03ee9ae9d61130c53c5a0f5b80e5bd4ae34d7
3
+ metadata.gz: 322b2a912a69a61b5f4a4b977148f68fe26e194b
4
+ data.tar.gz: 4fc391c4a8f67d88f5fa0880f10c3a833b569847
5
5
  SHA512:
6
- metadata.gz: 7c446fa5ddc162c53763695e8b36d5ca6ec213d5e677ecf5df0fe31097e5aae445ae34176642dcdfbe51b05085d5fc82d771a1572588386c9587c22c9f3b2b8a
7
- data.tar.gz: 7c7e563beeede3252e82ed481a9e4bba1785bf4e6f1abfb508d602041a8ad37df6db7a6f5d6b61ba78c042274d2a8b5abc13a3423f75edda8cdbbe4c05624220
6
+ metadata.gz: 6059553638e649414fed1cce92a0cb097e3a432ef53f472d014e917459e491384415424e7a52b7dae5e8a5e3c8fc89afcba1d33a886ee6f470c068a0d52140f2
7
+ data.tar.gz: bd60f4ca7f1b9e815eb90174552449e9510fdca4b28717cc53ba6d4de7e489bc90358bc67c909f5b0554a57c4b5fc255e2cbdd872b754d3388957961e54d6d44
data/build.gradle CHANGED
@@ -13,7 +13,7 @@ configurations {
13
13
  provided
14
14
  }
15
15
 
16
- version = "0.2.0"
16
+ version = "0.3.0"
17
17
 
18
18
  sourceCompatibility = 1.7
19
19
  targetCompatibility = 1.7
data/gradlew.bat CHANGED
@@ -1,90 +1,90 @@
1
- @if "%DEBUG%" == "" @echo off
2
- @rem ##########################################################################
3
- @rem
4
- @rem Gradle startup script for Windows
5
- @rem
6
- @rem ##########################################################################
7
-
8
- @rem Set local scope for the variables with windows NT shell
9
- if "%OS%"=="Windows_NT" setlocal
10
-
11
- @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
12
- set DEFAULT_JVM_OPTS=
13
-
14
- set DIRNAME=%~dp0
15
- if "%DIRNAME%" == "" set DIRNAME=.
16
- set APP_BASE_NAME=%~n0
17
- set APP_HOME=%DIRNAME%
18
-
19
- @rem Find java.exe
20
- if defined JAVA_HOME goto findJavaFromJavaHome
21
-
22
- set JAVA_EXE=java.exe
23
- %JAVA_EXE% -version >NUL 2>&1
24
- if "%ERRORLEVEL%" == "0" goto init
25
-
26
- echo.
27
- echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28
- echo.
29
- echo Please set the JAVA_HOME variable in your environment to match the
30
- echo location of your Java installation.
31
-
32
- goto fail
33
-
34
- :findJavaFromJavaHome
35
- set JAVA_HOME=%JAVA_HOME:"=%
36
- set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37
-
38
- if exist "%JAVA_EXE%" goto init
39
-
40
- echo.
41
- echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42
- echo.
43
- echo Please set the JAVA_HOME variable in your environment to match the
44
- echo location of your Java installation.
45
-
46
- goto fail
47
-
48
- :init
49
- @rem Get command-line arguments, handling Windowz variants
50
-
51
- if not "%OS%" == "Windows_NT" goto win9xME_args
52
- if "%@eval[2+2]" == "4" goto 4NT_args
53
-
54
- :win9xME_args
55
- @rem Slurp the command line arguments.
56
- set CMD_LINE_ARGS=
57
- set _SKIP=2
58
-
59
- :win9xME_args_slurp
60
- if "x%~1" == "x" goto execute
61
-
62
- set CMD_LINE_ARGS=%*
63
- goto execute
64
-
65
- :4NT_args
66
- @rem Get arguments from the 4NT Shell from JP Software
67
- set CMD_LINE_ARGS=%$
68
-
69
- :execute
70
- @rem Setup the command line
71
-
72
- set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
73
-
74
- @rem Execute Gradle
75
- "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
76
-
77
- :end
78
- @rem End local scope for the variables with windows NT shell
79
- if "%ERRORLEVEL%"=="0" goto mainEnd
80
-
81
- :fail
82
- rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
83
- rem the _cmd.exe /c_ return code!
84
- if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
85
- exit /b 1
86
-
87
- :mainEnd
88
- if "%OS%"=="Windows_NT" endlocal
89
-
90
- :omega
1
+ @if "%DEBUG%" == "" @echo off
2
+ @rem ##########################################################################
3
+ @rem
4
+ @rem Gradle startup script for Windows
5
+ @rem
6
+ @rem ##########################################################################
7
+
8
+ @rem Set local scope for the variables with windows NT shell
9
+ if "%OS%"=="Windows_NT" setlocal
10
+
11
+ @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
12
+ set DEFAULT_JVM_OPTS=
13
+
14
+ set DIRNAME=%~dp0
15
+ if "%DIRNAME%" == "" set DIRNAME=.
16
+ set APP_BASE_NAME=%~n0
17
+ set APP_HOME=%DIRNAME%
18
+
19
+ @rem Find java.exe
20
+ if defined JAVA_HOME goto findJavaFromJavaHome
21
+
22
+ set JAVA_EXE=java.exe
23
+ %JAVA_EXE% -version >NUL 2>&1
24
+ if "%ERRORLEVEL%" == "0" goto init
25
+
26
+ echo.
27
+ echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28
+ echo.
29
+ echo Please set the JAVA_HOME variable in your environment to match the
30
+ echo location of your Java installation.
31
+
32
+ goto fail
33
+
34
+ :findJavaFromJavaHome
35
+ set JAVA_HOME=%JAVA_HOME:"=%
36
+ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37
+
38
+ if exist "%JAVA_EXE%" goto init
39
+
40
+ echo.
41
+ echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42
+ echo.
43
+ echo Please set the JAVA_HOME variable in your environment to match the
44
+ echo location of your Java installation.
45
+
46
+ goto fail
47
+
48
+ :init
49
+ @rem Get command-line arguments, handling Windowz variants
50
+
51
+ if not "%OS%" == "Windows_NT" goto win9xME_args
52
+ if "%@eval[2+2]" == "4" goto 4NT_args
53
+
54
+ :win9xME_args
55
+ @rem Slurp the command line arguments.
56
+ set CMD_LINE_ARGS=
57
+ set _SKIP=2
58
+
59
+ :win9xME_args_slurp
60
+ if "x%~1" == "x" goto execute
61
+
62
+ set CMD_LINE_ARGS=%*
63
+ goto execute
64
+
65
+ :4NT_args
66
+ @rem Get arguments from the 4NT Shell from JP Software
67
+ set CMD_LINE_ARGS=%$
68
+
69
+ :execute
70
+ @rem Setup the command line
71
+
72
+ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
73
+
74
+ @rem Execute Gradle
75
+ "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
76
+
77
+ :end
78
+ @rem End local scope for the variables with windows NT shell
79
+ if "%ERRORLEVEL%"=="0" goto mainEnd
80
+
81
+ :fail
82
+ rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
83
+ rem the _cmd.exe /c_ return code!
84
+ if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
85
+ exit /b 1
86
+
87
+ :mainEnd
88
+ if "%OS%"=="Windows_NT" endlocal
89
+
90
+ :omega
@@ -10,6 +10,7 @@ import java.util.concurrent.atomic.AtomicLong;
10
10
 
11
11
  import javax.validation.constraints.NotNull;
12
12
 
13
+ import com.amazonaws.services.s3.model.AmazonS3Exception;
13
14
  import org.embulk.config.Config;
14
15
  import org.embulk.config.ConfigDefault;
15
16
  import org.embulk.config.ConfigDiff;
@@ -121,6 +122,7 @@ public class S3PerRecordOutputPlugin
121
122
  private final Column dataColumn;
122
123
  private final Schema schema;
123
124
  private final boolean decodeBase64;
125
+ private final int retryLimit = 2;
124
126
 
125
127
  public S3PerRecordPageOutput(PluginTask task, Schema schema) {
126
128
  this.schema = schema;
@@ -180,17 +182,37 @@ public class S3PerRecordOutputPlugin
180
182
  ObjectMetadata metadata = new ObjectMetadata();
181
183
  metadata.setContentLength(payloadBytes.length);
182
184
 
183
- try (InputStream is = new ByteArrayInputStream(payloadBytes)) {
184
- Upload upload = transferManager.upload(bucket, key, is, metadata);
185
- upload.waitForUploadResult();
186
- long rows = processedRows.incrementAndGet();
187
- if (rows == nextLoggingRowCount) {
188
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
189
- logger.info(String.format("> Uploaded %,d rows in %.2f seconds", rows, seconds));
190
- nextLoggingRowCount *= 2;
185
+ int retryCount = 0;
186
+ int retryWait = 1000; // ms
187
+ while (true) {
188
+ try (InputStream is = new ByteArrayInputStream(payloadBytes)) {
189
+ Upload upload = transferManager.upload(bucket, key, is, metadata);
190
+ upload.waitForUploadResult();
191
+ long rows = processedRows.incrementAndGet();
192
+ if (rows == nextLoggingRowCount) {
193
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
194
+ logger.info(String.format("> Uploaded %,d rows in %.2f seconds", rows, seconds));
195
+ nextLoggingRowCount *= 2;
196
+ }
197
+ break;
198
+ } catch (AmazonS3Exception e) {
199
+ if (!e.isRetryable())
200
+ throw e;
201
+
202
+ if (retryCount > retryLimit)
203
+ throw e;
204
+
205
+ retryCount++;
206
+ logger.warn(String.format("> Upload failed by %s, Retry Uploading in after %d ms (%d of %d)", e.getMessage(), retryWait, retryCount, retryLimit));
207
+ try {
208
+ Thread.sleep(retryWait);
209
+ retryWait = retryWait * 2;
210
+ } catch (InterruptedException e1) {
211
+ Thread.currentThread().interrupt();
212
+ }
213
+ } catch (InterruptedException | IOException e) {
214
+ throw new RuntimeException(e);
191
215
  }
192
- } catch (InterruptedException | IOException e) {
193
- throw new RuntimeException(e);
194
216
  }
195
217
  }
196
218
  }
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-s3_per_record
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - tomykaira
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-05-02 00:00:00.000000000 Z
11
+ date: 2016-06-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: bundler
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - ~>
17
18
  - !ruby/object:Gem::Version
18
19
  version: '1.0'
19
- name: bundler
20
- prerelease: false
21
- type: :development
22
- version_requirements: !ruby/object:Gem::Requirement
20
+ requirement: !ruby/object:Gem::Requirement
23
21
  requirements:
24
22
  - - ~>
25
23
  - !ruby/object:Gem::Version
26
24
  version: '1.0'
25
+ prerelease: false
26
+ type: :development
27
27
  - !ruby/object:Gem::Dependency
28
- requirement: !ruby/object:Gem::Requirement
28
+ name: rake
29
+ version_requirements: !ruby/object:Gem::Requirement
29
30
  requirements:
30
31
  - - '>='
31
32
  - !ruby/object:Gem::Version
32
33
  version: '10.0'
33
- name: rake
34
- prerelease: false
35
- type: :development
36
- version_requirements: !ruby/object:Gem::Requirement
34
+ requirement: !ruby/object:Gem::Requirement
37
35
  requirements:
38
36
  - - '>='
39
37
  - !ruby/object:Gem::Version
40
38
  version: '10.0'
39
+ prerelease: false
40
+ type: :development
41
41
  description: Upload a column's value to S3 as one S3 object per row.
42
42
  email:
43
43
  - tomykaira@gmail.com
@@ -58,15 +58,15 @@ files:
58
58
  - lib/embulk/output/s3_per_record.rb
59
59
  - src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java
60
60
  - src/test/java/org/embulk/output/s3_per_record/TestS3PerRecordOutputPlugin.java
61
- - classpath/jackson-dataformat-cbor-2.5.3.jar
62
- - classpath/httpclient-4.3.6.jar
63
- - classpath/commons-logging-1.1.3.jar
64
- - classpath/commons-codec-1.6.jar
65
- - classpath/aws-java-sdk-s3-1.10.71.jar
66
61
  - classpath/aws-java-sdk-core-1.10.71.jar
67
- - classpath/embulk-output-s3_per_record-0.2.0.jar
68
- - classpath/httpcore-4.3.3.jar
69
62
  - classpath/aws-java-sdk-kms-1.10.71.jar
63
+ - classpath/aws-java-sdk-s3-1.10.71.jar
64
+ - classpath/commons-codec-1.6.jar
65
+ - classpath/commons-logging-1.1.3.jar
66
+ - classpath/embulk-output-s3_per_record-0.3.0.jar
67
+ - classpath/httpclient-4.3.6.jar
68
+ - classpath/httpcore-4.3.3.jar
69
+ - classpath/jackson-dataformat-cbor-2.5.3.jar
70
70
  homepage: https://github.com/tomykaira/embulk-output-s3_per_record
71
71
  licenses:
72
72
  - MIT