embulk-output-azure_blob_storage 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 5cc1a4b0540a0db2bb640ffff57daef127b63eec
4
- data.tar.gz: 09115e5d83c014bac1b3d8d19b12fb8adad3a4be
3
+ metadata.gz: 6f0517ecae2588f48dd8266de574626f89534a7c
4
+ data.tar.gz: 2487729f05d378c29cf76b81a8a921d1710df3c6
5
5
  SHA512:
6
- metadata.gz: 6b167e589e571d77ed3b33552505090aa58bdc1c4470fd36fc66bbc90b8dca8129a0b2f031b1832640a2b0154855fd207a2100477f26ef34fec7ac6a8c4ee818
7
- data.tar.gz: 9aedb87b4cdb6750ded5ef47108fd9574392f3a244ac11c051c287595c04195abd54ccbccb0c84f92a961e0aa48f8f015a6bfd94110252865dc437238e2515f5
6
+ metadata.gz: 0bcd72b765c0d90581a38ae5b6b095c3eaa18e3adf5819469f3da9eb61d899c01401b68be87e01c4d1ec0940d200bbd5a602e56ad05b94e1ce7b0d72594d6907
7
+ data.tar.gz: 6677c17408925c38d2ac0896686fae03fcfdcb89a7777f1fb4f33d26de9b1d9321c6462fde2bda05a7e7c51e1088c35f2ade0c1b4ef5004cb8773ede29aefde1
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 0.1.4 - 2015-03-20
2
+
3
+ * [maintenance] Use RetryExecutor when retry is needed [#7](https://github.com/sakama/embulk-output-azure_blob_storage/pull/7)
4
+
1
5
  ## 0.1.3 - 2015-03-15
2
6
 
3
7
  * [maintenance] Add retry logic using expotential backoff [#6](https://github.com/sakama/embulk-output-azure_blob_storage/pull/6)
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
- # Azure blob Storage file output plugin for Embulk
1
+ # Azure Blob Storage file output plugin for Embulk
2
2
 
3
- [Embulk](http://www.embulk.org/) file output plugin stores files on [Microsoft Azure](https://azure.microsoft.com/) [blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
3
+ [Embulk](http://www.embulk.org/) file output plugin stores files on [Microsoft Azure](https://azure.microsoft.com/) [Blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
4
4
 
5
5
 
6
6
  ## Overview
@@ -22,10 +22,10 @@ First, create Azure [Storage Account](https://azure.microsoft.com/en-us/document
22
22
 
23
23
  ### Auto create container
24
24
 
25
- container will create automatically when container isn't exists.
25
+ container will create automatically when container doesn't exists.
26
26
 
27
- When a container is deleted, a container with the same name cannot be created for at least 30 seconds.
28
- It's a [service specification](https://technet.microsoft.com/en-us/library/dd179408.aspx#Anchor_3) of Azure blob Storage.
27
+ When a container was deleted, a container with same name cannot be created for at least 30 seconds.
28
+ It's a [service specification](https://technet.microsoft.com/en-us/library/dd179408.aspx#Anchor_3) of Azure Blob Storage.
29
29
 
30
30
  ## Example
31
31
 
data/build.gradle CHANGED
@@ -17,7 +17,7 @@ configurations {
17
17
  sourceCompatibility = 1.7
18
18
  targetCompatibility = 1.7
19
19
 
20
- version = "0.1.3"
20
+ version = "0.1.4"
21
21
 
22
22
  dependencies {
23
23
  compile "org.embulk:embulk-core:0.8.2"
@@ -65,8 +65,8 @@ Gem::Specification.new do |spec|
65
65
  spec.name = "${project.name}"
66
66
  spec.version = "${project.version}"
67
67
  spec.authors = ["Satoshi Akama"]
68
- spec.summary = %[Microsoft Azure blob Storage file output plugin for Embulk]
69
- spec.description = %[Stores files on Microsoft Azure blob Storage.]
68
+ spec.summary = %[Microsoft Azure Blob Storage file output plugin for Embulk]
69
+ spec.description = %[Stores files on Microsoft Azure Blob Storage.]
70
70
  spec.email = ["satoshiakama@gmail.com"]
71
71
  spec.licenses = ["Apache-2.0"]
72
72
  spec.homepage = "https://github.com/sakama/embulk-output-azure_blob_storage"
@@ -18,8 +18,10 @@ import org.embulk.spi.Buffer;
18
18
  import org.embulk.spi.Exec;
19
19
  import org.embulk.spi.FileOutputPlugin;
20
20
  import org.embulk.spi.TransactionalFileOutput;
21
-
21
+ import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
22
+ import org.embulk.spi.util.RetryExecutor.Retryable;
22
23
  import org.slf4j.Logger;
24
+ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
23
25
 
24
26
  import java.io.BufferedOutputStream;
25
27
  import java.io.File;
@@ -58,7 +60,7 @@ public class AzureBlobStorageFileOutputPlugin
58
60
  String getSequenceFormat();
59
61
 
60
62
  @Config("max_connection_retry")
61
- @ConfigDefault("5") // 5 times retry to connect sftp server if failed.
63
+ @ConfigDefault("10") // 5 times retry to connect sftp server if failed.
62
64
  int getMaxConnectionRetry();
63
65
  }
64
66
 
@@ -203,48 +205,71 @@ public class AzureBlobStorageFileOutputPlugin
203
205
  public void finish()
204
206
  {
205
207
  close();
206
- try {
207
- Thread.sleep(1000 * 10);
208
- }
209
- catch (Exception ex) {
210
- // null;
211
- }
208
+ uploadFile();
209
+ }
210
+
211
+ private Void uploadFile()
212
+ {
212
213
  if (filePath != null) {
213
- int count = 0;
214
- while (true) {
215
- try {
216
- CloudBlockBlob blob = container.getBlockBlobReference(filePath);
217
- log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
218
- blob.upload(new FileInputStream(file), file.length());
219
- log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
220
- log.info("Delete completed local file {}", file.getAbsolutePath());
221
- if (!file.delete()) {
222
- throw new ConfigException("Couldn't delete local file " + file.getAbsolutePath());
223
- }
224
- break;
225
- }
226
- catch (FileNotFoundException | URISyntaxException ex) {
227
- throw new ConfigException(ex);
228
- }
229
- catch (StorageException | IOException ex) {
230
- if (++count == maxConnectionRetry) {
231
- Throwables.propagate(ex);
232
- }
233
- log.warn("failed to connect SFTP server: " + ex.getMessage(), ex);
234
-
235
- try {
236
- long sleepTime = ((long) Math.pow(2, count) * 1000);
237
- log.warn("sleep in next connection retry: {} milliseconds", sleepTime);
238
- Thread.sleep(sleepTime); // milliseconds
239
- }
240
- catch (InterruptedException ex2) {
241
- // Ignore this exception because this exception is just about `sleep`.
242
- log.warn(ex2.getMessage(), ex2);
243
- }
244
- log.warn("retrying to connect SFTP server: " + count + " times");
245
- }
214
+ try {
215
+ return retryExecutor()
216
+ .withRetryLimit(maxConnectionRetry)
217
+ .withInitialRetryWait(500)
218
+ .withMaxRetryWait(30 * 1000)
219
+ .runInterruptible(new Retryable<Void>() {
220
+ @Override
221
+ public Void call() throws StorageException, URISyntaxException, IOException, RetryGiveupException
222
+ {
223
+ CloudBlockBlob blob = container.getBlockBlobReference(filePath);
224
+ log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
225
+ blob.upload(new FileInputStream(file), file.length());
226
+ log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
227
+ log.info("Delete completed local file {}", file.getAbsolutePath());
228
+ if (!file.delete()) {
229
+ throw new ConfigException("Couldn't delete local file " + file.getAbsolutePath());
230
+ }
231
+ return null;
232
+ }
233
+
234
+ @Override
235
+ public boolean isRetryableException(Exception exception)
236
+ {
237
+ return true;
238
+ }
239
+
240
+ @Override
241
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
242
+ throws RetryGiveupException
243
+ {
244
+ Class clazz = exception.getClass();
245
+ if (clazz.equals(FileNotFoundException.class) || clazz.equals(URISyntaxException.class) || clazz.equals(ConfigException.class)) {
246
+ throw new RetryGiveupException(exception);
247
+ }
248
+ String message = String.format("Azure Blob Storage put request failed. Retrying %d/%d after %d seconds. Message: %s",
249
+ retryCount, retryLimit, retryWait / 1000, exception.getMessage());
250
+ if (retryCount % 3 == 0) {
251
+ log.warn(message, exception);
252
+ }
253
+ else {
254
+ log.warn(message);
255
+ }
256
+ }
257
+
258
+ @Override
259
+ public void onGiveup(Exception firstException, Exception lastException)
260
+ throws RetryGiveupException
261
+ {
262
+ }
263
+ });
264
+ }
265
+ catch (RetryGiveupException ex) {
266
+ throw Throwables.propagate(ex.getCause());
267
+ }
268
+ catch (InterruptedException ex) {
269
+ throw Throwables.propagate(ex);
246
270
  }
247
271
  }
272
+ return null;
248
273
  }
249
274
 
250
275
  @Override
@@ -9,7 +9,6 @@ import com.microsoft.azure.storage.blob.CloudBlobClient;
9
9
  import com.microsoft.azure.storage.blob.CloudBlobContainer;
10
10
  import org.embulk.EmbulkTestRuntime;
11
11
  import org.embulk.config.ConfigDiff;
12
- import org.embulk.config.ConfigException;
13
12
  import org.embulk.config.ConfigSource;
14
13
  import org.embulk.config.TaskReport;
15
14
  import org.embulk.config.TaskSource;
@@ -34,6 +33,7 @@ import java.io.BufferedReader;
34
33
  import java.io.ByteArrayOutputStream;
35
34
  import java.io.File;
36
35
  import java.io.FileInputStream;
36
+ import java.io.FileNotFoundException;
37
37
  import java.io.IOException;
38
38
  import java.io.InputStream;
39
39
  import java.io.InputStreamReader;
@@ -100,7 +100,11 @@ public class TestAzureBlobStorageFileOutputPlugin
100
100
  .set("formatter", formatterConfig());
101
101
 
102
102
  PluginTask task = config.loadConfig(PluginTask.class);
103
+
103
104
  assertEquals(AZURE_ACCOUNT_NAME, task.getAccountName());
105
+ assertEquals(AZURE_ACCOUNT_KEY, task.getAccountKey());
106
+ assertEquals(AZURE_CONTAINER, task.getContainer());
107
+ assertEquals(10, task.getMaxConnectionRetry());
104
108
  }
105
109
 
106
110
  @Test
@@ -214,7 +218,7 @@ public class TestAzureBlobStorageFileOutputPlugin
214
218
  assertRecords(remotePath);
215
219
  }
216
220
 
217
- @Test(expected = ConfigException.class)
221
+ @Test
218
222
  public void testAzureFileOutputByOpenWithNonReadableFile() throws Exception
219
223
  {
220
224
  ConfigSource configSource = config();
@@ -236,7 +240,12 @@ public class TestAzureBlobStorageFileOutputPlugin
236
240
  File file = File.createTempFile("non-readable-file", ".tmp");
237
241
  file.setReadable(false);
238
242
  field.set(output, file);
239
- output.finish();
243
+ try {
244
+ output.finish();
245
+ }
246
+ catch (Exception ex) {
247
+ assertEquals(FileNotFoundException.class, ex.getCause().getClass());
248
+ }
240
249
  }
241
250
 
242
251
  @Test(expected = RuntimeException.class)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-azure_blob_storage
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-14 00:00:00.000000000 Z
11
+ date: 2016-03-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
38
38
  - - '>='
39
39
  - !ruby/object:Gem::Version
40
40
  version: '10.0'
41
- description: Stores files on Microsoft Azure blob Storage.
41
+ description: Stores files on Microsoft Azure Blob Storage.
42
42
  email:
43
43
  - satoshiakama@gmail.com
44
44
  executables: []
@@ -63,7 +63,7 @@ files:
63
63
  - src/test/resources/sample_02.csv
64
64
  - classpath/azure-storage-4.0.0.jar
65
65
  - classpath/commons-lang3-3.4.jar
66
- - classpath/embulk-output-azure_blob_storage-0.1.3.jar
66
+ - classpath/embulk-output-azure_blob_storage-0.1.4.jar
67
67
  - classpath/jackson-core-2.6.0.jar
68
68
  homepage: https://github.com/sakama/embulk-output-azure_blob_storage
69
69
  licenses:
@@ -88,5 +88,5 @@ rubyforge_project:
88
88
  rubygems_version: 2.1.9
89
89
  signing_key:
90
90
  specification_version: 4
91
- summary: Microsoft Azure blob Storage file output plugin for Embulk
91
+ summary: Microsoft Azure Blob Storage file output plugin for Embulk
92
92
  test_files: []