embulk-output-azure_blob_storage 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +5 -5
- data/build.gradle +3 -3
- data/classpath/embulk-output-azure_blob_storage-0.1.4.jar +0 -0
- data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java +66 -41
- data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java +12 -3
- metadata +5 -5
- data/classpath/embulk-output-azure_blob_storage-0.1.3.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6f0517ecae2588f48dd8266de574626f89534a7c
|
4
|
+
data.tar.gz: 2487729f05d378c29cf76b81a8a921d1710df3c6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0bcd72b765c0d90581a38ae5b6b095c3eaa18e3adf5819469f3da9eb61d899c01401b68be87e01c4d1ec0940d200bbd5a602e56ad05b94e1ce7b0d72594d6907
|
7
|
+
data.tar.gz: 6677c17408925c38d2ac0896686fae03fcfdcb89a7777f1fb4f33d26de9b1d9321c6462fde2bda05a7e7c51e1088c35f2ade0c1b4ef5004cb8773ede29aefde1
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.1.4 - 2015-03-20
|
2
|
+
|
3
|
+
* [maintenance] Use RetryExecutor when retry is needed [#7](https://github.com/sakama/embulk-output-azure_blob_storage/pull/7)
|
4
|
+
|
1
5
|
## 0.1.3 - 2015-03-15
|
2
6
|
|
3
7
|
* [maintenance] Add retry logic using expotential backoff [#6](https://github.com/sakama/embulk-output-azure_blob_storage/pull/6)
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
# Azure
|
1
|
+
# Azure Blob Storage file output plugin for Embulk
|
2
2
|
|
3
|
-
[Embulk](http://www.embulk.org/) file output plugin stores files on [Microsoft Azure](https://azure.microsoft.com/) [
|
3
|
+
[Embulk](http://www.embulk.org/) file output plugin stores files on [Microsoft Azure](https://azure.microsoft.com/) [Blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
|
4
4
|
|
5
5
|
|
6
6
|
## Overview
|
@@ -22,10 +22,10 @@ First, create Azure [Storage Account](https://azure.microsoft.com/en-us/document
|
|
22
22
|
|
23
23
|
### Auto create container
|
24
24
|
|
25
|
-
container will create automatically when container
|
25
|
+
container will create automatically when container doesn't exists.
|
26
26
|
|
27
|
-
When a container
|
28
|
-
It's a [service specification](https://technet.microsoft.com/en-us/library/dd179408.aspx#Anchor_3) of Azure
|
27
|
+
When a container was deleted, a container with same name cannot be created for at least 30 seconds.
|
28
|
+
It's a [service specification](https://technet.microsoft.com/en-us/library/dd179408.aspx#Anchor_3) of Azure Blob Storage.
|
29
29
|
|
30
30
|
## Example
|
31
31
|
|
data/build.gradle
CHANGED
@@ -17,7 +17,7 @@ configurations {
|
|
17
17
|
sourceCompatibility = 1.7
|
18
18
|
targetCompatibility = 1.7
|
19
19
|
|
20
|
-
version = "0.1.
|
20
|
+
version = "0.1.4"
|
21
21
|
|
22
22
|
dependencies {
|
23
23
|
compile "org.embulk:embulk-core:0.8.2"
|
@@ -65,8 +65,8 @@ Gem::Specification.new do |spec|
|
|
65
65
|
spec.name = "${project.name}"
|
66
66
|
spec.version = "${project.version}"
|
67
67
|
spec.authors = ["Satoshi Akama"]
|
68
|
-
spec.summary = %[Microsoft Azure
|
69
|
-
spec.description = %[Stores files on Microsoft Azure
|
68
|
+
spec.summary = %[Microsoft Azure Blob Storage file output plugin for Embulk]
|
69
|
+
spec.description = %[Stores files on Microsoft Azure Blob Storage.]
|
70
70
|
spec.email = ["satoshiakama@gmail.com"]
|
71
71
|
spec.licenses = ["Apache-2.0"]
|
72
72
|
spec.homepage = "https://github.com/sakama/embulk-output-azure_blob_storage"
|
Binary file
|
data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java
CHANGED
@@ -18,8 +18,10 @@ import org.embulk.spi.Buffer;
|
|
18
18
|
import org.embulk.spi.Exec;
|
19
19
|
import org.embulk.spi.FileOutputPlugin;
|
20
20
|
import org.embulk.spi.TransactionalFileOutput;
|
21
|
-
|
21
|
+
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
22
|
+
import org.embulk.spi.util.RetryExecutor.Retryable;
|
22
23
|
import org.slf4j.Logger;
|
24
|
+
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
23
25
|
|
24
26
|
import java.io.BufferedOutputStream;
|
25
27
|
import java.io.File;
|
@@ -58,7 +60,7 @@ public class AzureBlobStorageFileOutputPlugin
|
|
58
60
|
String getSequenceFormat();
|
59
61
|
|
60
62
|
@Config("max_connection_retry")
|
61
|
-
@ConfigDefault("
|
63
|
+
@ConfigDefault("10") // 5 times retry to connect sftp server if failed.
|
62
64
|
int getMaxConnectionRetry();
|
63
65
|
}
|
64
66
|
|
@@ -203,48 +205,71 @@ public class AzureBlobStorageFileOutputPlugin
|
|
203
205
|
public void finish()
|
204
206
|
{
|
205
207
|
close();
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
}
|
208
|
+
uploadFile();
|
209
|
+
}
|
210
|
+
|
211
|
+
private Void uploadFile()
|
212
|
+
{
|
212
213
|
if (filePath != null) {
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
214
|
+
try {
|
215
|
+
return retryExecutor()
|
216
|
+
.withRetryLimit(maxConnectionRetry)
|
217
|
+
.withInitialRetryWait(500)
|
218
|
+
.withMaxRetryWait(30 * 1000)
|
219
|
+
.runInterruptible(new Retryable<Void>() {
|
220
|
+
@Override
|
221
|
+
public Void call() throws StorageException, URISyntaxException, IOException, RetryGiveupException
|
222
|
+
{
|
223
|
+
CloudBlockBlob blob = container.getBlockBlobReference(filePath);
|
224
|
+
log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
|
225
|
+
blob.upload(new FileInputStream(file), file.length());
|
226
|
+
log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
|
227
|
+
log.info("Delete completed local file {}", file.getAbsolutePath());
|
228
|
+
if (!file.delete()) {
|
229
|
+
throw new ConfigException("Couldn't delete local file " + file.getAbsolutePath());
|
230
|
+
}
|
231
|
+
return null;
|
232
|
+
}
|
233
|
+
|
234
|
+
@Override
|
235
|
+
public boolean isRetryableException(Exception exception)
|
236
|
+
{
|
237
|
+
return true;
|
238
|
+
}
|
239
|
+
|
240
|
+
@Override
|
241
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
242
|
+
throws RetryGiveupException
|
243
|
+
{
|
244
|
+
Class clazz = exception.getClass();
|
245
|
+
if (clazz.equals(FileNotFoundException.class) || clazz.equals(URISyntaxException.class) || clazz.equals(ConfigException.class)) {
|
246
|
+
throw new RetryGiveupException(exception);
|
247
|
+
}
|
248
|
+
String message = String.format("Azure Blob Storage put request failed. Retrying %d/%d after %d seconds. Message: %s",
|
249
|
+
retryCount, retryLimit, retryWait / 1000, exception.getMessage());
|
250
|
+
if (retryCount % 3 == 0) {
|
251
|
+
log.warn(message, exception);
|
252
|
+
}
|
253
|
+
else {
|
254
|
+
log.warn(message);
|
255
|
+
}
|
256
|
+
}
|
257
|
+
|
258
|
+
@Override
|
259
|
+
public void onGiveup(Exception firstException, Exception lastException)
|
260
|
+
throws RetryGiveupException
|
261
|
+
{
|
262
|
+
}
|
263
|
+
});
|
264
|
+
}
|
265
|
+
catch (RetryGiveupException ex) {
|
266
|
+
throw Throwables.propagate(ex.getCause());
|
267
|
+
}
|
268
|
+
catch (InterruptedException ex) {
|
269
|
+
throw Throwables.propagate(ex);
|
246
270
|
}
|
247
271
|
}
|
272
|
+
return null;
|
248
273
|
}
|
249
274
|
|
250
275
|
@Override
|
data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java
CHANGED
@@ -9,7 +9,6 @@ import com.microsoft.azure.storage.blob.CloudBlobClient;
|
|
9
9
|
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
10
10
|
import org.embulk.EmbulkTestRuntime;
|
11
11
|
import org.embulk.config.ConfigDiff;
|
12
|
-
import org.embulk.config.ConfigException;
|
13
12
|
import org.embulk.config.ConfigSource;
|
14
13
|
import org.embulk.config.TaskReport;
|
15
14
|
import org.embulk.config.TaskSource;
|
@@ -34,6 +33,7 @@ import java.io.BufferedReader;
|
|
34
33
|
import java.io.ByteArrayOutputStream;
|
35
34
|
import java.io.File;
|
36
35
|
import java.io.FileInputStream;
|
36
|
+
import java.io.FileNotFoundException;
|
37
37
|
import java.io.IOException;
|
38
38
|
import java.io.InputStream;
|
39
39
|
import java.io.InputStreamReader;
|
@@ -100,7 +100,11 @@ public class TestAzureBlobStorageFileOutputPlugin
|
|
100
100
|
.set("formatter", formatterConfig());
|
101
101
|
|
102
102
|
PluginTask task = config.loadConfig(PluginTask.class);
|
103
|
+
|
103
104
|
assertEquals(AZURE_ACCOUNT_NAME, task.getAccountName());
|
105
|
+
assertEquals(AZURE_ACCOUNT_KEY, task.getAccountKey());
|
106
|
+
assertEquals(AZURE_CONTAINER, task.getContainer());
|
107
|
+
assertEquals(10, task.getMaxConnectionRetry());
|
104
108
|
}
|
105
109
|
|
106
110
|
@Test
|
@@ -214,7 +218,7 @@ public class TestAzureBlobStorageFileOutputPlugin
|
|
214
218
|
assertRecords(remotePath);
|
215
219
|
}
|
216
220
|
|
217
|
-
@Test
|
221
|
+
@Test
|
218
222
|
public void testAzureFileOutputByOpenWithNonReadableFile() throws Exception
|
219
223
|
{
|
220
224
|
ConfigSource configSource = config();
|
@@ -236,7 +240,12 @@ public class TestAzureBlobStorageFileOutputPlugin
|
|
236
240
|
File file = File.createTempFile("non-readable-file", ".tmp");
|
237
241
|
file.setReadable(false);
|
238
242
|
field.set(output, file);
|
239
|
-
|
243
|
+
try {
|
244
|
+
output.finish();
|
245
|
+
}
|
246
|
+
catch (Exception ex) {
|
247
|
+
assertEquals(FileNotFoundException.class, ex.getCause().getClass());
|
248
|
+
}
|
240
249
|
}
|
241
250
|
|
242
251
|
@Test(expected = RuntimeException.class)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-azure_blob_storage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-03-
|
11
|
+
date: 2016-03-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -38,7 +38,7 @@ dependencies:
|
|
38
38
|
- - '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '10.0'
|
41
|
-
description: Stores files on Microsoft Azure
|
41
|
+
description: Stores files on Microsoft Azure Blob Storage.
|
42
42
|
email:
|
43
43
|
- satoshiakama@gmail.com
|
44
44
|
executables: []
|
@@ -63,7 +63,7 @@ files:
|
|
63
63
|
- src/test/resources/sample_02.csv
|
64
64
|
- classpath/azure-storage-4.0.0.jar
|
65
65
|
- classpath/commons-lang3-3.4.jar
|
66
|
-
- classpath/embulk-output-azure_blob_storage-0.1.
|
66
|
+
- classpath/embulk-output-azure_blob_storage-0.1.4.jar
|
67
67
|
- classpath/jackson-core-2.6.0.jar
|
68
68
|
homepage: https://github.com/sakama/embulk-output-azure_blob_storage
|
69
69
|
licenses:
|
@@ -88,5 +88,5 @@ rubyforge_project:
|
|
88
88
|
rubygems_version: 2.1.9
|
89
89
|
signing_key:
|
90
90
|
specification_version: 4
|
91
|
-
summary: Microsoft Azure
|
91
|
+
summary: Microsoft Azure Blob Storage file output plugin for Embulk
|
92
92
|
test_files: []
|
Binary file
|