embulk-output-azure_blob_storage 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 45c7205b0ee914070c15b48038ec9afd82134136
|
4
|
+
data.tar.gz: 1173936e3234bd114085b28545f1fc6df1e1cec3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8d19051e9154907145766a16215242996ca51abf3a968d2cfea6861c278f5bcfa6730d9e12fccaa9b97d90d7542bf08a75a8f408889b47c5e7f913c087d26f6f
|
7
|
+
data.tar.gz: db41bfb30ec06404c9d3d6691f0d2b12cb080f227ca37b2559db10dc4fa0dd2544474b0da7ff65756164c5d65adda6a14889bbd0a54f14280ea128ee4837d65f
|
data/CHANGELOG.md
CHANGED
@@ -1,14 +1,19 @@
|
|
1
|
-
## 0.1.
|
1
|
+
## 0.1.5 - 2016-04-06
|
2
|
+
|
3
|
+
* [maintenance] Refactoring codes [#8](https://github.com/sakama/embulk-output-azure_blob_storage/pull/8)
|
4
|
+
|
5
|
+
|
6
|
+
## 0.1.4 - 2016-03-20
|
2
7
|
|
3
8
|
* [maintenance] Use RetryExecutor when retry is needed [#7](https://github.com/sakama/embulk-output-azure_blob_storage/pull/7)
|
4
9
|
|
5
|
-
## 0.1.3 -
|
10
|
+
## 0.1.3 - 2016-03-15
|
6
11
|
|
7
12
|
* [maintenance] Add retry logic using expotential backoff [#6](https://github.com/sakama/embulk-output-azure_blob_storage/pull/6)
|
8
13
|
* [maintenance] Change local file output path to temporary directory
|
9
14
|
[#5](https://github.com/sakama/embulk-output-azure_blob_storage/pull/5)
|
10
15
|
|
11
|
-
## 0.1.2 -
|
16
|
+
## 0.1.2 - 2016-02-02
|
12
17
|
|
13
18
|
* [maintenance] Upgrade embulk version to v0.8.2 [#3](https://github.com/sakama/embulk-output-azure_blob_storage/pull/3)
|
14
19
|
|
data/build.gradle
CHANGED
@@ -17,7 +17,7 @@ configurations {
|
|
17
17
|
sourceCompatibility = 1.7
|
18
18
|
targetCompatibility = 1.7
|
19
19
|
|
20
|
-
version = "0.1.
|
20
|
+
version = "0.1.5"
|
21
21
|
|
22
22
|
dependencies {
|
23
23
|
compile "org.embulk:embulk-core:0.8.2"
|
@@ -69,7 +69,7 @@ Gem::Specification.new do |spec|
|
|
69
69
|
spec.description = %[Stores files on Microsoft Azure Blob Storage.]
|
70
70
|
spec.email = ["satoshiakama@gmail.com"]
|
71
71
|
spec.licenses = ["Apache-2.0"]
|
72
|
-
spec.homepage = "https://github.com/
|
72
|
+
spec.homepage = "https://github.com/embulk/embulk-output-azure_blob_storage"
|
73
73
|
|
74
74
|
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
75
75
|
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java
CHANGED
@@ -23,6 +23,7 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
|
|
23
23
|
import org.slf4j.Logger;
|
24
24
|
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
25
25
|
|
26
|
+
import java.io.BufferedInputStream;
|
26
27
|
import java.io.BufferedOutputStream;
|
27
28
|
import java.io.File;
|
28
29
|
import java.io.FileInputStream;
|
@@ -60,7 +61,7 @@ public class AzureBlobStorageFileOutputPlugin
|
|
60
61
|
String getSequenceFormat();
|
61
62
|
|
62
63
|
@Config("max_connection_retry")
|
63
|
-
@ConfigDefault("10") //
|
64
|
+
@ConfigDefault("10") // 10 times retry to connect Azure Blob Storage if failed.
|
64
65
|
int getMaxConnectionRetry();
|
65
66
|
}
|
66
67
|
|
@@ -77,12 +78,12 @@ public class AzureBlobStorageFileOutputPlugin
|
|
77
78
|
String containerName = task.getContainer();
|
78
79
|
CloudBlobContainer container = blobClient.getContainerReference(containerName);
|
79
80
|
if (!container.exists()) {
|
80
|
-
log.info("container {} doesn't
|
81
|
+
log.info("container {} doesn't exist and is created.", containerName);
|
81
82
|
container.createIfNotExists();
|
82
83
|
}
|
83
84
|
}
|
84
|
-
catch (StorageException | URISyntaxException
|
85
|
-
|
85
|
+
catch (StorageException | URISyntaxException ex) {
|
86
|
+
throw new ConfigException(ex);
|
86
87
|
}
|
87
88
|
|
88
89
|
return resume(task.dump(), taskCount, control);
|
@@ -121,37 +122,33 @@ public class AzureBlobStorageFileOutputPlugin
|
|
121
122
|
public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
|
122
123
|
{
|
123
124
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
124
|
-
|
125
|
+
CloudBlobClient client = newAzureClient(task.getAccountName(), task.getAccountKey());
|
126
|
+
return new AzureFileOutput(client, task, taskIndex);
|
125
127
|
}
|
126
128
|
|
127
129
|
public static class AzureFileOutput implements TransactionalFileOutput
|
128
130
|
{
|
131
|
+
private final CloudBlobClient client;
|
132
|
+
private final String containerName;
|
129
133
|
private final String pathPrefix;
|
130
134
|
private final String sequenceFormat;
|
131
135
|
private final String pathSuffix;
|
132
|
-
private final CloudBlobClient client;
|
133
136
|
private final int maxConnectionRetry;
|
134
|
-
private CloudBlobContainer container = null;
|
135
137
|
private BufferedOutputStream output = null;
|
136
138
|
private int fileIndex;
|
137
139
|
private File file;
|
138
140
|
private String filePath;
|
139
141
|
private int taskIndex;
|
140
142
|
|
141
|
-
public AzureFileOutput(PluginTask task, int taskIndex)
|
143
|
+
public AzureFileOutput(CloudBlobClient client, PluginTask task, int taskIndex)
|
142
144
|
{
|
145
|
+
this.client = client;
|
146
|
+
this.containerName = task.getContainer();
|
143
147
|
this.taskIndex = taskIndex;
|
144
148
|
this.pathPrefix = task.getPathPrefix();
|
145
149
|
this.sequenceFormat = task.getSequenceFormat();
|
146
150
|
this.pathSuffix = task.getFileNameExtension();
|
147
|
-
this.client = newAzureClient(task.getAccountName(), task.getAccountKey());
|
148
151
|
this.maxConnectionRetry = task.getMaxConnectionRetry();
|
149
|
-
try {
|
150
|
-
this.container = client.getContainerReference(task.getContainer());
|
151
|
-
}
|
152
|
-
catch (URISyntaxException | StorageException ex) {
|
153
|
-
Throwables.propagate(ex);
|
154
|
-
}
|
155
152
|
}
|
156
153
|
|
157
154
|
@Override
|
@@ -220,9 +217,10 @@ public class AzureBlobStorageFileOutputPlugin
|
|
220
217
|
@Override
|
221
218
|
public Void call() throws StorageException, URISyntaxException, IOException, RetryGiveupException
|
222
219
|
{
|
220
|
+
CloudBlobContainer container = client.getContainerReference(containerName);
|
223
221
|
CloudBlockBlob blob = container.getBlockBlobReference(filePath);
|
224
222
|
log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
|
225
|
-
blob.upload(new FileInputStream(file), file.length());
|
223
|
+
blob.upload(new BufferedInputStream(new FileInputStream(file)), file.length());
|
226
224
|
log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
|
227
225
|
log.info("Delete completed local file {}", file.getAbsolutePath());
|
228
226
|
if (!file.delete()) {
|
@@ -241,8 +239,7 @@ public class AzureBlobStorageFileOutputPlugin
|
|
241
239
|
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
242
240
|
throws RetryGiveupException
|
243
241
|
{
|
244
|
-
|
245
|
-
if (clazz.equals(FileNotFoundException.class) || clazz.equals(URISyntaxException.class) || clazz.equals(ConfigException.class)) {
|
242
|
+
if (exception instanceof FileNotFoundException || exception instanceof URISyntaxException || exception instanceof ConfigException) {
|
246
243
|
throw new RetryGiveupException(exception);
|
247
244
|
}
|
248
245
|
String message = String.format("Azure Blob Storage put request failed. Retrying %d/%d after %d seconds. Message: %s",
|
data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java
CHANGED
@@ -270,14 +270,9 @@ public class TestAzureBlobStorageFileOutputPlugin
|
|
270
270
|
maxConnectionRetry.setAccessible(true);
|
271
271
|
maxConnectionRetry.set(output, 1);
|
272
272
|
|
273
|
-
|
274
|
-
Method method = AzureBlobStorageFileOutputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
|
275
|
-
method.setAccessible(true);
|
276
|
-
CloudBlobClient client = (CloudBlobClient) method.invoke(plugin, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY);
|
277
|
-
|
278
|
-
Field container = AzureBlobStorageFileOutputPlugin.AzureFileOutput.class.getDeclaredField("container");
|
273
|
+
Field container = AzureBlobStorageFileOutputPlugin.AzureFileOutput.class.getDeclaredField("containerName");
|
279
274
|
container.setAccessible(true);
|
280
|
-
container.set(output,
|
275
|
+
container.set(output, "non-existing-container");
|
281
276
|
output.finish();
|
282
277
|
}
|
283
278
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-azure_blob_storage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-04-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -63,9 +63,9 @@ files:
|
|
63
63
|
- src/test/resources/sample_02.csv
|
64
64
|
- classpath/azure-storage-4.0.0.jar
|
65
65
|
- classpath/commons-lang3-3.4.jar
|
66
|
-
- classpath/embulk-output-azure_blob_storage-0.1.
|
66
|
+
- classpath/embulk-output-azure_blob_storage-0.1.5.jar
|
67
67
|
- classpath/jackson-core-2.6.0.jar
|
68
|
-
homepage: https://github.com/
|
68
|
+
homepage: https://github.com/embulk/embulk-output-azure_blob_storage
|
69
69
|
licenses:
|
70
70
|
- Apache-2.0
|
71
71
|
metadata: {}
|