embulk-output-azure_blob_storage 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6f0517ecae2588f48dd8266de574626f89534a7c
4
- data.tar.gz: 2487729f05d378c29cf76b81a8a921d1710df3c6
3
+ metadata.gz: 45c7205b0ee914070c15b48038ec9afd82134136
4
+ data.tar.gz: 1173936e3234bd114085b28545f1fc6df1e1cec3
5
5
  SHA512:
6
- metadata.gz: 0bcd72b765c0d90581a38ae5b6b095c3eaa18e3adf5819469f3da9eb61d899c01401b68be87e01c4d1ec0940d200bbd5a602e56ad05b94e1ce7b0d72594d6907
7
- data.tar.gz: 6677c17408925c38d2ac0896686fae03fcfdcb89a7777f1fb4f33d26de9b1d9321c6462fde2bda05a7e7c51e1088c35f2ade0c1b4ef5004cb8773ede29aefde1
6
+ metadata.gz: 8d19051e9154907145766a16215242996ca51abf3a968d2cfea6861c278f5bcfa6730d9e12fccaa9b97d90d7542bf08a75a8f408889b47c5e7f913c087d26f6f
7
+ data.tar.gz: db41bfb30ec06404c9d3d6691f0d2b12cb080f227ca37b2559db10dc4fa0dd2544474b0da7ff65756164c5d65adda6a14889bbd0a54f14280ea128ee4837d65f
data/CHANGELOG.md CHANGED
@@ -1,14 +1,19 @@
1
- ## 0.1.4 - 2015-03-20
1
+ ## 0.1.5 - 2016-04-06
2
+
3
+ * [maintenance] Refactoring codes [#8](https://github.com/sakama/embulk-output-azure_blob_storage/pull/8)
4
+
5
+
6
+ ## 0.1.4 - 2016-03-20
2
7
 
3
8
  * [maintenance] Use RetryExecutor when retry is needed [#7](https://github.com/sakama/embulk-output-azure_blob_storage/pull/7)
4
9
 
5
- ## 0.1.3 - 2015-03-15
10
+ ## 0.1.3 - 2016-03-15
6
11
 
7
12
  * [maintenance] Add retry logic using expotential backoff [#6](https://github.com/sakama/embulk-output-azure_blob_storage/pull/6)
8
13
  * [maintenance] Change local file output path to temporary directory
9
14
  [#5](https://github.com/sakama/embulk-output-azure_blob_storage/pull/5)
10
15
 
11
- ## 0.1.2 - 2015-02-02
16
+ ## 0.1.2 - 2016-02-02
12
17
 
13
18
  * [maintenance] Upgrade embulk version to v0.8.2 [#3](https://github.com/sakama/embulk-output-azure_blob_storage/pull/3)
14
19
 
data/build.gradle CHANGED
@@ -17,7 +17,7 @@ configurations {
17
17
  sourceCompatibility = 1.7
18
18
  targetCompatibility = 1.7
19
19
 
20
- version = "0.1.4"
20
+ version = "0.1.5"
21
21
 
22
22
  dependencies {
23
23
  compile "org.embulk:embulk-core:0.8.2"
@@ -69,7 +69,7 @@ Gem::Specification.new do |spec|
69
69
  spec.description = %[Stores files on Microsoft Azure Blob Storage.]
70
70
  spec.email = ["satoshiakama@gmail.com"]
71
71
  spec.licenses = ["Apache-2.0"]
72
- spec.homepage = "https://github.com/sakama/embulk-output-azure_blob_storage"
72
+ spec.homepage = "https://github.com/embulk/embulk-output-azure_blob_storage"
73
73
 
74
74
  spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
75
75
  spec.test_files = spec.files.grep(%r"^(test|spec)/")
@@ -23,6 +23,7 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
23
23
  import org.slf4j.Logger;
24
24
  import static org.embulk.spi.util.RetryExecutor.retryExecutor;
25
25
 
26
+ import java.io.BufferedInputStream;
26
27
  import java.io.BufferedOutputStream;
27
28
  import java.io.File;
28
29
  import java.io.FileInputStream;
@@ -60,7 +61,7 @@ public class AzureBlobStorageFileOutputPlugin
60
61
  String getSequenceFormat();
61
62
 
62
63
  @Config("max_connection_retry")
63
- @ConfigDefault("10") // 5 times retry to connect sftp server if failed.
64
+ @ConfigDefault("10") // 10 times retry to connect Azure Blob Storage if failed.
64
65
  int getMaxConnectionRetry();
65
66
  }
66
67
 
@@ -77,12 +78,12 @@ public class AzureBlobStorageFileOutputPlugin
77
78
  String containerName = task.getContainer();
78
79
  CloudBlobContainer container = blobClient.getContainerReference(containerName);
79
80
  if (!container.exists()) {
80
- log.info("container {} doesn't exists and created.", containerName);
81
+ log.info("container {} doesn't exist and is created.", containerName);
81
82
  container.createIfNotExists();
82
83
  }
83
84
  }
84
- catch (StorageException | URISyntaxException | ConfigException ex) {
85
- Throwables.propagate(ex);
85
+ catch (StorageException | URISyntaxException ex) {
86
+ throw new ConfigException(ex);
86
87
  }
87
88
 
88
89
  return resume(task.dump(), taskCount, control);
@@ -121,37 +122,33 @@ public class AzureBlobStorageFileOutputPlugin
121
122
  public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
122
123
  {
123
124
  final PluginTask task = taskSource.loadTask(PluginTask.class);
124
- return new AzureFileOutput(task, taskIndex);
125
+ CloudBlobClient client = newAzureClient(task.getAccountName(), task.getAccountKey());
126
+ return new AzureFileOutput(client, task, taskIndex);
125
127
  }
126
128
 
127
129
  public static class AzureFileOutput implements TransactionalFileOutput
128
130
  {
131
+ private final CloudBlobClient client;
132
+ private final String containerName;
129
133
  private final String pathPrefix;
130
134
  private final String sequenceFormat;
131
135
  private final String pathSuffix;
132
- private final CloudBlobClient client;
133
136
  private final int maxConnectionRetry;
134
- private CloudBlobContainer container = null;
135
137
  private BufferedOutputStream output = null;
136
138
  private int fileIndex;
137
139
  private File file;
138
140
  private String filePath;
139
141
  private int taskIndex;
140
142
 
141
- public AzureFileOutput(PluginTask task, int taskIndex)
143
+ public AzureFileOutput(CloudBlobClient client, PluginTask task, int taskIndex)
142
144
  {
145
+ this.client = client;
146
+ this.containerName = task.getContainer();
143
147
  this.taskIndex = taskIndex;
144
148
  this.pathPrefix = task.getPathPrefix();
145
149
  this.sequenceFormat = task.getSequenceFormat();
146
150
  this.pathSuffix = task.getFileNameExtension();
147
- this.client = newAzureClient(task.getAccountName(), task.getAccountKey());
148
151
  this.maxConnectionRetry = task.getMaxConnectionRetry();
149
- try {
150
- this.container = client.getContainerReference(task.getContainer());
151
- }
152
- catch (URISyntaxException | StorageException ex) {
153
- Throwables.propagate(ex);
154
- }
155
152
  }
156
153
 
157
154
  @Override
@@ -220,9 +217,10 @@ public class AzureBlobStorageFileOutputPlugin
220
217
  @Override
221
218
  public Void call() throws StorageException, URISyntaxException, IOException, RetryGiveupException
222
219
  {
220
+ CloudBlobContainer container = client.getContainerReference(containerName);
223
221
  CloudBlockBlob blob = container.getBlockBlobReference(filePath);
224
222
  log.info("Upload start {} to {}", file.getAbsolutePath(), filePath);
225
- blob.upload(new FileInputStream(file), file.length());
223
+ blob.upload(new BufferedInputStream(new FileInputStream(file)), file.length());
226
224
  log.info("Upload completed {} to {}", file.getAbsolutePath(), filePath);
227
225
  log.info("Delete completed local file {}", file.getAbsolutePath());
228
226
  if (!file.delete()) {
@@ -241,8 +239,7 @@ public class AzureBlobStorageFileOutputPlugin
241
239
  public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
242
240
  throws RetryGiveupException
243
241
  {
244
- Class clazz = exception.getClass();
245
- if (clazz.equals(FileNotFoundException.class) || clazz.equals(URISyntaxException.class) || clazz.equals(ConfigException.class)) {
242
+ if (exception instanceof FileNotFoundException || exception instanceof URISyntaxException || exception instanceof ConfigException) {
246
243
  throw new RetryGiveupException(exception);
247
244
  }
248
245
  String message = String.format("Azure Blob Storage put request failed. Retrying %d/%d after %d seconds. Message: %s",
@@ -270,14 +270,9 @@ public class TestAzureBlobStorageFileOutputPlugin
270
270
  maxConnectionRetry.setAccessible(true);
271
271
  maxConnectionRetry.set(output, 1);
272
272
 
273
- // set non-existing-container for Test
274
- Method method = AzureBlobStorageFileOutputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
275
- method.setAccessible(true);
276
- CloudBlobClient client = (CloudBlobClient) method.invoke(plugin, AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY);
277
-
278
- Field container = AzureBlobStorageFileOutputPlugin.AzureFileOutput.class.getDeclaredField("container");
273
+ Field container = AzureBlobStorageFileOutputPlugin.AzureFileOutput.class.getDeclaredField("containerName");
279
274
  container.setAccessible(true);
280
- container.set(output, client.getContainerReference("non-exists-container"));
275
+ container.set(output, "non-existing-container");
281
276
  output.finish();
282
277
  }
283
278
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-azure_blob_storage
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-30 00:00:00.000000000 Z
11
+ date: 2016-04-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -63,9 +63,9 @@ files:
63
63
  - src/test/resources/sample_02.csv
64
64
  - classpath/azure-storage-4.0.0.jar
65
65
  - classpath/commons-lang3-3.4.jar
66
- - classpath/embulk-output-azure_blob_storage-0.1.4.jar
66
+ - classpath/embulk-output-azure_blob_storage-0.1.5.jar
67
67
  - classpath/jackson-core-2.6.0.jar
68
- homepage: https://github.com/sakama/embulk-output-azure_blob_storage
68
+ homepage: https://github.com/embulk/embulk-output-azure_blob_storage
69
69
  licenses:
70
70
  - Apache-2.0
71
71
  metadata: {}