embulk-input-azure_blob_storage 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -0
- data/CHANGELOG.md +3 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java +6 -0
- data/src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java +23 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 69d9a2cbec36f6021ed35909a0d3e056858510f0
|
4
|
+
data.tar.gz: 4db321c8e905485c2f65d77d1dd61173c7160c81
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85e0487c28e5a01e60cbcd9f16eee326b828fbd5ffe945389c91c63f95e783bc9d30c4703ecaf88b86687c01a5aba8db4bc8e224054ef1063c74599e19d9a67c
|
7
|
+
data.tar.gz: 9a7b078e40011a203893619b8a17576f281db783937b110c01884d500680dd7711d0e42b46e2493ad224381f2974f83db81e6e341c8799b7b366d133106c22b8
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 0.3.0 - 2020-02-17
|
2
|
+
* [maintenance] Enhance the incremental process to skip file in the last_path to avoid duplication [#21](https://github.com/embulk/embulk-input-azure_blob_storage/pull/21)
|
3
|
+
|
1
4
|
## 0.2.0 - 2018-12-26
|
2
5
|
* [new feature] Use file name output feature in Embulk core to show file name instead of logging [#17](https://github.com/sakama/embulk-input-azure_blob_storage/pull/17)
|
3
6
|
|
data/build.gradle
CHANGED
@@ -82,6 +82,8 @@ public class AzureBlobStorageFileInputPlugin
|
|
82
82
|
|
83
83
|
private static final Logger log = Exec.getLogger(AzureBlobStorageFileInputPlugin.class);
|
84
84
|
|
85
|
+
private static boolean IS_REMOVE_FIRST_RECORD = true;
|
86
|
+
|
85
87
|
@Override
|
86
88
|
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
|
87
89
|
{
|
@@ -167,6 +169,10 @@ public class AzureBlobStorageFileInputPlugin
|
|
167
169
|
do {
|
168
170
|
blobs = container.listBlobsSegmented(prefix, true, null, maxResults, token, null, null);
|
169
171
|
log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength(), blobs.getContinuationToken()));
|
172
|
+
if (lastKey != null && !blobs.getResults().isEmpty() && IS_REMOVE_FIRST_RECORD) {
|
173
|
+
log.info("Remove first item " + blobs.getResults().remove(0).getStorageUri().getPrimaryUri());
|
174
|
+
IS_REMOVE_FIRST_RECORD = false;
|
175
|
+
}
|
170
176
|
for (ListBlobItem blobItem : blobs.getResults()) {
|
171
177
|
if (blobItem instanceof CloudBlob) {
|
172
178
|
CloudBlob blob = (CloudBlob) blobItem;
|
data/src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java
CHANGED
@@ -210,6 +210,29 @@ public class TestAzureBlobStorageFileInputPlugin
|
|
210
210
|
assertRecords(config, output);
|
211
211
|
}
|
212
212
|
|
213
|
+
@Test
|
214
|
+
public void testAzureIncrementalWithoutDuplicate()
|
215
|
+
throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
|
216
|
+
{
|
217
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
218
|
+
ConfigDiff configDiff = runner.transaction(config, new Control());
|
219
|
+
|
220
|
+
Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", PluginTask.class);
|
221
|
+
listFiles.setAccessible(true);
|
222
|
+
task.setFiles((FileList) listFiles.invoke(plugin, task));
|
223
|
+
|
224
|
+
assertRecords(config, output);
|
225
|
+
|
226
|
+
output.pages.clear();
|
227
|
+
|
228
|
+
config.set("last_path", configDiff.get(String.class, "last_path"));
|
229
|
+
task = config.loadConfig(PluginTask.class);
|
230
|
+
runner.transaction(config, new Control());
|
231
|
+
task.setFiles((FileList) listFiles.invoke(plugin, task));
|
232
|
+
List<Object[]> records = getRecords(config, output);
|
233
|
+
assertEquals(0, records.size());
|
234
|
+
}
|
235
|
+
|
213
236
|
@Test
|
214
237
|
public void testCreateNextToken() throws Exception
|
215
238
|
{
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-azure_blob_storage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-02-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -65,9 +65,9 @@ files:
|
|
65
65
|
- src/test/resources/sample_01.csv
|
66
66
|
- src/test/resources/sample_02.csv
|
67
67
|
- classpath/azure-storage-8.0.0.jar
|
68
|
-
- classpath/embulk-input-azure_blob_storage-0.2.0.jar
|
69
68
|
- classpath/guava-20.0.jar
|
70
69
|
- classpath/azure-keyvault-core-1.0.0.jar
|
70
|
+
- classpath/embulk-input-azure_blob_storage-0.3.0.jar
|
71
71
|
- classpath/jackson-core-2.9.4.jar
|
72
72
|
homepage: https://github.com/embulk/embulk-input-azure_blob_storage
|
73
73
|
licenses:
|