embulk-input-azure_blob_storage 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 023f5fb334a7684e46155d3a3538168ca6c10d6c
4
- data.tar.gz: 2a7ed41ce1b0f56bca0a3134add627295d5d359d
3
+ metadata.gz: 69d9a2cbec36f6021ed35909a0d3e056858510f0
4
+ data.tar.gz: 4db321c8e905485c2f65d77d1dd61173c7160c81
5
5
  SHA512:
6
- metadata.gz: fa524db4147ff03b7fe7ae509c8096c0cb1b191239a2959b9e969217ec48398bb552e588fe2b05f4f3610cbab0934dd87b14eae288f3ca0ff445959134e7879a
7
- data.tar.gz: af78a9bc48198e812795189134bcaf449fd246d7ee90fc2f6e9ac9e70bfa1ddbf4192dd9f7863545b22a522591de91e0f67881a9a2dee65d2fed947d498f25b1
6
+ metadata.gz: 85e0487c28e5a01e60cbcd9f16eee326b828fbd5ffe945389c91c63f95e783bc9d30c4703ecaf88b86687c01a5aba8db4bc8e224054ef1063c74599e19d9a67c
7
+ data.tar.gz: 9a7b078e40011a203893619b8a17576f281db783937b110c01884d500680dd7711d0e42b46e2493ad224381f2974f83db81e6e341c8799b7b366d133106c22b8
@@ -1,5 +1,7 @@
1
1
  language: java
2
2
 
3
+ dist: trusty
4
+
3
5
  jdk:
4
6
  - oraclejdk8
5
7
 
@@ -1,3 +1,6 @@
1
+ ## 0.3.0 - 2020-02-17
2
+ * [maintenance] Enhance the incremental process to skip file in the last_path to avoid duplication [#21](https://github.com/embulk/embulk-input-azure_blob_storage/pull/21)
3
+
1
4
  ## 0.2.0 - 2018-12-26
2
5
  * [new feature] Use file name output feature in Embulk core to show file name instead of logging [#17](https://github.com/sakama/embulk-input-azure_blob_storage/pull/17)
3
6
 
@@ -17,7 +17,7 @@ configurations {
17
17
  sourceCompatibility = 1.8
18
18
  targetCompatibility = 1.8
19
19
 
20
- version = "0.2.0"
20
+ version = "0.3.0"
21
21
 
22
22
  dependencies {
23
23
  compile "org.embulk:embulk-core:0.9.12"
@@ -82,6 +82,8 @@ public class AzureBlobStorageFileInputPlugin
82
82
 
83
83
  private static final Logger log = Exec.getLogger(AzureBlobStorageFileInputPlugin.class);
84
84
 
85
+ private static boolean IS_REMOVE_FIRST_RECORD = true;
86
+
85
87
  @Override
86
88
  public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
87
89
  {
@@ -167,6 +169,10 @@ public class AzureBlobStorageFileInputPlugin
167
169
  do {
168
170
  blobs = container.listBlobsSegmented(prefix, true, null, maxResults, token, null, null);
169
171
  log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength(), blobs.getContinuationToken()));
172
+ if (lastKey != null && !blobs.getResults().isEmpty() && IS_REMOVE_FIRST_RECORD) {
173
+ log.info("Remove first item " + blobs.getResults().remove(0).getStorageUri().getPrimaryUri());
174
+ IS_REMOVE_FIRST_RECORD = false;
175
+ }
170
176
  for (ListBlobItem blobItem : blobs.getResults()) {
171
177
  if (blobItem instanceof CloudBlob) {
172
178
  CloudBlob blob = (CloudBlob) blobItem;
@@ -210,6 +210,29 @@ public class TestAzureBlobStorageFileInputPlugin
210
210
  assertRecords(config, output);
211
211
  }
212
212
 
213
+ @Test
214
+ public void testAzureIncrementalWithoutDuplicate()
215
+ throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
216
+ {
217
+ PluginTask task = config.loadConfig(PluginTask.class);
218
+ ConfigDiff configDiff = runner.transaction(config, new Control());
219
+
220
+ Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", PluginTask.class);
221
+ listFiles.setAccessible(true);
222
+ task.setFiles((FileList) listFiles.invoke(plugin, task));
223
+
224
+ assertRecords(config, output);
225
+
226
+ output.pages.clear();
227
+
228
+ config.set("last_path", configDiff.get(String.class, "last_path"));
229
+ task = config.loadConfig(PluginTask.class);
230
+ runner.transaction(config, new Control());
231
+ task.setFiles((FileList) listFiles.invoke(plugin, task));
232
+ List<Object[]> records = getRecords(config, output);
233
+ assertEquals(0, records.size());
234
+ }
235
+
213
236
  @Test
214
237
  public void testCreateNextToken() throws Exception
215
238
  {
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-azure_blob_storage
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-12-26 00:00:00.000000000 Z
11
+ date: 2020-02-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -65,9 +65,9 @@ files:
65
65
  - src/test/resources/sample_01.csv
66
66
  - src/test/resources/sample_02.csv
67
67
  - classpath/azure-storage-8.0.0.jar
68
- - classpath/embulk-input-azure_blob_storage-0.2.0.jar
69
68
  - classpath/guava-20.0.jar
70
69
  - classpath/azure-keyvault-core-1.0.0.jar
70
+ - classpath/embulk-input-azure_blob_storage-0.3.0.jar
71
71
  - classpath/jackson-core-2.9.4.jar
72
72
  homepage: https://github.com/embulk/embulk-input-azure_blob_storage
73
73
  licenses: