embulk-input-azure_blob_storage 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 023f5fb334a7684e46155d3a3538168ca6c10d6c
4
- data.tar.gz: 2a7ed41ce1b0f56bca0a3134add627295d5d359d
3
+ metadata.gz: 69d9a2cbec36f6021ed35909a0d3e056858510f0
4
+ data.tar.gz: 4db321c8e905485c2f65d77d1dd61173c7160c81
5
5
  SHA512:
6
- metadata.gz: fa524db4147ff03b7fe7ae509c8096c0cb1b191239a2959b9e969217ec48398bb552e588fe2b05f4f3610cbab0934dd87b14eae288f3ca0ff445959134e7879a
7
- data.tar.gz: af78a9bc48198e812795189134bcaf449fd246d7ee90fc2f6e9ac9e70bfa1ddbf4192dd9f7863545b22a522591de91e0f67881a9a2dee65d2fed947d498f25b1
6
+ metadata.gz: 85e0487c28e5a01e60cbcd9f16eee326b828fbd5ffe945389c91c63f95e783bc9d30c4703ecaf88b86687c01a5aba8db4bc8e224054ef1063c74599e19d9a67c
7
+ data.tar.gz: 9a7b078e40011a203893619b8a17576f281db783937b110c01884d500680dd7711d0e42b46e2493ad224381f2974f83db81e6e341c8799b7b366d133106c22b8
@@ -1,5 +1,7 @@
1
1
  language: java
2
2
 
3
+ dist: trusty
4
+
3
5
  jdk:
4
6
  - oraclejdk8
5
7
 
@@ -1,3 +1,6 @@
1
+ ## 0.3.0 - 2020-02-17
2
+ * [maintenance] Enhance the incremental process to skip file in the last_path to avoid duplication [#21](https://github.com/embulk/embulk-input-azure_blob_storage/pull/21)
3
+
1
4
  ## 0.2.0 - 2018-12-26
2
5
  * [new feature] Use file name output feature in Embulk core to show file name instead of logging [#17](https://github.com/sakama/embulk-input-azure_blob_storage/pull/17)
3
6
 
@@ -17,7 +17,7 @@ configurations {
17
17
  sourceCompatibility = 1.8
18
18
  targetCompatibility = 1.8
19
19
 
20
- version = "0.2.0"
20
+ version = "0.3.0"
21
21
 
22
22
  dependencies {
23
23
  compile "org.embulk:embulk-core:0.9.12"
@@ -82,6 +82,8 @@ public class AzureBlobStorageFileInputPlugin
82
82
 
83
83
  private static final Logger log = Exec.getLogger(AzureBlobStorageFileInputPlugin.class);
84
84
 
85
+ private static boolean IS_REMOVE_FIRST_RECORD = true;
86
+
85
87
  @Override
86
88
  public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
87
89
  {
@@ -167,6 +169,10 @@ public class AzureBlobStorageFileInputPlugin
167
169
  do {
168
170
  blobs = container.listBlobsSegmented(prefix, true, null, maxResults, token, null, null);
169
171
  log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength(), blobs.getContinuationToken()));
172
+ if (lastKey != null && !blobs.getResults().isEmpty() && IS_REMOVE_FIRST_RECORD) {
173
+ log.info("Remove first item " + blobs.getResults().remove(0).getStorageUri().getPrimaryUri());
174
+ IS_REMOVE_FIRST_RECORD = false;
175
+ }
170
176
  for (ListBlobItem blobItem : blobs.getResults()) {
171
177
  if (blobItem instanceof CloudBlob) {
172
178
  CloudBlob blob = (CloudBlob) blobItem;
@@ -210,6 +210,29 @@ public class TestAzureBlobStorageFileInputPlugin
210
210
  assertRecords(config, output);
211
211
  }
212
212
 
213
+ @Test
214
+ public void testAzureIncrementalWithoutDuplicate()
215
+ throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
216
+ {
217
+ PluginTask task = config.loadConfig(PluginTask.class);
218
+ ConfigDiff configDiff = runner.transaction(config, new Control());
219
+
220
+ Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", PluginTask.class);
221
+ listFiles.setAccessible(true);
222
+ task.setFiles((FileList) listFiles.invoke(plugin, task));
223
+
224
+ assertRecords(config, output);
225
+
226
+ output.pages.clear();
227
+
228
+ config.set("last_path", configDiff.get(String.class, "last_path"));
229
+ task = config.loadConfig(PluginTask.class);
230
+ runner.transaction(config, new Control());
231
+ task.setFiles((FileList) listFiles.invoke(plugin, task));
232
+ List<Object[]> records = getRecords(config, output);
233
+ assertEquals(0, records.size());
234
+ }
235
+
213
236
  @Test
214
237
  public void testCreateNextToken() throws Exception
215
238
  {
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-azure_blob_storage
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-12-26 00:00:00.000000000 Z
11
+ date: 2020-02-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -65,9 +65,9 @@ files:
65
65
  - src/test/resources/sample_01.csv
66
66
  - src/test/resources/sample_02.csv
67
67
  - classpath/azure-storage-8.0.0.jar
68
- - classpath/embulk-input-azure_blob_storage-0.2.0.jar
69
68
  - classpath/guava-20.0.jar
70
69
  - classpath/azure-keyvault-core-1.0.0.jar
70
+ - classpath/embulk-input-azure_blob_storage-0.3.0.jar
71
71
  - classpath/jackson-core-2.9.4.jar
72
72
  homepage: https://github.com/embulk/embulk-input-azure_blob_storage
73
73
  licenses: