embulk-input-azure_blob_storage 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 023f5fb334a7684e46155d3a3538168ca6c10d6c
-  data.tar.gz: 2a7ed41ce1b0f56bca0a3134add627295d5d359d
+  metadata.gz: 69d9a2cbec36f6021ed35909a0d3e056858510f0
+  data.tar.gz: 4db321c8e905485c2f65d77d1dd61173c7160c81
 SHA512:
-  metadata.gz: fa524db4147ff03b7fe7ae509c8096c0cb1b191239a2959b9e969217ec48398bb552e588fe2b05f4f3610cbab0934dd87b14eae288f3ca0ff445959134e7879a
-  data.tar.gz: af78a9bc48198e812795189134bcaf449fd246d7ee90fc2f6e9ac9e70bfa1ddbf4192dd9f7863545b22a522591de91e0f67881a9a2dee65d2fed947d498f25b1
+  metadata.gz: 85e0487c28e5a01e60cbcd9f16eee326b828fbd5ffe945389c91c63f95e783bc9d30c4703ecaf88b86687c01a5aba8db4bc8e224054ef1063c74599e19d9a67c
+  data.tar.gz: 9a7b078e40011a203893619b8a17576f281db783937b110c01884d500680dd7711d0e42b46e2493ad224381f2974f83db81e6e341c8799b7b366d133106c22b8

data/.travis.yml

@@ -1,5 +1,7 @@
 language: java
 
+dist: trusty
+
 jdk:
   - oraclejdk8
 

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.0 - 2020-02-17
+* [maintenance] Enhance the incremental process to skip file in the last_path to avoid duplication [#21](https://github.com/embulk/embulk-input-azure_blob_storage/pull/21)
+
 ## 0.2.0 - 2018-12-26
 * [new feature] Use file name output feature in Embulk core to show file name instead of logging [#17](https://github.com/sakama/embulk-input-azure_blob_storage/pull/17)
 

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-version = "0.2.0"
+version = "0.3.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.9.12"

data/src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java

@@ -82,6 +82,8 @@ public class AzureBlobStorageFileInputPlugin
 
     private static final Logger log = Exec.getLogger(AzureBlobStorageFileInputPlugin.class);
 
+    private static boolean IS_REMOVE_FIRST_RECORD = true;
+
     @Override
     public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
     {
@@ -167,6 +169,10 @@ public class AzureBlobStorageFileInputPlugin
                             do {
                                 blobs = container.listBlobsSegmented(prefix, true, null, maxResults, token, null, null);
                                 log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength(), blobs.getContinuationToken()));
+                                if (lastKey != null && !blobs.getResults().isEmpty() && IS_REMOVE_FIRST_RECORD) {
+                                    log.info("Remove first item " + blobs.getResults().remove(0).getStorageUri().getPrimaryUri());
+                                    IS_REMOVE_FIRST_RECORD = false;
+                                }
                                 for (ListBlobItem blobItem : blobs.getResults()) {
                                     if (blobItem instanceof CloudBlob) {
                                         CloudBlob blob = (CloudBlob) blobItem;

data/src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java

@@ -210,6 +210,29 @@ public class TestAzureBlobStorageFileInputPlugin
         assertRecords(config, output);
     }
 
+    @Test
+    public void testAzureIncrementalWithoutDuplicate()
+            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigDiff configDiff = runner.transaction(config, new Control());
+
+        Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", PluginTask.class);
+        listFiles.setAccessible(true);
+        task.setFiles((FileList) listFiles.invoke(plugin, task));
+
+        assertRecords(config, output);
+
+        output.pages.clear();
+
+        config.set("last_path", configDiff.get(String.class, "last_path"));
+        task = config.loadConfig(PluginTask.class);
+        runner.transaction(config, new Control());
+        task.setFiles((FileList) listFiles.invoke(plugin, task));
+        List<Object[]> records = getRecords(config, output);
+        assertEquals(0, records.size());
+    }
+
     @Test
     public void testCreateNextToken() throws Exception
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-26 00:00:00.000000000 Z
+date: 2020-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,9 +65,9 @@ files:
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
 - classpath/azure-storage-8.0.0.jar
-- classpath/embulk-input-azure_blob_storage-0.2.0.jar
 - classpath/guava-20.0.jar
 - classpath/azure-keyvault-core-1.0.0.jar
+- classpath/embulk-input-azure_blob_storage-0.3.0.jar
 - classpath/jackson-core-2.9.4.jar
 homepage: https://github.com/embulk/embulk-input-azure_blob_storage
 licenses: