embulk-input-azure_blob_storage 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8a820c0389e246171f6f0ccd9ff535ae4be3b56
-  data.tar.gz: 9a31f3ba0a98659a4745923cadb485b87507ad60
+  metadata.gz: 5f42f0f0c779cd135beaa548cb3d3f4a50dc1bbc
+  data.tar.gz: f6c5aac931cf45ec5ba5e0d662c2c8e0b3b2918a
 SHA512:
-  metadata.gz: d636e2392ad6c94b8d218e03ac84e9b1e95e29229bd30af937b9e0cd85a247f37588bea12c4ca336ea6d992ac89812a2a0d0834e087f3b3a342e92d897f3032e
-  data.tar.gz: aa4c141450109ad1dca6d04c1c9f7dfade3f471dbd5f821b5b87220971ea12387fa6278c4265622e70640d0ed2fbe699653b98146a1ca3093ce1fc7d92e9a04d
+  metadata.gz: 66e6b34d630431264a158b83115da874ffebb12b02e8e020e27bc915e31fbf3b283cb9aa5812fad51c933568ce96f5a8b6234e0a711b50d10f4f3c4ee014e411
+  data.tar.gz: 67286da3771c3c7a264ed9b9f28233fc46e3fed181a2b9e3bfe260621fec13d0c9197a64121c17befbbb401bcba6e1b80ce5c12d313c7ae37533318033bcbd5f

data/.travis.yml

@@ -0,0 +1,18 @@
+language: java
+
+jdk:
+  - oraclejdk8
+  - oraclejdk7
+  - openjdk7
+
+env:
+  global:
+    - AZURE_ACCOUNT_NAME=sakamatest
+    - AZURE_CONTAINER=blobcontainers
+    - AZURE_CONTAINER_IMPORT_DIRECTORY=unittests_import
+    - secure: "kcQtK4pbq9ldVFOQXRiPd+ieVokGfPWfc45bZu9iAags68sFhUHxs0WrT4Io2RZvYSIQ7t5OHYwF8TUqgecaMPmoQmNkUHozgVoJ1oBu9BwA2ceh+u7/pkWPJ0VsAPW2+LE2uMC67MrvayeIxV3wuCCDdBHd2F64Ja8AE7p8Re8vrrDvD8BVF6+OqHma0S8i7PCZE+4rYeJpITbAAQ8PqMa8P/6Wkbp0LSJCAiDhwc/89wGn8wlZkju/aUGz0tG89Yq3Iw+xonhsPhLi910rg31vIMEH/K14I/uiUOaxavvxzc37fHAJgMzhQnRN67qhVuX5eocMosDAVAsXByURYAGZ4/f4sLqsghGkh4f+b96AchiGvM62hbwPyn1BMBl6fOezap5LOMAKjSZBReHNN2Hau+7YXDhqv2aVxVLWwXHDJQ8KhgMn10EZ+W3U5KuNLaGrfcrPfWIDxcR3rU81SXkzAyoHri895xlXq+CxsWOlYsXpWwpX14hNIJzcqtq9KQt+U34Xyw/whBsrSZbCitTZvPiTC9pFvCXBddpz07KqaUeUYwneGbVdCjpq42OO5bybSOAHEI8U7Ay7ldEFs0mRq8HzEOrhwZIdAgODsujQwV1jFgz6yKNwKTE6jx8UOqUQKalQjiAt0he4Pz8BjH+uxmRRvALNjF0ZWOMl5UU="
+
+script:
+  - ./gradlew gem
+  - ./gradlew --info check jacocoTestReport
+

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+
+## 0.1.7 - 2016-08-19
+* [new feature] Support `incremental` option [#13](https://github.com/sakama/embulk-input-azure_blob_storage/pull/13)
+
 ## 0.1.6 - 2016-04-01
 
 * [maintenance] Small refactoring [#9](https://github.com/sakama/embulk-input-azure_blob_storage/pull/9)

data/README.md

@@ -1,4 +1,5 @@
 # Azure Blob Storage file input plugin for Embulk
+[![Build Status](https://travis-ci.org/embulk/embulk-input-azure_blob_storage.svg?branch=master)](https://travis-ci.org/embulk/embulk-input-azure_blob_storage)
 
 [Embulk](http://www.embulk.org/) file input plugin read files stored on [Microsoft Azure](https://azure.microsoft.com/) [Blob Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/#blob-storage)
 
@@ -16,6 +17,7 @@ First, create Azure [Storage Account](https://azure.microsoft.com/en-us/document
 - **account_key**: primary access key (string, required)
 - **container**: container name data stored (string, required)
 - **path_prefix**: prefix of target keys (string, required) (string, required)
+- **incremental**: enables incremental loading(boolean, optional. default: true). If incremental loading is enabled, config diff for the next execution will include `last_path` parameter so that next execution skips files before the path. Otherwise, `last_path` will not be included.
 - **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)
 - **total_file_count_limit**: maximum number of files to read (integer, optional)
 

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.6"
+version = "0.1.7"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"
@@ -69,7 +69,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[Reads files stored on Microsoft Azure Blob Storage.]
   spec.email         = ["satoshiakama@gmail.com"]
   spec.licenses      = ["Apache-2.0"]
-  spec.homepage      = "https://github.com/sakama/embulk-input-azure_blob_storage"
+  spec.homepage      = "https://github.com/embulk/embulk-input-azure_blob_storage"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java

@@ -57,6 +57,10 @@ public class AzureBlobStorageFileInputPlugin
         @Config("path_prefix")
         String getPathPrefix();
 
+        @Config("incremental")
+        @ConfigDefault("true")
+        boolean getIncremental();
+
         @Config("last_path")
         @ConfigDefault("null")
         Optional<String> getLastPath();
@@ -95,7 +99,9 @@ public class AzureBlobStorageFileInputPlugin
         control.run(taskSource, taskCount);
 
         ConfigDiff configDiff = Exec.newConfigDiff();
-        configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
+        if (task.getIncremental()) {
+            configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
+        }
 
         return configDiff;
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-01 00:00:00.000000000 Z
+date: 2016-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - CHANGELOG.md
 - README.md
 - build.gradle
@@ -64,9 +65,9 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/azure-storage-4.0.0.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-input-azure_blob_storage-0.1.6.jar
+- classpath/embulk-input-azure_blob_storage-0.1.7.jar
 - classpath/jackson-core-2.6.0.jar
-homepage: https://github.com/sakama/embulk-input-azure_blob_storage
+homepage: https://github.com/embulk/embulk-input-azure_blob_storage
 licenses:
 - Apache-2.0
 metadata: {}