embulk-input-s3 0.2.7 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fde7e23b02972cc6b615c1845ab820c3e960770
-  data.tar.gz: 7ab94c1a292321bbee5c4642be906c1404db35d2
+  metadata.gz: 4f8bc775d296e8ae5cfcaffe8d335f1f09de52cc
+  data.tar.gz: ddfb843558dc281243bc6925f9f5abfefe72f4bc
 SHA512:
-  metadata.gz: ed3cfe60161e427fdffe19b0622feae7aa1e5138d2c3f10243125eb82349ee91efbda869ab21644fb7141125415c29172e36c77f20cf352128c85855222a8fc6
-  data.tar.gz: 7235b4b49d0f7df056098084c70d8dea60be33dbe0335b589eac8e0478a835514caa4a82ab1ee1b5a45370a2b0353985b2f6b66a01ca9ce123202417917fcbbf
+  metadata.gz: 757deefb26f6d1ef1ca859d87d0f7aba7b347bfb3a98b845af112e0c786ca4c68775c52fad9dd79d4013e212f1975cbe6f507800dca9bbf04c1a2e02ed87f302
+  data.tar.gz: 435a0194c28b19152c631cc78bf616e3ee1a50aa701b8f913723ac45c5b0517968f516db0cd5f841822215524214bf6916cac36d071be944e01cefb64dcca7a7

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -65,6 +65,10 @@ public abstract class AbstractS3FileInputPlugin
         @ConfigDefault("null")
         public Optional<String> getAccessKeyId();
 
+        @Config("incremental")
+        @ConfigDefault("true")
+        public boolean getIncremental();
+
         // TODO timeout, ssl, etc
 
         public FileList getFiles();
@@ -104,7 +108,9 @@ public abstract class AbstractS3FileInputPlugin
         ConfigDiff configDiff = Exec.newConfigDiff();
 
         // last_path
-        configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
+        if (task.getIncremental()) {
+            configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
+        }
 
         return configDiff;
     }

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assume.assumeNotNull;
 
@@ -91,6 +92,15 @@ public class TestS3FileInputPlugin
         assertEquals(0, getRecords(config, output).size());
     }
 
+    @Test
+    public void useIncremental()
+    {
+        ConfigSource config = this.config.deepCopy().set("incremental", false);
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
+
+        assertFalse(configDiff.has("last_path"));
+    }
+
     @Test
     public void emptyFilesWithLastPath()
             throws Exception

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.8
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-21 00:00:00.000000000 Z
+date: 2016-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.0
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
 description: Reads files stored on Amazon S3
 email:
 - frsyuki@gmail.com
@@ -61,7 +61,7 @@ files:
 - classpath/aws-java-sdk-kms-1.10.33.jar
 - classpath/aws-java-sdk-s3-1.10.33.jar
 - classpath/commons-codec-1.6.jar
-- classpath/embulk-input-s3-0.2.7.jar
+- classpath/embulk-input-s3-0.2.8.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/jcl-over-slf4j-1.7.12.jar