embulk-input-s3 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 089138d39186ee5b3be9c5568b5fa406ab1950b8
-  data.tar.gz: d1ffa739756982425d9d57a8c7f967f99c2c478e
+  metadata.gz: 83d9c51fe52f61880b83a5d97238ad16858eca69
+  data.tar.gz: 11b3a0a14d2606f93bb3dff5b3c469d75471dbaa
 SHA512:
-  metadata.gz: f3063c5fba94ea815a858a099e4ba6d9b945aaa9664be0864f23534c352c8fea597b28bc21fa81901a1557b1cd12c0eea6eecf033621738c4487bc2903fb5b28
-  data.tar.gz: f9c80a61ab30392a56cac8730656760575110c82148d96c8054e45a35db2b031648bd43d702c49079a05e5e0369158c0f3f792313032539582221493b6c8861a
+  metadata.gz: 34cd105f626902545a8250f2e6a5bd48e8d3b5c898e9df8112e333d6a6e51ea69bfa8d081175e6f07b1a9dc8d37b71ab6c728ea21c0188b9d1110fa6ab22a069
+  data.tar.gz: e1b82af62b79634ea454d114603ab6e5721bf0c01e1abe4e1ddbfead7fab2c7478a2a62c0c70fcbf839d24d9d2b03362bc3f9ea4b1c2f976576848ebf85dd122

data/ChangeLog

@@ -0,0 +1,9 @@
+Release 0.1.2 - 2015-02-20
+
+* Sets last_path to the next config.
+* Upgraded embulk version to 0.4.6.
+
+Release 0.1.1 - 2015-02-20
+
+* Upgraded embulk version to 0.4.2.
+

data/build.gradle

@@ -13,11 +13,11 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.4.2"
-    provided "org.embulk:embulk-core:0.4.2"
+    compile  "org.embulk:embulk-core:0.4.6"
+    provided "org.embulk:embulk-core:0.4.6"
     compile "com.amazonaws:aws-java-sdk-s3:1.9.17"
     testCompile "junit:junit:4.+"
     testCompile "org.mockito:mockito-core:1.+"

data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java

@@ -1,40 +1,13 @@
 package org.embulk.input.s3;
 
 import java.util.List;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.Files;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.FileVisitResult;
-import java.nio.file.attribute.BasicFileAttributes;
-import javax.validation.constraints.NotNull;
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.Config;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.CommitReport;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
-import org.slf4j.Logger;
-
-import static org.embulk.spi.util.Inputs.formatPath;
-
-import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.io.IOException;
 import java.io.InputStream;
 import com.google.common.collect.ImmutableList;
 import com.google.common.base.Optional;
-import com.fasterxml.jackson.annotation.JacksonInject;
+import org.slf4j.Logger;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
@@ -47,6 +20,7 @@ import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
@@ -104,22 +78,30 @@ public class S3FileInputPlugin
         // list files recursively
         task.setFiles(listFiles(task));
 
+        // TODO what if task.getFiles().isEmpty()?
+
         // number of processors is same with number of files
         return resume(task.dump(), task.getFiles().size(), control);
     }
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             FileInputPlugin.Control control)
     {
-        control.run(taskSource, processorCount);
-        return Exec.newConfigDiff();
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        control.run(taskSource, taskCount);
+
+        List<String> files = new ArrayList<String>(task.getFiles());
+        Collections.sort(files);
+        return Exec.newConfigDiff().
+            set("last_path", files.get(files.size() - 1));
     }
 
     @Override
     public void cleanup(TaskSource taskSource,
-            int processorCount,
+            int taskCount,
             List<CommitReport> successCommitReports)
     {
         // do nothing
@@ -201,10 +183,10 @@ public class S3FileInputPlugin
     }
 
     @Override
-    public TransactionalFileInput open(TaskSource taskSource, int processorIndex)
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new S3FileInput(task, processorIndex);
+        return new S3FileInput(task, taskIndex);
     }
 
     public static class S3FileInput
@@ -220,11 +202,11 @@ public class S3FileInputPlugin
             private final String key;
             private boolean opened = false;
 
-            public SingleFileProvider(PluginTask task, int processorIndex)
+            public SingleFileProvider(PluginTask task, int taskIndex)
             {
                 this.client = newS3Client(task);
                 this.bucket = task.getBucket();
-                this.key = task.getFiles().get(processorIndex);
+                this.key = task.getFiles().get(taskIndex);
             }
 
             @Override
@@ -250,9 +232,9 @@ public class S3FileInputPlugin
             public void close() { }
         }
 
-        public S3FileInput(PluginTask task, int processorIndex)
+        public S3FileInput(PluginTask task, int taskIndex)
         {
-            super(task.getBufferAllocator(), new SingleFileProvider(task, processorIndex));
+            super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
         }
 
         public void abort() { }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-18 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,6 +60,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- ChangeLog
 - README.md
 - build.gradle
 - gradle/wrapper/gradle-wrapper.jar
@@ -74,7 +75,7 @@ files:
 - classpath/aws-java-sdk-s3-1.9.17.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-input-s3-0.1.1.jar
+- classpath/embulk-input-s3-0.1.2.jar
 - classpath/httpclient-4.3.4.jar
 - classpath/httpcore-4.3.2.jar
 homepage: https://github.com/embulk/embulk-input-s3