embulk 0.3.2 → 0.4.0

data/embulk-standards/build.gradle

@@ -1,6 +1,5 @@
 dependencies {
     compile project(':embulk-core')
-    compile 'com.amazonaws:aws-java-sdk:1.5.2'
 
     testCompile project(':embulk-core').sourceSets.test.output
 }

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -11,7 +11,7 @@ import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.Column;
 import org.embulk.spi.Schema;
-import org.embulk.spi.SchemaVisitor;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.FormatterPlugin;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageOutput;
@@ -78,7 +78,7 @@ public class CsvFormatterPlugin
             {
                 pageReader.setPage(page);
                 while (pageReader.nextRecord()) {
-                    schema.visitColumns(new SchemaVisitor() {
+                    schema.visitColumns(new ColumnVisitor() {
                         public void booleanColumn(Column column)
                         {
                             addDelimiter(column);

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.standards;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
 import org.embulk.config.Task;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -14,7 +15,7 @@ import org.embulk.spi.time.TimestampParseException;
 import org.embulk.spi.Column;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.SchemaVisitor;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Exec;
@@ -29,6 +30,14 @@ import java.util.Map;
 public class CsvParserPlugin
         implements ParserPlugin
 {
+    private static final ImmutableSet<String> TRUE_STRINGS =
+        ImmutableSet.of(
+                "true", "True", "TRUE",
+                "yes", "Yes", "YES",
+                "y", "Y",
+                "on", "On", "ON",
+                "1");
+
     public interface PluginTask
             extends Task, LineDecoder.DecoderTask, TimestampParser.ParserTask
     {
@@ -120,14 +129,14 @@ public class CsvParserPlugin
                             break;
                         }
 
-                        schema.visitColumns(new SchemaVisitor() {
+                        schema.visitColumns(new ColumnVisitor() {
                             public void booleanColumn(Column column)
                             {
                                 String v = nextColumn(schema, tokenizer, nullStringOrNull);
                                 if (v == null) {
                                     pageBuilder.setNull(column);
                                 } else {
-                                    pageBuilder.setBoolean(column, Boolean.parseBoolean(v));
+                                    pageBuilder.setBoolean(column, TRUE_STRINGS.contains(v));
                                 }
                             }
 

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java

@@ -3,10 +3,10 @@ package org.embulk.standards;
 import java.io.InputStream;
 import java.io.IOException;
 import java.util.zip.GZIPInputStream;
-import com.fasterxml.jackson.annotation.JacksonInject;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigInject;
 import org.embulk.spi.DecoderPlugin;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.FileInput;
@@ -19,7 +19,7 @@ public class GzipFileDecoderPlugin
     public interface PluginTask
             extends Task
     {
-        @JacksonInject
+        @ConfigInject
         public BufferAllocator getBufferAllocator();
     }
 

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -13,12 +13,14 @@ import java.nio.file.FileVisitResult;
 import java.nio.file.attribute.BasicFileAttributes;
 import javax.validation.constraints.NotNull;
 import com.google.common.collect.ImmutableList;
-import com.fasterxml.jackson.annotation.JacksonInject;
+import com.google.common.base.Optional;
 import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.CommitReport;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
@@ -27,34 +29,37 @@ import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
 import org.slf4j.Logger;
 
-import static org.embulk.spi.util.Inputs.formatPath;
-
 public class LocalFileInputPlugin
         implements FileInputPlugin
 {
     public interface PluginTask
             extends Task
     {
-        @Config("paths")
-        @NotNull
-        public List<String> getPathPrefixes();
+        @Config("path_prefix")
+        public String getPathPrefix();
+
+        @Config("last_path")
+        @ConfigDefault("null")
+        public Optional<String> getLastPath();
 
         public List<String> getFiles();
         public void setFiles(List<String> files);
 
-        @JacksonInject
+        @ConfigInject
         public BufferAllocator getBufferAllocator();
     }
 
     private final Logger log = Exec.getLogger(getClass());
 
     @Override
-    public NextConfig transaction(ConfigSource config, FileInputPlugin.Control control)
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
         // list files recursively
-        task.setFiles(listFiles(task));
+        List<String> files = listFiles(task);
+        log.info("Loading files {}", files);
+        task.setFiles(files);
 
         // number of processors is same with number of files
         int processorCount = task.getFiles().size();
@@ -62,12 +67,12 @@ public class LocalFileInputPlugin
     }
 
     @Override
-    public NextConfig resume(TaskSource taskSource,
+    public ConfigDiff resume(TaskSource taskSource,
             int processorCount,
             FileInputPlugin.Control control)
     {
         control.run(taskSource, processorCount);
-        return Exec.newNextConfig();
+        return Exec.newConfigDiff();
     }
 
     @Override
@@ -78,22 +83,40 @@ public class LocalFileInputPlugin
 
     public List<String> listFiles(PluginTask task)
     {
+        Path pathPrefix = Paths.get(task.getPathPrefix()).normalize();
+        final Path directory;
+        final String fileNamePrefix;
+        if (Files.isDirectory(pathPrefix)) {
+            directory = pathPrefix;
+            fileNamePrefix = "";
+        } else {
+            fileNamePrefix = pathPrefix.getFileName().toString();
+            Path d = pathPrefix.getParent();
+            directory = (d == null ? Paths.get(".") : d);
+        }
+
         final ImmutableList.Builder<String> builder = ImmutableList.builder();
-        for (String prefix : task.getPathPrefixes()) {
-            String formatted = formatPath(prefix);
-            try {
-                log.info("Listing local files with prefix '{}'", formatted);
-                Files.walkFileTree(Paths.get(formatted), new SimpleFileVisitor<Path>() {
-                    @Override
-                    public FileVisitResult visitFile(Path file, BasicFileAttributes aAttrs)
-                    {
-                        builder.add(file.toString());
-                        return FileVisitResult.CONTINUE;
+        final String lastPath = task.getLastPath().orNull();
+        try {
+            log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory, fileNamePrefix);
+            Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path path, BasicFileAttributes aAttrs)
+                {
+                    if (lastPath == null || path.toString().compareTo(lastPath) > 0) {
+                        if (path.getParent().equals(directory)) {
+                            if (path.getFileName().toString().startsWith(fileNamePrefix)) {
+                                builder.add(path.toString());
+                            }
+                        } else {
+                            builder.add(path.toString());
+                        }
                     }
-                });
-            } catch (IOException ex) {
-                throw new RuntimeException(String.format("Failed get a list of local files at '%s'", formatted), ex);
-            }
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException ex) {
+            throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex);
         }
         return builder.build();
     }

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -8,8 +8,9 @@ import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.CommitReport;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
@@ -25,38 +26,38 @@ public class LocalFileOutputPlugin
     public interface PluginTask
             extends Task
     {
-        @Config("directory")
-        public String getDirectory();
-
-        @Config("file_name")
-        public String getFileNameFormat();
+        @Config("path_prefix")
+        public String getPathPrefix();
 
         @Config("file_ext")
         public String getFileNameExtension();
 
-        // TODO support in FileInputPlugin and FileOutputPlugin
-        //@Config("compress_type")
-        //public String getCompressType();
+        @Config("sequence_format")
+        @ConfigDefault("\".%03d.%02d\"")
+        public String getSequenceFormat();
     }
 
     private final Logger log = Exec.getLogger(getClass());
 
     @Override
-    public NextConfig transaction(ConfigSource config, int processorCount,
+    public ConfigDiff transaction(ConfigSource config, int processorCount,
             FileOutputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
+        // validate sequence_format
+        String.format(task.getSequenceFormat(), 0, 0);
+
         return resume(task.dump(), processorCount, control);
     }
 
     @Override
-    public NextConfig resume(TaskSource taskSource,
+    public ConfigDiff resume(TaskSource taskSource,
             int processorCount,
             FileOutputPlugin.Control control)
     {
         control.run(taskSource);
-        return Exec.newNextConfig();
+        return Exec.newConfigDiff();
     }
 
     @Override
@@ -70,11 +71,9 @@ public class LocalFileOutputPlugin
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
-        // TODO format path using timestamp
-        final String fileName = task.getFileNameFormat();
-
-        final String pathPrefix = task.getDirectory() + File.separator + fileName;
+        final String pathPrefix = task.getPathPrefix();
         final String pathSuffix = task.getFileNameExtension();
+        final String sequenceFormat = task.getSequenceFormat();
 
         final List<String> fileNames = new ArrayList<>();
 
@@ -85,7 +84,7 @@ public class LocalFileOutputPlugin
             public void nextFile()
             {
                 closeFile();
-                String path = pathPrefix + String.format(".%03d.%02d.", processorIndex, fileIndex) + pathSuffix;
+                String path = pathPrefix + String.format(sequenceFormat, processorIndex, fileIndex) + pathSuffix;
                 log.info("Writing local file '{}'", path);
                 fileNames.add(path);
                 try {

data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java

@@ -3,7 +3,7 @@ package org.embulk.standards;
 import java.util.List;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.CommitReport;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Page;
@@ -15,19 +15,19 @@ public class NullOutputPlugin
         implements OutputPlugin
 {
     @Override
-    public NextConfig transaction(ConfigSource config,
+    public ConfigDiff transaction(ConfigSource config,
             Schema schema, int processorCount,
             OutputPlugin.Control control)
     {
         return resume(Exec.newTaskSource(), schema, processorCount, control);
     }
 
-    public NextConfig resume(TaskSource taskSource,
+    public ConfigDiff resume(TaskSource taskSource,
             Schema schema, int processorCount,
             OutputPlugin.Control control)
     {
         control.run(taskSource);
-        return Exec.newNextConfig();
+        return Exec.newConfigDiff();
     }
 
     public void cleanup(TaskSource taskSource,

data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java

@@ -22,7 +22,6 @@ public class StandardPluginModule
 
         // input plugins
         registerPluginTo(binder, InputPlugin.class, "file", LocalFileInputPlugin.class);
-        registerPluginTo(binder, InputPlugin.class, "s3_file", S3FileInputPlugin.class);
 
         // parser plugins
         registerPluginTo(binder, ParserPlugin.class, "csv", CsvParserPlugin.class);

data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java

@@ -3,12 +3,12 @@ package org.embulk.standards;
 import java.util.List;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
-import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.CommitReport;
 import org.embulk.config.Task;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.Schema;
-import org.embulk.spi.SchemaVisitor;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Column;
 import org.embulk.spi.Page;
 import org.embulk.spi.Exec;
@@ -26,7 +26,7 @@ public class StdoutOutputPlugin
     }
 
     @Override
-    public NextConfig transaction(ConfigSource config,
+    public ConfigDiff transaction(ConfigSource config,
             Schema schema, int processorCount,
             OutputPlugin.Control control)
     {
@@ -35,12 +35,12 @@ public class StdoutOutputPlugin
     }
 
     @Override
-    public NextConfig resume(TaskSource taskSource,
+    public ConfigDiff resume(TaskSource taskSource,
             Schema schema, int processorCount,
             OutputPlugin.Control control)
     {
         control.run(taskSource);
-        return Exec.newNextConfig();
+        return Exec.newConfigDiff();
     }
 
     public void cleanup(TaskSource taskSource,

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 21 18:25:48 PST 2015
+#Wed Feb 04 13:46:12 PST 2015
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip

data/lib/embulk/buffer.rb

@@ -2,14 +2,14 @@
 module Embulk
   class Buffer < String
     if Embulk.java?
-      def self.from_java_object(java_buffer)
+      def self.from_java(java_buffer)
         byte_list = org.jruby.util.ByteList.new(java_buffer.array(), java_buffer.offset(), java_buffer.limit(), false)
         buffer = new
         buffer.replace(org.jruby.RubyString.new(JRuby.runtime, self, byte_list).dup)  # TODO simplify
         buffer
       end
 
-      def java_object
+      def to_java
         Java::Buffer.wrap(to_java_bytes)
       end
     end

data/lib/embulk/column.rb

@@ -6,26 +6,26 @@ module Embulk
     end
 
     if Embulk.java?
-      def self.from_java_object(java_column)
+      def self.from_java(java_column)
         Column.new(
           java_column.getIndex,
           java_column.getName,
-          Type.from_java_object(java_column.getType))
+          Type.from_java(java_column.getType))
       end
 
-      def java_object
-        Java::Column.new(index, name, Type.to_java_object(type))
+      def to_java
+        Java::Column.new(index, name, Type.new_java_type(type))
       end
     end
   end
 
   module Type
     if Embulk.java?
-      def self.from_java_object(java_type)
+      def self.from_java(java_type)
         java_type.getName.to_sym
       end
 
-      def self.to_java_object(ruby_type)
+      def self.new_java_type(ruby_type)
         case ruby_type
         when :boolean
           Java::Types::BOOLEAN