embulk 0.6.16 → 0.6.17

data/lib/embulk/data/new/java/build.gradle.erb

@@ -57,7 +57,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[<%= description %>]
   spec.email         = [<%= email.dump %>]
   spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = <%= "https://github.com/#{email[/([^@]*)/]}/#{project_name}".dump %>
+  # TODO set this: spec.homepage      = <%= "https://github.com/#{email[/([^@]*)/]}/#{full_project_name}".dump %>
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/lib/embulk/data/new/java/decoder.java.erb

@@ -1,12 +1,19 @@
-package org.embulk.<%= embulk_category %>;
+package <%= java_package_name %>;
 
+import java.io.InputStream;
+import java.io.IOException;
+import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.DecoderPlugin;
 import org.embulk.spi.FileInput;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.util.FileInputInputStream;
+import org.embulk.spi.util.InputStreamFileInput;
 
 public class <%= java_class_name %>
         implements DecoderPlugin
@@ -14,12 +21,22 @@ public class <%= java_class_name %>
     public interface PluginTask
             extends Task
     {
-        @Config("property1")
-        public String getProperty1();
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
 
-        @Config("property2")
-        @ConfigDefault("0")
-        public int getProperty2();
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
     }
 
     @Override
@@ -31,10 +48,37 @@ public class <%= java_class_name %>
     }
 
     @Override
-    public FileInput open(TaskSource taskSource, FileInput input)
+    public FileInput open(TaskSource taskSource, FileInput fileInput)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
 
-        // TODO
+        // If expect InputStream, you can use this code:
+
+        //final FileInputInputStream files = new FileInputInputStream(fileInput);
+        //
+        //return new InputStreamFileInput(
+        //        task.getBufferAllocator(),
+        //        new InputStreamFileInput.Provider() {
+        //            public InputStream openNext() throws IOException
+        //            {
+        //                if (!files.nextFile()) {
+        //                    return null;
+        //                }
+        //                return newDecoderInputStream(task, files);
+        //            }
+        //
+        //            public void close() throws IOException
+        //            {
+        //                files.close();
+        //            }
+        //        });
     }
+
+    //private static InputStream newDecoderInputStream(PluginTask task, InputStream file) throws IOException
+    //{
+    //    return new MyInputStream(file);
+    //}
 }

data/lib/embulk/data/new/java/encoder.java.erb

@@ -1,12 +1,19 @@
-package org.embulk.<%= embulk_category %>;
+package <%= java_package_name %>;
 
+import java.io.OutputStream;
+import java.io.IOException;
+import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.EncoderPlugin;
 import org.embulk.spi.FileOutput;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.util.FileOutputOutputStream;
+import org.embulk.spi.util.OutputStreamFileOutput;
 
 public class <%= java_class_name %>
         implements EncoderPlugin
@@ -14,12 +21,22 @@ public class <%= java_class_name %>
     public interface PluginTask
             extends Task
     {
-        @Config("property1")
-        public String getProperty1();
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
 
-        @Config("property2")
-        @ConfigDefault("0")
-        public int getProperty2();
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
     }
 
     @Override
@@ -33,8 +50,37 @@ public class <%= java_class_name %>
     @Override
     public FileOutput open(TaskSource taskSource, FileOutput fileOutput)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
 
-        // TODO
+        // If expect OutputStream, you can use this code:
+
+        //final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput,
+        //    task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);
+        //
+        //return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() {
+        //    public OutputStream openNext() throws IOException
+        //    {
+        //        output.nextFile();
+        //        return newEncoderOutputStream(output);
+        //    }
+        //
+        //    public void finish() throws IOException
+        //    {
+        //        fileOutput.finish();
+        //    }
+        //
+        //    public void close() throws IOException
+        //    {
+        //        fileOutput.close();
+        //    }
+        //});
     }
+
+    //private static OutputStream newEncoderOutputStream(PluginTask task, OutputStream file) throws IOException
+    //{
+    //    return new MyOutputStream(file);
+    //}
 }

data/lib/embulk/data/new/java/file_input.java.erb

@@ -1,16 +1,22 @@
-package org.embulk.<%= embulk_category %>;
+package <%= java_package_name %>;
 
 import java.util.List;
+import java.util.ArrayList;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
 import org.embulk.config.CommitReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamTransactionalFileInput;
 
 public class <%= java_class_name %>
         implements FileInputPlugin
@@ -18,12 +24,33 @@ public class <%= java_class_name %>
     public interface PluginTask
             extends Task
     {
-        @Config("property1")
-        public String getProperty1();
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
 
-        @Config("property2")
-        @ConfigDefault("0")
-        public int getProperty2();
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        //@Config("path_prefix")
+        //public String getPathPrefix();
+
+        //@Config("last_path")
+        //@ConfigDefault("null")
+        //public Optional<String> getLastPath();
+
+        // usually, you store list of files in task to pass them from transaction() to run().
+        //public List<String> getFiles();
+        //public void setFiles(List<String> files);
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
     }
 
     @Override
@@ -31,19 +58,50 @@ public class <%= java_class_name %>
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        // taskCount is usually number of input files
-        int taskCount = 1;  // number of run() method calls
+        // run() method is called for this number of times in parallel.
+        int taskCount = 1;
+
+        // usually, taskCount is number of input files.
+        //task.setFiles(listFiles(task));
+        //int taskCount = task.getFiles().size();
 
         return resume(task.dump(), taskCount, control);
     }
 
+    // usually, you have an method to create list of files
+    //List<String> listFiles(PluginTask task)
+    //{
+    //    final ImmutableList.Builder<String> builder = ImmutableList.builder();
+    //    for (String path : listFilesWithPrefix(task.getPathPrefix())) {
+    //        if (task.getLastPath().isPresent() && path.compareTo(task.getLastPath().get())) {
+    //            continue;
+    //        }
+    //        builder.add(path);
+    //    }
+    //    return builder.build();
+    //}
+
     @Override
     public ConfigDiff resume(TaskSource taskSource,
             int taskCount,
             FileInputPlugin.Control control)
     {
         control.run(taskSource, taskCount);
-        return Exec.newConfigDiff();
+
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        // usually, yo uset last_path
+        //if (task.getFiles().isEmpty()) {
+        //    if (task.getLastPath().isPresent()) {
+        //        configDiff.set("last_path", task.getLastPath().get());
+        //    }
+        //} else {
+        //    List<String> files = new ArrayList<String>(task.getFiles());
+        //    Collections.sort(files);
+        //    configDiff.set("last_path", files.get(files.size() - 1));
+        //}
+
+        return configDiff;
     }
 
     @Override
@@ -56,9 +114,30 @@ public class <%= java_class_name %>
     @Override
     public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
 
-        // TODO
-        throw new UnsupportedOperationException("The 'open' method needs to be implemented");
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
+
+        // if you expect InputStream, you can use this code:
+
+        //InputStream input = openInputStream(task, task.getFiles().get(taskIndex));
+        //
+        //return new InputStreamTransactionalFileInput(task.getBufferAllocator(), input) {
+        //    @Override
+        //    public void abort()
+        //    { }
+        //
+        //    @Override
+        //    public CommitReport commit()
+        //    {
+        //        return Exec.newCommitReport();
+        //    }
+        //}
     }
+
+    //private static InputStream openInputStream(PluginTask task, String path)
+    //{
+    //    return new MyInputStream(file);
+    //}
 }

data/lib/embulk/data/new/java/file_output.java.erb

@@ -1,6 +1,7 @@
-package org.embulk.<%= embulk_category %>;
+package <%= java_package_name %>;
 
 import java.util.List;
+import com.google.common.base.Optional;
 import org.embulk.config.CommitReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -18,12 +19,35 @@ public class <%= java_class_name %>
     public interface PluginTask
             extends Task
     {
-        @Config("property1")
-        public String getProperty1();
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
 
-        @Config("property2")
-        @ConfigDefault("0")
-        public int getProperty2();
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        // usually, run() method needs to write multiple files because size of a file
+        // can be very large. So, file name will be:
+        //
+        //    path_prefix + String.format(sequence_format, taskIndex, sequenceCounterInRunMethod) + file_ext
+        //
+
+        //@Config("path_prefix")
+        //public String getPathPrefix();
+
+        //@Config("file_ext")
+        //public String getFileNameExtension();
+
+        //@Config("sequence_format")
+        //@ConfigDefault("\"%03d.%02d.\"")
+        //public String getSequenceFormat();
     }
 
     @Override
@@ -60,7 +84,10 @@ public class <%= java_class_name %>
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
-        // TODO
-        throw new UnsupportedOperationException("The 'open' method needs to be implemented");
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
+
+        // See LocalFileOutputPlugin as an example implementation:
+        // https://github.com/embulk/embulk/blob/master/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java
     }
 }

data/lib/embulk/data/new/java/filter.java.erb

@@ -1,5 +1,6 @@
 package org.embulk.<%= embulk_category %>;
 
+import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -17,12 +18,19 @@ public class <%= java_class_name %>
     public interface PluginTask
             extends Task
     {
-        @Config("property1")
-        public String getProperty1();
-
-        @Config("property2")
-        @ConfigDefault("0")
-        public int getProperty2();
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
+
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
     }
 
     @Override
@@ -42,6 +50,7 @@ public class <%= java_class_name %>
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
-        // TODO
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
     }
 }

data/lib/embulk/data/new/java/formatter.java.erb

@@ -1,5 +1,6 @@
-package org.embulk.<%= embulk_category %>;
+package <%= java_package_name %>;
 
+import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -17,12 +18,19 @@ public class <%= java_class_name %>
     public interface PluginTask
             extends Task
     {
-        @Config("property1")
-        public String getProperty1();
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
 
-        @Config("property2")
-        @ConfigDefault("0")
-        public int getProperty2();
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
     }
 
     @Override
@@ -40,6 +48,7 @@ public class <%= java_class_name %>
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
-        // TODO
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
     }
 }