embulk 0.7.0-java

data/lib/embulk/data/new/java/decoder.java.erb

@@ -0,0 +1,84 @@
+package <%= java_package_name %>;
+
+import java.io.InputStream;
+import java.io.IOException;
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.DecoderPlugin;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.util.FileInputInputStream;
+import org.embulk.spi.util.InputStreamFileInput;
+
+public class <%= java_class_name %>
+        implements DecoderPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
+
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, DecoderPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        control.run(task.dump());
+    }
+
+    @Override
+    public FileInput open(TaskSource taskSource, FileInput fileInput)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
+
+        // If expect InputStream, you can use this code:
+
+        //final FileInputInputStream files = new FileInputInputStream(fileInput);
+        //
+        //return new InputStreamFileInput(
+        //        task.getBufferAllocator(),
+        //        new InputStreamFileInput.Provider() {
+        //            public InputStream openNext() throws IOException
+        //            {
+        //                if (!files.nextFile()) {
+        //                    return null;
+        //                }
+        //                return newDecoderInputStream(task, files);
+        //            }
+        //
+        //            public void close() throws IOException
+        //            {
+        //                files.close();
+        //            }
+        //        });
+    }
+
+    //private static InputStream newDecoderInputStream(PluginTask task, InputStream file) throws IOException
+    //{
+    //    return new MyInputStream(file);
+    //}
+}

data/lib/embulk/data/new/java/encoder.java.erb

@@ -0,0 +1,86 @@
+package <%= java_package_name %>;
+
+import java.io.OutputStream;
+import java.io.IOException;
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.EncoderPlugin;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.util.FileOutputOutputStream;
+import org.embulk.spi.util.OutputStreamFileOutput;
+
+public class <%= java_class_name %>
+        implements EncoderPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
+
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, EncoderPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        control.run(task.dump());
+    }
+
+    @Override
+    public FileOutput open(TaskSource taskSource, FileOutput fileOutput)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
+
+        // If expect OutputStream, you can use this code:
+
+        //final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput,
+        //    task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);
+        //
+        //return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() {
+        //    public OutputStream openNext() throws IOException
+        //    {
+        //        output.nextFile();
+        //        return newEncoderOutputStream(task, output);
+        //    }
+        //
+        //    public void finish() throws IOException
+        //    {
+        //        output.finish();
+        //    }
+        //
+        //    public void close() throws IOException
+        //    {
+        //        output.close();
+        //    }
+        //});
+    }
+
+    //private static OutputStream newEncoderOutputStream(PluginTask task, OutputStream file) throws IOException
+    //{
+    //    return new MyOutputStream(file);
+    //}
+}

data/lib/embulk/data/new/java/file_input.java.erb

@@ -0,0 +1,143 @@
+package <%= java_package_name %>;
+
+import java.util.List;
+import java.util.ArrayList;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.TaskReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamTransactionalFileInput;
+
+public class <%= java_class_name %>
+        implements FileInputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
+
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        //@Config("path_prefix")
+        //public String getPathPrefix();
+
+        //@Config("last_path")
+        //@ConfigDefault("null")
+        //public Optional<String> getLastPath();
+
+        // usually, you store list of files in task to pass them from transaction() to run().
+        //public List<String> getFiles();
+        //public void setFiles(List<String> files);
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // run() method is called for this number of times in parallel.
+        int taskCount = 1;
+
+        // usually, taskCount is number of input files.
+        //task.setFiles(listFiles(task));
+        //int taskCount = task.getFiles().size();
+
+        return resume(task.dump(), taskCount, control);
+    }
+
+    // usually, you have an method to create list of files
+    //List<String> listFiles(PluginTask task)
+    //{
+    //    final ImmutableList.Builder<String> builder = ImmutableList.builder();
+    //    for (String path : listFilesWithPrefix(task.getPathPrefix())) {
+    //        if (task.getLastPath().isPresent() && path.compareTo(task.getLastPath().get())) {
+    //            continue;
+    //        }
+    //        builder.add(path);
+    //    }
+    //    return builder.build();
+    //}
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control)
+    {
+        control.run(taskSource, taskCount);
+
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        // usually, yo uset last_path
+        //if (task.getFiles().isEmpty()) {
+        //    if (task.getLastPath().isPresent()) {
+        //        configDiff.set("last_path", task.getLastPath().get());
+        //    }
+        //} else {
+        //    List<String> files = new ArrayList<String>(task.getFiles());
+        //    Collections.sort(files);
+        //    configDiff.set("last_path", files.get(files.size() - 1));
+        //}
+
+        return configDiff;
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
+
+        // if you expect InputStream, you can use this code:
+
+        //InputStream input = openInputStream(task, task.getFiles().get(taskIndex));
+        //
+        //return new InputStreamTransactionalFileInput(task.getBufferAllocator(), input) {
+        //    @Override
+        //    public void abort()
+        //    { }
+        //
+        //    @Override
+        //    public TaskReport commit()
+        //    {
+        //        return Exec.newTaskReport();
+        //    }
+        //}
+    }
+
+    //private static InputStream openInputStream(PluginTask task, String path)
+    //{
+    //    return new MyInputStream(file);
+    //}
+}

data/lib/embulk/data/new/java/file_output.java.erb

@@ -0,0 +1,93 @@
+package <%= java_package_name %>;
+
+import java.util.List;
+import com.google.common.base.Optional;
+import org.embulk.config.TaskReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.TransactionalFileOutput;
+
+public class <%= java_class_name %>
+        implements FileOutputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
+
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        // usually, run() method needs to write multiple files because size of a file
+        // can be very large. So, file name will be:
+        //
+        //    path_prefix + String.format(sequence_format, taskIndex, sequenceCounterInRunMethod) + file_ext
+        //
+
+        //@Config("path_prefix")
+        //public String getPathPrefix();
+
+        //@Config("file_ext")
+        //public String getFileNameExtension();
+
+        //@Config("sequence_format")
+        //@ConfigDefault("\"%03d.%02d.\"")
+        //public String getSequenceFormat();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, int taskCount,
+            FileOutputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // retryable (idempotent) output:
+        // return resume(task.dump(), taskCount, control);
+
+        // non-retryable (non-idempotent) output:
+        control.run(task.dump());
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileOutputPlugin.Control control)
+    {
+        throw new UnsupportedOperationException("<%= name %> output plugin does not support resuming");
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
+
+        // See LocalFileOutputPlugin as an example implementation:
+        // https://github.com/embulk/embulk/blob/master/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java
+    }
+}

data/lib/embulk/data/new/java/filter.java.erb

@@ -0,0 +1,56 @@
+package <%= java_package_name %>;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+
+public class <%= java_class_name %>
+        implements FilterPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
+
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema inputSchema,
+            FilterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        Schema outputSchema = inputSchema;
+
+        control.run(task.dump(), outputSchema);
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, Schema inputSchema,
+            Schema outputSchema, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.open method is not implemented yet");
+    }
+}