embulk 0.7.0-java

data/lib/embulk/data/new/java/parser.java.erb

@@ -0,0 +1,60 @@
+package <%= java_package_name %>;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+
+public class <%= java_class_name %>
+        implements ParserPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        // configuration option 1 (required integer)
+        @Config("option1")
+        public int getOption1();
+
+        // configuration option 2 (optional string, null is not allowed)
+        @Config("optoin2")
+        @ConfigDefault("\"myvalue\"")
+        public String getOption2();
+
+        // configuration option 3 (optional string, null is allowed)
+        @Config("optoin3")
+        @ConfigDefault("null")
+        public Optional<String> getOption3();
+
+        // if you get schema from config or data source
+        @Config("columns")
+        public SchemaConfig getColumns();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        Schema schema = task.getColumns().toSchema();
+
+        control.run(task.dump(), schema);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // Write your code here :)
+        throw new UnsupportedOperationException("<%= java_class_name %>.run method is not implemented yet");
+    }
+}

data/lib/embulk/data/new/java/plugin_loader.rb.erb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_<%= embulk_category %>(
+  <%= name.dump %>, <%= "#{java_package_name}.#{java_class_name}".dump %>,
+  File.expand_path('../../../../classpath', __FILE__))

data/lib/embulk/data/new/java/test.java.erb

@@ -0,0 +1,5 @@
+package <%= java_package_name %>;
+
+public class Test<%= java_class_name %>
+{
+}

data/lib/embulk/data/new/ruby/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.0.0

data/lib/embulk/data/new/ruby/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/lib/embulk/data/new/ruby/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/lib/embulk/data/new/ruby/decoder_guess.rb.erb

@@ -0,0 +1,25 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g <%= name.dump %> partial-config.yml
+%if language == :ruby
+
+    #require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
+%end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  FOO_BAR_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
+    #
+    #  def guess(config, sample_buffer)
+    #    if sample_buffer[0,2] == FOO_BAR_HEADER
+    #      return {"decoders" => [{"type" => <%= name.dump %>}]}
+    #    end
+    #    return {}
+    #  end
+    #end
+
+  end
+end

data/lib/embulk/data/new/ruby/filter.rb.erb

@@ -0,0 +1,41 @@
+module Embulk
+  module Filter
+
+    class <%= ruby_class_name %> < FilterPlugin
+      Plugin.register_filter(<%= name.dump %>, self)
+
+      def self.transaction(config, in_schema, &control)
+        # configuration code:
+        task = {
+          "option1" => config.param("option1", :integer),                     # integer, required
+          "option2" => config.param("option2", :string, default: "myvalue"),  # string, optional
+          "option3" => config.param("option3", :string, default: nil),        # string, optional
+        }
+
+        yield(task, out_columns)
+      end
+
+      def init
+        # initialization code:
+        @option1 = task["option1"]
+        @option2 = task["option2"]
+        @option3 = task["option3"]
+      end
+
+      def close
+      end
+
+      def add(page)
+        # filtering code:
+        page.each do |record|
+          page_builder.add(record)
+        end
+      end
+
+      def finish
+        page_builder.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/formatter.rb.erb

@@ -0,0 +1,49 @@
+module Embulk
+  module Formatter
+
+    class <%= ruby_class_name %> < FormatterPlugin
+      Plugin.register_formatter(<%= name.dump %>, self)
+
+      def self.transaction(config, schema, &control)
+        # configuration code:
+        task = {
+          "option1" => config.param("option1", :integer),                     # integer, required
+          "option2" => config.param("option2", :string, default: "myvalue"),  # string, optional
+          "option3" => config.param("option3", :string, default: nil),        # string, optional
+        }
+
+        yield(task)
+      end
+
+      def init
+        # initialization code:
+        @option1 = task["option1"]
+        @option2 = task["option2"]
+        @option3 = task["option3"]
+
+        # your data
+        @current_file == nil
+        @current_file_size = 0
+      end
+
+      def close
+      end
+
+      def add(page)
+        # output code:
+        page.each do |record|
+          if @current_file == nil || @current_file_size > 32*1024
+            @current_file = file_output.next_file
+            @current_file_size = 0
+          end
+          @current_file.write "|mydata|"
+        end
+      end
+
+      def finish
+        file_output.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/gemspec.erb

@@ -0,0 +1,20 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "<%= full_project_name %>"
+  spec.version       = "0.1.0"
+  spec.authors       = [<%= author.dump %>]
+  spec.summary       = <%= "#{display_name} #{display_category} plugin for Embulk".dump %>
+  spec.description   = <%= "#{description}".dump %>
+  spec.email         = [<%= email.dump %>]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/<%= email[/([^@]*)/] %>/<%= full_project_name %>"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'embulk', ['~> <%= Embulk::VERSION %>']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/lib/embulk/data/new/ruby/input.rb.erb

@@ -0,0 +1,59 @@
+module Embulk
+  module Input
+
+    class <%= ruby_class_name %> < InputPlugin
+      Plugin.register_input(<%= name.dump %>, self)
+
+      def self.transaction(config, &control)
+        # configuration code:
+        task = {
+          "option1" => config.param("option1", :integer),                     # integer, required
+          "option2" => config.param("option2", :string, default: "myvalue"),  # string, optional
+          "option3" => config.param("option3", :string, default: nil),        # string, optional
+        }
+
+        columns = [
+          Column.new(0, "example", :string),
+          Column.new(1, "column", :long),
+          Column.new(2, "value", :double),
+        ]
+
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        task_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      # TODO
+      #def self.guess(config)
+      #  sample_records = [
+      #    {"example"=>"a", "column"=>1, "value"=>0.1},
+      #    {"example"=>"a", "column"=>2, "value"=>0.2},
+      #  ]
+      #  columns = Guess::SchemaGuess.from_hash_records(sample_records)
+      #  return {"columns" => columns}
+      #end
+
+      def init
+        # initialization code:
+        @option1 = task["option1"]
+        @option2 = task["option2"]
+        @option3 = task["option3"]
+      end
+
+      def run
+        page_builder.add(["example-value", 1, 0.1])
+        page_builder.add(["example-value", 2, 0.2])
+        page_builder.finish
+
+        task_report = {}
+        return task_report
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/output.rb.erb

@@ -0,0 +1,61 @@
+module Embulk
+  module Output
+
+    class <%= ruby_class_name %> < OutputPlugin
+      Plugin.register_output(<%= name.dump %>, self)
+
+      def self.transaction(config, schema, count, &control)
+        # configuration code:
+        task = {
+          "option1" => config.param("option1", :integer),                     # integer, required
+          "option2" => config.param("option2", :string, default: "myvalue"),  # string, optional
+          "option3" => config.param("option3", :string, default: nil),        # string, optional
+        }
+
+        # resumable output:
+        # resume(task, schema, count, &control)
+
+        # non-resumable output:
+        task_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  task_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
+      def init
+        # initialization code:
+        @option1 = task["option1"]
+        @option2 = task["option2"]
+        @option3 = task["option3"]
+      end
+
+      def close
+      end
+
+      def add(page)
+        # output code:
+        page.each do |record|
+          #hash = Hash[schema.names.zip(record)]
+        end
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        task_report = {}
+        return task_report
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/parser.rb.erb

@@ -0,0 +1,44 @@
+module Embulk
+  module Parser
+
+    class <%= ruby_class_name %> < ParserPlugin
+      Plugin.register_parser(<%= name.dump %>, self)
+
+      def self.transaction(config, &control)
+        # configuration code:
+        task = {
+          "option1" => config.param("option1", :integer),                     # integer, required
+          "option2" => config.param("option2", :string, default: "myvalue"),  # string, optional
+          "option3" => config.param("option3", :string, default: nil),        # string, optional
+        }
+
+        columns = [
+          Column.new(0, "example", :string),
+          Column.new(1, "column", :long),
+          Column.new(2, "name", :double),
+        ]
+
+        yield(task, columns)
+      end
+
+      def init
+        # initialization code:
+        @option1 = task["option1"]
+        @option2 = task["option2"]
+        @option3 = task["option3"]
+      end
+
+      def run(file_input)
+        while file = file_input.next_file
+          file.each do |buffer|
+            # parsering code
+            record = ["col1", 2, 3.0]
+            page_builder.add(record)
+          end
+        end
+        page_builder.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/parser_guess.rb.erb

@@ -0,0 +1,65 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g <%= name.dump %> partial-config.yml
+    #
+    #      Depending on the file format the plugin uses, you can use choose
+    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
+    #      or line guess (LineGuessPlugin).
+%if language == :ruby
+
+    #require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
+%end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  def guess(config, sample_buffer)
+    #    if sample_buffer[0,2] == GZIP_HEADER
+    #      guessed = {}
+    #      guessed["type"] = <%= name.dump %>
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < TextGuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  def guess_text(config, sample_text)
+    #    js = JSON.parse(sample_text) rescue nil
+    #    if js && js["mykeyword"] == "keyword"
+    #      guessed = {}
+    #      guessed["type"] = <%= name.dump %>
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < LineGuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  def guess_lines(config, sample_lines)
+    #    all_line_matched = sample_lines.all? do |line|
+    #      line =~ /mypattern/
+    #    end
+    #    if all_line_matched
+    #      guessed = {}
+    #      guessed["type"] = <%= name.dump %>
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+  end
+end