embulk 0.3.2 → 0.4.0

data/lib/embulk/command/embulk_example.rb

@@ -23,7 +23,7 @@ EOF
       f.write <<EOF
 in:
   type: file
-  paths: ["#{File.expand_path File.join(path, 'csv')}"]
+  path_prefix: "#{File.expand_path File.join(path, 'csv', 'sample_')}"
 out:
   type: stdout
 EOF

data/lib/embulk/command/embulk_new_plugin.rb

@@ -0,0 +1,87 @@
+module Embulk
+  def self.new_plugin(name, language, category)
+    require 'embulk/data/package_data'
+    require 'embulk/version'
+    require 'fileutils'
+
+    embulk_category = category
+    embulk_category = :input if category == :file_input
+    embulk_category = :output if category == :file_output
+
+    project_name = "embulk-#{embulk_category}-#{name}"
+    plugin_path = "lib/embulk/#{embulk_category}/#{name}.rb"
+
+    if File.exist?(project_name)
+      raise "./#{project_name} already exists. Please delete it first."
+    end
+    FileUtils.mkdir_p(project_name)
+
+    puts "Creating #{project_name}/"
+
+    success = false
+    begin
+      author = `git config user.name`.strip
+      author = "YOUR_NAME" if author.empty?
+      email = `git config user.email`.strip
+      email = "YOUR_NAME" if email.empty?
+
+      ruby_class_name = name.split('-').map {|a| a.capitalize }.join + category.to_s.split('_').map {|a| a.capitalize }.join + "Plugin"
+      java_iface = category.to_s.split('_').map {|a| a.capitalize }.join
+      java_class_name = name.split('-').map {|a| a.capitalize }.join + java_iface + "Plugin"
+      display_name = name.split('-').map {|a| a.capitalize }.join(' ')
+      display_category = category.to_s.gsub('_', ' ')
+
+      description =
+        case category
+        when :input
+          %[that loads records from #{display_name} so that any output plugins can receive the records. Search the output plugins by "embulk-output" keyword.]
+        when :file_input
+          %[that reads files from #{display_name} and parses the file using any parser plugins. Search the parser plugins by "embulk-parser" keyword.]
+        when :parser
+          %[that parses #{display_name} file format read by any file input plugins. Search the file input plugins by "embulk-input file" keywords.]
+        when :decoder
+          %[that decodes files encoded by #{display_name} read by any file input plugins. Search the file input plugins by "embulk-input file" keywords.]
+        when :output
+          %[that loads records to #{display_name} read by any input plugins. Search the input plugins by "embulk-input" keyword.]
+        when :file_output
+          %[that stores files to #{display_name} formatted by any formatter plugins. Search the formatter plugins by "embulk-formatter" keyword.]
+        when :formtter
+          %[that formats records using #{display_name} file format and so that any file output plugins can store the files. Search the file output plugins by "embulk-output file" keywords.]
+        when :encoder
+          %[that encodes files using #{display_name} so that any file output plugins can store the files. Search the file output plugins by "embulk-output file" keywords.]
+        when :filter
+          %[that converts records read by an input plugin before passing it to an output plugins. Search the input and plugins by "embulk-input" and "embulk-output" plugins.]
+        end
+
+      pkg = Embulk::PackageData.new("new", project_name, binding())
+
+      pkg.cp_erb("README.md.erb", "README.md")
+      pkg.cp("LICENSE.txt", "LICENSE.txt")
+      pkg.cp_erb("gitignore.erb", ".gitignore")
+
+      case language
+      when :ruby
+        pkg.cp("ruby/Rakefile", "Rakefile")
+        pkg.cp("ruby/Gemfile", "Gemfile")
+        pkg.cp_erb("ruby/gemspec.erb", "#{project_name}.gemspec")
+        pkg.cp_erb("ruby/#{category}.rb.erb", plugin_path)
+
+      when :java
+        pkg.cp("java/gradle/wrapper/gradle-wrapper.jar", "gradle/wrapper/gradle-wrapper.jar")
+        pkg.cp("java/gradle/wrapper/gradle-wrapper.properties", "gradle/wrapper/gradle-wrapper.properties")
+        pkg.cp("java/gradlew.bat", "gradlew.bat")
+        pkg.cp("java/gradlew", "gradlew")
+        pkg.set_executable("gradlew")
+        pkg.cp_erb("java/build.gradle.erb", "build.gradle")
+        pkg.cp_erb("java/plugin_loader.rb.erb", plugin_path)
+        pkg.cp_erb("java/#{category}.java.erb", "src/main/java/org/embulk/#{embulk_category}/#{java_class_name}.java")
+        pkg.cp_erb("java/test.java.erb", "src/test/java/org/embulk/#{embulk_category}/Test#{java_class_name}.java")
+      end
+
+      success = true
+      puts ""
+    ensure
+      FileUtils.rm_rf project_name unless success
+    end
+  end
+end

data/lib/embulk/command/embulk_run.rb

@@ -14,13 +14,19 @@ module Embulk
       Gem.clear_paths  # force rubygems to reload GEM_HOME
     end
 
+    # to make sure org.embulk.jruby.JRubyScriptingModule can require 'embulk/java/bootstrap'
+    $LOAD_PATH << Embulk.home('lib')
+
+    if argv.include?('--version')
+      require 'embulk/version'
+      puts "embulk #{Embulk::VERSION}"
+      exit 1
+    end
+
     i = argv.find_index {|arg| arg !~ /^\-/ }
     usage nil unless i
     subcmd = argv.slice!(i)
 
-    # to make sure org.embulk.jruby.JRubyScriptingModule can require 'embulk/java/bootstrap'
-    $LOAD_PATH << Embulk.home('lib')
-
     require 'java'
     require 'optparse'
     op = OptionParser.new
@@ -36,6 +42,7 @@ module Embulk
 
     case subcmd.to_sym
     when :bundle
+      op.remove  # remove --bundle
       if default_bundle_path
         op.banner = "Usage: bundle [directory=#{default_bundle_path}]"
         args = 0..1
@@ -46,8 +53,6 @@ module Embulk
 
     when :run
       op.banner = "Usage: run <config.yml>"
-      op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
-      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -64,8 +69,6 @@ module Embulk
 
     when :cleanup
       op.banner = "Usage: run <config.yml>"
-      op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
-      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -79,8 +82,6 @@ module Embulk
 
     when :preview
       op.banner = "Usage: preview <config.yml>"
-      op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
-      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -102,8 +103,34 @@ module Embulk
       end
       args = 1..1
 
-    #when :generate  # or :new
-      # TODO create plugin templates
+    when :new
+      op.remove  # remove --bundle
+      op.banner = "Usage: new <category> <name>" + %[
+categories:
+    ruby-input                 Ruby record input plugin    (like "mysql")
+    ruby-output                Ruby record output plugin   (like "mysql")
+    ruby-filter                Ruby record filter plugin   (like "add-hostname")
+    #ruby-file-input           Ruby file input plugin      (like "ftp")          # not implemented yet [#21]
+    #ruby-file-output          Ruby file output plugin     (like "ftp")          # not implemented yet [#22]
+    #ruby-parser               Ruby file parser plugin     (like "csv")          # not implemented yet [#33]
+    #ruby-formatter            Ruby file formatter plugin  (like "csv")          # not implemented yet [#34]
+    #ruby-decoder              Ruby file decoder plugin    (like "gzip")         # not implemented yet [#31]
+    #ruby-encoder              Ruby file encoder plugin    (like "gzip")         # not implemented yet [#32]
+    java-input                 Java record input plugin    (like "mysql")
+    java-output                Java record output plugin   (like "mysql")
+    java-filter                Java record filter plugin   (like "add-hostname")
+    java-file-input            Java file input plugin      (like "ftp")
+    java-file-output           Java file output plugin     (like "ftp")
+    java-parser                Java file parser plugin     (like "csv")
+    java-formatter             Java file formatter plugin  (like "csv")
+    java-decoder               Java file decoder plugin    (like "gzip")
+    java-encoder               Java file encoder plugin    (like "gzip")
+
+examples:
+    new ruby-output hbase
+    new ruby-filter int-to-string
+]
+      args = 2..2
 
     when :gem
       require 'rubygems/gem_runner'
@@ -124,10 +151,10 @@ module Embulk
     begin
       op.parse!(argv)
       unless args.include?(argv.length)
-        usage nil
+        usage_op op, nil
       end
     rescue => e
-      usage e.to_s
+      usage_op op, e.to_s
     end
 
     case subcmd.to_sym
@@ -146,19 +173,10 @@ module Embulk
           success = false
 
           # copy embulk/data/bundle/ directory
-          if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
-            # data is in embulk-core jar
-            resource_class = org.embulk.command.Runner.java_class
-            %w[.bundle/config embulk/input_example.rb embulk/filter_example.rb embulk/output_example.rb Gemfile Gemfile.lock].each do |file|  # TODO get file list from the jar
-              url = resource_class.resource("/embulk/data/bundle/#{file}").to_s
-              dst = File.join(path, file)
-              FileUtils.mkdir_p File.dirname(dst)
-              FileUtils.cp(url, dst)
-            end
-          else
-            #tmpl = File.join(File.dirname(__FILE__), '../data/bundle')
-            tmpl = File.join(Embulk.home('lib'), 'embulk', 'data', 'bundle')
-            FileUtils.cp_r tmpl, path
+          require 'embulk/data/package_data'
+          pkg = PackageData.new("bundle", path)
+          %w[.bundle/config embulk/input/example.rb embulk/output/example.rb embulk/filter/example.rb Gemfile].each do |file|
+            pkg.cp(file, file)
           end
 
           ## TODO this is disabled for now. enable this if you want to use
@@ -207,6 +225,36 @@ module Embulk
       puts "   3. run config.yml"
       puts ""
 
+    when :new
+      lang_cate = ARGV[0]
+      name = ARGV[1]
+
+      language, category = case lang_cate
+        when "java-input"       then [:java, :input]
+        when "java-output"      then [:java, :output]
+        when "java-filter"      then [:java, :filter]
+        when "java-file-input"  then [:java, :file_input]
+        when "java-file-output" then [:java, :file_output]
+        when "java-parser"      then [:java, :parser]
+        when "java-formatter"   then [:java, :formatter]
+        when "java-decoder"     then [:java, :decoder]
+        when "java-encoder"     then [:java, :encoder]
+        when "ruby-input"       then [:ruby, :input]
+        when "ruby-output"      then [:ruby, :output]
+        when "ruby-filter"      then [:ruby, :filter]
+        when "ruby-file-input"  then raise "ruby-file-input is not implemented yet. See #21 on github." #[:ruby, :file_input]
+        when "ruby-file-output" then raise "ruby-file-output is not implemented yet. See #22 on github." #[:ruby, :file_output]
+        when "ruby-parser"      then raise "ruby-parser is not implemented yet. See #33 on github." #[:ruby, :parser]
+        when "ruby-formatter"   then raise "ruby-formatter is not implemented yet. See #34 on github." #[:ruby, :formatter]
+        when "ruby-decoder"     then raise "ruby-decoder is not implemented yet. See #31 on github." #[:ruby, :decoder]
+        when "ruby-encoder"     then raise "ruby-decoder is not implemented yet. See #32 on github." #[:ruby, :encoder]
+        else
+          usage_op op, "Unknown category '#{lang_cate}'"
+        end
+
+      require 'embulk/command/embulk_new_plugin'
+      Embulk.new_plugin(name, language, category)
+
     else
       require 'json'
 
@@ -265,6 +313,7 @@ module Embulk
     STDERR.puts "   guess     <partial-config.yml> -o <output.yml>     # guess missing parameters to create a complete configuration file."
     STDERR.puts "   gem       <install | list | help>                  # install a plugin or show installed plugins."
     STDERR.puts "                                                      # plugin path is #{ENV['GEM_HOME']}"
+    STDERR.puts "   new       <category> <name>                        # generates new plugin template"
     STDERR.puts "   example   [path]                                   # creates an example config file and csv file to try embulk."
     STDERR.puts ""
     if message
@@ -274,4 +323,13 @@ module Embulk
     end
     exit 1
   end
+
+  def self.usage_op(op, message)
+    STDERR.puts op.help
+    STDERR.puts
+    if message
+      STDERR.puts message
+    end
+    exit 1
+  end
 end

data/lib/embulk/data/bundle/Gemfile

@@ -1,31 +1,23 @@
 source 'https://rubygems.org/'
-
-# Plugin bundle:
 #
-# a) Using plugins released on RubyGems:
-#   You can write plugin names (and versions) to this file to include
-#   the plugins released on RubyGems. To use those plugins,
+# Note: After updating this file, run this command to update Gemfile.lock:
 #
-#   1. modify this Gemfile. Examples are at the end of this file.
-#   2. update Gemfile.lock using: embulk bundle <this dir>
-#   3. run embulk with -b option: embulk run -b <this dir> example.yml
+#       embulk bundle <this_dir>
 #
-# b) Using simple single-file ruby plugins:
-#   You can put a simple ruby plugins to <this dir>/embulk directory.
-#   When you run embulk with -b <this dir> option, embulk loads them:
+
 #
-#   $ embulk run -b <this dir> example.yml
+# RubyGems Plugins
 #
-#   See <this dir>/embulk/* files for example. When the plugin works,
-#   you can release the plugin to RubyGems with "embulk-plugin-foobar"
-#   name to deploy them to servers.
+#gem 'embulk-output-mysql'
+#gem 'embulk-output-postgresql', '>= 0.1.0'
+#gem 'embulk-input-baz', '= 0.2.0'
+#gem 'embulk-input-xyz', '~> 0.3.0'
 #
 
-# Plugins
 #
-#gem 'embulk-plugin-foo'
-#gem 'embulk-plugin-bar', '~> 0.1.0'
-#gem 'embulk-plugin-baz', '= 0.2.0'
-#gem 'embulk-plugin-xyz', '>= 0.3.0'
+# Private plugins
+#
+#gem 'embulk-inupt-awesome', path: '/path/to/your/embulk-input-awesome'
+#gem 'embulk-output-awesome', git: 'https://github.com/you/embulk-output-awesome.git', branch: 'master'
 #
 

data/lib/embulk/data/bundle/embulk/{filter_example.rb → filter/example.rb}

@@ -1,8 +1,8 @@
 module Embulk
-  module Plugin
+  module Filter
 
-    class FilterExample < FilterPlugin
-      # filter plugin file name must be: embulk/filter_<name>.rb
+    class ExampleFilterPlugin < FilterPlugin
+      # filter plugin file name must be: embulk/filter/<name>.rb
       Plugin.register_filter('example', self)
 
       def self.transaction(config, in_schema, &control)

data/lib/embulk/data/bundle/embulk/{input_example.rb → input/example.rb}

@@ -1,8 +1,8 @@
 module Embulk
-  module Plugin
+  module Input
 
-    class InputExample < InputPlugin
-      # input plugin file name must be: embulk/input_<name>.rb
+    class ExampleInputPlugin < InputPlugin
+      # input plugin file name must be: embulk/input/<name>.rb
       Plugin.register_input('example', self)
 
       def self.transaction(config, &control)

data/lib/embulk/data/bundle/embulk/{output_example.rb → output/example.rb}

@@ -1,8 +1,8 @@
 module Embulk
-  module Plugin
+  module Output
 
-    class OutputExample < OutputPlugin
-      # output plugin file name must be: embulk/output_<name>.rb
+    class ExampleOutputPlugin < OutputPlugin
+      # output plugin file name must be: embulk/output/<name>.rb
       Plugin.register_output('example', self)
 
       def self.transaction(config, schema, count, &control)

data/lib/embulk/data/new/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/embulk/data/new/README.md.erb

@@ -0,0 +1,75 @@
+# <%= display_name %> <%= display_category %> plugin for Embulk
+
+TODO: Write short description here
+
+## Overview
+
+* **Plugin type**: <%= embulk_category %>
+* **Load all or nothing**: yes
+* **Resume supported**: no
+
+## Configuration
+
+- **property1**: description (string, required)
+- **property2**: description (integer, default: default-value)
+
+## Example
+
+```yaml
+%case category
+%when :input, :file_input
+in:
+  type: <%= name %>
+  property1: example1
+  property2: example2
+%when :output, :file_output
+out:
+  type: <%= name %>
+  property1: example1
+  property2: example2
+%when :filter
+filters:
+  - type: <%= name %>
+    property1: example1
+    property2: example2
+%when :parser
+in:
+  type: any file input plugin type
+  parser:
+    type: <%= name %>
+    property1: example1
+    property2: example2
+%when :formatter
+out:
+  type: any output input plugin type
+  formatter:
+    type: <%= name %>
+    property1: example1
+    property2: example2
+%when :decoder
+in:
+  type: any output input plugin type
+  decoders:
+    - type: <%= name %>
+      property1: example1
+      property2: example2
+%when :encoder
+out:
+  type: any output input plugin type
+  encoders:
+    - type: <%= name %>
+      property1: example1
+      property2: example2
+%end
+```
+
+## Build
+
+```
+%case language
+%when :ruby
+$ rake
+%when :java
+$ ./gradlew gem
+% end
+```