embulk 0.6.2 → 0.6.3

data/embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java

@@ -0,0 +1,79 @@
+package org.embulk.spi.unit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import org.junit.Test;
+
+public class TestByteSize
+{
+    @Test
+    public void testUnitPatterns()
+    {
+        assertByteSize(42L, "42");
+        assertByteSize(42L, "42B");
+        assertByteSize(42L*(1L << 10), "42KB");
+        assertByteSize(42L*(1L << 20), "42MB");
+        assertByteSize(42L*(1L << 30), "42GB");
+        assertByteSize(42L*(1L << 40), "42TB");
+        assertByteSize(42L*(1L << 50), "42PB");
+        assertByteSize(42L, "42 B");
+        assertByteSize(42L*(1L << 10), "42 KB");
+    }
+
+    @Test
+    public void testUnknownUnits()
+    {
+        assertInvalidByteSize("42XB");
+        assertInvalidByteSize("42 XB");
+    }
+
+    @Test
+    public void testInvalidPatterns()
+    {
+        assertInvalidByteSize(" 42");
+        assertInvalidByteSize("42  B");
+        assertInvalidByteSize("42 B ");
+        assertInvalidByteSize("42B ");
+        assertInvalidByteSize("42  KB");
+        assertInvalidByteSize("42 KB ");
+        assertInvalidByteSize("42KB ");
+    }
+
+    @Test
+    public void testInvalidValues()
+    {
+        assertInvalidByteSize("9223372036854775KB");
+    }
+
+    @Test
+    public void testToString()
+    {
+        assertByteSizeString("42B", "42 B");
+        assertByteSizeString("42KB", "42 KB");
+        assertByteSizeString("42MB", "42 MB");
+        assertByteSizeString("42GB", "42 GB");
+        assertByteSizeString("42TB", "42 TB");
+        assertByteSizeString("42PB", "42 PB");
+        assertByteSizeString("42.20KB", "42.2 KB");
+        assertByteSizeString("42.33KB", "42.33KB");
+    }
+
+    private static void assertByteSize(long bytes, String string)
+    {
+        assertEquals(bytes, ByteSize.parseByteSize(string).getBytes());
+    }
+
+    private static void assertByteSizeString(String expected, String string)
+    {
+        assertEquals(expected, ByteSize.parseByteSize(string).toString());
+    }
+
+    private static void assertInvalidByteSize(String string)
+    {
+        try {
+            ByteSize.parseByteSize(string);
+            fail();
+        } catch (IllegalArgumentException ex) {
+        }
+    }
+}

data/embulk-docs/build.gradle

@@ -1,5 +1,10 @@
 apply plugin: 'com.github.jruby-gradle.base'
 
+import com.github.jrubygradle.JRubyExec
+jruby {
+    execVersion = project.jrubyVersion
+}
+
 dependencies {
     jrubyExec 'rubygems:yard:0.8.7.6'
 }
@@ -16,7 +21,6 @@ task javadoc_html(type: Copy, dependsOn: [':embulk-core:javadoc']) {
     into 'build/html/javadoc'
 }
 
-import com.github.jrubygradle.JRubyExec
 task rdoc_html(type: JRubyExec) {
     workingDir '..'
     jrubyArgs '-ryard', '-eYARD::CLI::Yardoc.run(*ARGV)', '--', '-o', 'embulk-docs/build/html/rdoc'

data/embulk-docs/src/customization.rst

@@ -0,0 +1,184 @@
+Customization
+==================================
+
+.. contents::
+   :local:
+   :depth: 2
+
+
+Creating plugins
+------------------
+
+Creating a new plugin is 4 steps:
+
+1. Create a new project using a template generator
+2. Build the project (Java only)
+3. Confirm it works
+4. Modify the code as you need
+
+This article describes how to create a plugin step by step.
+
+Step 1: Creating a new project
+~~~~~~~~~~~~~~~~~~
+
+Embulk comes with a number of templates that generates a new project so that you can start development instantly. Because the generated project contains completely working code without additional code, you can focus on the necessary coding.
+
+Usage of the generator is where ``<type>`` is one of the plugin templates:
+
+::
+
+    $ embulk new <type> <name>
+
+Here is the list of available templates. Please find the best option:
+
+====================  ===============================  =================
+Type                  description                      example
+====================  ===============================  =================
+**java-input**        Java record input plugin         ``mysql``
+**java-output**       Java record output plugin        ``mysql``
+**java-filter**       Java record filter plugin        ``add-hostname``
+**java-file-input**   Java file input plugin           ``ftp``
+**java-file-output**  Java file output plugin          ``ftp``
+**java-parser**       Java file parser plugin          ``csv``
+**java-formatter**    Java file formatter plugin       ``csv``
+**java-decoder**      Java file decoder plugin         ``gzip``
+**java-encoder**      Java file encoder plugin         ``gzip``
+**ruby-input**        Ruby record input plugin         ``mysql``
+**ruby-output**       Ruby record output plugin        ``mysql``
+**ruby-filter**       Ruby record filter plugin        ``add-hostname``
+**ruby-parser**       Ruby file parser plugin          ``csv``
+**ruby-formatter**    Ruby file formatter plugin       ``csv```
+====================  ===============================  =================
+
+For example, if you want to parse a new file format using Java, type this command:
+
+::
+
+    $ embulk new java-parser myformat
+
+This will create a Java-based parser plugin called ``myformat`` in ``embulk-parser-myformat`` directory.
+
+Step 2: Building the project
+~~~~~~~~~~~~~~~~~~
+
+If the plugin is Java-based, you need to build the project. To build, type this command:
+
+::
+
+    $ cd embulk-parser-myformat
+    $ ./gradlew package
+
+Now, the plugin is ready to use.
+
+Step 3: Confirm it works
+~~~~~~~~~~~~~~~~~~
+
+The next step is to actually use the plugin.
+
+Let's supporse you have a configuration file named ``your-config.yml``. You can use the plugin using embulk with ``-L`` argument:
+
+::
+
+    $ embulk run -L ./embulk-parser-myformat/ your-config.yml
+
+Step 4: Modifying the code
+~~~~~~~~~~~~~~~~~~
+
+The final step is to modify code as you want!
+
+The code is located at
+
+* Java-based plugins
+
+  * src/org/embulk/*
+
+* Ruby-based plugins
+
+  * lib/embulk/*
+
+There are a lot of good code examples on Github. Search repositories by `embulk-<type> keyword <https://github.com/search?q=embulk-output>`_.
+
+Releasing plugins
+------------------
+
+You can release publicly so that all people can use your awesome plugins.
+
+Checking plugin description
+~~~~~~~~~~~~~~~~~~
+
+To prepare the plugin ready to release, you need to include some additional information. The plugin information is written in this file:
+
+* Java-based plugins
+
+  * ``build.gradle`` file
+
+* Ruby-based plugins
+
+  * ``embulk-<type>-<name>.gemspec`` file (``<type>`` is plugin type and ``<name>`` is plugin name)
+
+You will find following section in the file.
+
+.. code-block:: ruby
+
+    Gem::Specification.new do |spec|
+        # ...
+
+        spec.authors       = ["Your Name"]
+        spec.summary       = %[Myformat parser plugin for Embulk]
+        spec.description   = %[Parses Myformat files read by other file input plugins.]
+        spec.email         = ["you@example.org"]
+        spec.licenses      = ["MIT"]
+        spec.homepage      = "https://github.com/frsyuki/embulk-parser-myformat"
+
+        # ...
+    end
+
+The items in above example are important. Please make sure that they are good.
+
+Creating account on RubyGems.org
+~~~~~~~~~~~~~~~~~~
+
+Embulk uses `RubyGems.org <https://rubygems.org/>`_ as a package distribution service. Please create an account there to release plugins at `Sign Up <https://rubygems.org/sign_up>`_ page.
+
+Don't forget the password! It will be necessary at the next step.
+
+Releasing the plugin to RubyGems.org
+~~~~~~~~~~~~~~~~~~
+
+Now, you're ready to release the plugin. To release, type following command:
+
+* Java-based plugins
+
+  * ``$ ./gradlew gemPush``
+
+* Ruby-based plugins
+
+  * ``$ rake release``
+
+If everything is good, you can find your plugin at https://rubygems.org/. Congratulations!
+
+Installing your plugin
+~~~~~~~~~~~~~~~~~~
+
+Usage of plugin installer is:
+
+::
+
+    $ embulk gem install embulk-<type>-<name>
+
+``<type>`` is plugin type and ``<name>`` is plugin name.
+
+If your plugin is ``embulk-parser-myformat``, then type this command:
+
+::
+
+    $ embulk gem install embulk-parser-myformat
+
+This command installs the plugin to ``~/.embulk`` directory.
+
+To check the list of installed plugins and their versions, use this command:
+
+::
+
+    $ embulk gem list
+

data/embulk-docs/src/index.rst

@@ -63,7 +63,11 @@ Documents
    :maxdepth: 2
 
    built-in
-   release
+
+.. toctree::
+   :maxdepth: 3
+
+   customization
 
 * `JavaDoc <javadoc/index.html>`_
 
@@ -71,3 +75,8 @@ Documents
 
 * `Github <https://github.com/embulk/embulk>`_
 
+.. toctree::
+   :maxdepth: 2
+
+   release
+

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.3
     release/release-0.6.2
     release/release-0.6.1
     release/release-0.6.0

data/embulk-docs/src/release/release-0.6.3.rst

@@ -0,0 +1,23 @@
+Release 0.6.3
+==================================
+
+Plugin API
+------------------
+
+* Added ``org.embulk.spi.unit.ByteSize`` utility class. Plugins can use this class to allow units such as 'KB' or 'MB' in the configuration file.
+
+Command-line
+------------------
+
+* Added '-L, --load PATH' argument to load a plugin form a local directory.
+
+General Changes
+------------------
+
+* Updated template generator for Java so that generated build.gradle file has ``package`` task to create gemspec file.
+* PreviewExecutor and GuessExecutor unwrap ExecutionException. Error stacktrace becomes simple.
+* EmbulkService creates JRuby runtime and its execution environment for each Injector instances.
+
+Release Date
+------------------
+2015-04-21

data/lib/embulk/command/embulk_run.rb

@@ -36,10 +36,16 @@ module Embulk
 
     puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: Embulk v#{Embulk::VERSION}"
 
+    plugin_paths = []
     load_paths = []
     classpaths = []
     classpath_separator = java.io.File.pathSeparator
-    options = {}
+
+    options = {
+      # use the global ruby runtime (the jruby Runtime running this embulk_run.rb script) for all
+      # ScriptingContainer injected by the org.embulk.command.Runner.
+      useGlobalRubyRuntime: true,
+    }
 
     op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
       # only for help message. implemented at lib/embulk/command/embulk.rb
@@ -61,6 +67,9 @@ module Embulk
       op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
+      op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
+        plugin_paths << plugin_path
+      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -80,6 +89,9 @@ module Embulk
       op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
+      op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
+        plugin_paths << plugin_path
+      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -96,6 +108,9 @@ module Embulk
       op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
+      op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
+        plugin_paths << plugin_path
+      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -115,6 +130,9 @@ module Embulk
       op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|
         options[:nextConfigOutputPath] = path
       end
+      op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
+        plugin_paths << plugin_path
+      end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
         load_paths << load_path
       end
@@ -186,6 +204,7 @@ examples:
 
       require 'fileutils'
       require 'rubygems/gem_runner'
+      setup_plugin_paths(plugin_paths)
       setup_load_paths(load_paths)
       setup_classpaths(classpaths)
 
@@ -292,6 +311,7 @@ examples:
         end
       end
 
+      setup_plugin_paths(plugin_paths)
       setup_load_paths(load_paths)
       setup_classpaths(classpaths)
 
@@ -331,6 +351,24 @@ examples:
     Gem.clear_paths  # force rubygems to reload GEM_HOME
   end
 
+  def self.setup_plugin_paths(plugin_paths)
+    plugin_paths.each do |path|
+      unless File.directory?(path)
+        raise "Path '#{path}' is not a directory"
+      end
+      specs = Dir[File.join(File.expand_path(path), "*.gemspec")]
+      if specs.empty?
+        raise "Path '#{path}' does not include *.gemspec file. Hint: Did you run './gradlew package' command?"
+      end
+      specs.each do |spec|
+        gem_path = File.dirname(spec)
+        stub = Gem::StubSpecification.new(spec)
+        stub.define_singleton_method(:full_gem_path) { gem_path }
+        Gem::Specification.add_spec(stub)
+      end
+    end
+  end
+
   def self.setup_load_paths(load_paths)
     # first $LOAD_PATH has highet priority. later load_paths should have highest priority.
     load_paths.each do |load_path|
@@ -340,9 +378,9 @@ examples:
   end
 
   def self.setup_classpaths(classpaths)
-    classpaths.each {|classpath|
+    classpaths.each do |classpath|
       $CLASSPATH << classpath  # $CLASSPATH object doesn't have concat method
-    }
+    end
   end
 
   def self.usage(message)

data/lib/embulk/data/new/gitignore.erb

@@ -2,6 +2,7 @@
 /pkg/
 /tmp/
 %if language == :java
+*.gemspec
 .gradle/
 /classpath/
 build/