embulk 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 157c521d28f82705b9162cca7ca03e052d5e478f
-  data.tar.gz: 5063ed2e3544fede216449ea7cebc11c427faf9c
+  metadata.gz: 57cffd859609745fafc0e644e0e8466c0511c622
+  data.tar.gz: 5df593df4e75dce977ac7ab295f9fe089ababa1a
 SHA512:
-  metadata.gz: 694c3da663fbb60de1ac1529e9d17c561785bc0637713d3d6d325e11fdd69204e200bb359d2592b177cf5a05fb743b53d80083dfb9216807e3f1cc65a54de47b
-  data.tar.gz: 3b4dcaf0704cce4a6bf6af89f8e1929412b7d85c54b59d98da84efdc24ef2d55dc28f4e347fe8b15f7f887c49c77a70a405ea3777723c5b2aed7f0c0a89fb1d2
+  metadata.gz: 2c613261f18c551e6eb1ea9f748148422054d0818a76d8367de8eb68a071445686e871ce304d1413b10eca8570d220b4d53dbd6f3b7f060b008ce3c500903fe9
+  data.tar.gz: 0e6c8086818d6f7d81ac8009ee5fbf76d0aa0a34fc962af60f3647d189ca488cd3c4aa09c8b49f62b3551fcd9b4ad42e2582b69f6a56f2c78f9f6e06f6986439

data/README.md

@@ -24,7 +24,7 @@ You can release plugins to share your efforts of data cleaning, error handling,
 The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java:
 
 ```
-wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.4.jar -O embulk.jar
+wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.5.jar -O embulk.jar
 java -jar embulk.jar --help
 ```
 
@@ -40,10 +40,10 @@ java -jar embulk.jar run     config.yml
 ### Using plugins
 
 You can use plugins to load data from/to various systems and file formats.
-An example is [embulk-plugin-postgres-json](https://github.com/frsyuki/embulk-plugin-postgres-json) plugin. It outputs data into PostgreSQL server using "json" column type.
+An example is [embulk-output-postgres-json](https://github.com/frsyuki/embulk-output-postgres-json) plugin. It outputs data into PostgreSQL server using "json" column type.
 
 ```
-java -jar embulk.jar gem install embulk-plugin-postgres-json
+java -jar embulk.jar gem install embulk-output-postgres-json
 java -jar embulk.jar gem list
 ```
 

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.4.4'
+    version = '0.4.5'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'

data/embulk-docs/src/release.rst

@@ -15,4 +15,5 @@ Release Notes
     release/release-0.4.2
     release/release-0.4.3
     release/release-0.4.4
+    release/release-0.4.5
 

data/embulk-docs/src/release/release-0.4.5.rst

@@ -0,0 +1,24 @@
+Release 0.4.5
+==================================
+
+CLI
+------------------
+
+* Fixed a problem where ``embulk gem install foo --version x.y.z`` shows embulk's version.
+
+Plugin SPI
+------------------
+
+* Added ParserPlugin SPI for JRuby. You can write parser plugins in Ruby.
+* Added FormatterPlugin SPI for JRuby. You can write formatter plugins in Ruby.
+
+
+Plugin API Changes
+------------------
+
+* Added ``Embulk::FileInput`` API used mainly by parser plugins.
+* Added ``Embulk::FileOutput`` API used mainly by formatter plugins.
+
+Release Date
+------------------
+2015-02-19

data/lib/embulk/buffer.rb

@@ -9,6 +9,11 @@ module Embulk
         buffer
       end
 
+      def self.from_ruby_string(string)
+        b = Buffer.new(string)
+        b.force_encoding('ASCII-8BIT')
+      end
+
       def to_java
         Java::Buffer.wrap(to_java_bytes)
       end

data/lib/embulk/command/embulk_run.rb

@@ -19,18 +19,20 @@ module Embulk
 
     require 'embulk/version'
 
-    if argv.include?('--version')
-      puts "embulk #{Embulk::VERSION}"
-      exit 0
-    end
-
     i = argv.find_index {|arg| arg !~ /^\-/ }
-    usage nil unless i
+    unless i
+      if argv.include?('--version')
+        puts "embulk #{Embulk::VERSION}"
+        exit 0
+      end
+      usage nil
+    end
     subcmd = argv.slice!(i)
 
     require 'java'
     require 'optparse'
     op = OptionParser.new
+    op.version = Embulk::VERSION
 
     puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S,%3N %z")}: Embulk v#{Embulk::VERSION}"
 
@@ -127,8 +129,8 @@ categories:
     ruby-filter                Ruby record filter plugin   (like "add-hostname")
     #ruby-file-input           Ruby file input plugin      (like "ftp")          # not implemented yet [#21]
     #ruby-file-output          Ruby file output plugin     (like "ftp")          # not implemented yet [#22]
-    #ruby-parser               Ruby file parser plugin     (like "csv")          # not implemented yet [#33]
-    #ruby-formatter            Ruby file formatter plugin  (like "csv")          # not implemented yet [#34]
+    ruby-parser                Ruby file parser plugin     (like "csv")          # not implemented yet [#33]
+    ruby-formatter             Ruby file formatter plugin  (like "csv")          # not implemented yet [#34]
     #ruby-decoder              Ruby file decoder plugin    (like "gzip")         # not implemented yet [#31]
     #ruby-encoder              Ruby file encoder plugin    (like "gzip")         # not implemented yet [#32]
     java-input                 Java record input plugin    (like "mysql")
@@ -259,8 +261,8 @@ examples:
         when "ruby-filter"      then [:ruby, :filter]
         when "ruby-file-input"  then raise "ruby-file-input is not implemented yet. See #21 on github." #[:ruby, :file_input]
         when "ruby-file-output" then raise "ruby-file-output is not implemented yet. See #22 on github." #[:ruby, :file_output]
-        when "ruby-parser"      then raise "ruby-parser is not implemented yet. See #33 on github." #[:ruby, :parser]
-        when "ruby-formatter"   then raise "ruby-formatter is not implemented yet. See #34 on github." #[:ruby, :formatter]
+        when "ruby-parser"      then [:ruby, :parser]
+        when "ruby-formatter"   then [:ruby, :formatter]
         when "ruby-decoder"     then raise "ruby-decoder is not implemented yet. See #31 on github." #[:ruby, :decoder]
         when "ruby-encoder"     then raise "ruby-decoder is not implemented yet. See #32 on github." #[:ruby, :encoder]
         else

data/lib/embulk/data/new/ruby/filter.rb.erb

@@ -7,8 +7,8 @@ module Embulk
       def self.transaction(config, in_schema, &control)
         # configuration code:
         task = {
-          "property1" => config.param("property1", :string)
-          "property2" => config.param("property2", :integer, default: 0)
+          "property1" => config.param("property1", :string),
+          "property2" => config.param("property2", :integer, default: 0),
         }
 
         yield(task, out_columns)

data/lib/embulk/data/new/ruby/formatter.rb.erb

@@ -0,0 +1,47 @@
+module Embulk
+  module Formatter
+
+    class <%= ruby_class_name %> < FormatterPlugin
+      Plugin.register_formatter(<%= name.dump %>, self)
+
+      def self.transaction(config, schema, &control)
+        # configuration code:
+        task = {
+          "property1" => config.param("property1", :string),
+          "property2" => config.param("property2", :integer, default: 0),
+        }
+
+        yield(task)
+      end
+
+      def init
+        # initialization code:
+        @property1 = task["property1"]
+        @property2 = task["property2"]
+
+        # your data
+        @current_file == nil
+        @current_file_size = 0
+      end
+
+      def close
+      end
+
+      def add(page)
+        # output code:
+        page.each do |record|
+          if @current_file == nil || @current_file_size > 32*1024
+            @current_file = @file_output.next_file
+            @current_file_size = 0
+          end
+          @current_file.write "|mydata|"
+        end
+      end
+
+      def finish
+        @file_output.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/input.rb.erb

@@ -7,13 +7,13 @@ module Embulk
       def self.transaction(config, &control)
         # configuration code:
         task = {
-          "property1" => config.param("property1", :string)
-          "property2" => config.param("property2", :integer, default: 0)
+          "property1" => config.param("property1", :string),
+          "property2" => config.param("property2", :integer, default: 0),
         }
 
         columns = [
           Column.new(0, "example", :string),
-          Column.new(1, "column", :integer),
+          Column.new(1, "column", :long),
           Column.new(2, "name", :double),
         ]
 

data/lib/embulk/data/new/ruby/output.rb.erb

@@ -7,8 +7,8 @@ module Embulk
       def self.transaction(config, schema, count, &control)
         # configuration code:
         task = {
-          "property1" => config.param("property1", :string)
-          "property2" => config.param("property2", :integer, default: 0)
+          "property1" => config.param("property1", :string),
+          "property2" => config.param("property2", :integer, default: 0),
         }
 
         # resumable output:

data/lib/embulk/data/new/ruby/parser.rb.erb

@@ -0,0 +1,42 @@
+module Embulk
+  module Parser
+
+    class <%= ruby_class_name %> < ParserPlugin
+      Plugin.register_parser(<%= name.dump %>, self)
+
+      def self.transaction(config, &control)
+        # configuration code:
+        task = {
+          "property1" => config.param("property1", :string),
+          "property2" => config.param("property2", :integer, default: 0),
+        }
+
+        columns = [
+          Column.new(0, "example", :string),
+          Column.new(1, "column", :long),
+          Column.new(2, "name", :double),
+        ]
+
+        yield(task, columns)
+      end
+
+      def init
+        # initialization code:
+        @property1 = task["property1"]
+        @property2 = task["property2"]
+      end
+
+      def run(file_input)
+        while file = file_input.next_file
+          file.each do |buffer|
+            # parsering code
+            record = ["col1", 2, 3.0]
+            @page_builder.add(record)
+          end
+        end
+        @page_builder.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/file_input.rb

@@ -0,0 +1,83 @@
+
+module Embulk
+  require 'embulk/buffer'
+
+  class FileInput
+    def initialize(java_file_input)
+      @java_file_input = java_file_input
+      @buffer = nil
+    end
+
+    def next_file
+      if @java_file_input.nextFile
+        return self
+      else
+        return nil
+      end
+    end
+
+    def each(&block)
+      if @buffer
+        yield @buffer
+        @buffer = nil
+      end
+
+      while java_buffer = @java_file_input.poll
+        buffer = Buffer.from_java(java_buffer)
+        java_buffer.release
+        yield buffer
+      end
+    end
+
+    def read(count=nil, dest=nil)
+      if count == nil
+        @buffer ||= Buffer.new
+        while java_buffer = @java_file_input.poll
+          @buffer << Buffer.from_java(java_buffer)
+          java_buffer.release
+        end
+
+        return nil if @buffer.empty? && count != 0
+
+        if dest
+          dest.replace(@buffer)
+        else
+          dest = @buffer
+        end
+        @buffer = nil
+
+      else
+        @buffer ||= Buffer.new
+        until @buffer.size >= count
+          java_buffer = @java_file_input.poll
+          break unless java_buffer
+          @buffer << Buffer.from_java(java_buffer)
+          java_buffer.release
+        end
+
+        return nil if @buffer.empty? && count != 0
+
+        if @buffer.size <= count
+          if dest
+            dest.replace(@buffer)
+          else
+            dest = @buffer
+          end
+          @buffer = nil
+        else
+          data = @buffer.slice!(0, count)
+          if dest
+            dest.replace(data)
+          else
+            dest = data
+          end
+        end
+      end
+      return dest
+    end
+
+    def close
+      @java_file_input.close
+    end
+  end
+end

data/lib/embulk/file_output.rb

@@ -0,0 +1,51 @@
+
+module Embulk
+  require 'embulk/buffer'
+
+  class FileOutput
+    def initialize(java_file_output)
+      @java_file_output = java_file_output
+      @buffer = Buffer.new
+      @buffer.force_encoding('ASCII-8BIT')
+      @flush_size = 32*1024
+    end
+
+    def next_file
+      flush
+      @java_file_output.nextFile
+      self
+    end
+
+    def write(buffer)
+      buffer.force_encoding('ASCII-8BIT')  # TODO this is destructively change buffer
+      @buffer << buffer
+      if @buffer.size > @flush_size
+        flush
+      end
+      nil
+    end
+
+    def add(buffer)
+      flush
+      @java_file_output.add(Buffer.from_ruby_string(buffer))
+      nil
+    end
+
+    def flush
+      unless @buffer.empty?
+        @java_file_output.add(@buffer.to_java)
+      end
+      nil
+    end
+
+    def finish
+      flush
+      @java_file_output.finish
+    end
+
+    def close
+      @java_file_output.finish
+    end
+  end
+
+end

data/lib/embulk/formatter_plugin.rb

@@ -3,7 +3,7 @@ module Embulk
   require 'embulk/data_source'
   require 'embulk/schema'
   require 'embulk/page'
-  #require 'embulk/file_output'  TODO not implemented
+  require 'embulk/file_output'
 
   class FormatterPlugin
     def self.transaction(config, schema, &control)
@@ -14,7 +14,7 @@ module Embulk
     def initialize(task, schema, file_output)
       @task = task
       @schema = schema
-      @file_output
+      @file_output = file_output
       init
     end
 
@@ -58,7 +58,7 @@ module Embulk
         def open(java_task_source, java_schema, java_file_output)
           task_source = DataSource.from_java(java_task_source)
           schema = Schema.from_java(java_schema)
-          file_output = FileOutput.from_java(java_file_output)
+          file_output = FileOutput.new(java_file_output)
           ruby_object = @ruby_class.new(task_source, schema, file_output)
           return OutputAdapter.new(ruby_object, schema, file_output)
         end

data/lib/embulk/parser_plugin.rb

@@ -3,7 +3,7 @@ module Embulk
   require 'embulk/data_source'
   require 'embulk/schema'
   require 'embulk/page_builder'
-  #require 'embulk/file_input'  TODO not implemented
+  require 'embulk/file_input'
 
   class ParserPlugin
     def self.transaction(config, &control)
@@ -49,7 +49,7 @@ module Embulk
         def run(java_task_source, java_schema, java_file_input, java_output)
           task_source = DataSource.from_java(java_task_source)
           schema = Schema.from_java(java_schema)
-          file_input = FileInput.from_java(java_file_input)
+          file_input = FileInput.new(java_file_input)
           page_builder = PageBuilder.new(schema, java_output)
           begin
             @ruby_class.new(task_source, schema, page_builder).run(file_input)

data/lib/embulk/plugin.rb

@@ -36,15 +36,13 @@ module Embulk
       register_plugin(:filter, type, klass, FilterPlugin)
     end
 
-    ## TODO FileInput is not implemented yet.
-    #def register_parser(type, klass)
-    #  register_plugin(:parser, type, klass, ParserPlugin)
-    #end
+    def register_parser(type, klass)
+      register_plugin(:parser, type, klass, ParserPlugin)
+    end
 
-    ## TODO FileOutput is not implemented yet.
-    #def register_formatter(type, klass)
-    #  register_plugin(:formatter, type, klass, FormatterPlugin)
-    #end
+    def register_formatter(type, klass)
+      register_plugin(:formatter, type, klass, FormatterPlugin)
+    end
 
     ## TODO DecoderPlugin JRuby API is not written by anyone yet
     #def register_decoder(type, klass)

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.4.4'
+  VERSION = '0.4.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-19 00:00:00.000000000 Z
+date: 2015-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -271,6 +271,7 @@ files:
 - embulk-docs/src/release/release-0.4.2.rst
 - embulk-docs/src/release/release-0.4.3.rst
 - embulk-docs/src/release/release-0.4.4.rst
+- embulk-docs/src/release/release-0.4.5.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -327,15 +328,19 @@ files:
 - lib/embulk/data/new/ruby/Gemfile
 - lib/embulk/data/new/ruby/Rakefile
 - lib/embulk/data/new/ruby/filter.rb.erb
+- lib/embulk/data/new/ruby/formatter.rb.erb
 - lib/embulk/data/new/ruby/gemspec.erb
 - lib/embulk/data/new/ruby/input.rb.erb
 - lib/embulk/data/new/ruby/output.rb.erb
+- lib/embulk/data/new/ruby/parser.rb.erb
 - lib/embulk/data/package_data.rb
 - lib/embulk/data_source.rb
 - lib/embulk/decoder_plugin.rb
 - lib/embulk/encoder_plugin.rb
 - lib/embulk/error.rb
+- lib/embulk/file_input.rb
 - lib/embulk/file_input_plugin.rb
+- lib/embulk/file_output.rb
 - lib/embulk/file_output_plugin.rb
 - lib/embulk/filter_plugin.rb
 - lib/embulk/formatter_plugin.rb
@@ -365,8 +370,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.4.4.jar
-- classpath/embulk-standards-0.4.4.jar
+- classpath/embulk-core-0.4.5.jar
+- classpath/embulk-standards-0.4.5.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar