embulk 0.4.10 → 0.5.0

data/lib/embulk/guess/time_format_guess.rb

@@ -375,7 +375,8 @@ module Embulk::Guess
     ]
 
     def self.guess(texts)
-      texts = Array(texts).select {|text| text != "" }
+      texts = Array(texts).map {|text| text.to_s }
+      texts.reject! {|text| text == "" }
       matches = texts.map do |text|
         PATTERNS.map {|pattern| pattern.match(text) }.compact
       end.flatten

data/lib/embulk/guess_plugin.rb

@@ -1,5 +1,7 @@
 module Embulk
 
+  require 'embulk/guess/schema_guess'
+
   class GuessPlugin
     def guess(config, sample_buffer)
       raise NotImplementedError, "GuessPlugin#guess(config, sample_buffer) must be implemented"
@@ -45,6 +47,12 @@ module Embulk
 
   class TextGuessPlugin < GuessPlugin
     def guess(config, sample)
+      if config.fetch('parser', {}).fetch('charset', nil).nil?
+        require 'embulk/guess/charset'
+        charset_guess = Guess::CharsetGuessPlugin.new
+        return charset_guess.guess(config, sample)
+      end
+
       # TODO pure-ruby LineDecoder implementation?
       begin
         parser_task = config.param("parser", :hash, default: {}).load_config(Java::LineDecoder::DecoderTask)
@@ -79,6 +87,18 @@ module Embulk
 
   class LineGuessPlugin < GuessPlugin
     def guess(config, sample)
+      if config.fetch('parser', {}).fetch('charset', nil).nil?
+        require 'embulk/guess/charset'
+        charset_guess = Guess::CharsetGuessPlugin.new
+        return charset_guess.guess(config, sample)
+      end
+
+      if config.fetch('parser', {}).fetch('newline', nil).nil?
+        require 'embulk/guess/newline'
+        newline_guess = Guess::NewlineGuessPlugin.new
+        return newline_guess.guess(config, sample)
+      end
+
       # TODO pure-ruby LineDecoder implementation?
       begin
         parser_task = config.param("parser", :hash, default: {}).load_config(Java::LineDecoder::DecoderTask)

data/lib/embulk/input_plugin.rb

@@ -17,6 +17,10 @@ module Embulk
       # do nothing by default
     end
 
+    def self.guess(config)
+      raise NotImplementedError, "#{self}.guess(config) is not implemented. This input plugin does not support guess."
+    end
+
     def initialize(task, schema, index, page_builder)
       @task = task
       @schema = schema
@@ -83,6 +87,12 @@ module Embulk
           return nil
         end
 
+        def guess(java_config)
+          config = DataSource.from_java(java_config)
+          config_diff_hash = @ruby_class.guess(config)
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
+        end
+
         def run(java_task_source, java_schema, processor_index, java_output)
           task_source = DataSource.from_java(java_task_source)
           schema = Schema.from_java(java_schema)

data/lib/embulk/schema.rb

@@ -3,8 +3,15 @@ module Embulk
   require 'embulk/column'
 
   class Schema < Array
-    def initialize(src)
-      super
+    def initialize(columns)
+      columns = columns.map.with_index {|c,index|
+        if c.index && c.index != index
+          # TODO ignore this error?
+          raise "Index of column '#{c.name}' is #{c.index} but it is at column #{index}."
+        end
+        Column.new(index, c.name, c.type, c.format)
+      }
+      super(columns)
 
       record_reader_script =
         "lambda do |reader|\n" <<

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.4.10'
+  VERSION = '0.5.0'
 end

data/test/guess/test_schema_guess.rb

@@ -0,0 +1,11 @@
+require 'helper'
+require 'time'
+require 'embulk/guess/schema_guess'
+
+class SchemaGuessTest < ::Test::Unit::TestCase
+  G = Embulk::Guess::SchemaGuess
+
+  def test_guess
+    G.from_hash_records([{"int" => "1", "str" => "a"}])
+  end
+end

data/test/helper.rb

@@ -1,6 +1,5 @@
 require 'test/unit'
 
-module Embulk
-end
+require 'embulk'
 
 # TODO simplecov

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.4.10
+  version: 0.5.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-27 00:00:00.000000000 Z
+date: 2015-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -129,6 +129,7 @@ files:
 - embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java
 - embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java
 - embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java
+- embulk-core/src/main/java/org/embulk/exec/ForGuess.java
 - embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java
 - embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java
 - embulk-core/src/main/java/org/embulk/exec/LocalExecutor.java
@@ -270,6 +271,7 @@ files:
 - embulk-docs/src/release/release-0.4.7.rst
 - embulk-docs/src/release/release-0.4.8.rst
 - embulk-docs/src/release/release-0.4.9.rst
+- embulk-docs/src/release/release-0.5.0.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -325,12 +327,14 @@ files:
 - lib/embulk/data/new/java/test.java.erb
 - lib/embulk/data/new/ruby/Gemfile
 - lib/embulk/data/new/ruby/Rakefile
+- lib/embulk/data/new/ruby/decoder_guess.rb.erb
 - lib/embulk/data/new/ruby/filter.rb.erb
 - lib/embulk/data/new/ruby/formatter.rb.erb
 - lib/embulk/data/new/ruby/gemspec.erb
 - lib/embulk/data/new/ruby/input.rb.erb
 - lib/embulk/data/new/ruby/output.rb.erb
 - lib/embulk/data/new/ruby/parser.rb.erb
+- lib/embulk/data/new/ruby/parser_guess.rb.erb
 - lib/embulk/data/package_data.rb
 - lib/embulk/data_source.rb
 - lib/embulk/decoder_plugin.rb
@@ -346,6 +350,7 @@ files:
 - lib/embulk/guess/csv.rb
 - lib/embulk/guess/gzip.rb
 - lib/embulk/guess/newline.rb
+- lib/embulk/guess/schema_guess.rb
 - lib/embulk/guess/time_format_guess.rb
 - lib/embulk/guess_plugin.rb
 - lib/embulk/input_plugin.rb
@@ -362,6 +367,7 @@ files:
 - lib/embulk/schema.rb
 - lib/embulk/version.rb
 - settings.gradle
+- test/guess/test_schema_guess.rb
 - test/guess/test_time_format_guess.rb
 - test/helper.rb
 - classpath/annotations-3.0.0.jar
@@ -370,8 +376,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.4.10.jar
-- classpath/embulk-standards-0.4.10.jar
+- classpath/embulk-core-0.5.0.jar
+- classpath/embulk-standards-0.5.0.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar
@@ -418,5 +424,6 @@ signing_key:
 specification_version: 4
 summary: Embulk, a plugin-based parallel bulk data loader
 test_files:
+- test/guess/test_schema_guess.rb
 - test/guess/test_time_format_guess.rb
 - test/helper.rb