embulk 0.4.1 → 0.4.2

data/lib/embulk/guess/newline.rb

@@ -0,0 +1,22 @@
+module Embulk
+  module Guess
+
+    class NewlineGuessPlugin < TextGuessPlugin
+      Plugin.register_guess('newline', self)
+
+      def guess_text(config, sample_text)
+        cr_count = sample_text.count("\r")
+        lf_count = sample_text.count("\n")
+        crlf_count = sample_text.scan(/\r\n/).length
+        if crlf_count > cr_count / 2 && crlf_count > lf_count / 2
+          return {"parser" => {"newline" => "CRLF"}}
+        elsif cr_count > lf_count / 2
+          return {"parser" => {"newline" => "CR"}}
+        else
+          return {"parser" => {"newline" => "LF"}}
+        end
+      end
+    end
+
+  end
+end

data/lib/embulk/guess/time_format_guess.rb

@@ -0,0 +1,333 @@
+module Embulk::Guess
+  module TimeFormatGuess
+    module Parts
+      YEAR = /[1-4][0-9]{3}/
+      MONTH         = /10|11|12|[0 ]?[0-9]/
+      MONTH_NODELIM = /10|11|12|[0][0-9]/
+      DAY         = /[1-2][0-9]|[0 ]?[1-9]|30|31/
+      DAY_NODELIM = /[1-2][0-9]|[0][1-9]|30|31/
+      HOUR         = /20|21|22|23|24|1[0-9]|[0 ]?[0-9]/
+      HOUR_NODELIM = /20|21|22|23|24|1[0-9]|[0][0-9]/
+      MINUTE         = SECOND         = /60|[1-5][0-9]|[0 ]?[0-9]/
+      MINUTE_NODELIM = SECOND_NODELIM = /60|[1-5][0-9]|[0][0-9]/
+
+      MONTH_NAME_SHORT = /Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec/
+      MONTH_NAME_FULL = /January|February|March|April|May|June|July|August|September|October|November|December/
+
+      WEEKDAY_NAME_SHORT = /Sun|Mon|Tue|Wed|Thu|Fri|Sat/
+      WEEKDAY_NAME_FULL = /Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday/
+    end
+
+    class GuessMatch
+      def initialize(delimiters, parts, part_options)
+        @delimiters = delimiters
+        @parts = parts
+        @part_options = part_options
+      end
+
+      def format
+        format = ''
+        @parts.size.times do |i|
+          format << @delimiters[i-1] if i != 0
+          option = @part_options[i]
+
+          case @parts[i]
+          when :year
+            format << '%Y'
+
+          when :month
+            case option
+            when :zero
+              format << '%m'
+            when :blank
+              #format << '%_m'  # not supported
+              format << '%m'
+            when :none
+              #format << '%-m'  # not supported
+              format << '%m'
+            else
+              format << '%m'
+            end
+
+          when :day
+            case option
+            when :zero
+              format << '%d'
+            when :blank
+              format << '%e'
+            when :none
+              format << '%d'  # not supported
+            else
+              format << '%d'
+            end
+
+          when :hour
+            case option
+            when :zero
+              format << '%H'
+            when :blank
+              format << '%k'
+            when :none
+              format << '%k'  # not supported
+            else
+              format << '%H'
+            end
+
+          when :minute
+            # heading options are not supported
+            format << '%M'
+
+          when :second
+            # heading options are not supported
+            format << '%S'
+
+          when :frac
+            if option <= 3
+              format << '%L'
+            #elsif option <= 6
+            #  format << '%6N'
+            #elsif option <= 6
+            #  format << '%6N'
+            #elsif option <= 9
+            #  format << '%9N'
+            #elsif option <= 12
+            #  format << '%12N'
+            #elsif option <= 15
+            #  format << '%15N'
+            #elsif option <= 18
+            #  format << '%18N'
+            #elsif option <= 21
+            #  format << '%21N'
+            #elsif option <= 24
+            #  format << '%24N'
+            else
+              format << '%N'
+            end
+
+          when :zone_off
+            format << '%z'
+
+          when :zone_abb
+            format << '%Z'
+
+          else
+            raise "Unknown part: #{@parts[i]}"
+          end
+        end
+
+        return format
+      end
+
+      def mergeable_group
+        [@delimiters, @parts]
+      end
+
+      attr_reader :part_options
+
+      def merge!(another_in_group)
+        part_options = another_in_group.part_options
+        @part_options.size.times do |i|
+          @part_options[i] ||= part_options[i]
+          if @part_options[i] == nil
+            part_options[i]
+          elsif part_options[i] == nil
+            @part_options[i]
+          else
+            [@part_options[i], part_options[i]].sort.last
+          end
+        end
+      end
+    end
+
+    class GuessPattern
+      include Parts
+
+      date_delims = /[\/\-]/
+      # yyyy-MM-dd
+      YMD         = /(?<year>#{YEAR})(?<date_delim>#{date_delims})(?<month>#{MONTH})\k<date_delim>(?<day>#{DAY})/
+      YMD_NODELIM = /(?<year>#{YEAR})(?<month>#{MONTH_NODELIM})(?<day>#{DAY_NODELIM})/
+      # dd/MM/yyyy
+      DMY         = /(?<year>#{YEAR})(?<date_delim>#{date_delims})(?<month>#{MONTH})\k<date_delim>(?<day>#{DAY})/
+      DMY_NODELIM = /(?<year>#{YEAR})(?<month>#{MONTH_NODELIM})(?<day>#{DAY_NODELIM})/
+
+      frac = /[0-9]{1,24}/
+      time_delims = /[\:\-]/
+      frac_delims = /[\.\,]/
+      TIME         = /(?<hour>#{HOUR})(?<time_delim>#{time_delims})(?<minute>#{MINUTE})(?:\k<time_delim>(?<second>#{SECOND})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?/
+      TIME_NODELIM = /(?<hour>#{HOUR_NODELIM})(?<minute>#{MINUTE_NODELIM})((?<second>#{SECOND_NODELIM})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?/
+
+      TZ = /(?<zone_space> )?(?<zone>(?<zone_off>[\-\+]\d\d(?::?\d\d)?)|(?<zone_abb>[A-Z]{3}))|(?<z>Z)/
+
+      def match(text)
+        delimiters = []
+        parts = []
+        part_options = []
+
+        if dm = (/^#{YMD}(?<rest>.*?)$/.match(text) or /^#{YMD_NODELIM}(?<rest>.*?)$/.match(text))
+          date_delim = dm["date_delim"] rescue ""
+
+          parts << :year
+          part_options << nil
+          delimiters << date_delim
+
+          parts << :month
+          part_options << part_heading_option(dm["month"])
+          delimiters << date_delim
+
+          parts << :day
+          part_options << part_heading_option(dm["day"])
+
+        elsif dm = (/^#{DMY}(?<rest>.*?)$/.match(text) or /^#{DMY_NODELIM}(?<rest>.*?)$/.match(text))
+          date_delim = dm["date_delim"] rescue ""
+
+          parts << :day
+          part_options << part_heading_option(dm["day"])
+          delimiters << date_delim
+
+          parts << :month
+          part_options << part_heading_option(dm["month"])
+          delimiters << date_delim
+
+          parts << :year
+          part_options << nil
+          delimiters << date_delim
+
+        else
+          date_delim = ""
+          return nil
+        end
+        rest = dm["rest"]
+
+        date_time_delims = /[ _T]/
+        if tm = (
+              /^(?<date_time_delim>#{date_time_delims})#{TIME}(?<rest>.*?)?$/.match(rest) or
+              /^(?<date_time_delim>#{date_time_delims})#{TIME_NODELIM}(?<rest>.*?)?$/.match(rest) or
+              (date_delim == "" && /^#{TIME_NODELIM}(?<rest>.*?)?$/.match(rest))
+            )
+          date_time_delim = tm["date_time_delim"] rescue ""
+          time_delim = tm["time_delim"] rescue ""
+
+          delimiters << date_time_delim
+          parts << :hour
+          part_options << part_heading_option(tm["hour"])
+
+          delimiters << time_delim
+          parts << :minute
+          part_options << part_heading_option(tm["minute"])
+
+          if tm["second"]
+            delimiters << time_delim
+            parts << :second
+            part_options << part_heading_option(tm["second"])
+          end
+
+          if tm["frac"]
+            delimiters << tm["frac_delim"]
+            parts << :frac
+            part_options << tm["frac"].size
+          end
+
+          rest = tm["rest"]
+        end
+
+        if zm = /^#{TZ}$/.match(rest)
+          delimiters << zm["zone_space"] || ''
+          if zm["z"]
+            # TODO ISO 8601
+            parts << :zone_off
+          elsif zm["zone_off"]
+            parts << :zone_off
+          else
+            parts << :zone_abb
+          end
+          part_options << nil
+
+          return GuessMatch.new(delimiters, parts, part_options)
+
+        elsif rest =~ /^\s*$/
+          return GuessMatch.new(delimiters, parts, part_options)
+
+        else
+          return nil
+        end
+      end
+
+      def part_heading_option(text)
+        if text[0] == '0'
+          :zero
+        elsif text[0] == ' '
+          :blank
+        elsif text.size == 1
+          :none
+        else
+          nil
+        end
+      end
+    end
+
+    class RegexpMatch
+      def initialize(format)
+        @format
+      end
+
+      attr_reader :format
+
+      def mergeable_group
+        @format
+      end
+
+      def merge!(another_in_group)
+      end
+    end
+
+    class RegexpPattern
+      def initialize(regexp, format)
+        @regexp = regexp
+        @match = RegexpMatch.new(format)
+      end
+
+      def match(text)
+        if @regexp =~ text
+          return @match
+        else
+          return nil
+        end
+      end
+    end
+
+    module StandardPatterns
+      include Parts
+
+      RFC_822_1123 = /^#{WEEKDAY_NAME_SHORT}, \d\d #{MONTH_NAME_SHORT} \d\d\d\d \d\d:\d\d:\d\d [a-zA-Z]{3}$/
+      RFC_850_1035 = /^#{WEEKDAY_NAME_FULL}, \d\d-#{MONTH_NAME_SHORT}-\d\d \d\d:\d\d:\d\d [a-zA-Z]{3}$/
+      APACHE_CLF = /^\d\d\/#{MONTH_NAME_SHORT}\/\d\d\d\d \d\d:\d\d:\d\d [\-\+]\d\d(?::?\d\d)?$/
+      ANSI_C_ASCTIME = /^#{WEEKDAY_NAME_SHORT} #{MONTH_NAME_SHORT} \d\d? \d\d:\d\d:\d\d \d\d\d\d$/
+    end
+
+    PATTERNS = [
+      GuessPattern.new,
+      RegexpPattern.new(StandardPatterns::RFC_822_1123, "%a, %d %b %Y %H:%M:%S %z"),
+      RegexpPattern.new(StandardPatterns::RFC_850_1035, "%A, %d-%b-%y %H:%M:%S %z"),
+      RegexpPattern.new(StandardPatterns::APACHE_CLF, "%d/%b/%Y %H:%M:%S %Z"),
+      RegexpPattern.new(StandardPatterns::ANSI_C_ASCTIME, "$a %b %e %H:%M:%S %Y"),
+    ]
+
+    def self.guess(texts)
+      texts = Array(texts).select {|text| text != "" }
+      matches = texts.map do |text|
+        PATTERNS.map {|pattern| pattern.match(text) }.compact
+      end.flatten
+      if matches.empty?
+        return nil
+      elsif matches.size == 1
+        return matches[0].format
+      else
+        match_groups = matches.group_by {|match| match.mergeable_group }
+        best_match_group = match_groups.sort_by {|group| group.size }.last[1]
+        best_match = best_match_group.shift
+        best_match_group.each {|m| best_match.merge!(m) }
+        return best_match.format
+      end
+    end
+  end
+end

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -267,6 +267,7 @@ files:
 - embulk-docs/src/release/release-0.3.2.rst
 - embulk-docs/src/release/release-0.4.0.rst
 - embulk-docs/src/release/release-0.4.1.rst
+- embulk-docs/src/release/release-0.4.2.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -335,10 +336,11 @@ files:
 - lib/embulk/file_output_plugin.rb
 - lib/embulk/filter_plugin.rb
 - lib/embulk/formatter_plugin.rb
-- lib/embulk/guess_charset.rb
-- lib/embulk/guess_csv.rb
-- lib/embulk/guess_gzip.rb
-- lib/embulk/guess_newline.rb
+- lib/embulk/guess/charset.rb
+- lib/embulk/guess/csv.rb
+- lib/embulk/guess/gzip.rb
+- lib/embulk/guess/newline.rb
+- lib/embulk/guess/time_format_guess.rb
 - lib/embulk/guess_plugin.rb
 - lib/embulk/input_plugin.rb
 - lib/embulk/java/bootstrap.rb
@@ -352,7 +354,6 @@ files:
 - lib/embulk/plugin.rb
 - lib/embulk/plugin_registry.rb
 - lib/embulk/schema.rb
-- lib/embulk/time_format_guess.rb
 - lib/embulk/version.rb
 - settings.gradle
 - classpath/annotations-3.0.0.jar
@@ -361,8 +362,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.4.1.jar
-- classpath/embulk-standards-0.4.1.jar
+- classpath/embulk-core-0.4.2.jar
+- classpath/embulk-standards-0.4.2.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar

data/lib/embulk/guess_charset.rb

@@ -1,26 +0,0 @@
-module Embulk
-
-  class GuessCharset < GuessPlugin
-    Plugin.register_guess('charset', self)
-
-    def guess(config, sample_buffer)
-      # ICU4J
-      detector = com.ibm.icu.text.CharsetDetector.new
-      detector.setText(sample_buffer.to_java_bytes)
-      best_match = detector.detect
-      if best_match.getConfidence < 50
-        name = "UTF-8"
-      else
-        name = best_match.getName
-        if name == "ISO-8859-1"
-          # ISO-8859-1 means ASCII which is a subset
-          # of UTF-8 in most of cases due to lack of
-          # sample data set
-          name = "UTF-8"
-        end
-      end
-      return {"parser" => {"charset" => name}}
-    end
-  end
-
-end

data/lib/embulk/guess_csv.rb

@@ -1,204 +0,0 @@
-module Embulk
-  require_relative 'time_format_guess'
-
-  class GuessCsv < LineGuessPlugin
-    Plugin.register_guess('csv', self)
-
-    DELIMITER_CANDIDATES = [
-      ",", "\t", "|"
-    ]
-
-    QUOTE_CANDIDATES = [
-      "\"", "'"
-    ]
-
-    # CsvParserPlugin.TRUE_STRINGS
-    TRUE_STRINGS = Hash[*%w[
-      true True TRUE
-      yes Yes YES
-      y Y
-      on On ON
-      1
-    ].map {|k| [k, true] }]
-
-    def guess_lines(config, sample_lines)
-      delim = guess_delimiter(sample_lines)
-      unless delim
-        # not CSV file
-        return {}
-      end
-
-      parser_config = config["parser"] || {}
-      parser_guessed = {"type" => "csv", "delimiter" => delim}
-
-      quote = guess_quote(sample_lines, delim)
-      parser_guessed["quote"] = quote ? quote : ''
-
-      sample_records = sample_lines.map {|line| line.split(delim) }  # TODO use CsvTokenizer
-      first_types = guess_field_types(sample_records[0, 1])
-      other_types = guess_field_types(sample_records[1..-1])
-
-      if first_types.size <= 1 || other_types.size <= 1
-        # guess failed
-        return {}
-      end
-
-      unless parser_config.has_key?("header_line")
-        parser_guessed["header_line"] = (first_types != other_types && !first_types.any? {|t| t != ["string"] })
-      end
-
-      unless parser_config.has_key?("columns")
-        if parser_guessed["header_line"] || parser_config["header_line"]
-          column_names = sample_records.first
-        else
-          column_names = (0..other_types.size).to_a.map {|i| "c#{i}" }
-        end
-        schema = []
-        column_names.zip(other_types).each do |name,(type,format)|
-          if name && type
-            if format
-              schema << {"name" => name, "type" => type, "format" => format}
-            else
-              schema << {"name" => name, "type" => type}
-            end
-          end
-        end
-        parser_guessed["columns"] = schema
-      end
-
-      return {"parser" => parser_guessed}
-    end
-
-    private
-
-    def guess_delimiter(sample_lines)
-      delim_weights = DELIMITER_CANDIDATES.map do |d|
-        counts = sample_lines.map {|line| line.count(d) }
-        total = array_sum(counts)
-        if total > 0
-          stddev = array_standard_deviation(counts)
-          stddev = 0.000000001 if stddev == 0.0
-          weight = total / stddev
-          [d, weight]
-        else
-          [nil, 0]
-        end
-      end
-
-      delim, weight = *delim_weights.sort_by {|d,weight| weight }.last
-      if delim != nil && weight > 1
-        return delim
-      else
-        return nil
-      end
-    end
-
-    def guess_quote(sample_lines, delim)
-      delim_regexp = Regexp.escape(delim)
-      quote_weights = QUOTE_CANDIDATES.map do |q|
-        weights = sample_lines.map do |line|
-          q_regexp = Regexp.escape(q)
-          count = line.count(q)
-          if count > 0
-            weight = count
-            weight += line.scan(/(?:\A|#{delim_regexp})\s*#{q_regexp}(?:(?!#{q_regexp}).)*\s*#{q_regexp}(?:$|#{delim_regexp})/).size * 20
-            weight += line.scan(/(?:\A|#{delim_regexp})\s*#{q_regexp}(?:(?!#{delim_regexp}).)*\s*#{q_regexp}(?:$|#{delim_regexp})/).size * 40
-            weight
-          else
-            nil
-          end
-        end.compact
-        weights.empty? ? 0 : array_avg(weights)
-      end
-      quote, weight = QUOTE_CANDIDATES.zip(quote_weights).sort_by {|q,w| w }.last
-      if weight >= 10.0
-        return quote
-      else
-        return nil
-      end
-    end
-
-    def guess_field_types(field_lines)
-      column_lines = []
-      field_lines.each do |fields|
-        fields.each_with_index {|field,i| (column_lines[i] ||= []) << guess_type(field) }
-      end
-      columns = column_lines.map do |types|
-        t = types.inject(nil) {|r,t| merge_type(r,t) } || "string"
-        if t.is_a?(TimestampMatch)
-          format = TimeFormatGuess.guess(types.map {|type| type.text })
-          ["timestamp", format]
-        else
-          [t]
-        end
-      end
-      return columns
-    end
-
-    TYPE_COALESCE = Hash[{
-      long: :double,
-      boolean: :long,
-    }.map {|k,v|
-      [[k.to_s, v.to_s].sort, v.to_s]
-    }]
-
-    def merge_type(type1, type2)
-      if type1 == type2
-        type1
-      elsif type1.nil? || type2.nil?
-        type1 || type2
-      else
-        TYPE_COALESCE[[type1, type2].sort] || "string"
-      end
-    end
-
-    class TimestampMatch < String
-      def initialize(text)
-        super("timestamp")
-        @text = text
-      end
-      attr_reader :text
-    end
-
-    def guess_type(str)
-      if TRUE_STRINGS[str]
-        return "boolean"
-      end
-
-      if TimeFormatGuess.guess(str)
-        return TimestampMatch.new(str)
-      end
-
-      if str.to_i.to_s == str
-        return "long"
-      end
-
-      if str.include?('.')
-        a, b = str.split(".", 2)
-        if a.to_i.to_s == a && b.to_i.to_s == b
-          return "double"
-        end
-      end
-
-      return "string"
-    end
-
-    def array_sum(array)
-      array.inject(0) {|r,i| r += i }
-    end
-
-    def array_avg(array)
-      array.inject(0.0) {|r,i| r += i } / array.size
-    end
-
-    def array_variance(array)
-      avg = array_avg(array)
-      array.inject(0.0) {|r,i| r += (i - avg) ** 2 } / array.size
-    end
-
-    def array_standard_deviation(array)
-      Math.sqrt(array_variance(array))
-    end
-  end
-
-end