embulk 0.7.0-java

data/lib/embulk/guess/gzip.rb

@@ -0,0 +1,18 @@
+module Embulk
+  module Guess
+
+    class GzipGuessPlugin < GuessPlugin
+      Plugin.register_guess('gzip', self)
+
+      GZIP_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
+
+      def guess(config, sample_buffer)
+        if sample_buffer[0,2] == GZIP_HEADER
+          return {"decoders" => [{"type" => "gzip"}]}
+        end
+        return {}
+      end
+    end
+
+  end
+end

data/lib/embulk/guess/newline.rb

@@ -0,0 +1,22 @@
+module Embulk
+  module Guess
+
+    class NewlineGuessPlugin < TextGuessPlugin
+      Plugin.register_guess('newline', self)
+
+      def guess_text(config, sample_text)
+        cr_count = sample_text.count("\r")
+        lf_count = sample_text.count("\n")
+        crlf_count = sample_text.scan(/\r\n/).length
+        if crlf_count > cr_count / 2 && crlf_count > lf_count / 2
+          return {"parser" => {"newline" => "CRLF"}}
+        elsif cr_count > lf_count / 2
+          return {"parser" => {"newline" => "CR"}}
+        else
+          return {"parser" => {"newline" => "LF"}}
+        end
+      end
+    end
+
+  end
+end

data/lib/embulk/guess/schema_guess.rb

@@ -0,0 +1,118 @@
+module Embulk::Guess
+  require 'embulk/column'
+  require 'embulk/guess/time_format_guess'
+
+  module SchemaGuess
+    class TimestampTypeMatch < String
+      def initialize(format)
+        super("timestamp")
+        @format = format
+      end
+
+      attr_reader :format
+    end
+
+    class << self
+      def from_hash_records(array_of_hash)
+        array_of_hash = Array(array_of_hash)
+        if array_of_hash.empty?
+          raise "SchemaGuess Can't guess schema from no records"
+        end
+        column_names = array_of_hash.first.keys
+        samples = array_of_hash.to_a.map {|hash| column_names.map {|name| hash[name] } }
+        from_array_records(column_names, samples)
+      end
+
+      def from_array_records(column_names, samples)
+        column_types = types_from_array_records(samples)
+        columns = column_types.zip(column_names).map do |(type,name)|
+          hash = {name: name, type: type.to_sym}
+          hash[:format] = type.format if type.is_a?(TimestampTypeMatch)
+          Embulk::Column.new(hash)
+        end
+        return Embulk::Schema.new(columns)
+      end
+
+      # TODO this method will be private once guess/csv is refactored
+      def types_from_array_records(samples)
+        columnar_types = []
+        samples.each do |record|
+          record.each_with_index {|value,i| (columnar_types[i] ||= []) << guess_type(value.to_s) }
+        end
+        columnar_types.map {|types| merge_types(types) }
+      end
+
+      private
+
+      def guess_type(str)
+        if TRUE_STRINGS[str] || FALSE_STRINGS[str]
+          return "boolean"
+        end
+
+        if TimeFormatGuess.guess(str)
+          return TimestampTypeMatch.new(str)
+        end
+
+        if str.to_i.to_s == str
+          return "long"
+        end
+
+        if str.include?('.')
+          a, b = str.split(".", 2)
+          if a.to_i.to_s == a && b.to_i.to_s == b
+            return "double"
+          end
+        end
+
+        if str.empty?
+          return nil
+        end
+
+        return "string"
+      end
+
+      def merge_types(types)
+        t = types.inject(nil) {|r,t| merge_type(r,t) } || "string"
+        if t.is_a?(TimestampTypeMatch)
+          format = TimeFormatGuess.guess(types.map {|type| type.is_a?(TimestampTypeMatch) ? type.format : nil }.compact)
+          return TimestampTypeMatch.new(format)
+        else
+          return t
+        end
+      end
+
+      # taken from CsvParserPlugin.TRUE_STRINGS
+      TRUE_STRINGS = Hash[%w[
+        true True TRUE
+        yes Yes YES
+        t T y Y
+        on On ON
+      ].map {|k| [k, true] }]
+
+      # When matching to false string, then retrun 'true'
+      FALSE_STRINGS = Hash[%w[
+        false False FALSE
+        no No NO
+        f N n N
+        off Off OFF
+      ].map {|k| [k, true] }]
+
+      TYPE_COALESCE = Hash[{
+        long: :double,
+        boolean: :long,
+      }.map {|k,v|
+        [[k.to_s, v.to_s].sort, v.to_s]
+      }]
+
+      def merge_type(type1, type2)
+        if type1 == type2
+          type1
+        elsif type1.nil? || type2.nil?
+          type1 || type2
+        else
+          TYPE_COALESCE[[type1, type2].sort] || "string"
+        end
+      end
+    end
+  end
+end

data/lib/embulk/guess/time_format_guess.rb

@@ -0,0 +1,394 @@
+module Embulk::Guess
+  module TimeFormatGuess
+    module Parts
+      YEAR = /[1-4][0-9]{3}/
+      MONTH         = /10|11|12|[0 ]?[0-9]/
+      MONTH_NODELIM = /10|11|12|[0][0-9]/
+      DAY         = /31|30|[1-2][0-9]|[0 ]?[1-9]/
+      DAY_NODELIM = /31|30|[1-2][0-9]|[0][1-9]/
+      HOUR         = /20|21|22|23|24|1[0-9]|[0 ]?[0-9]/
+      HOUR_NODELIM = /20|21|22|23|24|1[0-9]|[0][0-9]/
+      MINUTE         = SECOND         = /60|[1-5][0-9]|[0 ]?[0-9]/
+      MINUTE_NODELIM = SECOND_NODELIM = /60|[1-5][0-9]|[0][0-9]/
+
+      MONTH_NAME_SHORT = /Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec/
+      MONTH_NAME_FULL = /January|February|March|April|May|June|July|August|September|October|November|December/
+
+      WEEKDAY_NAME_SHORT = /Sun|Mon|Tue|Wed|Thu|Fri|Sat/
+      WEEKDAY_NAME_FULL = /Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday/
+
+      ZONE_OFF = /(?:Z|[\-\+]\d\d(?::?\d\d)?)/
+      ZONE_ABB = /[A-Z]{1,3}/
+    end
+
+    class GuessMatch
+      def initialize(delimiters, parts, part_options)
+        @delimiters = delimiters
+        @parts = parts
+        @part_options = part_options
+      end
+
+      def format
+        format = ''
+        @parts.size.times do |i|
+          format << @delimiters[i-1] if i != 0
+          option = @part_options[i]
+
+          case @parts[i]
+          when :year
+            format << '%Y'
+
+          when :month
+            case option
+            when :zero
+              format << '%m'
+            when :blank
+              #format << '%_m'  # not supported
+              format << '%m'
+            when :none
+              #format << '%-m'  # not supported
+              format << '%m'
+            else
+              format << '%m'
+            end
+
+          when :day
+            case option
+            when :zero
+              format << '%d'
+            when :blank
+              format << '%e'
+            when :none
+              format << '%d'  # not supported
+            else
+              format << '%d'
+            end
+
+          when :hour
+            case option
+            when :zero
+              format << '%H'
+            when :blank
+              format << '%k'
+            when :none
+              format << '%k'  # not supported
+            else
+              format << '%H'
+            end
+
+          when :minute
+            # heading options are not supported
+            format << '%M'
+
+          when :second
+            # heading options are not supported
+            format << '%S'
+
+          when :frac
+            if option <= 3
+              format << '%L'
+            #elsif option <= 6
+            #  format << '%6N'
+            #elsif option <= 6
+            #  format << '%6N'
+            #elsif option <= 9
+            #  format << '%9N'
+            #elsif option <= 12
+            #  format << '%12N'
+            #elsif option <= 15
+            #  format << '%15N'
+            #elsif option <= 18
+            #  format << '%18N'
+            #elsif option <= 21
+            #  format << '%21N'
+            #elsif option <= 24
+            #  format << '%24N'
+            else
+              format << '%N'
+            end
+
+          when :zone_off
+            format << '%z'
+
+          when :zone_abb
+            format << '%Z'
+
+          else
+            raise "Unknown part: #{@parts[i]}"
+          end
+        end
+
+        return format
+      end
+
+      def mergeable_group
+        # MDY is mergible with DMY
+        if i = array_sequence_find(@parts, [:day, :month, :year])
+          ps = @parts.dup
+          ps[i, 3] = [:month, :day, :year]
+          [@delimiters, ps]
+        else
+          [@delimiters, @parts]
+        end
+      end
+
+      attr_reader :parts
+      attr_reader :part_options
+
+      def merge!(another_in_group)
+        part_options = another_in_group.part_options
+        @part_options.size.times do |i|
+          @part_options[i] ||= part_options[i]
+          if @part_options[i] == nil
+            part_options[i]
+          elsif part_options[i] == nil
+            @part_options[i]
+          else
+            [@part_options[i], part_options[i]].sort.last
+          end
+        end
+
+        # if DMY matches, MDY is likely false match of DMY.
+        dmy = array_sequence_find(another_in_group.parts, [:day, :month, :year])
+        mdy = array_sequence_find(@parts, [:month, :day, :year])
+        if mdy && dmy
+          @parts[mdy, 3] = [:day, :month, :year]
+        end
+      end
+
+      def array_sequence_find(array, seq)
+        (array.size - seq.size + 1).times {|i|
+          return i if array[i, seq.size] == seq
+        }
+        return nil
+      end
+    end
+
+    class GuessPattern
+      include Parts
+
+      date_delims = /[\/\-\.]/
+      # yyyy-MM-dd
+      YMD         = /(?<year>#{YEAR})(?<date_delim>#{date_delims})(?<month>#{MONTH})\k<date_delim>(?<day>#{DAY})/
+      YMD_NODELIM = /(?<year>#{YEAR})(?<month>#{MONTH_NODELIM})(?<day>#{DAY_NODELIM})/
+      # MM/dd/yyyy
+      MDY         = /(?<month>#{MONTH})(?<date_delim>#{date_delims})(?<day>#{DAY})\k<date_delim>(?<year>#{YEAR})/
+      MDY_NODELIM = /(?<month>#{MONTH_NODELIM})(?<day>#{DAY_NODELIM})(?<year>#{YEAR})/
+      # dd.MM.yyyy
+      DMY         = /(?<day>#{DAY})(?<date_delim>#{date_delims})(?<month>#{MONTH})\k<date_delim>(?<year>#{YEAR})/
+      DMY_NODELIM = /(?<day>#{DAY_NODELIM})(?<month>#{MONTH_NODELIM})(?<year>#{YEAR})/
+
+      frac = /[0-9]{1,9}/
+      time_delims = /[\:\-]/
+      frac_delims = /[\.\,]/
+      TIME         = /(?<hour>#{HOUR})(?:(?<time_delim>#{time_delims})(?<minute>#{MINUTE})(?:\k<time_delim>(?<second>#{SECOND})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?)?/
+      TIME_NODELIM = /(?<hour>#{HOUR_NODELIM})(?:(?<minute>#{MINUTE_NODELIM})((?<second>#{SECOND_NODELIM})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?)?/
+
+      ZONE = /(?<zone_space> )?(?<zone>(?<zone_off>#{ZONE_OFF})|(?<zone_abb>#{ZONE_ABB}))/
+
+      def match(text)
+        delimiters = []
+        parts = []
+        part_options = []
+
+        if dm = (/^#{YMD}(?<rest>.*?)$/.match(text) or /^#{YMD_NODELIM}(?<rest>.*?)$/.match(text))
+          date_delim = dm["date_delim"] rescue ""
+
+          parts << :year
+          part_options << nil
+
+          delimiters << date_delim
+          parts << :month
+          part_options << part_heading_option(dm["month"])
+
+          delimiters << date_delim
+          parts << :day
+          part_options << part_heading_option(dm["day"])
+
+        elsif dm = (/^#{MDY}(?<rest>.*?)$/.match(text) or /^#{MDY_NODELIM}(?<rest>.*?)$/.match(text))
+          date_delim = dm["date_delim"] rescue ""
+
+          parts << :month
+          part_options << part_heading_option(dm["month"])
+
+          delimiters << date_delim
+          parts << :day
+          part_options << part_heading_option(dm["day"])
+
+          delimiters << date_delim
+          parts << :year
+          part_options << nil
+
+        elsif dm = (/^#{DMY}(?<rest>.*?)$/.match(text) or /^#{DMY_NODELIM}(?<rest>.*?)$/.match(text))
+          date_delim = dm["date_delim"] rescue ""
+
+          parts << :day
+          part_options << part_heading_option(dm["day"])
+
+          delimiters << date_delim
+          parts << :month
+          part_options << part_heading_option(dm["month"])
+
+          delimiters << date_delim
+          parts << :year
+          part_options << nil
+
+        else
+          date_delim = ""
+          return nil
+        end
+        rest = dm["rest"]
+
+        date_time_delims = /(:? |_|T|\. ?)/
+        if tm = (
+              /^(?<date_time_delim>#{date_time_delims})#{TIME}(?<rest>.*?)?$/.match(rest) or
+              /^(?<date_time_delim>#{date_time_delims})#{TIME_NODELIM}(?<rest>.*?)?$/.match(rest) or
+              (date_delim == "" && /^#{TIME_NODELIM}(?<rest>.*?)?$/.match(rest))
+            )
+          date_time_delim = tm["date_time_delim"] rescue ""
+          time_delim = tm["time_delim"] rescue ""
+
+          delimiters << date_time_delim
+          parts << :hour
+          part_options << part_heading_option(tm["hour"])
+
+          if tm["minute"]
+            delimiters << time_delim
+            parts << :minute
+            part_options << part_heading_option(tm["minute"])
+
+            if tm["second"]
+              delimiters << time_delim
+              parts << :second
+              part_options << part_heading_option(tm["second"])
+
+              if tm["frac"]
+                delimiters << tm["frac_delim"]
+                parts << :frac
+                part_options << tm["frac"].size
+              end
+            end
+          end
+
+          rest = tm["rest"]
+        end
+
+        if zm = /^#{ZONE}$/.match(rest)
+          delimiters << (zm["zone_space"] || '')
+          if zm["zone_off"]
+            parts << :zone_off
+          else
+            parts << :zone_abb
+          end
+          part_options << nil
+
+          return GuessMatch.new(delimiters, parts, part_options)
+
+        elsif rest =~ /^\s*$/
+          return GuessMatch.new(delimiters, parts, part_options)
+
+        else
+          return nil
+        end
+      end
+
+      def part_heading_option(text)
+        if text[0] == '0'
+          :zero
+        elsif text[0] == ' '
+          :blank
+        elsif text.size == 1
+          :none
+        else
+          nil
+        end
+      end
+    end
+
+    class SimpleMatch
+      def initialize(format)
+        @format = format
+      end
+
+      attr_reader :format
+
+      def mergeable_group
+        @format
+      end
+
+      def merge!(another_in_group)
+      end
+    end
+
+    class Rfc2822Pattern
+      include Parts
+
+      def initialize
+        @regexp = /^(?<weekday>#{WEEKDAY_NAME_SHORT}, )?\d\d #{MONTH_NAME_SHORT} \d\d\d\d(?<time> \d\d:\d\d(?<second>:\d\d)? (?:(?<zone_off>#{ZONE_OFF})|(?<zone_abb>#{ZONE_ABB})))?$/
+      end
+
+      def match(text)
+        if m = @regexp.match(text)
+          format = ''
+          format << "%a, " if m['weekday']
+          format << "%d %b %Y"
+          format << " %H:%M" if m['time']
+          format << ":%S" if m['second']
+          format << " %z" if m['zone_off']
+          format << " %Z" if m['zone_abb']
+          SimpleMatch.new(format)
+        else
+          nil
+        end
+      end
+    end
+
+    class RegexpPattern
+      def initialize(regexp, format)
+        @regexp = regexp
+        @match = SimpleMatch.new(format)
+      end
+
+      def match(text)
+        if @regexp =~ text
+          return @match
+        else
+          return nil
+        end
+      end
+    end
+
+    module StandardPatterns
+      include Parts
+
+      APACHE_CLF = /^\d\d\/#{MONTH_NAME_SHORT}\/\d\d\d\d:\d\d:\d\d:\d\d #{ZONE_OFF}?$/
+      ANSI_C_ASCTIME = /^#{WEEKDAY_NAME_SHORT} #{MONTH_NAME_SHORT} \d\d? \d\d:\d\d:\d\d \d\d\d\d$/
+    end
+
+    PATTERNS = [
+      GuessPattern.new,
+      Rfc2822Pattern.new,
+      RegexpPattern.new(StandardPatterns::APACHE_CLF, "%d/%b/%Y:%H:%M:%S %z"),
+      RegexpPattern.new(StandardPatterns::ANSI_C_ASCTIME, "%a %b %e %H:%M:%S %Y"),
+    ]
+
+    def self.guess(texts)
+      texts = Array(texts).map {|text| text.to_s }
+      texts.reject! {|text| text == "" }
+      matches = texts.map do |text|
+        PATTERNS.map {|pattern| pattern.match(text) }.compact
+      end.flatten
+      if matches.empty?
+        return nil
+      elsif matches.size == 1
+        return matches[0].format
+      else
+        match_groups = matches.group_by {|match| match.mergeable_group }.values
+        best_match_group = match_groups.sort_by {|group| group.size }.last
+        best_match = best_match_group.shift
+        best_match_group.each {|m| best_match.merge!(m) }
+        return best_match.format
+      end
+    end
+  end
+end