bmg 0.18.2 → 0.18.7

data/lib/bmg/reader/csv.rb

@@ -5,30 +5,36 @@ module Bmg
 
       DEFAULT_OPTIONS = {
         :headers => true,
-        :return_headers => false
+        :return_headers => false,
+        :smart => true
       }
 
-      def initialize(type, path, options = {})
+      def initialize(type, path_or_io, options = {})
         @type = type
-        @path = path
+        @path_or_io = path_or_io
         @options = DEFAULT_OPTIONS.merge(options)
-        @options[:col_sep] ||= infer_col_sep
-        @options[:quote_char] ||= infer_quote_char
+        if @options[:smart] && !@path_or_io.is_a?(IO)
+          @options[:col_sep] ||= infer_col_sep
+          @options[:quote_char] ||= infer_quote_char
+        end
       end
 
       def each
+        return to_enum unless block_given?
         require 'csv'
-        ::CSV.foreach(@path, @options) do |row|
-          yield tuple(row)
+        with_io do |io|
+          ::CSV.new(io, **csv_options).each do |row|
+            yield tuple(row)
+          end
         end
       end
 
       def to_ast
-        [ :csv, @path, @options ]
+        [ :csv, @path_or_io, @options ]
       end
 
       def to_s
-        "(csv #{path})"
+        "(csv #{@path_or_io})"
       end
       alias :inspect :to_s
 
@@ -47,7 +53,16 @@ module Bmg
       end
 
       def text_portion
-        @text_portion ||= File.foreach(@path).first(10).join("\n")
+        @text_portion ||= with_io{|io| io.readlines(10).join("\n") }
+      end
+
+      def with_io(&bl)
+        case @path_or_io
+        when IO, StringIO
+          bl.call(@path_or_io)
+        else
+          File.open(@path_or_io, "r", &bl)
+        end
       end
 
       # Finds the best candidate among `candidates` for a separator
@@ -61,6 +76,10 @@ module Bmg
         snif.size > 0 ? snif[0][0] : default
       end
 
+      def csv_options
+        @csv_options ||= @options.dup.tap{|opts| opts.delete(:smart) }
+      end
+
     end # class Csv
   end # module Reader
 end # module Bmg

data/lib/bmg/reader/excel.rb

@@ -4,7 +4,8 @@ module Bmg
       include Reader
 
       DEFAULT_OPTIONS = {
-        skip: 0
+        skip: 0,
+        row_num: true
       }
 
       def initialize(type, path, options = {})
@@ -14,6 +15,7 @@ module Bmg
       end
 
       def each
+        return to_enum unless block_given?
         require 'roo'
         xlsx = Roo::Spreadsheet.open(@path, @options)
         headers = nil
@@ -23,9 +25,13 @@ module Bmg
           .each_with_index
           .each do |row, i|
             if i==0
-              headers = row.map(&:to_sym)
+              headers = row.map{|c| c.to_s.strip.to_sym }
             else
-              tuple = (0...headers.size).each_with_object({}){|i,t| t[headers[i]] = row[i] }
+              init = init_tuple(i)
+              tuple = (0...headers.size)
+                .each_with_object(init){|i,t|
+                  t[headers[i]] = row[i]
+                }
               yield(tuple)
             end
           end
@@ -36,10 +42,23 @@ module Bmg
       end
 
       def to_s
-        "(excel #{path})"
+        "(excel #{@path})"
       end
       alias :inspect :to_s
 
+    private
+
+      def init_tuple(i)
+        case as = @options[:row_num]
+        when TrueClass
+          { :row_num => i }
+        when FalseClass
+          {}
+        when Symbol
+          { :"#{as}" => i }
+        end
+      end
+
     end # class Excel
   end # module Reader
 end # module Bmg

data/lib/bmg/relation.rb

@@ -27,6 +27,12 @@ module Bmg
       }
     end
 
+    def with_type_attrlist
+      return self if type.knows_attrlist?
+      attrs = self.first.keys
+      with_type(type.with_attrlist(attrs))
+    end
+
     def with_typecheck
       dup.tap{|r|
         r.type = r.type.with_typecheck

data/lib/bmg/relation/in_memory.rb

@@ -8,7 +8,6 @@ module Bmg
         @type = type
       end
       attr_accessor :type
-      protected :type=
       attr_reader :operand
 
     public

data/lib/bmg/sequel/relation.rb

@@ -9,6 +9,7 @@ module Bmg
       attr_reader :sequel_db
 
       def each(&bl)
+        return to_enum unless block_given?
         dataset.each(&bl)
       end
 

data/lib/bmg/sequel/translator.rb

@@ -78,8 +78,15 @@ module Bmg
       end
 
       def on_func_call(sexpr)
-        args = sexpr.func_args.map{|fa| apply(fa) }
-        ::Sequel.function(sexpr.func_name, *args)
+        case sexpr.func_name
+        when :cast
+          to_cast = apply(sexpr.func_args.first)
+          type = sexpr.func_args.last.last
+          to_cast.cast(type)
+        else
+          args = sexpr.func_args.map{|fa| apply(fa) }
+          ::Sequel.function(sexpr.func_name, *args)
+        end
       end
 
       def on_summarizer(sexpr)

data/lib/bmg/sql.rb

@@ -2,7 +2,10 @@ require 'sexpr'
 module Bmg
 
   module Sql
-  end
+
+    class NotSupportedError < Bmg::Error; end
+
+  end # module Sql
 
   def sql(table, type = Type::ANY)
     builder = Sql::Builder.new

data/lib/bmg/sql/processor.rb

@@ -85,4 +85,5 @@ require_relative 'processor/semi_join'
 require_relative 'processor/flatten'
 require_relative 'processor/requalify'
 require_relative 'processor/summarize'
+require_relative 'processor/transform'
 require_relative 'processor/bind'

data/lib/bmg/sql/processor/transform.rb

@@ -0,0 +1,105 @@
+module Bmg
+  module Sql
+    class Processor
+      class Transform < Processor
+
+        module SplitSupported
+          extend(self)
+
+          def split_supported(tr, &bl)
+            case tr
+            when Array
+              i = tr.find_index{|x| !bl.call(x) } || tr.size
+              [tr[0...i], tr[i..-1]].map{|a|
+                case a.size
+                when 0 then nil
+                when 1 then a.first
+                else a
+                end
+              }
+            when Hash
+              tr.inject([{}, {}]){|(sup,unsup),(k,v)|
+                mine, hers = _split_supported(v, &bl)
+                [
+                  sup.merge(k => mine),
+                  unsup.merge(k => hers)
+                ].map(&:compact)
+              }.map{|h| h.empty? ? nil : h }
+            else
+              _split_supported(tr, &bl)
+            end
+          end
+
+          def _split_supported(tr, &bl)
+            if tr.is_a?(Array)
+              split_supported(tr, &bl)
+            else
+              bl.call(tr) ? [tr, nil] : [nil, tr]
+            end
+          end
+        end # module SplitSupported
+
+        def initialize(transformation, options, builder)
+          raise NotSupportedError unless options.empty?
+          super(builder)
+          @transformation = transformation
+        end
+        attr_reader :transformation
+
+        def self.split_supported(*args, &bl)
+          SplitSupported.split_supported(*args, &bl)
+        end
+
+        def on_select_list(sexpr)
+          sexpr.each_with_index.map{|child,index|
+            index == 0 ? child : apply(child)
+          }
+        end
+
+        def on_select_item(sexpr)
+          as = sexpr.as_name.to_sym
+          case t = transformation_for(as)
+          when NilClass
+            sexpr
+          when Class, Array
+            sexpr([:select_item,
+              func_call_node(sexpr, Array(t).reverse),
+              sexpr[2]
+            ])
+          else
+            raise NotSupportedError
+          end
+        end
+
+      private
+
+        def func_call_node(sexpr, ts)
+          _func_call_node(sexpr, ts.first, ts[1..-1])
+        end
+
+        def _func_call_node(sexpr, head, tail)
+          inside = if tail.empty?
+            sexpr[1]
+          else
+            _func_call_node(sexpr, tail.first, tail[1..-1])
+          end
+          [:func_call,
+            :cast,
+            inside,
+            [ :literal, head ] ]
+        end
+
+        def transformation_for(as)
+          case t = transformation
+          when Class then t
+          when Hash  then t[as]
+          when Array then t
+          else
+            raise Sql::NotSupportedError, "Unable to use `#{as}` for `transform`"
+          end
+        end
+
+      end # class Transform
+    end # class Processor
+  end # module Sql
+end # module Bmg

data/lib/bmg/sql/relation.rb

@@ -123,27 +123,41 @@ module Bmg
 
       def _rename(type, renaming)
         expr = before_use(self.expr)
-        expr = Processor::Rename.new(renaming, builder).call(self.expr)
+        expr = Processor::Rename.new(renaming, builder).call(expr)
         _instance(type, builder, expr)
       end
 
       def _restrict(type, predicate)
         expr = before_use(self.expr)
-        expr = Processor::Where.new(predicate, builder).call(self.expr)
+        expr = Processor::Where.new(predicate, builder).call(expr)
         _instance(type, builder, expr)
       end
 
-      def _summarize(type, by, summarization)
-        summarization = Operator::Summarize.compile(summarization)
+      def _summarize(type, by, defs)
+        summarization = ::Bmg::Summarizer.summarization(defs)
         if can_compile_summarization?(summarization)
           expr = before_use(self.expr)
-          expr = Processor::Summarize.new(by, summarization, builder).call(self.expr)
+          expr = Processor::Summarize.new(by, summarization, builder).call(expr)
           _instance(type, builder, expr)
         else
           super
         end
       end
 
+      def _transform(type, transformation, options)
+        expr = before_use(self.expr)
+        sup, unsup = Processor::Transform.split_supported(transformation){|x|
+          [String, Integer, Float, Date, DateTime].include?(x)
+        }
+        return super if sup.nil?
+        expr = Processor::Transform.new(sup, options, builder).call(expr)
+        result = _instance(type, builder, expr)
+        result = result.transform(unsup, options) if unsup
+        result
+      rescue Sql::NotSupportedError
+        super
+      end
+
       def can_compile_summarization?(summarization)
         summarization.values.all?{|s|
           [:avg, :count, :max, :min, :sum].include?(s.to_summarizer_name)
@@ -153,7 +167,7 @@ module Bmg
       def _union(type, right, options)
         if right_expr = extract_compatible_sexpr(right)
           expr = before_use(self.expr)
-          expr = Processor::Merge.new(:union, !!options[:all], right_expr, builder).call(self.expr)
+          expr = Processor::Merge.new(:union, !!options[:all], right_expr, builder).call(expr)
           _instance(type, builder, expr)
         else
           super

data/lib/bmg/summarizer.rb

@@ -50,6 +50,21 @@ module Bmg
       end
     end
 
+    # Converts some summarization definitions to a Hash of
+    # summarizers.
+    def self.summarization(defs)
+      Hash[defs.map{|k,v|
+        summarizer = case v
+        when Summarizer then v
+        when Symbol     then Summarizer.send(v, k)
+        when Proc       then Summarizer.by_proc(&v)
+        else
+          raise ArgumentError, "Unexpected summarizer #{k} => #{v}"
+        end
+        [ k, summarizer ]
+      }]
+    end
+
     # Returns the default options to use
     #
     # @return the default aggregation options
@@ -80,7 +95,7 @@ module Bmg
     # @param the current iterated tuple
     # @return updated memo value
     def happens(memo, tuple)
-      value = @functor.is_a?(Proc) ? @functor.call(tuple) : tuple[@functor]
+      value = extract_value(tuple)
       _happens(memo, value)
     end
 
@@ -119,6 +134,21 @@ module Bmg
       self.class.name.downcase[/::([a-z]+)$/, 1].to_sym
     end
 
+  protected
+
+    def extract_value(tuple)
+      value = case @functor
+      when Proc
+        @functor.call(tuple)
+      when NilClass
+        tuple
+      when Symbol
+        tuple[@functor]
+      else
+        tuple[@functor]
+      end
+    end
+
   end # class Summarizer
 end # module Bmg
 require_relative 'summarizer/count'
@@ -128,5 +158,10 @@ require_relative 'summarizer/max'
 require_relative 'summarizer/avg'
 require_relative 'summarizer/variance'
 require_relative 'summarizer/stddev'
+require_relative 'summarizer/percentile'
 require_relative 'summarizer/collect'
+require_relative 'summarizer/distinct'
 require_relative 'summarizer/concat'
+require_relative 'summarizer/by_proc'
+require_relative 'summarizer/multiple'
+require_relative 'summarizer/value_by'

data/lib/bmg/summarizer/avg.rb

@@ -16,13 +16,13 @@ module Bmg
       end
 
       # Collects one more value + the sum of all
-      def _happens(memo, val) 
+      def _happens(memo, val)
         [memo.first + val, memo.last + 1]
       end
 
       # Finalizes the computation.
-      def finalize(memo) 
-        memo.first / memo.last 
+      def finalize(memo)
+        memo.first / memo.last
       end
 
     end # class Avg