bmg 0.18.0 → 0.18.5

data/lib/bmg/operator/restrict.rb

@@ -26,6 +26,7 @@ module Bmg
       end
 
       def each
+        return to_enum unless block_given?
         @operand.each do |tuple|
           yield(tuple) if @predicate.evaluate(tuple)
         end

data/lib/bmg/operator/rxmatch.rb

@@ -32,6 +32,7 @@ module Bmg
     public
 
       def each
+        return to_enum unless block_given?
         @operand.each do |tuple|
           against = attrs.map{|a| tuple[a] }.join(" ")
           matcher = self.matcher

data/lib/bmg/operator/summarize.rb

@@ -13,7 +13,7 @@ module Bmg
         @type = type
         @operand = operand
         @by = by
-        @summarization = Summarize.compile(summarization)
+        @summarization = Summarizer.summarization(summarization)
       end
 
     protected
@@ -23,6 +23,7 @@ module Bmg
     public
 
       def each
+        return to_enum unless block_given?
         # summary key => summarization memo, starting with least
         result = Hash.new{|h,k|
           h[k] = Hash[@summarization.map{|k,v|
@@ -56,22 +57,6 @@ module Bmg
         [ by, summarization ]
       end
 
-    private
-
-      # Compile a summarization hash so that every value is a Summarizer
-      # instance
-      def self.compile(summarization)
-        Hash[summarization.map{|k,v|
-          summarizer = case v
-          when Summarizer then v
-          when Symbol     then Summarizer.send(v, k)
-          else
-            raise ArgumentError, "Unexpected summarizer #{k} => #{v}"
-          end
-          [ k, summarizer ]
-        }]
-      end
-
     end # class Summarize
   end # module Operator
 end # module Bmg

data/lib/bmg/operator/transform.rb

@@ -23,11 +23,12 @@ module Bmg
 
     protected
 
-      attr_reader :transformation
+      attr_reader :transformation, :options
 
     public
 
       def each
+        return to_enum unless block_given?
         t = transformer
         @operand.each do |tuple|
           yield t.call(tuple)
@@ -40,6 +41,43 @@ module Bmg
 
     protected ### optimization
 
+      def _allbut(type, butlist)
+        # `allbut` can always be pushed down the tree. unlike
+        # `extend` the Proc that might be used cannot use attributes
+        # in butlist, so it's safe to strip them away.
+        if transformer.knows_attrlist?
+          # We just need to clean the transformation
+          attrlist = transformer.to_attrlist
+          thrown = attrlist & butlist
+          t = transformation.dup.reject{|k,v| thrown.include?(k) }
+          operand.allbut(butlist).transform(t, options)
+        else
+          operand.allbut(butlist).transform(transformation, options)
+        end
+      end
+
+      def _project(type, attrlist)
+        if transformer.knows_attrlist?
+          t = transformation.dup.select{|k,v| attrlist.include?(k) }
+          operand.project(attrlist).transform(t, options)
+        else
+          operand.project(attrlist).transform(transformation, options)
+        end
+      end
+
+      def _restrict(type, predicate)
+        return super unless transformer.knows_attrlist?
+        top, bottom = predicate.and_split(transformer.to_attrlist)
+        if top == predicate
+          super
+        else
+          operand
+            .restrict(bottom)
+            .transform(transformation, options)
+            .restrict(top)
+        end
+      end
+
     protected ### inspect
 
       def args

data/lib/bmg/operator/union.rb

@@ -37,6 +37,7 @@ module Bmg
       end
 
       def each(&bl)
+        return to_enum unless block_given?
         if all?
           operands.each do |op|
             op.each(&bl)

data/lib/bmg/reader/csv.rb

@@ -5,30 +5,36 @@ module Bmg
 
       DEFAULT_OPTIONS = {
         :headers => true,
-        :return_headers => false
+        :return_headers => false,
+        :smart => true
       }
 
-      def initialize(type, path, options = {})
+      def initialize(type, path_or_io, options = {})
         @type = type
-        @path = path
+        @path_or_io = path_or_io
         @options = DEFAULT_OPTIONS.merge(options)
-        @options[:col_sep] ||= infer_col_sep
-        @options[:quote_char] ||= infer_quote_char
+        if @options[:smart] && !@path_or_io.is_a?(IO)
+          @options[:col_sep] ||= infer_col_sep
+          @options[:quote_char] ||= infer_quote_char
+        end
       end
 
       def each
+        return to_enum unless block_given?
         require 'csv'
-        ::CSV.foreach(@path, @options) do |row|
-          yield tuple(row)
+        with_io do |io|
+          ::CSV.new(io, **csv_options).each do |row|
+            yield tuple(row)
+          end
         end
       end
 
       def to_ast
-        [ :csv, @path, @options ]
+        [ :csv, @path_or_io, @options ]
       end
 
       def to_s
-        "(csv #{path})"
+        "(csv #{@path_or_io})"
       end
       alias :inspect :to_s
 
@@ -47,7 +53,16 @@ module Bmg
       end
 
       def text_portion
-        @text_portion ||= File.foreach(@path).first(10).join("\n")
+        @text_portion ||= with_io{|io| io.readlines(10).join("\n") }
+      end
+
+      def with_io(&bl)
+        case @path_or_io
+        when IO, StringIO
+          bl.call(@path_or_io)
+        else
+          File.open(@path_or_io, "r", &bl)
+        end
       end
 
       # Finds the best candidate among `candidates` for a separator
@@ -61,6 +76,10 @@ module Bmg
         snif.size > 0 ? snif[0][0] : default
       end
 
+      def csv_options
+        @csv_options ||= @options.dup.tap{|opts| opts.delete(:smart) }
+      end
+
     end # class Csv
   end # module Reader
 end # module Bmg

data/lib/bmg/reader/excel.rb

@@ -4,7 +4,8 @@ module Bmg
       include Reader
 
       DEFAULT_OPTIONS = {
-        skip: 0
+        skip: 0,
+        row_num: true
       }
 
       def initialize(type, path, options = {})
@@ -14,6 +15,7 @@ module Bmg
       end
 
       def each
+        return to_enum unless block_given?
         require 'roo'
         xlsx = Roo::Spreadsheet.open(@path, @options)
         headers = nil
@@ -23,9 +25,13 @@ module Bmg
           .each_with_index
           .each do |row, i|
             if i==0
-              headers = row.map(&:to_sym)
+              headers = row.map{|c| c.to_s.strip.to_sym }
             else
-              tuple = (0...headers.size).each_with_object({}){|i,t| t[headers[i]] = row[i] }
+              init = init_tuple(i)
+              tuple = (0...headers.size)
+                .each_with_object(init){|i,t|
+                  t[headers[i]] = row[i]
+                }
               yield(tuple)
             end
           end
@@ -36,10 +42,23 @@ module Bmg
       end
 
       def to_s
-        "(excel #{path})"
+        "(excel #{@path})"
       end
       alias :inspect :to_s
 
+    private
+
+      def init_tuple(i)
+        case as = @options[:row_num]
+        when TrueClass
+          { :row_num => i }
+        when FalseClass
+          {}
+        when Symbol
+          { :"#{as}" => i }
+        end
+      end
+
     end # class Excel
   end # module Reader
 end # module Bmg

data/lib/bmg/relation.rb

@@ -27,6 +27,12 @@ module Bmg
       }
     end
 
+    def with_type_attrlist
+      return self if type.knows_attrlist?
+      attrs = self.first.keys
+      with_type(type.with_attrlist(attrs))
+    end
+
     def with_typecheck
       dup.tap{|r|
         r.type = r.type.with_typecheck
@@ -110,6 +116,18 @@ module Bmg
       end
     end
 
+    def count
+      if type.knows_keys?
+        project(type.keys.first)._count
+      else
+        self._count
+      end
+    end
+
+    def _count
+      to_a.size
+    end
+
     # Returns a json representation
     def to_json(*args, &bl)
       to_a.to_json(*args, &bl)

data/lib/bmg/relation/empty.rb

@@ -19,6 +19,10 @@ module Bmg
       def each(&bl)
       end
 
+      def _count
+        0
+      end
+
       def to_ast
         [ :empty ]
       end

data/lib/bmg/relation/in_memory.rb

@@ -8,7 +8,6 @@ module Bmg
         @type = type
       end
       attr_accessor :type
-      protected :type=
       attr_reader :operand
 
     public
@@ -17,6 +16,16 @@ module Bmg
         @operand.each(&bl)
       end
 
+      def _count
+        if operand.respond_to?(:count)
+          operand.count
+        elsif operand.respond_to?(:size)
+          operand.size
+        else
+          super
+        end
+      end
+
       def to_ast
         [ :in_memory, operand ]
       end

data/lib/bmg/relation/materialized.rb

@@ -16,6 +16,12 @@ module Bmg
       end
       protected :type=
 
+    public
+
+      def _count
+        operand._count
+      end
+
     public
 
       def each(&bl)

data/lib/bmg/relation/spied.rb

@@ -28,6 +28,11 @@ module Bmg
         operand.each(&bl)
       end
 
+      def count
+        spy.call(self) if bl
+        operand.count
+      end
+
       def to_ast
         [ :spied, operand.to_ast, spy ]
       end

data/lib/bmg/sequel/relation.rb

@@ -9,6 +9,7 @@ module Bmg
       attr_reader :sequel_db
 
       def each(&bl)
+        return to_enum unless block_given?
         dataset.each(&bl)
       end
 
@@ -33,6 +34,10 @@ module Bmg
         base_table.update(arg)
       end
 
+      def _count
+        dataset.count
+      end
+
       def to_ast
         [:sequel, dataset.sql]
       end

data/lib/bmg/sql/relation.rb

@@ -10,7 +10,6 @@ module Bmg
       end
 
       attr_accessor :type
-      protected :type=
 
     protected
 
@@ -134,8 +133,8 @@ module Bmg
         _instance(type, builder, expr)
       end
 
-      def _summarize(type, by, summarization)
-        summarization = Operator::Summarize.compile(summarization)
+      def _summarize(type, by, defs)
+        summarization = ::Bmg::Summarizer.summarization(defs)
         if can_compile_summarization?(summarization)
           expr = before_use(self.expr)
           expr = Processor::Summarize.new(by, summarization, builder).call(self.expr)

data/lib/bmg/summarizer.rb

@@ -50,6 +50,21 @@ module Bmg
       end
     end
 
+    # Converts some summarization definitions to a Hash of
+    # summarizers.
+    def self.summarization(defs)
+      Hash[defs.map{|k,v|
+        summarizer = case v
+        when Summarizer then v
+        when Symbol     then Summarizer.send(v, k)
+        when Proc       then Summarizer.by_proc(&v)
+        else
+          raise ArgumentError, "Unexpected summarizer #{k} => #{v}"
+        end
+        [ k, summarizer ]
+      }]
+    end
+
     # Returns the default options to use
     #
     # @return the default aggregation options
@@ -80,7 +95,7 @@ module Bmg
     # @param the current iterated tuple
     # @return updated memo value
     def happens(memo, tuple)
-      value = @functor.is_a?(Proc) ? @functor.call(tuple) : tuple[@functor]
+      value = extract_value(tuple)
       _happens(memo, value)
     end
 
@@ -119,6 +134,21 @@ module Bmg
       self.class.name.downcase[/::([a-z]+)$/, 1].to_sym
     end
 
+  protected
+
+    def extract_value(tuple)
+      value = case @functor
+      when Proc
+        @functor.call(tuple)
+      when NilClass
+        tuple
+      when Symbol
+        tuple[@functor]
+      else
+        tuple[@functor]
+      end
+    end
+
   end # class Summarizer
 end # module Bmg
 require_relative 'summarizer/count'
@@ -128,5 +158,10 @@ require_relative 'summarizer/max'
 require_relative 'summarizer/avg'
 require_relative 'summarizer/variance'
 require_relative 'summarizer/stddev'
+require_relative 'summarizer/percentile'
 require_relative 'summarizer/collect'
+require_relative 'summarizer/distinct'
 require_relative 'summarizer/concat'
+require_relative 'summarizer/by_proc'
+require_relative 'summarizer/multiple'
+require_relative 'summarizer/value_by'