bmg 0.17.8 → 0.18.4

data/lib/bmg/operator/union.rb

@@ -37,6 +37,7 @@ module Bmg
       end
 
       def each(&bl)
+        return to_enum unless block_given?
         if all?
           operands.each do |op|
             op.each(&bl)

data/lib/bmg/reader.rb

@@ -9,3 +9,4 @@ module Bmg
 end
 require_relative "reader/csv"
 require_relative "reader/excel"
+require_relative "reader/text_file"

data/lib/bmg/reader/csv.rb

@@ -5,30 +5,36 @@ module Bmg
 
       DEFAULT_OPTIONS = {
         :headers => true,
-        :return_headers => false
+        :return_headers => false,
+        :smart => true
       }
 
-      def initialize(type, path, options = {})
+      def initialize(type, path_or_io, options = {})
         @type = type
-        @path = path
+        @path_or_io = path_or_io
         @options = DEFAULT_OPTIONS.merge(options)
-        @options[:col_sep] ||= infer_col_sep
-        @options[:quote_char] ||= infer_quote_char
+        if @options[:smart] && !@path_or_io.is_a?(IO)
+          @options[:col_sep] ||= infer_col_sep
+          @options[:quote_char] ||= infer_quote_char
+        end
       end
 
       def each
+        return to_enum unless block_given?
         require 'csv'
-        ::CSV.foreach(@path, @options) do |row|
-          yield tuple(row)
+        with_io do |io|
+          ::CSV.new(io, csv_options).each do |row|
+            yield tuple(row)
+          end
         end
       end
 
       def to_ast
-        [ :csv, @path, @options ]
+        [ :csv, @path_or_io, @options ]
       end
 
       def to_s
-        "(csv #{path})"
+        "(csv #{@path_or_io})"
       end
       alias :inspect :to_s
 
@@ -47,7 +53,16 @@ module Bmg
       end
 
       def text_portion
-        @text_portion ||= File.foreach(@path).first(10).join("\n")
+        @text_portion ||= with_io{|io| io.readlines(10).join("\n") }
+      end
+
+      def with_io(&bl)
+        case @path_or_io
+        when IO, StringIO
+          bl.call(@path_or_io)
+        else
+          File.open(@path_or_io, "r", &bl)
+        end
       end
 
       # Finds the best candidate among `candidates` for a separator
@@ -61,6 +76,10 @@ module Bmg
         snif.size > 0 ? snif[0][0] : default
       end
 
+      def csv_options
+        @csv_options ||= @options.dup.tap{|opts| opts.delete(:smart) }
+      end
+
     end # class Csv
   end # module Reader
 end # module Bmg

data/lib/bmg/reader/excel.rb

@@ -4,7 +4,8 @@ module Bmg
       include Reader
 
       DEFAULT_OPTIONS = {
-        skip: 0
+        skip: 0,
+        row_num: true
       }
 
       def initialize(type, path, options = {})
@@ -14,6 +15,7 @@ module Bmg
       end
 
       def each
+        return to_enum unless block_given?
         require 'roo'
         xlsx = Roo::Spreadsheet.open(@path, @options)
         headers = nil
@@ -23,9 +25,13 @@ module Bmg
           .each_with_index
           .each do |row, i|
             if i==0
-              headers = row.map(&:to_sym)
+              headers = row.map{|c| c.to_s.strip.to_sym }
             else
-              tuple = (0...headers.size).each_with_object({}){|i,t| t[headers[i]] = row[i] }
+              init = init_tuple(i)
+              tuple = (0...headers.size)
+                .each_with_object(init){|i,t|
+                  t[headers[i]] = row[i]
+                }
               yield(tuple)
             end
           end
@@ -36,10 +42,23 @@ module Bmg
       end
 
       def to_s
-        "(excel #{path})"
+        "(excel #{@path})"
       end
       alias :inspect :to_s
 
+    private
+
+      def init_tuple(i)
+        case as = @options[:row_num]
+        when TrueClass
+          { :row_num => i }
+        when FalseClass
+          {}
+        when Symbol
+          { :"#{as}" => i }
+        end
+      end
+
     end # class Excel
   end # module Reader
 end # module Bmg

data/lib/bmg/reader/text_file.rb

@@ -0,0 +1,56 @@
+module Bmg
+  module Reader
+    class TextFile
+      include Reader
+
+      DEFAULT_OPTIONS = {
+        strip: true,
+        parse: nil
+      }
+
+      def initialize(type, path, options = {})
+        options = { parse: options } if options.is_a?(Regexp)
+        @path = path
+        @options = DEFAULT_OPTIONS.merge(options)
+        @type = infer_type(type)
+      end
+      attr_reader :path, :options
+
+    public # Relation
+
+      def each
+        path.each_line.each_with_index do |text, line|
+          text = text.strip if strip?
+          parsed = parse(text)
+          yield({line: 1+line}.merge(parsed)) if parsed
+        end
+      end
+
+    private
+
+      def infer_type(base)
+        return base unless base == Bmg::Type::ANY
+        attr_list = if rx = options[:parse]
+          [:line] + rx.names.map(&:to_sym)
+        else
+          [:line, :text]
+        end
+        base
+          .with_attrlist(attr_list)
+          .with_keys([[:line]])
+      end
+
+      def strip?
+        options[:strip]
+      end
+
+      def parse(text)
+        return { text: text } unless rx = options[:parse]
+        if match = rx.match(text)
+          TupleAlgebra.symbolize_keys(match.named_captures)
+        end
+      end
+
+    end # class TextFile
+  end # module Reader
+end # module Bmg

data/lib/bmg/relation.rb

@@ -17,6 +17,22 @@ module Bmg
       self
     end
 
+    def type
+      Bmg::Type::ANY
+    end
+
+    def with_type(type)
+      dup.tap{|r|
+        r.type = type
+      }
+    end
+
+    def with_type_attrlist
+      return self if type.knows_attrlist?
+      attrs = self.first.keys
+      with_type(type.with_attrlist(attrs))
+    end
+
     def with_typecheck
       dup.tap{|r|
         r.type = r.type.with_typecheck
@@ -100,6 +116,18 @@ module Bmg
       end
     end
 
+    def count
+      if type.knows_keys?
+        project(type.keys.first)._count
+      else
+        self._count
+      end
+    end
+
+    def _count
+      to_a.size
+    end
+
     # Returns a json representation
     def to_json(*args, &bl)
       to_a.to_json(*args, &bl)

data/lib/bmg/relation/empty.rb

@@ -19,6 +19,10 @@ module Bmg
       def each(&bl)
       end
 
+      def _count
+        0
+      end
+
       def to_ast
         [ :empty ]
       end

data/lib/bmg/relation/in_memory.rb

@@ -8,7 +8,6 @@ module Bmg
         @type = type
       end
       attr_accessor :type
-      protected :type=
       attr_reader :operand
 
     public
@@ -17,6 +16,16 @@ module Bmg
         @operand.each(&bl)
       end
 
+      def _count
+        if operand.respond_to?(:count)
+          operand.count
+        elsif operand.respond_to?(:size)
+          operand.size
+        else
+          super
+        end
+      end
+
       def to_ast
         [ :in_memory, operand ]
       end

data/lib/bmg/relation/materialized.rb

@@ -16,6 +16,12 @@ module Bmg
       end
       protected :type=
 
+    public
+
+      def _count
+        operand._count
+      end
+
     public
 
       def each(&bl)

data/lib/bmg/relation/spied.rb

@@ -24,10 +24,15 @@ module Bmg
       protected :type=
 
       def each(&bl)
-        spy.call(self)
+        spy.call(self) if bl
         operand.each(&bl)
       end
 
+      def count
+        spy.call(self) if bl
+        operand.count
+      end
+
       def to_ast
         [ :spied, operand.to_ast, spy ]
       end

data/lib/bmg/sequel/relation.rb

@@ -9,6 +9,7 @@ module Bmg
       attr_reader :sequel_db
 
       def each(&bl)
+        return to_enum unless block_given?
         dataset.each(&bl)
       end
 
@@ -33,6 +34,10 @@ module Bmg
         base_table.update(arg)
       end
 
+      def _count
+        dataset.count
+      end
+
       def to_ast
         [:sequel, dataset.sql]
       end

data/lib/bmg/sql/relation.rb

@@ -10,7 +10,6 @@ module Bmg
       end
 
       attr_accessor :type
-      protected :type=
 
     protected
 
@@ -134,8 +133,8 @@ module Bmg
         _instance(type, builder, expr)
       end
 
-      def _summarize(type, by, summarization)
-        summarization = Operator::Summarize.compile(summarization)
+      def _summarize(type, by, defs)
+        summarization = ::Bmg::Summarizer.summarization(defs)
         if can_compile_summarization?(summarization)
           expr = before_use(self.expr)
           expr = Processor::Summarize.new(by, summarization, builder).call(self.expr)

data/lib/bmg/summarizer.rb

@@ -50,6 +50,21 @@ module Bmg
       end
     end
 
+    # Converts some summarization definitions to a Hash of
+    # summarizers.
+    def self.summarization(defs)
+      Hash[defs.map{|k,v|
+        summarizer = case v
+        when Summarizer then v
+        when Symbol     then Summarizer.send(v, k)
+        when Proc       then Summarizer.by_proc(&v)
+        else
+          raise ArgumentError, "Unexpected summarizer #{k} => #{v}"
+        end
+        [ k, summarizer ]
+      }]
+    end
+
     # Returns the default options to use
     #
     # @return the default aggregation options
@@ -80,7 +95,16 @@ module Bmg
     # @param the current iterated tuple
     # @return updated memo value
     def happens(memo, tuple)
-      value = @functor.is_a?(Proc) ? @functor.call(tuple) : tuple[@functor]
+      value = case @functor
+      when Proc
+        @functor.call(tuple)
+      when NilClass
+        tuple
+      when Symbol
+        tuple[@functor]
+      else
+        tuple[@functor]
+      end
       _happens(memo, value)
     end
 
@@ -128,5 +152,9 @@ require_relative 'summarizer/max'
 require_relative 'summarizer/avg'
 require_relative 'summarizer/variance'
 require_relative 'summarizer/stddev'
+require_relative 'summarizer/percentile'
 require_relative 'summarizer/collect'
+require_relative 'summarizer/distinct'
 require_relative 'summarizer/concat'
+require_relative 'summarizer/by_proc'
+require_relative 'summarizer/multiple'

data/lib/bmg/summarizer/avg.rb

@@ -16,13 +16,13 @@ module Bmg
       end
 
       # Collects one more value + the sum of all
-      def _happens(memo, val) 
+      def _happens(memo, val)
         [memo.first + val, memo.last + 1]
       end
 
       # Finalizes the computation.
-      def finalize(memo) 
-        memo.first / memo.last 
+      def finalize(memo)
+        memo.first / memo.last
       end
 
     end # class Avg

data/lib/bmg/summarizer/by_proc.rb

@@ -0,0 +1,41 @@
+module Bmg
+  class Summarizer
+    #
+    # Generic summarizer that takes a Proc àla each_with_object.
+    #
+    # Example:
+    #
+    #   # direct ruby usage
+    #   Bmg::Summarizer.by_proc{|t,memo| ... }.summarize(...)
+    #
+    class ByProc < Summarizer
+
+      def initialize(least, by_proc)
+        @least = least
+        @by_proc = by_proc
+      end
+
+      # Returns [] as least value.
+      def least()
+        @least
+      end
+
+      # Adds val to the memo array
+      def happens(memo, val)
+        @by_proc.call(val, memo)
+      end
+
+      def finalize(memo)
+        memo
+      end
+
+    end # class ByProc
+
+    # Factors a distinct summarizer
+    def self.by_proc(least = nil, proc = nil, &bl)
+      least, proc = nil, least if least.is_a?(Proc)
+      ByProc.new(least, proc || bl)
+    end
+
+  end # class Summarizer
+end # module Bmg