omf_rete 0.5

data/lib/omf_rete/planner/join_plan.rb

@@ -0,0 +1,94 @@
+
+require 'omf_rete/tuple_stream'
+
+module OMF::Rete
+  module Planner
+    
+    
+    # This class represents a planned join op.
+    # 
+    #
+    class JoinPlan < AbstractPlan
+      
+      # stream1 - first stream to join
+      # stream2 - second stream to join
+      # joinSet - set of bindings to join on
+      # resultSet - set of bindings in result
+      # coverSet - set of leaf nodes contributing to this result
+      #
+      def initialize(stream1, stream2, joinSet, resultSet, coverSet, planBuilder)
+        super coverSet, resultSet 
+        
+        @planBuilder = planBuilder
+        @left = stream1
+        @right = stream2
+        @join_set = joinSet
+      end
+
+      # Materialize the plan. Create all the relevant operations and tuple sets
+      # to realize a configuration for the respective query. Returns the result
+      # set.
+      #
+      def materialize(indexPattern, resultSet, opts, &block)
+        unless resultSet
+          description = @result_set.to_a.sort
+          resultSet = IndexedTupleSet.new(description, indexPattern, nil, opts)
+        end
+          
+        indexPattern = @join_set.to_a
+        leftSet = @left.materialize(indexPattern, nil, opts)
+        rightSet = @right.materialize(indexPattern, nil, opts)
+        op = JoinOP.new(leftSet, rightSet, resultSet)
+        resultSet.source = op
+        resultSet
+      end
+
+      # Create a hash for this plan which allows us to
+      # to identify identical plans.
+      #
+      # Please note, that there is most likely a mroe efficient way to 
+      # calculate a hash with the above properties
+      #
+      def hash()
+        unless @hash
+          s = StringIO.new
+          describe(s, 0, 0, '|')
+          str = s.string
+          @hash = str.hash
+        end
+        @hash
+      end
+      
+      # Return the cost of this plan.
+      #
+      # TODO: Some more meaningful heuristic will be nice
+      #
+      def cost()
+        unless @cost
+          lcost = @left.cost()
+          rcost = @right.cost()
+          #@cost = 1 + 1.2 * (lcost > rcost ? lcost : rcost)
+          @cost = 1 + 1.2 * (lcost + rcost)
+          
+        end
+        @cost
+      end
+      
+      def describe(out = STDOUT, offset = 0, incr = 2, sep = "\n")
+        out.write(" " * offset)
+        result = @result_set.to_a.sort
+        join = @join_set.to_a.sort
+        out.write("join: [#{join.join(', ')}] => [#{result.join(', ')}] cost: #{cost}#{sep}")
+        @left.describe(out, offset + incr, incr, sep) 
+        @right.describe(out, offset + incr, incr, sep) 
+      end
+      
+      def to_s
+        result = @result_set.to_a.sort
+        join = @join_set.to_a.sort
+        "JoinPlan [#{join.join(', ')}] out: [#{result.join(', ')}]"
+      end        
+    end # PlanBuilder
+
+  end # Planner
+end # module

data/lib/omf_rete/planner/plan_builder.rb

@@ -0,0 +1,302 @@
+
+
+
+# Monkey patch symbol to allow consistent ordering of set keys
+unless (:test).respond_to? '<=>'
+  class Symbol
+    def <=>(o) 
+      self.to_s <=> o.to_s 
+    end
+  end
+end
+
+
+module OMF::Rete
+  module Planner
+    
+    # The base exception for all errors related 
+    class PlannerException < Exception; end
+
+    require 'omf_rete/planner/source_plan'
+    require 'omf_rete/planner/plan_level_builder'
+    require 'omf_rete/planner/plan_set'
+    require 'omf_rete/planner/filter_plan'      
+
+    # This class builds all the possible plans for a given 
+    # query
+    #
+    class PlanBuilder
+      
+      attr_reader :plan, :store
+      #
+      # query -- query consists of an array of tuple paterns with binding declarations
+      # store -- store to attach source sets to
+      #
+      def initialize(query, store, opts = {})
+        @store = store
+        @opts = opts
+        
+        _parse_query(query)
+        
+        @complete_plans = []
+        if (@source_cnt == 1)
+          # only one source means a trivial plan, the source itself
+          @complete_plans = @sources.to_a
+        end
+        
+      end
+      
+      def build()
+        level = 0
+        maxLevels = @source_cnt + 10 # pull the emergency breaks sometimes
+        while (@complete_plans.empty? && level < maxLevels) do
+          _iterate()
+          level += 1
+        end
+        if (@complete_plans.empty?)
+          raise PlannerException.new("Can't create plan")            
+        end
+        @complete_plans
+      end
+      
+      def each_plan()
+        @complete_plans.each do |p| yield(p) end
+      end
+      
+      # Return plan with lowest cost
+      #
+      def best_plan()
+        # best_plan = nil
+        # lowest_cost = 9999999999
+#           
+        # each_plan do |plan|
+          # cost = plan.cost
+          # if (cost < lowest_cost) 
+            # lowest_cost = cost
+            # best_plan = plan
+          # end
+        # end
+        best_plan = @complete_plans.min do |p1, p2|
+          p1.cost <=> p2.cost
+        end
+        best_plan
+      end
+      
+      
+      # Materialize the plan. Create all the relevant operations and tuple sets
+      # to realize a configuration for the respective query. Returns the result
+      # set.
+      #
+      def materialize(projectPattern = nil, plan = nil, opts = nil, &block)
+        unless plan
+          plan = best_plan()
+        end
+        unless plan
+          raise PlannerException.new("No plan to materialize");
+        end
+        if (plan.is_a?(SourcePlan))
+          # This is really just a simple pattern on the store
+          _materialize_simple_plan(projectPattern, plan, opts, &block)
+        else
+          # this is the root of the plan
+          if projectPattern 
+            description = projectPattern
+          else
+            description = plan.result_set.to_a.sort
+          end
+          frontS, endS = _materialize_result_stream(plan, projectPattern, opts, &block)
+          plan.materialize(nil, frontS, opts, &block)
+          endS
+        end
+      end
+      
+      
+      def describe(out = STDOUT, offset = 0, incr = 2, sep = "\n")
+        out << "\n=========\n"
+        @complete_plans.each do |p|
+          out << "------\n"
+          p.describe(out, offset, incr, sep)
+        end
+      end
+      
+      private
+
+      # Parse +query+ which is an array of query tuples or filters.
+      # 
+      # This method create a new +SourcePlan+ (to be attached to a store)
+      # for every query tuple in the +query+ array.
+      #      
+      def _parse_query(query)
+        @query = query
+        @sources = Set.new
+        @filters = []
+        @plans = PlanSet.new
+        query.each do |sp|
+          
+          if sp.is_a? FilterPlan
+            @filters << sp
+          elsif sp.is_a? SourcePlan
+            @sources << sp
+            @plans << sp
+          elsif sp.is_a? Array
+            unless sp.length == @store.length
+              raise PlannerException.new("SubPlan: Expected array of store size, but got '#{sp}'")
+            end
+            begin
+              p = SourcePlan.new(sp, @store)
+              @sources << p
+              @plans << p
+            rescue NoBindingException
+              # ignore sources with no bindings in them
+            end
+          else
+            raise PlannerException.new("SubPlan: Unknown sub goal definition '#{sp}'")
+          end
+        end
+        @source_cnt = @sources.size
+        if @sources.empty?
+          raise PlannerException.new("Query '#{query}' seems to be empty")
+        end
+      end
+      
+      #
+      # Array of sources from lower levels to build new plans from
+      #
+      #  plans -- array of plans to combine
+      #  fullCoverSet -- set containing all initial sources
+      #
+      def _iterate()
+#          puts ">>>>>>>> LEVEL >>>>>"
+        plans = @plans.to_a.dup
+        plans.each_with_index do |plan, i|
+          unless (plan.complete?) # don't combine complete plans anymore
+            unless (rem = plans[i + 1 .. -1]).empty?
+              _build_for_plan(plan, rem)
+            end
+            unless plan.used?
+              _add_plan(plan)
+            end
+          end
+        end
+        @plans
+      end
+        
+      # Compare +plan+ with all remaining plans and create
+      # a new plan if it can be combined. If no new plan
+      # is created for +plan+ elevated it to this level.
+      #
+      def _build_for_plan(plan, otherPlans)
+        otherPlans.each do |other|
+          unless (other.complete?) # don't combine complete plans anymore
+            _build_for(plan, other)
+          end
+        end
+      end
+      
+      
+      def _build_for(left, right)
+#          STDOUT.puts "CHECKING"
+#          STDOUT.puts "  LEFT"
+#          left.describe
+#          STDOUT.puts "  RIGHT"
+#          right.describe
+
+        lcover = left.cover_set
+        rcover = right.cover_set
+        combinedCover = lcover + rcover
+        combinedSize = combinedCover.size
+        if (lcover.size == combinedSize || rcover.size == combinedSize)
+          return nil # doesn't get us closer to a solution
+        end
+        
+        joinSet = left.result_set.intersection(right.result_set)
+        if (joinSet.empty?)
+          return nil # nothing to join            
+        end
+        
+        resultSet = left.result_set + right.result_set
+        left.used
+        right.used
+        jp = JoinPlan.new(left, right, joinSet, resultSet, combinedCover, self)
+        _add_plan(jp)
+      end
+      
+      def _add_plan(plan)
+        action = 'DUPLICATE: '
+        if (@plans << plan) 
+          action = 'ADDED: '
+          if (plan.cover_set.size == @source_cnt)
+            action = 'COMPLETE: '
+            @complete_plans << plan
+            plan.complete()
+          end
+        end
+#          STDOUT << action
+#          plan.describe
+      end
+      
+      # The +plan+ consists only of a source plan. Create
+      # a processing stream and attach a block which extracts
+      # the 'bound' elements from the incoming tuple.
+      #
+      def _materialize_simple_plan(projectPattern, plan, opts, &block)
+        
+        unless projectPattern
+          # create one from the binding varibales in plan.description
+          projectPattern = []
+          plan.description.each do |name|
+            if name.to_s.end_with?('?')
+              projectPattern << name.to_sym
+            end
+          end
+          if (projectPattern.empty?)
+            raise NoBindingException.new("No binding declaration in source plan '#{plan.description.join(', ')}'")
+          end              
+        end
+        description = projectPattern
+        
+        #src = plan.materialize(nil, projectPattern, opts)
+        src = ProcessingTupleStream.new(projectPattern, projectPattern, plan.description) 
+        frontS, endS = _materialize_result_stream(plan, projectPattern, opts, &block)
+        
+        src.receiver = frontS
+        frontS.source = src
+
+        @store.registerTSet(src, plan.description) if @store
+
+        endS
+      end
+    
+      # This creates the result stream and stacks all filters on top (if any)
+      # It returns the first and last element as an array.
+      #
+      def _materialize_result_stream(plan, projectPattern, opts, &block)
+        plan_description = plan.result_description
+        description = projectPattern || plan.result_description
+        rs = ResultTupleStream.new(description, &block)
+          
+        # This is a very naive plan to add filters. It simple stacks them all at the end.
+        # It would be much better to put them right after each source or join which produces 
+        # the matching binding stream.
+        #
+        first_filter = nil
+        last_filter = nil
+        @filters.each do |f|
+          fs = f.materialize(plan_description, last_filter, opts)
+          if (last_filter)
+            last_filter.receiver = fs
+          end
+          first_filter ||= fs
+          last_filter = fs
+        end
+        if (last_filter)
+          last_filter.receiver = rs
+          rs.source = last_filter
+        end
+        [first_filter || rs, rs]
+      end
+      
+
+    end # PlanBuilder
+  end # Planner
+end # module

data/lib/omf_rete/planner/plan_level_builder.rb

@@ -0,0 +1,94 @@
+
+require 'omf_rete/planner/join_plan'
+
+module OMF::Rete
+  module Planner
+      
+      
+    # This class builds all the possible plans for a given 
+    # level of the plan forest.
+    #
+    class PlanLevelBuilder
+      
+      attr_reader :plans, :complete_plans
+      
+      # fullCover -- Set of all sources to cover
+      # 
+      def initialize(sources)
+        @fullCover = sources
+        @complete_plans = []
+        @plans = sources.clone
+      end
+      
+      #
+      # Array of sources from lower levels to build new plans from
+      #
+      #  plans -- array of plans to combine
+      #  fullCoverSet -- set containing all initial sources
+      #
+      def build()
+        plans = @plans.to_a
+        plans.each_with_index do |plan, i|
+          unless (rem = plans[i + 1 .. -1]).nil?
+            build_for_plan(plan, rem)
+          end
+          unless plan.used?
+            add_plan(plan)
+          end
+        end
+        @plans
+      end
+      
+      def describe(out = STDOUT, offset = 0, incr = 2, sep = "\n")
+        @plans.each do |p|
+          p.describe(out, offset, incr, sep)
+        end
+      end
+      
+
+      private
+
+      
+      # Compare +plan+ with all remaining plans and create
+      # a new plan if it can be combined. If no new plan
+      # is created for +plan+ elevated it to this level.
+      #
+      def build_for_plan(plan, otherPlans)
+        otherPlans.each do |other|
+          build_for(plan, other)
+        end
+      end
+      
+      
+      def build_for(left, right)
+        lcover = left.cover_set
+        rcover = right.cover_set
+        combinedCover = lcover + rcover
+        combinedSize = combinedCover.size
+        if (lcover.size == combinedSize || rcover.size == combinedSize)
+          return nil # doesn't get us closer to a solution
+        end
+        
+        joinSet = left.result_set.intersection(right.result_set)
+        if (joinSet.empty?)
+          return nil # nothing to join            
+        end
+        
+        resultSet = left.result_set + right.result_set
+        left.used
+        right.used
+        jp = JoinPlan.new(left, right, joinSet, resultSet, combinedCover)
+        add_plan(jp)
+      end
+      
+      def add_plan(plan)
+        if (plan.cover_set == @fullCover)
+          @complete_plans << plan
+        end
+        @plans << plan
+      end
+    
+    end # PlanBuilder
+
+  end # Planner
+end # module

data/lib/omf_rete/planner/plan_set.rb

@@ -0,0 +1,82 @@
+#
+#
+
+require 'set'
+
+module OMF::Rete
+  module Planner
+      
+      # This is a specialisation of the Set class which uses the
+    # hash of an object to determine identity. 
+    #
+    class PlanSet
+      
+      def initialize()
+        @hash = {}
+        @plans = []
+      end
+      
+      # Converts the set to an array.  The order of elements is uncertain.
+      def to_a
+        @plans
+      end
+      
+      def empty?
+        @plans.empty?
+      end
+
+      def length
+        @plans.length
+      end
+      
+      
+      # Returns true if two sets are equal.  The equality of each couple
+      # of elements is defined according to Object#eql?.
+      def ==(set)
+        equal?(set) and return true
+    
+        set.is_a?(PlanSet) && size == set.size or return false
+    
+        hash = @hash.dup
+        set.all? { |o| hash.include?(o) }
+      end
+      
+      def eql?(o) # :nodoc:
+        return false unless o.is_a?(PlanSet)
+        @plans.eql?(o.instance_eval{@plans})
+      end
+
+      # Returns true if the set contains the given object.
+      def include?(o)
+        @hash.has_value?(o)
+      end
+    
+      # Calls the given block once for each element in the set, passing
+      # the element as parameter.  Returns an enumerator if no block is
+      # given.
+      def each
+        block_given? or return enum_for(__method__)
+        @plans.each do |o| yield(o) end
+        self
+      end
+    
+      # Adds the given object to the set and returns false if object is already
+      # in the set, true otherwise.
+      #
+      def add(o)
+        really_added = false
+        oh = o.hash
+        unless @hash.key?(oh)
+          really_added = true
+          @hash[oh] = o
+          @plans << o
+        end
+        really_added 
+      end
+      alias << add
+
+    end # class
+  end # module
+end # module
+
+      

data/lib/omf_rete/planner/source_plan.rb

@@ -0,0 +1,81 @@
+
+require 'omf_rete/planner/plan_builder'
+require 'omf_rete/planner/abstract_plan'
+require 'set'
+
+module OMF::Rete
+  module Planner
+    
+    # Thrown if the plan doesn't contain any bindings
+    class NoBindingException < PlannerException; end
+
+    # This class represents a planned join op.
+    # 
+    #
+    class SourcePlan < AbstractPlan
+      attr_reader :description
+      attr_reader :source_set  # tuple set created by this plan
+      
+      #
+      # description - description of tuples contained in set
+      # store - store to attach +source_set+ to
+      #
+      def initialize(description, store = nil)
+        @description = description
+        # the result set consists of all the binding declarations 
+        # which are symbols with trailing '?'
+        resultSet = Set.new
+        description.each do |name|
+          if name.to_s.end_with?('?')
+            resultSet << name.to_sym
+          end
+        end
+        if (resultSet.empty?)
+          raise NoBindingException.new("No binding declaration in sub plan '#{description.join(', ')}'")
+        end
+        coverSet = Set.new([self])
+        super coverSet, resultSet 
+        
+        #raise Exception unless store.kind_of?(Moana::Filter::Store)
+        @store = store
+      end
+      
+      # Materialize the plan. Returns a tuple set.
+      #
+      def materialize(indexPattern, projectPattern, opts)
+        unless indexPattern
+          # this plan only consists of a single source
+          projectPattern ||= result_description
+          @source_set = ProcessingTupleStream.new(projectPattern, projectPattern, @description) 
+        else
+          @source_set = OMF::Rete::IndexedTupleSet.new(@description, indexPattern)
+        end
+        @store.registerTSet(@source_set, @description) if @store
+      end
+      
+      # Return the cost of this plan.
+      #
+      # TODO: Some more meaningful heuristic will be nice
+      #
+      def cost()
+        unless @cost
+          @cost = @description.inject(0) do |val, el|
+            val + ((el.nil? || el.to_s.end_with?('?')) ? 1 : 0.1)
+          end
+        end
+        @cost
+      end
+      
+      
+      def describe(out = STDOUT, offset = 0, incr = 2, sep = "\n")
+        out.write(" " * offset)
+        desc = @description.collect do |e| e || '*' end
+#          index = @result_set.to_a.sort
+#          out.write("src: [#{desc.join(', ')}] index: [#{index.join(', ')}] cost: #{cost}#{sep}")
+        out.write("src: [#{desc.join(', ')}] cost: #{cost}#{sep}")
+      end
+
+    end # SourcePlan
+
+  end # module
+end # module