flockdb 0.6.1 → 0.7.0

data/lib/flock/gen-rb/flockdb_types.rb

@@ -103,6 +103,28 @@ module Flock
           ::Thrift::Struct.generate_accessors self
         end
 
+        class Metadata
+          include ::Thrift::Struct, ::Thrift::Struct_Union
+          SOURCE_ID = 1
+          STATE_ID = 2
+          COUNT = 3
+          UPDATED_AT = 4
+
+          FIELDS = {
+            SOURCE_ID => {:type => ::Thrift::Types::I64, :name => 'source_id'},
+            STATE_ID => {:type => ::Thrift::Types::I32, :name => 'state_id'},
+            COUNT => {:type => ::Thrift::Types::I32, :name => 'count'},
+            UPDATED_AT => {:type => ::Thrift::Types::I32, :name => 'updated_at'}
+          }
+
+          def struct_fields; FIELDS; end
+
+          def validate
+          end
+
+          ::Thrift::Struct.generate_accessors self
+        end
+
         class Edge
           include ::Thrift::Struct, ::Thrift::Struct_Union
           SOURCE_ID = 1

data/lib/flock/mixins/sizeable.rb

@@ -2,13 +2,9 @@ module Flock
   module Mixins
     module Sizeable
       def any?(&block)
-        if !block_given?
-          size > 0
-        else
-          map(&block).any?
-        end
+        block_given? ? map(&block).any? : size > 0
       end
-      
+
       def empty?
         size == 0
       end

data/lib/flock/mock_service.rb

@@ -7,24 +7,30 @@ module Flock
       Edges::ExecuteOperationType::Archive => :archive,
       Edges::ExecuteOperationType::Negate => :negate
     }
+    OP_COLOR_MAP = {:add => 0, :remove => 1, :archive => 2, :negate => 3}
 
     attr_accessor :timeout, :fixtures
 
     def clear
-      @forward_edges = nil
-      @backward_edges = nil
+      @graphs = nil
+    end
+
+    def load_yml(file)
+      @data_for_fixture ||= Hash.new do |h, k|
+        h[k] = YAML::load(ERB.new(File.open(k, 'r').read).result(binding)).sort
+      end
+      @data_for_fixture[file]
     end
 
     def load(fixtures = nil)
       fixtures ||= self.fixtures or raise "No flock fixtures specified. either pass fixtures to load, or set Flock::MockService.fixtures."
-
       clear
 
       fixtures.each do |fixture|
         file, graph, source, dest = fixture.values_at(:file, :graph, :source, :destination)
 
-        YAML::load(ERB.new(File.open(file, 'r').read).result(binding)).sort.each do |key, row|
-          add(row[source], graph, row[dest])
+        load_yml(file).each do |key, row|
+          color_node(row[source], graph, row[dest], OP_COLOR_MAP[:add])
         end
       end
     end
@@ -37,21 +43,52 @@ module Flock
       operations = operations.operations
       operations.each do |operation|
         term = operation.term
+        position = operation.position
         graph, source = term.graph_id, term.source_id
-        dest = term.destination_ids && term.destination_ids.unpack('Q*')
+        destinations = term.destination_ids && term.destination_ids.unpack('Q*')
+        dest_state = OP_COLOR_MAP[EXEC_OPS[operation.operation_type]]
 
-        source, dest = dest, source unless term.is_forward
-
-        self.send(EXEC_OPS[operation.operation_type], source, graph, dest)
+        source, destinations = destinations, source unless term.is_forward
+        color_node(source, graph, destinations, dest_state, position)
       end
     end
 
     def select(select_operations, page)
-      iterate(select_query(select_operations), page)
+      data, next_cursor, prev_cursor = paginate(select_query(select_operations), page)
+
+      result = Flock::Results.new
+      result.ids = data.pack("Q*")
+      result.next_cursor = next_cursor
+      result.prev_cursor = prev_cursor
+      result
+    end
+
+    def select2(queries)
+      queries.map do |query|
+        select(query.operations, query.page)
+      end
+    end
+
+    def select_edges(queries)
+      queries.map do |query|
+        edges, next_cursor, prev_cursor = paginate(simple_query(query.term), query.page)
+        result = Edges::EdgeResults.new
+        result.edges = if query.term.is_forward
+           edges.map(&:dup)
+         else
+           edges.map(&:dup).map do |edge|
+             edge.source_id, edge.destination_id = edge.destination_id, edge.source_id
+             edge
+           end
+         end
+        result.next_cursor = next_cursor
+        result.prev_cursor = prev_cursor
+        result
+      end
     end
 
     def contains(source, graph, dest)
-      forward_edges[graph][Edges::EdgeState::Positive][source].include?(dest)
+      graphs(graph).contains?(source, dest, Edges::EdgeState::Positive)
     end
 
     def count(select_operations)
@@ -66,10 +103,13 @@ module Flock
         case select_operation.operation_type
         when Edges::SelectOperationType::SimpleQuery
           term = select_operation.term
-          source = term.is_forward ? forward_edges : backward_edges
-          states = term.state_ids || [Edges::EdgeState::Positive]
-          data = source[term.graph_id].inject([]) {|r, (s, e)| states.include?(s) ? r.concat(e[term.source_id]) : r }.uniq
-          stack.push(term.destination_ids ? (term.destination_ids.unpack('Q*') & data) : data)
+          matching_edges = simple_query(term)
+          ids = if term.is_forward
+            matching_edges.map(&:destination_id)
+          else
+            matching_edges.map(&:source_id)
+          end
+          stack.push(ids)
         when Edges::SelectOperationType::Intersection
           stack.push(stack.pop & stack.pop)
         when Edges::SelectOperationType::Union
@@ -83,7 +123,18 @@ module Flock
       stack.pop
     end
 
-    def iterate(data, page)
+    def simple_query(term)
+      destination_ids = term.destination_ids && term.destination_ids.unpack("Q*")
+      states = term.state_ids || [Edges::EdgeState::Positive]
+      graph = graphs(term.graph_id)
+      if term.is_forward
+        graph.select_edges(term.source_id, destination_ids, states)
+      else
+        graph.select_edges(destination_ids, term.source_id, states)
+      end
+    end
+
+    def paginate(data, page)
       return empty_result if page.cursor == Flock::CursorEnd
 
       start =
@@ -95,13 +146,8 @@ module Flock
       rv = data.slice(start, page.count)
       next_cursor = (start + page.count >= data.size) ? Flock::CursorEnd : start + page.count
       prev_cursor = (start <= 0) ? Flock::CursorEnd : -start
-
-      result = Flock::Results.new
-      result.ids = Array(rv).pack("Q*")
-      result.next_cursor = next_cursor
-      result.prev_cursor = prev_cursor
-      result
-    end
+      [Array(rv), next_cursor, prev_cursor]
+   end
 
     def empty_result
       @empty_result ||=
@@ -114,81 +160,100 @@ module Flock
         end
     end
 
-
-    [:forward_edges, :backward_edges].each do |name|
-      class_eval("def #{name}; @#{name} ||= new_edges_hash end")
+    def graphs(graph_id)
+      (@graphs ||= Hash.new do |h, k|
+        h[k] = Graph.new
+      end)[graph_id]
     end
 
-    def new_edges_hash
-      Hash.new { |h,k| h[k] = Hash.new { |h,k| h[k] = Hash.new { |h2,k2| h2[k2] = [] } } }
-    end
 
-    def add_row(store, source, dest)
-      (store[source] << dest).uniq!
-    end
-
-    def add_edge(source, graph, dest, state)
-      add_row(forward_edges[graph][state], source, dest)
-      add_row(backward_edges[graph][state], dest, source)
-      [source, graph, dest]
-    end
+    def color_node(source, graph_id, dest, dest_state, position = nil)
+      raise ArgumentError unless Edges::EdgeState::VALUE_MAP.keys.include? dest_state
+      raise ArgumentError if source.nil? && dest.nil?
+      position ||= Time.now.to_i
 
-    def remove_row(store, source, dest)
-      store[source].delete(dest).tap do
-        store.delete(source) if store[source].empty?
+      if source.nil? || dest.nil?
+        existing_edges = graphs(graph_id).select_edges(source, dest)
+        existing_edges.each do |existing_edge|
+          graphs(graph_id).add_edge(dest_state, existing_edge.source_id, existing_edge.destination_id, Time.now, position)
+        end
+      else
+        Array(source).each do |s|
+          Array(dest).each do |d|
+            graphs(graph_id).add_edge(dest_state, s, d, Time.now, position)
+          end
+        end
       end
     end
 
-    def remove_edge(source, graph, dest, state)
-      forward = remove_row(forward_edges[graph][state], source, dest)
-      backward = remove_row(backward_edges[graph][state], dest, source)
+  end
 
-      [source, graph, dest] if forward and backward
+  class Graph
+    def initialize
+      @by_pair, @by_source, @by_destination = {}, Hash.new { |h, k| h[k] = [] }, Hash.new { |h, k| h[k] = [] }
     end
 
-    def remove_node(source, graph, dest, state)
-      raise unless source or dest
-
-      sources, dests = Array(source), Array(dest)
-
-      sources = dests.map{|dest| backward_edges[graph][state][dest] }.inject([]) {|a,b| a.concat b } if sources.empty?
-      dests = sources.map{|source| forward_edges[graph][state][source] }.inject([]) {|a,b| a.concat b } if dests.empty?
-
-      [].tap do |deleted|
-        sources.each {|s| dests.each {|d| deleted << remove_edge(s, graph, d, state) } }
-      end.compact
+    def add_edge(state, source, dest, time, pos)
+      if existing_edge = @by_pair[[source, dest]]
+        if ![Edges::EdgeState::Positive, Edges::EdgeState::Archived].include?(existing_edge.state_id) && state == Edges::EdgeState::Positive
+          existing_edge.position = pos
+        end
+        existing_edge.state_id = state
+        existing_edge.updated_at = time.to_i
+      else
+        edge = make_edge(state, source, dest, time, pos)
+        @by_pair[[source, dest]] = edge
+        @by_source[source] << edge
+        @by_destination[dest] << edge
+      end
     end
 
-    def color_node(source, graph, dest, dest_state)
-      raise ArgumentError unless Edges::EdgeState::VALUE_MAP.keys.include? dest_state
-
-      source_states = (Edges::EdgeState::VALUE_MAP.keys - [dest_state])
+    def select_edges(source_ids, destination_ids, states = [])
+      source_ids, destination_ids, states = Array(source_ids), Array(destination_ids), Array(states)
+      result = []
+      if source_ids.empty?
+        destination_ids.each do |destination_id|
+          @by_destination[destination_id].each do |edge|
+            next unless states.empty? || states.include?(edge.state_id)
 
-      if source.nil? or dest.nil?
-        source_states.each do |source_state|
-          remove_node(source, graph, dest, source_state).tap do |deleted|
-            deleted.each {|source, graph, dest| add_edge(source, graph, dest, dest_state) }
+            result << edge
           end
         end
+      elsif destination_ids.empty?
+        source_ids.each do |source_id|
+          @by_source[source_id].each do |edge|
+            next unless states.empty? || states.include?(edge.state_id)
 
+            result << edge
+          end
+        end
       else
-        sources, dests = Array(source), Array(dest)
+        source_ids.each do |source_id|
+          destination_ids.each do |destination_id|
+            next unless existing_edge = @by_pair[[source_id, destination_id]]
+            next unless states.empty? || states.include?(existing_edge.state_id)
 
-        sources.each do |s|
-          dests.each do |d|
-            add_edge(s, graph, d, dest_state)
-            source_states.each { |state| remove_edge(s, graph, d, state) }
+            result << existing_edge
           end
         end
       end
+      result
     end
 
-    # must map manually since execute operations do not line up with
-    # their actual colors
-    op_color_map = {:add => 0, :remove => 1, :archive => 2, :negate => 3}
+    def contains?(source, dest, state)
+      select_edges(source, dest, state).any?
+    end
 
-    op_color_map.each do |op, color|
-      class_eval "def #{op}(s, g, d); color_node(s, g, d, #{color}) end", __FILE__, __LINE__
+    private
+    def make_edge(state, source, dest, time, pos)
+      Edges::Edge.new.tap do |edge|
+        edge.source_id = source
+        edge.destination_id = dest
+        edge.updated_at = time.to_i
+        edge.position = pos
+        edge.count = 1
+        edge.state_id = state
+      end
     end
   end
 end

data/lib/flock/operations/execute_operation.rb

@@ -8,7 +8,7 @@ module Flock
       op = Edges::ExecuteOperation.new
       op.operation_type = @operation_type
       op.position = @position if @position
-      op.term = QueryTerm.new(@query).to_thrift
+      op.term = @query.to_thrift
       op
     end
   end

data/lib/flock/operations/execute_operations.rb

@@ -6,7 +6,12 @@ module Flock
 
     Flock::Edges::ExecuteOperationType::VALUE_MAP.each do |op_id, op|
       op = op.downcase
-      class_eval "def #{op}(s, g, d, p = nil); @operations << ExecuteOperation.new(#{op_id}, [s, g, d], p); self end", __FILE__, __LINE__
+      class_eval <<-EOT, __FILE__, __LINE__ + 1
+        def #{op}(s, g, d, p = nil)
+          @operations << ExecuteOperation.new(#{op_id}, Flock::QueryTerm.new([s, g, d]), p)
+          self
+        end
+      EOT
     end
 
     def apply

data/lib/flock/operations/query_term.rb

@@ -1,20 +1,27 @@
 module Flock
   class QueryTerm
+    # symbol => state_id map
+    STATES = Flock::Edges::EdgeState::VALUE_MAP.inject({}) do |states, (id, name)|
+      states.update name.downcase.to_sym => id
+    end.freeze
+
     attr_accessor :source, :graph, :destination, :states
 
-    def initialize(query)
-      case query.size
-      when 3, 4
-        @source, @graph, @destination, @states = *query
-      else
-        raise ArgumentError
-      end
+    def initialize(query, graphs = nil)
+      raise ArgumentError if query.size < 3
+      @graphs = graphs
+
+      @source, @graph, @destination, @states = *query_args(query)
     end
 
     def forward?
       @source.is_a? Numeric
     end
 
+    def unapply
+      [@source, @graph, @destination, @states]
+    end
+
     def to_thrift
       term = Edges::QueryTerm.new
       term.graph_id = @graph
@@ -33,5 +40,42 @@ module Flock
 
       term
     end
+
+    def ==(other)
+      self.source == other.source &&
+        self.graph == other.graph &&
+        self.destination == other.destination &&
+        self.states == other.states
+    end
+
+    private
+
+    def lookup_graph(key)
+      if @graphs.nil? or key.is_a? Integer
+        key
+      else
+        @graphs[key] or raise UnknownGraphError.new(key)
+      end
+    end
+
+    def lookup_states(states)
+      states = states.flatten.compact.map do |s|
+        if s.is_a? Integer
+          s
+        else
+          STATES[s] or raise UnknownStateError.new(s)
+        end
+      end
+    end
+
+    def query_args(args)
+      source, graph, destination, *states = *((args.length == 1) ? args.first : args)
+      [node_arg(source), lookup_graph(graph), node_arg(destination), lookup_states(states)]
+    end
+
+    def node_arg(node)
+      return node.map {|n| n.to_i if n } if node.respond_to? :map
+      node.to_i if node
+    end
   end
 end

data/lib/flock/operations/select_operation.rb

@@ -1,4 +1,3 @@
-
 module Flock
   class SelectOperation
     [:each, :paginate, :to_ary, :size, :first].each do |method|