pacer 0.9.1.1-java

data/lib/pacer/pipe/simple_visitor_pipe.rb

@@ -0,0 +1,43 @@
+module Pacer
+  module Pipes
+    class SimpleVisitorPipe < Pacer::Pipes::RubyPipe
+      attr_reader :visitor, :in_section
+
+      attr_accessor :use_on_element, :use_after_element, :use_reset
+
+      def initialize(visitor = nil)
+        super()
+        self.visitor = visitor if visitor
+        @in_section = false
+      end
+
+      def visitor=(visitor)
+        @visitor = visitor
+        @use_on_element      = visitor.respond_to? :on_element
+        @use_after_element   = visitor.respond_to? :after_element
+        @use_reset           = visitor.respond_to? :reset
+      end
+
+      def processNextStart
+        visitor.after_element if use_after_element and in_section
+        current = starts.next
+        @in_section = true unless in_section
+        visitor.on_element(current) if use_on_element
+        return current
+      rescue NativeException => e
+        if e.cause.getClass == Pacer::NoSuchElementException.getClass
+          @in_section = false
+          raise e.cause
+        else
+          raise
+        end
+      end
+
+      def reset
+        visitor.reset if use_reset
+        @in_section = false
+        super
+      end
+    end
+  end
+end

data/lib/pacer/pipe/stream_sort_pipe.rb

@@ -0,0 +1,84 @@
+module Pacer::Pipes
+  # has 3 states:
+  # - starts in rebalancing mode to build up a working set which stores
+  #   elements in 4 silos
+  # - if the first silo becomes empty, it goes into clearing mode
+  # - in clearing mode, it will empty all elements out of the first non-empty silo
+  # - it then goes back into rebalancing mode
+  #
+  # Number of silos is 4 by default, a number I pulled out of my ass without any testing.
+  class StreamSortPipe < RubyPipe
+    def initialize(queue_length = 100, silo_size = 10)
+      super()
+      @queue_length = queue_length
+      @rebalancing = []
+      @rebal_length = 0
+      @first_silo = []
+      @second_silo = []
+      @third_silo = []
+      @clearing = []
+      @silo_size = silo_size
+    end
+
+    def setSiloSize(n)
+      @silo_size = n
+    end
+
+    def processNextStart
+      if @clearing.any?
+        return @clearing.shift
+      end
+      while @rebal_length < @queue_length and @starts.hasNext
+        @rebalancing << @starts.next
+        @rebal_length += 1
+      end
+      if @rebalancing.any?
+        if @starts.hasNext
+          @rebalancing.sort!
+          @first_silo = @rebalancing.slice(0, @silo_size)
+          @second_silo = @rebalancing.slice(@silo_size, @silo_size * 2) || []
+          @third_silo = @rebalancing.slice(@silo_size * 2..-1) || []
+          @rebalancing = []
+        else
+          @clearing = @rebalancing.sort
+          @rebalancing = []
+          return processNextStart
+        end
+      end
+      if @starts.hasNext
+        if @first_silo.any?
+          element = @starts.next
+          begin
+            if element < @first_silo.last
+              @first_silo << element
+              @first_silo.sort!
+            elsif @second_silo.none? or element < @second_silo.last
+              @second_silo.unshift element
+            else
+              @third_silo << element
+            end
+          rescue => e
+            @first_silo.unshift element
+          end
+          return @first_silo.shift
+        else
+          @clearing = @second_silo
+          @clearing.sort!
+          @rebalancing = @third_silo
+          @rebal_length = @rebalancing.length
+          return @rebalancing.shift
+        end
+      else
+        @clearing = @first_silo + @second_silo.sort! + @third_silo.sort!
+        return processNextStart if @clearing.any?
+      end
+      raise Pacer::NoSuchElementException
+    rescue NativeException => e
+      if e.cause.getClass == NoSuchElementException.getClass
+        raise e.cause
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/pacer/pipe/stream_uniq_pipe.rb

@@ -0,0 +1,33 @@
+module Pacer::Pipes
+  class StreamUniqPipe < RubyPipe
+    def initialize(buffer = 1000)
+      super()
+      @list = java.util.LinkedList.new
+      @buffer = buffer
+    end
+
+    protected
+
+    def processNextStart
+      while true
+        obj = @starts.next
+        duplicate = @list.removeLastOccurrence(obj)
+        @list.addLast obj
+        if not duplicate
+          if @buffer == 0
+            @list.removeFirst
+          else
+            @buffer -= 1
+          end
+          return obj
+        end
+      end
+    rescue NativeException => e
+      if e.cause.getClass == NoSuchElementException.getClass
+        raise e.cause
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/pacer/pipe/type_filter_pipe.rb

@@ -0,0 +1,22 @@
+module Pacer::Pipes
+  class TypeFilterPipe < RubyPipe
+    def initialize(type)
+      super()
+      @type = type
+    end
+
+    def processNextStart()
+      while @starts.hasNext
+        s = @starts.next
+        return s if s.is_a? @type
+      end
+      raise Pacer::NoSuchElementException
+    rescue NativeException => e
+      if e.cause.getClass == Pacer::NoSuchElementException.getClass
+        raise e.cause
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/pacer/pipe/unary_transform_pipe.rb

@@ -0,0 +1,59 @@
+module Pacer::Pipes
+  class UnaryTransformPipe < RubyPipe
+    import com.tinkerpop.pipes.Pipe
+    import com.tinkerpop.pipes.util.SingleIterator
+
+    attr_reader :branch_a, :method
+
+    def initialize(method, branch_a)
+      super()
+      if branch_a.is_a? Pipe
+        @branch_a = branch_a
+        @a = nil
+      else
+        @a = branch_a
+      end
+      @first = true
+      @method = method
+      @element = nil
+    end
+
+    def processNextStart
+      next_a
+      a.send method rescue nil if a.respond_to? method
+    rescue NativeException => e
+      if e.cause.getClass == Pacer::NoSuchElementException.getClass
+        raise e.cause
+      else
+        raise e
+      end
+    end
+
+    protected
+
+    attr_accessor :element, :a
+
+    def next_element
+      @first = false
+      self.element = starts.next
+    end
+
+    def next_a
+      if branch_a
+        begin
+          self.a = branch_a.next
+        rescue NativeException => e
+          if e.cause.getClass == Pacer::NoSuchElementException.getClass or @first
+            next_element
+            branch_a.setStarts SingleIterator.new(element) if branch_a
+            retry
+          else
+            raise e
+          end
+        end
+      else
+        next_element
+      end
+    end
+  end
+end

data/lib/pacer/pipe/variable_store_iterator_wrapper.rb

@@ -0,0 +1,26 @@
+module Pacer::Pipes
+  class VariableStoreIteratorWrapper < AbstractPipe
+
+    field_reader :starts
+    attr_accessor :vars
+
+    def initialize(pipe, vars, variable_name)
+      super()
+      setStarts pipe if pipe
+      @vars = vars
+      @variable_name = variable_name
+    end
+
+    protected
+
+    def processNextStart
+      @vars[@variable_name] = starts.next
+    rescue NativeException => e
+      if e.cause.getClass == Pacer::NoSuchElementException.getClass
+        raise e.cause
+      else
+        raise e
+      end
+    end
+  end
+end

data/lib/pacer/pipe/visitor_pipe.rb

@@ -0,0 +1,67 @@
+module Pacer
+  module Pipes
+    class VisitorPipe < Pacer::Pipes::RubyPipe
+      attr_reader :visitor, :queue, :in_section
+
+      attr_accessor :use_on_element, :use_replace_element,
+        :use_after_element, :use_reset, :use_hasNext, :use_next
+
+      def initialize(visitor = nil)
+        super()
+        self.visitor = visitor if visitor
+        @queue          = []
+        @in_section = false
+      end
+
+      def visitor=(visitor)
+        if visitor.respond_to? :on_pipe
+          @visitor = visitor.on_pipe(self)
+        else
+          @visitor = visitor
+        end
+        @use_hasNext         = visitor.respond_to? :hasNext
+        @use_next            = visitor.respond_to? :next
+        @use_on_element      = visitor.respond_to? :on_element
+        @use_replace_element = visitor.respond_to? :replace_element
+        @use_after_element   = visitor.respond_to? :after_element
+        @use_reset           = visitor.respond_to? :reset
+      end
+
+      def processNextStart
+        while true
+          visitor.after_element if use_after_element and in_section
+          if use_next and (not use_hasNext or visitor.hasNext)
+            return visitor.next
+          elsif queue.any?
+            return queue.shift
+          else
+            current = starts.next
+            @in_section = true unless in_section
+            visitor.on_element(current) if use_on_element
+            if use_replace_element
+              visitor.replace_element(current) do |e|
+                queue << e
+              end
+            else
+              return current
+            end
+          end
+        end
+      rescue NativeException => e
+        if e.cause.getClass == Pacer::NoSuchElementException.getClass
+          @in_section = false
+          raise e.cause
+        else
+          raise
+        end
+      end
+
+      def reset
+        visitor.reset if use_reset
+        @in_section = false
+        @queue = []
+        super
+      end
+    end
+  end
+end

data/lib/pacer/pipes.rb

@@ -0,0 +1,61 @@
+module Pacer
+  # Import the Pipes and related objects that we'll be using.
+  module Pipes
+    # TODO: move pipe imports to the modules that actually use them.
+    import com.tinkerpop.pipes.AbstractPipe
+    import com.tinkerpop.pipes.transform.IdentityPipe
+    import com.tinkerpop.pipes.util.Pipeline
+    import com.tinkerpop.pipes.util.ExpandableIterator
+    import com.tinkerpop.pipes.util.MultiIterator
+    import com.tinkerpop.pipes.util.PipeHelper
+
+    import com.tinkerpop.pipes.filter.RandomFilterPipe
+    import com.tinkerpop.pipes.filter.DuplicateFilterPipe
+    import com.tinkerpop.pipes.filter.RangeFilterPipe
+    import com.tinkerpop.pipes.filter.FilterPipe
+
+    import com.tinkerpop.pipes.transform.IdPipe
+    import com.tinkerpop.pipes.transform.PropertyPipe
+
+    EQUAL = FilterPipe::Filter::EQUAL
+    NOT_EQUAL = FilterPipe::Filter::NOT_EQUAL
+    #GREATER_THAN, LESS_THAN, GREATER_THAN_EQUAL, LESS_THAN_EQUAL
+  end
+
+  import java.util.Iterator
+  begin
+    java.util.ArrayList.new.iterator.next
+  rescue NativeException => e
+    NoSuchElementException = e.cause
+    Pipes::NoSuchElementException = e.cause
+  end
+end
+
+require 'pacer/pipe/ruby_pipe'
+
+require 'pacer/pipe/never_pipe'
+require 'pacer/pipe/block_filter_pipe'
+require 'pacer/pipe/collection_filter_pipe'
+require 'pacer/pipe/enumerable_pipe'
+require 'pacer/pipe/expandable_pipe'
+require 'pacer/pipe/loop_pipe'
+require 'pacer/pipe/map_pipe'
+require 'pacer/pipe/process_pipe'
+require 'pacer/pipe/visitor_pipe'
+require 'pacer/pipe/simple_visitor_pipe'
+require 'pacer/pipe/is_unique_pipe'
+require 'pacer/pipe/is_empty_pipe'
+require 'pacer/pipe/stream_sort_pipe'
+require 'pacer/pipe/stream_uniq_pipe'
+require 'pacer/pipe/type_filter_pipe'
+require 'pacer/pipe/label_collection_filter_pipe'
+require 'pacer/pipe/id_collection_filter_pipe'
+require 'pacer/pipe/label_prefix_pipe'
+require 'pacer/pipe/variable_store_iterator_wrapper'
+
+require 'pacer/pipe/property_comparison_pipe'
+
+require 'pacer/pipe/blackbox_pipeline'
+
+require 'pacer/pipe/unary_transform_pipe'
+require 'pacer/pipe/cross_product_transform_pipe'

data/lib/pacer/route/mixin/bulk_operations.rb

@@ -0,0 +1,52 @@
+module Pacer::Routes
+
+  # Additional iteration methods that allow for rapid data
+  # manipulation in transactional graphs. Bulk operations automatically
+  # manage transactions in larger batches rather than on every
+  # element created or removed or every property set.
+  module BulkOperations
+    # Like bulk_job that also returns an array of results
+    def bulk_map(size = nil, target_graph = nil)
+      result = []
+      bulk_job(size, target_graph) do |e|
+        result << yield(e)
+      end
+      result
+    end
+
+    # Iterates over each element in the route, controlling
+    # transactions so that they are only committed once every
+    # +size+ records.
+    def bulk_job(size = nil, target_graph = nil)
+      target_graph ||= graph
+      if target_graph and not target_graph.in_bulk_job?
+        begin
+          target_graph.in_bulk_job = true
+          size ||= target_graph.bulk_job_size
+          counter = 0
+          each_slice(size) do |slice|
+            print counter if Pacer.verbose?
+            counter += size
+            target_graph.managed_manual_transaction do
+              target_graph.unmanaged_transactions do
+                slice.each do |element|
+                  yield element
+                end
+              end
+              print '.' if Pacer.verbose?
+            end
+          end
+        ensure
+          puts '!' if Pacer.verbose?
+          target_graph.in_bulk_job = false
+        end
+      elsif target_graph
+        each do |element|
+          yield element
+        end
+      else
+        raise 'No graph in route for bulk job'
+      end
+    end
+  end
+end

data/lib/pacer/route/mixin/route_operations.rb

@@ -0,0 +1,107 @@
+module Pacer::Routes
+
+  # Additional convenience and data analysis methods that can be mixed into
+  # routes if they support the full route interface.
+  module RouteOperations
+    include BulkOperations
+
+    def has?(element)
+      any? { |e| e == element }
+    end
+
+    # Creates a hash where the key is the properties and return value of the
+    # given block, and the value is the number of times each key was found in
+    # the results set.
+    def group_count(*props)
+      result = Hash.new(0)
+      props = props.collect { |p| p.to_s }
+      if props.empty? and block_given?
+        each { |e| result[yield(e)] += 1 }
+      elsif block_given?
+        each do |e|
+          key = props.collect { |p| e.getProperty(p) }
+          key << yield(e)
+          result[key] += 1
+        end
+      elsif props.count == 1
+        prop = props.first
+        each do |e|
+          result[e.getProperty(prop)] += 1
+        end
+      elsif props.any?
+        each do |e|
+          result[props.collect { |p| e.getProperty(p) }] += 1
+        end
+      else
+        each do |e|
+          result[e] += 1
+        end
+      end
+      result
+    end
+
+    def most_frequent(range = 0, include_counts = false)
+      if include_counts
+        result = group_count.sort_by { |k, v| -v }[range]
+        if not result and range.is_a? Fixnum
+          []
+        else
+          result
+        end
+      else
+        result = group_count.sort_by { |k, v| -v }[range]
+        if range.is_a? Fixnum
+          result.first if result
+        elsif result
+          result.collect { |k, v| k }.to_route(:based_on => self)
+        else
+          [].to_route(:based_on => self)
+        end
+      end
+    end
+
+    # Store the current intermediate element in the route's vars hash by the
+    # given name so that it is accessible subsequently in the processing of the
+    # route.
+    def as(name)
+      chain_route :modules => VariableRouteModule, :variable_name => name
+    end
+
+    # Returns true if this route could contain both vertices and edges.
+    def mixed_route?
+      self.is_a? Pacer::Core::Graph::MixedRoute
+    end
+
+    # Returns true if this route countains only vertices.
+    def vertices_route?
+      self.is_a? Pacer::Core::Graph::VerticesRoute
+    end
+
+    # Returns true if this route countains only edges.
+    def edges_route?
+      self.is_a? Pacer::Core::Graph::EdgesRoute
+    end
+
+    def pages(elements_per_page = 1000)
+      page = []
+      results = []
+      idx = 0
+      each do |e|
+        page << e
+        idx += 1
+        if idx % elements_per_page == 0
+          results << yield(page)
+          page = []
+        end
+      end
+      yield page unless page.empty?
+      results
+    end
+
+    protected
+
+    def has_routable_class?
+      true
+    end
+  end
+end

data/lib/pacer/route/mixin/variable_route_module.rb

@@ -0,0 +1,26 @@
+module Pacer::Routes
+
+  # Adds support to a route to store a variable inline during processing. See
+  # Pacer::Routes::RouteOperations#as
+  module VariableRouteModule
+    attr_accessor :variable_name
+
+    def root?
+      false
+    end
+
+    protected
+
+    def attach_pipe(pipe)
+      Pacer::Pipes::VariableStoreIteratorWrapper.new(pipe, vars, @variable_name)
+    end
+
+    def has_routable_class?
+      false
+    end
+
+    def inspect_class_name
+      @variable_name.inspect
+    end
+  end
+end

data/lib/pacer/route/mixins.rb

@@ -0,0 +1,3 @@
+require 'pacer/route/mixin/bulk_operations'
+require 'pacer/route/mixin/route_operations'
+require 'pacer/route/mixin/variable_route_module'