pacer 2.0.10.pre-java → 2.0.12-java

data/ext/src/test/java/com/xnlogic/pacer/pipes/VerticesPipeTest.java

@@ -0,0 +1,97 @@
+package com.xnlogic.pacer.pipes;
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+import org.junit.Before;
+import org.junit.After;
+import com.tinkerpop.blueprints.impls.tg.TinkerGraph;
+import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.blueprints.Vertex;
+import java.util.Collection;
+import java.util.Arrays;
+import java.util.ArrayList;
+import com.xnlogic.pacer.pipes.VerticesPipe;
+
+public class VerticesPipeTest {
+    private TinkerGraph graph = null;
+    private Collection<Graph> graphs;
+    private Collection<Vertex> verticesThatCount;
+
+    @Before
+    public void setup() throws Exception {
+        this.graph = new TinkerGraph();
+        this.graphs = Arrays.asList((Graph)this.graph);
+    }
+
+    private void createVertices() {
+        Vertex v1 = this.graph.addVertex(1);
+        Vertex v2 = this.graph.addVertex(2);
+        Vertex v3 = this.graph.addVertex(3);
+
+        this.verticesThatCount = Arrays.asList(v1, v2, v3);
+    }
+  
+    @After
+    public void teardown() throws Exception {
+        this.graph.shutdown();
+        this.graph = null;
+    }
+
+    @Test
+    public void getVerticesFromGraphTest() {
+        this.createVertices();
+
+        VerticesPipe verticesPipe = new VerticesPipe();
+        verticesPipe.setStarts(this.graphs);
+        
+        Collection<Vertex> vertices = new ArrayList<Vertex>();
+
+        while (verticesPipe.hasNext()) {
+            vertices.add(verticesPipe.next());
+        }
+
+        assertEquals(3, vertices.size());
+        assertTrue(vertices.containsAll(this.verticesThatCount));
+    }
+
+    @Test
+    public void getVerticesFromGraphAfterResetTest() {
+        this.createVertices();
+        
+        VerticesPipe verticesPipe = new VerticesPipe();
+        verticesPipe.setStarts(this.graphs);
+        
+        Collection<Vertex> vertices = new ArrayList<Vertex>();
+
+        while (verticesPipe.hasNext()) {
+            vertices.add(verticesPipe.next());
+        }
+
+        assertEquals(3, vertices.size());
+        assertTrue(vertices.containsAll(this.verticesThatCount));
+        
+        verticesPipe.reset();
+        vertices.clear();
+        
+        while (verticesPipe.hasNext()) {
+            vertices.add(verticesPipe.next());
+        }
+
+        assertEquals(3, vertices.size());
+        assertTrue(vertices.containsAll(this.verticesThatCount));
+    }
+    
+    @Test
+    public void getNoVerticesFromGraphTest() {
+        VerticesPipe verticesPipe = new VerticesPipe();
+        verticesPipe.setStarts(this.graphs);
+        
+        Collection<Vertex> vertices = new ArrayList<Vertex>();
+
+        while (verticesPipe.hasNext()) {
+            vertices.add(verticesPipe.next());
+        }
+
+        assertEquals(0, vertices.size());
+    }
+}

data/lib/pacer/core/graph/element_route.rb

@@ -166,8 +166,8 @@ module Pacer::Core::Graph
 
     protected
 
-    def configure_iterator(iter)
-      pipe = Pacer::Pipes::WrappingPipe.new graph, element_type, extensions
+    def configure_iterator(iter = nil, g = nil)
+      pipe = Pacer::Pipes::WrappingPipe.new((g || graph), element_type, extensions)
       pipe.wrapper = wrapper if wrapper
       pipe.setStarts iter
       pipe

data/lib/pacer/core/graph/path_route.rb

@@ -96,9 +96,9 @@ HELP
     end
     protected
 
-    def configure_iterator(iter)
+    def configure_iterator(iter = nil, g = nil)
       if respond_to? :graph
-        pipe = Pacer::Pipes::PathWrappingPipe.new(graph)
+        pipe = Pacer::Pipes::PathWrappingPipe.new(g || graph)
         pipe.setStarts iter
         pipe
       else

data/lib/pacer/core/route.rb

@@ -380,6 +380,25 @@ HELP
         chain_route(:extensions => [], :wrapper => nil)
       end
 
+      # Returns an outer proc that returns an inner proc that will execute the given route for a given input data or element.
+      #
+      # The outer proc contains the already-compiled route (the most expensive part of the process). When you call it, it returns
+      # the inner proc. If you call it with no arguments, it will prepare the proc in the context of the original route used to create
+      # it. In a multi-tenant or multi-graph system, call the outer proc with the current graph to prepare so that it correctly wraps
+      # returned elements.
+      #
+      # The inner proc contains a non-thread-safe data pipeline compiled from the route. Call it with an item of input data for the detached
+      # pipe and it will execute the pipeline for that piece of data and return the result or results.
+      #
+      # A detached route can be configured with or without gather enabled. If enabled (default), the result will be an ArrayList of
+      # all result data produced by the detached route. If not, the result will be either the first thing produced by the route or null if
+      # nothing was produced.
+      def detach
+        route = yield Pacer::Route.empty(self)
+        DetachedRoute.new route
+      end
+
+
       # Change the source of this route.
       #
       # @note all routes derived from any route in the chain will be
@@ -421,7 +440,7 @@ HELP
 
       # Overridden to extend the iterator to apply mixins
       # or wrap elements
-      def configure_iterator(iter)
+      def configure_iterator(iter = nil, g = nil)
         iter
       end
 
@@ -617,6 +636,63 @@ HELP
         end
       end
 
+
+      class DetachedRoute
+        attr_reader :route, :preconfig
+
+        def initialize(route)
+          @route = route
+          w = route.send(:configure_iterator)
+          if w.respond_to? :instance
+            @preconfig = w
+          end
+        end
+
+        def build(graph, gather = true)
+          if route.is_a? Pacer::Filter::EmptyFilter
+            pipe = Pacer::Pipes::IdentityPipe.new
+          else
+            pipe = Pacer::Route.pipeline route
+          end
+          expando = Pacer::Pipes::ExpandablePipe.new
+          expando.enablePath true
+          expando.setStarts(Pacer::Pipes::EmptyIterator::INSTANCE)
+          pipe.setStarts expando
+          if preconfig
+            pipe = preconfig.instance pipe, graph
+          else
+            pipe = route.send(:configure_iterator, pipe, graph)
+          end
+          pipe = yield pipe if block_given?
+          if gather
+            g = Pacer::Pipes::GatherPipe.new
+            g.setStarts pipe
+            DetachedPipe.new(expando, g, true)
+          else
+            DetachedPipe.new(expando, pipe, false)
+          end
+        end
+      end
+
+      class DetachedPipe
+        attr_reader :expando, :pipe, :collection
+
+        def initialize(expando, pipe, collection)
+          @expando = expando
+          @pipe = pipe
+          @collection = collection
+        end
+
+        def read(element)
+          pipe.reset unless collection
+          expando.add element
+          if pipe.hasNext
+            pipe.next
+          elsif collection
+            []
+          end
+        end
+      end
     end
   end
 end

data/lib/pacer/filter/property_filter/edge_filters.rb

@@ -31,7 +31,7 @@ module Pacer
         def build_pipeline(route, start_pipe = nil, pipe = nil)
           pipe ||= start_pipe
           if labels.any?
-            label_pipe = Pacer::Pipes::LabelCollectionFilterPipe.new labels
+            label_pipe = Pacer::Pipes::LabelCollectionFilterPipe.new labels.map(&:to_s)
             label_pipe.set_starts pipe if pipe
             Pacer.debug_pipes << { :name => labels.inspect, :start => pipe, :end => block_pipe } if Pacer.debug_pipes
             pipe = label_pipe
@@ -44,6 +44,10 @@ module Pacer
           labels.any? or super
         end
 
+        def extensions_only?
+          labels.empty? and super
+        end
+
         def to_s
           if labels.any?
             [labels.map { |l| l.to_sym.inspect }.join(', '), super].reject { |s| s == '' }.join ', '

data/lib/pacer/filter/property_filter/filters.rb

@@ -134,15 +134,30 @@ module Pacer
           end
         end
 
+        # Not used for regular filtering, but useful to test an element against
+        # a complex set of conditions without having to build a route.
+        #
+        # Returns a proc that can be called with an element to test and returns
+        # true if the element matches the conditions.
+        def to_predicate
+          expando, pipe = build_pipeline(nil, com.tinkerpop.pipes.util.iterators.ExpandableIterator.new)
+          proc do |e|
+            expando.add e
+            pipe.next if pipe.hasNext
+          end
+        end
+
         def build_pipeline(route, start_pipe, pipe = nil)
-          self.graph = route.graph
           pipe ||= start_pipe
-          route_modules.each do |mod|
-            extension_route = mod.route(Pacer::Route.empty(route))
-            s, e = extension_route.send :build_pipeline
-            s.setStarts(pipe) if pipe
-            start_pipe ||= s
-            pipe = e
+          if route
+            self.graph = route.graph
+            route_modules.each do |mod|
+              extension_route = mod.route(Pacer::Route.empty(route))
+              s, e = extension_route.send :build_pipeline
+              s.setStarts(pipe) if pipe
+              start_pipe ||= s
+              pipe = e
+            end
           end
           encoded_properties.each do |key, value|
             case value
@@ -161,6 +176,7 @@ module Pacer
             end
           end
           blocks.each do |block|
+            # Will work if route is nil.
             block_pipe = Pacer::Pipes::BlockFilterPipe.new(route, block)
             block_pipe.set_starts pipe if pipe
             Pacer.debug_pipes << { :name => 'block', :start => pipe, :end => block_pipe } if Pacer.debug_pipes

data/lib/pacer/pipe/path_wrapping_pipe.rb

@@ -6,9 +6,20 @@ module Pacer
 
       def initialize(graph, vertex_extensions = [], edge_extensions = [])
         super()
-        @graph = graph
-        @vertex_wrapper = Pacer::Wrappers::WrapperSelector.build graph, :vertex, vertex_extensions || Set[]
-        @edge_wrapper = Pacer::Wrappers::WrapperSelector.build graph, :edge, edge_extensions || Set[]
+        if graph.is_a? Array
+          @graph, @vertex_wrapper, @edge_wrapper = graph
+        else
+          @graph = graph
+          @vertex_wrapper = Pacer::Wrappers::WrapperSelector.build graph, :vertex, vertex_extensions || Set[]
+          @edge_wrapper = Pacer::Wrappers::WrapperSelector.build graph, :edge, edge_extensions || Set[]
+        end
+      end
+
+      def instance(pipe, g)
+        g ||= graph
+        p = PathWrappingPipe.new [g, vertex_wrapper, edge_wrapper]
+        p.setStarts pipe
+        p
       end
 
       def getCurrentPath

data/lib/pacer/pipe/wrapping_pipe.rb

@@ -5,10 +5,21 @@ module Pacer
 
       def initialize(graph, element_type = nil, extensions = [])
         super()
-        @graph = graph
-        @element_type = element_type
-        @extensions = extensions || []
-        @wrapper = Pacer::Wrappers::WrapperSelector.build graph, element_type, @extensions
+        if graph.is_a? Array
+          @graph, @wrapper = graph
+        else
+          @graph = graph
+          @element_type = element_type
+          @extensions = extensions || []
+          @wrapper = Pacer::Wrappers::WrapperSelector.build graph, element_type, @extensions
+        end
+      end
+
+      def instance(pipe, g)
+        g ||= graph
+        p = WrappingPipe.new [g, wrapper]
+        p.setStarts pipe
+        p
       end
 
       def getSideEffect

data/lib/pacer/pipes.rb

@@ -15,6 +15,21 @@ module Pacer
     import com.tinkerpop.pipes.transform.PropertyPipe
     import com.tinkerpop.pipes.transform.PropertyMapPipe
 
+    import com.xnlogic.pacer.pipes.BlackboxPipeline
+    import com.xnlogic.pacer.pipes.EdgesPipe
+    import com.xnlogic.pacer.pipes.CollectionFilterPipe
+    import com.xnlogic.pacer.pipes.ExpandablePipe
+    SimpleExpandablePipe = com.tinkerpop.pipes.util.ExpandablePipe
+    import com.xnlogic.pacer.pipes.IdCollectionFilterPipe
+    import com.xnlogic.pacer.pipes.IsEmptyPipe
+    import com.xnlogic.pacer.pipes.LabelCollectionFilterPipe
+    import com.xnlogic.pacer.pipes.IsUniquePipe
+    import com.xnlogic.pacer.pipes.VerticesPipe
+    import com.xnlogic.pacer.pipes.LabelPrefixPipe
+    import com.xnlogic.pacer.pipes.NeverPipe
+    import com.tinkerpop.pipes.util.iterators.EmptyIterator
+    import com.tinkerpop.pipes.transform.GatherPipe
+
     IN = com.tinkerpop.blueprints.Direction::IN
     OUT = com.tinkerpop.blueprints.Direction::OUT
     BOTH = com.tinkerpop.blueprints.Direction::BOTH
@@ -33,32 +48,19 @@ end
 
 require 'pacer/pipe/ruby_pipe'
 
-require 'pacer/pipe/vertices_pipe'
-require 'pacer/pipe/edges_pipe'
-
-require 'pacer/pipe/never_pipe'
 require 'pacer/pipe/multi_pipe'
 require 'pacer/pipe/block_filter_pipe'
-require 'pacer/pipe/collection_filter_pipe'
 require 'pacer/pipe/enumerable_pipe'
-require 'pacer/pipe/expandable_pipe'
 require 'pacer/pipe/loop_pipe'
 require 'pacer/pipe/process_pipe'
 require 'pacer/pipe/visitor_pipe'
 require 'pacer/pipe/simple_visitor_pipe'
-require 'pacer/pipe/is_unique_pipe'
-require 'pacer/pipe/is_empty_pipe'
 require 'pacer/pipe/stream_sort_pipe'
 require 'pacer/pipe/stream_uniq_pipe'
 require 'pacer/pipe/type_filter_pipe'
-require 'pacer/pipe/label_collection_filter_pipe'
-require 'pacer/pipe/id_collection_filter_pipe'
-require 'pacer/pipe/label_prefix_pipe'
 
 require 'pacer/pipe/property_comparison_pipe'
 
-require 'pacer/pipe/blackbox_pipeline'
-
 require 'pacer/pipe/unary_transform_pipe'
 require 'pacer/pipe/cross_product_transform_pipe'
 

data/lib/pacer/route.rb

@@ -18,6 +18,7 @@ module Pacer
       # @return [Pacer::Pipes::BlackboxPipeline] an instantiated pipeline
       def pipeline(route)
         s, e = route.send(:build_pipeline)
+        e = yield e if block_given?
         if s.equal?(e)
           s
         else

data/lib/pacer/side_effect/as_var.rb

@@ -44,7 +44,7 @@ module Pacer
       def attach_pipe(pipe)
         if element_type == :vertex or element_type == :edge or element_type == :mixed
           wrapped = Pacer::Pipes::WrappingPipe.new graph, element_type, extensions
-          wrapped.setStarts pipe
+          wrapped.setStarts pipe if pipe
           as_pipe = AsPipe.new(wrapped, vars, variable_name)
           unwrapped = Pacer::Pipes::UnwrappingPipe.new
           unwrapped.setStarts as_pipe

data/lib/pacer/side_effect/is_unique.rb

@@ -14,7 +14,7 @@ module Pacer
       def unique?
         check = nil
         is_unique { |pipe| check = pipe }.each do
-          return false unless check.unique?
+          return false unless check.isUnique
         end
         true
       end
@@ -30,7 +30,7 @@ module Pacer
 
       def attach_pipe(end_pipe)
         checked = Pacer::Pipes::IsUniquePipe.new
-        checked.setStarts end_pipe
+        checked.setStarts end_pipe if end_pipe
         on_build_pipe.call checked if on_build_pipe
         checked
       end

data/lib/pacer/transform/branch.rb

@@ -44,7 +44,7 @@ module Pacer
       def clone(conf = {})
         back.chain_route({transform: Branch,
                           branches: branches.clone,
-                          element_type: back.element_type,
+                          element_type: element_type,
                           conds: conds.clone,
                           extensions: extensions.clone,
                           split_pipe_class: split_pipe_class,

data/lib/pacer/transform/lookup_ids.rb

@@ -28,7 +28,7 @@ module Pacer
         else
           fail ClientError, 'Can not look up elements without the element_type'
         end
-        pipe.setStarts end_pipe
+        pipe.setStarts end_pipe if end_pipe
         pipe
       end
     end

data/lib/pacer/transform/path_tree.rb

@@ -29,7 +29,7 @@ module Pacer
 
       def attach_pipe(end_pipe)
         pipe = PathTreePipe.new compare
-        pipe.setStarts end_pipe
+        pipe.setStarts end_pipe if end_pipe
         pipe
       end
 

data/lib/pacer/transform/reduce.rb

@@ -86,7 +86,7 @@ HELP
       def attach_pipe(end_pipe)
         if @enter and @reduce and @leave
           pipe = ReducerPipe.new self, @enter, @reduce, @leave
-          pipe.setStarts end_pipe
+          pipe.setStarts end_pipe if end_pipe
           pipe
         else
           fail Pacer::ClientError, 'enter, reduce, and leave must all be specified for reducers'

data/lib/pacer/transform/sort_section.rb

@@ -1,10 +1,18 @@
 module Pacer
   module Routes
     module RouteOperations
+      # Arity 2 uses custom sort logic. Arity 1 uses sort_by logic.
       def sort_section(section = nil, &block)
-        chain_route transform: :sort_section, sort_by_block: block, section: section
+        if not block
+          chain_route transform: :sort_section, section: section
+        elsif block.arity == 2
+          chain_route transform: :sort_section, custom_sort_block: block, section: section
+        else
+          chain_route transform: :sort_section, sort_by_block: block, section: section
+        end
       end
 
+      # Deprecated: use sort_section
       def custom_sort_section(section = nil, &block)
         chain_route transform: :sort_section, custom_sort_block: block, section: section
       end
@@ -40,27 +48,27 @@ module Pacer
         attr_reader :pf_1, :pf_2, :to_sort, :to_emit, :section, :route
         attr_reader :getPathToHere
 
-        def initialize(route, section, pipe_function)
+        def initialize(route, visitor_pipe, pipe_function)
           super()
           @to_emit = []
-          @section = section
+          @visitor_pipe = visitor_pipe
           @route = route
           @to_sort = []
           @paths = []
-          if section
-            section.visitor = self
+          if visitor_pipe
+            visitor_pipe.visitor = self
           else
             on_element nil
           end
           if pipe_function
             if pipe_function.arity == 1
               @pf_1 = pipe_function
-              section.use_on_element = false
+              visitor_pipe.use_on_element = false
             else
               @pf_2 = pipe_function
             end
           else
-            section.use_on_element = false
+            visitor_pipe.use_on_element = false
           end
         end
 
@@ -119,8 +127,8 @@ module Pacer
       class CustomSortPipe < SortBySectionPipe
         attr_reader :sort_block
 
-        def initialize(route, section, sort_block, graph, wrapper)
-          super route, section, nil
+        def initialize(route, visitor_pipe, sort_block, graph, wrapper)
+          super route, visitor_pipe, nil
           @sort_block = sort_block
           @graph = graph
           @wrapper = wrapper

data/lib/pacer/transform/stream_sort.rb

@@ -1,5 +1,6 @@
 module Pacer
   module Routes::RouteOperations
+    # Deprecated: use sort_section instead.
     def stream_sort(buffer = 1000, silo = 100)
       chain_route :transform => :stream_sort, :buffer => buffer, :silo => silo
     end

data/lib/pacer/transform/stream_uniq.rb

@@ -1,5 +1,6 @@
 module Pacer
   module Routes::RouteOperations
+    # Deprecated: use uniq_section instead.
     def stream_uniq(buffer = 1000)
       chain_route :transform => :stream_uniq, :buffer => buffer
     end

data/lib/pacer/transform/wrapped_path.rb

@@ -15,7 +15,7 @@ module Pacer
 
       def attach_pipe(end_pipe)
         pipe = Pacer::Pipes::PathWrappingPipe.new(graph)
-        pipe.setStarts end_pipe
+        pipe.setStarts end_pipe if end_pipe
         pipe
       end
     end

data/lib/pacer/version.rb

@@ -1,5 +1,5 @@
 module Pacer
-  VERSION = "2.0.10.pre"
+  VERSION = "2.0.12"
   # Clients may optionally define the following constants in the Pacer namespace:
   # - LOAD_JARS : set to false to manage jar loading in the client. Be sure to load the jars defined in JARFILES.
   # - LOCKJAR_LOCK_OPTS : set some options to be passed to LockJar.lock (ie. :lockfile, :download_artifacts, :local_repo)

data/lib/pacer/visitors/section.rb

@@ -18,10 +18,14 @@ module Pacer
 
       def will_visit!
         @visitor_count = visitor_count + 1
+        visitor_count - 1
       end
 
-      def section_visitor
-        section_visitors.pop
+      def section_visitor!(visitor_num)
+        vpipes = Thread.current["visitors_#{object_id}"]
+        vpipe = vpipes[visitor_num]
+        vpipes[visitor_num] = nil
+        vpipe
       end
 
       protected
@@ -34,12 +38,15 @@ module Pacer
       attr_reader :section_visitors
 
       def attach_pipe(end_pipe)
+        # With detached pipes, pipe construction happens in
+        # multiple threads, multiple times.
         pipe = end_pipe
-        @section_visitors = (1..visitor_count).map do
+        vpipes = (1..visitor_count).map do
           pipe = Pacer::Pipes::SimpleVisitorPipe.new Pacer::Wrappers::WrapperSelector.build(graph, element_type, extensions), graph
           pipe.setStarts end_pipe if end_pipe
           end_pipe = pipe
         end
+        Thread.current["visitors_#{object_id}"] = vpipes
         pipe
       end
 

data/lib/pacer/visitors/visits_section.rb

@@ -2,7 +2,7 @@ module Pacer
   module Visitors
     # This module is mixed in to the route that actually refers to this section.
     module VisitsSection
-      attr_reader :section, :section_route
+      attr_reader :section, :section_route, :visitor_num
 
       def section=(section)
         if section.is_a? Symbol
@@ -14,14 +14,14 @@ module Pacer
         else
           raise ArgumentError, "Unknown section #{ section }. Provide either a name or a route created with the #section methed."
         end
-        @section_route.will_visit!
-        @section_route
+        @visitor_num = section_route.will_visit!
+        section_route
       end
 
       protected
 
       def section_visitor
-        section_route.section_visitor if section_route
+        section_route.section_visitor!(visitor_num) if visitor_num
       end
 
       def section_visitor_target

data/lib/pacer.rb

@@ -23,6 +23,7 @@ require 'pp'
 require 'rubygems'
 require 'lock_jar'
 require 'pacer/support/lock_jar'
+require 'pacer-ext.jar'
 
 if (not defined? Pacer::LOAD_JARS) or Pacer::LOAD_JARS == true
   bundle_jarfiles = LockJar.register_bundled_jarfiles # defined in pacer/support/lock_jar.rb

data/pacer.gemspec

@@ -13,10 +13,11 @@ Gem::Specification.new do |s|
   s.summary = %Q{A very efficient and easy to use graph traversal engine.}
   s.description = %Q{Pacer defines composeable routes through a graph and then traverses them very quickly.}
 
-  s.files = `git ls-files`.split("\n")
+  s.files = `git ls-files`.split("\n") + ['lib/pacer-ext.jar']
   s.test_files = `git ls-files -- spec/*`.split("\n")
   s.require_paths = ['lib']
   s.add_dependency "lock_jar", "~> 0.10.2"
   s.add_development_dependency 'xn_gem_release_tasks'
+  s.add_development_dependency "rake-compiler",  "~> 0.9.2"
 end