pacer 1.1.0-java → 1.1.1-java

data/lib/pacer/core/array_route.rb

@@ -0,0 +1,78 @@
+module Pacer
+  module Core
+    module ArrayRoute
+      def help(section = nil)
+        case section
+        when :arrays
+          puts <<HELP
+The following array route methods are available:
+
+#lengths            Return the length of each array
+
+#transpose          Route version of Ruby's Array#transpase
+
+#compacted          Removes nils from each array
+
+#heads              Route to only the first element from each array
+
+#tails              Route to only the last element from each array
+
+#pairs(head, tail)  Route to an array of only the head and tail elements
+    head: Number    Array index of the : first  : element in the pair
+    tail: Number                       : second :
+
+#len(n)             Filter paths by length
+    n: Number | Range
+
+HELP
+        else
+          super
+        end
+        description
+      end
+
+      def lengths
+        map(element_type: :integer) { |s| s.length }
+      end
+
+      # This could be done more efficiently by reimplementing
+      # transpose... Right now it needs 2n memory.
+      def transpose
+        gather { [] }.
+          map(element_type: :array) { |a| a.transpose }.
+          scatter(element_type: :array)
+      end
+
+      def compacted
+        map element_type: element_type, route_name: 'compact' do |a|
+          a.compact
+        end
+      end
+
+      def heads(et = nil)
+        map element_type: et, route_name: 'heads' do |a|
+          a.first
+        end
+      end
+
+      def tails(et = nil)
+        map element_type: et, route_name: 'tails' do |a|
+          a.last
+        end
+      end
+
+      def pairs(head = 0, tail = -1)
+        map element_type: element_type, route_name: "pairs[#{ head },#{ tail }]" do |a|
+          [a[head], a[tail]]
+        end
+      end
+
+      def len(n)
+        select do |path|
+          n === path.length
+        end
+      end
+    end
+  end
+end
+

data/lib/pacer/core/graph/path_route.rb

@@ -4,9 +4,9 @@ module Pacer::Core::Graph
       case section
       when :paths
         puts <<HELP
-The following path helper methods are available:
+The following path-specific route methods are available:
 
-#transpose       Works the same as Array transpase
+See also the :arrays section for more available methods
 
 #subgraph(target_graph, opts)   Add each element in the path to the graph
     target_graph: PacerGraph (optional) if not specified creates a new TG.
@@ -18,19 +18,6 @@ The following path helper methods are available:
       ignore_missing_vertices: Boolean  Squelches the above mentioned exception
       show_missing_vertices: Boolean    Complain about missing vertices
 
-#compact_paths      Removes nils from paths
-
-#heads              Route to only the first element from each path
-
-#tails              Route to only the last element from each path
-
-#pairs(head, tail)  Route to a mini path of only the first and last elements
-    head: Number    Array index of the : first  : element in the pair
-    tail: Number                       : second :
-
-#len(n)             Filter paths by length
-    n: Number | Range
-
 #hashify            Make a hash of the properties and relationships of the path
     This is just a simple view on the data to facilitate analysis
 
@@ -41,10 +28,6 @@ HELP
       description
     end
 
-    def transpose
-      collect { |arraylist| arraylist.to_a }.transpose
-    end
-
     def subgraph(target_graph = nil, opts = {})
       raise "Can't create a subgraph within itself." if target_graph == graph
       target_graph ||= Pacer.tg
@@ -82,34 +65,16 @@ HELP
       end
     end
 
-    def compact_paths
-      map element_type: :path, route_name: 'compact' do |path|
-        path.compact
-      end
+    def transpose
+      map(element_type: :array, &:to_a).transpose
     end
 
     def heads(et = :vertex)
-      map element_type: et, route_name: 'heads' do |path|
-        path.first
-      end
+      super et
     end
 
     def tails(et = :vertex)
-      map element_type: et, route_name: 'tails' do |path|
-        path.last
-      end
-    end
-
-    def pairs(head = 0, tail = -1)
-      map element_type: :path, route_name: "pairs[#{ head },#{ tail }]" do |path|
-        [path[head], path[tail]]
-      end
-    end
-
-    def len(n)
-      select do |path|
-        n === path.length
-      end
+      super et
     end
 
     def hashify

data/lib/pacer/core/hash_route.rb

@@ -0,0 +1,10 @@
+module Pacer
+  module Core
+    module HashRoute
+      def lengths
+        map(element_type: :integer) { |s| s.length }
+      end
+    end
+  end
+end
+

data/lib/pacer/core/string_route.rb

@@ -0,0 +1,10 @@
+module Pacer
+  module Core
+    module StringRoute
+      def lengths
+        map(element_type: :integer) { |s| s.length }
+      end
+    end
+  end
+end
+

data/lib/pacer/loader.rb

@@ -19,6 +19,9 @@ require 'pacer/exceptions'
 require 'pacer/pipes'
 
 require 'pacer/core/route'
+require 'pacer/core/string_route'
+require 'pacer/core/array_route'
+require 'pacer/core/hash_route'
 require 'pacer/core/graph'
 require 'pacer/core/side_effect'
 
@@ -83,6 +86,7 @@ require 'pacer/transform/stream_uniq'
 require 'pacer/transform/gather'
 require 'pacer/transform/map'
 require 'pacer/transform/flat_map'
+require 'pacer/transform/reduce'
 require 'pacer/transform/make_pairs'
 require 'pacer/transform/process'
 require 'pacer/transform/join'

data/lib/pacer/route_builder.rb

@@ -25,7 +25,10 @@ module Pacer
       element_types[:vertex] = [Pacer::Core::Graph::ElementRoute, Pacer::Core::Graph::VerticesRoute]
       element_types[:edge] = [Pacer::Core::Graph::ElementRoute, Pacer::Core::Graph::EdgesRoute]
       element_types[:mixed] = [Pacer::Core::Graph::ElementRoute, Pacer::Core::Graph::MixedRoute]
-      element_types[:path] = [Pacer::Core::Graph::PathRoute]
+      element_types[:path] = [Pacer::Core::ArrayRoute, Pacer::Core::Graph::PathRoute]
+      element_types[:string] = [Pacer::Core::StringRoute]
+      element_types[:array] = [Pacer::Core::ArrayRoute]
+      element_types[:hash] = [Pacer::Core::HashRoute]
     end
 
     def chain(source, args)

data/lib/pacer/side_effect/aggregate.rb

@@ -1,9 +1,10 @@
 module Pacer
   module Routes::RouteOperations
-    def aggregate(into = nil)
+    def aggregate(into = nil, &block)
       aggregate = ::Pacer::SideEffect::Aggregate
       r = self
       r = section(into, aggregate::ElementSet) if into.is_a? Symbol
+      into = block if block
       r.chain_route :side_effect => aggregate, :into => into
     end
   end
@@ -25,13 +26,16 @@ module Pacer
       protected
 
       def attach_pipe(end_pipe)
-        if into.is_a? Symbol
+        case into
+        when Symbol
           hs = vars[into] = HashSet.new
           pipe = AggregatePipe.new hs
-        elsif into
-          pipe = AggregatePipe.new into
-        else
+        when Proc
+          pipe = AggregatePipe.new into.call(self)
+        when nil
           pipe = AggregatePipe.new HashSet.new
+        else
+          pipe = AggregatePipe.new into
         end
         pipe.setStarts end_pipe if end_pipe
         pipe

data/lib/pacer/transform/cap.rb

@@ -1,7 +1,7 @@
 module Pacer
   module Core::SideEffect
-    def cap
-      back.chain_route :transform => :cap, :with => self, :element_type => :object
+    def cap(opts = {})
+      back.chain_route({:transform => :cap, :with => self, :element_type => :object}.merge(opts))
     end
   end
 

data/lib/pacer/transform/gather.rb

@@ -1,8 +1,8 @@
 module Pacer
   module Core
     module Route
-      def gather
-        aggregate.cap
+      def gather(into = nil, &block)
+        aggregate(into, &block).cap(element_type: :array)
       end
     end
   end

data/lib/pacer/transform/reduce.rb

@@ -0,0 +1,165 @@
+module Pacer
+  module Routes
+    module RouteOperations
+      def reducer(opts = {}, &block)
+        chain_route({transform: :reduce, reduce: block}.merge(opts))
+      end
+    end
+  end
+
+  module Transform
+    module Reduce
+      # The goal is to break down the xml stream from being a black
+      # box iterator to doing the job in a few steps:
+
+      def help(section = nil)
+        case section
+        when nil
+          puts <<HELP
+
+HELP
+        when :example
+          puts <<HELP
+This example usage is from pacer-xml plugin v0.2. I transform a raw
+stream of lines from a 79MB file that contains > 4000 concatinated xml
+documents averaging 600 lines each. to a stream of imported nodes:
+
+First, a little setup: create a graph, open the file and make a route of
+its lines
+
+  graph = Pacer.tg
+  f = File.open '/tmp/ipgb20120103.xml'
+  lines = f.each_line.to_route(element_type: :string).route
+
+Create a simple reducer that delimits sections when it hits a DTD tag
+and when it gets to the end of the file (that's the s.nil?). and reduces
+the stream by pushing each section's lines into an array. When a section
+is entered, the initial value is provided by the return value of the
+enter block.
+
+  reducer = lines.reducer(element_type: :array).route
+  reducer.enter  { |s|        []     if s =~ /<\?xml/ }
+  reducer.reduce { |s, lines| lines << s              }
+  reducer.leave  { |s, lines| s.nil? or s =~ /<\?xml/ }
+
+Now we're back in the territory of fairly vanilla routes. We join each
+section, use the pacer-xml gem's StringRoute#xml method to parse the XML
+with Nokogiri and then its XmlRoute#import method to turn those XML
+nodes into graph elements.
+
+  vertex = reducer.map(element_type: :string, &:join).xml.limit(1).import(graph).first
+
+  graph           #=> #<PacerGraph tinkergraph[vertices:88 edges:90]
+  vertex          #=> #<V[0] us-patent-grant>
+
+We can see that we've now got a graph with 88 vertices and 90 edges.
+
+HELP
+        end
+      end
+
+      attr_writer :enter, :reduce, :leave
+
+      def enter(&block)
+        if block
+          @enter = block
+        end
+        self
+      end
+
+      def reduce(&block)
+        if block
+          @reduce = block
+        end
+        self
+      end
+
+      def leave(same_as = nil, &block)
+        if same_as == :enter
+          @leave = @enter
+        elsif block
+          @leave = block
+        end
+        self
+      end
+
+      def attach_pipe(end_pipe)
+        if @enter and @reduce and @leave
+          pipe = ReducerPipe.new self, @enter, @reduce, @leave
+          pipe.setStarts end_pipe
+          pipe
+        else
+          fail Pacer::ClientError, 'enter, reduce, and leave must all be specified for reducers'
+        end
+      end
+
+      class ReducerPipe < Pacer::Pipes::RubyPipe
+        attr_reader :enter, :reduce, :leave, :change_value
+        attr_accessor :changed_value, :value_changed
+        attr_accessor :next_value
+
+        def initialize(back, enter, reduce, leave)
+          super()
+          @change_value = proc do |new_value|
+            self.changed_value = new_value
+            self.value_changed = true
+          end
+          @enter = Pacer::Wrappers::WrappingPipeFunction.new back, enter
+          @reduce = Pacer::Wrappers::WrappingPipeFunction.new back, reduce
+          @leave = Pacer::Wrappers::WrappingPipeFunction.new back, leave
+          @next_value = nil
+        end
+
+        def processNextStart
+          if next_value
+            collecting = true
+            value = next_value
+            self.next_value = nil
+          else
+            collecting = false
+          end
+          leaving = false
+          final_value = nil
+          while raw_element = starts.next
+            if collecting
+              if leave.call_with_args(raw_element, value, change_value)
+                leaving = true
+                return_value = final_value(value)
+                collecting = false
+              else
+                value = reduce.call_with_args(raw_element, value)
+              end
+            end
+            if not collecting
+              value = enter.call raw_element
+              if value
+                collecting = true
+                value = reduce.call_with_args(raw_element, value)
+              end
+            end
+            if leaving
+              self.next_value = value if collecting
+              return return_value
+            end
+          end
+        rescue Pacer::EmptyPipe, java.util.NoSuchElementException
+          if collecting and leave.call_with_args(nil, value, change_value)
+            return final_value(value)
+          end
+          raise EmptyPipe.instance
+        end
+
+        private
+
+        def final_value(value)
+          if value_changed
+            self.value_changed = false
+            value = changed_value
+            self.changed_value = nil
+          end
+          value
+        end
+      end
+    end
+  end
+end

data/lib/pacer/version.rb

@@ -1,6 +1,6 @@
 module Pacer
   unless const_defined? :VERSION
-    VERSION = "1.1.0"
+    VERSION = "1.1.1"
 
     JAR = "pacer-#{ VERSION }-standalone.jar"
     JAR_PATH = "lib/#{ JAR }"

data/pom.xml

@@ -7,7 +7,7 @@
     <artifactId>pacer</artifactId>
     <!-- NOTE: the following properties are automatically updated based on the values in lib/pacer-neo4j/version.rb -->
     <properties>
-      <gem.version>1.1.0</gem.version>
+      <gem.version>1.1.1</gem.version>
       <blueprints.version>2.1.0</blueprints.version>
       <pipes.version>2.1.0</pipes.version>
       <gremlin.version>2.1.0</gremlin.version>

data/spec/pacer/transform/path_tree_spec.rb

@@ -20,7 +20,7 @@ Run.tg :read_only do
     end
     let(:paths) { second.paths }
     let(:payloads) { paths.payloads }
-    let(:compacted) { payloads.compact_paths }
+    let(:compacted) { payloads.compacted }
 
     describe 'basic paths' do
       subject { paths }

metadata

@@ -2,7 +2,7 @@
 name: pacer
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.1.0
+  version: 1.1.1
 platform: java
 authors:
 - Darrick Wiebe
@@ -40,6 +40,7 @@ files:
 - lib/pacer/blueprints/payload_elements.rb
 - lib/pacer/blueprints/ruby_graph.rb
 - lib/pacer/blueprints/tg.rb
+- lib/pacer/core/array_route.rb
 - lib/pacer/core/graph.rb
 - lib/pacer/core/graph/edges_route.rb
 - lib/pacer/core/graph/element_route.rb
@@ -48,8 +49,10 @@ files:
 - lib/pacer/core/graph/mixed_route.rb
 - lib/pacer/core/graph/path_route.rb
 - lib/pacer/core/graph/vertices_route.rb
+- lib/pacer/core/hash_route.rb
 - lib/pacer/core/route.rb
 - lib/pacer/core/side_effect.rb
+- lib/pacer/core/string_route.rb
 - lib/pacer/exceptions.rb
 - lib/pacer/filter/block_filter.rb
 - lib/pacer/filter/collection_filter.rb
@@ -132,6 +135,7 @@ files:
 - lib/pacer/transform/path_tree.rb
 - lib/pacer/transform/payload.rb
 - lib/pacer/transform/process.rb
+- lib/pacer/transform/reduce.rb
 - lib/pacer/transform/scatter.rb
 - lib/pacer/transform/sort_section.rb
 - lib/pacer/transform/stream_sort.rb
@@ -212,7 +216,7 @@ files:
 - spec/support/use_transactions.rb
 - spec/tackle/simple_mixin.rb
 - spec/tackle/tinkerpop_graph_mixins.rb
-- lib/pacer-1.1.0-standalone.jar
+- lib/pacer-1.1.1-standalone.jar
 homepage: http://github.com/pangloss/pacer
 licenses:
 - MIT