related 0.4.0 → 0.5.0

data/CHANGELOG

@@ -1,4 +1,9 @@
 
+*0.5*
+
+* Real-time stream processing
+* Related::Node::Query#find
+
 *0.4*
 
 * Related.root

data/README.md

@@ -72,6 +72,7 @@ node.outgoing(:friends).relationships
 node.outgoing(:friends).nodes
 node.outgoing(:friends).limit(5)
 node.outgoing(:friends).options(:fields => ..., :model => ...)
+node1.outgoing(:friends).relationships.find(node2)
 node1.path_to(node2).outgoing(:friends).depth(3)
 node1.shortest_path_to(node2).outgoing(:friends).depth(3)
 ```
@@ -268,6 +269,69 @@ user1.following_count
 The two nodes does not need to be of the same type. You can for example have
 a User following a Page or whatever makes sense in your app.
 
+Real-time Stream Processing
+---------------------------
+
+When working with graphs you often want to take the rich and interconnected
+web of data and actually do something with it. Stream processing is a powerful
+and flexible way to do that. It allows you to implement complex graph
+algorithms in a scalable way that is also easy to understand and work with.
+
+Stream processing in Related works by defining a data flow that new or
+existing data will be streamed through. A data flow is triggered when a
+Relationship is created, updated or deleted. You setup data flows for
+different relationship types, so for example when a "friend" relationship
+between two nodes is created or updated that relationship will be
+automatically sent through the data flows you have defined for the "friend"
+type.
+
+A data flow can consist of one or more steps and can branch out in a tree.
+You define the steps for a data flow using a simple Hash syntax.
+
+```ruby
+Related.data_flow :comment, Tokenize => { CountWords => { TotalSum => nil, MovingAverage => nil } }
+```
+
+In the example above a new comment will first sent to the Tokenize step that
+will split the comment text into words. The list of words will then
+automatically be sent to the CountWords step that will count the number of
+unique words. That number will then be sent to both the TotalSum step that
+adds the number to a global counter as well as the MovingAverage step that
+will calculate and store a moving average. The nil indicates the end of the
+data flow. You can define as many data flows for a relationship type as you
+want.
+
+A data flow step is simply a Ruby class that responds to the `process` message
+and takes a single argument that holds the input data. Any data yielded from
+the process method will be automatically sent to the next step in the data
+flow. The only limitation is that the data sent between steps is a Hash and
+only contains JSON serializable data. The first step in the data flow will
+receive the Relationship object that triggered it as a Ruby hash with all of
+its attributes.
+
+```ruby
+class Tokenize
+  def self.process(data)
+    data['text'].split(' ').each do |word|
+      yield({ :word => word })
+    end
+  end
+end
+```
+
+To actually run the data flows you have defined you need to start one or more
+data flow workers. Related uses Resque which supplies persistent queues and
+reliable workers. If you don't have Resque required in your application
+Related will simply run the work flow directly in process instead which can
+be useful when testing, but is not recommended for production.
+
+To start a stream processor:
+
+    $ QUEUE=related rake resque:work
+
+You can start as many stream processors as you may need to scale
+up.
+
 Development
 -----------
 

data/lib/related/data_flow.rb

@@ -0,0 +1,55 @@
+module Related
+  module DataFlow
+
+    def data_flow(name, steps)
+      @data_flows ||= {}
+      @data_flows[name.to_sym] ||= []
+      @data_flows[name.to_sym] << steps
+    end
+
+    def data_flows
+      @data_flows
+    end
+
+    def clear_data_flows
+      @data_flows = {}
+    end
+
+    def execute_data_flow(name_or_flow, data)
+      @data_flows ||= {}
+      if name_or_flow.is_a?(Hash)
+        enqueue_flow(name_or_flow, data)
+      else
+        flows = @data_flows[name_or_flow.to_sym] || []
+        flows.each do |flow|
+          enqueue_flow(flow, data)
+        end
+      end
+    end
+
+    class DataFlowJob
+      @queue = :related
+      def self.perform(flow, data)
+        flow.keys.each do |key|
+          step = key.constantize
+          step.perform(data) do |result|
+            if flow[key]
+              Related.execute_data_flow(flow[key], result)
+            end
+          end
+        end
+      end
+    end
+
+  protected
+
+    def enqueue_flow(flow, data)
+      if defined?(Resque)
+        Resque.enqueue(DataFlowJob, flow, data)
+      else
+        DataFlowJob.perform(JSON.parse(flow.to_json), JSON.parse(data.to_json))
+      end
+    end
+
+  end
+end

data/lib/related/entity.rb

@@ -102,7 +102,7 @@ module Related
 
     def self.find(*args)
       options = args.size > 1 && args.last.is_a?(Hash) ? args.pop : {}
-      args.size == 1 && args.first.is_a?(String) ?
+      args.size == 1 && !args.first.is_a?(Array) ?
         find_one(args.first, options) :
         find_many(args.flatten, options)
     end

data/lib/related/follower.rb

@@ -5,11 +5,8 @@ module Related
     end
 
     def unfollow!(other)
-      self.following.relationships.each do |rel|
-        if rel.end_node_id == other.id
-          rel.destroy
-        end
-      end
+      rel = self.following.relationships.find(other)
+      rel.destroy if rel
     end
 
     def followers

data/lib/related/node.rb

@@ -144,6 +144,18 @@ module Related
         end
       end
 
+      def find(node)
+        if @result_type == :nodes
+          if Related.redis.sismember(key, node.to_s)
+            Related::Node.find(node.to_s, @options)
+          end
+        else
+          if id = Related.redis.get(dir_key(node))
+            Related::Relationship.find(id, @options)
+          end
+        end
+      end
+
       def union(query)
         @result_type = :nodes
         @result = Related.redis.sunion(key, query.key)
@@ -193,6 +205,14 @@ module Related
         end
       end
 
+      def dir_key(node)
+        if @direction == :out
+          "#{@node.to_s}:#{@relationship_type}:#{node.to_s}"
+        elsif @direction == :in
+          "#{node.to_s}:#{@relationship_type}:#{@node.to_s}"
+        end
+      end
+
       def query
         self
       end

data/lib/related/relationship.rb

@@ -56,6 +56,10 @@ module Related
       end
     end
 
+    def dir_key
+      "#{self.start_node_id}:#{self.label}:#{self.end_node_id}"
+    end
+
     def self.weight_for(relationship, direction)
       if @weight
         relationship.instance_exec(direction, &@weight).to_i
@@ -71,7 +75,9 @@ module Related
         Related.redis.zadd(r_key(:in), self.class.weight_for(self, :in), self.id)
         Related.redis.sadd(n_key(:out), self.end_node_id)
         Related.redis.sadd(n_key(:in), self.start_node_id)
+        Related.redis.set(dir_key, self.id)
       end
+      Related.execute_data_flow(self.label, self)
       self
     end
 
@@ -80,9 +86,8 @@ module Related
         super
         Related.redis.zadd(r_key(:out), self.class.weight_for(self, :out), self.id)
         Related.redis.zadd(r_key(:in), self.class.weight_for(self, :in), self.id)
-        Related.redis.sadd(n_key(:out), self.end_node_id)
-        Related.redis.sadd(n_key(:in), self.start_node_id)
       end
+      Related.execute_data_flow(self.label, self)
       self
     end
 
@@ -92,8 +97,10 @@ module Related
         Related.redis.zrem(r_key(:in), self.id)
         Related.redis.srem(n_key(:out), self.end_node_id)
         Related.redis.srem(n_key(:in), self.start_node_id)
+        Related.redis.del(dir_key)
         super
       end
+      Related.execute_data_flow(self.label, self)
       self
     end
 

data/lib/related/version.rb

@@ -1,3 +1,3 @@
 module Related
-  Version = VERSION = '0.4.0'
+  Version = VERSION = '0.5.0'
 end

data/lib/related.rb

@@ -9,9 +9,11 @@ require 'related/entity'
 require 'related/node'
 require 'related/relationship'
 require 'related/root'
+require 'related/data_flow'
 
 module Related
   include Helpers
+  include DataFlow
   extend self
 
   # Accepts:

data/test/data_flow_test.rb

@@ -0,0 +1,55 @@
+require File.expand_path('test/test_helper')
+
+class ModelTest < ActiveModel::TestCase
+
+  class StepOne
+    def self.perform(data)
+      yield({ :text => data['text'] })
+    end
+  end
+
+  class StepTwo
+    def self.perform(data)
+      yield({ :id => 'StepTwo', :text => data['text'].downcase })
+    end
+  end
+
+  class StepThree
+    def self.perform(data)
+      yield({ :id => 'StepThree', :text => data['text'].upcase })
+    end
+  end
+
+  class LastStep
+    def self.perform(data)
+      Related.redis.set("DataFlowResult#{data['id']}", data['text'])
+    end
+  end
+
+  def setup
+    Related.redis.flushall
+  end
+
+  def teardown
+    Related.clear_data_flows
+  end
+
+  def test_defining_a_data_flow
+    Related.data_flow :like, StepOne => { StepTwo => nil }
+    assert_equal({ :like => [{ StepOne => { StepTwo => nil } }] }, Related.data_flows)
+  end
+
+  def test_executing_a_simple_data_flow
+    Related.data_flow :like, StepOne => { LastStep => nil }
+    Related::Relationship.create(:like, Related::Node.create, Related::Node.create, :text => 'Hello world!')
+    assert_equal 'Hello world!', Related.redis.get('DataFlowResult')
+  end
+
+  def test_executing_a_complicated_data_flow
+    Related.data_flow :like, StepOne => { StepTwo => { LastStep => nil }, StepThree => { LastStep => nil } }
+    Related::Relationship.create(:like, Related::Node.create, Related::Node.create, :text => 'Hello world!')
+    assert_equal 'hello world!', Related.redis.get('DataFlowResultStepTwo')
+    assert_equal 'HELLO WORLD!', Related.redis.get('DataFlowResultStepThree')
+  end
+
+end

data/test/model_test.rb

@@ -72,10 +72,10 @@ class ModelTest < ActiveModel::TestCase
     rel2 = Like.create(:like, node1, node3, :in_score => 2, :out_score => 2)
     rel3 = Like.create(:like, node2, node1, :in_score => 1, :out_score => 1)
     rel4 = Like.create(:like, node3, node1, :in_score => 2, :out_score => 2)
-    assert_equal [rel2,rel1], node1.outgoing(:like).relationships.options(:model => lambda { Like }).to_a
-    assert_equal [rel4,rel3], node1.incoming(:like).relationships.options(:model => lambda { Like }).to_a
-    assert_equal [rel1], node1.outgoing(:like).relationships.options(:model => lambda { Like }).per_page(2).page(rel2).to_a
-    assert_equal [rel3], node1.incoming(:like).relationships.options(:model => lambda { Like }).per_page(2).page(rel4).to_a
+    assert_equal [rel2,rel1], node1.outgoing(:like).relationships.options(:model => lambda {|a| Like }).to_a
+    assert_equal [rel4,rel3], node1.incoming(:like).relationships.options(:model => lambda {|a| Like }).to_a
+    assert_equal [rel1], node1.outgoing(:like).relationships.options(:model => lambda {|a| Like }).per_page(2).page(rel2).to_a
+    assert_equal [rel3], node1.incoming(:like).relationships.options(:model => lambda {|a| Like }).per_page(2).page(rel4).to_a
   end
 
 end

data/test/related_test.rb

@@ -45,6 +45,9 @@ class RelatedTest < Test::Unit::TestCase
     assert_raises Related::NotFound do
       Related::Node.find('foo')
     end
+    assert_raises Related::NotFound do
+      Related::Node.find(nil)
+    end
   end
 
   def test_can_update_node_with_new_attributes
@@ -295,4 +298,18 @@ class RelatedTest < Test::Unit::TestCase
     Related.root.save
   end
 
+  def test_find
+    node1 = Related::Node.create
+    node2 = Related::Node.create
+    rel = Related::Relationship.create(:friend, node1, node2)
+    assert_equal node2, node1.outgoing(:friend).find(node2)
+    assert_equal node1, node2.incoming(:friend).find(node1)
+    assert_equal nil, node1.outgoing(:friend).find(node1)
+    assert_equal nil, node2.incoming(:friend).find(node2)
+    assert_equal rel, node1.outgoing(:friend).relationships.find(node2)
+    assert_equal rel, node2.incoming(:friend).relationships.find(node1)
+    assert_equal nil, node1.outgoing(:friend).relationships.find(node1)
+    assert_equal nil, node2.incoming(:friend).relationships.find(node2)
+  end
+
 end

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: related
 version: !ruby/object:Gem::Version 
-  hash: 15
   prerelease: false
   segments: 
   - 0
-  - 4
+  - 5
   - 0
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Niklas Holmgren
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-11 00:00:00 +02:00
+date: 2011-11-02 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,7 +25,6 @@ dependencies:
     requirements: 
     - - ">"
       - !ruby/object:Gem::Version 
-        hash: 15
         segments: 
         - 2
         - 0
@@ -42,7 +40,6 @@ dependencies:
     requirements: 
     - - ">"
       - !ruby/object:Gem::Version 
-        hash: 63
         segments: 
         - 0
         - 8
@@ -58,7 +55,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -78,6 +74,7 @@ files:
 - Rakefile
 - LICENSE
 - CHANGELOG
+- lib/related/data_flow.rb
 - lib/related/entity.rb
 - lib/related/exceptions.rb
 - lib/related/follower.rb
@@ -88,6 +85,7 @@ files:
 - lib/related/version.rb
 - lib/related.rb
 - test/active_model_test.rb
+- test/data_flow_test.rb
 - test/follower_test.rb
 - test/model_test.rb
 - test/performance_test.rb
@@ -108,7 +106,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -117,7 +114,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"