resque_jobs_tree 0.3.4 → 0.4.0

data/lib/resque_jobs_tree/storage/node.rb

@@ -0,0 +1,80 @@
+module ResqueJobsTree::Storage::Node
+	include ResqueJobsTree::Storage
+
+  def store
+    raise 'Can\'t store a root node' if root?
+		redis.hset PARENTS_KEY, key, parent.key
+		unless redis.sadd parent.childs_key, key
+			raise ResqueJobsTree::JobNotUniq,
+				"Job #{parent.name} already has the child #{name} with resources: #{resources}"
+		end
+  end
+
+  def unstore
+		redis.srem parent.childs_key, key
+		redis.hdel PARENTS_KEY, key
+  end
+
+  def cleanup
+		unless definition.leaf?
+			stored_childs.each &:cleanup
+			redis.del childs_key
+		end
+    redis.hdel PARENTS_KEY, key
+    tree.unstore if root?
+  end
+
+  def childs_key
+    "#{key}:childs"
+  end
+
+  def key
+    "ResqueJobsTree:Node:#{serialize}"
+  end
+
+  def only_stored_child?
+    (redis.smembers(parent.childs_key) - [key]).empty?
+  end
+
+  def stored_childs
+    redis.smembers(childs_key).map do |_key|
+      node_name, _resources = node_info_from_key _key
+      definition.find(node_name).spawn _resources
+    end
+  end
+
+  def parent
+    @parent ||= definition.parent.spawn node_info_from_key(parent_key).last
+  end
+  
+  def lock_key
+    "#{key}:lock"
+  end
+
+	private
+
+  def lock
+    _key = parent.lock_key
+    while !redis.setnx(_key, 'locked')
+      sleep 0.05 # 50 ms
+    end
+    yield
+  ensure
+    redis.del _key
+  end
+
+  def parent_key
+    redis.hget PARENTS_KEY, key
+  end
+
+  def node_info_from_key _key
+    tree_name, node_name, *resources_arguments = JSON.load(_key.gsub /ResqueJobsTree:Node:/, '')
+    _resources = ResqueJobsTree::ResourcesSerializer.instancize(resources_arguments)
+		[node_name, _resources]
+  end
+
+	def main_arguments
+		[definition.tree.name, name]
+	end
+  
+end

data/lib/resque_jobs_tree/storage/tree.rb

@@ -0,0 +1,26 @@
+module ResqueJobsTree::Storage::Tree
+	include ResqueJobsTree::Storage
+
+  def store
+    redis.sadd LAUNCHED_TREES, key
+  end
+
+  def unstore
+    redis.srem LAUNCHED_TREES, key
+  end
+
+  def key
+    "ResqueJobsTree:Tree:#{serialize}"
+  end
+
+  def uniq?
+    !redis.sismember LAUNCHED_TREES, key
+  end
+  
+  private
+
+	def main_arguments
+		[name]
+	end
+  
+end

data/lib/resque_jobs_tree/tree.rb

@@ -1,55 +1,60 @@
 class ResqueJobsTree::Tree
 
-  attr_accessor :name
-  attr_reader :jobs
+  include ResqueJobsTree::Storage::Tree
 
-  def initialize name
-    @name = name.to_s
-    @jobs = []
-  end
+  attr_reader :definition, :resources, :leaves
 
-  def root name=nil, &block
-    @root ||= ResqueJobsTree::Node.new(name, self).tap do |root|
-      root.instance_eval &block
-    end
+  def initialize definition, resources
+    @definition = definition
+    @resources = resources
+    @leaves = []
   end
 
-  def on_failure &block
-    @on_failure ||= block
+  def name
+    @definition.name
   end
 
-  def launch *resources
-    ResqueJobsTree::Storage.track_launch self, resources do
-      @root.launch resources
+  def launch
+    if uniq?
+      before_perform
+      store
+      root.launch
       enqueue_leaves_jobs
     end
   end
 
-  def enqueue job_name, *resources
-    @jobs << ([name, job_name] + ResqueJobsTree::ResourcesSerializer.to_args(resources))
+  %w(before_perform after_perform on_failure).each do |callback|
+    class_eval %Q{def #{callback} ; run_callback :#{callback} ; end}
   end
 
-  def find_node_by_name name
-    root.find_node_by_name name.to_s
+  def root
+    @root ||= ResqueJobsTree::Node.new(definition.root, resources, nil, self)
   end
 
-  def validate!
-    raise(ResqueJobsTree::TreeInvalid, "`#{name}` has no root node") unless @root
-    root.validate!
+  def register_a_leaf node
+    @leaves << node
   end
 
-  def nodes
-    [root, root.nodes].flatten
+  def inspect
+    "<ResqueJobsTree::Tree @name=#{name} @resources=#{resources} >"
   end
 
-  def inspect
-    "<ResqueJobsTree::Tree @name=#{name}>"
+  def finish
+    after_perform
+    unstore
   end
 
   private
 
   def enqueue_leaves_jobs
-    @jobs.each{ |job_args| Resque.enqueue_to name, ResqueJobsTree::Job, *job_args }
+    @leaves.each do |leaf|
+      leaf.enqueue unless leaf.definition.options[:async]
+    end
+  end
+
+  def run_callback callback
+    callback = definition.send(callback)
+    callback.call(*resources) if callback.kind_of? Proc
   end
 
 end

data/lib/resque_jobs_tree/version.rb

@@ -1,3 +1,3 @@
 module ResqueJobsTree
-  VERSION = "0.3.4"
+  VERSION = "0.4.0"
 end

data/test/definitions_test.rb

@@ -0,0 +1,99 @@
+require 'test_helper'
+
+class DefinitionsTest < MiniTest::Unit::TestCase
+
+  def setup
+    @tree = ResqueJobsTree::Definitions::Tree.new :tree1
+    @root = ResqueJobsTree::Definitions::Node.new :node1, @tree
+    @tree.root = @root
+    @leaf = ResqueJobsTree::Definitions::Node.new :node2, @tree, @root
+    @root.node_childs = [@leaf]
+  end
+
+  def test_perform
+    variable = 1
+    @root.perform do |n|
+      variable = n
+    end
+    @root.perform.call 2
+    assert_equal variable, 2
+  end
+
+  def test_childs
+    variable = 1
+    @leaf.childs do |n|
+      variable = n
+    end
+    @leaf.childs.call 2
+    assert_equal variable, 2
+  end
+
+  def test_node
+    node3 = @root.node :node3
+    assert_equal @root.find('node3').object_id, node3.object_id
+  end
+
+  def test_leaf
+    assert @leaf.leaf?
+    assert !@root.leaf?
+  end
+
+  def test_root
+    assert @root.root?
+    assert !@leaf.root?
+  end
+
+  def test_siblings
+    node3 = ResqueJobsTree::Node.new :node3, @tree, @root
+    assert @leaf.siblings, [node3]
+  end
+
+  def test_childs_validation
+    assert_raises ResqueJobsTree::NodeDefinitionInvalid do
+      ResqueJobsTree::Factory.create :tree1 do
+        root :job1 do
+          perform {}
+          childs {}
+        end
+      end
+    end
+    ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform {}
+      end
+    end
+  end
+
+  def test_perform_validation
+    assert_raises ResqueJobsTree::NodeDefinitionInvalid do
+      ResqueJobsTree::Factory.create :tree1 do
+        root :job1 do
+        end
+      end
+    end
+  end
+
+  def test_perform_validation
+    assert_raises ResqueJobsTree::NodeDefinitionInvalid do
+      ResqueJobsTree::Factory.create :tree1 do
+        root :job1 do
+          perform {}
+          childs {}
+          node :job2 do
+            perform {}
+          end
+          node :job2 do
+            perform {}
+          end
+        end
+      end
+    end
+  end
+
+  def test_options
+    options = { async: true }
+    @leaf.options = options
+    assert_equal options, @tree.find(:node2).options
+  end
+
+end

data/test/factory_test.rb

@@ -7,12 +7,11 @@ class FactoryTest < MiniTest::Unit::TestCase
   end
 
   def test_tree_creation
-    assert_equal ResqueJobsTree::Factory.trees.first.object_id, @tree.object_id
+    assert_equal ResqueJobsTree::Factory.trees.values.first.name, @tree_definition.name
   end
 
-  def test_find_by_name
-    assert_equal ResqueJobsTree::Factory.find_tree_by_name(@tree.name).object_id,
-      @tree.object_id
+  def test_find
+    assert_equal ResqueJobsTree::Factory.find(@tree_definition.name).name, @tree_definition.name
   end
 
 end

data/test/job_test.rb

@@ -1,15 +1,13 @@
 require 'test_helper'
 
 class JobTest < MiniTest::Unit::TestCase
-	
-	def setup
-		create_tree
-		@args = [@tree.name, @tree.find_node_by_name('job1').name, 1, 2, 3]
-	end
 
-	def test_tree_node_and_resources
-		result = [@tree.find_node_by_name('job1'), [1, 2, 3]]
-		assert_equal result, ResqueJobsTree::Job.send(:node_and_resources, @args)
+	def test_node
+		create_tree
+		args = [@tree_definition.name, @tree_definition.find('job1').name, 1, 2, 3]
+		assert_equal @tree_definition.find(:job1), ResqueJobsTree::Job.send(:node, *args).definition
+		assert_equal ResqueJobsTree.find(:tree1), ResqueJobsTree::Job.send(:node, *args).definition.tree
+		assert_equal [1, 2, 3], ResqueJobsTree::Job.send(:node, *args).resources
 	end
 
 end

data/test/process_test.rb

@@ -0,0 +1,214 @@
+require 'test_helper'
+
+class ProcessTest < MiniTest::Unit::TestCase
+
+  def test_launch
+    create_tree
+    resources = [1, 2, 3]
+    @tree_definition.spawn(resources).launch
+    history = ['tree1 job2']*3+['tree1 job1']
+    assert_equal history, redis.lrange('history', 0, -1)
+  end
+
+  def test_launch_with_no_resources
+    create_tree
+    @tree_definition.spawn([]).launch
+  end
+
+  def test_leaf_failure
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform {}
+        childs { [:job2] }
+        node :job2 do
+          perform { raise ExpectedException, 'an expected exception'}
+        end
+      end
+    end
+    assert_raises ExpectedException do
+      tree_definition.spawn([1, 2, 3]).launch
+    end
+    assert_redis_empty
+  end
+
+  def test_launch_async
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform { raise 'should not arrive here' }
+        childs { [:job2] }
+        node :job2, async: true do
+          perform {}
+        end
+      end
+    end
+    tree = tree_definition.spawn [1, 2, 3]
+    tree.launch
+  end
+
+  def test_launch_continue_on_failure
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform do
+          Resque.redis.rpush 'history', 'tree1 job1'
+          raise ExpectedException, 'an expected failure'
+        end
+        childs { [:job2] }
+        node :job2, continue_on_failure: true do
+          perform do
+            Resque.redis.rpush 'history', 'tree1 job2'
+            raise 'an expected failure'
+          end
+        end
+      end
+    end
+    assert_raises ExpectedException do
+      tree = tree_definition.spawn [1, 2, 3]
+      tree.launch
+    end
+    assert_equal ['tree1 job2','tree1 job1'], redis.lrange('history', 0, -1)
+    redis.del 'history'
+    assert_redis_empty
+  end
+
+  def test_root_failure
+    Resque.inline = false
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform do
+          Resque.redis.rpush 'history', 'tree1 job1'
+          raise ExpectedException, 'an expected exception'
+        end
+        childs { [:job2] }
+        node :job2 do
+          perform do
+            Resque.redis.rpush 'history', 'tree1 job2'
+          end
+        end
+      end
+    end
+    tree = tree_definition.spawn [1,2,3]
+    tree.launch
+    run_resque_workers tree_definition.name
+    assert_raises ExpectedException do
+      run_resque_workers tree_definition.name
+    end
+    assert_equal ['tree1 job2','tree1 job1'], redis.lrange('history', 0, -1)
+    redis.del 'history'
+    assert_redis_empty
+  end
+
+	def test_store_already_stored
+		wrong_tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform {}
+        childs do |resources|
+					[ [:job2], [:job2] ]
+        end
+        node :job2 do
+          perform {}
+        end
+      end
+    end
+		assert_raises ResqueJobsTree::JobNotUniq do
+			wrong_tree_definition.spawn([1]).launch
+		end
+	end
+
+  def test_on_failure
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform { raise }
+      end
+      on_failure do
+        raise ExpectedException, 'called from on_failure block'
+      end
+    end
+    assert_raises ExpectedException do
+      tree_definition.spawn([1]).launch
+    end
+  end
+
+  def test_tree_with_resource
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform do |resource, number|
+          raise 'unknown resource' unless resource.kind_of?(Model)
+          raise 'unknown resource' unless number.kind_of?(Integer)
+        end
+      end
+    end
+    ResqueJobsTree.launch tree_definition.name, Model.new(1), 1
+  end
+
+  def test_nested_tree
+    Resque.inline = false
+    create_nested_tree
+    @tree_definition.spawn([1,2,3]).launch
+    assert_raises RuntimeError do # job4 error
+      run_resque_workers @tree_definition.name
+    end
+    Resque.enqueue_to 'tree1', ResqueJobsTree::Job, 'tree1', 'job3'
+    run_resque_workers @tree_definition.name # job3 error
+    assert_raises RuntimeError do # job2 error
+      run_resque_workers @tree_definition.name
+    end
+    assert_raises ExpectedException do # job1 error
+      run_resque_workers @tree_definition.name
+    end
+  end
+
+  def test_async_tree
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform { raise 'should not arrive here' }
+        childs { [ [:job2], [:job3] ] }
+        node :job2, async: true do
+          perform {}
+        end
+        node :job3 do
+          perform {}
+        end
+      end
+    end
+    ResqueJobsTree.launch tree_definition.name
+    assert_equal ["ResqueJobsTree:Node:[\"tree1\",\"job2\"]"],
+      Resque.redis.smembers("ResqueJobsTree:Node:[\"tree1\",\"job1\"]:childs")
+    parents_hash = { 'ResqueJobsTree:Node:["tree1","job2"]'=>'ResqueJobsTree:Node:["tree1","job1"]' }
+    assert_equal parents_hash, Resque.redis.hgetall(ResqueJobsTree::Storage::PARENTS_KEY)
+  end
+
+  def test_async_tree_with_fail
+    Resque.inline = false
+    tree_definition = ResqueJobsTree::Factory.create :tree1 do
+      root :job1 do
+        perform { raise 'should not arrive here' }
+        childs { [ [:job2], [:job3] ] }
+        node :job2, async: true do
+          perform {}
+        end
+        node :job3, continue_on_failure: true do
+          perform { raise ExpectedException, 'an expected failure' }
+        end
+      end
+    end
+    tree_definition.spawn([]).launch
+    assert_raises ExpectedException do # job3 exception
+      run_resque_workers tree_definition.name
+    end
+    assert_equal ["ResqueJobsTree:Node:[\"tree1\",\"job2\"]"],
+      Resque.redis.smembers("ResqueJobsTree:Node:[\"tree1\",\"job1\"]:childs")
+  end
+
+  private
+
+  def assert_redis_empty
+    assert_equal [], Resque.keys
+  end
+
+  # Inline mode is messy when dealing with on failure callbacks.
+  def run_resque_workers queue_name
+    Resque::Job.reserve(queue_name).perform
+    redis.srem 'queues', queue_name
+  end
+
+end