gush 2.0.1 → 2.1.0

data/lib/gush/client.rb

@@ -1,9 +1,22 @@
-require 'connection_pool'
+require 'redis'
+require 'concurrent-ruby'
 
 module Gush
   class Client
     attr_reader :configuration
 
+    @@redis_connection = Concurrent::ThreadLocalVar.new(nil)
+
+    def self.redis_connection(config)
+      cached = (@@redis_connection.value ||= { url: config.redis_url, connection: nil })
+      return cached[:connection] if !cached[:connection].nil? && config.redis_url == cached[:url]
+
+      Redis.new(url: config.redis_url).tap do |instance|
+        RedisClassy.redis = instance
+        @@redis_connection.value = { url: config.redis_url, connection: instance }
+      end
+    end
+
     def initialize(config = Gush.configuration)
       @configuration = config
     end
@@ -47,9 +60,7 @@ module Gush
 
       loop do
         job_id = SecureRandom.uuid
-        available = connection_pool.with do |redis|
-          !redis.hexists("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
-        end
+        available = !redis.hexists("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
 
         break if available
       end
@@ -61,9 +72,7 @@ module Gush
       id = nil
       loop do
         id = SecureRandom.uuid
-        available = connection_pool.with do |redis|
-          !redis.exists("gush.workflow.#{id}")
-        end
+        available = !redis.exists?("gush.workflow.#{id}")
 
         break if available
       end
@@ -72,37 +81,31 @@ module Gush
     end
 
     def all_workflows
-      connection_pool.with do |redis|
-        redis.scan_each(match: "gush.workflows.*").map do |key|
-          id = key.sub("gush.workflows.", "")
-          find_workflow(id)
-        end
+      redis.scan_each(match: "gush.workflows.*").map do |key|
+        id = key.sub("gush.workflows.", "")
+        find_workflow(id)
       end
     end
 
     def find_workflow(id)
-      connection_pool.with do |redis|
-        data = redis.get("gush.workflows.#{id}")
+      data = redis.get("gush.workflows.#{id}")
 
-        unless data.nil?
-          hash = Gush::JSON.decode(data, symbolize_keys: true)
-          keys = redis.scan_each(match: "gush.jobs.#{id}.*")
+      unless data.nil?
+        hash = Gush::JSON.decode(data, symbolize_keys: true)
+        keys = redis.scan_each(match: "gush.jobs.#{id}.*")
 
-          nodes = keys.each_with_object([]) do |key, array|
-            array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
-          end
-
-          workflow_from_hash(hash, nodes)
-        else
-          raise WorkflowNotFound.new("Workflow with given id doesn't exist")
+        nodes = keys.each_with_object([]) do |key, array|
+          array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
         end
+
+        workflow_from_hash(hash, nodes)
+      else
+        raise WorkflowNotFound.new("Workflow with given id doesn't exist")
       end
     end
 
     def persist_workflow(workflow)
-      connection_pool.with do |redis|
-        redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
-      end
+      redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
 
       workflow.jobs.each {|job| persist_job(workflow.id, job) }
       workflow.mark_as_persisted
@@ -111,9 +114,7 @@ module Gush
     end
 
     def persist_job(workflow_id, job)
-      connection_pool.with do |redis|
-        redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
-      end
+      redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
     end
 
     def find_job(workflow_id, job_name)
@@ -132,31 +133,23 @@ module Gush
     end
 
     def destroy_workflow(workflow)
-      connection_pool.with do |redis|
-        redis.del("gush.workflows.#{workflow.id}")
-      end
+      redis.del("gush.workflows.#{workflow.id}")
       workflow.jobs.each {|job| destroy_job(workflow.id, job) }
     end
 
     def destroy_job(workflow_id, job)
-      connection_pool.with do |redis|
-        redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
-      end
+      redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
     end
 
     def expire_workflow(workflow, ttl=nil)
       ttl = ttl || configuration.ttl
-      connection_pool.with do |redis|
-        redis.expire("gush.workflows.#{workflow.id}", ttl)
-      end
+      redis.expire("gush.workflows.#{workflow.id}", ttl)
       workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
     end
 
     def expire_job(workflow_id, job, ttl=nil)
       ttl = ttl || configuration.ttl
-      connection_pool.with do |redis|
-        redis.expire("gush.jobs.#{workflow_id}.#{job.name}", ttl)
-      end
+      redis.expire("gush.jobs.#{workflow_id}.#{job.klass}", ttl)
     end
 
     def enqueue_job(workflow_id, job)
@@ -172,16 +165,11 @@ module Gush
     def find_job_by_klass_and_id(workflow_id, job_name)
       job_klass, job_id = job_name.split('|')
 
-      connection_pool.with do |redis|
-        redis.hget("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
-      end
+      redis.hget("gush.jobs.#{workflow_id}.#{job_klass}", job_id)
     end
 
     def find_job_by_klass(workflow_id, job_name)
-      new_cursor, result = connection_pool.with do |redis|
-        redis.hscan("gush.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
-      end
-
+      new_cursor, result = redis.hscan("gush.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
       return nil if result.empty?
 
       job_id, job = *result[0]
@@ -202,14 +190,8 @@ module Gush
       flow
     end
 
-    def build_redis
-      Redis.new(url: configuration.redis_url).tap do |instance|
-        RedisClassy.redis = instance
-      end
-    end
-
-    def connection_pool
-      @connection_pool ||= ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
+    def redis
+      self.class.redis_connection(configuration)
     end
   end
 end

data/lib/gush/configuration.rb

@@ -1,17 +1,19 @@
 module Gush
   class Configuration
-    attr_accessor :concurrency, :namespace, :redis_url, :ttl
+    attr_accessor :concurrency, :namespace, :redis_url, :ttl, :locking_duration, :polling_interval
 
     def self.from_json(json)
       new(Gush::JSON.decode(json, symbolize_keys: true))
     end
 
     def initialize(hash = {})
-      self.concurrency = hash.fetch(:concurrency, 5)
-      self.namespace   = hash.fetch(:namespace, 'gush')
-      self.redis_url   = hash.fetch(:redis_url, 'redis://localhost:6379')
-      self.gushfile    = hash.fetch(:gushfile, 'Gushfile')
-      self.ttl         = hash.fetch(:ttl, -1)
+      self.concurrency      = hash.fetch(:concurrency, 5)
+      self.namespace        = hash.fetch(:namespace, 'gush')
+      self.redis_url        = hash.fetch(:redis_url, 'redis://localhost:6379')
+      self.gushfile         = hash.fetch(:gushfile, 'Gushfile')
+      self.ttl              = hash.fetch(:ttl, -1)
+      self.locking_duration = hash.fetch(:locking_duration, 2) # how long you want to wait for the lock to be released, in seconds
+      self.polling_interval = hash.fetch(:polling_internal, 0.3) # how long the polling interval should be, in seconds
     end
 
     def gushfile=(path)
@@ -24,10 +26,12 @@ module Gush
 
     def to_hash
       {
-        concurrency: concurrency,
-        namespace:   namespace,
-        redis_url:   redis_url,
-        ttl:         ttl
+        concurrency:      concurrency,
+        namespace:        namespace,
+        redis_url:        redis_url,
+        ttl:              ttl,
+        locking_duration: locking_duration,
+        polling_interval: polling_interval
       }
     end
 

data/lib/gush/graph.rb

@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+
 module Gush
   class Graph
-    attr_reader :workflow, :filename, :path, :start, :end_node
+    attr_reader :workflow, :filename, :path, :start_node, :end_node
 
     def initialize(workflow, options = {})
       @workflow = workflow
@@ -9,19 +13,26 @@ module Gush
     end
 
     def viz
-      GraphViz.new(:G, graph_options) do |graph|
-        set_node_options!(graph)
-        set_edge_options!(graph)
+      @graph = Graphviz::Graph.new(**graph_options)
+      @start_node = add_node('start', shape: 'diamond', fillcolor: '#CFF09E')
+      @end_node = add_node('end', shape: 'diamond', fillcolor: '#F56991')
 
-        @start = graph.start(shape: 'diamond', fillcolor: '#CFF09E')
-        @end_node = graph.end(shape: 'diamond', fillcolor: '#F56991')
-
-        workflow.jobs.each do |job|
-          add_job(graph, job)
-        end
+      # First, create nodes for all jobs
+      @job_name_to_node_map = {}
+      workflow.jobs.each do |job|
+        add_job_node(job)
+      end
 
-        graph.output(png: path)
+      # Next, link up the jobs with edges
+      workflow.jobs.each do |job|
+        link_job_edges(job)
       end
+
+      format = 'png'
+      file_format = path.split('.')[-1]
+      format = file_format if file_format.length == 3
+
+      Graphviz::output(@graph, path: path, format: format)
     end
 
     def path
@@ -29,43 +40,43 @@ module Gush
     end
 
     private
-    def add_job(graph, job)
-      name = job.class.to_s
-      graph.add_nodes(job.name, label: name)
+
+    def add_node(name, **specific_options)
+      @graph.add_node(name, **node_options.merge(specific_options))
+    end
+
+    def add_job_node(job)
+      @job_name_to_node_map[job.name] = add_node(job.name, label: node_label_for_job(job))
+    end
+
+    def link_job_edges(job)
+      job_node = @job_name_to_node_map[job.name]
 
       if job.incoming.empty?
-        graph.add_edges(start, job.name)
+        @start_node.connect(job_node, **edge_options)
       end
 
       if job.outgoing.empty?
-        graph.add_edges(job.name, end_node)
+        job_node.connect(@end_node, **edge_options)
       else
         job.outgoing.each do |id|
           outgoing_job = workflow.find_job(id)
-          graph.add_edges(job.name, outgoing_job.name)
+          job_node.connect(@job_name_to_node_map[outgoing_job.name], **edge_options)
         end
       end
     end
 
-    def set_node_options!(graph)
-      node_options.each do |key, value|
-        graph.node[key] = value
-      end
-    end
-
-    def set_edge_options!(graph)
-      edge_options.each do |key, value|
-        graph.edge[key] = value
-      end
+    def node_label_for_job(job)
+      job.class.to_s
     end
 
     def graph_options
       {
-        type: :digraph,
-        dpi: 200,
-        compound: true,
-        rankdir: "LR",
-        center: true
+          dpi: 200,
+          compound: true,
+          rankdir: "LR",
+          center: true,
+          format: 'png'
       }
     end
 

data/lib/gush/worker.rb

@@ -6,6 +6,12 @@ module Gush
     def perform(workflow_id, job_id)
       setup_job(workflow_id, job_id)
 
+      if job.succeeded?
+        # Try to enqueue outgoing jobs again because the last job has redis mutex lock error
+        enqueue_outgoing_jobs
+        return
+      end
+
       job.payloads = incoming_payloads
 
       error = nil
@@ -24,12 +30,16 @@ module Gush
 
     private
 
-    attr_reader :client, :workflow_id, :job
+    attr_reader :client, :workflow_id, :job, :configuration
 
     def client
       @client ||= Gush::Client.new(Gush.configuration)
     end
 
+    def configuration
+      @configuration ||= client.configuration
+    end
+
     def setup_job(workflow_id, job_id)
       @workflow_id = workflow_id
       @job ||= client.find_job(workflow_id, job_id)
@@ -67,7 +77,11 @@ module Gush
 
     def enqueue_outgoing_jobs
       job.outgoing.each do |job_name|
-        RedisMutex.with_lock("gush_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}", sleep: 0.3, block: 2) do
+        RedisMutex.with_lock(
+          "gush_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}",
+          sleep: configuration.polling_interval,
+          block: configuration.locking_duration
+        ) do
           out = client.find_job(workflow_id, job_name)
 
           if out.ready_to_start?
@@ -75,6 +89,8 @@ module Gush
           end
         end
       end
+    rescue RedisMutex::LockError
+      Worker.set(wait: 2.seconds).perform_later(workflow_id, job.name)
     end
   end
 end

data/spec/features/integration_spec.rb

@@ -152,17 +152,15 @@ describe "Workflows" do
     flow = PayloadWorkflow.create
     flow.start!
 
-    perform_one
-    expect(flow.reload.find_job(flow.jobs[0].name).output_payload).to eq('first')
+    3.times { perform_one }
 
-    perform_one
-    expect(flow.reload.find_job(flow.jobs[1].name).output_payload).to eq('second')
+    outputs = flow.reload.jobs.select { |j| j.klass == 'RepetitiveJob' }.map { |j| j.output_payload }
+    expect(outputs).to match_array(['first', 'second', 'third'])
 
     perform_one
-    expect(flow.reload.find_job(flow.jobs[2].name).output_payload).to eq('third')
 
-    perform_one
-    expect(flow.reload.find_job(flow.jobs[3].name).output_payload).to eq(%w(first second third))
+    summary_job = flow.reload.jobs.find { |j| j.klass == 'SummaryJob' }
+    expect(summary_job.output_payload).to eq(%w(first second third))
   end
 
   it "does not execute `configure` on each job for huge workflows" do

data/spec/gush/client_spec.rb

@@ -95,12 +95,18 @@ describe Gush::Client do
   end
 
   describe "#expire_workflow" do
+    let(:ttl) { 2000 }
+
     it "sets TTL for all Redis keys related to the workflow" do
       workflow = TestWorkflow.create
 
-      client.expire_workflow(workflow, -1)
+      client.expire_workflow(workflow, ttl)
+
+      expect(redis.ttl("gush.workflows.#{workflow.id}")).to eq(ttl)
 
-      # => TODO - I believe fakeredis does not handle TTL the same.   
+      workflow.jobs.each do |job|
+        expect(redis.ttl("gush.jobs.#{workflow.id}.#{job.klass}")).to eq(ttl)
+      end
     end
   end
 

data/spec/gush/configuration_spec.rb

@@ -8,6 +8,8 @@ describe Gush::Configuration do
     expect(subject.concurrency).to eq(5)
     expect(subject.namespace).to eq('gush')
     expect(subject.gushfile).to eq(GUSHFILE.realpath)
+    expect(subject.locking_duration).to eq(2)
+    expect(subject.polling_interval).to eq(0.3)
   end
 
   describe "#configure" do
@@ -15,10 +17,14 @@ describe Gush::Configuration do
       Gush.configure do |config|
         config.redis_url = "redis://localhost"
         config.concurrency = 25
+        config.locking_duration = 5
+        config.polling_interval = 0.5
       end
 
       expect(Gush.configuration.redis_url).to eq("redis://localhost")
       expect(Gush.configuration.concurrency).to eq(25)
+      expect(Gush.configuration.locking_duration).to eq(5)
+      expect(Gush.configuration.polling_interval).to eq(0.5)
     end
   end
 end

data/spec/gush/graph_spec.rb

@@ -10,26 +10,43 @@ describe Gush::Graph do
       edge = double("edge", :[]= => true)
       graph = double("graph", node: node, edge: edge)
       path = Pathname.new(Dir.tmpdir).join(filename)
-      expect(graph).to receive(:start).with(shape: 'diamond', fillcolor: '#CFF09E')
-      expect(graph).to receive(:end).with(shape: 'diamond', fillcolor: '#F56991')
-
-      expect(graph).to receive(:output).with(png: path.to_s)
-
-      expect(graph).to receive(:add_nodes).with(/Prepare/, label: "Prepare")
-      expect(graph).to receive(:add_nodes).with(/FetchFirstJob/, label: "FetchFirstJob")
-      expect(graph).to receive(:add_nodes).with(/FetchSecondJob/, label: "FetchSecondJob")
-      expect(graph).to receive(:add_nodes).with(/NormalizeJob/, label: "NormalizeJob")
-      expect(graph).to receive(:add_nodes).with(/PersistFirstJob/, label: "PersistFirstJob")
-
-      expect(graph).to receive(:add_edges).with(nil, /Prepare/)
-      expect(graph).to receive(:add_edges).with(/Prepare/, /FetchFirstJob/)
-      expect(graph).to receive(:add_edges).with(/Prepare/, /FetchSecondJob/)
-      expect(graph).to receive(:add_edges).with(/FetchFirstJob/, /PersistFirstJob/)
-      expect(graph).to receive(:add_edges).with(/FetchSecondJob/, /NormalizeJob/)
-      expect(graph).to receive(:add_edges).with(/PersistFirstJob/, /NormalizeJob/)
-      expect(graph).to receive(:add_edges).with(/NormalizeJob/, nil)
-
-      expect(GraphViz).to receive(:new).and_yield(graph)
+
+      expect(Graphviz::Graph).to receive(:new).and_return(graph)
+
+      node_start = double('start')
+      node_end = double('end')
+      node_prepare = double('Prepare')
+      node_fetch_first_job = double('FetchFirstJob')
+      node_fetch_second_job = double('FetchSecondJob')
+      node_normalize_job = double('NormalizeJob')
+      node_persist_first_job = double('PersistFirstJob')
+
+      expect(graph).to receive(:add_node).with('start', {shape: 'diamond', fillcolor: '#CFF09E', color: "#555555", style: 'filled'}).and_return(node_start)
+      expect(graph).to receive(:add_node).with('end', {shape: 'diamond', fillcolor: '#F56991', color: "#555555", style: 'filled'}).and_return(node_end)
+
+      standard_options = {:color=>"#555555", :fillcolor=>"white", :label=>"Prepare", :shape=>"ellipse", :style=>"filled"}
+
+      expect(graph).to receive(:add_node).with(/Prepare/, standard_options.merge(label: "Prepare")).and_return(node_prepare)
+      expect(graph).to receive(:add_node).with(/FetchFirstJob/, standard_options.merge(label: "FetchFirstJob")).and_return(node_fetch_first_job)
+      expect(graph).to receive(:add_node).with(/FetchSecondJob/, standard_options.merge(label: "FetchSecondJob")).and_return(node_fetch_second_job)
+      expect(graph).to receive(:add_node).with(/NormalizeJob/, standard_options.merge(label: "NormalizeJob")).and_return(node_normalize_job)
+      expect(graph).to receive(:add_node).with(/PersistFirstJob/, standard_options.merge(label: "PersistFirstJob")).and_return(node_persist_first_job)
+
+      edge_options = {
+          dir: "forward",
+          penwidth: 1,
+          color: "#555555"
+      }
+
+      expect(node_start).to receive(:connect).with(node_prepare, **edge_options)
+      expect(node_prepare).to receive(:connect).with(node_fetch_first_job, **edge_options)
+      expect(node_prepare).to receive(:connect).with(node_fetch_second_job, **edge_options)
+      expect(node_fetch_first_job).to receive(:connect).with(node_persist_first_job, **edge_options)
+      expect(node_fetch_second_job).to receive(:connect).with(node_normalize_job, **edge_options)
+      expect(node_persist_first_job).to receive(:connect).with(node_normalize_job, **edge_options)
+      expect(node_normalize_job).to receive(:connect).with(node_end, **edge_options)
+
+      expect(graph).to receive(:dump_graph).and_return(nil)
 
       subject.viz
     end

data/spec/gush/worker_spec.rb

@@ -4,6 +4,8 @@ describe Gush::Worker do
   subject { described_class.new }
 
   let!(:workflow)   { TestWorkflow.create }
+  let(:locking_duration) { 5 }
+  let(:polling_interval) { 0.5 }
   let!(:job)        { client.find_job(workflow.id, "Prepare")  }
   let(:config)      { Gush.configuration.to_json  }
   let!(:client)     { Gush::Client.new }
@@ -39,6 +41,18 @@ describe Gush::Worker do
       end
     end
 
+    context 'when job failed to enqueue outgoing jobs' do
+      it 'enqeues another job to handling enqueue_outgoing_jobs' do
+        allow(RedisMutex).to receive(:with_lock).and_raise(RedisMutex::LockError)
+        subject.perform(workflow.id, 'Prepare')
+        expect(Gush::Worker).to have_no_jobs(workflow.id, jobs_with_id(["FetchFirstJob", "FetchSecondJob"]))
+
+        allow(RedisMutex).to receive(:with_lock).and_call_original
+        perform_one
+        expect(Gush::Worker).to have_jobs(workflow.id, jobs_with_id(["FetchFirstJob", "FetchSecondJob"]))
+      end
+    end
+
     it "calls job.perform method" do
       SPY = double()
       expect(SPY).to receive(:some_method)
@@ -59,5 +73,11 @@ describe Gush::Worker do
 
       subject.perform(workflow.id, 'OkayJob')
     end
+
+    it 'calls RedisMutex.with_lock with customizable locking_duration and polling_interval' do
+      expect(RedisMutex).to receive(:with_lock)
+        .with(anything, block: 5, sleep: 0.5).twice
+      subject.perform(workflow.id, 'Prepare')
+    end
   end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,4 @@
 require 'gush'
-require 'fakeredis'
 require 'json'
 require 'pry'
 
@@ -35,12 +34,8 @@ class ParameterTestWorkflow < Gush::Workflow
   end
 end
 
-class Redis
-  def publish(*)
-  end
-end
 
-REDIS_URL = "redis://localhost:6379/12"
+REDIS_URL = ENV["REDIS_URL"] || "redis://localhost:6379/12"
 
 module GushHelpers
   def redis
@@ -78,6 +73,19 @@ RSpec::Matchers.define :have_jobs do |flow, jobs|
   end
 end
 
+RSpec::Matchers.define :have_no_jobs do |flow, jobs|
+  match do |actual|
+    expected = jobs.map do |job|
+      hash_including(args: include(flow, job))
+    end
+    expect(ActiveJob::Base.queue_adapter.enqueued_jobs).not_to match_array(expected)
+  end
+
+  failure_message do |actual|
+    "expected queue to have no #{jobs}, but instead has: #{ActiveJob::Base.queue_adapter.enqueued_jobs.map{ |j| j[:args][1]}}"
+  end
+end
+
 RSpec.configure do |config|
   config.include ActiveJob::TestHelper
   config.include GushHelpers
@@ -91,12 +99,13 @@ RSpec.configure do |config|
     clear_performed_jobs
 
     Gush.configure do |config|
-      config.redis_url = REDIS_URL
-      config.gushfile = GUSHFILE
+      config.redis_url        = REDIS_URL
+      config.gushfile         = GUSHFILE
+      config.locking_duration = defined?(locking_duration) ? locking_duration : 2
+      config.polling_interval = defined?(polling_interval) ? polling_interval : 0.3
     end
   end
 
-
   config.after(:each) do
     clear_enqueued_jobs
     clear_performed_jobs