gush 0.4.1 → 1.0.0

data/gush.gemspec

@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "gush"
-  spec.version       = "0.4.1"
+  spec.version       = "1.0.0"
   spec.authors       = ["Piotrek Okoński"]
   spec.email         = ["piotrek@okonski.org"]
-  spec.summary       = "Fast and distributed workflow runner using only Sidekiq and Redis"
-  spec.description   = "Gush is a parallel workflow runner using only Redis as its message broker and Sidekiq for workers."
-  spec.homepage      = "https://github.com/pokonski/gush"
+  spec.summary       = "Fast and distributed workflow runner based on ActiveJob and Redis"
+  spec.description   = "Gush is a parallel workflow runner using Redis as storage and ActiveJob for executing jobs."
+  spec.homepage      = "https://github.com/chaps-io/gush"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -17,7 +17,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq", "~> 4.0"
+  spec.add_dependency "activejob", "~> 5.0"
+  spec.add_dependency "connection_pool", "~> 2.2.1"
   spec.add_dependency "multi_json", "~> 1.11"
   spec.add_dependency "redis", "~> 3.2"
   spec.add_dependency "hiredis", "~> 0.6"

data/lib/gush.rb

@@ -5,7 +5,6 @@ require "hiredis"
 require "pathname"
 require "redis"
 require "securerandom"
-require "sidekiq"
 require "multi_json"
 
 require "gush/json"
@@ -34,18 +33,5 @@ module Gush
 
   def self.configure
     yield configuration
-    reconfigure_sidekiq
-  end
-
-  def self.reconfigure_sidekiq
-    Sidekiq.configure_server do |config|
-      config.redis = { url: configuration.redis_url, queue: configuration.namespace}
-    end
-
-    Sidekiq.configure_client do |config|
-      config.redis = { url: configuration.redis_url, queue: configuration.namespace}
-    end
   end
 end
-
-Gush.reconfigure_sidekiq

data/lib/gush/cli.rb

@@ -2,16 +2,12 @@ require 'terminal-table'
 require 'colorize'
 require 'thor'
 require 'launchy'
-require 'sidekiq'
-require 'sidekiq/api'
 
 module Gush
   class CLI < Thor
     class_option :gushfile, desc: "configuration file to use", aliases: "-f"
-    class_option :concurrency, desc: "concurrency setting for Sidekiq", aliases: "-c"
     class_option :redis, desc: "Redis URL to use", aliases: "-r"
     class_option :namespace, desc: "namespace to run jobs in", aliases: "-n"
-    class_option :env, desc: "Sidekiq environment", aliases: "-e"
 
     def initialize(*)
       super
@@ -20,7 +16,6 @@ module Gush
         config.concurrency = options.fetch("concurrency", config.concurrency)
         config.redis_url   = options.fetch("redis",       config.redis_url)
         config.namespace   = options.fetch("namespace",   config.namespace)
-        config.environment = options.fetch("environment", config.environment)
       end
       load_gushfile
     end
@@ -52,11 +47,6 @@ module Gush
       client.stop_workflow(id)
     end
 
-    desc "clear", "Clears all jobs from Sidekiq queue"
-    def clear
-      Sidekiq::Queue.new(client.configuration.namespace).clear
-    end
-
     desc "show [workflow_id]", "Shows details about workflow with given ID"
     option :skip_overview, type: :boolean
     option :skip_jobs, type: :boolean
@@ -79,22 +69,17 @@ module Gush
     def list
       workflows = client.all_workflows
       rows = workflows.map do |workflow|
-        [workflow.id, workflow.class, {alignment: :center, value: status_for(workflow)}]
+        [workflow.id, (Time.at(workflow.started_at) if workflow.started_at), workflow.class, {alignment: :center, value: status_for(workflow)}]
       end
       headers = [
         {alignment: :center, value: 'id'},
+        {alignment: :center, value: 'started at'},
         {alignment: :center, value: 'name'},
         {alignment: :center, value: 'status'}
       ]
       puts Terminal::Table.new(headings: headers, rows: rows)
     end
 
-    desc "workers", "Starts Sidekiq workers"
-    def workers
-      config = client.configuration
-      Kernel.exec "bundle exec sidekiq -r #{config.gushfile} -c #{config.concurrency} -q #{config.namespace} -e #{config.environment} -v"
-    end
-
     desc "viz [WorkflowClass]", "Displays graph, visualising job dependencies"
     def viz(name)
       client
@@ -136,7 +121,7 @@ module Gush
         raise Thor::Error, "#{file} not found, please add it to your project".colorize(:red)
       end
 
-      require file
+      load file.to_s
     rescue LoadError
       raise Thor::Error, "failed to require #{file}".colorize(:red)
     end

data/lib/gush/cli/overview.rb

@@ -65,7 +65,7 @@ module Gush
       end
 
       def started_at
-        workflow.started_at.inspect
+        Time.at(workflow.started_at) if workflow.started_at
       end
 
       def failed_status

data/lib/gush/client.rb

@@ -1,15 +1,15 @@
+require 'connection_pool'
+
 module Gush
   class Client
-    attr_reader :configuration, :sidekiq
+    attr_reader :configuration
 
     def initialize(config = Gush.configuration)
       @configuration = config
-      @sidekiq = build_sidekiq
     end
 
     def configure
       yield configuration
-      @sidekiq = build_sidekiq
     end
 
     def create_workflow(name)
@@ -111,8 +111,7 @@ module Gush
       end
     end
 
-    def load_job(workflow_id, job_id)
-      workflow = find_workflow(workflow_id)
+    def find_job(workflow_id, job_id)
       job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_id)
       hypen = '-' if job_name_match.nil?
 
@@ -129,7 +128,7 @@ module Gush
       return nil if data.nil?
 
       data = Gush::JSON.decode(data, symbolize_keys: true)
-      Gush::Job.from_hash(workflow, data)
+      Gush::Job.from_hash(data)
     end
 
     def destroy_workflow(workflow)
@@ -145,51 +144,28 @@ module Gush
       end
     end
 
-    def worker_report(message)
-      report("gush.workers.status", message)
-    end
-
-    def workflow_report(message)
-      report("gush.workflows.status", message)
-    end
-
     def enqueue_job(workflow_id, job)
       job.enqueue!
       persist_job(workflow_id, job)
 
-      sidekiq.push(
-        'class' => Gush::Worker,
-        'queue' => configuration.namespace,
-        'args'  => [workflow_id, job.name]
-      )
+      Gush::Worker.set(queue: configuration.namespace).perform_later(*[workflow_id, job.name])
     end
 
     private
 
     def workflow_from_hash(hash, nodes = nil)
-      flow = hash[:klass].constantize.new *hash[:arguments]
+      flow = hash[:klass].constantize.new(*hash[:arguments])
       flow.jobs = []
       flow.stopped = hash.fetch(:stopped, false)
       flow.id = hash[:id]
 
       (nodes || hash[:nodes]).each do |node|
-        flow.jobs << Gush::Job.from_hash(flow, node)
+        flow.jobs << Gush::Job.from_hash(node)
       end
 
       flow
     end
 
-    def report(key, message)
-      connection_pool.with do |redis|
-        redis.publish(key, Gush::JSON.encode(message))
-      end
-    end
-
-
-    def build_sidekiq
-      Sidekiq::Client.new(connection_pool)
-    end
-
     def build_redis
       Redis.new(url: configuration.redis_url)
     end

data/lib/gush/configuration.rb

@@ -1,6 +1,6 @@
 module Gush
   class Configuration
-    attr_accessor :concurrency, :namespace, :redis_url, :environment
+    attr_accessor :concurrency, :namespace, :redis_url
 
     def self.from_json(json)
       new(Gush::JSON.decode(json, symbolize_keys: true))
@@ -10,8 +10,7 @@ module Gush
       self.concurrency = hash.fetch(:concurrency, 5)
       self.namespace   = hash.fetch(:namespace, 'gush')
       self.redis_url   = hash.fetch(:redis_url, 'redis://localhost:6379')
-      self.gushfile    = hash.fetch(:gushfile, 'Gushfile.rb')
-      self.environment = hash.fetch(:environment, 'development')
+      self.gushfile    = hash.fetch(:gushfile, 'Gushfile')
     end
 
     def gushfile=(path)
@@ -19,15 +18,14 @@ module Gush
     end
 
     def gushfile
-      @gushfile.realpath
+      @gushfile.realpath if @gushfile.exist?
     end
 
     def to_hash
       {
         concurrency: concurrency,
         namespace:   namespace,
-        redis_url:   redis_url,
-        environment: environment
+        redis_url:   redis_url
       }
     end
 

data/lib/gush/graph.rb

@@ -41,7 +41,8 @@ module Gush
         graph.add_edges(job.name, end_node)
       else
         job.outgoing.each do |id|
-          graph.add_edges(job.name, id)
+          outgoing_job = workflow.find_job(id)
+          graph.add_edges(job.name, outgoing_job.name)
         end
       end
     end

data/lib/gush/job.rb

@@ -1,11 +1,10 @@
 module Gush
   class Job
     attr_accessor :workflow_id, :incoming, :outgoing, :params,
-      :finished_at, :failed_at, :started_at, :enqueued_at, :payloads_hash, :klass
-    attr_reader :name, :output_payload, :params, :payloads
+      :finished_at, :failed_at, :started_at, :enqueued_at, :payloads, :klass
+    attr_reader :name, :output_payload, :params
 
-    def initialize(workflow, opts = {})
-      @workflow = workflow
+    def initialize(opts = {})
       options = opts.dup
       assign_variables(options)
     end
@@ -21,6 +20,7 @@ module Gush
         started_at: started_at,
         failed_at: failed_at,
         params: params,
+        workflow_id: workflow_id,
         output_payload: output_payload
       }
     end
@@ -29,21 +29,15 @@ module Gush
       Gush::JSON.encode(as_json)
     end
 
-    def self.from_hash(flow, hash)
-      hash[:klass].constantize.new(flow, hash)
+    def self.from_hash(hash)
+      hash[:klass].constantize.new(hash)
     end
 
     def output(data)
       @output_payload = data
     end
 
-    def payloads
-      payload_h = {}
-      payloads_hash.each {|k,val| payload_h[k.to_s] = val.map {|h| h[:payload] }}
-      payload_h
-    end
-
-    def work
+    def perform
     end
 
     def start!
@@ -95,7 +89,7 @@ module Gush
 
     def parents_succeeded?
       incoming.all? do |name|
-        @workflow.find_job(name).succeeded?
+        client.find_job(workflow_id, name).succeeded?
       end
     end
 
@@ -104,8 +98,9 @@ module Gush
     end
 
     private
-    def logger
-      Sidekiq.logger
+
+    def client
+      @client ||= Client.new
     end
 
     def current_timestamp
@@ -123,6 +118,7 @@ module Gush
       @params         = opts[:params] || {}
       @klass          = opts[:klass]
       @output_payload = opts[:output_payload]
+      @workflow_id    = opts[:workflow_id]
     end
   end
 end

data/lib/gush/worker.rb

@@ -1,91 +1,63 @@
-require 'sidekiq'
+require 'active_job'
 
 module Gush
-  class Worker
-    include ::Sidekiq::Worker
-    sidekiq_options retry: false
-
+  class Worker < ::ActiveJob::Base
     def perform(workflow_id, job_id)
       setup_job(workflow_id, job_id)
 
-      job.payloads_hash = incoming_payloads
-
-      start = Time.now
-      report(:started, start)
+      job.payloads = incoming_payloads
 
-      failed = false
       error = nil
 
       mark_as_started
       begin
-        job.work
-      rescue Exception => error
+        job.perform
+      rescue StandardError => error
         mark_as_failed
-        report(:failed, start, error.message)
         raise error
       else
         mark_as_finished
-        report(:finished, start)
-
         enqueue_outgoing_jobs
       end
     end
 
     private
-    attr_reader :client, :workflow, :job
+
+    attr_reader :client, :workflow_id, :job
 
     def client
       @client ||= Gush::Client.new(Gush.configuration)
     end
 
     def setup_job(workflow_id, job_id)
-      @workflow ||= client.find_workflow(workflow_id)
-      @job ||= workflow.find_job(job_id)
+      @workflow_id = workflow_id
+      @job ||= client.find_job(workflow_id, job_id)
     end
 
     def incoming_payloads
-      payloads = {}
-      job.incoming.each do |job_name|
-       job = client.load_job(workflow.id, job_name)
-       payloads[job.klass.to_s] ||= []
-       payloads[job.klass.to_s] << {:id => job.name, :payload => job.output_payload}
+      job.incoming.map do |job_name|
+        job = client.find_job(workflow_id, job_name)
+        {
+          id: job.name,
+          class: job.klass.to_s,
+          output: job.output_payload
+        }
       end
-      payloads
     end
 
     def mark_as_finished
       job.finish!
-      client.persist_job(workflow.id, job)
+      client.persist_job(workflow_id, job)
     end
 
     def mark_as_failed
       job.fail!
-      client.persist_job(workflow.id, job)
+      client.persist_job(workflow_id, job)
     end
 
     def mark_as_started
       job.start!
-      client.persist_job(workflow.id, job)
-    end
-
-    def report_workflow_status
-      client.workflow_report({
-        workflow_id:  workflow.id,
-        status:       workflow.status,
-        started_at:   workflow.started_at,
-        finished_at:  workflow.finished_at
-      })
-    end
-
-    def report(status, start, error = nil)
-      message = {
-        status: status,
-        workflow_id: workflow.id,
-        job: job.name,
-        duration: elapsed(start)
-      }
-      message[:error] = error if error
-      client.worker_report(message)
+      client.persist_job(workflow_id, job)
     end
 
     def elapsed(start)
@@ -94,9 +66,9 @@ module Gush
 
     def enqueue_outgoing_jobs
       job.outgoing.each do |job_name|
-        out = client.load_job(workflow.id, job_name)
+        out = client.find_job(workflow_id, job_name)
         if out.ready_to_start?
-          client.enqueue_job(workflow.id, out)
+          client.enqueue_job(workflow_id, out)
         end
       end
     end