gush 2.1.0 → 4.0.0

data/lib/gush/cli.rb

@@ -70,9 +70,14 @@ module Gush
       client.destroy_workflow(workflow)
     end
 
-    desc "list", "Lists all workflows with their statuses"
-    def list
-      workflows = client.all_workflows
+    desc "list START STOP", "Lists workflows from START index through STOP index with their statuses"
+    option :start, type: :numeric, default: nil
+    option :stop, type: :numeric, default: nil
+    def list(start=nil, stop=nil)
+      workflows = client.workflow_ids(start, stop).map do |id|
+        client.find_workflow(id)
+      end
+
       rows = workflows.map do |workflow|
         [workflow.id, (Time.at(workflow.started_at) if workflow.started_at), workflow.class, {alignment: :center, value: status_for(workflow)}]
       end
@@ -101,7 +106,7 @@ module Gush
         begin
           workflow = class_or_id.constantize.new
         rescue NameError => e
-          STDERR.puts Paint["'#{class_or_id}' is not a valid workflow class or id", :red]
+          warn Paint["'#{class_or_id}' is not a valid workflow class or id", :red]
           exit 1
         end
       end
@@ -120,6 +125,24 @@ module Gush
       end
     end
 
+    desc "migrate", "Runs all unapplied migrations to Gush storage"
+    def migrate
+      Dir[File.join(__dir__, 'migrate', '*.rb')].each {|file| require file }
+
+      applied = Gush::Migration.subclasses.sort(&:version).count do |klass|
+        migration = klass.new
+        next if migration.migrated?
+
+        puts "Migrating to #{klass.name} (#{migration.version})"
+        migration.migrate
+        puts "== #{migration.version} #{klass.name}: migrated ==="
+
+        true
+      end
+
+      puts "#{applied} #{'migrations'.pluralize(applied)} applied"
+    end
+
     private
 
     def client

data/lib/gush/client.rb

@@ -72,7 +72,7 @@ module Gush
       id = nil
       loop do
         id = SecureRandom.uuid
-        available = !redis.exists?("gush.workflow.#{id}")
+        available = !redis.exists?("gush.workflows.#{id}")
 
         break if available
       end
@@ -80,6 +80,41 @@ module Gush
       id
     end
 
+    # Returns the specified range of workflow ids, sorted by created timestamp.
+    #
+    # @param start, stop [Integer] see https://redis.io/docs/latest/commands/zrange/#index-ranges
+    #   for details on the start and stop parameters.
+    # @param by_ts [Boolean] if true, start and stop are treated as timestamps
+    #   rather than as element indexes, which allows the workflows to be indexed
+    #   by created timestamp
+    # @param order [Symbol] if :asc, finds ids in ascending created timestamp;
+    #   if :desc, finds ids in descending created timestamp
+    # @returns [Array<String>] array of workflow ids
+    def workflow_ids(start=nil, stop=nil, by_ts: false, order: :asc)
+      start ||= 0
+      stop ||= 99
+
+      redis.zrange(
+        "gush.idx.workflows.created_at",
+        start,
+        stop,
+        by_score: by_ts,
+        rev: order&.to_sym == :desc
+      )
+    end
+
+    def workflows(start=nil, stop=nil, **kwargs)
+      workflow_ids(start, stop, **kwargs).map { |id| find_workflow(id) }
+    end
+
+    def workflows_count
+      redis.zcard('gush.idx.workflows.created_at')
+    end
+
+    # Deprecated.
+    #
+    # This method is not performant when there are a large number of workflows
+    # or when the redis keyspace is large. Use workflows instead with pagination.
     def all_workflows
       redis.scan_each(match: "gush.workflows.*").map do |key|
         id = key.sub("gush.workflows.", "")
@@ -92,10 +127,16 @@ module Gush
 
       unless data.nil?
         hash = Gush::JSON.decode(data, symbolize_keys: true)
-        keys = redis.scan_each(match: "gush.jobs.#{id}.*")
+
+        if hash[:job_klasses]
+          keys = hash[:job_klasses].map { |klass| "gush.jobs.#{id}.#{klass}" }
+        else
+          # For backwards compatibility, get job keys via a full keyspace scan
+          keys = redis.scan_each(match: "gush.jobs.#{id}.*")
+        end
 
         nodes = keys.each_with_object([]) do |key, array|
-          array.concat redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
+          array.concat(redis.hvals(key).map { |json| Gush::JSON.decode(json, symbolize_keys: true) })
         end
 
         workflow_from_hash(hash, nodes)
@@ -105,15 +146,25 @@ module Gush
     end
 
     def persist_workflow(workflow)
+      created_at = Time.now.to_f
+      added = redis.zadd("gush.idx.workflows.created_at", created_at, workflow.id, nx: true)
+
+      if added && configuration.ttl&.positive?
+        expires_at = created_at + configuration.ttl
+        redis.zadd("gush.idx.workflows.expires_at", expires_at, workflow.id, nx: true)
+      end
+
       redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
 
-      workflow.jobs.each {|job| persist_job(workflow.id, job) }
+      workflow.jobs.each {|job| persist_job(workflow.id, job, expires_at: expires_at) }
       workflow.mark_as_persisted
 
       true
     end
 
-    def persist_job(workflow_id, job)
+    def persist_job(workflow_id, job, expires_at: nil)
+      redis.zadd("gush.idx.jobs.expires_at", expires_at, "#{workflow_id}.#{job.klass}", nx: true) if expires_at
+
       redis.hset("gush.jobs.#{workflow_id}.#{job.klass}", job.id, job.to_json)
     end
 
@@ -134,30 +185,67 @@ module Gush
 
     def destroy_workflow(workflow)
       redis.del("gush.workflows.#{workflow.id}")
+      redis.zrem("gush.idx.workflows.created_at", workflow.id)
+      redis.zrem("gush.idx.workflows.expires_at", workflow.id)
       workflow.jobs.each {|job| destroy_job(workflow.id, job) }
     end
 
     def destroy_job(workflow_id, job)
       redis.del("gush.jobs.#{workflow_id}.#{job.klass}")
+      redis.zrem("gush.idx.jobs.expires_at", "#{workflow_id}.#{job.klass}")
+    end
+
+    def expire_workflows(expires_at=nil)
+      expires_at ||= Time.now.to_f
+
+      ids = redis.zrange("gush.idx.workflows.expires_at", "-inf", expires_at, by_score: true)
+      return if ids.empty?
+
+      redis.del(ids.map { |id| "gush.workflows.#{id}" })
+      redis.zrem("gush.idx.workflows.created_at", ids)
+      redis.zrem("gush.idx.workflows.expires_at", ids)
+
+      expire_jobs(expires_at)
+    end
+
+    def expire_jobs(expires_at=nil)
+      expires_at ||= Time.now.to_f
+
+      keys = redis.zrange("gush.idx.jobs.expires_at", "-inf", expires_at, by_score: true)
+      return if keys.empty?
+
+      redis.del(keys.map { |key| "gush.jobs.#{key}" })
+      redis.zrem("gush.idx.jobs.expires_at", keys)
     end
 
     def expire_workflow(workflow, ttl=nil)
-      ttl = ttl || configuration.ttl
-      redis.expire("gush.workflows.#{workflow.id}", ttl)
+      ttl ||= configuration.ttl
+
+      if ttl&.positive?
+        redis.zadd("gush.idx.workflows.expires_at", Time.now.to_f + ttl, workflow.id)
+      else
+        redis.zrem("gush.idx.workflows.expires_at", workflow.id)
+      end
+
       workflow.jobs.each {|job| expire_job(workflow.id, job, ttl) }
     end
 
     def expire_job(workflow_id, job, ttl=nil)
-      ttl = ttl || configuration.ttl
-      redis.expire("gush.jobs.#{workflow_id}.#{job.klass}", ttl)
+      ttl ||= configuration.ttl
+
+      if ttl&.positive?
+        redis.zadd("gush.idx.jobs.expires_at", Time.now.to_f + ttl, "#{workflow_id}.#{job.klass}")
+      else
+        redis.zrem("gush.idx.jobs.expires_at", "#{workflow_id}.#{job.klass}")
+      end
     end
 
     def enqueue_job(workflow_id, job)
       job.enqueue!
       persist_job(workflow_id, job)
-      queue = job.queue || configuration.namespace
 
-      Gush::Worker.set(queue: queue).perform_later(*[workflow_id, job.name])
+      options = { queue: configuration.namespace }.merge(job.worker_options)
+      job.enqueue_worker!(options)
     end
 
     private
@@ -178,16 +266,25 @@ module Gush
     end
 
     def workflow_from_hash(hash, nodes = [])
-      flow = hash[:klass].constantize.new(*hash[:arguments])
-      flow.jobs = []
-      flow.stopped = hash.fetch(:stopped, false)
-      flow.id = hash[:id]
-
-      flow.jobs = nodes.map do |node|
+      jobs = nodes.map do |node|
         Gush::Job.from_hash(node)
       end
 
-      flow
+      internal_state = {
+        persisted: true,
+        jobs: jobs,
+        # For backwards compatibility, setup can only be skipped for a persisted
+        # workflow if there is no data missing from the persistence.
+        # 2024-07-23: dependencies added to persistence
+        skip_setup: !hash[:dependencies].nil?
+      }.merge(hash)
+
+      hash[:klass].constantize.new(
+        *hash[:arguments],
+        **hash[:kwargs],
+        globals: hash[:globals],
+        internal_state: internal_state
+      )
     end
 
     def redis

data/lib/gush/graph.rb

@@ -4,7 +4,7 @@ require 'tmpdir'
 
 module Gush
   class Graph
-    attr_reader :workflow, :filename, :path, :start_node, :end_node
+    attr_reader :workflow, :filename, :start_node, :end_node
 
     def initialize(workflow, options = {})
       @workflow = workflow
@@ -32,7 +32,7 @@ module Gush
       file_format = path.split('.')[-1]
       format = file_format if file_format.length == 3
 
-      Graphviz::output(@graph, path: path, format: format)
+      Graphviz.output(@graph, path: path, format: format)
     end
 
     def path

data/lib/gush/job.rb

@@ -1,8 +1,9 @@
 module Gush
   class Job
     attr_accessor :workflow_id, :incoming, :outgoing, :params,
-      :finished_at, :failed_at, :started_at, :enqueued_at, :payloads, :klass, :queue
-    attr_reader :id, :klass, :output_payload, :params
+      :finished_at, :failed_at, :started_at, :enqueued_at, :payloads,
+      :klass, :queue, :wait
+    attr_reader :id, :output_payload
 
     def initialize(opts = {})
       options = opts.dup
@@ -22,7 +23,8 @@ module Gush
         failed_at: failed_at,
         params: params,
         workflow_id: workflow_id,
-        output_payload: output_payload
+        output_payload: output_payload,
+        wait: wait
       }
     end
 
@@ -57,6 +59,14 @@ module Gush
       @failed_at = nil
     end
 
+    def enqueue_worker!(options = {})
+      Gush::Worker.set(options).perform_later(workflow_id, name)
+    end
+
+    def worker_options
+      { queue: queue, wait: wait }.compact
+    end
+
     def finish!
       @finished_at = current_timestamp
     end
@@ -126,6 +136,7 @@ module Gush
       @output_payload = opts[:output_payload]
       @workflow_id    = opts[:workflow_id]
       @queue          = opts[:queue]
+      @wait           = opts[:wait]
     end
   end
 end

data/lib/gush/migrate/1_create_gush_workflows_created.rb

@@ -0,0 +1,21 @@
+module Gush
+  class IndexWorkflowsByCreatedAtAndExpiresAt < Gush::Migration
+    def self.version
+      1
+    end
+
+    def up
+      redis.scan_each(match: "gush.workflows.*").map do |key|
+        id = key.sub("gush.workflows.", "")
+        workflow = client.find_workflow(id)
+
+        ttl = redis.ttl(key)
+        redis.persist(key)
+        workflow.jobs.each { |job| redis.persist("gush.jobs.#{id}.#{job.klass}") }
+
+        client.persist_workflow(workflow)
+        client.expire_workflow(workflow, ttl.positive? ? ttl : -1)
+      end
+    end
+  end
+end

data/lib/gush/migration.rb

@@ -0,0 +1,37 @@
+module Gush
+  class Migration
+    def migrate
+      return if migrated?
+
+      up
+      migrated!
+    end
+
+    def up
+      # subclass responsibility
+      raise NotImplementedError
+    end
+
+    def version
+      self.class.version
+    end
+
+    def migrated?
+      redis.sismember("gush.migration.schema_migrations", version)
+    end
+
+    private
+
+    def migrated!
+      redis.sadd("gush.migration.schema_migrations", version)
+    end
+
+    def client
+      @client ||= Client.new
+    end
+
+    def redis
+      Gush::Client.redis_connection(client.configuration)
+    end
+  end
+end

data/lib/gush/version.rb

@@ -0,0 +1,3 @@
+module Gush
+  VERSION = '4.0.0'.freeze
+end

data/lib/gush/worker.rb

@@ -30,7 +30,7 @@ module Gush
 
     private
 
-    attr_reader :client, :workflow_id, :job, :configuration
+    attr_reader :workflow_id, :job
 
     def client
       @client ||= Gush::Client.new(Gush.configuration)

data/lib/gush/workflow.rb

@@ -2,25 +2,33 @@ require 'securerandom'
 
 module Gush
   class Workflow
-    attr_accessor :id, :jobs, :stopped, :persisted, :arguments
+    attr_accessor :jobs, :dependencies, :stopped, :persisted, :arguments, :kwargs, :globals
+    attr_writer :id
 
-    def initialize(*args)
-      @id = id
-      @jobs = []
-      @dependencies = []
-      @persisted = false
-      @stopped = false
+    def initialize(*args, globals: nil, internal_state: {}, **kwargs)
       @arguments = args
+      @kwargs = kwargs
+      @globals = globals || {}
+
+      @id = internal_state[:id] || id
+      @jobs = internal_state[:jobs] || []
+      @dependencies = internal_state[:dependencies] || []
+      @persisted = internal_state[:persisted] || false
+      @stopped = internal_state[:stopped] || false
 
-      setup
+      setup unless internal_state[:skip_setup]
     end
 
     def self.find(id)
       Gush::Client.new.find_workflow(id)
     end
 
-    def self.create(*args)
-      flow = new(*args)
+    def self.page(start=0, stop=99, order: :asc)
+      Gush::Client.new.workflows(start, stop, order: order)
+    end
+
+    def self.create(*args, **kwargs)
+      flow = new(*args, **kwargs)
       flow.save
       flow
     end
@@ -38,7 +46,7 @@ module Gush
       persist!
     end
 
-    def configure(*args)
+    def configure(*args, **kwargs)
     end
 
     def mark_as_stopped
@@ -53,7 +61,7 @@ module Gush
       client.persist_workflow(self)
     end
 
-    def expire! (ttl=nil)
+    def expire!(ttl=nil)
       client.expire_workflow(self, ttl)
     end
 
@@ -111,8 +119,9 @@ module Gush
       node = klass.new({
         workflow_id: id,
         id: client.next_free_job_id(id, klass.to_s),
-        params: opts.fetch(:params, {}),
-        queue: opts[:queue]
+        params: (@globals || {}).merge(opts.fetch(:params, {})),
+        queue: opts[:queue],
+        wait: opts[:wait]
       })
 
       jobs << node
@@ -156,7 +165,7 @@ module Gush
         when stopped?
           :stopped
         else
-          :running
+          :pending
       end
     end
 
@@ -174,9 +183,13 @@ module Gush
         name: name,
         id: id,
         arguments: @arguments,
+        kwargs: @kwargs,
+        globals: @globals,
+        dependencies: @dependencies,
         total: jobs.count,
         finished: jobs.count(&:finished?),
         klass: name,
+        job_klasses: jobs.map(&:class).map(&:to_s).uniq,
         status: status,
         stopped: stopped,
         started_at: started_at,
@@ -199,7 +212,7 @@ module Gush
     private
 
     def setup
-      configure(*@arguments)
+      configure(*@arguments, **@kwargs)
       resolve_dependencies
     end
 

data/lib/gush.rb

@@ -15,6 +15,7 @@ require "gush/client"
 require "gush/configuration"
 require "gush/errors"
 require "gush/job"
+require "gush/migration"
 require "gush/worker"
 require "gush/workflow"
 

data/spec/features/integration_spec.rb

@@ -5,9 +5,10 @@ describe "Workflows" do
   context "when all jobs finish successfuly" do
     it "marks workflow as completed" do
       flow = TestWorkflow.create
-      perform_enqueued_jobs do
-        flow.start!
-      end
+
+      ActiveJob::Base.queue_adapter.perform_enqueued_jobs = true
+      flow.start!
+      ActiveJob::Base.queue_adapter.perform_enqueued_jobs = false
 
       flow = flow.reload
       expect(flow).to be_finished
@@ -135,7 +136,7 @@ describe "Workflows" do
 
     class SummaryJob < Gush::Job
       def perform
-        output payloads.map { |payload| payload[:output] }
+        output(payloads.map { |payload| payload[:output] })
       end
     end
 
@@ -168,8 +169,8 @@ describe "Workflows" do
     INTERNAL_CONFIGURE_SPY = double('configure spy')
     expect(INTERNAL_SPY).to receive(:some_method).exactly(110).times
 
-    # One time when persisting, second time when reloading in the spec
-    expect(INTERNAL_CONFIGURE_SPY).to receive(:some_method).exactly(2).times
+    # One time when persisting; reloading does not call configure again
+    expect(INTERNAL_CONFIGURE_SPY).to receive(:some_method).exactly(1).time
 
     class SimpleJob < Gush::Job
       def perform