gush 0.0.1

data/lib/gush/client.rb

@@ -0,0 +1,146 @@
+module Gush
+  class Client
+    attr_reader :configuration
+
+    def initialize(config = Gush.configuration)
+      @configuration = config
+      @sidekiq = build_sidekiq
+      @redis = build_redis
+      load_gushfile
+    end
+
+    def configure
+      yield configuration
+      @sidekiq = build_sidekiq
+      @redis = build_redis
+    end
+
+    def create_workflow(name)
+      id = SecureRandom.uuid.split("-").first
+
+      begin
+        workflow = name.constantize.new(id)
+      rescue NameError
+        raise WorkflowNotFound.new("Workflow with given name doesn't exist")
+      end
+
+      persist_workflow(workflow)
+      workflow
+    end
+
+    def start_workflow(id, jobs = [])
+      workflow = find_workflow(id)
+      workflow.start!
+      persist_workflow(workflow)
+
+      jobs = if jobs.empty?
+               workflow.next_jobs
+             else
+               jobs.map {|name| workflow.find_job(name) }
+             end
+
+      jobs.each do |job|
+        job.enqueue!
+        persist_job(workflow.id, job)
+        enqueue_job(workflow.id, job)
+      end
+    end
+
+    def stop_workflow(id)
+      workflow = find_workflow(id)
+      workflow.stop!
+      persist_workflow(workflow)
+    end
+
+    def all_workflows
+      redis.keys("gush.workflows.*").map do |key|
+        id = key.sub("gush.workflows.", "")
+        find_workflow(id)
+      end
+    end
+
+    def find_workflow(id)
+      data = redis.get("gush.workflows.#{id}")
+      unless data.nil?
+        hash = Yajl::Parser.parse(data, symbolize_keys: true)
+        keys = redis.keys("gush.jobs.#{id}.*")
+        nodes = redis.mget(*keys).map { |json| Yajl::Parser.parse(json, symbolize_keys: true) }
+        workflow_from_hash(hash, nodes)
+      else
+        raise WorkflowNotFound.new("Workflow with given id doesn't exist")
+      end
+    end
+
+    def persist_workflow(workflow)
+      redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
+      workflow.nodes.each {|job| persist_job(workflow.id, job) }
+    end
+
+    def persist_job(workflow_id, job)
+      redis.set("gush.jobs.#{workflow_id}.#{job.class.to_s}", job.to_json)
+    end
+
+    def destroy_workflow(workflow)
+      redis.del("gush.workflows.#{workflow.id}")
+      workflow.nodes.each {|job| destroy_job(workflow.id, job) }
+    end
+
+    def destroy_job(workflow_id, job)
+      redis.del("gush.jobs.#{workflow_id}.#{job.class.to_s}")
+    end
+
+    def worker_report(message)
+      report("gush.workers.status", message)
+    end
+
+    def workflow_report(message)
+      report("gush.workflows.status", message)
+    end
+
+    private
+
+    attr_reader :sidekiq, :redis
+
+    def workflow_from_hash(hash, nodes = nil)
+      flow = hash[:klass].constantize.new(hash[:id], configure: false)
+      flow.logger_builder(hash.fetch(:logger_builder, 'Gush::LoggerBuilder').constantize)
+      flow.stopped = hash.fetch(:stopped, false)
+
+      (nodes || hash[:nodes]).each do |node|
+        flow.nodes << Gush::Job.from_hash(node)
+      end
+
+      flow
+    end
+
+    def report(key, message)
+      redis.publish(key, Yajl::Encoder.new.encode(message))
+    end
+
+    def enqueue_job(workflow_id, job)
+      sidekiq.push(
+        'class' => Gush::Worker,
+        'queue' => configuration.namespace,
+        'args'  => [workflow_id, job.class.to_s, configuration.to_json]
+      )
+    end
+
+    def build_sidekiq
+      Sidekiq::Client.new(connection_pool)
+    end
+
+    def build_redis
+      Redis.new(url: configuration.redis_url)
+    end
+
+    def connection_pool
+      ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
+    end
+
+    def load_gushfile
+      require configuration.gushfile
+    rescue LoadError
+      raise Thor::Error, "failed to load #{configuration.gushfile.basename}".colorize(:red)
+    end
+  end
+end

data/lib/gush/configuration.rb

@@ -0,0 +1,42 @@
+require 'yajl'
+
+module Gush
+  class Configuration
+    attr_accessor :concurrency, :namespace, :redis_url, :environment
+
+    def self.from_json(json)
+      new(Yajl::Parser.parse(json, symbolize_keys: true))
+    end
+
+    def initialize(hash = {})
+      self.concurrency = hash.fetch(:concurrency, 5)
+      self.namespace   = hash.fetch(:namespace, 'gush')
+      self.redis_url   = hash.fetch(:redis_url, 'redis://localhost:6379')
+      self.gushfile    = hash.fetch(:gushfile, 'Gushfile.rb')
+      self.environment = hash.fetch(:environment, 'development')
+    end
+
+    def gushfile=(path)
+      @gushfile = Pathname(path)
+    end
+
+    def gushfile
+      raise Thor::Error, "#{@gushfile} not found, please add it to your project".colorize(:red) unless @gushfile.exist?
+      @gushfile.realpath
+    end
+
+    def to_hash
+      {
+        concurrency: concurrency,
+        namespace:   namespace,
+        redis_url:   redis_url,
+        environment: environment,
+        gushfile:    gushfile.to_path
+      }
+    end
+
+    def to_json
+      Yajl::Encoder.new.encode(to_hash)
+    end
+  end
+end

data/lib/gush/errors.rb

@@ -0,0 +1,3 @@
+class WorkflowNotFound < StandardError; end
+class DependencyLevelTooDeep < StandardError; end
+

data/lib/gush/job.rb

@@ -0,0 +1,161 @@
+require 'gush/metadata'
+
+module Gush
+  class Job
+    include Gush::Metadata
+
+    RECURSION_LIMIT = 1000
+
+    DEFAULTS = {
+      finished: false,
+      enqueued: false,
+      failed: false,
+      running: false
+    }
+
+    attr_accessor :finished, :enqueued, :failed, :workflow_id, :incoming, :outgoing,
+      :finished_at, :failed_at, :started_at, :jid, :running
+
+    attr_reader :name
+
+    attr_writer :logger
+
+    def initialize(opts = {})
+      options = DEFAULTS.dup.merge(opts)
+      assign_variables(options)
+    end
+
+    def as_json
+      {
+        name: @name,
+        klass: self.class.to_s,
+        finished: @finished,
+        enqueued: @enqueued,
+        failed: @failed,
+        incoming: @incoming,
+        outgoing: @outgoing,
+        finished_at: @finished_at,
+        started_at: @started_at,
+        failed_at: @failed_at,
+        running: @running
+      }
+    end
+
+    def to_json(options = {})
+      Yajl::Encoder.new.encode(as_json)
+    end
+
+    def self.from_hash(hash)
+      hash[:klass].constantize.new(
+        name:     hash[:name],
+        finished: hash[:finished],
+        enqueued: hash[:enqueued],
+        failed: hash[:failed],
+        incoming: hash[:incoming],
+        outgoing: hash[:outgoing],
+        failed_at: hash[:failed_at],
+        finished_at: hash[:finished_at],
+        started_at: hash[:started_at],
+        running: hash[:running]
+      )
+    end
+
+    def before_work
+    end
+
+    def work
+    end
+
+    def after_work
+    end
+
+    def start!
+      @enqueued = false
+      @running = true
+      @started_at = Time.now.to_i
+    end
+
+    def enqueue!
+      @enqueued = true
+      @running = false
+      @failed = false
+      @started_at = nil
+      @finished_at = nil
+      @failed_at = nil
+    end
+
+    def finish!
+      @running = false
+      @finished = true
+      @enqueued = false
+      @failed = false
+      @finished_at = Time.now.to_i
+    end
+
+    def fail!
+      @finished = true
+      @running = false
+      @failed = true
+      @enqueued = false
+      @finished_at = Time.now.to_i
+      @failed_at = Time.now.to_i
+    end
+
+    def enqueued?
+      !!enqueued
+    end
+
+    def finished?
+      !!finished
+    end
+
+    def failed?
+      !!failed
+    end
+
+    def succeeded?
+      finished? && !failed?
+    end
+
+    def running?
+      !!running
+    end
+
+    def can_be_started?(flow)
+      !running? &&
+        !enqueued? &&
+          !finished? &&
+            !failed? &&
+              dependencies_satisfied?(flow)
+    end
+
+    def dependencies(flow, level = 0)
+      fail DependencyLevelTooDeep if level > RECURSION_LIMIT
+      (incoming.map {|name| flow.find_job(name) } + incoming.flat_map{ |name| flow.find_job(name).dependencies(flow, level + 1) }).uniq
+    end
+
+    def logger
+      fail "You cannot log when the job is not running" unless running?
+      @logger
+    end
+
+    private
+
+    def assign_variables(options)
+      @name        = options[:name]
+      @finished    = options[:finished]
+      @enqueued    = options[:enqueued]
+      @failed      = options[:failed]
+      @incoming    = options[:incoming] || []
+      @outgoing    = options[:outgoing] || []
+      @failed_at   = options[:failed_at]
+      @finished_at = options[:finished_at]
+      @started_at  = options[:started_at]
+      @running     = options[:running]
+    end
+
+    def dependencies_satisfied?(flow)
+      dependencies(flow).all? { |dep| !dep.enqueued? && dep.finished? && !dep.failed? }
+    end
+  end
+end

data/lib/gush/logger_builder.rb

@@ -0,0 +1,15 @@
+module Gush
+  class LoggerBuilder
+    attr_reader :workflow, :job, :jid
+
+    def initialize(workflow, job, jid)
+      @workflow = workflow
+      @job = job
+      @jid = jid
+    end
+
+    def build
+      NullLogger.new
+    end
+  end
+end

data/lib/gush/metadata.rb

@@ -0,0 +1,24 @@
+module Gush
+  module Metadata
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def metadata(params = {})
+        @metadata = (@metadata || {}).merge(params)
+      end
+    end
+
+    def name
+      metadata[:name] || @name
+    end
+
+    private
+
+    def metadata
+      self.class.metadata
+    end
+  end
+end

data/lib/gush/null_logger.rb

@@ -0,0 +1,6 @@
+module Gush
+  class NullLogger
+    def method_missing(*)
+    end
+  end
+end

data/lib/gush/version.rb

@@ -0,0 +1,3 @@
+module Gush
+  VERSION = "0.0.1"
+end

data/lib/gush/worker.rb

@@ -0,0 +1,100 @@
+require 'sidekiq'
+require 'yajl'
+
+module Gush
+  class Worker
+    include ::Sidekiq::Worker
+    sidekiq_options retry: false
+
+    def perform(workflow_id, job_id, configuration_json)
+      configure_client(configuration_json)
+
+      workflow = client.find_workflow(workflow_id)
+      job = workflow.find_job(job_id)
+
+      start = Time.now
+      report(workflow, job, :started, start)
+
+      job.logger = workflow.build_logger_for_job(job, job_id)
+      job.jid = jid
+
+      failed = false
+      error = nil
+
+      mark_as_started(workflow, job)
+      begin
+        job.before_work
+        job.work
+        job.after_work
+      rescue Exception => e
+        failed = true
+        error = e
+      end
+
+      unless failed
+        report(workflow, job, :finished, start)
+        mark_as_finished(workflow, job)
+
+        continue_workflow(workflow)
+      else
+        log_exception(job.logger, error)
+        mark_as_failed(workflow, job)
+        report(workflow, job, :failed, start, error.message)
+      end
+    end
+
+    private
+
+    attr_reader :client
+
+    def configure_client(config_json)
+      @client = Client.new(Configuration.from_json(config_json))
+    end
+
+    def mark_as_finished(workflow, job)
+      job.finish!
+      client.persist_job(workflow.id, job)
+    end
+
+    def mark_as_failed(workflow, job)
+      job.fail!
+      client.persist_job(workflow.id, job)
+    end
+
+    def mark_as_started(workflow, job)
+      job.start!
+      client.persist_job(workflow.id, job)
+    end
+
+    def report_workflow_status(workflow, job)
+      message = {workflow_id: workflow.id, status: workflow.status, started_at: workflow.started_at, finished_at: workflow.finished_at }
+      client.workflow_report(message)
+    end
+
+    def report(workflow, job, status, start, error = nil)
+      message = {status: status, workflow_id: workflow.id, job: job.name, duration: elapsed(start)}
+      message[:error] = error if error
+      client.worker_report(message)
+    end
+
+    def elapsed(start)
+      (Time.now - start).to_f.round(3)
+    end
+
+    def continue_workflow(workflow)
+      # refetch is important to get correct workflow status
+      unless client.find_workflow(workflow.id).stopped?
+        client.start_workflow(workflow.id)
+      end
+    end
+
+    def log_exception(logger, exception)
+      first, *rest = exception.backtrace
+
+      logger << "#{first}: #{exception.message} (#{exception.class})\n"
+      rest.each do |line|
+        logger << "        from #{line}\n"
+      end
+    end
+  end
+end