gush 0.0.1 → 0.1

data/lib/gush/client.rb

@@ -6,7 +6,6 @@ module Gush
       @configuration = config
       @sidekiq = build_sidekiq
       @redis = build_redis
-      load_gushfile
     end
 
     def configure
@@ -16,42 +15,47 @@ module Gush
     end
 
     def create_workflow(name)
-      id = SecureRandom.uuid.split("-").first
-
       begin
-        workflow = name.constantize.new(id)
+        flow = name.constantize.new
+        flow.save
       rescue NameError
         raise WorkflowNotFound.new("Workflow with given name doesn't exist")
       end
 
-      persist_workflow(workflow)
-      workflow
+      flow
     end
 
-    def start_workflow(id, jobs = [])
-      workflow = find_workflow(id)
-      workflow.start!
+    def start_workflow(workflow, job_names = [])
+      workflow.mark_as_started
       persist_workflow(workflow)
 
-      jobs = if jobs.empty?
-               workflow.next_jobs
+      jobs = if job_names.empty?
+               workflow.initial_jobs
              else
-               jobs.map {|name| workflow.find_job(name) }
+               job_names.map {|name| workflow.find_job(name) }
              end
 
       jobs.each do |job|
-        job.enqueue!
-        persist_job(workflow.id, job)
         enqueue_job(workflow.id, job)
       end
     end
 
     def stop_workflow(id)
       workflow = find_workflow(id)
-      workflow.stop!
+      workflow.mark_as_stopped
       persist_workflow(workflow)
     end
 
+    def next_free_id
+      id = nil
+      loop do
+        id = SecureRandom.uuid
+        break if !redis.exists("gush.workflow.#{id}")
+      end
+
+      id
+    end
+
     def all_workflows
       redis.keys("gush.workflows.*").map do |key|
         id = key.sub("gush.workflows.", "")
@@ -62,9 +66,9 @@ module Gush
     def find_workflow(id)
       data = redis.get("gush.workflows.#{id}")
       unless data.nil?
-        hash = Yajl::Parser.parse(data, symbolize_keys: true)
+        hash = Gush::JSON.decode(data, symbolize_keys: true)
         keys = redis.keys("gush.jobs.#{id}.*")
-        nodes = redis.mget(*keys).map { |json| Yajl::Parser.parse(json, symbolize_keys: true) }
+        nodes = redis.mget(*keys).map { |json| Gush::JSON.decode(json, symbolize_keys: true) }
         workflow_from_hash(hash, nodes)
       else
         raise WorkflowNotFound.new("Workflow with given id doesn't exist")
@@ -73,16 +77,25 @@ module Gush
 
     def persist_workflow(workflow)
       redis.set("gush.workflows.#{workflow.id}", workflow.to_json)
-      workflow.nodes.each {|job| persist_job(workflow.id, job) }
+      workflow.jobs.each {|job| persist_job(workflow.id, job) }
+      workflow.mark_as_persisted
+      true
     end
 
     def persist_job(workflow_id, job)
       redis.set("gush.jobs.#{workflow_id}.#{job.class.to_s}", job.to_json)
     end
 
+    def load_job(workflow_id, job_id)
+      data = redis.get("gush.jobs.#{workflow_id}.#{job_id}")
+      return nil if data.nil?
+      data = Gush::JSON.decode(data, symbolize_keys: true)
+      Gush::Job.from_hash(nil, data)
+    end
+
     def destroy_workflow(workflow)
       redis.del("gush.workflows.#{workflow.id}")
-      workflow.nodes.each {|job| destroy_job(workflow.id, job) }
+      workflow.jobs.each {|job| destroy_job(workflow.id, job) }
     end
 
     def destroy_job(workflow_id, job)
@@ -97,33 +110,37 @@ module Gush
       report("gush.workflows.status", message)
     end
 
+    def enqueue_job(workflow_id, job)
+      job.enqueue!
+      persist_job(workflow_id, job)
+
+      sidekiq.push(
+        'class' => Gush::Worker,
+        'queue' => configuration.namespace,
+        'args'  => [workflow_id, job.class.to_s, configuration.to_json]
+      )
+    end
+
     private
 
     attr_reader :sidekiq, :redis
 
     def workflow_from_hash(hash, nodes = nil)
-      flow = hash[:klass].constantize.new(hash[:id], configure: false)
-      flow.logger_builder(hash.fetch(:logger_builder, 'Gush::LoggerBuilder').constantize)
+      flow = hash[:klass].constantize.new(false)
       flow.stopped = hash.fetch(:stopped, false)
+      flow.id = hash[:id]
 
       (nodes || hash[:nodes]).each do |node|
-        flow.nodes << Gush::Job.from_hash(node)
+        flow.jobs << Gush::Job.from_hash(flow, node)
       end
 
       flow
     end
 
     def report(key, message)
-      redis.publish(key, Yajl::Encoder.new.encode(message))
+      redis.publish(key, Gush::JSON.encode(message))
     end
 
-    def enqueue_job(workflow_id, job)
-      sidekiq.push(
-        'class' => Gush::Worker,
-        'queue' => configuration.namespace,
-        'args'  => [workflow_id, job.class.to_s, configuration.to_json]
-      )
-    end
 
     def build_sidekiq
       Sidekiq::Client.new(connection_pool)
@@ -136,11 +153,5 @@ module Gush
     def connection_pool
       ConnectionPool.new(size: configuration.concurrency, timeout: 1) { build_redis }
     end
-
-    def load_gushfile
-      require configuration.gushfile
-    rescue LoadError
-      raise Thor::Error, "failed to load #{configuration.gushfile.basename}".colorize(:red)
-    end
   end
 end

data/lib/gush/configuration.rb

@@ -5,7 +5,7 @@ module Gush
     attr_accessor :concurrency, :namespace, :redis_url, :environment
 
     def self.from_json(json)
-      new(Yajl::Parser.parse(json, symbolize_keys: true))
+      new(Gush::JSON.decode(json, symbolize_keys: true))
     end
 
     def initialize(hash = {})
@@ -21,7 +21,6 @@ module Gush
     end
 
     def gushfile
-      raise Thor::Error, "#{@gushfile} not found, please add it to your project".colorize(:red) unless @gushfile.exist?
       @gushfile.realpath
     end
 
@@ -30,13 +29,12 @@ module Gush
         concurrency: concurrency,
         namespace:   namespace,
         redis_url:   redis_url,
-        environment: environment,
-        gushfile:    gushfile.to_path
+        environment: environment
       }
     end
 
     def to_json
-      Yajl::Encoder.new.encode(to_hash)
+      Gush::JSON.encode(to_hash)
     end
   end
 end

data/lib/gush/errors.rb

@@ -1,3 +1,4 @@
-class WorkflowNotFound < StandardError; end
-class DependencyLevelTooDeep < StandardError; end
-
+module Gush
+  class WorkflowNotFound < StandardError; end
+  class DependencyLevelTooDeep < StandardError; end
+end

data/lib/gush/graph.rb

@@ -0,0 +1,88 @@
+module Gush
+  class Graph
+    attr_reader :workflow, :filename, :path, :start, :end_node
+
+    def initialize(workflow, options: {})
+      @workflow = workflow
+      @filename = options.fetch(:filename, "graph.png")
+      @path = options.fetch(:path, Pathname.new(Dir.tmpdir).join(filename))
+    end
+
+    def viz
+      GraphViz.new(:G, graph_options) do |graph|
+        set_node_options!(graph)
+        set_edge_options!(graph)
+
+        @start = graph.start(shape: 'diamond', fillcolor: '#CFF09E')
+        @end_node = graph.end(shape: 'diamond', fillcolor: '#F56991')
+
+        workflow.jobs.each do |job|
+          add_job(graph, job)
+        end
+
+        graph.output(png: path)
+      end
+    end
+
+    def path
+      @path.to_s
+    end
+
+    private
+    def add_job(graph, job)
+      name = job.class.to_s
+      graph.add_nodes(name)
+
+      if job.incoming.empty?
+        graph.add_edges(start, name)
+      end
+
+      if job.outgoing.empty?
+        graph.add_edges(name, end_node)
+      else
+        job.outgoing.each do |out|
+          graph.add_edges(name, out)
+        end
+      end
+    end
+
+    def set_node_options!(graph)
+      node_options.each do |key, value|
+        graph.node[key] = value
+      end
+    end
+
+    def set_edge_options!(graph)
+      edge_options.each do |key, value|
+        graph.edge[key] = value
+      end
+    end
+
+    def graph_options
+      {
+        type: :digraph,
+        dpi: 200,
+        compound: true,
+        rankdir: "LR",
+        center: true
+      }
+    end
+
+    def node_options
+      {
+        shape: "ellipse",
+        style: "filled",
+        color: "#555555",
+        fillcolor: "white"
+      }
+    end
+
+    def edge_options
+      {
+        dir: "forward",
+        penwidth: 1,
+        color: "#555555"
+      }
+    end
+  end
+end

data/lib/gush/job.rb

@@ -1,27 +1,14 @@
-require 'gush/metadata'
-
 module Gush
   class Job
-    include Gush::Metadata
-
-    RECURSION_LIMIT = 1000
 
-    DEFAULTS = {
-      finished: false,
-      enqueued: false,
-      failed: false,
-      running: false
-    }
-
-    attr_accessor :finished, :enqueued, :failed, :workflow_id, :incoming, :outgoing,
-      :finished_at, :failed_at, :started_at, :jid, :running
+    attr_accessor :workflow_id, :incoming, :outgoing,
+      :finished_at, :failed_at, :started_at, :enqueued_at
 
     attr_reader :name
 
-    attr_writer :logger
-
-    def initialize(opts = {})
-      options = DEFAULTS.dup.merge(opts)
+    def initialize(workflow, opts = {})
+      @workflow = workflow
+      options = opts.dup
       assign_variables(options)
     end
 
@@ -29,88 +16,60 @@ module Gush
       {
         name: @name,
         klass: self.class.to_s,
-        finished: @finished,
-        enqueued: @enqueued,
-        failed: @failed,
+        finished: finished?,
+        enqueued: enqueued?,
+        failed: failed?,
         incoming: @incoming,
         outgoing: @outgoing,
-        finished_at: @finished_at,
-        started_at: @started_at,
-        failed_at: @failed_at,
-        running: @running
+        finished_at: finished_at,
+        enqueued_at: enqueued_at,
+        started_at: started_at,
+        failed_at: failed_at,
+        running: running?
       }
     end
 
     def to_json(options = {})
-      Yajl::Encoder.new.encode(as_json)
+      Gush::JSON.encode(as_json)
     end
 
-    def self.from_hash(hash)
-      hash[:klass].constantize.new(
-        name:     hash[:name],
-        finished: hash[:finished],
-        enqueued: hash[:enqueued],
-        failed: hash[:failed],
-        incoming: hash[:incoming],
-        outgoing: hash[:outgoing],
-        failed_at: hash[:failed_at],
-        finished_at: hash[:finished_at],
-        started_at: hash[:started_at],
-        running: hash[:running]
-      )
-    end
-
-    def before_work
+    def self.from_hash(flow, hash)
+      hash[:klass].constantize.new(flow, hash)
     end
 
     def work
     end
 
-    def after_work
-    end
-
     def start!
-      @enqueued = false
-      @running = true
-      @started_at = Time.now.to_i
+      @started_at = current_timestamp
     end
 
     def enqueue!
-      @enqueued = true
-      @running = false
-      @failed = false
+      @enqueued_at = current_timestamp
       @started_at = nil
       @finished_at = nil
       @failed_at = nil
     end
 
     def finish!
-      @running = false
-      @finished = true
-      @enqueued = false
-      @failed = false
-      @finished_at = Time.now.to_i
+      @finished_at = current_timestamp
     end
 
     def fail!
-      @finished = true
-      @running = false
-      @failed = true
-      @enqueued = false
-      @finished_at = Time.now.to_i
-      @failed_at = Time.now.to_i
+      @finished_at = current_timestamp
+      @failed_at = current_timestamp
     end
 
     def enqueued?
-      !!enqueued
+      !!enqueued_at
     end
 
     def finished?
-      !!finished
+      !!finished_at
     end
 
     def failed?
-      !!failed
+      !!failed_at
     end
 
     def succeeded?
@@ -118,44 +77,31 @@ module Gush
     end
 
     def running?
-      !!running
-    end
-
-    def can_be_started?(flow)
-      !running? &&
-        !enqueued? &&
-          !finished? &&
-            !failed? &&
-              dependencies_satisfied?(flow)
+      !!started_at && !finished?
     end
 
-    def dependencies(flow, level = 0)
-      fail DependencyLevelTooDeep if level > RECURSION_LIMIT
-      (incoming.map {|name| flow.find_job(name) } + incoming.flat_map{ |name| flow.find_job(name).dependencies(flow, level + 1) }).uniq
+    def ready_to_start?
+      !running? && !enqueued? && !finished? && !failed?
     end
 
-    def logger
-      fail "You cannot log when the job is not running" unless running?
-      @logger
+    def has_no_dependencies?
+      incoming.empty?
     end
 
     private
 
+    def current_timestamp
+      Time.now.to_i
+    end
+
     def assign_variables(options)
       @name        = options[:name]
-      @finished    = options[:finished]
-      @enqueued    = options[:enqueued]
-      @failed      = options[:failed]
       @incoming    = options[:incoming] || []
       @outgoing    = options[:outgoing] || []
       @failed_at   = options[:failed_at]
       @finished_at = options[:finished_at]
       @started_at  = options[:started_at]
-      @running     = options[:running]
-    end
-
-    def dependencies_satisfied?(flow)
-      dependencies(flow).all? { |dep| !dep.enqueued? && dep.finished? && !dep.failed? }
+      @enqueued_at = options[:enqueued_at]
     end
   end
 end

data/lib/gush/json.rb

@@ -0,0 +1,12 @@
+module Gush
+  class JSON
+
+    def self.encode(data)
+      Yajl::Encoder.new.encode(data)
+    end
+
+    def self.decode(data, options = {})
+      Yajl::Parser.parse(data, options)
+    end
+  end
+end

data/lib/gush/worker.rb

@@ -15,17 +15,12 @@ module Gush
       start = Time.now
       report(workflow, job, :started, start)
 
-      job.logger = workflow.build_logger_for_job(job, job_id)
-      job.jid = jid
-
       failed = false
       error = nil
 
       mark_as_started(workflow, job)
       begin
-        job.before_work
         job.work
-        job.after_work
       rescue Exception => e
         failed = true
         error = e
@@ -35,9 +30,8 @@ module Gush
         report(workflow, job, :finished, start)
         mark_as_finished(workflow, job)
 
-        continue_workflow(workflow)
+        enqueue_outgoing_jobs(workflow.id, job)
       else
-        log_exception(job.logger, error)
         mark_as_failed(workflow, job)
         report(workflow, job, :failed, start, error.message)
       end
@@ -81,19 +75,12 @@ module Gush
       (Time.now - start).to_f.round(3)
     end
 
-    def continue_workflow(workflow)
-      # refetch is important to get correct workflow status
-      unless client.find_workflow(workflow.id).stopped?
-        client.start_workflow(workflow.id)
-      end
-    end
-
-    def log_exception(logger, exception)
-      first, *rest = exception.backtrace
-
-      logger << "#{first}: #{exception.message} (#{exception.class})\n"
-      rest.each do |line|
-        logger << "        from #{line}\n"
+    def enqueue_outgoing_jobs(workflow_id, job)
+      job.outgoing.each do |job_name|
+        out = client.load_job(workflow_id, job_name)
+        if out.ready_to_start?
+          client.enqueue_job(workflow_id, out)
+        end
       end
     end
   end