gush 0.0.1 → 0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee767bd97e343cac8bf395a65d2d19c63abd638b
-  data.tar.gz: d5a41126c1895e505871ae108613955466724829
+  metadata.gz: 7fa19439bb45ef12b63b393f7956aca1edd96ebe
+  data.tar.gz: 0ce4dc6cdc209c66e33248da59390b017a54a4e7
 SHA512:
-  metadata.gz: ce471a04c947ff17fc3864644223463da25ca52aa59e34cc2118bcb607526db2b394c79b6b708af23bc29a7ae4efb5647e5bd19a09e51d9df253e1b8442fac42
-  data.tar.gz: dda4e7862751f8c7a7cc625f7bdafaca54dc6718a96689346d010d4ae8f6b3350dff9cd07cf461ad369930eb90f75e0bf33242f685be85ed0482065e8f8f367d
+  metadata.gz: 79fa056cf641d929d5f61985a12cc734a9fd4946c3b6ada2d31fd1bbec3b34b2bb7fd2dbc22821a0e42710a37baa1fc72955e867fcbab7a1582cea2ba82df9db
+  data.tar.gz: 229f3550ff30f57ff4cb27eb0cd177d962cd604be9f2d9e8b420172a4543461eef3c430f00096287d88a136e7dacd5b11160d133dab94c7831171b076368fee2

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+script: "bundle exec rspec"
+rvm:
+  - 2.0.0
+  - 2.1.6
+  - 2.2.2
+services:
+  - redis-server
+email:
+  recipients:
+    - piotrek@okonski.org
+  on_success: change
+  on_failure: always

data/Gemfile

@@ -5,4 +5,4 @@ gemspec
 
 gem 'pry'
 gem 'yajl-ruby'
-gem 'bbq-spawn', git: 'git@github.com:drugpl/bbq-spawn.git'
+gem "fakeredis", require: false

data/README.md

@@ -1,4 +1,4 @@
-# Gush
+# Gush [![Build Status](https://travis-ci.org/pokonski/gush.svg?branch=master)](https://travis-ci.org/pokonski/gush)
 
 Gush is a parallel workflow runner using only Redis as its message broker and Sidekiq for workers.
 
@@ -18,21 +18,9 @@ Or install it yourself as:
 
 ## Usage
 
-Your project should contain a file called `Gushfile.rb` which loads all the necessary workflows for Sidekiq to use.
-
-Example:
-
-```ruby
-require_relative './lib/your_project'
-
-Dir[Rowlf.root.join("workflows/**/*.rb")].each do |file|
-  require file
-end
-```
-
 ### Defining workflows
 
-The DSL for defining jobs consists of a single `run` method. 
+The DSL for defining jobs consists of a single `run` method.
 Here is a complete example of a workflow you can create:
 
 ```ruby
@@ -63,6 +51,7 @@ bundle exec gush viz SampleWorkflow
 For the Workflow above, the graph will look like this:
 
 ![SampleWorkflow](http://i.imgur.com/SmeRRVT.png)
+
 ### Defining jobs
 
 Jobs are classes inheriting from `Gush::Job`:
@@ -76,35 +65,54 @@ class FetchJob1 < Gush::Job
 end
 ```
 
-### Running
+### Running workflows
 
-#### 1. Register workflow
+Now that we have defined our workflow we can use it:
 
-After you define your workflows and jobs, all you have to do is register them:
+#### 1. Initialize and save it
 
+```ruby
+flow = SampleWorkflow.new
+flow.save # saves workflow and its jobs to Redis
 ```
-bundle exec gush create SampleWorkflow
-```
 
-the command will return a unique workflow id you will use in later commands.
+**or:** you can also use a shortcut:
+
+```ruby
+flow = SampleWorkflow.create
+```
 
-#### 2. Run workers
+#### 2. Start workflow
 
-This will start Sidekiq workers responsible for processing jobs
+First you need to start Sidekiq workers:
 
 ```
 bundle exec gush workers
 ```
 
-#### 3. Start the workflow
-
-Use your workflow_id returned by `create` command.
+and then start your workflow:
 
+```ruby
+flow.start!
 ```
-bundle gush start <workflow_id>
+
+Now Gush will start processing jobs in background using Sidekiq
+in the order defined in `configure` method inside Workflow.
+
+
+### Checking status:
+
+#### In Ruby:
+
+```ruby
+flow.reload
+flow.status
+#=> :running|:pending|:finished|:failed
 ```
 
-### 5. Check the status
+`reload` is needed to see the latest status, since workflows are updated asynchronously.
+
+#### Via CLI:
 
 - of a specific workflow:
 
@@ -113,15 +121,29 @@ bundle gush start <workflow_id>
   ```
 
 - of all created workflows:
-  
+
   ```
   bundle gush list
   ```
 
+### Requiring workflows inside your projects
+
+**Skip this step if using Gush inside Rails application, workflows will already be loaded**
+
+When using Gush and its CLI commands you need a Gushfile.rb in root directory.
+Gushfile should require all your Workflows and jobs, for example:
+
+```ruby
+require_relative './lib/your_project'
+
+Dir[Rails.root.join("app/workflows/**/*.rb")].each do |file|
+  require file
+end
+```
 
 ## Contributing
 
-1. Fork it ( http://github.com/lonelyplanet/gush/fork )
+1. Fork it ( http://github.com/pokonski/gush/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/bin/gush

@@ -9,4 +9,10 @@ $:.unshift File.expand_path("../../lib", bin_file)
 
 require 'gush'
 
-Gush::CLI.start(ARGV)
+begin
+  Gush::CLI.start(ARGV)
+rescue Gush::WorkflowNotFound
+  puts "Workflow not found".red
+rescue Gush::DependencyLevelTooDeep
+  puts "Dependency level too deep. Perhaps you have a dependency cycle?".red
+end

data/gush.gemspec

@@ -1,11 +1,10 @@
 # coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'gush/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "gush"
-  spec.version       = Gush::VERSION
+  spec.version       = "0.1"
   spec.authors       = ["Piotrek Okoński"]
   spec.email         = ["piotrek@okonski.org"]
   spec.summary       = "Fast and distributed workflow runner using only Sidekiq and Redis"
@@ -17,15 +16,15 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq", "~> 3.2.2"
-  spec.add_dependency "yajl-ruby"
-  spec.add_dependency "redis", "~> 3.0.0"
-  spec.add_dependency "hiredis", "~> 0.5.2"
-  spec.add_dependency "ruby-graphviz"
-  spec.add_dependency "terminal-table"
-  spec.add_dependency "colorize"
-  spec.add_dependency "thor"
-  spec.add_dependency "launchy"
+  spec.add_dependency "sidekiq", "~> 3.3.4"
+  spec.add_dependency "yajl-ruby", "~> 1.2.1"
+  spec.add_dependency "redis", "~> 3.2.1"
+  spec.add_dependency "hiredis", "~> 0.6.0"
+  spec.add_dependency "ruby-graphviz", "~> 1.2.2"
+  spec.add_dependency "terminal-table", "~> 1.4.5"
+  spec.add_dependency "colorize", "~> 0.7.7"
+  spec.add_dependency "thor", "~> 0.19.1"
+  spec.add_dependency "launchy", "~> 2.4.3"
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", '~> 3.0.0'

data/lib/gush/cli/overview.rb

@@ -0,0 +1,138 @@
+module Gush
+  class CLI
+    class Overview
+      attr_reader :workflow
+
+      def initialize(workflow)
+        @workflow = workflow
+      end
+
+      def table
+        Terminal::Table.new(rows: rows)
+      end
+
+      def status
+        if workflow.failed?
+          failed_status
+        elsif workflow.running?
+          running_status
+        elsif workflow.finished?
+          "done".green
+        elsif workflow.stopped?
+          "stopped".red
+        else
+          "pending".light_white
+        end
+      end
+
+      def jobs_list(jobs)
+        "\nJobs list:\n".tap do |output|
+          jobs_by_type(jobs).each do |job|
+            output << job_to_list_element(job)
+          end
+        end
+      end
+
+      private
+      def rows
+        [].tap do |rows|
+          columns.each_pair do |name, value|
+            rows << [{alignment: :center, value: name}, value]
+            rows << :separator if name != "Status"
+          end
+        end
+      end
+
+      def columns
+        {
+          "ID" => workflow.id,
+          "Name" => workflow.class.to_s,
+          "Jobs" => workflow.jobs.count,
+          "Failed jobs" => failed_jobs_count.red,
+          "Succeeded jobs" => succeeded_jobs_count.green,
+          "Enqueued jobs" => enqueued_jobs_count.yellow,
+          "Running jobs" => running_jobs_count.blue,
+          "Remaining jobs" => remaining_jobs_count,
+          "Status" => status
+        }
+      end
+
+      def running_status
+        finished = succeeded_jobs_count.to_i
+        status = "running".yellow
+        status += "\n#{finished}/#{total_jobs_count} [#{(finished*100)/total_jobs_count}%]"
+      end
+
+      def failed_status
+        status = "failed".light_red
+        status += "\n#{failed_job} failed"
+      end
+
+      def job_to_list_element(job)
+        name = job.name
+        case
+        when job.failed?
+          "[✗] #{name.red} \n"
+        when job.finished?
+          "[✓] #{name.green} \n"
+        when job.enqueued?
+          "[•] #{name.yellow} \n"
+        when job.running?
+          "[•] #{name.blue} \n"
+        else
+          "[ ] #{name} \n"
+        end
+      end
+
+      def jobs_by_type(type)
+        return sorted_jobs if type == :all
+        jobs.select{|j| j.public_send("#{type}?") }
+      end
+
+      def sorted_jobs
+        workflow.jobs.sort_by do |job|
+          case
+          when job.failed?
+            0
+          when job.finished?
+            1
+          when job.enqueued?
+            2
+          when job.running?
+            3
+          else
+            4
+          end
+        end
+      end
+
+      def failed_job
+        workflow.jobs.find(&:failed).name
+      end
+
+      def total_jobs_count
+        workflow.jobs.count
+      end
+
+      def failed_jobs_count
+        workflow.jobs.count(&:failed?).to_s
+      end
+
+      def succeeded_jobs_count
+        workflow.jobs.count(&:succeeded?).to_s
+      end
+
+      def enqueued_jobs_count
+        workflow.jobs.count(&:enqueued?).to_s
+      end
+
+      def running_jobs_count
+        workflow.jobs.count(&:running?).to_s
+      end
+
+      def remaining_jobs_count
+        workflow.jobs.count{|j| [j.finished, j.failed, j.enqueued].none? }.to_s
+      end
+    end
+  end
+end

data/lib/gush/cli.rb

@@ -22,6 +22,7 @@ module Gush
         config.namespace   = options.fetch("namespace",   config.namespace)
         config.environment = options.fetch("environment", config.environment)
       end
+      load_gushfile
     end
 
     desc "create [WorkflowClass]", "Registers new workflow"
@@ -29,18 +30,12 @@ module Gush
       workflow = client.create_workflow(name)
       puts "Workflow created with id: #{workflow.id}"
       puts "Start it with command: gush start #{workflow.id}"
-    rescue
-      puts "Workflow not found."
     end
 
     desc "start [workflow_id]", "Starts Workflow with given ID"
     def start(*args)
       id = args.shift
       client.start_workflow(id, args)
-    rescue WorkflowNotFound
-      puts "Workflow not found."
-    rescue DependencyLevelTooDeep
-      puts "Dependency level too deep. Perhaps you have a dependency cycle?"
     end
 
     desc "create_and_start [WorkflowClass]", "Create and instantly start the new workflow"
@@ -48,18 +43,12 @@ module Gush
       workflow = client.create_workflow(name)
       client.start_workflow(workflow.id, args)
       puts "Created and started workflow with id: #{workflow.id}"
-    rescue WorkflowNotFound
-      puts "Workflow not found."
-    rescue DependencyLevelTooDeep
-      puts "Dependency level too deep. Perhaps you have a dependency cycle?"
     end
 
     desc "stop [workflow_id]", "Stops Workflow with given ID"
     def stop(*args)
       id = args.shift
       client.stop_workflow(id)
-    rescue WorkflowNotFound
-      puts "Workflow not found."
     end
 
     desc "clear", "Clears all jobs from Sidekiq queue"
@@ -77,16 +66,12 @@ module Gush
       display_overview_for(workflow) unless options[:skip_overview]
 
       display_jobs_list_for(workflow, options[:jobs]) unless options[:skip_jobs]
-    rescue WorkflowNotFound
-      puts "Workflow not found."
     end
 
     desc "rm [workflow_id]", "Delete workflow with given ID"
     def rm(workflow_id)
       workflow = client.find_workflow(workflow_id)
       client.destroy_workflow(workflow)
-    rescue WorkflowNotFound
-      puts "Workflow not found."
     end
 
     desc "list", "Lists all workflows with their statuses"
@@ -112,44 +97,10 @@ module Gush
     desc "viz [WorkflowClass]", "Displays graph, visualising job dependencies"
     def viz(name)
       client
-      workflow = name.constantize.new("start")
-      GraphViz.new(:G, type: :digraph, dpi: 200, compound: true) do |g|
-        g[:compound] = true
-        g[:rankdir] = "LR"
-        g[:center] = true
-        g.node[:shape] = "ellipse"
-        g.node[:style] = "filled"
-        g.node[:color] = "#555555"
-        g.node[:fillcolor] = "white"
-        g.edge[:dir] = "forward"
-        g.edge[:penwidth] = 1
-        g.edge[:color] = "#555555"
-        start = g.start(shape: 'diamond', fillcolor: '#CFF09E')
-        end_node = g.end(shape: 'diamond', fillcolor: '#F56991')
-
-
-        workflow.nodes.each do |job|
-          name = job.class.to_s
-          g.add_nodes(name)
-
-          if job.incoming.empty?
-            g.add_edges(start, name)
-          end
-
-
-          if job.outgoing.empty?
-            g.add_edges(name, end_node)
-          else
-            job.outgoing.each do |out|
-              g.add_edges(name, out)
-            end
-          end
-        end
-
-        g.output(png: Pathname.new(Dir.tmpdir).join("graph.png"))
-      end
-
-      Launchy.open(Pathname.new(Dir.tmpdir).join("graph.png").to_s)
+      workflow = name.constantize.new
+      graph = Graph.new(workflow)
+      graph.viz
+      Launchy.open graph.path
     end
 
     private
@@ -158,88 +109,35 @@ module Gush
       @client ||= Client.new
     end
 
-    def display_overview_for(workflow)
-      rows = []
-      columns  = {
-        "id" => workflow.id,
-        "name" => workflow.class.to_s,
-        "jobs" => workflow.nodes.count,
-        "failed jobs" => workflow.nodes.count(&:failed?).to_s.red,
-        "succeeded jobs" => workflow.nodes.count(&:succeeded?).to_s.green,
-        "enqueued jobs" => workflow.nodes.count(&:enqueued?).to_s.yellow,
-        "running jobs" => workflow.nodes.count(&:running?).to_s.blue,
-        "remaining jobs" => workflow.nodes.count{|j| [j.finished, j.failed, j.enqueued].all? {|b| !b} },
-        "status" => status_for(workflow)
-      }
-
-      columns.each_pair do |name, value|
-        rows << [{alignment: :center, value: name}, value]
-        rows << :separator if name != "status"
-      end
+    def overview(workflow)
+      CLI::Overview.new(workflow)
+    end
 
-      puts Terminal::Table.new(rows: rows)
+    def display_overview_for(workflow)
+      puts overview(workflow).table
     end
 
     def status_for(workflow)
-      if workflow.failed?
-        status = "failed".light_red
-        status += "\n#{workflow.nodes.find(&:failed).name} failed"
-      elsif workflow.running?
-        status = "running".yellow
-        finished = workflow.nodes.count {|job| job.finished }
-        total = workflow.nodes.count
-        status += "\n#{finished}/#{total} [#{(finished*100)/total}%]"
-      elsif workflow.finished?
-        status = "done".green
-      elsif workflow.stopped?
-        status = "stopped".red
-      else
-        status = "pending".light_white
-      end
+      overview(workflow).status
     end
 
     def display_jobs_list_for(workflow, jobs)
-      puts "\nJobs list:\n"
-
-      jobs_by_type(workflow, jobs).each do |job|
-        name = job.name
-        puts case
-        when job.failed?
-          "[✗] #{name.red}"
-        when job.finished?
-          "[✓] #{name.green}"
-        when job.enqueued?
-          "[•] #{name.yellow}"
-        when job.running?
-          "[•] #{name.blue}"
-        else
-          "[ ] #{name}"
-        end
-      end
-    end
-
-    def jobs_by_type(workflow, type)
-      jobs = workflow.nodes.sort_by do |job|
-        case
-        when job.failed?
-          0
-        when job.finished?
-          1
-        when job.enqueued?
-          2
-        when job.running?
-          3
-        else
-          4
-        end
-      end
-
-      jobs.select!{|j| j.public_send("#{type}?") } unless type == :all
-      jobs
+      puts overview(workflow).jobs_list(jobs)
     end
 
     def gushfile
       Pathname.pwd.join(options[:gushfile])
     end
+
+    def load_gushfile
+      file = client.configuration.gushfile
+      if !@gushfile.exist?
+        raise Thor::Error, "#{file} not found, please add it to your project".colorize(:red)
+      end
+
+      require file
+    rescue LoadError
+      raise Thor::Error, "failed to require #{file}".colorize(:red)
+    end
   end
 end