gush 0.0.1 → 0.1

data/lib/gush/workflow.rb

@@ -1,46 +1,61 @@
 require 'securerandom'
-require 'gush/metadata'
 
 module Gush
   class Workflow
-    include Gush::Metadata
+    attr_accessor :id, :jobs, :stopped, :persisted
 
-    attr_accessor :id, :nodes, :stopped
-
-    def initialize(id, options = {})
+    def initialize(should_run_configure = true)
       @id = id
-      @nodes = []
+      @jobs = []
       @dependencies = []
-      @logger_builder = default_logger_builder
+      @persisted = false
       @stopped = false
 
-      unless options[:configure] == false
+      if should_run_configure
         configure
         create_dependencies
       end
     end
 
-    def default_logger_builder
-      LoggerBuilder
+    def self.find(id)
+      Gush::Client.new.find_workflow(id)
+    end
+
+    def self.create(*args)
+      flow = new(*args)
+      flow.save
+      flow
+    end
+
+    def save
+      if @id.nil?
+        assign_id
+      end
+
+      client.persist_workflow(self)
     end
 
     def configure
     end
 
-    def stop!
+    def mark_as_stopped
       @stopped = true
     end
 
     def start!
-      @stopped = false
+      client.start_workflow(self)
+    end
+
+    def persist!
+      client.persist_workflow(self)
     end
 
-    def logger_builder(klass)
-      @logger_builder = klass
+    def mark_as_persisted
+      @persisted = true
     end
 
-    def build_logger_for_job(job, jid)
-      @logger_builder.new(self, job, jid).build
+    def mark_as_started
+      @stopped = false
     end
 
     def create_dependencies
@@ -54,19 +69,19 @@ module Gush
     end
 
     def find_job(name)
-      @nodes.find { |node| node.name == name.to_s || node.class.to_s == name.to_s }
+      @jobs.find { |node| node.name == name.to_s || node.class.to_s == name.to_s }
     end
 
     def finished?
-      nodes.all?(&:finished)
+      jobs.all?(&:finished?)
     end
 
     def running?
-      nodes.any? {|j| j.enqueued? || j.running? } && !stopped?
+      !stopped? && jobs.any? {|j| j.enqueued? || j.running? }
     end
 
     def failed?
-      nodes.any?(&:failed)
+      jobs.any?(&:failed?)
     end
 
     def stopped?
@@ -74,8 +89,8 @@ module Gush
     end
 
     def run(klass, deps = {})
-      node = klass.new(name: klass.to_s)
-      @nodes << node
+      node = klass.new(self, name: klass.to_s)
+      @jobs << node
 
       deps_after = [*deps[:after]]
       deps_after.each do |dep|
@@ -88,18 +103,26 @@ module Gush
       end
     end
 
+    def reload
+      self.class.find(@id)
+    end
+
+    def initial_jobs
+      jobs.select(&:has_no_dependencies?)
+    end
+
     def status
       case
         when failed?
-          "Failed"
+          :failed
         when running?
-          "Running"
+          :running
         when finished?
-          "Finished"
+          :finished
         when stopped?
-          "Stopped"
+          :stopped
         else
-          "Pending"
+          :pending
       end
     end
 
@@ -116,26 +139,19 @@ module Gush
       {
         name: name,
         id: @id,
-        total: @nodes.count,
-        finished: @nodes.count(&:finished?),
+        total: @jobs.count,
+        finished: @jobs.count(&:finished?),
         klass: name,
-        nodes: @nodes.map(&:as_json),
+        jobs: @jobs.map(&:as_json),
         status: status,
         stopped: stopped,
         started_at: started_at,
-        finished_at: finished_at,
-        logger_builder: @logger_builder.to_s
+        finished_at: finished_at
       }
     end
 
     def to_json(options = {})
-      JSON.dump(to_hash)
-    end
-
-    def next_jobs
-      @nodes.select do |job|
-        job.can_be_started?(self)
-      end
+      Gush::JSON.encode(to_hash)
     end
 
     def self.descendants
@@ -143,12 +159,21 @@ module Gush
     end
 
     private
+
+    def assign_id
+      @id = client.next_free_id
+    end
+
+    def client
+      @client ||= Client.new
+    end
+
     def first_job
-      nodes.min_by{ |n| n.started_at || Time.now.to_i }
+      jobs.min_by{ |n| n.started_at || Time.now.to_i }
     end
 
     def last_job
-      nodes.max_by{ |n| n.finished_at || 0 } if nodes.all?(&:finished?)
+      jobs.max_by{ |n| n.finished_at || 0 } if jobs.all?(&:finished?)
     end
   end
 end

data/lib/gush.rb

@@ -7,14 +7,14 @@ require "redis"
 require "securerandom"
 require "sidekiq"
 
+require "gush/json"
 require "gush/cli"
+require "gush/cli/overview"
+require "gush/graph"
 require "gush/client"
 require "gush/configuration"
 require "gush/errors"
 require "gush/job"
-require "gush/logger_builder"
-require "gush/metadata"
-require "gush/null_logger"
 require "gush/version"
 require "gush/worker"
 require "gush/workflow"

data/spec/features/workflows_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+
+describe "Workflows" do
+  it "runs the whole workflow in proper order" do
+    flow = TestWorkflow.create
+    flow.start!
+
+    expect(Gush::Worker).to have_jobs(flow.id, ["Prepare"])
+
+    Gush::Worker.perform_one
+    expect(Gush::Worker).to have_jobs(flow.id, ["FetchFirstJob", "FetchSecondJob"])
+
+    Gush::Worker.perform_one
+    expect(Gush::Worker).to have_jobs(flow.id, ["FetchSecondJob", "PersistFirstJob"])
+
+    Gush::Worker.perform_one
+    expect(Gush::Worker).to have_jobs(flow.id, ["PersistFirstJob", "NormalizeJob"])
+
+    Gush::Worker.perform_one
+    expect(Gush::Worker).to have_jobs(flow.id, ["NormalizeJob"])
+
+    Gush::Worker.perform_one
+
+    expect(Gush::Worker.jobs).to be_empty
+
+    flow = flow.reload
+    expect(flow).to be_finished
+    expect(flow).to_not be_failed
+  end
+end

data/spec/lib/gush/client_spec.rb

@@ -1,12 +1,16 @@
 require 'spec_helper'
 
 describe Gush::Client do
+  let(:client) do
+    Gush::Client.new(Gush::Configuration.new(gushfile: GUSHFILE, redis_url: REDIS_URL))
+  end
+
   describe "#find_workflow" do
     context "when workflow doesn't exist" do
       it "returns raises WorkflowNotFound" do
         expect {
           client.find_workflow('nope')
-        }.to raise_error(WorkflowNotFound)
+        }.to raise_error(Gush::WorkflowNotFound)
       end
     end
 
@@ -17,57 +21,54 @@ describe Gush::Client do
         workflow = client.find_workflow(expected_workflow.id)
 
         expect(workflow.id).to eq(expected_workflow.id)
-        expect(workflow.nodes.map(&:name)).to match_array(expected_workflow.nodes.map(&:name))
+        expect(workflow.jobs.map(&:name)).to match_array(expected_workflow.jobs.map(&:name))
       end
     end
   end
 
   describe "#start_workflow" do
     it "enqueues next jobs from the workflow" do
-      id = SecureRandom.uuid
-      workflow = TestWorkflow.new(id)
+      workflow = TestWorkflow.new
       client.persist_workflow(workflow)
       expect {
-        client.start_workflow(id)
+        client.start_workflow(workflow)
       }.to change{Gush::Worker.jobs.count}.from(0).to(1)
     end
 
     it "removes stopped flag when the workflow is started" do
-      id = SecureRandom.uuid
-      workflow = TestWorkflow.new(id)
-      workflow.stop!
+      workflow = TestWorkflow.new
+      workflow.mark_as_stopped
       client.persist_workflow(workflow)
       expect {
-        client.start_workflow(id)
-      }.to change{client.find_workflow(id).stopped?}.from(true).to(false)
+        client.start_workflow(workflow)
+      }.to change{client.find_workflow(workflow.id).stopped?}.from(true).to(false)
     end
 
     it "marks the enqueued jobs as enqueued" do
-      id = SecureRandom.uuid
-      workflow = TestWorkflow.new(id)
+      workflow = TestWorkflow.new
       client.persist_workflow(workflow)
-      client.start_workflow(id)
-      job = client.find_workflow(id).find_job("Prepare")
+      client.start_workflow(workflow)
+      job = workflow.reload.find_job("Prepare")
       expect(job.enqueued?).to eq(true)
     end
   end
 
   describe "#stop_workflow" do
     it "marks the workflow as stopped" do
-      id = SecureRandom.uuid
-      workflow = TestWorkflow.new(id)
+      workflow = TestWorkflow.new
       client.persist_workflow(workflow)
       expect {
-        client.stop_workflow(id)
-      }.to change{client.find_workflow(id).stopped?}.from(false).to(true)
+        client.stop_workflow(workflow.id)
+      }.to change{client.find_workflow(workflow.id).stopped?}.from(false).to(true)
     end
   end
 
   describe "#persist_workflow" do
     it "persists JSON dump of the Workflow and its jobs" do
       job = double("job", to_json: 'json')
-      workflow = double("workflow", id: 'abcd', nodes: [job, job, job], to_json: '"json"')
+      workflow = double("workflow", id: 'abcd', jobs: [job, job, job], to_json: '"json"')
       expect(client).to receive(:persist_job).exactly(3).times.with(workflow.id, job)
+      expect(workflow).to receive(:mark_as_persisted)
       client.persist_workflow(workflow)
       expect(redis.keys("gush.workflows.abcd").length).to eq(1)
     end
@@ -75,16 +76,15 @@ describe Gush::Client do
 
   describe "#destroy_workflow" do
     it "removes all Redis keys related to the workflow" do
-      id = SecureRandom.uuid
-      workflow = TestWorkflow.new(id)
+      workflow = TestWorkflow.new
       client.persist_workflow(workflow)
-      expect(redis.keys("gush.workflows.#{id}").length).to eq(1)
-      expect(redis.keys("gush.jobs.#{id}.*").length).to eq(5)
+      expect(redis.keys("gush.workflows.#{workflow.id}").length).to eq(1)
+      expect(redis.keys("gush.jobs.#{workflow.id}.*").length).to eq(5)
 
       client.destroy_workflow(workflow)
 
-      expect(redis.keys("gush.workflows.#{id}").length).to eq(0)
-      expect(redis.keys("gush.jobs.#{id}.*").length).to eq(0)
+      expect(redis.keys("gush.workflows.#{workflow.id}").length).to eq(0)
+      expect(redis.keys("gush.jobs.#{workflow.id}.*").length).to eq(0)
     end
   end
 
@@ -106,20 +106,17 @@ describe Gush::Client do
   end
 
   it "should be able to handle outdated data format" do
-    workflow_id = SecureRandom.uuid
-    workflow = TestWorkflow.new(workflow_id)
+    workflow = TestWorkflow.new
     client.persist_workflow(workflow)
 
     # malform the data
-    hash = Yajl::Parser.parse(redis.get("gush.workflows.#{workflow_id}"), symbolize_keys: true)
-    hash.delete(:logger_builder)
+    hash = Gush::JSON.decode(redis.get("gush.workflows.#{workflow.id}"), symbolize_keys: true)
     hash.delete(:stopped)
-    redis.set("gush.workflows.#{workflow_id}", Yajl::Encoder.new.encode(hash))
+    redis.set("gush.workflows.#{workflow.id}", Gush::JSON.encode(hash))
 
     expect {
-      workflow = client.find_workflow(workflow_id)
+      workflow = client.find_workflow(workflow.id)
       expect(workflow.stopped?).to be false
-      expect(workflow.instance_variable_get(:@logger_builder)).to be Gush::LoggerBuilder
     }.not_to raise_error
   end
 end

data/spec/lib/gush/job_spec.rb

@@ -7,10 +7,10 @@ describe Gush::Job do
       job = described_class.new(name: "a-job")
       job.fail!
       expect(job.failed_at).to eq(Time.now.to_i)
-      expect(job.failed).to eq(true)
-      expect(job.finished).to eq(true)
-      expect(job.running).to eq(false)
-      expect(job.enqueued).to eq(false)
+      expect(job.failed?).to eq(true)
+      expect(job.finished?).to eq(true)
+      expect(job.running?).to eq(false)
+      expect(job.enqueued?).to eq(false)
     end
   end
 
@@ -19,10 +19,10 @@ describe Gush::Job do
       job = described_class.new(name: "a-job")
       job.finish!
       expect(job.finished_at).to eq(Time.now.to_i)
-      expect(job.failed).to eq(false)
-      expect(job.running).to eq(false)
-      expect(job.finished).to eq(true)
-      expect(job.enqueued).to eq(false)
+      expect(job.failed?).to eq(false)
+      expect(job.running?).to eq(false)
+      expect(job.finished?).to eq(true)
+      expect(job.enqueued?).to eq(false)
     end
   end
 
@@ -35,28 +35,27 @@ describe Gush::Job do
       expect(job.started_at).to eq(nil)
       expect(job.finished_at).to eq(nil)
       expect(job.failed_at).to eq(nil)
-      expect(job.failed).to eq(false)
-      expect(job.finished).to eq(false)
-      expect(job.enqueued).to eq(true)
-      expect(job.running).to eq(false)
+      expect(job.failed?).to eq(false)
+      expect(job.finished?).to eq(false)
+      expect(job.enqueued?).to eq(true)
+      expect(job.running?).to eq(false)
     end
   end
 
   describe "#start!" do
-    it "resets flags to false and sets running to true" do
+    it "resets flags and marks as running" do
       job = described_class.new(name: "a-job")
-      job.enqueue!
       job.start!
       expect(job.started_at).to eq(Time.now.to_i)
-      expect(job.enqueued).to eq(false)
-      expect(job.running).to eq(true)
+      expect(job.enqueued?).to eq(false)
+      expect(job.running?).to eq(true)
     end
   end
 
   describe "#as_json" do
     context "finished and enqueued set to true" do
       it "returns correct hash" do
-        job = described_class.new(name: "a-job", finished: true, enqueued: true)
+        job = described_class.new(double('flow'), name: "a-job", finished_at: 123, enqueued_at: 120)
         expected = {
           name: "a-job",
           klass: "Gush::Job",
@@ -67,7 +66,8 @@ describe Gush::Job do
           outgoing: [],
           failed_at: nil,
           started_at: nil,
-          finished_at: nil,
+          finished_at: 123,
+          enqueued_at: 120,
           running: false
         }
         expect(job.as_json).to eq(expected)
@@ -77,38 +77,34 @@ describe Gush::Job do
 
   describe ".from_hash" do
     it "properly restores state of the job from hash" do
-      job = described_class.from_hash({
-        klass: 'Gush::Job',
-        name: 'gob',
-        finished: true,
-        failed: true,
-        enqueued: true,
-        incoming: ['a', 'b'],
-        outgoing: ['c'],
-        failed_at: 123,
-        finished_at: 122,
-        started_at: 55
-      })
+      job = described_class.from_hash(
+        double('flow'),
+        {
+          klass: 'Gush::Job',
+          name: 'gob',
+          finished: true,
+          failed: true,
+          enqueued: true,
+          incoming: ['a', 'b'],
+          outgoing: ['c'],
+          failed_at: 123,
+          finished_at: 122,
+          started_at: 55,
+          enqueued_at: 444
+        }
+      )
 
       expect(job.name).to eq('gob')
       expect(job.class).to eq(Gush::Job)
-      expect(job.finished).to eq(true)
-      expect(job.failed).to eq(true)
-      expect(job.enqueued).to eq(true)
+      expect(job.finished?).to eq(true)
+      expect(job.failed?).to eq(true)
+      expect(job.enqueued?).to eq(true)
       expect(job.incoming).to eq(['a', 'b'])
       expect(job.outgoing).to eq(['c'])
       expect(job.failed_at).to eq(123)
       expect(job.finished_at).to eq(122)
       expect(job.started_at).to eq(55)
-    end
-  end
-
-  describe "#logger" do
-    it "returns a logger for the job" do
-      job = described_class.new(name: "a-job", finished: true, running: true)
-      job.logger = TestLoggerBuilder.new(:workflow, job, 1234).build
-      expect(job.logger).to be_a TestLogger
-      expect(job.logger.name).to eq(job.name)
+      expect(job.enqueued_at).to eq(444)
     end
   end
 end

data/spec/lib/gush/worker_spec.rb

@@ -1,96 +1,67 @@
 require 'spec_helper'
 
 describe Gush::Worker do
-  let(:workflow_id) { '1234'                        }
-  let(:workflow)    { TestWorkflow.new(workflow_id) }
+  subject { described_class.new }
+
+  let!(:workflow)   { TestWorkflow.create }
   let(:job)         { workflow.find_job("Prepare")  }
-  let(:config)      { client.configuration.to_json  }
+  let(:config)      { Gush.configuration.to_json  }
+  let!(:client)     { double("client") }
 
   before :each do
-    allow(client).to receive(:find_workflow).with(workflow_id).and_return(workflow)
-    allow(Gush::Client).to receive(:new).and_return(client)
+    allow(subject).to receive(:client).and_return(client)
+    allow(subject).to receive(:enqueue_outgoing_jobs)
+
+    allow(client).to receive(:find_workflow).with(workflow.id).and_return(workflow)
+    expect(client).to receive(:persist_job).at_least(1).times
+    expect(client).to receive(:worker_report).with(hash_including(status: :started)).ordered
   end
 
   describe "#perform" do
     context "when job fails" do
-      before :each do
-        expect(job).to receive(:work).and_raise(StandardError)
-        job.enqueue!
-        job.start!
-      end
-
       it "should mark it as failed" do
-        allow(client).to receive(:persist_job)
-        Gush::Worker.new.perform(workflow_id, "Prepare", config)
-
-        expect(client).to have_received(:persist_job).with(workflow_id, job).at_least(1).times do |_, job|
-          expect(job).to be_failed
-        end
+        allow(job).to receive(:work).and_raise(StandardError)
+        expect(client).to receive(:worker_report).with(hash_including(status: :failed)).ordered
 
+        subject.perform(workflow.id, "Prepare", config)
+        expect(workflow.find_job("Prepare")).to be_failed
       end
 
       it "reports that job failed" do
-        allow(client).to receive(:worker_report)
-        Gush::Worker.new.perform(workflow_id, "Prepare", config)
-        expect(client).to have_received(:worker_report).with(hash_including(status: :failed))
-      end
+        allow(job).to receive(:work).and_raise(StandardError)
+        expect(client).to receive(:worker_report).with(hash_including(status: :failed)).ordered
 
-      it "logs the exception" do
-        logger = TestLogger.new(1234, 'Prepare')
-        expect(logger).to receive(:<<).with(instance_of(String)).at_least(1).times
-        expect(workflow).to receive(:build_logger_for_job).and_return(logger)
-
-        Gush::Worker.new.perform(workflow_id, "Prepare", config)
+        subject.perform(workflow.id, "Prepare", config)
       end
     end
 
     context "when job completes successfully" do
       it "should mark it as succedeed" do
-        allow(client).to receive(:persist_job)
-
-        Gush::Worker.new.perform(workflow_id, "Prepare", config)
+        expect(subject).to receive(:mark_as_finished)
+        expect(client).to receive(:worker_report).with(hash_including(status: :finished)).ordered
 
-        expect(client).to have_received(:persist_job).at_least(1).times.with(workflow_id, job) do |_, job|
-          expect(job).to be_succeeded
-        end
+        subject.perform(workflow.id, "Prepare", config)
       end
 
       it "reports that job succedeed" do
-        allow(client).to receive(:worker_report)
-        Gush::Worker.new.perform(workflow_id, "Prepare", config)
+        expect(client).to receive(:worker_report).with(hash_including(status: :finished)).ordered
 
-        expect(client).to have_received(:worker_report).with(hash_including(status: :finished))
+        subject.perform(workflow.id, "Prepare", config)
       end
     end
 
-    [:before_work, :work, :after_work].each do |method|
-      it "calls job.#{method} hook" do
-        expect(job).to receive(method)
-        Gush::Worker.new.perform(workflow_id, "Prepare", config)
-      end
-    end
+    it "calls job.work method" do
+      expect(job).to receive(:work)
+      expect(client).to receive(:worker_report).with(hash_including(status: :finished)).ordered
 
-    it "sets up a logger for the job" do
-      Gush::Worker.new.perform(workflow_id, "Prepare", config)
-      job.start!
-      expect(job.logger).to be_a TestLogger
-    end
-
-    it "sets a job id" do
-      job_id = 1234
-      worker = Gush::Worker.new
-
-      allow(worker).to receive(:jid).and_return(job_id)
-
-      worker.perform(workflow_id, "Prepare", config)
-      job.enqueue!
-      expect(job.jid).to eq job_id
+      subject.perform(workflow.id, "Prepare", config)
     end
 
     it "reports when the job is started" do
       allow(client).to receive(:worker_report)
-      Gush::Worker.new.perform(workflow_id, "Prepare", config)
-      expect(client).to have_received(:worker_report).with(hash_including(status: :started))
+      expect(client).to receive(:worker_report).with(hash_including(status: :finished)).ordered
+
+      subject.perform(workflow.id, "Prepare", config)
     end
   end
 end