octopusci 0.2.3 → 0.3.0

data/lib/octopusci/job.rb

@@ -1,16 +1,141 @@
 module Octopusci
   class Job
+    class << self
+      attr_accessor :context_stack
+
+      def inherited(subclass)
+        subclass.context_stack = []
+      end
+    end
+
     def self.run(job_rec)
-      raise PureVirtualMethod, "The self.commit_run method needs to be defined on your Octopusci::Job."
+      raise Octopusci::PureVirtualMethod, "The self.run method needs to be defined on your Octopusci::Job based class."
+    end
+
+    def self.context(desc_str)
+      num_contexts_before = context_stack.length
+      context_lead_in = '  '*num_contexts_before
+      context_stack.push(desc_str)
+      num_contexts_now = context_stack.length
+      @output_lead_in = '  '*num_contexts_now
+      context_str = context_stack.join(' ')
+      output("\n\n#{desc_str}:\n\n")
+      begin
+        yield
+        Octopusci::Notifier.job_complete(@job, get_recip_email(), context_str, true).deliver
+      rescue Octopusci::JobRunFailed => e
+        write_exception(e)
+        Octopusci::Notifier.job_complete(@job, get_recip_email(), context_str, false).deliver
+        # setup notification and send notification
+        raise JobHalted.new("JobRunFailed:\n  Context: #{context_str}\n  Message: #{e.message}")
+      ensure
+        context_stack.pop
+      end
+    end
+
+    def self.run_shell_cmd(cmd_str, output_to_log=false)
+      horiz_line = "-"*30
+      @io.write_raw_output(output_to_log) do |out_f|
+        out_f << "\n\n#{@output_lead_in}Running: #{cmd_str}\n"
+        out_f << "#{@output_lead_in}#{horiz_line}"
+        out_f << "\n"
+        out_f.flush
+      
+        in_f = ::IO.popen(cmd_str)
+        while(cur_line = in_f.gets) do
+          out_f << "#{@output_lead_in}#{cur_line}"
+          out_f.flush
+        end
+
+        in_f.close
+      end
+
+      return $?.exitstatus.to_i
+    end
+
+    def self.run_shell_cmd!(cmd_str, output_to_log=false)
+      r = self.run_shell_cmd(cmd_str, output_to_log)
+      if (r != 0)
+        raise Octopusci::JobRunFailed.new("#{cmd_str} exited with non-zero return value (#{r})")
+      else
+        return 0
+      end
+    end
+
+    def self.failed!(msg = "")
+      raise Octopusci::JobRunFailed.new(msg)
+    end
+
+    def self.output(msg)
+      @io.write_raw_output(false, "#{@output_lead_in}#{msg}")
+    end
+
+    def self.log(msg)
+      @io.write_raw_output(true) do |f|
+        f << "\n#{context_stack.join(' ')}:\n\t#{msg}" 
+      end
+    end
+
+    def self.write_exception(e)
+      @io.write_raw_output do |f|
+        f << "\n\nException: #{e.message}\n"
+        f << "-"*30
+        f << "\n"
+        f << e.backtrace.join("\n")
+      end
+    end
+
+    def self.record_start(stage)
+      @job['started_at'] = Time.new
+      @job['stage'] = stage
+      @job['status'] = 'running'
+      Octopusci::JobStore.set(@job['id'], @job)
+    end
+
+    def self.workspace_path
+      return "#{Octopusci::Config['general']['workspace_base_path']}/#{@job['stage']}"
+    end
+
+    def self.repository_path
+      return "#{workspace_path}/#{@job['repo_name']}"
+    end
+
+    def self.clone_code(job_conf)
+      if File.directory?(repository_path)
+        return 0
+      else
+        if !Dir.exists?(workspace_path)
+          FileUtils.mkdir_p(workspace_path)
+        end
+        return run_shell_cmd("cd #{workspace_path} 2>&1 && git clone #{job_conf['repo_uri']} #{@job['repo_name']} 2>&1", true)
+      end
+    end
+
+    def self.checkout_branch(job_conf)
+      return run_shell_cmd("cd #{repository_path} 2>&1 && git fetch --all -p 2>&1 && git checkout #{@job['branch_name']} 2>&1 && git pull -f origin #{@job['branch_name']}:#{@job['branch_name']} 2>&1", true)
+    end
+
+    def self.get_recip_email
+      recip_email = nil
+      if @job['branch_name'] == 'master'
+        recip_email = @job_conf['default_email']
+      else
+        if @job['payload']['pusher']['email']
+          recip_email = @job['payload']['pusher']['email']
+        else
+          recip_email = @job_conf['default_email']
+        end
+      end
     end
-    
+
     def self.perform(project_name, branch_name, job_id, job_conf)
-      ActiveRecord::Base.verify_active_connections!
+      context_stack = []
+      @job_conf = job_conf
 
       # Note: There is no logic for handling stage coming back as nil because
       # it should never happen because there are the same number of resque
       # workers as there are stages at all times.
-      if Octopusci::CONFIG.has_key?('stages')
+      if Octopusci::Config.has_key?('stages')
         # Get the next available stage from redis which locks it by removing it
         # from the list of available
         stage = Octopusci::StageLocker.pop
@@ -20,44 +145,30 @@ module Octopusci
         # Using redis to get the associated github_payload
         github_payload = Octopusci::Queue.github_payload(project_name, branch_name)
         
-        job = ::Job.where("jobs.repo_name = ? && jobs.ref = ?", github_payload['repository']['name'], github_payload['ref']).order('jobs.created_at DESC').first
-        if job
-          job.started_at = Time.new
-          job.stage = stage
-          job.status = 'running'
-          job.save
+        @job = Octopusci::JobStore.list_repo_branch(github_payload['repository']['name'], github_payload['ref'].split('/').last, 0, 1).first
+        if @job
+          @io = Octopusci::IO.new(@job)
+          record_start(stage)
           
           begin
-            job.clone_code(job_conf)
-            job.checkout_branch(job_conf)
+            clone_code(job_conf)
+            checkout_branch(job_conf)
             
-            rv = self.run(job)
-            if ::Job::STATUS.keys.include?(rv)
-              job.status = 0
-            else
-              if rv == 0
-                job.status = 'successful'
-              else
-                job.status = 'failed'
-              end
-            end
+            self.run(@job)
+
+            @job['status'] = 'successful'
+          rescue JobHalted => e
+            @job['status'] = 'failed'
           rescue => e
-            File.open(job.abs_output_file_path, 'a') { |f|
-              f << "\n\nException: #{e.message}\n"
-              f << "-"*30
-              f << "\n"
-              f << e.backtrace.join("\n")
-            }
-            job.status = 'error'
+            write_exception(e)
+            @job['status'] = 'error'
           ensure
-            job.ended_at = Time.new
-            job.save
-            
-            Octopusci::Notifier.job_complete(job, job_conf, job.successful?)
+            @job['ended_at'] = Time.new
+            Octopusci::JobStore.set(@job['id'], @job)
           end
         end
       ensure
-        if Octopusci::CONFIG.has_key?('stages')
+        if Octopusci::Config.has_key?('stages')
           # Unlock the stage by adding it back to the list of available stages
           Octopusci::StageLocker.push(stage)
         end

data/lib/octopusci/job_store.rb

@@ -0,0 +1,67 @@
+module Octopusci
+  class JobStore
+    def self.prepend(job)
+      job_id = redis.incr('octopusci:job_count')
+      self.set(job_id, job.merge({ 'id' => job_id }))
+      redis.lpush("octopusci:jobs", job_id)
+      redis.lpush("octopusci:#{job['repo_name']}:#{job['ref'].split('/').last}:jobs", job_id)
+      return job_id
+    end
+
+    def self.set(job_id, job)
+      redis.set("octopusci:jobs:#{job_id}", YAML.dump(job))
+    end
+
+    def self.get(job_id)
+      job = redis.get("octopusci:jobs:#{job_id}")
+      if job
+        return YAML.load(job)
+      end
+      return nil
+    end
+
+    def self.size
+      redis.llen("octopusci:jobs")
+    end
+
+    def self.repo_branch_size(repo_name, branch_name)
+      redis.llen("octopusci:#{repo_name}:#{branch_name}:jobs")
+    end
+
+    def self.list_job_ids(start_idx, num_jobs)
+      len = size()
+      end_idx = len - 1
+
+      range_idx = start_idx + num_jobs
+      if (end_idx - start_idx < num_jobs)
+        range_idx = end_idx
+      end
+      redis.lrange("octopusci:jobs", 0, range_idx)
+    end
+
+    def self.list_repo_branch_job_ids(repo_name, branch_name, start_idx, num_jobs)
+      len = repo_branch_size(repo_name, branch_name)
+      end_idx = len - 1
+
+      range_idx = start_idx + num_jobs
+      if (end_idx - start_idx < num_jobs)
+        range_idx = end_idx
+      end
+      redis.lrange("octopusci:#{repo_name}:#{branch_name}:jobs", 0, range_idx)
+    end
+
+    def self.list(start_idx, num_jobs)
+      job_ids = list_job_ids(start_idx, num_jobs)
+      job_ids.map { |id| self.get(id) }
+    end
+
+    def self.list_repo_branch(repo_name, branch_name, start_idx, num_jobs)
+      job_ids = list_repo_branch_job_ids(repo_name, branch_name, start_idx, num_jobs)
+      job_ids.map { |id| self.get(id) }
+    end
+
+    def self.redis
+      Resque.redis
+    end
+  end
+end

data/lib/octopusci/notifier.rb

@@ -6,28 +6,18 @@ require 'action_mailer'
 ActionMailer::Base.view_paths = File.dirname(__FILE__) + '/../'
 
 module Octopusci
-  class Notifier < ActionMailer::Base    
-    def job_complete(job_rec, job_conf, success=false)
-      @job = job_rec
+  class Notifier < ActionMailer::Base
+    def job_complete(job, recip_email, context_str, success=false)
+      @job = job
+      @success = success
+      @context_str = context_str
       if success
         @status_str = 'success'
       else
         @status_str = 'failed'
       end
-      
-      recip_email = nil
-      if job_rec.branch_name == 'master'
-        recip_email = job_conf['default_email']
-      else
-        if job_rec.payload['pusher']['email']
-          recip_email = job_rec.payload['pusher']['email']
-        else
-          recip_email = job_conf['default_email']
-        end
-      end
-      
-      mail(:to => recip_email, :subject => "Octopusci Build (#{@status_str}) - #{@job.repo_name} / #{@job.branch_name}") do |format|
-        format.text
+            
+      mail(:to => recip_email, :subject => "Octopusci Build (#{@status_str}) - #{context_str} - #{@job['repo_name']} / #{@job['branch_name']}") do |format|
         format.html
       end
     end

data/lib/octopusci/notifier/job_complete.html.erb

@@ -1,11 +1,84 @@
+<% j = @job %>
 <!DOCTYPE html>
 <html>
   <head>
     <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
   </head>
   <body>
-    <h1>Octopusci Build (<%= @status_str %>) - <%= @job.repo_name %> / <%= @job.branch_name %></h1>
-    <h2>Output:</h2>
-    <pre><code><%= @job.output %></code></pre>
+  	<div style="background-color: #333; color: #fff; width: 100%;">
+  		<div style="padding: 10px; font-size: 16px;">Octopusci Build (<%= @status_str %>) - <a style="color: #fff" href="<%= @job['payload']['repository']['url'] %>"><%= @job['repo_name'] %></a> / <a style="color: #fff" href="<%= Octopusci::Config['general']['base_url'] %>/<%= @job['repo_name'] %>/<%= @job['branch_name'] %>"><%= @job['branch_name'] %></a></div>
+	  	<div style="background-color: #e2e2e2; padding: 20px;">
+	  		<div style="background-color: #ffffff; color: #333;">
+	  			<div style="background-color: #F9F9F9; padding: 10px; border: 1px dotted #E6E6E6;">
+	  				<div style="padding: 10px; color: #7A7A7A;">
+				      <table style="color: #7A7A7A;">
+				      	<tr>
+				      		<td style="text-align: right;">Status:</td>
+				      		<td style="width: 200px;"><span style="<% if @success %>color: #78be3f;<% else %>color: red;<% end %>"><%= @status_str %></span></td>
+				      	</tr>
+						    <tr>
+						      <td style="text-align: right;">Job:</td>
+						      <td style="width: 200px;"><a style="color: #333;" href="<%= Octopusci::Config['general']['base_url'] %>/jobs/<%= j['id'] %>"><%= j['id'] %></a></td>
+						      <td style="text-align: right;">Before Commit:</td>
+						      <td>
+						        <% if !j['payload']['created'] %>
+						          <%= j['payload']['before'] %>
+						        <% else %>
+						          No before commit, remote branch was created with this push.
+						        <% end %>
+						      </td>
+						    </tr>
+						    <tr>
+						      <td style="text-align: right;">Finished:</td>
+						      <td>
+						        <% if j['ended_at'] %>
+						          <%= j['ended_at'].ago_in_words %>
+						        <% else %>
+						          -
+						        <% end %>
+						      </td>
+						      <td style="text-align: right;">After Commit:</td>
+						      <td>
+						        <%= j['payload']['after'] %>
+						      </td>
+						    </tr>
+						    <tr>
+						      <td style="text-align: right;">Duration:</td>
+						      <td>
+						        <% if j['ended_at'] && j['started_at'] %>
+						          <%= ((j['ended_at'] - j['started_at'])/60).to_i %> minutes
+						        <% else %>
+						          -
+						        <% end %>
+						      </td>
+						      <td style="text-align: right;">Compare:</td>
+						      <td><a style="color: #333;" href="<%= j['compare'] %>"><%= j['compare'] %></a></td>
+						    </tr>
+						    <tr>
+						      <td style="text-align: right;">Stage:</td>
+						      <td><%= j['stage'] %></td>
+						      <td style="text-align: right;">Pusher:</td>
+						      <td>
+						        <% if j['payload']['pusher']['email'] %>
+						          <a style="color: #333;" href="mailto:<%= j['payload']['pusher']['name'] %>"><%= j['payload']['pusher']['name'] %></a>
+						        <% else %>
+						          <%= j['payload']['pusher']['name'] %>
+						        <% end %>
+						      </td>
+						    </tr>
+						    <tr>
+						    	<td style="text-align: right;">Output:</td>
+						    	<td></td>
+						    	<td style="text-align: right;">Context:</td>
+				      		<td><%= @context_str %></td>
+						    </tr>
+						  </table>
+					  </div>
+				    <div style="background-color: #333; color: #fff; padding: 10px;">
+				    	<pre><code><%= Octopusci::IO.new(@job).read_all_out %></code></pre>
+				    </div>
+				   </div>
+	  		</div>
+  	</div>
   </body>
 </html>

data/lib/octopusci/queue.rb

@@ -7,18 +7,18 @@ module Octopusci
       gh_pl_key = github_payload_key(proj_name, branch_name)
 
       if lismember('octopusci:commit', resque_opts)
-        Resque.redis.set(gh_pl_key, Resque::encode(github_payload))
-        # Get the most recent job for this project and update it with the data
-        job = ::Job.where("jobs.repo_name = ? && jobs.ref = ?", proj_name, '/refs/heads/' + branch_name).order('jobs.created_at DESC').first
+        self.redis.set(gh_pl_key, Resque::encode(github_payload))
+
+        job = Octopusci::JobStore.list_repo_branch(proj_name, branch_name, 0, 1).first
         if job
-          job.update_attributes(Octopusci::Helpers.gh_payload_to_job_attrs(github_payload))
-        end
+          Octopusci::JobStore.set(job['id'], job.merge(Octopusci::Helpers.gh_payload_to_job_attrs(github_payload)))
+        end        
       else
         # Create a new job for this project with the appropriate data
-        job = ::Job.create(Octopusci::Helpers.gh_payload_to_job_attrs(github_payload).merge(:status => 'pending'))
-        resque_opts["args"] << job.id
+        job_id = Octopusci::JobStore.prepend(Octopusci::Helpers.gh_payload_to_job_attrs(github_payload).merge('status' => 'pending'))
+        resque_opts["args"] << job_id
         resque_opts["args"] << job_conf
-        Resque.redis.set(gh_pl_key, Resque::encode(github_payload))
+        self.redis.set(gh_pl_key, Resque::encode(github_payload))
         Resque.push('octopusci:commit', resque_opts)
       end
     end
@@ -31,11 +31,15 @@ module Octopusci
     end
     
     def self.github_payload(project_name, branch_name)
-      Resque::decode(Resque.redis.get(github_payload_key(project_name, branch_name)))
+      Resque::decode(self.redis.get(github_payload_key(project_name, branch_name)))
     end
     
     def self.github_payload_key(proj_name, branch_name)
-      "octpusci:github_payload:#{proj_name}:#{branch_name}"
+      "octopusci:github_payload:#{proj_name}:#{branch_name}"
+    end
+
+    def self.redis
+      Resque.redis
     end
   end
 end

data/lib/octopusci/server.rb

@@ -8,13 +8,9 @@ module Octopusci
     dir = File.dirname(File.expand_path(__FILE__))
     
     set :views, "#{dir}/server/views"
-    set :public, "#{dir}/server/public"
+    set :public_folder, "#{dir}/server/public"
     set :static, true
     
-    before do
-      ActiveRecord::Base.verify_active_connections!
-    end
-    
     helpers do
       def protected!
         unless authorized?
@@ -25,53 +21,53 @@ module Octopusci
 
       def authorized?
         @auth ||=  Rack::Auth::Basic::Request.new(request.env)        
-        @auth.provided? && @auth.basic? && @auth.credentials && @auth.credentials == [ Octopusci::CONFIG['http_basic']['username'], Octopusci::CONFIG['http_basic']['password'] ]
+        @auth.provided? && @auth.basic? && @auth.credentials && @auth.credentials == [ Octopusci::Config['http_basic']['username'], Octopusci::Config['http_basic']['password'] ]
       end
     end
     
     get '/' do
       protected!
-      @jobs = ::Job.order('jobs.created_at DESC').limit(20)
+      @jobs = Octopusci::JobStore.list(0, 20)
       erb :index
     end
-    
+
     get '/:repo_name/:branch_name/jobs' do
       protected!
       @page_logo = "#{params[:repo_name]} / #{params[:branch_name]}"
-      @jobs = ::Job.where(:repo_name => params[:repo_name], :ref => "refs/heads/#{params[:branch_name]}").order('jobs.created_at DESC').limit(20)
+      @jobs = Octopusci::JobStore.list_repo_branch(params[:repo_name], params[:branch_name], 0, 20)
       erb :index
     end
     
     get '/jobs/:job_id' do
       protected!
-      @job = ::Job.find(params[:job_id])
+      @job = Octopusci::JobStore.get(params[:job_id])
       erb :job
     end
     
     get '/jobs/:job_id/output' do
       protected!
-      @job = ::Job.find(params[:job_id])
+      @job = Octopusci::JobStore.get(params[:job_id])
       content_type('text/plain')
-      return @job.output
+      return Octopusci::IO.new(@job).read_all_out
     end
 
     get '/jobs/:job_id/silent_output' do
       protected!
-      @job = ::Job.find(params[:job_id])
+      @job = Octopusci::JobStore.get(params[:job_id])
       content_type('text/plain')
-      return @job.silent_output
+      return Octopusci::IO.new(@job).read_all_log
     end
 
     get '/jobs/:job_id/status' do
       protected!
-      @job = ::Job.find(params[:job_id])
+      @job = Octopusci::JobStore.get(params[:job_id])
       content_type('text/plain')
-      return @job.status
+      return @job['status']
     end
     
     get '/jobs/:job_id/ajax_summary' do
       protected!
-      @job = ::Job.find(params[:job_id])
+      @job = Octopusci::JobStore.get(params[:job_id])
       erb :job_summary, :layout => false, :locals => { :j => @job }
     end
     
@@ -80,22 +76,23 @@ module Octopusci
         raise "No payload paramater found, it is a required parameter."
       end
       github_payload = Octopusci::Helpers.decode(params["payload"])
-            
+
       # Make sure that the request is for a project Octopusci knows about
       proj_info = Octopusci::Helpers.get_project_info(github_payload["repository"]["name"], github_payload["repository"]["owner"]["name"])
       if proj_info.nil?
         return 404
       end
-      
+
       if (github_payload["ref"] =~ /refs\/heads\//) && (github_payload["deleted"] != true)
         branch_name = github_payload["ref"].gsub(/refs\/heads\//, '')
       
         # Queue the job appropriately
         Octopusci::Queue.enqueue(proj_info['job_klass'], github_payload["repository"]["name"], branch_name, github_payload, proj_info)
+        return 200
       else
         return 200
       end
     end
-        
+
   end
 end