build-buddy 1.6.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a199c1158908a1d5fe2f9a2d577e7aefa2ac2f15
-  data.tar.gz: f48e1c69d14502d6ed8ab7d5f3026feb39ace36e
+  metadata.gz: 81ee8f317ad72e56104078228407ee03776e40e4
+  data.tar.gz: ac43883aa9dd4188844043bb8370d178b5345348
 SHA512:
-  metadata.gz: 002f8b81d5788d26834339b5046cea2484d6737ef29f5c4a03bad386962c55641a5b33f07029543b3c33d7f6c5d0f8fef7bc5de26dac6054df8c8c302dd52025
-  data.tar.gz: 74fbe2352deeb4eb614cbd8ff38438663918259f8ddcc806cc5eebfc60084b80b9762395e069a137932e4f09f64d3ea440fb06b6c4498b0c751c5798f7202587
+  metadata.gz: 6f4a4d782e9a013622f05680fdf9275f001953a3ad0e45ec8a0dd7f5318c92e73fa1fc779be56d7a4b836303c3a845a82d302756e51d976535a4c50aeae036ad
+  data.tar.gz: 14e79e052d4d1a771ea538532c257f60fe21e02195fdf687a22a9f343131d3b1a0886392e09fc33b4aec92e6e5aaf3c7f76e0a6b0cc31b7ad9700d8b325adc56

data/lib/build_buddy.rb

@@ -9,5 +9,5 @@ require 'build_buddy/recorder'
 require 'build_buddy/build_data'
 
 module BuildBuddy
-  VERSION = "1.6.1"
+  VERSION = "1.7.0"
 end

data/lib/build_buddy/build_data.rb

@@ -1,9 +1,9 @@
 module BuildBuddy
   class BuildData
     attr_accessor :_id # Mongo ID
-    attr_accessor :build_type # one of :master, :release or :pull_request
+    attr_accessor :type # one of :master, :release or :pull_request
     attr_accessor :repo_full_name
-    attr_accessor :build_version
+    attr_accessor :branch
     attr_accessor :pull_request
     attr_accessor :repo_full_name
     attr_accessor :repo_sha
@@ -11,7 +11,9 @@ module BuildBuddy
     attr_accessor :exit_code
     attr_accessor :start_time
     attr_accessor :end_time
-    attr_accessor :build_log_filename
+    attr_accessor :log_filename
+    attr_accessor :flags # :no_upload, :test_channel
+    attr_accessor :metrics
 
     def initialize(args)
       args.each do |key, value|

data/lib/build_buddy/builder.rb

@@ -1,6 +1,7 @@
 require 'rubygems'
 require 'bundler'
 require 'celluloid'
+require 'psych'
 require_relative './watcher.rb'
 require_relative './config.rb'
 
@@ -9,52 +10,59 @@ module BuildBuddy
     include Celluloid
     include Celluloid::Internals::Logger
 
-    # TODO: Respond to request to kill the build.
-    # TODO: Kill the build pid after a certain amount of time has elapsed and report.
-
     def initialize
       @pid = nil
       @gid = nil
       @watcher = nil
+      @metrics_tempfile = nil
     end
 
     def start_build(build_data)
       @build_data = build_data
+      @metrics_tempfile = Tempfile.new('build-metrics')
+      @metrics_tempfile.close()
+
       repo_parts = build_data.repo_full_name.split('/')
       command = "bash "
       env = {
           "GIT_REPO_OWNER" => repo_parts[0],
           "GIT_REPO_NAME" => repo_parts[1],
+          "METRICS_DATA_FILE" => @metrics_tempfile.path,
           "RBENV_DIR" => nil,
           "RBENV_VERSION" => nil,
           "RBENV_HOOK_PATH" => nil,
           "RBENV_ROOT" => nil,
           "PATH" => ENV['PATH'].split(':').select { |v| !v.match(/\.rbenv\/versions|Cellar\/rbenv/) }.join(':')
       }
+      unless build_data.flags.nil?
+        build_data.flags.each do |flag|
+         env["BUILD_FLAG_#{flag.to_s.upcase}"] = '1'
+        end
+      end
 
-      case build_data.build_type
+      case build_data.type
         when :pull_request
           env["GIT_PULL_REQUEST"] = build_data.pull_request.to_s
           command += Config.pull_request_build_script
         when :master
           command += Config.master_build_script
         when :release
-          env["GIT_BRANCH"] = build_data.build_version
+          env["GIT_BRANCH"] = build_data.branch
           command += Config.release_build_script
         else
           raise "Unknown build type"
       end
 
       @build_data.start_time = Time.now.utc
-      build_log_filename = File.join(Config.build_log_dir,
-        "build_#{build_data.build_type.to_s}_#{build_data.start_time.strftime('%Y%m%d_%H%M%S')}.log")
-      @build_data.build_log_filename = build_log_filename
+      log_filename = File.join(Config.build_log_dir,
+        "build_#{build_data.type.to_s}_#{build_data.start_time.strftime('%Y%m%d_%H%M%S')}.log")
+      @build_data.log_filename = log_filename
 
       Bundler.with_clean_env do
-        @pid = Process.spawn(env, command, :pgroup => true, [:out, :err] => build_log_filename)
+        @pid = Process.spawn(env, command, :pgroup => true, [:out, :err] => log_filename)
         @gid = Process.getpgid(@pid)
       end
-      info "Running #{File.basename(command)} (pid #{@pid}, gid #{@gid}) : Log #{build_log_filename}"
+      info "Running #{File.basename(command)} (pid #{@pid}, gid #{@gid}) : Log #{log_filename}"
 
       if @watcher
         @watcher.terminate
@@ -68,6 +76,18 @@ module BuildBuddy
       @build_data.end_time = Time.now.utc
       @build_data.termination_type = (status.signaled? ? :killed : :exited)
       @build_data.exit_code = (status.exited? ? status.exitstatus : -1)
+
+      # Collect any data written to the build metrics YAML file
+      begin
+        metrics = Psych.load_file(@metrics_tempfile.path)
+      rescue Psych::SyntaxError => ex
+        error "There was a problem collecting bulid metrics: #{ex.message}"
+      end
+      if !metrics
+        metrics = {}
+      end
+      @build_data.metrics = metrics
+
       info "Process #{status.pid} #{@build_data.termination_type == :killed ? 'was terminated' : "exited (#{@build_data.exit_code})"}"
       Celluloid::Actor[:scheduler].async.on_build_completed(@build_data)
       @watcher.terminate

data/lib/build_buddy/config.rb

@@ -8,6 +8,7 @@ module BuildBuddy
     attr_accessor :github_api_token
     attr_accessor :slack_api_token
     attr_accessor :slack_build_channel
+    attr_accessor :slack_test_channel
     attr_accessor :slack_builders
     attr_accessor :xcode_workspace
     attr_accessor :xcode_test_scheme

data/lib/build_buddy/recorder.rb

@@ -10,8 +10,10 @@ module BuildBuddy
     include Celluloid::Internals::Logger
 
     def initialize()
+      Mongo::Logger.logger.level = ::Logger::FATAL
       @mongo ||= Mongo::Client.new([ '127.0.0.1:27017' ], :database => 'build-buddy')
       info "Connected to MongoDB"
+      @mongo[:builds].indexes.create_one({:start_time => -1}, :name => "reverse_build_order")
     end
 
     def record_build_data(build_data)
@@ -28,12 +30,17 @@ module BuildBuddy
     end
 
     def get_build_data(id)
-      builds = @mongo[:builds]
-      document = builds.find({ :_id => BSON::ObjectId.from_string(id) }, { :limit => 1 }).first
+      document = @mongo[:builds].find({ :_id => BSON::ObjectId.from_string(id) }, { :limit => 1 }).first
       if document.nil?
         return nil
       end
       BuildData.new(document)
     end
+
+    def get_build_data_history(limit)
+      @mongo[:builds].find().sort(:start_time => -1).limit(limit).map do |document|
+        BuildData.new(document)
+      end
+    end
   end
 end

data/lib/build_buddy/scheduler.rb

@@ -20,7 +20,7 @@ module BuildBuddy
     def queue_a_build(build_data)
       @build_queue.push(build_data)
 
-      case build_data.build_type
+      case build_data.type
         when :pull_request
           Celluloid::Actor[:gitter].async.set_status(
               build_data.repo_full_name, build_data.repo_sha, :pending, "This build is in the queue")
@@ -57,34 +57,14 @@ module BuildBuddy
           build_data = @build_queue.pop()
           @active_build = build_data
           Celluloid::Actor[:recorder].async.record_build_data(build_data)
-          if build_data.build_type == :pull_request
+          if build_data.type == :pull_request
             Celluloid::Actor[:gitter].async.set_status(
                 build_data.repo_full_name, build_data.repo_sha, :pending, "This build has started")
           end
           Celluloid::Actor[:builder].async.start_build(build_data)
         elsif @done_queue.length > 0
           build_data = @done_queue.pop
-          status_message = build_data.termination_type == :killed ? "was stopped" : build_data.exit_code != 0 ? "failed" : "succeeded"
-          status_message += '. '
-
-          if build_data.build_type == :pull_request
-            message = "The buddy build #{status_message}"
-            Celluloid::Actor[:gitter].async.set_status(
-                build_data.repo_full_name, build_data.repo_sha,
-                build_data.termination_type == :killed ? :failure : build_data.exit_code != 0 ? :error : :success,
-                message)
-            info "Pull request build #{status_message}"
-          else
-            status_message += "Log file at #{Config.server_base_uri + '/log/' + build_data._id.to_s}."
-            if build_data.build_type == :master
-              message = "A build of the `master` branch #{status_message}"
-              info "`master` branch build #{status_message}"
-            else
-              message = "A build of the `#{build_data.build_version}` branch #{status_message}"
-              info "Release branch build #{status_message}"
-            end
-            Celluloid::Actor[:slacker].async.notify_channel(message)
-          end
+          Celluloid::Actor[:slacker].async.notify_channel(build_data)
           Celluloid::Actor[:recorder].async.update_build_data(build_data)
         else
           @build_timer.cancel

data/lib/build_buddy/server.rb

@@ -29,8 +29,9 @@ module BuildBuddy
                 payload = JSON.parse(payload_text)
                 pull_request = payload['pull_request']
                 build_data = BuildData.new(
-                    :build_type => :pull_request,
+                    :type => :pull_request,
                     :pull_request => pull_request['number'],
+                    :flags => [],
                     :repo_sha => pull_request['head']['sha'],
                     :repo_full_name => pull_request['base']['repo']['full_name'])
                 info "Got pull request #{build_data.pull_request} from GitHub"
@@ -49,13 +50,12 @@ module BuildBuddy
           case request.method
           when 'GET'
             build_data = Celluloid::Actor[:recorder].get_build_data($1)
-            log_contents = nil
-            if build_data.nil? or build_data.build_log_filename.nil? or !File.exist?(build_data.build_log_filename)
+            if build_data.nil? or build_data.log_filename.nil? or !File.exist?(build_data.log_filename)
               sleep 1
               request.respond 404, "Not found"
             end
             log_contents = 'Log file has been deleted.'
-            File.open(build_data.build_log_filename) do |io|
+            File.open(build_data.log_filename) do |io|
               log_contents = io.read
             end
             html = %Q(

data/lib/build_buddy/slacker.rb

@@ -25,7 +25,141 @@ module BuildBuddy
         raise "Slack connection was closed"
       end
       @rt_client.start_async
-      @notify_slack_channel = nil
+      @build_slack_channel = nil
+      @test_slack_channel = nil
+    end
+
+    def self.get_build_flags message
+      flags = []
+      message.split(',').each do |s|
+        flags.push(s.lstrip.rstrip.gsub(' ', '_').to_sym)
+      end
+      flags
+    end
+
+    def do_build(message, from_slack_channel, slack_user_name)
+      response = ''
+      sender_is_a_builder = (Config.slack_builders.nil? ? true : Config.slack_builders.include?('@' + slack_user_name))
+      unless sender_is_a_builder
+        if from_slack_channel
+          response = "I'm sorry @#{slack_user_name} you are not on my list of allowed builders."
+        else
+          response = "I'm sorry but you are not on my list of allowed builders."
+        end
+      else
+        scheduler = Celluloid::Actor[:scheduler]
+
+        case message
+        when /^master(?: with )?(?<flags>.*)?/i
+          flags = Slacker.get_build_flags($~[:flags])
+          response = "OK, I've queued a build of the `master` branch."
+          scheduler.queue_a_build(BuildData.new(
+              :type => :master,
+              :flags => flags,
+              :repo_full_name => Config.github_webhook_repo_full_name))
+        when /^(?<version>v\d+\.\d+)(?: with )?(?<flags>.*)?/
+          flags = Slacker.get_build_flags($~[:flags])
+          version = $~[:version]
+          if Config.valid_release_versions.include?(version)
+            response = "OK, I've queued a build of the `#{version}` branch."
+            scheduler.queue_a_build(BuildData.new(
+                :type => :release,
+                :branch => version,
+                :flags => flags,
+                :repo_full_name => Config.github_webhook_repo_full_name))
+          else
+            response = "I'm sorry, I am not allowed to build the `#{version}` release branch"
+          end
+        when /stop/i
+          if scheduler.stop_build
+            response = "OK, I stopped the currently running build."
+          else
+            response = "There is no build running to stop."
+          end
+        else
+          response = "Sorry#{from_slack_channel ? " <@#{data['user']}>" : ""}, I'm not sure if you want do a `master` or release branch build, or maybe `stop` any running build?"
+        end
+      end
+      response
+    end
+
+    def do_status
+      scheduler = Celluloid::Actor[:scheduler]
+      build_data = scheduler.active_build
+      queue_length = scheduler.queue_length
+      response = ''
+      if build_data.nil?
+        response = "There is currently no build running"
+        if queue_length == 0
+          response += " and no builds in the queue."
+        else
+          response += " and #{queue_length} in the queue."
+        end
+      else
+        case build_data.type
+        when :pull_request
+          response = "There is a pull request build in progress for https://github.com/#{build_data.repo_full_name}/pull/#{build_data.pull_request}."
+        when :master
+          response = "There is a build of the `master` branch of https://github.com/#{build_data.repo_full_name} in progress."
+        when :release
+          response = "There is a build of the `#{build_data.branch}` branch of https://github.com/#{build_data.repo_full_name} in progress."
+        end
+        if queue_length == 1
+          response += " There is one build in the queue."
+        elsif queue_length > 1
+          response += " There are #{queue_length} builds in the queue."
+        end
+      end
+      response
+    end
+
+    def do_help
+      # TODO: The repository should be a link to GitHub
+      %Q(Hello#{from_slack_channel ? " <@#{data['user']}>" : ""}, I'm the *@#{@rt_client.self['name']}* build bot version #{BuildBuddy::VERSION}! I look after 3 types of build: pull request, master and release.
+
+A pull request build happens when you make a pull request to the *#{Config.github_webhook_repo_full_name}* GitHub repository.
+
+I can run builds of the master branch if you say `build master`. I can do builds of release branches, e.g. `build v2.3` but only for those branches that I am allowed to build.
+
+I can stop any running build if you ask me to `stop build`, even pull request builds.  I am configured to let the *#{Config.slack_build_channel}* channel know if master or release builds are stopped.
+
+You can also ask me for `status` and I'll tell you what's being built and what's in the queue and `history` to get a list of recent builds.
+)
+    end
+
+    def do_history(message)
+      case message
+      when /([0-9]+)/
+        limit = $1
+      else
+        limit = 5
+      end
+
+      recorder = Celluloid::Actor[:recorder]
+      build_datas = recorder.get_build_data_history(limit)
+
+      if build_datas.count == 0
+        response = "No builds have performed yet"
+      else
+        response = "Here are the last #{build_datas.count} builds:\n"
+        build_datas.each do |build_data|
+          response += "A "
+          response += case build_data.type
+                      when :master
+                        "`master` branch build"
+                      when :release
+                        "`#{build_data.branch}` release branch build"
+                      when :pull_request
+                        "pull request `#{build_data.pull_request}` build"
+                      end
+          response += " at #{build_data.start_time.to_s}. #{Config.server_base_uri + '/log/' + build_data._id.to_s}\n"
+        end
+      end
+      response
+    end
+
+    def self.get_channel_id(channel, map_channel_name_to_id, map_group_name_to_id)
+      (channel.start_with?('#') ? map_channel_name_to_id[channel[1..-1]] : map_group_name_to_id[channel])
     end
 
     def on_slack_hello
@@ -35,14 +169,21 @@ module BuildBuddy
 
       map_channel_name_to_id = @rt_client.channels.map {|id, channel| [channel.name, id]}.to_h
       map_group_name_to_id = @rt_client.groups.map {|id, group| [group.name, id]}.to_h
-      channel = Config.slack_build_channel
-      is_channel = (channel[0] == '#')
 
-      @notify_slack_channel = (is_channel ? map_channel_name_to_id[channel[1..-1]] : map_group_name_to_id[channel])
-      if @notify_slack_channel.nil?
-        error "Unable to identify the slack channel #{channel}"
+      @build_slack_channel = Slacker.get_channel_id(Config.slack_build_channel, map_channel_name_to_id, map_group_name_to_id)
+
+      if @build_slack_channel.nil?
+        error "Unable to identify the build slack channel #{channel}"
+      else
+        info "Slack build notification channel is #{@build_slack_channel} (#{Config.slack_build_channel})"
+      end
+
+      @test_slack_channel = Slacker.get_channel_id(Config.slack_test_channel, map_channel_name_to_id, map_group_name_to_id)
+
+      if @test_slack_channel.nil?
+        error "Unable to identify the test slack channel #{channel}"
       else
-        info "Slack notification channel is #{@notify_slack_channel} (#{channel})"
+        info "Slack test notification channel is #{@test_slack_channel} (#{Config.slack_test_channel})"
       end
     end
 
@@ -54,124 +195,83 @@ module BuildBuddy
         return
       end
 
-      sending_user_id = data['user']
+      slack_user_id = data['user']
 
       # Only respond to messages from users and bots
-      if sending_user_id.nil?
+      if slack_user_id.nil?
         if data['username'].nil? or data['subtype'] != 'bot_message'
           return
         end
-        sending_user_name = data['username']
+        slack_user_name = data['username']
       else
         map_user_id_to_name = @rt_client.users.map {|id, user| [id, user.name]}.to_h
-        sending_user_name = map_user_id_to_name[sending_user_id]
+        slack_user_name = map_user_id_to_name[slack_user_id]
 
-        if sending_user_name.nil?
-          error "User #{sending_user_id} is not known"
+        if slack_user_name.nil?
+          error "User #{slack_user_id} is not known"
           return
         end
       end
 
       # Don't respond if _we_ sent the message!
-      if sending_user_id == @rt_client.self['id']
+      if slack_user_id == @rt_client.self['id']
         return
       end
 
-      sender_is_a_builder = (Config.slack_builders.nil? ? true : Config.slack_builders.include?('@' + sending_user_name))
-
       c = data['channel'][0]
-      in_channel = (c == 'C' || c == 'G')
+      from_slack_channel = (c == 'C' || c == 'G')
 
       # Don't respond if the message is to a channel and our name is not in the message
-      if in_channel and !message.match(@rt_client.self['id'])
+      if from_slack_channel and !message.match(@rt_client.self['id'])
         return
       end
 
-      scheduler = Celluloid::Actor[:scheduler]
-
-      case message
-        when /build/i
-          unless sender_is_a_builder
-            if in_channel
-              response = "I'm sorry @#{sending_user_name} you are not on my list of allowed builders."
-            else
-              response = "I'm sorry but you are not on my list of allowed builders."
-            end
-          else
-            case message
-              when /master/i
-                response = "OK, I've queued a build of the `master` branch."
-                scheduler.queue_a_build(BuildData.new(
-                    :build_type => :master,
-                    :repo_full_name => Config.github_webhook_repo_full_name))
-              when /(?<version>v\d+\.\d+)/
-                version = $~[:version]
-                if Config.valid_release_versions.include?(version)
-                  response = "OK, I've queued a build of `#{version}` branch."
-                  scheduler.queue_a_build(BuildData.new(
-                    :build_type => :release,
-                    :build_version => version,
-                    :repo_full_name => Config.github_webhook_repo_full_name))
-                else
-                  response = "I'm sorry, I cannot build the #{version} release branch"
-                end
-              when /stop/i
-                if scheduler.stop_build
-                  response = "OK, I stopped the currently running build."
-                else
-                  response = "There is no build running to stop."
-                end
-              else
-                response = "Sorry#{in_channel ? " <@#{data['user']}>" : ""}, I'm not sure if you want do a `master` or release branch build, or maybe `stop` any running build?"
-            end
-          end
+      response = case message
+        when /build (.*)/i
+          do_build $1, from_slack_channel, slack_user_name
         when /status/i
-          build_data = scheduler.active_build
-          queue_length = scheduler.queue_length
-          if build_data.nil?
-            response = "There is currently no build running"
-            if queue_length == 0
-              response += " and no builds in the queue."
-            else
-              response += " and #{queue_length} in the queue."
-            end
-          else
-            case build_data.build_type
-              when :pull_request
-                response = "There is a pull request build in progress for https://github.com/#{build_data.repo_full_name}/pull/#{build_data.pull_request}."
-              when :master
-                response = "There is a build of the `master` branch of https://github.com/#{build_data.repo_full_name} in progress."
-              when :release
-                response = "There is a build of the `#{build_data.build_version}` branch of https://github.com/#{build_data.repo_full_name} in progress."
-            end
-            if queue_length == 1
-              response += " There is one build in the queue."
-            elsif queue_length > 1
-              response += " There are #{queue_length} builds in the queue."
-            end
-          end
+          do_status
+        when /history (.*)/
+          do_history $1
         when /help/i, /what can/i
-          # TODO: The repository should be a link to GitHub
-          response = %Q(Hello#{in_channel ? " <@#{data['user']}>" : ""}, I'm the *@#{@rt_client.self['name']}* build bot version #{BuildBuddy::VERSION}! I look after 3 types of build: pull request, master and release.
-
-A pull request build happens when you make a pull request to the *#{Config.github_webhook_repo_full_name}* GitHub repository.
-
-I can run builds of the master branch if you say `build master`. I can do builds of release branches, e.g. `build v2.3` but only for those branches that I am allowed to build.
-
-I can stop any running build if you ask me to `stop build`, even pull request builds.  I am configured to let the *#{Config.slack_build_channel}* channel know if master or release builds are stopped.
-
-You can also ask me for `status` and I'll tell you what's being built and what's in the queue.
-)
+          do_help
         else
-          response = "Sorry#{in_channel ? " <@#{data['user']}>" : ""}, I'm not sure how to respond."
-      end
+          "Sorry#{from_slack_channel ? " <@#{data['user']}>" : ""}, I'm not sure how to respond."
+        end
       @rt_client.message channel: data['channel'], text: response
       info "Slack message '#{message}' from #{data['channel']} handled"
     end
 
-    def notify_channel(message)
-      unless @notify_slack_channel.nil?
-        @rt_client.message(channel: @notify_slack_channel, text: message)
+    def notify_channel(build_data)
+      status_message = build_data.termination_type == :killed ? "was stopped" : build_data.exit_code != 0 ? "failed" : "succeeded"
+      status_message += '. '
+
+      if build_data.type == :pull_request
+        message = "The buddy build #{status_message}"
+        Celluloid::Actor[:gitter].async.set_status(
+            build_data.repo_full_name, build_data.repo_sha,
+            build_data.termination_type == :killed ? :failure : build_data.exit_code != 0 ? :error : :success,
+            message)
+        info "Pull request build #{status_message}"
+      else
+        status_message += "Log file at #{Config.server_base_uri + '/log/' + build_data._id.to_s}."
+        if build_data.type == :master
+          message = "A build of the `master` branch #{status_message}"
+          info "`master` branch build #{status_message}"
+        else
+          message = "A build of the `#{build_data.branch}` branch #{status_message}"
+          info "Release branch build #{status_message}"
+        end
+
+        if build_data.flags.include?(:test_channel)
+          unless @test_slack_channel.nil?
+            @rt_client.message(channel: @test_slack_channel, text: message)
+          end
+        else
+          unless @build_slack_channel.nil?
+            @rt_client.message(channel: @build_slack_channel, text: message)
+          end
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: build-buddy
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.0
 platform: ruby
 authors:
 - John Lyon-smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-07 00:00:00.000000000 Z
+date: 2016-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: timers