build-buddy 1.11.0 → 1.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c3ac2498d85759e142f61aead4aebf0968721a4e
-  data.tar.gz: 6af814ee68ac2f59f23b84eeef9ca9bfd454707c
+  metadata.gz: 4d50fe65cd2806a970546102b56304bdae80f2b6
+  data.tar.gz: 7fdd7e59a6b62df8fe884ec009020bdb85ec9cdd
 SHA512:
-  metadata.gz: c04cbb95121ccd2dafc9d0445a7c03e94999e48dfdc8421f4f89c7dea5d857446208e193f51451370a2917964032f031b0dc0d5798884b99abc87f7d7f1918ea
-  data.tar.gz: c9f08e085110a8a733032bc01a68d824a477f7d5b3149c0c5a968a64b085c057b92961a4602e7f35428702a306434670b5841b28afc6f32d02dc461c521b7289
+  metadata.gz: cdd33b96f5b8e83ec9eff07b0555038c03f629dca285de77975928d264ab68a5a6942d5550d0fedb4a01b0211757a55512dd66ea883f262abc30bbe03015d617
+  data.tar.gz: afec166a8cce90aa46b85ff38bc3b4eddc8204afc2eeeca57a1b994dc75c151bcd6da50591d171fa772e259d65d5dfca6797a18f11cc0274d5f5e0504b362e18

data/bin/build-buddy

@@ -2,8 +2,6 @@
 
 require 'rubygems'
 require 'celluloid/current'
-require 'celluloid/supervision'
-require 'celluloid/supervision/container'
 require 'methadone'
 require 'build_buddy'
 
@@ -21,12 +19,8 @@ module BuildBuddy
 
       load config_file_name
 
-      config_path = File.dirname(config_file_name)
-      build_log_dir = File.expand_path(Config.build_log_dir, config_path)
-
-      unless Dir.exist?(build_log_dir)
-        Dir.mkdir(build_log_dir)
-      end
+      Dir.mkdir(Config.build_log_dir) unless Dir.exist?(Config.build_log_dir)
+      Dir.mkdir(Config.hud_image_dir) unless Dir.exist?(Config.hud_image_dir)
 
       Slack.configure do |config|
         config.token = Config.slack_api_token
@@ -34,15 +28,23 @@ module BuildBuddy
 
       Celluloid.logger = Reel::Logger.logger
 
-      Builder.supervise as: :builder
-      Slacker.supervise as: :slacker
-      Gitter.supervise as: :gitter
-      Scheduler.supervise as: :scheduler
-      Server.supervise as: :server
-      Recorder.supervise as: :recorder
+      Celluloid::Actor[:builder] = Builder.new
+      Celluloid::Actor[:slacker] = Slacker.new
+      Celluloid::Actor[:gitter] = Gitter.new
+      Celluloid::Actor[:scheduler] = Scheduler.new
+      Celluloid::Actor[:server] = Server.new
+      Celluloid::Actor[:recorder] = Recorder.new
+
+      Celluloid::Actor[:recorder].async.gen_charts
 
       begin
-        sleep
+        loop {
+          sleep(5)
+
+          unless Celluloid.actor_system.running.any? {|k| k.is_a?(BuildBuddy::Slacker)}
+            Celluloid::Actor[:slacker] = Slacker.new
+          end
+        }
       rescue Interrupt => e
         puts
       end

data/lib/build_buddy.rb

@@ -9,5 +9,5 @@ require 'build_buddy/recorder'
 require 'build_buddy/build_data'
 
 module BuildBuddy
-  VERSION = "1.11.0"
+  VERSION = "1.12.0"
 end

data/lib/build_buddy/build_data.rb

@@ -5,24 +5,27 @@ module BuildBuddy
     attr_accessor :repo_full_name
     attr_accessor :branch
     attr_accessor :pull_request
-    attr_accessor :repo_full_name
     attr_accessor :repo_sha
     attr_accessor :termination_type # :killed or :exited
+    attr_accessor :started_by
+    attr_accessor :stopped_by
     attr_accessor :exit_code
     attr_accessor :start_time
     attr_accessor :end_time
-    attr_accessor :log_filename
-    attr_accessor :flags # :no_upload, :test_channel
+    attr_accessor :flags
     attr_accessor :metrics
 
     def initialize(args)
       args.each do |key, value|
-        setter = self.method((key.to_s + '=').to_sym)
-
-        unless setter.nil?
-          setter.call(value)
+        begin
+          self.method((key.to_s + '=').to_sym).call(value)
+        rescue NameError
+          # Ignore fields in the database we don't understand
         end
       end
+
+      # Set the id here so that we can use it to refer to this build_data in queue
+      @_id = BSON::ObjectId.new
     end
 
     def to_h
@@ -30,5 +33,26 @@ module BuildBuddy
       instance_variables.each {|var| hash[var.to_s.delete("@")] = instance_variable_get(var) }
       hash
     end
+
+    def server_log_uri
+      Config.server_base_uri + '/log/' + @_id.to_s
+    end
+
+    def status_verb
+      if @termination_type == :killed
+        'was stopped'
+      else
+        @exit_code != 0 ? 'failed' : 'succeeded'
+      end
+    end
+
+    def pull_request_uri
+      "https://github.com/#{@repo_full_name}/pull/#{@pull_request}"
+    end
+
+    def log_file_name
+      return nil if @start_time.nil?
+      File.join(Config.build_log_dir, "#{@start_time.strftime('%Y%m%d-%H%M%S')}.log")
+    end
   end
 end

data/lib/build_buddy/builder.rb

@@ -27,7 +27,7 @@ module BuildBuddy
       @metrics_tempfile = Tempfile.new('build-metrics')
       @metrics_tempfile.close()
 
-      repo_parts = build_data.repo_full_name.split('/')
+      repo_parts = @build_data.repo_full_name.split('/')
       git_repo_owner = repo_parts[0]
       git_repo_name = repo_parts[1]
       env = {}
@@ -71,6 +71,8 @@ fi
 cd ${GIT_REPO_NAME}
 )
 
+      build_root_dir = nil
+
       case build_data.type
       when :pull_request
         build_root_dir = expand_vars(Config.pull_request_root_dir)
@@ -113,14 +115,12 @@ source ${BUILD_SCRIPT}
 
       unless build_data.flags.nil?
         build_data.flags.each do |flag|
-         env["BUILD_FLAG_#{flag.to_s.upcase}"] = '1'
+         env["BUILD_FLAG_#{key.to_s.upcase}"] = 1
         end
       end
 
       @build_data.start_time = Time.now.utc
-      log_filename = File.join(File.expand_path(Config.build_log_dir),
-        "build_#{build_data.type.to_s}_#{build_data.start_time.strftime('%Y%m%d_%H%M%S')}.log")
-      @build_data.log_filename = log_filename
+      log_file_name = @build_data.log_file_name
 
       # Ensure build root and git user directory exists
       clone_dir = File.join(build_root_dir, git_repo_owner)
@@ -130,12 +130,19 @@ source ${BUILD_SCRIPT}
       script_filename = File.join(clone_dir, "build-buddy-bootstrap.sh")
       File.write(script_filename, build_script)
 
+      # Notify GitHub that build has started
+      if @build_data.type == :pull_request
+        Celluloid::Actor[:gitter].async.set_status(
+            build_data.repo_full_name, build_data.repo_sha, :pending, "Build has started",
+            build_data.server_log_uri)
+      end
+
       # Run the build script
       Bundler.with_clean_env do
-        @pid = Process.spawn(env, "bash #{script_filename}", :pgroup => true, :chdir => clone_dir, [:out, :err] => log_filename)
+        @pid = Process.spawn(env, "bash #{script_filename}", :pgroup => true, :chdir => clone_dir, [:out, :err] => log_file_name)
         @gid = Process.getpgid(@pid)
       end
-      info "Running build script (pid #{@pid}, gid #{@gid}) : Log #{log_filename}"
+      info "Running build script (pid #{@pid}, gid #{@gid}) : Log #{log_file_name}"
 
       if @watcher
         @watcher.terminate
@@ -164,10 +171,28 @@ source ${BUILD_SCRIPT}
 
       @build_data.metrics = metrics
 
+      # Send status to GitHub if pull request
+      if @build_data.type == :pull_request
+        git_state = (@build_data.termination_type == :killed ? :failure : @build_data.exit_code != 0 ? :error : :success)
+        status_verb = @build_data.status_verb
+        Celluloid::Actor[:gitter].async.set_status(
+            @build_data.repo_full_name, @build_data.repo_sha, git_state, "Build #{status_verb}",
+            @build_data.server_log_uri)
+      end
+
+      # Log the build completion and clean-up
       info "Process #{status.pid} #{@build_data.termination_type == :killed ? 'was terminated' : "exited (#{@build_data.exit_code})"}"
       Celluloid::Actor[:scheduler].async.on_build_completed(@build_data)
       @watcher.terminate
       @watcher = nil
+
+      # Delete older log files
+      log_file_names = Dir.entries(Config.build_log_dir).select { |f| !f.match(/\d{8}-\d{6}\.log/).nil? }.sort()
+      while log_file_names.count > Config.build_log_limit
+        file_name = log_file_names.shift
+        FileUtils.rm(File.join(Config.build_log_dir, file_name))
+        info "Removing oldest log file #{file_name}"
+      end
     end
 
     def stop_build

data/lib/build_buddy/config.rb

@@ -2,33 +2,44 @@ module BuildBuddy
   module Config
     extend self
 
-    attr_accessor :github_webhook_port
-    attr_accessor :github_webhook_secret_token
-    attr_accessor :github_webhook_repo_full_name
-    attr_accessor :github_api_token
-    attr_accessor :slack_api_token
-    attr_accessor :slack_build_channel
-    attr_accessor :slack_test_channel
-    attr_accessor :slack_builders
-    attr_accessor :xcode_workspace
-    attr_accessor :xcode_test_scheme
-    attr_accessor :build_log_dir
-    attr_accessor :pull_request_build_script
-    attr_accessor :branch_build_script
-    attr_accessor :pull_request_root_dir
-    attr_accessor :branch_root_dir
-    attr_accessor :allowed_build_branches
-    attr_accessor :kill_build_after_mins
-    attr_accessor :server_base_uri
-    attr_accessor :mongo_uri
+    ATTRS = [
+      :github_webhook_port,
+      :github_webhook_secret_token,
+      :github_webhook_repo_full_name,
+      :github_api_token,
+      :slack_api_token,
+      :slack_build_channel,
+      :slack_builders,
+      :build_log_dir,
+      :build_log_limit,
+      :pull_request_build_script,
+      :branch_build_script,
+      :pull_request_root_dir,
+      :branch_root_dir,
+      :allowed_build_branches,
+      :kill_build_after_mins,
+      :server_base_uri,
+      :mongo_uri,
+      :hud_secret_token,
+      :hud_image_dir,
+    ]
+    attr_accessor(*ATTRS)
   end
 
   class << self
     def configure
-      Config.github_webhook_port = 4567
-      Config.kill_build_after_mins = 30
-      Config.mongo_uri = 'mongodb://localhost:27017/build-buddy'
+      config.github_webhook_port = 4567
+      config.kill_build_after_mins = 30
+      config.mongo_uri = 'mongodb://localhost:27017/build-buddy'
+      config.build_log_limit = 30
       block_given? ? yield(Config) : Config
+      config.build_log_dir = File.expand_path(Config.build_log_dir.gsub(/\$(\w+)/) { ENV[$1] })
+      config.hud_image_dir = File.expand_path(Config.hud_image_dir.gsub(/\$(\w+)/) { ENV[$1] })
+      Config::ATTRS.map {|attr| ('@' + attr.to_s).to_sym }.each {|var|
+        if config.instance_variable_get(var).nil?
+          raise "Config value '#{var.to_s.delete('@')}' not set"
+        end
+      }
     end
 
     def config

data/lib/build_buddy/gitter.rb

@@ -14,14 +14,13 @@ module BuildBuddy
       info "Connected to Github"
     end
 
+    # state is one of :pending, :killed, :failure, :error, :success
     def set_status(repo_full_name, repo_sha, state, description, target_url)
       options = {
-          :description => description.length > 140 ? "#{description[0..136]}..." : description,
-          :context => 'build-buddy'
+        :description => description.length > 140 ? "#{description[0..136]}..." : description,
+        :context => 'build-buddy',
+        :target_url => target_url || ''
       }
-      unless target_url.nil?
-        options[:target_url] = target_url
-      end
       @gh_client.create_status(repo_full_name, repo_sha, state.to_s, options)
     end
   end

data/lib/build_buddy/recorder.rb

@@ -2,6 +2,7 @@ require 'rubygems'
 require 'bundler'
 require 'celluloid'
 require 'mongo'
+require 'gruff'
 require_relative './config.rb'
 
 module BuildBuddy
@@ -9,6 +10,8 @@ module BuildBuddy
     include Celluloid
     include Celluloid::Internals::Logger
 
+    LIMIT = 50
+
     def initialize()
       Mongo::Logger.logger.level = ::Logger::FATAL
       mongo_uri = BuildBuddy::Config.mongo_uri
@@ -20,28 +23,261 @@ module BuildBuddy
     def record_build_data(build_data)
       builds = @mongo[:builds]
       result = builds.insert_one(build_data.to_h)
-      build_data._id = result.inserted_id
+      if build_data._id.nil?
+        build_data._id = result.inserted_id
+      end
     end
 
     def update_build_data(build_data)
-      unless build_data._id.nil?
-        builds = @mongo[:builds]
-        builds.replace_one({ :_id => build_data._id }, build_data.to_h)
+      if build_data._id.nil?
+        return
       end
+
+      builds = @mongo[:builds]
+      builds.replace_one({ :_id => build_data._id }, build_data.to_h)
+
+      gen_charts
     end
 
     def get_build_data(id)
-      document = @mongo[:builds].find({ :_id => BSON::ObjectId.from_string(id) }, { :limit => 1 }).first
-      if document.nil?
+      doc = @mongo[:builds].find({ :_id => BSON::ObjectId.from_string(id) }, { :limit => 1 }).first
+      if doc.nil?
         return nil
       end
-      BuildData.new(document)
+      BuildData.new(doc)
     end
 
     def get_build_data_history(limit)
-      @mongo[:builds].find().sort(:start_time => -1).limit(limit).map do |document|
-        BuildData.new(document)
+      @mongo[:builds].find().sort(:start_time => -1).limit(limit).map do |doc|
+        BuildData.new(doc)
+      end
+    end
+    
+    def gen_charts
+      gen_daily_builds_chart
+      gen_coverage_chart
+      gen_lines_chart
+      gen_localized_chart
+      gen_times_chart
+      gen_warnings_chart
+    end
+    
+    def gen_daily_builds_chart
+      # Generates a chart showing builds per day
+      data = []
+      keys = []
+      @mongo[:builds].find().sort(:start_time => -1).limit(LIMIT).each do |doc|
+        next if doc['start_time'].nil?
+        start_data_components = doc['start_time'].to_s.split(' ')[0].split('-')
+        date_string = start_data_components[1] + "/" + start_data_components[2]
+    
+        if keys.include?(date_string)
+          data.each { |arr|
+            if arr[0] == date_string
+              if doc['pull_request'] != nil
+                arr[1] = arr[1] + 1
+              else
+                arr[2] = arr[2] + 1
+              end
+            end
+          }
+        else
+          keys.push(date_string)
+          if doc['pull_request'] != nil
+            data.insert(0, [date_string, 1, 0])
+          else
+            data.insert(0, [date_string, 0, 1])
+          end
+        end
+        end
+  
+      g = Gruff::StackedBar.new
+      g.title = "Builds per Day"
+      g.labels = data.each_with_index.map {|d, i| [ i, d[0] ]}.to_h
+      g.theme = {
+          :colors => ['#12a702', '#aedaa9'],
+          :font_color => 'black',
+          :background_colors => 'white'
+      }
+      g.font = '/Library/Fonts/Verdana.ttf'
+      g.data("Pull Request (#{data.size > 0 ? data[data.size - 1][1] : 0})", data.map {|d| d[1]})
+      g.data("Other (#{data.size > 0 ? data[data.size - 1][2] : 0})", data.map {|d| d[2]})
+      g.x_axis_label = 'Date'
+      g.y_axis_label = 'Number of Builds'
+      g.y_axis_increment = 2
+      g.write File.join(Config.hud_image_dir, "daily_builds.png")
+    end
+
+    def gen_coverage_chart
+      # Generate code coverage numbers
+      data = []
+      keys = []
+      @mongo[:builds].find().sort(:start_time => -1).limit(LIMIT).map do |doc|
+        next if doc['start_time'].nil?
+        start_date_components = doc['start_time'].to_s.split(' ')[0].split('-')
+        date_string = start_date_components[1] + "/" + start_date_components[2]
+        pull_number = doc['pull_request'].to_i
+        if pull_number == 0 && doc['metrics']['coverage'] != nil && !keys.include?(date_string)
+          keys.push(date_string)
+
+          missed_lines = doc['metrics']['coverage']['swift_files']['missed_lines'] + doc['metrics']['coverage']['m_files']['missed_lines']
+          total_lines = doc['metrics']['coverage']['swift_files']['total_lines'] + doc['metrics']['coverage']['m_files']['total_lines']
+          coverage_percent = ((1.0 * (total_lines - missed_lines) / total_lines) * 100).to_i
+          data.insert(0, [date_string, coverage_percent])
+        end
+      end
+
+      g = Gruff::Bar.new
+      g.minimum_value = 0
+      g.maximum_value = 100
+      g.title = "Coverage"
+      g.labels = data.each_with_index.map {|d, i| [ i, d[0] ]}.to_h
+      g.theme = {
+          :colors => ['#822F92'],
+          :font_color => 'black',
+          :background_colors => 'white'
+      }
+      g.font = '/Library/Fonts/Verdana.ttf'
+      g.data("covered % (#{data.size > 0 ? data[data.size - 1][1] : 0})", data.map {|d| d[1]})
+      g.x_axis_label = 'Date'
+      g.y_axis_label = 'Coverage %'
+      g.write File.join(Config.hud_image_dir, "code_coverage.png")    
+    end
+
+    def gen_lines_chart
+      # Generates a chart displaying number of lines of code by Swift/Objective-C.
+      data = []
+      keys = []
+      @mongo[:builds].find().sort(:start_time => -1).limit(LIMIT).map do |doc|
+        next if doc['start_time'].nil?
+        start_date_components = doc['start_time'].to_s.split(' ')[0].split('-')
+        date_string = start_date_components[1] + "/" + start_date_components[2]
+        pull_number = doc['pull_request'].to_i
+        if pull_number == 0 && doc['metrics']['coverage'] != nil && !keys.include?(date_string)
+          keys.push(date_string)
+          data.insert(0, [date_string, doc['metrics']['coverage']['swift_files']['total_lines'], doc['metrics']['coverage']['m_files']['total_lines']])
+        end
+      end
+
+      g = Gruff::StackedBar.new
+      g.title = "Lines of Code"
+      g.labels = data.each_with_index.map {|d, i| [ i, d[0] ]}.to_h
+      g.theme = {
+          :colors => ['#C17B3A', '#C1BA3A'],
+          :font_color => 'black',
+          :background_colors => 'white'
+      }
+      g.font = '/Library/Fonts/Verdana.ttf'
+      g.data("Swift (#{data.size > 0 ? data[data.size - 1][1] : 0})", data.map {|d| d[1]})
+      g.data("Objective-C (#{data.size > 0 ? data[data.size - 1][2] : 0})", data.map {|d| d[2]})
+      g.x_axis_label = 'Day'
+      g.y_axis_label = 'Lines of Code'
+      g.write File.join(Config.hud_image_dir, "lines_of_code.png")
+    end
+
+    def gen_localized_chart
+      # Generates a chart displaying localization information for past pull requests.
+      # Displays # strings, # descriptions, and # words.
+      data = []
+      keys = []
+      @mongo[:builds].find().sort(:start_time => -1).limit(LIMIT).map do |doc|
+        pull_number = doc['pull_request'].to_i
+        if pull_number != 0 && !keys.include?(pull_number)
+          keys.push(pull_number)
+          data.insert(0, [pull_number, doc['metrics']['strings'], doc['metrics']['descriptions'], doc['metrics']['words']])
+        end
       end
+
+      g = Gruff::Line.new
+      g.title = 'Localization'
+      g.labels = data.each_with_index.map {|d, i| [ i, d[0] ]}.to_h
+      g.dot_style = :square
+      g.font = '/Library/Fonts/Verdana.ttf'
+      g.data("# strings (#{data.size > 0 ? data[data.size - 1][1] : 0})", data.map {|d| d[1]})
+      g.data("# descriptions (#{data.size > 0 ? data[data.size - 1][2] : 0})", data.map {|d| d[2]})
+      g.data("# words (#{data.size > 0 ? data[data.size - 1][3] : 0})", data.map {|d| d[3]})
+      g.x_axis_label = 'Pull Request'
+
+      g.theme = {
+          :colors => ['#e65954', '#0a2154', '#2a7b20'],
+          :font_color => 'black',
+          :background_colors => 'white'
+      }
+
+      g.write(File.join(Config.hud_image_dir, 'localization_data.png'))
+    end
+
+    def gen_times_chart
+      # Generates a chart displaying build time information for past pull requests.
+      data = []
+      keys = []
+      @mongo[:builds].find().sort(:start_time => -1).limit(LIMIT).map do |doc|
+        next if doc['metrics'].nil?
+        pull_number = doc['pull_request'].to_i
+        if pull_number != 0 && !keys.include?(pull_number) && doc['metrics']['test_build_end'] != nil && doc['metrics']['test_build_start'] != nil
+          end_time_components = doc['metrics']['test_build_end'].to_s.split(' ')[1].split(':')
+          end_date_components = doc['metrics']['test_build_end'].to_s.split(' ')[0].split('-')
+          end_time = Time.new(end_date_components[0], end_date_components[1], end_date_components[2], end_time_components[0], end_time_components[1], end_time_components[2])
+
+          start_time_components = doc['metrics']['test_build_start'].to_s.split(' ')[1].split(':')
+          start_date_components = doc['metrics']['test_build_start'].to_s.split(' ')[0].split('-')
+          start_time = Time.new(start_date_components[0], start_date_components[1], start_date_components[2], start_time_components[0], start_time_components[1], start_time_components[2])
+
+          build_time_mins = ((end_time - start_time) / 60).to_f.round(2)
+
+          keys.push(pull_number)
+          data.insert(0, [pull_number, build_time_mins])
+        end
+      end
+
+      g = Gruff::Line.new
+      g.minimum_value = 0
+      g.maximum_value = 10
+      g.title = 'Build Times'
+      g.labels = data.each_with_index.map {|d, i| [ i, d[0] ]}.to_h
+      g.dot_style = :square
+      g.font = '/Library/Fonts/Verdana.ttf'
+      g.data("minutes (#{data.size > 0 ? data[data.size - 1][1] : 0})", data.map {|d| d[1]})
+      g.x_axis_label = 'Pull Request'
+      g.y_axis_label = 'Time (mins)'
+      g.theme = {
+          :colors => ['#38c0df'],
+          :font_color => 'black',
+          :background_colors => 'white'
+      }
+
+      g.write(File.join(Config.hud_image_dir, 'build_times.png'))
+    end
+
+    def gen_warnings_chart
+      # Script that generates a chart displaying warning count information for past pull requests.
+      data = []
+      keys = []
+      @mongo[:builds].find().sort(:start_time => -1).limit(LIMIT).map do |doc|
+        next if doc['start_time'].nil?
+        start_date_components = doc['start_time'].to_s.split(' ')[0].split('-')
+        date_string = start_date_components[1] + "/" + start_date_components[2]
+        pull_number = doc['pull_request'].to_i
+        if pull_number == 0 && doc['metrics']['test_build_start'] != nil && !keys.include?(date_string)
+          keys.push(date_string)
+          data.insert(0, [date_string, doc['metrics']['warning_count']])
+        end
+      end
+
+      g = Gruff::Line.new
+      g.title = 'Warnings'
+      g.labels = data.each_with_index.map {|d, i| [ i, d[0] ]}.to_h
+      g.dot_style = :square
+      g.font = '/Library/Fonts/Verdana.ttf'
+      g.data("# warnings (#{data.size > 0 ? data[data.size - 1][1] : 0})", data.map {|d| d[1]})
+      g.x_axis_label = 'Date'
+      g.y_axis_label = 'Number of Warnings'
+      g.theme = {
+          :colors => ['#c3a00b'],
+          :font_color => 'black',
+          :background_colors => 'white'
+      }
+      g.write(File.join(Config.hud_image_dir, 'warning_count.png'))
     end
   end
 end