shiftzilla 0.2.6

data/lib/shiftzilla/milestones.rb

@@ -0,0 +1,28 @@
+require 'shiftzilla/milestone'
+
+module Shiftzilla
+  class Milestones
+    def initialize(msrc)
+      @milestones = {}
+      msrc.each do |key,val|
+        @milestones[key.to_sym] = Shiftzilla::Milestone.new(val)
+      end
+    end
+
+    def start
+      @milestones[:start]
+    end
+
+    def feature_complete
+      @milestones[:feature_complete]
+    end
+
+    def code_freeze
+      @milestones[:code_freeze]
+    end
+
+    def ga
+      @milestones[:ga]
+    end
+  end
+end

data/lib/shiftzilla/org_data.rb

@@ -0,0 +1,332 @@
+require 'json'
+require 'shiftzilla/bug'
+require 'shiftzilla/helpers'
+require 'shiftzilla/team_data'
+
+include Shiftzilla::Helpers
+
+module Shiftzilla
+  class OrgData
+    attr_reader :tmp_dir
+
+    def initialize(config)
+      @config          = config
+      @teams           = config.teams
+      @releases        = config.releases
+      @tmp_dir         = Shiftzilla::Helpers.tmp_dir
+      @org_data        = { '_overall' => Shiftzilla::TeamData.new('_overall') }
+    end
+
+    def populate_releases
+      all_snapshots.each do |snapshot|
+        snapdate = Date.parse(snapshot)
+        next if snapdate < @config.earliest_milestone
+        break if snapdate > @config.latest_milestone
+        break if snapdate > Date.today
+
+        dbh.execute(all_bugs_query(snapshot)) do |row|
+          bzid = row[0].strip
+          comp = row[1].strip
+          tgtr = row[2].strip
+          owns = row[3].strip
+          stat = row[4].strip
+          summ = row[5].strip
+          keyw = row[6].nil? ? '' : row[6].strip
+          pmsc = row[7].nil? ? '0' : row[7].strip
+          cust = row[8].nil? ? 0 : row[8]
+
+          # Package up bug data
+          binfo = {
+            :snapdate     => snapdate,
+            :test_blocker => keyw.include?('TestBlocker'),
+            :ops_blocker  => keyw.include?('OpsBlocker'),
+            :owner        => owns,
+            :summary      => summ,
+            :component    => comp,
+            :pm_score     => pmsc,
+            :cust_cases   => (cust == 1),
+            :tgt_release  => tgtr,
+          }
+
+          tgt_release = @config.release_by_target(tgtr)
+          all_release = @config.release('All')
+
+          # If this component isn't mapped to a team, stub out a fake team.
+          tname = comp_map.has_key?(comp) ? comp_map[comp] : "(?) #{comp}"
+          unless @org_data.has_key?(tname)
+            @config.add_ad_hoc_team({ 'name' => tname, 'components' => [comp] })
+            @org_data[tname] = Shiftzilla::TeamData.new(tname)
+          end
+
+          team_rdata = tgt_release.nil? ? nil : @org_data[tname].get_release_data(tgt_release)
+          team_adata = @org_data[tname].get_release_data(all_release)
+          over_rdata = tgt_release.nil? ? nil : @org_data['_overall'].get_release_data(tgt_release)
+          over_adata = @org_data['_overall'].get_release_data(all_release)
+
+          # Do some bean counting
+          [over_rdata,team_rdata,over_adata,team_adata].each do |group|
+            next if group.nil?
+            snapdata = group.get_snapdata(snapshot)
+            if group.first_snap.nil?
+              group.first_snap     = snapshot
+              group.first_snapdate = snapdate
+            end
+            group.latest_snap     = snapshot
+            group.latest_snapdate = snapdate
+
+            bug = group.add_or_update_bug(bzid,binfo)
+
+            # Add info to the snapshot
+            snapdata.bug_ids << bzid
+            if bug.test_blocker or bug.ops_blocker
+              snapdata.tb_ids << bzid
+            end
+            if bug.cust_cases
+              snapdata.cc_ids << bzid
+            end
+          end
+        end
+
+        # Determine new and closed bug counts by comparing the current
+        # snapshot to the previous snapshot
+        @org_data.each do |tname,tdata|
+          @releases.each do |release|
+            next unless tdata.has_release_data?(release)
+
+            rdata = tdata.get_release_data(release)
+            next unless rdata.has_snapdata?(snapshot)
+
+            if rdata.prev_snap.nil?
+              rdata.prev_snap = snapshot
+              next
+            end
+
+            currdata = rdata.get_snapdata(snapshot)
+            prevdata = rdata.get_snapdata(rdata.prev_snap)
+
+            prev_bzids = prevdata.bug_ids
+            curr_bzids = currdata.bug_ids
+            prev_tbids = prevdata.tb_ids
+            curr_tbids = currdata.tb_ids
+            prev_ccids = prevdata.cc_ids
+            curr_ccids = currdata.cc_ids
+            currdata.closed_bugs = prev_bzids.select{ |bzid| not curr_bzids.include?(bzid) }.length
+            currdata.new_bugs    = curr_bzids.select{ |bzid| not prev_bzids.include?(bzid) }.length
+            currdata.closed_tb   = prev_tbids.select{ |tbid| not curr_tbids.include?(tbid) }.length
+            currdata.new_tb      = curr_tbids.select{ |tbid| not prev_tbids.include?(tbid) }.length
+            currdata.closed_cc   = prev_ccids.select{ |ccid| not curr_ccids.include?(ccid) }.length
+            currdata.new_cc      = curr_ccids.select{ |ccid| not prev_ccids.include?(ccid) }.length
+
+            rdata.prev_snap = snapshot
+          end
+        end
+      end
+      @all_teams     = @teams.map{ |t| t.name }.concat(@org_data.keys).uniq
+      @ordered_teams = ['_overall'].concat(@all_teams.select{ |t| t != '_overall'}.sort)
+    end
+
+    def get_ordered_teams
+      @ordered_teams
+    end
+
+    def get_release_data(team_name,release)
+      if @org_data.has_key?(team_name) and @org_data[team_name].has_release_data?(release)
+        return @org_data[team_name].get_release_data(release)
+      end
+      return nil
+    end
+
+    def build_series
+      @team_files    = []
+      @ordered_teams.each do |tname|
+        unless @org_data.has_key?(tname)
+          @org_data[tname] = Shiftzilla::TeamData.new(tname)
+        end
+        tdata = @org_data[tname]
+        tinfo = @config.team(tname)
+
+        @team_files << {
+          :tname          => tdata.title,
+          :file           => tdata.file,
+          :releases       => {},
+        }
+
+        @releases.each do |release|
+          rname     = release.name
+          bug_total = 0
+          unless tdata.has_release_data?(release)
+            @team_files[-1][:releases][release.name] = bug_total
+            next
+          end
+          rdata = tdata.get_release_data(release)
+          if rdata.snaps.has_key?(latest_snapshot)
+            bug_total = rdata.snaps[latest_snapshot].total_bugs
+          end
+          @team_files[-1][:releases][release.name] = bug_total
+
+          next if rdata.first_snap.nil? and not release.uses_milestones?
+          next if release.built_in?
+
+          rdata.populate_series
+        end
+      end
+    end
+
+    def generate_reports
+      build_time  = timestamp
+      all_release = @config.release('All')
+      @ordered_teams.each do |tname|
+        tinfo = @config.team(tname)
+        tdata = @org_data[tname]
+
+        team_pinfo = {
+          :build_time      => build_time,
+          :tname           => tdata.title,
+          :tinfo           => tinfo,
+          :tdata           => tdata,
+          :team_files      => @team_files,
+          :bug_url         => BZ_URL,
+          :chart_order     => ((1..(@releases.length - 1)).to_a << 0),
+          :releases        => [],
+          :latest_snapshot => latest_snapshot,
+          :all_bugs        => [],
+        }
+
+        @releases.each do |release|
+          rname    = release.name
+          rdata    = tdata.has_release_data?(release) ? tdata.get_release_data(release) : nil
+          snapdata = nil
+          if not rdata.nil? and rdata.snaps.has_key?(latest_snapshot)
+            snapdata = rdata.snaps[latest_snapshot]
+          else
+            snapdata = Shiftzilla::SnapData.new(latest_snapshot)
+          end
+
+          release_info = {
+            :release     => release,
+            :snapdata    => snapdata,
+            :no_rdata    => (rdata.nil? ? true : false),
+            :bug_avg_age => (rdata.nil? ? 0 : rdata.bug_avg_age),
+            :tb_avg_age  => (rdata.nil? ? 0 : rdata.tb_avg_age),
+          }
+          unless rdata.nil?
+            release_info[:charts] = {
+              :burndown   => chartify('Bug Burndown',rdata.series[:date],[
+                {
+                  :label => 'Ideal Trend',
+                  :data  => rdata.series[:ideal],
+                },
+                {
+                  :label => 'Total',
+                  :data  => rdata.series[:total_bugs],
+                },
+                {
+                  :label => 'w/ Customer Cases',
+                  :data  => rdata.series[:total_cc],
+                },
+              ]),
+              :new_closed => chartify('New vs. Closed',rdata.series[:date],[
+                  {
+                    :label => 'New',
+                    :data  => rdata.series[:new_bugs],
+                  },
+                  {
+                    :label => 'Closed',
+                    :data  => rdata.series[:closed_bugs],
+                  },
+              ]),
+              :blockers   => chartify('Test / Ops Blockers',rdata.series[:date],[
+                  {
+                    :label => 'Total',
+                    :data  => rdata.series[:total_tb],
+                  },
+                  {
+                    :label => 'New',
+                    :data  => rdata.series[:new_tb],
+                  },
+                  {
+                    :label => 'Closed',
+                    :data  => rdata.series[:closed_tb],
+                  },
+              ]),
+            }
+          end
+          team_pinfo[:releases] << release_info
+
+          if rname == 'All'
+            team_pinfo[:all_bugs] = snapdata.bug_ids.map{ |id| rdata.bugs[id] }
+          end
+        end
+        team_page = haml_engine.render(Object.new,team_pinfo)
+        File.write(File.join(@tmp_dir,tdata.file), team_page)
+      end
+      # Copy flot library to build area
+      jsdir = File.join(@tmp_dir,'js')
+      Dir.mkdir(jsdir)
+      FileUtils.cp(File.join(VENDOR_DIR,'flot','jquery.flot.min.js'),jsdir)
+    end
+
+    def show_local_reports
+      system("open file://#{@tmp_dir}/index.html")
+    end
+
+    def publish_reports(ssh)
+      system("ssh #{ssh[:host]} 'rm -rf #{ssh[:path]}/*'")
+      system("rsync -avPq #{@tmp_dir}/* #{ssh[:host]}:#{ssh[:path]}/")
+    end
+
+    private
+
+    def comp_map
+      @comp_map ||= begin
+        comp_map = {}
+        @teams.each do |team|
+          team.components.each do |comp|
+            comp_map[comp] = team.name
+          end
+        end
+        comp_map
+      end
+    end
+
+    def chartify(title,dates,series)
+      modulo = label_modulo(dates.length)
+      tlist  = []
+      dates.each_with_index do |val,idx|
+        next unless (idx + 1) % modulo == 0
+        tlist << [idx,Date.parse(val).strftime('%m/%d')]
+      end
+      data = []
+      series.each do |s|
+        dlist = []
+        s[:data].each_with_index do |val,idx|
+          dlist <<[idx,val]
+        end
+        data << { :label => s[:label], :data => dlist }
+      end
+      options = {
+        :xaxis => {
+          :ticks => tlist,
+        },
+        :yaxis => {
+          :min        => 0,
+          :labelWidth => 50,
+        },
+        :tickSize => 5,
+      }
+      return { :title => title, :data => data.to_json, :options => options.to_json }
+    end
+
+    def label_modulo(date_count)
+      if date_count < 15
+        return 1
+      elsif date_count < 30
+        return 2
+      elsif date_count < 60
+        return 5
+      else
+        return 10
+      end
+    end
+  end
+end

data/lib/shiftzilla/release.rb

@@ -0,0 +1,34 @@
+require 'shiftzilla/milestones'
+
+module Shiftzilla
+  class Release
+    attr_reader :name, :targets, :milestones, :default, :token
+
+    def initialize(release,builtin=false)
+      @name       = release['name']
+      @token      = @name.tr(' .', '_')
+      @targets    = release['targets']
+      @default    = release.has_key?('default') ? release['default'] : false
+      @builtin    = builtin
+      @milestones = nil
+      if release.has_key?('milestones')
+        @milestones = Shiftzilla::Milestones.new(release['milestones'])
+      end
+    end
+
+    def uses_milestones?
+      return @milestones.nil? ? false : true
+    end
+
+    def built_in?
+      @builtin
+    end
+
+    def no_tgt_rel?
+      if @targets.length == 1 and @targets[0] == '---'
+        return true
+      end
+      false
+    end
+  end
+end

data/lib/shiftzilla/release_data.rb

@@ -0,0 +1,185 @@
+require 'shiftzilla/bug'
+require 'shiftzilla/snap_data'
+
+module Shiftzilla
+  class ReleaseData
+    attr_accessor :snaps, :bugs, :prev_snap, :latest_snap, :latest_snapdate, :first_snap, :first_snapdate, :labels, :series
+
+    def initialize(release)
+      @release         = release
+      @snaps           = {}
+      @bugs            = {}
+      @prev_snap       = nil
+      @latest_snap     = nil
+      @latest_snapdate = nil
+      @first_snap      = nil
+      @first_snapdate  = nil
+      @labels          = {}
+      @series          = {
+        :date        => [],
+        :ideal       => [],
+        :total_bugs  => [],
+        :new_bugs    => [],
+        :closed_bugs => [],
+        :total_tb    => [],
+        :new_tb      => [],
+        :closed_tb   => [],
+        :total_cc    => [],
+        :new_cc      => [],
+        :closed_cc   => [],
+      }
+    end
+
+    def max_total
+      pick_max([@snaps.values.map{ |s| s.total_bugs }])
+    end
+
+    def max_new_closed
+      pick_max([@snaps.values.map{ |s| s.new_bugs },
+                @snaps.values.map{ |s| s.closed_bugs }
+              ])
+    end
+
+    def max_tb
+      pick_max([@snaps.values.map{ |s| s.total_tb },
+                @snaps.values.map{ |s| s.new_tb },
+                @snaps.values.map{ |s| s.closed_tb }
+              ])
+    end
+
+    def max_cc
+      pick_max([@snaps.values.map{ |s| s.total_cc },
+                @snaps.values.map{ |s| s.new_cc },
+                @snaps.values.map{ |s| s.closed_cc }
+              ])
+    end
+
+    def populate_series
+      return if first_snap.nil?
+
+      # Set up the 'ideal' series
+      ideal_slope = max_total.to_f / burndown_span.to_f
+      ideal_total = max_total
+
+      range_idx = 0
+      series_range.each do |date|
+        next if date.saturday? or date.sunday?
+        snapshot = date.strftime('%Y-%m-%d')
+
+        @series[:date] << snapshot
+        @series[:ideal] << ideal_total
+        ideal_total = ideal_total < ideal_slope ? 0 : ideal_total - ideal_slope
+
+        snapdata = @snaps.has_key?(snapshot) ? @snaps[snapshot] : nil
+        if date < first_snapdate
+          snapdata = @snaps[first_snap]
+        end
+
+        ['bugs','tb','cc'].each do |set|
+          ['total','new','closed'].each do |count|
+            set_key = "#{count}_#{set}".to_sym
+            @series[set_key] << (snapdata.nil? ? nil : snapdata.send(set_key))
+          end
+        end
+
+        if range_idx % label_modulo == 0
+          @labels[range_idx] = date.strftime('%m/%d')
+        end
+        range_idx += 1
+      end
+    end
+
+    def series_range
+      if @release.uses_milestones?
+        return (@release.milestones.start.date..@release.milestones.ga.date)
+      elsif not @first_snapdate.nil? and not @latest_snapdate.nil?
+        return (@first_snapdate..@latest_snapdate)
+      end
+      return nil
+    end
+
+    def series_span
+      if @release.uses_milestones?
+        return business_days_between(@release.milestones.start.date,@release.milestones.ga.date)
+      elsif not @first_snapdate.nil? and not @latest_snapdate.nil?
+        return business_days_between(@first_snapdate,@latest_snapdate)
+      end
+      return nil
+    end
+
+    def burndown_span
+      if @release.uses_milestones?
+        return business_days_between(@release.milestones.start.date,@release.milestones.code_freeze.date) - 2
+      elsif not @first_snapdate.nil? and not @latest_snapdate.nil?
+        return business_days_between(@first_snapdate,@latest_snapdate)
+      end
+      return nil
+    end
+
+    def bug_avg_age(blockers_only=false)
+      bug_list  = blockers_only ? @bugs.values.select{ |b| b.test_blocker } : @bugs.values
+      bug_count = bug_list.length
+      age_total = bug_list.map{ |b| b.age }.sum
+      return bug_count == 0 ? 0 : (age_total / bug_count).round(1).to_s
+    end
+
+    def tb_avg_age
+      bug_avg_age(true)
+    end
+
+    def has_snapdata?(snapshot)
+      @snaps.has_key?(snapshot)
+    end
+
+    def get_snapdata(snapshot)
+      unless @snaps.has_key?(snapshot)
+        @snaps[snapshot] = Shiftzilla::SnapData.new(snapshot)
+      end
+      @snaps[snapshot]
+    end
+
+    def add_or_update_bug(bzid,binfo)
+      unless @bugs.has_key?(bzid)
+        @bugs[bzid] = Shiftzilla::Bug.new(bzid,binfo)
+      else
+        @bugs[bzid].update(binfo)
+      end
+      @bugs[bzid]
+    end
+
+    private
+
+    # Hat tip to https://stackoverflow.com/a/24753003 for this:
+    def business_days_between(start_date, end_date)
+      days_between = (end_date - start_date).to_i
+      return 0 unless days_between > 0
+      whole_weeks, extra_days = days_between.divmod(7)
+      unless extra_days.zero?
+        extra_days -= if (start_date + 1).wday <= end_date.wday
+                        [(start_date + 1).sunday?, end_date.saturday?].count(true)
+                      else
+                        2
+                      end
+      end
+      (whole_weeks * 5) + extra_days
+    end
+
+    def pick_max(series_list)
+      max_val = 0
+      series_list.each do |s|
+        smax = s.select{ |v| not v.nil? }.max
+        next if smax.nil?
+        next unless smax > max_val
+        max_val = smax
+      end
+      return max_val
+    end
+
+    def label_modulo
+      if series_span < 50
+        return 5
+      end
+      return 10
+    end
+  end
+end

data/lib/shiftzilla/snap_data.rb

@@ -0,0 +1,31 @@
+module Shiftzilla
+  class SnapData
+    attr_reader :id
+    attr_accessor :bug_ids, :tb_ids, :cc_ids, :new_bugs, :closed_bugs, :new_tb, :closed_tb, :new_cc, :closed_cc
+
+    def initialize(snapshot)
+      @id          = snapshot
+      @bug_ids     = []
+      @tb_ids      = []
+      @cc_ids      = []
+      @new_bugs    = 0
+      @closed_bugs = 0
+      @new_tb      = 0
+      @closed_tb   = 0
+      @new_cc      = 0
+      @closed_cc   = 0
+    end
+
+    def total_bugs
+      @bug_ids.length
+    end
+
+    def total_tb
+      @tb_ids.length
+    end
+
+    def total_cc
+      @cc_ids.length
+    end
+  end
+end

data/lib/shiftzilla/source.rb

@@ -0,0 +1,51 @@
+require 'shiftzilla/helpers'
+
+module Shiftzilla
+  class Source
+    attr_reader :id, :table
+    def initialize(qid,qinfo)
+      @id           = qid.to_sym
+      @search       = qinfo['search']
+      @sharer       = qinfo['sharer']
+      @table        = qinfo['table']
+      @external_sub = qinfo['external_sub']
+      @fields       = qinfo['fields'].map{ |f| f.to_sym }
+      @external_bugs_idx = @fields.index(:external_bugs)
+    end
+
+    def has_records_for_today?
+      count = 0
+      dbh.execute("SELECT count(*) FROM #{@table} WHERE Snapshot = date('now')") do |row|
+        count = row[0].to_i
+      end
+      return count > 0
+    end
+
+    def load_records
+      output_format = @fields.map{ |fld| "%{#{fld.to_s}}" }.join("\x1F")
+      table_fields  = @fields.map{ |fld| "\"#{field_map[fld]}\"" }.join(',')
+      insert_frame  = @fields.map{ |fld| '?' }.join(', ')
+      bz_command    = "bugzilla query --savedsearch #{@search} --savedsearch-sharer-id=#{@sharer} --outputformat='#{output_format}'"
+      bz_csv        = `#{bz_command}`
+      row_count     = 0
+      bz_csv.split("\n").each do |row|
+        values = row.split("\x1F")
+        if not @external_bugs_idx.nil?
+          if not @external_sub.nil? and values[@external_bugs_idx].include?(@external_sub)
+            values[@external_bugs_idx] = 1
+          else
+            values[@external_bugs_idx] = 0
+          end
+        end  
+        dbh.execute("INSERT INTO #{@table} (#{table_fields}) VALUES (#{insert_frame})", values)
+        row_count += 1
+      end
+      dbh.execute("UPDATE #{@table} SET Snapshot = date('now') WHERE Snapshot ISNULL")
+      return row_count
+    end
+
+    def purge_records
+      dbh.execute("DELETE FROM #{@table} WHERE Snapshot == date('now') OR Snapshot ISNULL")
+    end
+  end
+end

data/lib/shiftzilla/team.rb

@@ -0,0 +1,17 @@
+module Shiftzilla
+  class Team
+    attr_reader :name, :lead, :group, :components
+
+    def initialize(tinfo,group_map,ad_hoc=false)
+      @name       = tinfo['name']
+      @lead       = ad_hoc ? nil : tinfo['lead']
+      @group      = ad_hoc ? nil : group_map[tinfo['group']]
+      @components = tinfo.has_key?('components') ? tinfo['components'] : []
+      @ad_hoc     = ad_hoc
+    end
+
+    def ad_hoc?
+      @ad_hoc
+    end
+  end
+end