shiftzilla 0.2.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 97d7153e749c49c28648b85cd0e2a9704d938b0a462b36d08d506458943d0441
+  data.tar.gz: 85af213bfdf7b6ae5c8854dbef2e5f6bd99233edc3f91a292397cf4de5e1f494
+SHA512:
+  metadata.gz: 2d00e6716bdffab9b6266021b95aafd6f5d38f809500a12a9f46bdbf9a3225ab5ab6d17cfa0ec61be8a2d138b7e527e7eddfbdb31779f346cc29c562050f6770
+  data.tar.gz: ac4499fd1bcf2f048672b528ab422494386f4a2ec18fb2cf0d6e9327d7f5af0ecff3b7bcc7f397b3f7d10ed41104c3207e60777cace213ba71c397ca81b1fedc

data/.gitignore

@@ -0,0 +1,4 @@
+*.swp
+*.png
+Gemfile.lock
+*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,20 @@
+# Shiftzilla
+a.k.a "The tool that we made because Bugzilla lacks any meaingful aggreation reporting."
+
+This is a specialized tool for aggregating Bugzilla records in a way that is useful for some development teams. In order to use it:
+
+1. This utility depends on a python-based tool called [python-bugzilla](https://pypi.python.org/pypi/python-bugzilla).
+    * Install it so that the `bugzilla` executable is in your $PATH
+    * Configure it by running `bugzilla login`
+2. Next grab this utility from RubyGems:
+    * gem install shiftzilla
+    * Run any command (like `shiftzilla summary`) to have the utility set up your local $HOME/.shiftzilla directory
+3. Edit $HOME/.shiftzilla/shiftzilla_cfg.yml to reflect the right organizational info for your teams and groups, plus the saved reports in Bugzilla that you want to draw data from. The utlity expects three tables:
+    * One for _all_ team bugs
+    * One for bugs filtered by the release that you are tracking
+    * One for bugs identified as test blockers by your QE team
+
+With all of this done, you can start to run reports (or even set up cron jobs around them):
+* `shiftzilla load` polls bugzilla and stores info in a local SQLite3 database
+* `shiftzilla summary` gives you an overview report in your terminal
+* `shiftzilla build` generates an overall and team-by-team reports that it will push to a web server as static web pages

data/bin/shiftzilla

@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+
+require 'shiftzilla/engine'
+require 'shiftzilla/helpers'
+require 'shiftzilla/version'
+require 'trollop'
+
+include Shiftzilla::Engine
+include Shiftzilla::Helpers
+
+SUB_COMMANDS = %w{help version load build purge triage}
+Trollop::options do
+  version Shiftzilla::VERSION
+  banner <<-EOF
+Usage:
+  #$0 <command>
+
+Commands:
+  load
+    Queries Bugzilla and loads records into the local DB
+  build
+    Generates reports based local DB contents
+  purge
+    Removes today's records from the local DB
+  triage
+    Generate a report of teams that need to do bug triage
+
+Options:
+EOF
+  stop_on SUB_COMMANDS
+end
+
+cmd = ARGV.shift
+cmd_opts = case cmd
+when 'load'
+  Trollop::options do
+    banner <<-EOF
+Usage:
+  #$0 load <options>
+
+Description:
+  Runs all of the queries in the 'Queries' portion of the
+  config file, and stores info about each retrieved Bugzilla record
+  in a local database
+
+Options:
+EOF
+  end
+when 'build'
+  Trollop::options do
+    banner <<-EOF
+Usage:
+  #$0 build <options>
+
+Description:
+  Performs aggregations and generates reports based on those,
+  which are rsynced to a target location.
+
+Options:
+EOF
+    opt :local_preview, "Don't publish, just show the report locally", :default => false
+    opt :quiet, "For cron use; don't print anything to STDOUT.", :default => false
+  end
+when 'purge'
+  Trollop::options do
+    banner <<-EOF
+Usage:
+  #$0 purge <options>
+
+Description:
+  Purges today's data from the local database
+
+Options:
+EOF
+  end
+when 'triage'
+  Trollop::options do
+    banner <<-EOF
+Usage:
+  #$0 triage <options>
+
+Description:
+  Generate a report of bugs with no target release,
+  ranked by team / component and age.
+
+Options:
+EOF
+  end
+when 'help'
+  Trollop::educate
+when 'version'
+  puts Shiftzilla::VERSION
+  exit 0
+end
+
+# Check for the config info
+check_config
+
+case cmd
+when 'load'
+  load_records
+when 'build'
+  build_reports(cmd_opts)
+when 'purge'
+  purge_records
+when 'triage'
+  triage_report
+end
+
+exit

data/lib/shiftzilla/bug.rb

@@ -0,0 +1,67 @@
+module Shiftzilla
+  class Bug
+    attr_reader :id, :first_seen, :last_seen, :test_blocker, :ops_blocker, :owner, :component, :pm_score, :cust_cases, :tgt_release
+
+    def initialize(bzid,binfo)
+      @id           = bzid
+      @first_seen   = binfo[:snapdate]
+      @last_seen    = binfo[:snapdate]
+      @test_blocker = binfo[:test_blocker]
+      @ops_blocker  = binfo[:ops_blocker]
+      @owner        = binfo[:owner]
+      @summary      = binfo[:summary]
+      @component    = binfo[:component]
+      @pm_score     = binfo[:pm_score]
+      @cust_cases   = binfo[:cust_cases]
+      @tgt_release  = binfo[:tgt_release]
+    end
+
+    def update(binfo)
+      @last_seen    = binfo[:snapdate]
+      @test_blocker = binfo[:test_blocker]
+      @ops_blocker  = binfo[:ops_blocker]
+      @owner        = binfo[:owner]
+      @summary      = binfo[:summary]
+      @component    = binfo[:component]
+      @pm_score     = binfo[:pm_score]
+      @cust_cases   = binfo[:cust_cases]
+      @tgt_release  = binfo[:tgt_release]
+    end
+
+    def summary
+      @summary[0..30].gsub(/\s\w+\s*$/, '...')
+    end
+
+    def age
+      (@last_seen - @first_seen).to_i
+    end
+
+    def semver
+      parts = @tgt_release.split('.')
+      if parts.length == 1
+        return @tgt_release
+      end
+      semver = ''
+      first_part = true
+      parts.each do |part|
+        unless is_number?(part)
+          semver += part
+        else
+          semver += ("%09d" % part).to_s
+        end
+        # A version like '3.z' gets a middle 0 for sort purposes.
+        if first_part and parts.length == 2
+          semver += ("%09d" % 0).to_s
+        end
+        first_part = false
+      end
+      return semver
+    end
+
+    private
+
+    def is_number?(val)
+      val.to_f.to_s == val.to_s || val.to_i.to_s == val.to_s
+    end
+  end
+end

data/lib/shiftzilla/config.rb

@@ -0,0 +1,98 @@
+require 'date'
+require 'shiftzilla/group'
+require 'shiftzilla/release'
+require 'shiftzilla/source'
+require 'shiftzilla/team'
+
+module Shiftzilla
+  class Config
+    attr_reader :teams, :groups, :sources, :releases, :ssh
+
+    def initialize
+      @teams    = []
+      @groups   = []
+      @sources  = []
+      group_map = {}
+      cfg_file['Groups'].each do |group|
+        gobj = Shiftzilla::Group.new(group)
+        @groups << gobj
+        group_map[gobj.id] = gobj
+      end
+      cfg_file['Teams'].each do |team|
+        @teams << Shiftzilla::Team.new(team,group_map)
+      end
+      cfg_file['Sources'].each do |sid,sinfo|
+        @sources << Shiftzilla::Source.new(sid,sinfo)
+      end
+      # Always track a release for bugs with no target release
+      @releases = [Shiftzilla::Release.new({ 'name' => '"---"', 'targets' => ['---'] },true)]
+      cfg_file['Releases'].each do |release|
+        @releases << Shiftzilla::Release.new(release)
+      end
+      @releases << Shiftzilla::Release.new({ 'name' => 'All', 'targets' => [] },true)
+      @ssh = {
+        :host => cfg_file['SSH']['host'],
+        :path => cfg_file['SSH']['path'],
+        :url  => cfg_file['SSH']['url'],
+      }
+    end
+
+    def earliest_milestone
+      milestone_boundaries[:earliest]
+    end
+
+    def latest_milestone
+      milestone_boundaries[:latest]
+    end
+
+    def team(tname)
+      @teams.select{ |t| t.name == tname }[0]
+    end
+
+    def add_ad_hoc_team(tinfo)
+      @teams << Shiftzilla::Team.new(tinfo,{},true)
+    end
+
+    def release(rname)
+      @releases.select{ |r| r.name == rname }[0]
+    end
+
+    def release_by_target(tgt)
+      return target_map[tgt]
+    end
+
+    private
+
+    def target_map
+      @target_map ||= begin
+        tmap = {}
+        @releases.each do |release|
+          release.targets.each do |target|
+            tmap[target] = release
+          end
+        end
+        tmap
+      end
+    end
+
+    def milestone_boundaries
+      @milestone_boundaries ||= begin
+        boundaries = { :earliest => Date.today, :latest => (Date.today - 1800) }
+        @releases.each do |release|
+          next unless release.uses_milestones?
+          ms = release.milestones
+          [ms.start,ms.feature_complete,ms.code_freeze,ms.ga].each do |m|
+            next if m.date.nil?
+            if m.date < boundaries[:earliest]
+              boundaries[:earliest] = m.date
+            end
+            if m.date > boundaries[:latest]
+              boundaries[:latest] = m.date
+            end
+          end
+        end
+        boundaries
+      end
+    end
+  end
+end

data/lib/shiftzilla/engine.rb

@@ -0,0 +1,183 @@
+require 'highline/import'
+require 'shiftzilla/config'
+require 'shiftzilla/helpers'
+require 'shiftzilla/org_data'
+require 'terminal-table'
+require 'trollop'
+
+include Shiftzilla::Helpers
+
+module Shiftzilla
+  module Engine
+    def check_config
+      if not File.directory?(SZA_DIR)
+        choose do |menu|
+          menu.header = 'You don\'t have a Shiftzilla config directory at $HOME/.shiftzilla. Should I create one?'
+          menu.prompt = 'Choice?'
+          menu.choice(:yes) {
+            say('Okay. Creating config directory.')
+            Dir.mkdir(SZA_DIR)
+            Dir.mkdir(ARCH_DIR)
+          }
+          menu.choice(:no) {
+            say('Okay. Exiting Shiftzilla.')
+            exit
+          }
+        end
+      end
+      if not File.directory?(ARCH_DIR)
+        puts 'You don\'t have an archive directory at $HOME/.shiftzilla/archive. Creating it.'
+        Dir.mkdir(ARCH_DIR)
+      end
+      if not File.exists?(CFG_FILE)
+        choose do |menu|
+          menu.header = "\nYou don't have a shiftzilla_cfg.yml file in $HOME/.shiftzilla. Should I create one?"
+          menu.prompt = 'Choice?'
+          menu.choice(:yes) {
+            say('Okay. Creating shiftzilla_cfg.yml')
+            FileUtils.cp(CFG_TMPL,CFG_FILE)
+          }
+          menu.choice(:no) {
+            say('Okay. Exiting Shiftzilla.')
+            exit
+          }
+        end
+      end
+      if not File.exists?(DB_FPATH)
+        choose do |menu|
+          menu.header = "\nYou don't have a shiftzilla.sqlite file in $HOME/.shiftzilla.\nI can create it for you, but it is very important for you to\nconfigure Shiftzilla by puttng the proper settings in\n$HOME/.shiftzilla/shiftzilla_cfg.yml first. Do you want me to proceed with creating the database?"
+          menu.prompt = 'Choice?'
+          menu.choice(:yes) {
+            say('Okay. Creating shiftzilla.sqlite')
+            sql_tmpl       = File.read(SQL_TMPL)
+            tgt_rel_clause = release_clause(releases)
+            sql_tmpl.gsub! '$RELEASE_CLAUSE', tgt_rel_clause
+            dbh.execute_batch(sql_tmpl)
+            dbh.close
+            exit
+          }
+          menu.choice(:no) {
+            say('Okay. Exiting Shiftzilla.')
+            exit
+          }
+        end
+      end
+    end
+
+    def load_records
+      sources.each do |s|
+        if s.has_records_for_today?
+          puts "Skipping query for #{s.id}; it already has records for today."
+        else
+          backup_db
+          puts "Querying bugzilla for #{s.id}"
+          added_count = s.load_records
+          puts "Added #{added_count} records to #{s.table}"
+        end
+      end
+    end
+
+    def purge_records
+      sources.each do |s|
+        s.purge_records
+      end
+      puts "Purged #{sources.length} tables."
+    end
+
+    def triage_report
+      org_data = Shiftzilla::OrgData.new(shiftzilla_config)
+      org_data.populate_releases
+      teams      = org_data.get_ordered_teams
+      no_tgt_rel = shiftzilla_config.releases[0]
+
+      teams.each do |tname|
+        next if tname == '_overall'
+        rdata = org_data.get_release_data(tname,no_tgt_rel)
+        next if rdata.nil? or rdata.snaps.empty?
+        recipients = {}
+        team  = shiftzilla_config.team(tname)
+        unless team.nil? or team.ad_hoc?
+          recipients[team.lead] = 1
+          recipients[team.group.lead] = 1
+        end
+        bzids = rdata.snaps.has_key?(latest_snapshot) ? rdata.snaps[latest_snapshot].bug_ids : []
+        next if bzids.length == 0
+        bugs = rdata.bugs
+        table = Terminal::Table.new do |t|
+          t << ['URL','Age','Component','Owner','Summary']
+          t << :separator
+          bzids.sort_by{ |b| [bugs[b].first_seen,bugs[b].component] }.each do |bzid|
+            bug = bugs[bzid]
+            if team.nil?
+              recipients[bug.owner] = 1
+            end
+            t << [
+              bug_url(bzid),
+              bug.age,
+              bug.component,
+              bug.owner,
+              bug.summary
+            ]
+          end
+        end
+        puts "#{tname}#{team.nil? ? ' Component' : ' Team'} - #{bzids.length} bugs with no Target Release"
+        puts "To: #{recipients.keys.sort.join(',')}"
+        puts "#{table}\n\n"
+      end
+    end
+
+    def build_reports(options)
+      org_data = Shiftzilla::OrgData.new(shiftzilla_config)
+      org_data.populate_releases
+      org_data.build_series
+      org_data.generate_reports
+      if options[:local_preview]
+        org_data.show_local_reports
+      else
+        org_data.publish_reports(ssh)
+        system("rm -rf #{org_data.tmp_dir}")
+        unless options[:quiet]
+          system("open #{ssh[:url]}")
+        end
+      end
+    end
+
+    private
+
+    def shiftzilla_config
+      @shiftzilla_config ||= Shiftzilla::Config.new
+    end
+
+    def teams
+      @teams ||= shiftzilla_config.teams
+    end
+
+    def groups
+      @groups ||= shiftzilla_config.groups
+    end
+
+    def sources
+      @sources ||= shiftzilla_config.sources
+    end
+
+    def releases
+      @releases ||= shiftzilla_config.releases
+    end
+
+    def ssh
+      @ssh ||= shiftzilla_config.ssh
+    end
+
+    def component_team_map
+      @component_team_map ||= begin
+        ctm = {}
+        teams.each do |team|
+          team.components.each do |component|
+            ctm[component] = team
+          end
+        end
+        ctm
+      end
+    end
+  end
+end

data/lib/shiftzilla/group.rb

@@ -0,0 +1,10 @@
+module Shiftzilla
+  class Group
+    attr_reader :id, :lead
+
+    def initialize(ginfo)
+      @id    = ginfo['id']
+      @lead  = ginfo['lead']
+    end
+  end
+end

data/lib/shiftzilla/helpers.rb

@@ -0,0 +1,174 @@
+require 'date'
+require 'fileutils'
+require 'haml'
+require 'fileutils'
+require 'sqlite3'
+require 'tmpdir'
+require 'yaml'
+
+module Shiftzilla
+  module Helpers
+    BZ_URL     = 'https://bugzilla.redhat.com/show_bug.cgi?id='
+    SZA_DIR    = File.join(ENV['HOME'],'.shiftzilla')
+    ARCH_DIR   = File.join(SZA_DIR,'archive')
+    CFG_FILE   = File.join(SZA_DIR,'shiftzilla_cfg.yml')
+    DB_FNAME   = 'shiftzilla.sqlite'
+    DB_FPATH   = File.join(SZA_DIR,DB_FNAME)
+    THIS_PATH  = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+    HAML_TMPL  = File.expand_path(File.join(File.dirname(THIS_PATH), '../../template.haml'))
+    CFG_TMPL   = File.expand_path(File.join(File.dirname(THIS_PATH), '../../shiftzilla_cfg.yml.tmpl'))
+    SQL_TMPL   = File.expand_path(File.join(File.dirname(THIS_PATH), '../../shiftzilla.sql.tmpl'))
+    VENDOR_DIR = File.expand_path(File.join(File.dirname(THIS_PATH), '../../vendor'))
+
+    GRAPH_DIMENSIONS = '800x400'
+    GRAPH_THEME      = {
+      :colors => [
+        '#268bd2', # Blue
+        '#cb4b16', # Orange
+        '#859900', # Green
+        '#2aa198', # Cyan
+        '#d33682', # Magenta
+        '#6c71c4', # Violet
+        '#b58900', # Yellow
+        '#dc322f', # Red
+      ],
+      :marker_color      => '#93a1a1', # Base1
+      :font_color        => '#586e75', # Base01
+      :background_colors => '#fdf6e3', # Base3
+      :background_image  => nil,
+    }
+
+    def tmp_dir
+      @tmp_dir ||= Dir.mktmpdir('shiftzilla-reports-')
+    end
+
+    def cfg_file
+      @cfg_file ||= YAML.load_file(CFG_FILE)
+    end
+
+    def dbh
+      @dbh ||= SQLite3::Database.new(DB_FPATH)
+    end
+
+    def haml_engine
+      @haml_engine ||= Haml::Engine.new(File.read(HAML_TMPL))
+    end
+
+    def new_graph(labels,max_y)
+      g = Gruff::Line.new(GRAPH_DIMENSIONS)
+      g.theme            = GRAPH_THEME
+      g.line_width       = 2
+      g.labels           = labels
+      g.y_axis_increment = set_axis_increment(max_y)
+      g.hide_dots        = true
+      return g
+    end
+
+    def backup_db
+      unless db_backed_up
+        today = Date.today.strftime('%Y-%m-%d')
+        tpath = File.join(ARCH_DIR,today)
+        unless Dir.exists?(tpath)
+          Dir.mkdir(tpath)
+        end
+        apath    = ''
+        copy_idx = 0
+        loop do
+          copynum = "%02d" % copy_idx
+          apath   = File.join(tpath,"#{copynum}-#{DB_FNAME}")
+          break unless File.exists?(apath)
+          copy_idx += 1
+        end
+        FileUtils.cp DB_FPATH, apath
+        puts "Backed up the database."
+        @db_backed_up = true
+      end
+    end
+
+    def timestamp
+      DateTime.now.to_s
+    end
+
+    def bug_url(bug_id)
+      return "#{BZ_URL}#{bug_id}"
+    end
+
+    def set_axis_increment(initial_value)
+      case
+      when initial_value < 10
+        return 1
+      when initial_value < 20
+        return 2
+      when initial_value < 50
+        return 5
+      when initial_value < 100
+        return 10
+      when initial_value < 200
+        return 20
+      when initial_value < 400
+        return 25
+      else
+        return 100
+      end
+    end
+
+    def all_bugs_query(snapshot)
+      return "SELECT AB.'Bug ID', AB.'Component', AB.'Target Release', AB.'Assignee', AB.'Status', AB.'Summary', AB.'Keywords', AB.'PM Score', AB.'External Bugs' FROM ALL_BUGS AB WHERE AB.'Snapshot' = date('#{snapshot}')"
+    end
+
+    def component_bugs_query(components,snapshot)
+      rclause = list_clause(components)
+      rfilter = rclause == '' ? '' : " AND AB.'Component' #{rclause}"
+      return "SELECT AB.'Bug ID', AB.'Component', AB.'Target Release', AB.'Assignee', AB.'Status', AB.'Summary', AB.'Keywords', AB.'PM Score', AB.'External Bugs' FROM ALL_BUGS AB WHERE AB.'Snapshot' = date('#{snapshot}')#{rfilter} ORDER BY AB.'Target Release' DESC"
+    end
+
+    def component_bugs_count(components,snapshot)
+      rclause = list_clause(components)
+      rfilter = rclause == '' ? '' : " AND AB.'Component' #{rclause}"
+      return "SELECT count(*) FROM ALL_BUGS AB WHERE AB.'Snapshot' = date('#{snapshot}')#{rfilter}"
+    end
+
+    def all_snapshots
+      @all_snapshots ||= begin
+        all_snapshots = []
+        dbh.execute('SELECT DISTINCT Snapshot FROM ALL_BUGS ORDER BY Snapshot ASC') do |row|
+          all_snapshots << row[0]
+        end
+        all_snapshots
+      end
+    end
+
+    def latest_snapshot
+      all_snapshots[-1]
+    end
+
+    def field_map
+      {
+        :assigned_to      => 'Assignee',
+        :component        => 'Component',
+        :id               => 'Bug ID',
+        :keywords         => 'Keywords',
+        :cf_pm_score      => 'PM Score',
+        :status           => 'Status',
+        :summary          => 'Summary',
+        :target_release   => 'Target Release',
+        :external_bugs    => 'External Bugs',
+      }
+    end
+
+    private
+
+    def list_clause(list)
+      return '' if list.length == 0
+      if list.length > 1
+        stmt = list.map{ |t| "'#{t}'" }.join(',')
+        return "IN (#{stmt})"
+      end
+      return "== '#{list[0]}'"
+    end
+
+    def db_backed_up
+      @db_backed_up ||= false
+    end
+  end
+end

data/lib/shiftzilla/milestone.rb

@@ -0,0 +1,20 @@
+module Shiftzilla
+  class Milestone
+    def initialize(mtxt)
+      @date  = nil
+      @stamp = ''
+      unless (mtxt.nil? or mtxt == '')
+        @date  = Date.parse(mtxt)
+        @stamp = mtxt
+      end
+    end
+
+    def date
+      @date
+    end
+
+    def stamp
+      @stamp
+    end
+  end
+end