oss-stats 0.0.1

data/bin/promise_stats

@@ -0,0 +1,312 @@
+#!/usr/bin/env ruby
+
+require 'sqlite3'
+require 'optparse'
+require 'date'
+
+require_relative '../lib/oss_stats/log'
+require_relative '../lib/oss_stats/config/promise_stats'
+
+log.level = :info
+
+def initialize_db(path)
+  db = SQLite3::Database.new(path)
+  db.execute <<-SQL
+    CREATE TABLE IF NOT EXISTS promises (
+      id INTEGER PRIMARY KEY,
+      description TEXT NOT NULL,
+      promised_on DATE NOT NULL,
+      resolved_on DATE,
+      reference TEXT,
+      status TEXT DEFAULT 'pending'
+    );
+  SQL
+  db.close
+end
+
+def parse_date(str)
+  Date.parse(str)
+rescue ArgumentError
+  puts "Invalid date: #{str}"
+  exit 1
+end
+
+def add_promise(config, desc, date, ref)
+  db = SQLite3::Database.new(config.db_file)
+  db.execute(
+    "INSERT INTO promises (description, promised_on, reference)
+     VALUES (?, ?, ?)",
+    [desc, date.to_s, ref],
+  )
+  db.close
+  puts 'Promise added.'
+end
+
+def resolve_promise(config, date)
+  update_promise_status(config, date, 'resolved')
+end
+
+def abandon_promise(config, date)
+  update_promise_status(config, date, 'abandoned')
+end
+
+def update_promise_status(config, date, new_status)
+  db = SQLite3::Database.new(config.db_file)
+  rows = db.execute(
+    "SELECT id, description FROM promises WHERE status = 'pending'",
+  )
+  if rows.empty?
+    puts 'No pending promises.'
+    db.close
+    return
+  end
+
+  puts 'Pending promises:'
+  rows.each_with_index do |(id, desc), _i|
+    puts "#{id}. #{desc}"
+  end
+
+  print 'Enter ID to update: '
+  chosen_id = gets.strip.to_i
+  if rows.any? { |r| r[0] == chosen_id }
+    if new_status == 'resolved'
+      db.execute(
+        "UPDATE promises SET resolved_on = ?, status = 'resolved' " +
+        'WHERE id = ?',
+        [date.to_s, chosen_id],
+      )
+    elsif new_status == 'abandoned'
+      db.execute(
+        "UPDATE promises SET resolved_on = ?, status = 'abandoned' " +
+        'WHERE id = ?',
+        [date.to_s, chosen_id],
+      )
+    end
+    puts "Promise marked as #{new_status}."
+  else
+    puts 'Invalid ID.'
+  end
+  db.close
+end
+
+def edit_promise(config)
+  db = SQLite3::Database.new(config.db_file)
+  rows = db.execute(
+    'SELECT id, description, promised_on, reference FROM promises',
+  )
+  if rows.empty?
+    puts 'No promises available.'
+    db.close
+    return
+  end
+
+  puts 'All promises:'
+  rows.each_with_index do |(id, desc, date, ref), i|
+    puts "#{i + 1}. #{desc} (ID: #{id}, Date: #{date}, Ref: #{ref})"
+  end
+
+  print 'Enter ID to edit: '
+  chosen_id = gets.strip.to_i
+  entry = rows.find { |r| r[0] == chosen_id }
+
+  unless entry
+    puts 'Invalid ID.'
+    db.close
+    return
+  end
+
+  print "New description [#{entry[1]}]: "
+  new_desc = gets.strip
+  new_desc = entry[1] if new_desc.empty?
+
+  print "New date (YYYY-MM-DD) [#{entry[2]}]: "
+  new_date = gets.strip
+  new_date = entry[2] if new_date.empty?
+  new_date = parse_date(new_date)
+
+  print "New reference [#{entry[3]}]: "
+  new_ref = gets.strip
+  new_ref = entry[3] if new_ref.empty?
+
+  db.execute(
+    "UPDATE promises
+     SET description = ?, promised_on = ?, reference = ?
+     WHERE id = ?",
+    [new_desc, new_date.to_s, new_ref, chosen_id],
+  )
+  puts 'Promise updated.'
+  db.close
+end
+
+def output(fh, msg)
+  if fh
+    fh.write("#{msg}\n")
+  else
+    log.info(msg)
+  end
+end
+
+def show_status(config, include_abandoned: false)
+  db = SQLite3::Database.new(config.db_file)
+  query = "SELECT description, promised_on, reference, status
+           FROM promises WHERE status = 'pending'"
+  if include_abandoned
+    query += " OR status = 'abandoned'"
+  end
+  rows = db.execute(query)
+  db.close
+
+  fh = nil
+  if config.output
+    fh = open(config.output, 'w')
+    log.info("Generating report and writing to #{config.output}")
+  end
+
+  output(fh, config.header)
+
+  if rows.empty?
+    output(fh, 'No matching promises.')
+    return
+  end
+
+  today = Date.today
+  rows.each do |desc, promised_on, ref, status|
+    days = (today - Date.parse(promised_on)).to_i
+    label = "#{desc} (#{days} days ago)"
+    label += " [ref: #{ref}]" unless ref.empty?
+    label += " [#{status}]" if status == 'abandoned'
+    output(fh, "* #{label}")
+  end
+end
+
+def prompt_date(txt = 'Date')
+  parse_date(prompt(txt, Date.today.to_s))
+end
+
+def prompt(txt, default = nil)
+  p = txt
+  if default
+    p << " [#{default}]"
+  end
+  print "#{p}: "
+  resp = gets.strip
+  if resp.empty? && default
+    return default
+  end
+  resp
+end
+
+def main
+  if ARGV.empty? || %w{--help -h}.include?(ARGV[0])
+    puts <<~HELP
+      Usage: #{$PROGRAM_NAME} [subcommand] [options]
+
+      Subcommands:
+        add-promise         Add a new promise
+        resolve-promise     Resolve a pending promise
+        abandon-promise     Mark a promise as abandoned
+        edit-promise        Edit an existing promise
+        status              Show unresolved promises
+
+      Options:
+        --date=YYYY-MM-DD         Date of action
+        --promise="text"          Promise description
+        --reference="text"        Optional reference
+        --db-file=FILE            SQLite3 DB file
+        --include-abandoned       Show abandoned in status
+
+      Example:
+        #{$PROGRAM_NAME} add-promise --promise="Call mom" --date=2025-05-08
+    HELP
+    exit
+  end
+
+  options = {}
+  OptionParser.new do |opts|
+    opts.on('--date=DATE', 'Date (YYYY-MM-DD)') do |d|
+      options[:date] = parse_date(d)
+    end
+
+    opts.on('--promise=TEXT', 'Promise text') do |p|
+      options[:promise] = p
+    end
+
+    opts.on('--reference=TEXT', 'Optional reference') do |r|
+      options[:reference] = r
+    end
+
+    opts.on(
+      '-l LEVEL',
+      '--log-level LEVEL',
+      'Set logging level to LEVEL. [default: info]',
+    ) do |level|
+      options[:log_level] = level.to_sym
+    end
+
+    opts.on('--db-file=FILE', 'Path to DB file') do |f|
+      options[:db_file] = f
+    end
+
+    opts.on('--include-abandoned', 'Include abandoned in status') do
+      options[:include_abandoned] = true
+    end
+
+    opts.on(
+      '-o FILE',
+      '--output FILE',
+      'Write the output to FILE',
+    ) { |v| options[:output] = v }
+
+    opts.on('-m MODE', '--mode MODE',
+            %w{add resolve abandon edit status},
+            'Mode to operate in') do |m|
+      options[:mode] = m
+    end
+
+    opts.on('-h', '--help', 'Show help') do
+      puts opts
+      exit
+    end
+  end.parse!
+  log.level = options[:log_level] if options[:log_level]
+  config = OssStats::Config::Promises
+
+  if options[:config]
+    expanded_config = File.expand_path(options[:config])
+  else
+    f = config.config_file
+    expanded_config = File.expand_path(f) if f
+  end
+
+  if expanded_config && File.exist?(expanded_config)
+    log.debug("Loading config from #{expanded_config}")
+    config.from_file(expanded_config)
+  end
+  config.merge!(options)
+  log.level = config.log_level
+
+  initialize_db(config.db_file)
+
+  case config.mode
+  when 'add'
+    desc = options[:promise] || prompt('Promise')
+    date = options[:date] || promt_date
+    ref = options[:reference] || prompt('Reference (optional)')
+    add_promise(config, desc, date, ref)
+  when 'resolve'
+    date = options[:date] || prompt_date('Resolution date')
+    resolve_promise(config, date)
+  when 'abandon'
+    date = options[:date] || prompt_date('Resolution date')
+    abandon_promise(config, date)
+  when 'edit'
+    edit_promise(config)
+  when 'status'
+    show_status(config, include_abandoned: options[:include_abandoned])
+  else
+    puts "Unknown mode: #{config.mode}"
+    exit 1
+  end
+end
+
+main if __FILE__ == $PROGRAM_NAME

data/bin/repo_stats

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/oss_stats/repo_stats'
+
+# quick hack to rename this without .rb and jam methods in lib
+extend OssStats::RepoStats
+
+def main
+  parse_options
+  config = OssStats::Config::RepoStats
+  mode = config.mode
+  mode = %w{ci pr issue} if mode.include?('all')
+
+  organizations_to_process = determine_orgs_to_process
+
+  # Prepare list of repositories to process based on configuration
+  repos_to_process = []
+  if organizations_to_process.empty?
+    log.warn('No organizations or repositories configured to process. Exiting.')
+    exit 0
+  end
+
+  # get tokens early so we fail if we're missing them
+  gh_token = get_github_token!(config)
+  gh_client = Octokit::Client.new(
+    access_token: gh_token,
+    api_endpoint: config.github_api_endpoint,
+  )
+
+  if mode.include?('ci') && config.buildkite_org
+    bk_token = get_buildkite_token!(config)
+    bk_client = OssStats::BuildkiteClient.new(bk_token)
+    bk_pipelines_by_repo = bk_client.pipelines_by_repo(config.buildkite_org)
+  end
+
+  organizations_to_process.each do |org_name, org_level_config|
+    log.debug("Processing configuration for organization: #{org_name}")
+    repos = org_level_config['repositories'] || {}
+    repos.each do |repo_name, repo_level_config|
+      log.debug("Processing configuration for repository: #{repo_name}")
+      repos_to_process << get_effective_repo_settings(
+        org_name, repo_name, org_level_config, repo_level_config
+      )
+    end
+  end
+
+  if repos_to_process.empty?
+    log.info(
+      'No repositories found to process after evaluating configurations.',
+    )
+    exit 0
+  end
+
+  # Process each configured repository
+  all_repo_stats = []
+  repos_to_process.each do |settings|
+    repo_full_name = "#{settings[:org]}/#{settings[:repo]}"
+    repo_url = "https://github.com/#{repo_full_name}"
+
+    repo_data = {
+      name: repo_full_name,
+      url: repo_url,
+      settings:,
+      pr_issue_stats: nil,
+      ci_failures: nil,
+    }
+
+    # Fetch PR and Issue stats if PR or Issue mode is active
+    if %w{pr issue}.any? { |m| mode.include?(m) }
+      repo_data[:pr_issue_stats] = get_pr_and_issue_stats(gh_client, settings)
+    end
+
+    # Fetch CI stats if CI mode is active
+    if mode.include?('ci')
+      repo_data[:ci_failures] = get_failed_tests_from_ci(
+        gh_client, bk_client, settings, bk_pipelines_by_repo
+      )
+    end
+    all_repo_stats << repo_data
+  end
+
+  filtered_repos = filter_repositories(all_repo_stats, config)
+
+  filtered_repos.each do |repo_data|
+    if OssStats::Config::RepoStats.no_links
+      log.info(
+        "\n* #{repo_data[:name]} Stats (Last #{repo_data[:settings][:days]}" +
+        ' days) *',
+      )
+    else
+      log.info(
+        "\n*_[#{repo_data[:name]}](#{repo_data[:url]}) Stats " +
+        "(Last #{repo_data[:settings][:days]} days)_*",
+      )
+    end
+
+    if repo_data[:pr_issue_stats]
+      %w{PR Issue}.each do |type|
+        next unless mode.include?(type.downcase)
+        print_pr_or_issue_stats(
+          repo_data[:pr_issue_stats], type, config.include_list
+        )
+      end
+    end
+
+    next unless repo_data[:ci_failures]
+    # Ensure CI mode was active for this data to be present and printed
+    next unless mode.include?('ci')
+    print_ci_status(repo_data[:ci_failures])
+  end
+end
+
+main if __FILE__ == $PROGRAM_NAME

data/docs/MeetingStats.md

@@ -0,0 +1,69 @@
+# MeetingStats
+
+Many open source projects have weekly meetings and it's important to know
+that the relevant teams are showing up and reporting the expected data.
+
+[meeting_stats](../bin/meeting_stats.rb) will allow you to record the results
+of a meeting and then generate a report, including images with trends over
+time.
+
+You will **definitely** want a config file to make this useful for your
+project, and you can see
+[examples/meeting_stats_config.rb](../examples/meeting_stats_config.rb) for an
+example.
+
+It keeps all data in a SQLite DB. You can specify where this is with
+`--db-file`, and the default is `./data/meeting_data.sqlite3`.
+
+All CLI options can be specified in the config file.
+
+There are several modes, discussed below.
+
+## Modes
+
+### Generate
+
+This mode generates the markdown file with a table for every meeting and graphs
+at the top to show statistics over time.The markdown file is written to stdout
+by default, unless `--output` is specified.
+
+This mode is activated with `--mode generate`.
+
+This mode implicitly runs the `generate_plot` mode to regenerate the graph
+images.  See the section on that mode for more details.
+
+### Generate Plot
+
+This mode regenerate the plot graphs that show data over time (which are linked
+in the Markdown generated by `generate` mode). These graphs are written to
+`./images` by default, but you can change this with `--image-dir`.
+
+This mode is activated with `--mode generate_plot`.
+
+### Summary
+
+This mode print short summary of the last three meetings in Slack-friendly
+Markdown format. Sample output is:
+
+```markdown
+* 2025-06-05:
+    * Teams reported: 7 out of 7 (100%)
+    * Teams reporting build status: 6 out of 7 (85.71%)
+* 2025-05-29:
+    * Teams reported: 7 out of 7 (100%)
+    * Teams reporting build status: 6 out of 7 (85.71%)
+* 2025-05-22:
+    * Teams reported: 7 out of 7 (100%)
+    * Teams reporting build status: 6 out of 7 (85.71%)
+```
+
+### Record
+
+This mode is how you feed data into the database. It will prompt you to select
+a team, and then ask information such as if they were present, and if they
+reported their build status. Then it'll ask you to pick another team, and will
+repeat until you hit `q`, at which point it'll prompt to commit the new meeting
+information to the database.
+
+It defaults to the most recent Thursday for historical reasons, use `--date` to
+specify the date of the meeting.

data/docs/PipelineVisibilityStats.md

@@ -0,0 +1,51 @@
+# Pipeline visibility stats
+
+[pipeline_visibility_stats](../bin/pipeline_visibility_stats.rb) is a tool
+which walks Buildkite pipelines associated with your public GitHub repositories
+to ensure they are visible to contributors. It has a variety of options to
+exclude pipelines intended to be private (for example, pipelines that may have
+secrets to do pushes).
+
+There are two providers: buildkite and expeditor. Expeditor is deprecated
+and will go away.
+
+## Buildkite Provider
+
+This attempts to find improperly configured pipelines in two ways:
+
+* Given the buildkite repo, gets a list of all pipelines and builds a
+  map of GitHub Repos to pipelines. Then, it walks all GH repos
+  repos (either in the GH Org, or specified in the config), and checks
+  to see if there are buildkite repos associated with it, and if there are,
+  checks their visibility settings
+* Walks the most recent 10 PRs, and checks for any status checks that are
+  on buildkite, and checks if it can see them, and if so, checks their
+  visibility (it reporst them as private if it cannot see them)
+
+This is likely to include pipelines expected to be public such as those
+added adhoc to specific PRs to do builds. You can use --skip to add skip
+patterns (partial-match text) to avoid counting those.
+
+Example output looks like (this is truncated for brevity):
+
+```markdown
+# Chef Pipeline Visibility Report 2025-06-14
+
+* [chef/chef-cli](https://github.com/chef/chef-cli)
+    * chef/chef-chef-cli-main-habitat-test
+* [chef/chef-foundation](https://github.com/chef/chef-foundation)
+    * chef/chef-chef-foundation-main-verify
+* [chef/chef-powershell-shim](https://github.com/chef/chef-powershell-shim)
+    * chef/chef-chef-powershell-shim-pipeline-18-stable-habitat-build
+    * chef/chef-chef-powershell-shim-pipeline-stable-18-verify
+```
+
+At a minimum you will need the optoins `--github-org` and `--buildkite-org`.
+
+## Expeditor Provider
+
+The Expeditor Provider parses expeditor configs, which is Chef-specific and not
+open source. However, much of the code is generic and this could be adapted to
+other things.
+
+If you do want to use it, you can do so with `--provider=expeditor`.

data/docs/PromiseStats.md

@@ -0,0 +1,56 @@
+# Promises
+
+[promises](../bin/promises.rb) allows you to add, edit, resolve, abandon, and
+report on promises made. This can be useful for both promises made to the
+community or promises made between teams.
+
+You likely will probably want a config file for this; a sample
+is provided in [../examples/promises_config.rb](../examples/promises_config.rb).
+
+There are several sub-commands, discussed below.
+
+## Subcommands
+
+### add-promise
+
+This subcommand allows you to add a new promise. By default the data will be
+assumed to be today, but it may be changed with `--date`. You will be prompted
+for the promise, but you can specify it with `--promise`.
+
+Promises have 3 pieces of data associated with them:
+
+* `promise` - what was actually promised
+* `date` - the date on which it was promised
+* `reference` - additional reference about this promise. This could be
+  a link to the message, post, notes, etc. in which the promise was made,
+  for example. It is arbitrary text and may be whatever you wish.
+
+### resolve-promise
+
+Mark a promise as resolved. It will no longer be reported on by default, and
+the date it was resolved will be recorded in the database.
+
+### abandon-promise
+
+This marks a promise as abandoned and is useful in the case of a promise that
+is either no longer relevant or is not expected to be resolved.
+
+### edit-promise
+
+If you want to alter information about a promise in the database, this will
+re-prompt you for the information and update the database accordingly.
+
+### status
+
+This will output information about the promises in a Slack-friendly Markdown
+format. It will list all open promises and how long they've been open. Example
+output is:
+
+```markdown
+# Promises Report 2025-06-14
+
+* Publish Chef 19 / 2025 plan (247 days ago)
+* Fedora 41+ support (227 days ago)
+```
+
+You can include abandoned promises with `--include-abandoned`.