gitolemy 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: '083b930c641ed21a49484825dcf23d74097bb07b'
+  data.tar.gz: c89bc22ac8076d07a01e7d95b86be3c0924aaef8
+SHA512:
+  metadata.gz: 88c97b2e09ae85864329201c1f91732c2c22aeb49bd25162e3556a8e1b751268c60260ea82a0e3abf3c3b7fcd2b252fefce0d81dcf0282fbfb431e327701cc71
+  data.tar.gz: 84cdb5564782d60da58bd49b12089edb283765e65e8e93ae6d5ed3b0204c62e026099798c3a3711d264ba38a0a8066bcb8145d9ab2cf75ea4274eef0650accd5

data/bin/conglomerate.rb

@@ -0,0 +1,145 @@
+#! /usr/bin/env ruby
+
+require "rubygems"
+require "json"
+require "date"
+require "optparse"
+require "dotenv"
+require "active_support/core_ext/string"
+
+require "rollbar"
+require "airbrake-ruby"
+
+require_relative "../lib/file_manager"
+require_relative "../lib/notifier"
+require_relative "../lib/risk_analyzer"
+require_relative "../lib/secure_file_store"
+
+
+Dotenv.load()
+
+def merge_files_hash!(output, file_hash)
+  changes = output[:summary][:file_changes]
+  file_hash.each do |key, value|
+    changes[key] ||= {}
+    changes[key].merge!(value)
+  end
+end
+
+def client_factory(config, keys, additions={})
+  key = keys.detect { |key| config.has_key?(key) }
+  return nil if key.nil?
+  require_relative "../lib/integrations/#{key}_client"
+  data = config[key].merge(additions)
+  "#{key.camelize}Client".constantize.new(data)
+end
+
+def scm_client(config)
+  client_factory(config, ["git"])
+end
+
+def issues(config, commits)
+  client_factory(config, ["jira"])
+    .try(:merge_and_fetch_issues!, commits) || {}
+end
+
+def errors(config)
+  client_factory(config, ["airbrake", "rollbar"])
+end
+
+def coverage(config)
+  client_factory(config, ["covhura"])
+end
+
+def bugs(config, commits)
+  client_factory(config, ["jira"])
+    .try(:merge_and_fetch_bugs!, commits) || {}
+end
+
+
+def index(branches)
+  config = SecureFileStore.new(ENV["SETTING_KEY"]).read_settings()
+  scm_client = scm_client(config)
+
+  branches = branches.count == 0 ?
+    scm_client.remote_branches() :
+    branches
+
+  branches.each do |branch|
+    commits = scm_client.commits(branch)
+    file_manager = FileManager.new(branch)
+    file_manager.apply!(commits, {
+      issues: issues(config, commits),
+      errors: errors(config),
+      bugs: bugs(config, commits),
+      coverage: coverage(config)
+    })
+
+    risk = RiskAnalyzer
+      .new
+      .analyze(file_manager, commits.last)
+
+    Notifier
+      .new
+      .notify(scm_client.notification_url, commits.last, risk)
+  end
+end
+
+def get_opts()
+  options = {}
+  OptionParser.new do |opts|
+    opts.banner = "Usage: conglomerate.rb [options]"
+
+    opts.on("-p", "--repo-path=val", String, "Repository Path") do |path|
+      Dir.chdir(path)
+    end
+
+    opts.on("-v", "--verbose", "Run Verbosely") do |verbose|
+      ENV["GITOLEMY_VERBOSE"] = "true"
+    end
+
+    opts.on("-c", "--compare", "Compare virtual files to Git objects") do |compare|
+      ENV["GITOLEMY_COMPARE"] = "true"
+    end
+
+    opts.on("-t", "--test", "Run as test: do not store results") do |test|
+      ENV["GITOLEMY_PERSIST"] = "false"
+    end
+
+    opts.on("-s", "--sync", "Sync branch even if last commit indexed") do |sync|
+      ENV["GITOLEMY_SYNC"] = "true"
+    end
+  end.parse!
+  options[:branches] = []
+  options[:branches] << ARGV.first if ARGV.count > 0
+  options
+end
+
+
+def main()
+  index(get_opts()[:branches])
+rescue => ex
+  #handler = proc do |options|
+  #  payload = options[:payload]
+  #  payload["data"]["environment"] = ENV["ROLLBAR_ENV"]
+  #end
+
+  #Rollbar.configure do |config|
+  #  config.access_token = ENV["ROLLBAR_API_KEY"]
+  #  config.transform << handler
+  #end
+
+  #Rollbar.error(ex) if ENV["ROLLBAR_ENV"] == "production"
+
+  #Airbrake.configure do |c|
+  #  c.project_id = ENV["AIRBRAKE_PROJECT_ID"]
+  #  c.project_key = ENV["AIRBRAKE_PROJECT_KEY"]
+  #  c.environment = ENV["AIRBRAKE_ENV"].to_sym
+  #end
+
+  #Airbrake.notify_sync(ex) if ENV["AIRBRAKE_ENV"] == "production"
+
+  raise ex
+end
+
+main()

data/bin/serve.rb

@@ -0,0 +1,59 @@
+#! /usr/bin/env ruby
+
+require "webrick"
+require "webrick/httpproxy"
+require "json"
+
+server = WEBrick::HTTPProxyServer.new(:BindAddress => "0.0.0.0", :Port => 8180)
+
+server.mount_proc "/sync" do |req, res|
+  user = req.query["user"].gsub(" ", "+")
+  repo = req.query["repo"]
+  branch = req.query["branch"]
+  is_sync = req.query["sync"] == "true"
+
+  path = File.join(ENV.fetch("PROJECT_ROOT"), user, repo)
+  if Dir.exist?(path)
+    bin_path = File.join(File.dirname(__FILE__), "conglomerate.rb")
+    pid = spawn("#{bin_path} #{'-s' if is_sync} #{branch} -p #{path}")
+    Process.detach(pid)
+    res.status = 200
+  else
+    res.status = 400
+  end
+end
+
+server.mount_proc "/progress" do |req, res|
+  user = req.query["user"].gsub(" ", "+")
+  repo = req.query["repo"]
+  branch = req.query["branch"]
+
+  path = File.join(ENV.fetch("PROJECT_ROOT"), user, repo)
+  if Dir.exist?(path)
+    resp = {
+      commits: commit_count(path),
+      indexed: indexed_count(path, branch)
+    }
+    res.status = 200
+    res.body = resp.to_json
+  else
+    res.status = 400
+  end
+end
+
+def commit_count(project_root)
+  git_dir = File.join(project_root, ".git")
+  `git --git-dir=#{git_dir} log --graph --oneline | grep '^\*' | wc -l`.to_i
+end
+
+def indexed_count(project_root, branch)
+  branch = branch.gsub(/^remotes\/origin\//, "")
+  branch_lock_path = File.join(project_root, ".gitolemy", "branches", "#{branch}.lock")
+  `wc -l #{branch_lock_path}`
+    .split(" ")
+    .first
+    .to_i
+end
+
+trap("INT") { server.shutdown }
+server.start

data/lib/cache.rb

@@ -0,0 +1,92 @@
+require "json"
+require "zlib"
+require "fileutils"
+require "active_support/json"
+
+module Cache
+  extend self
+
+  CACHE_BASE_PATH = ".gitolemy"
+  OBJECT_CACHE_BASE = "objects"
+  BRANCH_CACHE_BASE = "branches"
+  REMOTES_REGEX = /^remotes\/origin\//
+
+  def cache_path(key)
+    File.join(CACHE_BASE_PATH, "#{key}.json.gz")
+  end
+
+  def write(key, data, existing_data=nil)
+    return data if not persist?
+    filename = cache_path(key)
+    dirname = File.dirname(filename)
+    ensure_directory(dirname)
+
+    store_data = existing_data.nil? ? data : existing_data.merge(data)
+    Zlib::GzipWriter.open(filename) { |gz| gz.write(store_data.to_json) }
+    data
+  end
+
+  def read(key, default_val=nil)
+    filename = cache_path(key)
+    return default_val if not File.exist?(filename)
+    JSON.parse(Zlib::GzipReader.open(filename) { |gz| gz.read })
+  rescue JSON::ParserError, Zlib::GzipFile::Error
+    default_val
+  end
+
+  def index_commit(branch, commit_id)
+    return if not persist?
+    branch_path = branch_path(branch)
+    ensure_directory(File.dirname(branch_path))
+    FileUtils.touch(branch_path) if not File.exist?(branch_path)
+    File.open(branch_path, "a") do |file|
+      file << "#{commit_id.to_s}\n"
+    end
+  end
+
+  def index_commits(branch,  commits)
+    return if not persist?
+    File.write(branch_path(branch), commits.join("\n") + "\n")
+  end
+
+  def last_indexed_commit(branch, commit_ids)
+    cached_commit_ids = File
+      .read(branch_path(branch))
+      .lines
+      .map(&:chomp)
+      .reduce({}) do |acc, commit_id|
+        acc[commit_id.to_sym] = true
+        acc
+      end
+
+    commit_ids.detect { |commit_id| cached_commit_ids[commit_id] }
+  rescue Errno::ENOENT
+    nil
+  end
+
+  def ensure_directory(dirname)
+    return if File.directory?(dirname)
+    FileUtils.makedirs(dirname)
+  end
+
+  def read_object(key, default_val=nil)
+    read(object_rel_path(key.to_s), default_val)
+  end
+
+  def write_object(key, data)
+    write(object_rel_path(key.to_s), data)
+  end
+
+  def object_rel_path(object_id)
+    File.join(OBJECT_CACHE_BASE, object_id)
+  end
+
+  def branch_path(branch)
+    branch_name = branch.gsub(REMOTES_REGEX, "")
+    File.join(".gitolemy", BRANCH_CACHE_BASE, "#{branch_name}.lock")
+  end
+
+  def persist?
+    ENV["GITOLEMY_PERSIST"] != "false"
+  end
+end

data/lib/commit.rb

@@ -0,0 +1,139 @@
+require "mail"
+require "iconv"
+
+require_relative "util"
+require_relative "loggr"
+require_relative "file_diff"
+require_relative "line_tracker"
+
+class Commit
+  FILE_DIFF_REGEX = /^diff --git a\/.* b\//
+
+  attr_accessor :commit_id
+  attr_accessor :children
+  attr_accessor :author
+  attr_accessor :date
+  attr_accessor :subject
+  attr_accessor :trees
+  attr_accessor :file_diffs
+  attr_accessor :movements
+  attr_accessor :changes
+  attr_accessor :insertions_total
+  attr_accessor :deletions_total
+  attr_accessor :changes_total
+  attr_accessor :cached_files
+  attr_accessor :cached_trees
+  attr_accessor :issue_id
+  attr_accessor :bug_id
+
+  alias_method :id, :commit_id
+  alias_method :id=, :commit_id=
+
+  def initialize(attrs={})
+    attrs.each do |key, val|
+      instance_variable_set("@#{key}".to_sym, val)
+    end
+  end
+
+  def set_cached(tree)
+    @cached_files = tree.select { |obj| obj[:type] == :file }
+    @cached_trees = tree.select { |obj| obj[:type] == :tree }
+  end
+
+  def business_value
+    issue = ::Store::Issue[issue_id]
+    issue.present? ?
+      issue[:business_value] :
+      0
+  end
+
+  def as_json
+    {
+      commit_id: commit_id,
+      children: children,
+      author: author,
+      date: date,
+      subject: subject,
+      insertions_total: insertions_total,
+      deletions_total: deletions_total,
+      changes_total: changes.count,
+      movements_total: movements.count,
+      issue_id: issue_id,
+      bug_id: bug_id
+    }
+  end
+
+  class << self
+    def from_git(commit_lines, git_client)
+      Commit.new(parse_commit(commit_lines, git_client))
+    end
+
+    def parse_commit(commit_lines, git_client)
+      commit_id, author, date, children, subject = parse_commit_header(commit_lines.shift())
+      Loggr.instance.info("PARSE COMMIT: #{commit_id}")
+      if children.length > 1
+        commit_lines = git_client.diff(children.first, commit_id)
+      end
+
+      file_diffs = commit_lines
+        .reduce([], &fold_reducer(FILE_DIFF_REGEX))
+        .map { |file_diff_lines| FileDiff.from_git(file_diff_lines) }
+
+      insertions = file_diffs.reduce(0) { |sum, file_diff| sum + file_diff.insertions_total }
+      deletions = file_diffs.reduce(0) { |sum, file_diff| sum + file_diff.deletions_total }
+
+      file_diffs = file_diffs.reduce({}) do |obj, file_diff|
+        obj[file_diff.b_file_name] = file_diff
+        obj
+      end
+
+      if children.length > 1
+        diff_id = "#{children.first}..#{commit_id}"
+        trees = git_client.parse_diff_tree(git_client.diff_tree(diff_id))
+      elsif children.length == 1
+        trees = git_client.parse_diff_tree(git_client.diff_tree(commit_id))
+      else
+        trees = git_client.parse_ls_tree(git_client.ls_tree(commit_id))
+      end
+
+      mutations = LineTracker.new.track_mutations!(file_diffs)
+
+      {
+        commit_id: commit_id,
+        children: children,
+        author: {
+          name: author.display_name,
+          email: author.address
+        },
+        date: date,
+        subject: subject,
+        trees: trees,
+        file_diffs: file_diffs,
+        movements: mutations[:movements],
+        changes: mutations[:changes],
+        insertions_total: insertions,
+        deletions_total: deletions,
+        changes_total: insertions + deletions
+      }
+    end
+
+    def parse_commit_header(header)
+      commit_id, children, author, date, subject = header.split("|||")
+      commit_id = commit_id.to_sym
+      subject ||= ""
+      children = children
+        .split(" ")
+        .map { |child_id| child_id.to_sym }
+      author = parse_author(author)
+      date = DateTime.rfc2822(date)
+      [commit_id, author, date, children, subject]
+    end
+
+    def parse_author(author)
+      author = Iconv
+        .conv("ascii//translit", "UTF-8", author)
+        .tr("[]", "()")
+      Mail::Address.new(author)
+    end
+  end
+end

data/lib/commit_stats.rb

@@ -0,0 +1,225 @@
+require "diff_match_patch_native"
+
+module CommitStats
+  def self.link_mutations!(files, commit_mutations, type)
+    dmp = DiffMatchPatch.new()
+
+    commit_mutations.each do |dest_file, file_mutations|
+      file_mutations.each do |dest_line, mutation|
+        src_file = mutation[:from]
+        src_line = mutation[:line]
+
+        b_file = files[:b][dest_file][:b_file]
+        a_file = files[:a][src_file][:a_file]
+
+        b_line = b_file.is_a?(Hash) ? b_file[:lines][dest_line] : b_file.lines[dest_line]
+        a_line = a_file.is_a?(Hash) ? a_file[:lines][src_line] : a_file.lines[src_line]
+
+        if type == :change
+          diff = dmp.diff_main(a_line[:text], b_line[:text], false)
+
+          b_line[:diff_text] = diff.reduce("") do |acc, section|
+            if section.first == 0
+              acc += section.last
+            elsif section.first == 1
+              acc += "<ins>#{section.last}</ins>"
+            end
+            acc
+          end
+
+          a_line[:diff_text] = diff.reduce("") do |acc, section|
+            if section.first == 0
+              acc += section.last
+            elsif section.first == -1
+              acc += "<del>#{section.last}</del>"
+            end
+            acc
+          end
+        end
+
+        if Line.trailing?(b_line, a_line)
+          change_type = :trailing
+        elsif Line.beauty?(b_line, a_line)
+          change_type = :beauty
+        else
+          change_type = :normal
+        end
+
+        b_line[type] = {
+          link: "#/a/#{src_line + 1}/#{src_file}",
+          type: change_type,
+          change_text: a_line[:text]
+        }
+
+        a_line[type] = {
+          link: "#/b/#{dest_line + 1}/#{dest_file}",
+          type: change_type,
+          change_text: b_line[:text]
+        }
+      end
+    end
+  end
+
+  def self.link_diffs!(files, file_diffs)
+    file_diffs.reduce(diff_context()) do |acc, (file_name, file_diff)|
+      file_diff.diffs.each do |diff|
+        diff.insertions.each_with_index do |insertion, index|
+          line_num = diff.insert_start + index - 1
+          file = files[:b][file_diff.b_file_name][:b_file]
+          line = file.is_a?(Hash) ? file[:lines][line_num] : file.lines[line_num]
+          line[:insertion] = true
+          acc[:insertions] << line
+        end
+
+        del_index = diff.insert_count > 0 ? 1 : 0
+        diff.deletions.each_with_index do |deletion, index|
+          line_num = diff.delete_start + index - 1
+          file = files[:a][file_diff.a_file_name][:a_file]
+          line = file.is_a?(Hash) ? file[:lines][line_num] : file.lines[line_num]
+          line.merge!({
+            deletion: true,
+            a_pos: line_num + 1,
+            b_pos: diff.insert_start + index - del_index
+          })
+
+          del_index += 1 if line[:change].blank?
+          acc[:deletions] << line
+        end
+      end
+      acc
+    end
+  end
+
+  def self.stats(commit, line_diffs, files)
+    changes_total = commit
+      .changes
+      .reduce(0) { |acc, (filename, changeset)| acc += changeset.count } * 2
+
+    movements_total = commit
+      .movements
+      .reduce(0) { |acc, (filename, moveset)| acc += moveset.count } * 2
+
+    uncovered_deletions_total = 0
+    uncovered_insertions_total = 0
+    uncovered_changes_total = 0
+    uncovered_error_changes_total = 0
+    uncovered_buggy_changes_total = 0
+    error_deletions_total = 0
+    error_changes_total = 0
+    buggy_insertions_total = 0
+    buggy_deletions_total = 0
+    buggy_changes_total = 0
+    whitespace_insertions_total = 0
+    whitespace_deletions_total = 0
+    total_files_modified = files[:a].count
+    total_lines_modified = commit.changes_total
+    trailingspace_total = 0
+    beautyspace_total = 0
+
+    line_diffs[:insertions].each do |insertion|
+      if insertion[:coverage] == 0
+        uncovered_insertions_total += 1
+        if insertion[:change]
+          uncovered_changes_total += 1
+          uncovered_error_changes_total += 1 if insertion[:errors].count > 0
+          uncovered_buggy_changes_total += 1 if insertion[:bugs].count > 0
+        end
+      end
+      if insertion[:change]
+        buggy_changes_total += 1 if insertion[:bugs].count > 0
+        error_changes_total += 1 if insertion[:errors].count > 0
+        beautyspace_total += 2 if insertion[:change][:type] == :beauty # + deletion
+        trailingspace_total += 2 if insertion[:change][:type] == :trailing # + deletion
+      end
+      whitespace_insertions_total += 1 if insertion[:text].strip().blank?
+      buggy_insertions_total += 1 if insertion[:bugs].count > 0
+    end
+
+    line_diffs[:deletions].each do |deletion|
+      if deletion[:coverage] == 0
+        uncovered_deletions_total += 1
+      end
+      error_deletions_total += 1 if deletion[:errors].count > 0
+      whitespace_deletions_total += 1 if deletion[:text].strip().blank?
+      buggy_deletions_total += 1 if deletion[:bugs].count > 0
+    end
+
+    relevant_insertions_total = commit.insertions_total - whitespace_insertions_total
+    covered_insertions_total = relevant_insertions_total - uncovered_insertions_total
+    covered_insertions_percent = change_percent(covered_insertions_total, relevant_insertions_total)
+    uncovered_insertions_percent = change_percent(uncovered_insertions_total, relevant_insertions_total)
+    buggy_insertions_percent = change_percent(buggy_insertions_total, relevant_insertions_total)
+
+    relevant_deletions_total = commit.insertions_total - whitespace_insertions_total
+    covered_deletions_total = relevant_deletions_total - uncovered_deletions_total
+    covered_deletions_percent = change_percent(covered_deletions_total, relevant_deletions_total)
+    uncovered_deletions_percent = change_percent(uncovered_deletions_total, relevant_deletions_total)
+    buggy_deletions_percent = change_percent(buggy_deletions_total, relevant_deletions_total)
+    error_deletions_percent = change_percent(error_deletions_total, relevant_deletions_total)
+
+    covered_changes_total = changes_total - uncovered_changes_total
+    covered_changes_percent = change_percent(covered_changes_total, changes_total)
+    uncovered_changes_percent = change_percent(uncovered_changes_total, changes_total)
+    uncovered_error_changes_percent = change_percent(uncovered_error_changes_total, changes_total)
+    uncovered_buggy_changes_percent = change_percent(uncovered_buggy_changes_total, changes_total)
+
+    diffs_total = commit.deletions_total + commit.insertions_total
+    movements_percent = change_percent(movements_total, diffs_total)
+
+    whitespace_total = whitespace_insertions_total + whitespace_deletions_total
+    whitespace_percent = change_percent(whitespace_total, diffs_total)
+    trailingspace_percent = change_percent(trailingspace_total, diffs_total)
+    beautyspace_percent = change_percent(beautyspace_total, diffs_total)
+
+    {
+      changes: commit.changes,
+      movements: commit.movements,
+      deletions_total: commit.deletions_total,
+      insertions_total: commit.insertions_total,
+      changes_total: changes_total,
+      movements_total: movements_total,
+      whitespace_total: whitespace_total,
+      trailingspace_total: trailingspace_total,
+      beautyspace_total: beautyspace_total,
+      covered_insertions_percent: covered_insertions_percent,
+      covered_deletions_percent: covered_deletions_percent,
+      covered_changes_percent: covered_changes_percent,
+      uncovered_insertions_percent: uncovered_insertions_percent,
+      uncovered_deletions_percent: uncovered_deletions_percent,
+      uncovered_changes_percent: uncovered_changes_percent,
+      uncovered_error_changes_percent: uncovered_error_changes_percent,
+      uncovered_buggy_changes_percent: uncovered_buggy_changes_percent,
+      buggy_insertions_percent: buggy_insertions_percent,
+      buggy_deletions_percent: buggy_deletions_percent,
+      error_deletions_percent: error_deletions_percent,
+      movements_percent: movements_percent,
+      whitespace_percent: whitespace_percent,
+      trailingspace_percent: trailingspace_percent,
+      beautyspace_percent: beautyspace_percent,
+      uncovered_deletions_total: uncovered_deletions_total,
+      uncovered_insertions_total: uncovered_insertions_total,
+      uncovered_changes_total: uncovered_changes_total,
+      uncovered_error_changes_total: uncovered_error_changes_total,
+      buggy_insertions_total: buggy_insertions_total,
+      buggy_deletions_total: buggy_deletions_total,
+      buggy_changes_total: buggy_changes_total,
+      uncovered_buggy_changes_total: uncovered_buggy_changes_total,
+      error_deletions_total: error_deletions_total,
+      error_changes_total: error_changes_total,
+      total_files_modified: total_files_modified,
+      total_lines_modified: total_lines_modified
+    }
+  end
+
+  def self.change_percent(numerator, denominator)
+    denominator > 0 ?
+      (numerator / denominator.to_f * 100) :
+      0
+  end
+
+  def self.diff_context()
+    {
+      insertions: [],
+      deletions: [] }
+  end
+end