gitolemy 0.0.4

data/lib/line.rb

@@ -0,0 +1,124 @@
+require "active_support/core_ext/object"
+
+require "amatch"
+include Amatch
+
+module Line
+  JARO_SIMILARITY_THRESHOLD = 0.88
+  MIN_LENGTH_SIMILARITY_THRESHOLD = 0.2
+  MAX_LENGTH_SIMILARITY_THRESHOLD = 5
+  COVERAGE_INVALID = nil
+
+  TOKEN_SPLIT_REGEX = /[\s,.&|^+=*\/-]/
+  MAX_MEANING_LENGTH = 400
+  NOT_WHITESPACE_REGEX = /[^\s]/
+  BEAUTYSPACE_REGEX = /\s/
+  MEANINGLESS_KEYWORDS = {
+    "" => true,
+    "//" => true,
+    "/*" => true,
+    "*/" => true,
+    "#" => true,
+    "else" => true,
+    "end" => true,
+    "rescue" => true,
+    "{" => true,
+    "}" => true,
+    "[" => true,
+    "]" => true,
+    "(" => true,
+    ")" => true,
+    "<" => true,
+    ">" => true,
+    "/>" => true
+  }
+
+  # TODO: return line_id only.
+  def Line.new_line(text, file_name, line_num, commit)
+    line = {
+      text: text,
+      coverage: COVERAGE_INVALID,
+      commit: commit.id,
+      issues: commit.issue_id.blank? ? [] : [commit.issue_id],
+      bugs: commit.bug_id.blank? ? [] : [commit.bug_id],
+      errors: [],
+      revisions: []
+    }
+    line
+  end
+
+  # TODO: Don't move/change lines twice.
+  def Line.change!(old_line, line)
+    past_revisions = old_line.delete(:revisions) || []
+    {
+      text: line[:text],
+      coverage: line[:coverage],
+      commit: line[:commit],
+      issues: line[:issues],
+      errors: line[:errors],
+      bugs: line[:bugs],
+      revisions: line[:revisions] + [old_line] + past_revisions
+    }
+  end
+
+  def Line.move!(old_line, line)
+    Line.change!(old_line, line)
+  end
+
+  def Line.merge_error!(line, error, depth)
+    return if line[:errors].any? { |line_error| line_error[:error_id] == error[:error_id] }
+
+    line[:errors].unshift({
+      error_id: error[:error_id],
+      depth: depth
+    })
+  end
+
+  def Line.relevant?(line)
+    MEANINGLESS_KEYWORDS[line[:text].strip()].nil?
+  end
+
+  def Line.relevant_change?(line)
+    return false if not relevant?(line)
+    if line.has_key?(:change)
+      a_line = {text: line.dig(:change, :change_text)}
+      return trailing?(line, a_line) || beauty?(line, a_line)
+    end
+    return true
+  end
+
+  def Line.meaningless?(text)
+    MEANINGLESS_KEYWORDS[text] == true ||
+      text.length > MAX_MEANING_LENGTH ||
+      text.split(TOKEN_SPLIT_REGEX).length <= 1
+  end
+
+  def Line.similar?(text_a, text_b)
+    length_similarity = text_a.length.to_f / text_b.length.to_f
+    length_similarity > MIN_LENGTH_SIMILARITY_THRESHOLD &&
+      length_similarity < MAX_LENGTH_SIMILARITY_THRESHOLD &&
+      text_a.jarowinkler_similar(text_b) > JARO_SIMILARITY_THRESHOLD
+  end
+
+  def Line.score(line)
+    score = 0
+    line[:errors].each { |error| score += 2.0 / (error[:depth] + 1) }
+    score += 3 * line[:bugs].count
+    score += 0.75 * line[:revisions].count
+    score += 3 if line[:coverage] == 0
+    score += 1.5 if line[:coverage].nil?
+    score
+  end
+
+  def Line.beauty?(a_line, b_line)
+    a_start = a_line[:text].index(NOT_WHITESPACE_REGEX)
+    b_start = b_line[:text].index(NOT_WHITESPACE_REGEX)
+
+    a_start == b_start &&
+      a_line[:text].gsub(BEAUTYSPACE_REGEX, "") == b_line[:text].gsub(BEAUTYSPACE_REGEX, "")
+  end
+
+  def Line.trailing?(a_line, b_line)
+    a_line[:text].rstrip == b_line[:text].rstrip
+  end
+end

data/lib/line_tracker.rb

@@ -0,0 +1,90 @@
+require_relative "line"
+
+class LineTracker
+  def track_mutations!(file_diffs)
+    deletions = index_deletions(file_diffs)
+    file_deletions = deletions.delete(:files)
+
+    grouped_file_diffs(file_diffs).reduce(change_context()) do |acc, (file_name, file_diff)|
+      file_diff[:diffs].each do |diff|
+        file_deletes = (file_deletions[file_diff[:a_file_name]] || {})
+
+        diff.insertions.each_with_index do |insertion, i|
+          insertion = insertion.strip()
+          next if Line.meaningless?(insertion)
+          line_number = diff.insert_start + i - 1
+
+          if movement?(deletions, insertion)
+            index!(acc[:movements], file_name, line_number, deletions[insertion].shift())
+          else
+            changed_line = find_change(file_deletes, insertion)
+            index!(acc[:changes], file_name, line_number, changed_line) if changed_line
+          end
+        end
+      end
+      acc
+    end
+  end
+
+  private
+
+  def movement?(deletions, new_line)
+    (deletions[new_line] || []).count > 0
+  end
+
+  def find_change(deletions, new_line)
+    change = (deletions || {})
+      .keys
+      .detect { |deletion| Line::similar?(deletion, new_line) }
+    return change ? deletions[change].shift() : nil
+  end
+
+  def index!(acc, file_name, line_number, old_line)
+    acc[file_name] ||= {}
+    acc[file_name][line_number] = old_line
+  end
+
+  def change_context()
+    {
+      movements: {},
+      changes: {}
+    }
+  end
+
+  # TODO:
+  #    1: Machine learning based match to avoid false positives from other files.
+  #       a: Within commit, within file, be most lenient.
+  #       b: Within commit, be somewhat lenient.
+  #       c: Within entire codebase, be pretty strict. Exclude even exact matches some times.
+  #    2: Once a line is matched from the graveyard, remove it -- it's alive again.
+  def index_deletions(file_diffs)
+    file_diffs
+      .reduce({files: {}, }) do |acc, (file_name, file_diff)|
+        deleted_file = file_diff.a_file_name
+
+        file_diff.diffs.each do |diff|
+          diff.deletions.each_with_index do |deletion, index|
+            deletion = deletion.strip()
+            next if Line.meaningless?(deletion)
+            line_number = diff.delete_start + index - 1
+            line = {from: deleted_file, line: line_number}
+
+            acc[:files][file_name] ||= {}
+            acc[:files][file_name][deletion] ||= []
+            acc[:files][file_name][deletion] << line
+            acc[deletion] ||= []
+            acc[deletion] << line
+          end
+        end
+        acc
+      end
+  end
+
+  def grouped_file_diffs(file_diffs)
+    file_diffs.reduce({}) do |acc, (file_name, file_diff)|
+      acc[file_diff.b_file_name] ||= {diffs: [], a_file_name: file_diff.a_file_name}
+      acc[file_diff.b_file_name][:diffs].concat(file_diff.diffs)
+      acc
+    end
+  end
+end

data/lib/loggr.rb

@@ -0,0 +1,24 @@
+require "logger"
+require "singleton"
+require "active_support/core_ext/module/delegation"
+
+require_relative "cache"
+
+class Loggr
+  include Singleton
+
+  attr_reader :logger
+  delegate :info, :warn, :error, to: :logger
+
+  def initialize()
+    @logger = Logger.new(output)
+  end
+
+  private
+
+  def output()
+    return STDOUT if ENV["GITOLEMY_VERBOSE"] == "true"
+    Cache.ensure_directory(".gitolemy")
+    File.open(File.join(".gitolemy", "out.log"), "w")
+  end
+end

data/lib/notifier.rb

@@ -0,0 +1,20 @@
+class Notifier
+  def notify(url, commit, risk)
+    return if ENV["GITOLEMY_SYNC"] == "true" || commit.nil?
+    post(url, {
+      status: risk <= 30 ? "success" : "failure",
+      commit_id: commit.commit_id
+    })
+  end
+
+  private
+
+  def post(url, payload)
+    uri = URI(url)
+    request = Net::HTTP::Post.new(uri, {"Content-Type" => "application/json"})
+    request.body = payload.to_json
+    http = Net::HTTP.new(uri.hostname, uri.port)
+    http.use_ssl = url.index("https://") == 0
+    http.request(request)
+  end
+end

data/lib/project_cache.rb

@@ -0,0 +1,13 @@
+require_relative "cache"
+
+class ProjectCache
+  include Cache
+
+  def initialize(base_path)
+    @base_path = base_path
+  end
+
+  def cache_path(key)
+    File.join(@base_path, Cache.cache_path(key))
+  end
+end

data/lib/risk_analyzer.rb

@@ -0,0 +1,53 @@
+require "active_support/core_ext/object"
+
+require_relative "commit_stats"
+
+class RiskAnalyzer
+  # TODO: Add config to match test files
+  def analyze(file_manager, commit)
+    return if commit.nil?
+
+    files = files_for_diff(commit.file_diffs, file_manager)
+
+    CommitStats.link_mutations!(files, commit.changes, :change)
+    CommitStats.link_mutations!(files, commit.movements, :movement)
+    line_diffs = CommitStats.link_diffs!(files, commit.file_diffs)
+
+    score(line_diffs)
+  end
+
+  # TODO: Maybe include function score per line.
+  def score(line_diffs)
+    (line_diffs[:insertions] + line_diffs[:deletions])
+      .reduce(0) do |acc, line|
+        penalty = Line.relevant_change?(line) ? Line.score(line) : 0
+        penalty /= 3 if line.has_key?(:movement)
+        acc += penalty
+        acc
+      end
+  end
+
+  private
+
+  def files_for_diff(file_diffs, file_manager)
+    file_diffs.reduce({a: {}, b: {}}) do |acc, (b_path, file_diff)|
+      file = {
+        a_file: file_manager.vfs[file_diff.a_file_name] || load_file(file_diff),
+        b_file: file_manager.vfs[file_diff.b_file_name],
+        a_path: file_diff.a_file_name,
+        b_path: file_diff.b_file_name,
+        changes: [],
+        movements: []
+      }
+      acc[:a][file[:a_path]] = file
+      acc[:b][file[:b_path]] = file
+      acc
+    end
+  end
+
+  def load_file(file_diff)
+    Cache
+      .read_object(file_diff.a_file_id)
+      .deep_symbolize_keys
+  end
+end

data/lib/secure_file_store.rb

@@ -0,0 +1,61 @@
+require "openssl"
+require "json"
+
+
+class SecureFileStore
+  def initialize(key)
+    @key = unhex(key)
+  end
+
+  def write_file(data, file_path)
+    iv, data = encrypt(@key, data)
+    File.open(file_path, "wb") { |file| file.write(data) }
+  end
+
+  def read_file(iv, file_path)
+    data = File.read(file_path)
+    decrypt(@key, unhex(iv), data)
+  end
+
+  def write_settings(settings, dir=".gitolemy")
+    iv, data = encrypt(@key, settings.to_json())
+    file_path = File.join(dir, "config-#{hex(iv)}")
+    File.open(file_path, "wb") { |file| file.write(data) }
+  end
+
+  def read_settings(dir=".gitolemy")
+    file_path = Dir[File.join(dir, "config-*")].first
+    iv = File.basename(file_path).gsub(/^config-/, "")
+    JSON.parse(read_file(iv, file_path))
+  end
+
+  private
+
+  def hex(iv)
+    iv.unpack("H*").first
+  end
+
+  def unhex(iv)
+    iv
+      .scan(/../)
+      .map { |x| x.hex }
+      .pack("c*")
+  end
+
+  def encrypt(private_key, data)
+    cipher = OpenSSL::Cipher::AES.new(256, :CBC)
+    cipher.encrypt
+    cipher.key = private_key
+    iv = cipher.random_iv
+    out = cipher.update(data) + cipher.final
+    [iv, out]
+  end
+
+  def decrypt(private_key, iv, data)
+    cipher = OpenSSL::Cipher::AES.new(256, :CBC)
+    cipher.decrypt
+    cipher.key = private_key
+    cipher.iv = iv
+    cipher.update(data) + cipher.final
+  end
+end

data/lib/source_tree.rb

@@ -0,0 +1,23 @@
+class SourceTree
+
+  def merge_collapse(commits)
+    indexed_commits = commits.reduce({}, &method(:index_commits))
+
+    graph = []
+    head = commits.first
+    while head
+      graph << head
+      commit_id = head.children.first
+      head = indexed_commits[commit_id]
+    end
+
+    graph
+  end
+
+  private
+
+  def index_commits(acc, commit)
+    acc[commit.id] = commit
+    acc
+  end
+end

data/lib/stack_tracer.rb

@@ -0,0 +1,197 @@
+require "date"
+require "byebug"
+
+require "active_support/core_ext/object"
+
+require_relative "project_cache"
+require_relative "integrations/git_client"
+require_relative "line"
+
+class StackTracer
+  def initialize(base_path)
+    @git_client = GitClient.new({"git_dir" => File.join(base_path, ".git")})
+    @cache = ProjectCache.new(base_path)
+  end
+
+  def trace(error_id, deployed_commit_id=nil, last_deployed_commit_id=nil)
+    error = @cache.read("errors")[error_id.to_s].deep_symbolize_keys
+    commits = @cache.read("commits")
+    deploys = @cache.read("deploys")
+
+    deployed_commits = deploys
+      .map { |deploy| commits[deploy["revision"]] }
+      .reject(&:nil?)
+      .sort_by { |commit| commit["date"] }
+
+    deployed_commit_id ||= after_deployed_commit(error[:first_time], deployed_commits)
+    last_deployed_commit_id ||= prior_deployed_commit(error[:first_time], deployed_commits)
+
+    git_files = git_files(deployed_commit_id.to_s)
+    stacktrace = stacktrace(error, git_files)
+    trace_files = trace_files(stacktrace, git_files)
+    trace_lines = trace_lines(stacktrace, trace_files, commits, last_deployed_commit_id.to_s)
+
+    detail(error, trace_lines)
+  end
+
+  private
+
+  def git_files(commit_id)
+    @git_client
+      .file_tree(commit_id)
+      .reduce({}) do |acc, blob|
+        blob = blob.split(" ")
+        acc[blob.last] = blob[2]
+        acc
+      end
+  end
+
+  def trace_files(stacktrace, git_files)
+    stacktrace
+      .map { |trace_file| trace_file[:file] }
+      .uniq
+      .reduce({}) do |acc, file|
+        acc[file] = @cache.read_object(git_files[file]).deep_symbolize_keys
+        acc
+      end
+  end
+
+  def trace_lines(stacktrace, trace_files, commits, last_deployed_commit_id)
+    last_deployed_commit = commits[last_deployed_commit_id]
+    cutoff_time = DateTime.parse(last_deployed_commit["date"])
+
+    stacktrace
+      .each_with_index
+      .map do |trace, depth|
+        file = trace_files[trace[:file]]
+        line_num = trace[:line] - 1
+        line = describe_line(file[:lines][line_num], commits, cutoff_time)
+        line[:depth] = depth
+
+        function = file[:functions].detect do |function|
+          line_num >= function[:start] && line_num <= function[:end]
+        end
+
+        function_lines = function_lines(file, function)
+          .each_with_index
+          .map do |line, index|
+            line = describe_line(line, commits, cutoff_time)
+            line[:revisions] = line[:revisions].map do |revision|
+              revision[:author] = commits[revision[:commit]]["author"]
+              revision
+            end
+            line
+          end
+
+        {
+          file: trace[:file],
+          line_num: trace[:line],
+          line: line,
+          title: "#{trace[:file]}:#{trace[:line]} - #{trace[:function]}",
+          function: function,
+          function_lines: function_lines
+        }
+      end
+  end
+
+  def detail(error, trace_lines)
+    functions = trace_lines.reduce({}) do |acc, trace|
+      function_id = trace[:function].nil? ?
+        trace[:file] :
+        "#{trace[:file]}:#{trace[:function][:name]}"
+
+      acc[function_id] ||= trace
+      depth = trace[:line][:depth]
+      trace[:line][:trace_title] = trace[:title]
+
+      if acc[function_id].has_key?(:lines)
+        acc[function_id][:lines] << trace[:line]
+      else
+        acc[function_id][:lines] = [trace[:line]]
+        acc[function_id].delete(:line)
+      end
+
+      acc[function_id][:function_lines].each_with_index do |line, index|
+        function_line_num = trace[:line_num] - trace[:function][:start]
+        line[:depth] = depth if index == function_line_num
+      end
+
+      if not acc.has_key?(function_id)
+        acc[function_id] = trace
+      end
+      acc
+    end
+
+    details = functions
+      .reduce(detail_context()) do |acc, (file_path, function)|
+        function[:function_lines].each do |line|
+          score_line(acc[:experts], line[:author]["email"], line)
+          if line[:after_cutoff]
+            score_line(acc[:suspects], line[:author]["email"], line)
+            score_line(acc[:suspect_commits], line[:commit], line)
+          end
+          line[:revisions].each do |revision|
+            score_line(acc[:experts], revision[:author]["email"], line)
+          end
+        end
+        acc
+      end
+
+    details.merge({
+      message: error[:message],
+      first_time: error[:first_time],
+      last_time: error[:last_time],
+      total_occurrences: error[:total_occurrences],
+      functions: functions,
+    })
+  end
+
+  def score_line(hash, key, line)
+    hash[key] ||= 0
+    hash[key] += 1
+    hash[key] += 2.0 / (line[:depth] + 1) if line.has_key?(:depth)
+  end
+
+  def detail_context()
+    {
+      experts: {},
+      suspects: {},
+      suspect_commits: {}
+    }
+  end
+
+  def describe_line(line, commits, cutoff_time)
+    line = line.dup
+    line[:updated_at] = DateTime.parse(commits[line[:commit]]["date"])
+    line[:after_cutoff] = line[:updated_at] > cutoff_time
+    line[:author] = commits[line[:commit]]["author"]
+    line[:score] = Line.score(line)
+    line
+  end
+
+  def stacktrace(error, git_files)
+    error[:stack_trace]
+      .reduce([]) do |acc, trace|
+        trace[:file] = git_files.keys.detect { |app_file| trace[:file].include?(app_file) }
+        acc << trace if not trace[:file].nil?
+        acc
+      end
+  end
+
+  def function_lines(file, function)
+    function.nil? ? [] : file[:lines][(function[:start] - 1)..function[:end]]
+  end
+
+  def prior_deployed_commit(timestamp, commits)
+    commits
+      .detect { |commit| commit["date"] < timestamp }["commit_id"]
+  end
+
+  def after_deployed_commit(timestamp, commits)
+    commit = commits
+      .reverse()
+      .detect { |commit| commit["date"] > timestamp }
+    commit ||= commits.last
+    commit["commit_id"]
+  end
+end