gitolemy 0.0.4

data/lib/integrations/error_client.rb

@@ -0,0 +1,55 @@
+require "active_support/core_ext/object"
+
+require_relative "../cache"
+
+class ErrorClient
+  ERROR_CACHE_KEY = "errors"
+  DEPLOY_CACHE_KEY = "deploys"
+
+  # TODO: Shared
+  def get_errors!(commit)
+    deploy = commit_deploy(commit)
+    return [] if deploy.nil?
+
+    deploy_index = @deploys.index(deploy) || 0
+    @deploys = @deploys[(deploy_index + 1)..-1] || []
+    if @deploys.first.nil?
+      errors = @errors
+      @errors = []
+    else
+      errors = []
+      while @errors.any? && @errors.first[:last_time] < @deploys.first[:timestamp] do
+        errors << @errors.shift()
+      end
+    end
+    errors
+  end
+
+  # TODO:
+  #    1: sync to last deploy?
+  #    2: Shared
+  def sync!(commit)
+    while @errors.any? && @errors.first[:last_time] < commit.date do
+      @errors.shift()
+    end
+  end
+
+  private
+
+  def commit_deploy(commit)
+    @deploys.detect { |deploy| deploy[:commit_id] == commit.commit_id }
+  end
+
+  def load_from_cache()
+    Cache
+      .read(ERROR_CACHE_KEY, {})
+      .map do |error_id, error|
+        error = error.deep_symbolize_keys
+        error[:error_id] = error[:error_id].to_sym
+        error[:first_time] = DateTime.rfc3339(error[:first_time])
+        error[:last_time] = DateTime.rfc3339(error[:last_time])
+        error
+      end
+      .sort_by { |error| error[:last_time] }
+  end
+end

data/lib/integrations/git_client.rb

@@ -0,0 +1,183 @@
+require "active_support/core_ext/hash"
+
+require_relative "../source_tree"
+require_relative "../cache"
+require_relative "../loggr"
+require_relative "../store"
+require_relative "../util"
+require_relative "../commit"
+
+class GitClient
+  COMMIT_REGEX = /^[0-9a-f]{40}\|\|\|/
+  TREE_PERMISSION = "040000"
+
+  attr_accessor :notification_url
+
+  def initialize(config)
+    @git_dir = config["git_dir"]
+    @default_branch = config["default_branch"]
+    @notification_url = config["notification_url"]
+  end
+
+  def commits(branch)
+    commit_ids = short_log(branch)
+    cache, index = commits_to_index(branch, commit_ids)
+    return [] if index == 0
+
+    commits = full_log(branch, index)
+      .reduce([], &fold_reducer(COMMIT_REGEX))
+      .map { |commit_lines| Commit.from_git(commit_lines, self) }
+
+    commits = SourceTree
+      .new()
+      .merge_collapse(commits)
+
+    commits.last.set_cached(cache)
+    cache_commits(commits)
+
+    commits.reverse()
+  end
+
+  def diff_tree(commit_id)
+    exec("diff-tree -t -r #{commit_id}")
+  end
+
+  def parse_diff_tree(tree)
+    tree
+      .reduce([]) do |acc, object|
+        a_perm, b_perm, a_id, b_id, operation, path = object[1..-1].split(" ")
+        if a_perm == TREE_PERMISSION || b_perm == TREE_PERMISSION
+          acc << {
+            a_tree_id: a_id,
+            b_tree_id: b_id,
+            path: path,
+            operation: operation == "D" ? :delete : :change
+          }
+        end
+        acc
+      end
+      .sort_by { |tree| tree[:path].split(File::SEPARATOR).length }
+  end
+
+  def ls_tree(commit_id)
+    exec("ls-tree -r -t #{commit_id}")
+  end
+
+  def file_tree(commit_id)
+    exec("ls-tree -r #{commit_id}")
+  end
+
+  def parse_ls_tree(tree)
+    tree.reduce([]) do |acc, object|
+      perm, type, tree_id, path = object.split(" ")
+      if type == "tree"
+        acc << {
+          a_tree_id: "0"*40,
+          b_tree_id: tree_id,
+          path: path,
+          operation: :change
+        }
+      end
+      acc
+    end
+  end
+
+  def diff(commit_id_a, commit_id_b)
+    exec("diff --full-index -l 10000 -U0 #{commit_id_a} #{commit_id_b}")
+  end
+
+  def remote_branches()
+    exec("branch -r")
+      .map { |branch| branch.split(" -> ").last.strip() }
+      .select { |branch| branch.index("origin/") == 0 }
+      .map { |branch| "remotes/#{branch}" }
+  end
+
+  private
+
+  def exec(cmd)
+    `git --git-dir=#{@git_dir || ".git"} #{cmd}`
+      .encode("UTF-8", {invalid: :replace})
+      .lines
+      .map(&:chomp)
+  end
+
+  def full_log(branch, count)
+    Loggr.instance.info("SHOW COMMITS")
+    count_option = count.nil? ? "" : "-n #{count}"
+    exec("log \
+      --topo-order \
+      --oneline \
+      -U0 \
+      --diff-algorithm=histogram \
+      --full-history \
+      --full-index \
+      --date=rfc \
+      --pretty=format:'%H|||%P|||%an <%aE>|||%cd|||%s' \
+      #{count_option} #{branch}"
+    )
+  end
+
+  def short_log(branch)
+    exec("log \
+      --graph \
+      --topo-order \
+      --oneline \
+      --pretty=format:%H #{branch} \
+      | grep '^\*'")
+      .map { |line| line.split(" ").last.to_sym }
+  end
+
+  def tree(commit_id)
+    exec("ls-tree -r -t #{commit_id}")
+  end
+
+  def parse_tree(tree)
+    tree
+      .map do |object|
+        permission, type, object_id, path = object.split(" ")
+        {
+          object_id: object_id,
+          type: type == "tree" ? :tree : :file,
+          path: path.strip()
+        }
+      end
+      .sort_by { |object| object[:path].split(File::SEPARATOR).length }
+  end
+
+  def commits_to_index(branch, commit_ids)
+    last_indexed_commit = Cache.last_indexed_commit(branch, commit_ids)
+
+    # Maybe use default_branch, also git merge-base.
+    if last_indexed_commit == nil && branch != "master"
+      last_indexed_commit = Cache.last_indexed_commit("master", commit_ids)
+    end
+
+    index = commit_ids.index(last_indexed_commit)
+
+    if sync?(index, commit_ids)
+      index = 1
+      last_indexed_commit = commit_ids[index]
+    end
+
+    # When switching to new branch, save all previously indexed commits.
+    if not index.nil?
+      Cache.index_commits(branch, commit_ids[index..-1].reverse())
+    end
+
+    cache = index.nil? ? [] : parse_tree(tree(last_indexed_commit))
+
+    [cache, index]
+  end
+
+  def cache_commits(commits)
+    commits.each { |commit| Store::Commit.index(commit.as_json()) }
+    Store::Commit.cache()
+  end
+
+  def sync?(index, commit_ids)
+    index == 0 &&
+      commit_ids.length > 1 &&
+      ENV["GITOLEMY_SYNC"] == "true"
+  end
+end

data/lib/integrations/jira_client.rb

@@ -0,0 +1,145 @@
+require "json"
+require "net/http"
+require "active_support/core_ext/object"
+
+require_relative "../loggr"
+require_relative "../cache"
+require_relative "../store"
+
+class JiraClient
+  ISSUES_CACHE_KEY = "issues"
+  STORY_TYPE = "Story"
+  BUG_TYPE = "Bug"
+
+  def initialize(config)
+    @client_config = config["client_config"]
+    @issue_id_rgx = issue_id_rgx
+    @cached_issues = load_cached_issues()
+    @business_value_custom_field = "customfield_#{config["business_value_field_id"]}"
+  end
+
+  # TODO: Store Issues as they're fetched.
+  def merge_and_fetch_issues!(commits)
+    issue_ids = fetch_issues(merge_issue_ids!(commits))
+      .reject { |issue_id, issue| issue.empty? }
+      .map { |issue_id, issue| Store::Issue::index(issue) }
+
+    Store::Issue::cache(@cached_issues)
+
+    issue_ids
+  end
+
+  def merge_and_fetch_bugs!(commits)
+    commits
+      .reduce([]) do |acc, commit|
+        issue_id = parse_id(commit.subject).to_sym
+        issue = issue_id.present? ? find_by_id(issue_id) : nil
+        if issue.present? && issue[:issue_type] == "Bug"
+          bug = {}.merge(issue)
+          bug[:bug_id] = bug.delete(:issue_id)
+          commit.bug_id = bug[:bug_id]
+          acc << bug
+        end
+        acc
+      end
+      .map { |bug| Store::Bug::index(bug) }
+  end
+
+  private
+
+  def client(jira_config)
+    jira_config["auth_type"] = jira_config["auth_type"].to_sym
+    jira_config = Hash[jira_config.map{ |key, value| [key.to_sym, value] }]
+    JIRA::Client.new(jira_config)
+  end
+
+  # TODO: support many fix versions...
+  def find_by_id(id)
+    return @cached_issues[id] if @cached_issues.has_key?(id)
+
+    Loggr.instance.info("FETCHING ISSUE: #{id}")
+    issue = fetch_issue(id)
+    {
+      issue_id: id,
+      status: issue.dig("fields", "status", "name"),
+      assignee: extract_person(issue.dig("fields", "assignee")),
+      reporter: extract_person(issue.dig("fields", "reporter")),
+      issue_type: issue.dig("fields", "issuetype", "name"),
+      business_value: issue.dig("fields", @business_value_custom_field) || 0.0,
+      priority: issue.dig("fields", "priority", "name"),
+      fix_versions: issue.dig("fields", "fixVersions").map { |fix| fix["name"] }.first,
+      link: "#{@client_config["site"]}/browse/#{id}",
+      summary: issue.dig("fields", "summary")
+    }
+  rescue
+    {}
+  end
+
+  def parse_id(commit_subject)
+    @issue_id_rgx
+      .match(commit_subject)
+      .try(:[], 0) || ""
+  end
+
+  def fetch_issues(ids)
+    ids.reduce({}) do |obj, id|
+      obj[id] = find_by_id(id)
+      obj
+    end
+  end
+
+  def merge_issue_ids!(commits)
+    commits
+      .map do |commit|
+        commit.issue_id = parse_id(commit.subject).to_sym
+        commit.issue_id
+      end
+      .reject { |id| id.empty? }
+      .uniq
+  end
+
+  def load_cached_issues()
+    Cache.read(ISSUES_CACHE_KEY, {})
+      .reduce({}) do |acc, (issue_id, issue)|
+        issue = issue.symbolize_keys
+        issue[:issue_id] = issue_id.to_sym
+        acc[issue_id.to_sym] = issue
+        acc
+      end
+  end
+
+  # TODO: Cache projects
+  def issue_id_rgx()
+    Regexp.new("(#{project_keys.join("|")})-\\d+", Regexp::IGNORECASE)
+  end
+
+  def fetch_issue(issue_id)
+    request("/issue/#{issue_id}")
+  end
+
+  def extract_person(data)
+    return {} if data.nil?
+    {
+      email: data["emailAddress"],
+      name: data["displayName"]
+    }
+  end
+
+  def fetch_projects()
+    request("/project")
+  end
+
+  def request(uri)
+    uri = URI("#{@client_config["site"]}/rest/api/2#{uri}")
+    req = Net::HTTP::Get.new(uri)
+    req.basic_auth(@client_config["username"], @client_config["password"])
+    http = Net::HTTP.new(uri.hostname, uri.port)
+    http.use_ssl = true
+    JSON.parse(http.request(req).body)
+  end
+
+  def project_keys()
+    fetch_projects()
+      .map { |project| project["key"] }
+  end
+end

data/lib/integrations/rollbar_client.rb

@@ -0,0 +1,147 @@
+require "json"
+require "date"
+require "net/http"
+
+require_relative "error_client"
+require_relative "../store"
+require_relative "../loggr"
+
+class RollbarClient < ErrorClient
+
+  def initialize(config)
+    @user = config["username"]
+    @key = config["api_key"]
+    @project = config["project"]
+    @environment = config["environment"]
+    @errors = @key.nil? ? [] : errors()
+    @deploys = @key.nil? ? [] : deploys()
+  end
+
+  private
+
+  def deploys()
+    fetch_deploys()
+      .map do |deploy|
+        {
+          commit_id: deploy["revision"].to_sym,
+          environment: deploy["environment"],
+          timestamp: Time.at(deploy["start_time"]).to_datetime
+        }
+      end
+      .sort_by { |deploy| deploy[:timestamp] }
+      .select { |deploy| deploy[:environment] == @environment }
+  end
+
+  # TODO: Cached errors are already applied, maybe skip...
+  def errors()
+    cached_errors = load_from_cache()
+
+    errors = fetch_errors(cached_errors)["items"]
+      .flat_map(&method(:select_traces))
+      .reject(&:nil?)
+      .map do |trace|
+        stack_trace = trace["frames"].map do |frame|
+          file = frame["filename"]
+          line = frame["lineno"]
+          function = frame["method"]
+
+          {
+            file: file,
+            line: line,
+            function: function
+          }
+        end
+
+        {
+          error_id: trace[:id],
+          first_time: trace[:first_time],
+          last_time: trace[:last_time],
+          environment: trace[:environment],
+          type: trace[:type],
+          message: trace[:message],
+          link: "https://rollbar.com/#{@user}/#{@project}/items/#{trace[:counter]}/",
+          total_occurrences: trace[:total_occurrences],
+          stack_trace: stack_trace.reverse()
+        }
+      end
+      .concat(cached_errors)
+      .select { |error| error[:environment] == @environment }
+      .sort_by { |error| error[:last_time] }
+
+    errors.each { |error| Store::Error::index(error) }
+    Store::Error::cache()
+
+    errors
+  end
+
+  def select_traces(item)
+    error = detail_error(item["id"])["result"]["instances"].first
+    traces = error["data"]["body"]["trace"].present? ?
+      [error["data"]["body"]["trace"]] :
+      error["data"]["body"]["trace_chain"]
+
+    return nil if traces.nil?
+
+    traces.map do |trace|
+      trace[:id] = item["id"].to_s.to_sym
+      trace[:environment] = item["environment"]
+      trace[:total_occurrences] = item["total_occurrences"]
+      trace[:error] = error
+      trace[:type] = item["level"]
+      trace[:message] = item["title"]
+      trace[:first_time] = Time.at(item["first_occurrence_timestamp"]).to_datetime
+      trace[:last_time] = Time.at(item["last_occurrence_timestamp"]).to_datetime
+      trace[:counter] = item["counter"]
+      trace
+    end
+  end
+
+  def fetch_deploys(page=1)
+    deploys = []
+    while true do
+      resp = JSON.parse(Net::HTTP.get(
+        URI("https://api.rollbar.com/api/1/deploys/?access_token=#{@key}&page=#{page}")
+      ))["result"]["deploys"]
+
+      break if resp.count == 0
+
+      deploys.concat(resp)
+      page += 1
+    end
+    Cache.write(DEPLOY_CACHE_KEY, deploys)
+    deploys
+  rescue
+    deploys
+  end
+
+  def detail_error(item_id)
+    Loggr.instance.info("FETCHING ERROR: #{item_id}")
+    JSON.parse(Net::HTTP.get(
+      URI("https://api.rollbar.com/api/1/item/#{item_id}/instances/?access_token=#{@key}")
+    ))
+  end
+
+  def fetch_errors(cached_errors, page=1)
+    errors = {"items" => [], "count" => 1}
+    while errors["items"].length < errors["count"] do
+      resp = JSON.parse(Net::HTTP.get(
+        URI("https://api.rollbar.com/api/1/items/?access_token=#{@key}&page=#{page}")
+      ))
+
+      errors["count"] = resp["total_count"]
+
+      while item = resp["result"]["items"].shift() do
+        if cached_errors.detect { |cached_item| cached_item[:error_id] == item["id"].to_s.to_sym }
+          return errors
+        else
+          errors["items"] << item
+        end
+      end
+
+      page += 1
+    end
+    errors
+  rescue
+    errors
+  end
+end