how_is 24.0.0 → 25.0.0

data/lib/how_is/sources/ci/travis.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "date"
+require "okay/default"
+require "okay/http"
+require "how_is/constants"
+require "how_is/sources/github"
+require "how_is/text"
+
+module HowIs
+  module Sources
+    module CI
+      # Fetches metadata about CI builds from travis-ci.org.
+      class Travis
+        BadResponseError = Class.new(StandardError)
+
+        # @param repository [String] GitHub repository name, of the format user/repo.
+        # @param start_date [String] Start date for the report being generated.
+        # @param end_date   [String] End date for the report being generated.
+        # @param cache      [Cacheable] Instance of HowIs::Cacheable to cache API calls
+        def initialize(config, start_date, end_date, cache)
+          @config = config
+          @cache = cache
+          @repository = config["repository"]
+          raise "Travis.new() got nil repository." if @repository.nil?
+          @start_date = DateTime.parse(start_date)
+          @end_date = DateTime.parse(end_date)
+          @default_branch = Okay.default
+        end
+
+        # @return [String] The default branch name.
+        def default_branch
+          return @default_branch unless @default_branch == Okay.default
+
+          response = fetch("branches", {"sort_by" => "default_branch"})
+          validate_response!(response)
+
+          branches = response["branches"]
+          validate_branches!(branches)
+
+          branch = branches.find { |b| b["default_branch"] == true }
+          @default_branch = branch&.fetch("name", nil)
+        end
+
+        # Returns the builds for the default branch.
+        #
+        # @return [Hash] Hash containing the builds for the default branch.
+        def builds
+          @cache.cached("travis_builds") do
+            raw_builds \
+              .map(&method(:normalize_build)) \
+              .select(&method(:in_date_range?)) \
+              .map(&method(:add_build_urls))
+          end
+        end
+
+        private
+
+        def add_build_urls(build)
+          build["html_url"] = "https://travis-ci.org/#{build['repository']}#{build['@href']}"
+          build
+        end
+
+        def validate_response!(response)
+          return true if hash_with_key?(response, "branches")
+
+          raise BadResponseError,
+            "expected `response' (#{response.class}) to be a Hash with key `\"branches\"'."
+        end
+
+        def validate_branches!(branches)
+          return true if array_of_hashes?(branches)
+
+          raise BadResponseError, "expected `branches' to be Array of Hashes."
+        end
+
+        def array_of_hashes?(ary)
+          ary.is_a?(Array) && ary.all? { |obj| obj.is_a?(Hash) }
+        end
+
+        def hash_with_key?(hsh, key)
+          hsh.is_a?(Hash) && hsh.has_key?(key)
+        end
+
+        def in_date_range?(build, start_date = @start_date, end_date = @end_date)
+          (build["started_at"] >= start_date) \
+            && (build["finished_at"] <= end_date)
+        end
+
+        def raw_builds
+          results = fetch("builds", {
+            "event_type" => "push",
+            "branch.name" => default_branch,
+          })
+
+          results["builds"] || {}
+        rescue Net::HTTPServerException
+          # It's not elegant, but it works™.
+          {}
+        end
+
+        def normalize_build(build)
+          build_keys = ["@href", "pull_request_title", "pull_request_number",
+                        "started_at", "finished_at", "repository", "commit",
+                        "jobs"]
+          result = pluck_keys(build, build_keys)
+
+          commit_keys = ["sha", "ref", "message", "compare_url",
+                         "committed_at", "jobs"]
+          result["commit"] = pluck_keys(result["commit"], commit_keys)
+
+          job_keys = ["href", "id"]
+          result["jobs"] = result["jobs"].map { |j| pluck_keys(j, job_keys) }
+
+          result["repository"] = result["repository"]["slug"]
+
+          ["started_at", "finished_at"].each do |k|
+            next if k.nil?
+            result[k] = DateTime.parse(result[k])
+          end
+
+          result
+        end
+
+        def pluck_keys(hsh, keys)
+          keys.map { |k| [k, hsh[k]] }.to_h
+        end
+
+        # Returns API results for /repos/:user/:repo/<path>.
+        #
+        # @param path [String] Path suffix (appended to /repo/<repo name>/).
+        # @param parameters [Hash] Parameters.
+        # @return [String] JSON result.
+        def fetch(path, parameters = {})
+          @cache.cached("travis_path") do
+            HowIs::Text.print "Fetching Travis CI #{path.sub(/e?s$/, '')} data."
+
+            # Apparently this is required for the Travis CI API to work.
+            repo = @repository.sub("/", "%2F")
+
+            ret = Okay::HTTP.get(
+              "https://api.travis-ci.org/repo/#{repo}/#{path}",
+              parameters: parameters,
+              headers: {
+                "Travis-Api-Version" => "3",
+                "Accept" => "application/json",
+                "User-Agent" => HowIs::USER_AGENT,
+              }
+            ).or_raise!.from_json
+
+            HowIs::Text.puts
+
+            ret
+          end
+        end
+      end
+    end
+  end
+end

data/lib/how_is/sources/github/contributions.rb

@@ -1,162 +1,203 @@
 # frozen_string_literal: true
 
-require "how_is/version"
+require "github_api"
+require "how_is/cacheable"
 require "how_is/sources/github"
 require "how_is/sources/github_helpers"
+require "how_is/template"
+require "how_is/text"
 require "date"
 
-module HowIs::Sources
-  class Github
-    # Fetch information about who has contributed to a repository during a given
-    # period.
-    #
-    # Usage:
-    #
-    #     c = HowIs::Contributions.new(start_date: '2017-07-01', user: 'how-is', repo: 'how_is')
-    #     c.commits          #=> All commits during July 2017.
-    #     c.contributors #=> All contributors during July 2017.
-    #     c.new_contributors #=> New contributors during July 2017.
-    class Contributions
-      include HowIs::Sources::GithubHelpers
-
-      # Returns an object that fetches contributor information about a particular
-      # repository for a month-long period starting on +start_date+.
+module HowIs
+  module Sources
+    class Github
+      # Fetch information about who has contributed to a repository during
+      # a given period.
       #
-      # @param repository [String] GitHub repository in the form of "user/repo".
-      # @param start_date [String] Date in the format YYYY-MM-DD. The first date
-      #                            to include commits from.
-      # @param end_date [String] Date in the format YYYY-MM-DD. The last date
-      #                          to include commits from.
-      def initialize(repository, start_date, end_date)
-        @user, @repo = repository.split("/")
-        @github = ::Github.new(auto_pagination: true) do |config|
-          config.basic_auth = HowIs::Sources::Github::BASIC_AUTH
-        end
+      # Usage:
+      #
+      #     c = HowIs::Contributions.new(start_date: '2017-07-01', user: 'how-is', repo: 'how_is')
+      #     c.commits          #=> All commits during July 2017.
+      #     c.contributors #=> All contributors during July 2017.
+      #     c.new_contributors #=> New contributors during July 2017.
+      class Contributions
+        include HowIs::Sources::GithubHelpers
+
+        # Returns an object that fetches contributor information about a
+        # particular repository for a month-long period starting on +start_date+.
+        #
+        # @param config     [Hash]   A config object.
+        # @param start_date [String] Date in the format YYYY-MM-DD.
+        #                            The first date to include commits from.
+        # @param end_date   [String] Date in the format YYYY-MM-DD.
+        #                            The last date to include commits from.
+        # @param cache      [Cacheable] Instance of HowIs::Cacheable to cache API calls
+        def initialize(config, start_date, end_date, cache)
+          raise "Got String, need Hash. The Github::Contributions API changed." if config.is_a?(String)
+
+          @config = config
+          @cache = cache
+          @github = HowIs::Sources::Github.new(config)
+          @repository = config["repository"]
+
+          @user, @repo = @repository.split("/")
+          @github = ::Github.new(auto_pagination: true) { |conf|
+            conf.basic_auth = @github.basic_auth
+          }
 
-        # IMPL. DETAIL: The external API uses "end_date" so it's clearer,
-        #               but internally we use "until_date" to match GitHub's API.
-        @since_date = start_date
-        @until_date = end_date
+          # IMPL. DETAIL: The external API uses "end_date" so it's clearer,
+          #               but internally we use "until_date" to match GitHub's API.
+          @since_date = start_date
+          @until_date = end_date
 
-        @commit = {}
-        @stats = nil
-        @changed_files = nil
-      end
+          @commit = {}
+          @stats = nil
+          @changed_files = nil
+        end
 
-      # Returns a list of contributors that have zero commits before the @since_date.
-      #
-      # @return [Hash{String => Hash}] Contributors keyed by email
-      def new_contributors
-        # author: GitHub login, name or email by which to filter by commit author.
-        @new_contributors ||= contributors.select do |email, _committer|
-          # Returns true if +email+ never wrote a commit for +@repo+ before +@since_date+.
-          @github.repos.commits.list(user: @user,
-                                    repo: @repo,
-                                    until: @since_date,
-                                    author: email).count.zero?
+        # Returns a list of contributors that have zero commits before the @since_date.
+        #
+        # @return [Hash{String => Hash}] Contributors keyed by email
+        def new_contributors
+          @new_contributors ||= contributors.select { |email, _committer|
+            args = {
+              user: @user,
+              repo: @repo,
+              until: @since_date,
+              author: email,
+            }
+            # True if +email+ never wrote a commit for +@repo+ before +@since_date+, false otherwise.
+            commits = @cache.cached("repos_commits", args.to_json) do
+              @github.repos.commits.list(**args)
+            end
+            commits.count.zero?
+          }
         end
-      end
 
-      # @return [Hash{String => Hash}] Author information keyed by author's email.
-      def contributors
-        commits.map { |api_response|
-          [api_response.commit.author.email, api_response.commit.author.to_h]
-        }.to_h
-      end
+        def new_contributors_html
+          names = new_contributors.values.map { |c| c["name"] }
+          list_items = names.map { |n| "  <li>#{n}</li>" }.join("\n")
 
-      def commits
-        @commits ||= begin
-          @github.repos.commits.list(user: @user,
-                                    repo: @repo,
-                                    since: @since_date,
-                                    until: @until_date).map { |c|
-            # The commits list endpoint doesn't include all commit data, e.g. stats.
-            # So, we make N requests here, where N == number of commits returned,
-            # and then we die a bit inside.
-            commit(c.sha)
-          }
+          if names.length.zero?
+            num_new_contributors = "no"
+          else
+            num_new_contributors = names.length
+          end
+
+          if names.length == 1
+            was_were = "was"
+            contributor_s = ""
+          else
+            was_were = "were"
+            contributor_s = "s"
+          end
+
+          Template.apply("new_contributors_partial.html", {
+            was_were: was_were,
+            contributor_s: contributor_s,
+            number_of_new_contributors: num_new_contributors,
+            list_items: list_items,
+          }).strip
         end
-      end
 
-      def commit(sha)
-        @commit[sha] ||= @github.repos.commits.get(user: @user, repo: @repo, sha: sha)
-      end
+        # @return [Hash{String => Hash}] Author information keyed by author's email.
+        def contributors
+          commits.map { |api_response|
+            [api_response.commit.author.email, api_response.commit.author.to_h]
+          }.to_h
+        end
 
-      def changes
-        if @stats.nil? || @changed_files.nil?
-          @stats = {
-            "total" => 0,
-            "additions" => 0,
-            "deletions" => 0,
-          }
+        def commits
+          return @commits if instance_variable_defined?(:@commits)
 
-          @changed_files = []
+          args = {
+            user: @user,
+            repo: @repo,
+            since: @since_date,
+            until: @until_date,
+          }
 
-          commits.map do |commit|
-            @stats.keys.each do |key|
-              @stats[key] += commit.stats[key]
-            end
+          HowIs::Text.print "Fetching #{@repository} commit data."
 
-            @changed_files += commit.files.map { |file| file["filename"] }
+          # The commits list endpoint doesn't include all stats.
+          # So, to compensate, we make N requests here, where N is number
+          # of commits returned, and then we die a bit inside.
+          @commits = @cache.cached("repos_commits", args.to_json) do
+            @github.repos.commits.list(**args).map { |c|
+              HowIs::Text.print "."
+              commit(c.sha)
+            }
           end
-
-          @changed_files = @changed_files.sort.uniq
+          HowIs::Text.puts
+          @commits
         end
 
-        {"stats" => @stats, "files" => @changed_files}
-      end
-
-      def changed_files
-        changes["files"]
-      end
+        def commit(sha)
+          @commit[sha] ||= @cache.cached("repos_commit_#{sha}") do
+            @github.repos.commits.get(user: @user, repo: @repo, sha: sha)
+          end
+        end
 
-      def additions_count
-        changes["stats"]["additions"]
-      end
+        def stats
+          return @stats if @stats
 
-      def deletions_count
-        changes["stats"]["deletions"]
-      end
+          stats = {"total" => 0, "additions" => 0, "deletions" => 0}
+          commits.map do |commit|
+            stats.keys.each do |key|
+              stats[key] += commit.stats[key]
+            end
+          end
+          @stats = stats
+        end
 
-      def compare_url
-        "https://github.com/#{@user}/#{@repo}/compare/#{default_branch}@%7B#{@since_date}%7D...#{default_branch}@%7B#{@until_date}%7D" # rubocop:disable Metrics/LineLength
-      end
+        def changed_files
+          return @changed_files if @changed_files
+          files = commits.flat_map do |commit|
+            commit.files.map { |file| file["filename"] }
+          end
+          @changed_files = files.sort.uniq
+        end
 
-      def default_branch
-        @default_branch ||= @github.repos.get(user: @user,
-          repo: @repo).default_branch
-      end
+        def changes
+          {"stats" => stats, "files" => changed_files}
+        end
 
-      def to_html(start_text: nil)
-        # TODO: Pulse has information about _all_ branches. Do we want that?
-        #       If we do, we'd need to pass a branch name as the 'sha' parameter
-        #       to /repos/:owner/:repo/commits.
-        #       https://developer.github.com/v3/repos/commits/
-
-        start_text ||= "From #{pretty_date(@since_date)} through #{pretty_date(@until_date)}"
-
-        "#{start_text}, #{@user}/#{@repo} gained "\
-          "<a href=\"#{compare_url}\">#{pluralize('new commit', commits.length)}</a>, " \
-          "contributed by #{pluralize('author', contributors.length)}. There " \
-          "#{(additions_count == 1) ? 'was' : 'were'} " \
-          "#{pluralize('addition', additions_count)} and " \
-          "#{pluralize('deletion', deletions_count)} across " \
-          "#{pluralize('file', changed_files.length)}."
-      end
-      alias :summary :to_html # For backwards compatibility.
+        def additions_count
+          changes["stats"]["additions"]
+        end
 
-      private
+        def deletions_count
+          changes["stats"]["deletions"]
+        end
 
-      def date_to_dt(date)
-        DateTime.strptime(date, "%Y-%m-%d")
-      end
+        def compare_url
+          "https://github.com/#{@user}/#{@repo}/compare/#{default_branch}@%7B#{@since_date}%7D...#{default_branch}@%7B#{@until_date}%7D" # rubocop:disable Metrics/LineLength
+        end
 
-      def timestamp_for(date)
-        date_to_dt(date).strftime("%s")
-      end
+        def default_branch
+          @default_branch ||= @cache.cached("repos_default_branch") do
+            @github.repos.get(user: @user, repo: @repo).default_branch
+          end
+        end
 
-      def pretty_date(date)
-        date_to_dt(date).strftime("%b %d, %Y")
+        # rubocop:disable Metrics/AbcSize
+        def to_html(start_text: nil)
+          start_text ||= "From #{pretty_date(@since_date)} through #{pretty_date(@until_date)}"
+
+          HowIs::Template.apply("contributions_partial.html", {
+            start_text: start_text,
+            user: @user,
+            repo: @repo,
+            compare_url: compare_url,
+            additions_count_str: (additions_count == 1) ? "was" : "were",
+            authors: pluralize("author", contributors.length),
+            new_commits: pluralize("new commit", commits.length),
+            additions: pluralize("addition", additions_count),
+            deletions: pluralize("deletion", deletions_count),
+            changed_files: pluralize("file", changed_files.length),
+          }).strip
+        end
+        # rubocop:enable Metrics/AbcSize
       end
     end
   end

data/lib/how_is/sources/github/issue_fetcher.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "how_is/version"
+require "how_is/date_time_helpers"
+require "how_is/sources/github"
+require "how_is/text"
+
+module HowIs
+  module Sources
+    class Github
+      ##
+      # Fetches raw data for GitHub issues.
+      class IssueFetcher
+        include HowIs::DateTimeHelpers
+
+        END_LOOP = :terminate_graphql_loop
+
+        GRAPHQL_QUERY = <<~QUERY
+          repository(owner: %{user}, name: %{repo}) {
+            %{type}(first: %{chunk_size}%{after_str}, orderBy:{field: CREATED_AT, direction: ASC}) {
+              edges {
+                cursor
+                node {
+                  number
+                  createdAt
+                  closedAt
+                  updatedAt
+                  state
+                  title
+                  url
+                  labels(first: 100) {
+                    nodes {
+                      name
+                    }
+                  }
+                }
+              }
+            }
+          }
+        QUERY
+
+        CHUNK_SIZE = 100
+
+        attr_reader :type
+
+        # @param issues_source [Issues] HowIs::Issues (or HowIs::Pulls) instance for which to fetch issues
+        def initialize(issues_source)
+          @issues_source = issues_source
+          @cache = issues_source.cache
+          @github = HowIs::Sources::Github.new(issues_source.config)
+          @repository = issues_source.config["repository"]
+          @user, @repo = @repository.split("/", 2)
+          @start_date = issues_source.start_date
+          @end_date = issues_source.end_date
+          @type = issues_source.type
+        end
+
+        def data
+          return @data if instance_variable_defined?(:@data)
+
+          @data = []
+          return @data if last_cursor.nil?
+
+          HowIs::Text.print "Fetching #{@repository} #{@issues_source.pretty_type} data."
+
+          @data = @cache.cached("fetch-#{type}") do
+            data = []
+            after, data = fetch_issues(after, data) until after == END_LOOP
+            data.select(&method(:issue_is_relevant?))
+          end
+
+          HowIs::Text.puts
+
+          @data
+        end
+
+        def issue_is_relevant?(issue)
+          if !issue["closedAt"].nil? && date_le(issue["closedAt"], @start_date)
+            false
+          else
+            date_ge(issue["createdAt"], @start_date) && date_le(issue["createdAt"], @end_date)
+          end
+        end
+
+        def last_cursor
+          return @last_cursor if instance_variable_defined?(:@last_cursor)
+
+          raw_data = @github.graphql <<~QUERY
+            repository(owner: #{@user.inspect}, name: #{@repo.inspect}) {
+              #{type}(last: 1, orderBy:{field: CREATED_AT, direction: ASC}) {
+                edges {
+                  cursor
+                }
+              }
+            }
+          QUERY
+
+          edges = raw_data.dig("data", "repository", type, "edges")
+          @last_cursor =
+            if edges.nil? || edges.empty?
+              nil
+            else
+              edges.last["cursor"]
+            end
+        end
+
+        def fetch_issues(after, data)
+          HowIs::Text.print "."
+
+          after_str = ", after: #{after.inspect}" unless after.nil?
+
+          query = build_query(@user, @repo, type, after_str)
+          raw_data = @github.graphql(query)
+          edges = raw_data.dig("data", "repository", type, "edges")
+
+          data += edge_nodes(edges)
+
+          next_cursor = edges.last["cursor"]
+          next_cursor = END_LOOP if next_cursor == last_cursor
+
+          [next_cursor, data]
+        end
+
+        def build_query(user, repo, type, after_str)
+          format(GRAPHQL_QUERY, {
+            user: user.inspect,
+            repo: repo.inspect,
+            type: type,
+            chunk_size: CHUNK_SIZE,
+            after_str: after_str,
+          })
+        end
+
+        def edge_nodes(edges)
+          return [] if edges.nil?
+          new_data = edges.map { |issue|
+            node = issue["node"]
+            node["labels"] = node["labels"]["nodes"]
+
+            node
+          }
+
+          new_data
+        end
+      end
+    end
+  end
+end