github_changelog_generator 1.13.2 → 1.14.0

data/lib/github_changelog_generator/generator/generator_tags.rb

@@ -3,33 +3,59 @@ module GitHubChangelogGenerator
   class Generator
     # fetch, filter tags, fetch dates and sort them in time order
     def fetch_and_filter_tags
-      @filtered_tags = get_filtered_tags(@fetcher.get_all_tags)
-      fetch_tags_dates
+      detect_since_tag
+      detect_due_tag
+
+      all_tags      = @fetcher.get_all_tags
+      included_tags = filter_excluded_tags(all_tags)
+
+      fetch_tags_dates(all_tags) # Creates a Hash @tag_times_hash
+      @sorted_tags   = sort_tags_by_date(included_tags)
+      @filtered_tags = get_filtered_tags(included_tags)
+
+      @tag_section_mapping = build_tag_section_mapping(@filtered_tags, sorted_tags)
+
+      @filtered_tags
     end
 
-    # Sort all tags by date
+    # @param [Array] filtered_tags are the tags that need a subsection output
+    # @param [Array] all_tags is the list of all tags ordered from newest -> oldest
+    # @return [Hash] key is the tag to output, value is an array of [Left Tag, Right Tag]
+    # PRs to include in this section will be >= [Left Tag Date] and <= [Right Tag Date]
+    def build_tag_section_mapping(filtered_tags, all_tags)
+      tag_mapping = {}
+      filtered_tags.each do |tag|
+        older_tag_idx = all_tags.index(tag) + 1
+        older_tag = all_tags[older_tag_idx]
+        tag_mapping[tag] = [older_tag, tag]
+      end
+      tag_mapping
+    end
+
+    # Sort all tags by date, newest to oldest
     def sort_tags_by_date(tags)
-      puts "Sorting tags..." if @options[:verbose]
+      puts "Sorting tags..." if options[:verbose]
       tags.sort_by! do |x|
         get_time_of_tag(x)
       end.reverse!
     end
 
-    # Try to find tag date in local hash.
-    # Otherwise fFetch tag time and put it to local hash file.
-    # @param [Hash] tag_name name of the tag
+    # Returns date for given GitHub Tag hash
+    #
+    # Memoize the date by tag name.
+    #
+    # @param [Hash] tag_name
+    #
     # @return [Time] time of specified tag
     def get_time_of_tag(tag_name)
       raise ChangelogGeneratorError, "tag_name is nil" if tag_name.nil?
 
-      name_of_tag = tag_name["name"]
-      time_for_name = @tag_times_hash[name_of_tag]
-      if !time_for_name.nil?
-        time_for_name
-      else
-        time_string = @fetcher.fetch_date_of_tag tag_name
+      name_of_tag = tag_name.fetch("name")
+      time_for_tag_name = @tag_times_hash[name_of_tag]
+      return time_for_tag_name if time_for_tag_name
+
+      @fetcher.fetch_date_of_tag(tag_name).tap do |time_string|
         @tag_times_hash[name_of_tag] = time_string
-        time_string
       end
     end
 
@@ -42,12 +68,12 @@ module GitHubChangelogGenerator
       newer_tag_time = newer_tag.nil? ? Time.new : get_time_of_tag(newer_tag)
 
       # if it's future release tag - set this value
-      if newer_tag.nil? && @options[:future_release]
-        newer_tag_name = @options[:future_release]
-        newer_tag_link = @options[:future_release]
+      if newer_tag.nil? && options[:future_release]
+        newer_tag_name = options[:future_release]
+        newer_tag_link = options[:future_release]
       else
         # put unreleased label if there is no name for the tag
-        newer_tag_name = newer_tag.nil? ? @options[:unreleased_label] : newer_tag["name"]
+        newer_tag_name = newer_tag.nil? ? options[:unreleased_label] : newer_tag["name"]
         newer_tag_link = newer_tag.nil? ? "HEAD" : newer_tag_name
       end
       [newer_tag_link, newer_tag_name, newer_tag_time]
@@ -55,13 +81,17 @@ module GitHubChangelogGenerator
 
     # @return [Object] try to find newest tag using #Reader and :base option if specified otherwise returns nil
     def detect_since_tag
-      @since_tag ||= @options.fetch(:since_tag) { version_of_first_item }
+      @since_tag ||= options.fetch(:since_tag) { version_of_first_item }
+    end
+
+    def detect_due_tag
+      @due_tag ||= options.fetch(:due_tag, nil)
     end
 
     def version_of_first_item
-      return unless File.file?(@options[:base].to_s)
+      return unless File.file?(options[:base].to_s)
 
-      sections = GitHubChangelogGenerator::Reader.new.read(@options[:base])
+      sections = GitHubChangelogGenerator::Reader.new.read(options[:base])
       sections.first["version"] if sections && sections.any?
     end
 
@@ -70,8 +100,8 @@ module GitHubChangelogGenerator
     # @return [Array]
     def get_filtered_tags(all_tags)
       filtered_tags = filter_since_tag(all_tags)
-      filtered_tags = filter_between_tags(filtered_tags)
-      filter_excluded_tags(filtered_tags)
+      filtered_tags = filter_due_tag(filtered_tags)
+      filter_between_tags(filtered_tags)
     end
 
     # @param [Array] all_tags all tags
@@ -80,8 +110,8 @@ module GitHubChangelogGenerator
       filtered_tags = all_tags
       tag = detect_since_tag
       if tag
-        if all_tags.map(&:name).include? tag
-          idx = all_tags.index { |t| t.name == tag }
+        if all_tags.map { |t| t["name"] }.include? tag
+          idx = all_tags.index { |t| t["name"] == tag }
           filtered_tags = if idx > 0
                             all_tags[0..idx - 1]
                           else
@@ -98,13 +128,12 @@ module GitHubChangelogGenerator
     # @return [Array] filtered tags according :due_tag option
     def filter_due_tag(all_tags)
       filtered_tags = all_tags
-      tag = @options[:due_tag]
+      tag           = detect_due_tag
       if tag
-        if all_tags.any? && all_tags.map(&:name).include?(tag)
-          idx = all_tags.index { |t| t.name == tag }
-          last_index = all_tags.count - 1
-          filtered_tags = if idx > 0 && idx < last_index
-                            all_tags[idx + 1..last_index]
+        if all_tags.any? && all_tags.map { |t| t["name"] }.include?(tag)
+          idx = all_tags.index { |t| t["name"] == tag }
+          filtered_tags = if idx > 0
+                            all_tags[(idx + 1)..-1]
                           else
                             []
                           end
@@ -119,13 +148,15 @@ module GitHubChangelogGenerator
     # @return [Array] filtered tags according :between_tags option
     def filter_between_tags(all_tags)
       filtered_tags = all_tags
-      if @options[:between_tags]
-        @options[:between_tags].each do |tag|
-          unless all_tags.map(&:name).include? tag
+      tag_names     = filtered_tags.map { |ft| ft["name"] }
+
+      if options[:between_tags]
+        options[:between_tags].each do |tag|
+          unless tag_names.include?(tag)
             Helper.log.warn "Warning: can't find tag #{tag}, specified with --between-tags option."
           end
         end
-        filtered_tags = all_tags.select { |tag| @options[:between_tags].include? tag.name }
+        filtered_tags = all_tags.select { |tag| options[:between_tags].include?(tag["name"]) }
       end
       filtered_tags
     end
@@ -133,9 +164,9 @@ module GitHubChangelogGenerator
     # @param [Array] all_tags all tags
     # @return [Array] filtered tags according :exclude_tags or :exclude_tags_regex option
     def filter_excluded_tags(all_tags)
-      if @options[:exclude_tags]
+      if options[:exclude_tags]
         apply_exclude_tags(all_tags)
-      elsif @options[:exclude_tags_regex]
+      elsif options[:exclude_tags_regex]
         apply_exclude_tags_regex(all_tags)
       else
         all_tags
@@ -145,39 +176,39 @@ module GitHubChangelogGenerator
     private
 
     def apply_exclude_tags(all_tags)
-      if @options[:exclude_tags].is_a?(Regexp)
-        filter_tags_with_regex(all_tags, @options[:exclude_tags])
+      if options[:exclude_tags].is_a?(Regexp)
+        filter_tags_with_regex(all_tags, options[:exclude_tags])
       else
         filter_exact_tags(all_tags)
       end
     end
 
     def apply_exclude_tags_regex(all_tags)
-      filter_tags_with_regex(all_tags, Regexp.new(@options[:exclude_tags_regex]))
+      filter_tags_with_regex(all_tags, Regexp.new(options[:exclude_tags_regex]))
     end
 
     def filter_tags_with_regex(all_tags, regex)
       warn_if_nonmatching_regex(all_tags)
-      all_tags.reject { |tag| regex =~ tag.name }
+      all_tags.reject { |tag| regex =~ tag["name"] }
     end
 
     def filter_exact_tags(all_tags)
-      @options[:exclude_tags].each do |tag|
+      options[:exclude_tags].each do |tag|
         warn_if_tag_not_found(all_tags, tag)
       end
-      all_tags.reject { |tag| @options[:exclude_tags].include? tag.name }
+      all_tags.reject { |tag| options[:exclude_tags].include?(tag["name"]) }
     end
 
     def warn_if_nonmatching_regex(all_tags)
-      unless all_tags.map(&:name).any? { |t| @options[:exclude_tags] =~ t }
+      unless all_tags.map { |t| t["name"] }.any? { |t| options[:exclude_tags] =~ t }
         Helper.log.warn "Warning: unable to reject any tag, using regex "\
-                        "#{@options[:exclude_tags].inspect} in --exclude-tags "\
+                        "#{options[:exclude_tags].inspect} in --exclude-tags "\
                         "option."
       end
     end
 
     def warn_if_tag_not_found(all_tags, tag)
-      unless all_tags.map(&:name).include? tag
+      unless all_tags.map { |t| t["name"] }.include?(tag)
         Helper.log.warn "Warning: can't find tag #{tag}, specified with --exclude-tags option."
       end
     end

data/lib/github_changelog_generator/octo_fetcher.rb

@@ -0,0 +1,363 @@
+# frozen_string_literal: true
+require "retriable"
+module GitHubChangelogGenerator
+  # A Fetcher responsible for all requests to GitHub and all basic manipulation with related data
+  # (such as filtering, validating, e.t.c)
+  #
+  # Example:
+  # fetcher = GitHubChangelogGenerator::OctoFetcher.new(options)
+  class OctoFetcher
+    PER_PAGE_NUMBER   = 100
+    MAX_THREAD_NUMBER = 25
+    MAX_FORBIDDEN_RETRIES = 100
+    CHANGELOG_GITHUB_TOKEN = "CHANGELOG_GITHUB_TOKEN"
+    GH_RATE_LIMIT_EXCEEDED_MSG = "Warning: Can't finish operation: GitHub API rate limit exceeded, change log may be " \
+    "missing some issues. You can limit the number of issues fetched using the `--max-issues NUM` argument."
+    NO_TOKEN_PROVIDED = "Warning: No token provided (-t option) and variable $CHANGELOG_GITHUB_TOKEN was not found. " \
+    "This script can make only 50 requests to GitHub API per hour without token!"
+
+    # @param options [Hash] Options passed in
+    # @option options [String] :user GitHub username
+    # @option options [String] :project GitHub project
+    # @option options [String] :since Only issues updated at or after this time are returned. This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ. eg. Time.parse("2016-01-01 10:00:00").iso8601
+    # @option options [Boolean] :http_cache Use ActiveSupport::Cache::FileStore to cache http requests
+    # @option options [Boolean] :cache_file If using http_cache, this is the cache file path
+    # @option options [Boolean] :cache_log If using http_cache, this is the cache log file path
+    def initialize(options = {}) # rubocop:disable Metrics/CyclomaticComplexity
+      @options      = options || {}
+      @user         = @options[:user]
+      @project      = @options[:project]
+      @since        = @options[:since]
+      @http_cache   = @options[:http_cache]
+      @cache_file   = @options.fetch(:cache_file, "/tmp/github-changelog-http-cache") if @http_cache
+      @cache_log    = @options.fetch(:cache_log, "/tmp/github-changelog-logger.log") if @http_cache
+      init_cache if @http_cache
+
+      @github_token = fetch_github_token
+
+      @request_options               = { per_page: PER_PAGE_NUMBER }
+      @github_options                = {}
+      @github_options[:access_token] = @github_token unless @github_token.nil?
+      @github_options[:api_endpoint] = @options[:github_endpoint] unless @options[:github_endpoint].nil?
+
+      client_type = @options[:github_endpoint].nil? ? Octokit::Client : Octokit::EnterpriseAdminClient
+      @client     = client_type.new(@github_options)
+    end
+
+    def init_cache
+      middleware_opts = {
+        serializer: Marshal,
+        store: ActiveSupport::Cache::FileStore.new(@cache_file),
+        logger: Logger.new(@cache_log),
+        shared_cache: false
+      }
+      stack = Faraday::RackBuilder.new do |builder|
+        builder.use Faraday::HttpCache, middleware_opts
+        builder.use Octokit::Response::RaiseError
+        builder.adapter Faraday.default_adapter
+        # builder.response :logger
+      end
+      Octokit.middleware = stack
+    end
+
+    # Fetch all tags from repo
+    #
+    # @return [Array <Hash>] array of tags
+    def get_all_tags
+      print "Fetching tags...\r" if @options[:verbose]
+
+      check_github_response { github_fetch_tags }
+    end
+
+    # Returns the number of pages for a API call
+    #
+    # @return [Integer] number of pages for this API call in total
+    def calculate_pages(client, method, request_options)
+      # Makes the first API call so that we can call last_response
+      check_github_response do
+        client.send(method, user_project, @request_options.merge(request_options))
+      end
+
+      last_response = client.last_response
+
+      if (last_pg = last_response.rels[:last])
+        querystring_as_hash(last_pg.href)["page"].to_i
+      else
+        1
+      end
+    end
+
+    # Fill input array with tags
+    #
+    # @return [Array <Hash>] array of tags in repo
+    def github_fetch_tags
+      tags        = []
+      page_i      = 0
+      count_pages = calculate_pages(@client, "tags", {})
+
+      iterate_pages(@client, "tags", {}) do |new_tags|
+        page_i += PER_PAGE_NUMBER
+        print_in_same_line("Fetching tags... #{page_i}/#{count_pages * PER_PAGE_NUMBER}")
+        tags.concat(new_tags)
+      end
+      print_empty_line
+
+      if tags.count == 0
+        Helper.log.warn "Warning: Can't find any tags in repo.\
+Make sure, that you push tags to remote repo via 'git push --tags'"
+      else
+        Helper.log.info "Found #{tags.count} tags"
+      end
+      # tags are a Sawyer::Resource. Convert to hash
+      tags = tags.map { |h| stringify_keys_deep(h.to_hash) }
+      tags
+    end
+
+    # This method fetch all closed issues and separate them to pull requests and pure issues
+    # (pull request is kind of issue in term of GitHub)
+    #
+    # @return [Tuple] with (issues [Array <Hash>], pull-requests [Array <Hash>])
+    def fetch_closed_issues_and_pr
+      print "Fetching closed issues...\r" if @options[:verbose]
+      issues = []
+      options = {
+        state: "closed",
+        filter: "all",
+        labels: nil
+      }
+      options[:since] = @since unless @since.nil?
+
+      page_i      = 0
+      count_pages = calculate_pages(@client, "issues", options)
+
+      iterate_pages(@client, "issues", options) do |new_issues|
+        page_i += PER_PAGE_NUMBER
+        print_in_same_line("Fetching issues... #{page_i}/#{count_pages * PER_PAGE_NUMBER}")
+        issues.concat(new_issues)
+        break if @options[:max_issues] && issues.length >= @options[:max_issues]
+      end
+      print_empty_line
+      Helper.log.info "Received issues: #{issues.count}"
+
+      issues = issues.map { |h| stringify_keys_deep(h.to_hash) }
+
+      # separate arrays of issues and pull requests:
+      issues.partition do |x|
+        x["pull_request"].nil?
+      end
+    end
+
+    # Fetch all pull requests. We need them to detect :merged_at parameter
+    #
+    # @return [Array <Hash>] all pull requests
+    def fetch_closed_pull_requests
+      pull_requests = []
+      options = { state: "closed" }
+
+      unless @options[:release_branch].nil?
+        options[:base] = @options[:release_branch]
+      end
+
+      page_i      = 0
+      count_pages = calculate_pages(@client, "pull_requests", options)
+
+      iterate_pages(@client, "pull_requests", options) do |new_pr|
+        page_i += PER_PAGE_NUMBER
+        log_string = "Fetching merged dates... #{page_i}/#{count_pages * PER_PAGE_NUMBER}"
+        print_in_same_line(log_string)
+        pull_requests.concat(new_pr)
+      end
+      print_empty_line
+
+      Helper.log.info "Pull Request count: #{pull_requests.count}"
+      pull_requests = pull_requests.map { |h| stringify_keys_deep(h.to_hash) }
+      pull_requests
+    end
+
+    # Fetch event for all issues and add them to 'events'
+    #
+    # @param [Array] issues
+    # @return [Void]
+    def fetch_events_async(issues)
+      i       = 0
+      threads = []
+
+      issues.each_slice(MAX_THREAD_NUMBER) do |issues_slice|
+        issues_slice.each do |issue|
+          threads << Thread.new do
+            issue["events"] = []
+            iterate_pages(@client, "issue_events", issue["number"], {}) do |new_event|
+              issue["events"].concat(new_event)
+            end
+            issue["events"] = issue["events"].map { |h| stringify_keys_deep(h.to_hash) }
+            print_in_same_line("Fetching events for issues and PR: #{i + 1}/#{issues.count}")
+            i += 1
+          end
+        end
+        threads.each(&:join)
+        threads = []
+      end
+
+      # to clear line from prev print
+      print_empty_line
+
+      Helper.log.info "Fetching events for issues and PR: #{i}"
+    end
+
+    # Fetch tag time from repo
+    #
+    # @param [Hash] tag GitHub data item about a Tag
+    #
+    # @return [Time] time of specified tag
+    def fetch_date_of_tag(tag)
+      commit_data = check_github_response { @client.commit(user_project, tag["commit"]["sha"]) }
+      commit_data = stringify_keys_deep(commit_data.to_hash)
+
+      commit_data["commit"]["committer"]["date"]
+    end
+
+    # Fetch commit for specified event
+    #
+    # @return [Hash]
+    def fetch_commit(event)
+      check_github_response do
+        commit = @client.commit(user_project, event["commit_id"])
+        commit = stringify_keys_deep(commit.to_hash)
+        commit
+      end
+    end
+
+    private
+
+    def stringify_keys_deep(indata)
+      case indata
+      when Array
+        indata.map do |value|
+          stringify_keys_deep(value)
+        end
+      when Hash
+        indata.each_with_object({}) do |(k, v), output|
+          output[k.to_s] = stringify_keys_deep(v)
+        end
+      else
+        indata
+      end
+    end
+
+    # Iterates through all pages until there are no more :next pages to follow
+    # yields the result per page
+    #
+    # @param [Octokit::Client] client
+    # @param [String] method (eg. 'tags')
+    # @return [Integer] total number of pages
+    def iterate_pages(client, method, *args)
+      if args.size == 1 && args.first.is_a?(Hash)
+        request_options = args.delete_at(0)
+      elsif args.size > 1 && args.last.is_a?(Hash)
+        request_options = args.delete_at(args.length - 1)
+      end
+
+      args.push(@request_options.merge(request_options))
+
+      pages = 1
+
+      check_github_response do
+        client.send(method, user_project, *args)
+      end
+      last_response = client.last_response
+
+      yield last_response.data
+
+      until (next_one = last_response.rels[:next]).nil?
+        pages += 1
+
+        last_response = check_github_response { next_one.get }
+        yield last_response.data
+      end
+
+      pages
+    end
+
+    # This is wrapper with rescue block
+    #
+    # @return [Object] returns exactly the same, what you put in the block, but wrap it with begin-rescue block
+    def check_github_response
+      Retriable.retriable(retry_options) do
+        yield
+      end
+
+    rescue Octokit::Forbidden => e
+      Helper.log.error("#{e.class}: #{e.message}")
+      sys_abort("Exceeded retry limit")
+    rescue Octokit::Unauthorized => e
+      Helper.log.error("#{e.class}: #{e.message}")
+      sys_abort("Error: wrong GitHub token")
+    end
+
+    # Exponential backoff
+    def retry_options
+      {
+        on: [Octokit::Forbidden],
+        tries: MAX_FORBIDDEN_RETRIES,
+        base_interval: sleep_base_interval,
+        multiplier: 1.0,
+        rand_factor: 0.0,
+        on_retry: retry_callback
+      }
+    end
+
+    def sleep_base_interval
+      1.0
+    end
+
+    def retry_callback
+      proc do |exception, try, elapsed_time, next_interval|
+        Helper.log.warn("RETRY - #{exception.class}: '#{exception.message}'")
+        Helper.log.warn("#{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try")
+        Helper.log.warn GH_RATE_LIMIT_EXCEEDED_MSG
+        Helper.log.warn @client.rate_limit
+      end
+    end
+
+    def sys_abort(msg)
+      abort(msg)
+    end
+
+    # Print specified line on the same string
+    #
+    # @param [String] log_string
+    def print_in_same_line(log_string)
+      print log_string + "\r"
+    end
+
+    # Print long line with spaces on same line to clear prev message
+    def print_empty_line
+      print_in_same_line("                                                                       ")
+    end
+
+    # Returns GitHub token. First try to use variable, provided by --token option,
+    # otherwise try to fetch it from CHANGELOG_GITHUB_TOKEN env variable.
+    #
+    # @return [String]
+    def fetch_github_token
+      env_var = @options[:token] ? @options[:token] : (ENV.fetch CHANGELOG_GITHUB_TOKEN, nil)
+
+      Helper.log.warn NO_TOKEN_PROVIDED unless env_var
+
+      env_var
+    end
+
+    # @return [String] helper to return Github "user/project"
+    def user_project
+      "#{@options[:user]}/#{@options[:project]}"
+    end
+
+    # Returns Hash of all querystring variables in given URI.
+    #
+    # @param [String] uri eg. https://api.github.com/repositories/43914960/tags?page=37&foo=1
+    # @return [Hash] of all GET variables. eg. { 'page' => 37, 'foo' => 1 }
+    def querystring_as_hash(uri)
+      Hash[URI.decode_www_form(URI(uri).query || "")]
+    end
+  end
+end