github_changelog_generator 1.15.0.pre.rc → 1.15.0

data/lib/github_changelog_generator/generator/generator.rb

@@ -1,20 +1,32 @@
 # frozen_string_literal: true
 
 require "github_changelog_generator/octo_fetcher"
-require "github_changelog_generator/generator/generator_generation"
 require "github_changelog_generator/generator/generator_fetcher"
 require "github_changelog_generator/generator/generator_processor"
 require "github_changelog_generator/generator/generator_tags"
+require "github_changelog_generator/generator/entry"
+require "github_changelog_generator/generator/section"
 
 module GitHubChangelogGenerator
   # Default error for ChangelogGenerator
   class ChangelogGeneratorError < StandardError
   end
 
+  # This class is the high-level code for gathering issues and PRs for a github
+  # repository and generating a CHANGELOG.md file. A changelog is made up of a
+  # series of "entries" of all tagged releases, plus an extra entry for the
+  # unreleased changes. Entries are made up of various organizational
+  # "sections," and sections contain the github issues and PRs.
+  #
+  # So the changelog contains entries, entries contain sections, and sections
+  # contain issues and PRs.
+  #
+  # @see GitHubChangelogGenerator::Entry
+  # @see GitHubChangelogGenerator::Section
   class Generator
-    attr_accessor :options, :filtered_tags, :github, :tag_section_mapping, :sorted_tags
+    attr_accessor :options, :filtered_tags, :tag_section_mapping, :sorted_tags
 
-    # A Generator responsible for all logic, related with change log generation from ready-to-parse issues
+    # A Generator responsible for all logic, related with changelog generation from ready-to-parse issues
     #
     # Example:
     #   generator = GitHubChangelogGenerator::Generator.new
@@ -23,156 +35,121 @@ module GitHubChangelogGenerator
       @options        = options
       @tag_times_hash = {}
       @fetcher        = GitHubChangelogGenerator::OctoFetcher.new(options)
+      @sections       = []
     end
 
-    def fetch_issues_and_pr
-      issues, pull_requests = @fetcher.fetch_closed_issues_and_pr
+    # Main function to start changelog generation
+    #
+    # @return [String] Generated changelog file
+    def compound_changelog
+      @options.load_custom_ruby_files
+      fetch_and_filter_tags
+      fetch_issues_and_pr
 
-      @pull_requests = options[:pulls] ? get_filtered_pull_requests(pull_requests) : []
+      log = ""
+      log += @options[:frontmatter] if @options[:frontmatter]
+      log += "#{options[:header]}\n\n"
 
-      @issues = options[:issues] ? get_filtered_issues(issues) : []
+      log += if @options[:unreleased_only]
+               generate_entry_between_tags(@filtered_tags[0], nil)
+             else
+               generate_entries_for_all_tags
+             end
 
-      fetch_events_for_issues_and_pr
-      detect_actual_closed_dates(@issues + @pull_requests)
+      log += File.read(@options[:base]) if File.file?(@options[:base])
+
+      credit_line = "\n\n\\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*"
+      log.gsub!(/#{credit_line}(\n)?/, "") # Remove old credit lines
+      log += "#{credit_line}\n"
+
+      @log = log
     end
 
-    ENCAPSULATED_CHARACTERS = %w(< > * _ \( \) [ ] #)
+    private
 
-    # Encapsulate characters to make Markdown look as expected.
-    #
-    # @param [String] string
-    # @return [String] encapsulated input string
-    def encapsulate_string(string)
-      string = string.gsub('\\', '\\\\')
+    # Generate log only between 2 specified tags
+    # @param [String] older_tag all issues before this tag date will be excluded. May be nil, if it's first tag
+    # @param [String] newer_tag all issue after this tag will be excluded. May be nil for unreleased section
+    def generate_entry_between_tags(older_tag, newer_tag)
+      filtered_issues, filtered_pull_requests = filter_issues_for_tags(newer_tag, older_tag)
 
-      ENCAPSULATED_CHARACTERS.each do |char|
-        string = string.gsub(char, "\\#{char}")
+      if newer_tag.nil? && filtered_issues.empty? && filtered_pull_requests.empty?
+        # do not generate empty unreleased section
+        return ""
       end
 
-      string
-    end
-
-    # Generates log for section with header and body
-    #
-    # @param [Array] pull_requests List or PR's in new section
-    # @param [Array] issues List of issues in new section
-    # @param [String] newer_tag Name of the newer tag. Could be nil for `Unreleased` section
-    # @param [Hash, nil] older_tag Older tag, used for the links. Could be nil for last tag.
-    # @return [String] Ready and parsed section
-    def create_log_for_tag(pull_requests, issues, newer_tag, older_tag = nil)
       newer_tag_link, newer_tag_name, newer_tag_time = detect_link_tag_time(newer_tag)
 
-      github_site = options[:github_site] || "https://github.com"
-      project_url = "#{github_site}/#{options[:user]}/#{options[:project]}"
-
       # If the older tag is nil, go back in time from the latest tag and find
       # the SHA for the first commit.
       older_tag_name =
         if older_tag.nil?
-          @fetcher.commits_before(newer_tag_time).last["sha"]
+          @fetcher.oldest_commit["sha"]
         else
           older_tag["name"]
         end
 
-      log = generate_header(newer_tag_name, newer_tag_link, newer_tag_time, older_tag_name, project_url)
+      Entry.new(options).generate_entry_for_tag(filtered_pull_requests, filtered_issues, newer_tag_name, newer_tag_link, newer_tag_time, older_tag_name)
+    end
+
+    # Filters issues and pull requests based on, respectively, `actual_date`
+    # and `merged_at` timestamp fields. `actual_date` is the detected form of
+    # `closed_at` based on merge event SHA commit times.
+    #
+    # @return [Array] filtered issues and pull requests
+    def filter_issues_for_tags(newer_tag, older_tag)
+      filtered_pull_requests = filter_by_tag(@pull_requests, newer_tag)
+      filtered_issues        = delete_by_time(@issues, "actual_date", older_tag, newer_tag)
+
+      newer_tag_name = newer_tag.nil? ? nil : newer_tag["name"]
 
-      if options[:issues]
-        # Generate issues:
-        log += issues_to_log(issues, pull_requests)
+      if options[:filter_issues_by_milestone]
+        # delete excess irrelevant issues (according milestones). Issue #22.
+        filtered_issues = filter_by_milestone(filtered_issues, newer_tag_name, @issues)
+        filtered_pull_requests = filter_by_milestone(filtered_pull_requests, newer_tag_name, @pull_requests)
       end
+      [filtered_issues, filtered_pull_requests]
+    end
+
+    # The full cycle of generation for whole project
+    # @return [String] All entries in the changelog
+    def generate_entries_for_all_tags
+      puts "Generating entry..." if options[:verbose]
+
+      entries = generate_unreleased_entry
 
-      if options[:pulls] && options[:add_pr_wo_labels]
-        # Generate pull requests:
-        log += generate_sub_section(pull_requests, options[:merge_prefix])
+      @tag_section_mapping.each_pair do |_tag_section, left_right_tags|
+        older_tag, newer_tag = left_right_tags
+        entries += generate_entry_between_tags(older_tag, newer_tag)
       end
 
-      log
+      entries
     end
 
-    # Generate ready-to-paste log from list of issues and pull requests.
-    #
-    # @param [Array] issues
-    # @param [Array] pull_requests
-    # @return [String] generated log for issues
-    def issues_to_log(issues, pull_requests)
-      sections = parse_by_sections(issues, pull_requests)
-
-      log = ""
-      log += generate_sub_section(sections[:breaking], options[:breaking_prefix])
-      log += generate_sub_section(sections[:enhancements], options[:enhancement_prefix])
-      log += generate_sub_section(sections[:bugs], options[:bug_prefix])
-      log += generate_sub_section(sections[:issues], options[:issue_prefix])
-      log
+    def generate_unreleased_entry
+      entry = ""
+      if options[:unreleased]
+        start_tag        = @filtered_tags[0] || @sorted_tags.last
+        unreleased_entry = generate_entry_between_tags(start_tag, nil)
+        entry           += unreleased_entry if unreleased_entry
+      end
+      entry
     end
 
-    # This method sort issues by types
-    # (bugs, features, or just closed issues) by labels
+    # Fetches @pull_requests and @issues and filters them based on options.
     #
-    # @param [Array] issues
-    # @param [Array] pull_requests
-    # @return [Hash] Mapping of filtered arrays: (Bugs, Enhancements, Breaking stuff, Issues)
-    def parse_by_sections(issues, pull_requests)
-      sections = {
-        issues: [],
-        enhancements: [],
-        bugs: [],
-        breaking: []
-      }
-
-      issues.each do |dict|
-        added = false
-
-        dict["labels"].each do |label|
-          if options[:bug_labels].include?(label["name"])
-            sections[:bugs] << dict
-            added = true
-          elsif options[:enhancement_labels].include?(label["name"])
-            sections[:enhancements] << dict
-            added = true
-          elsif options[:breaking_labels].include?(label["name"])
-            sections[:breaking] << dict
-            added = true
-          end
-
-          break if added
-        end
+    # @return [Nil] No return.
+    def fetch_issues_and_pr
+      issues, pull_requests = @fetcher.fetch_closed_issues_and_pr
 
-        sections[:issues] << dict unless added
-      end
+      @pull_requests = options[:pulls] ? get_filtered_pull_requests(pull_requests) : []
 
-      sort_pull_requests(pull_requests, sections)
-    end
+      @issues = options[:issues] ? get_filtered_issues(issues) : []
 
-    # This method iterates through PRs and sorts them into sections
-    #
-    # @param [Array] pull_requests
-    # @param [Hash] sections
-    # @return [Hash] sections
-    def sort_pull_requests(pull_requests, sections)
-      added_pull_requests = []
-      pull_requests.each do |pr|
-        added = false
-
-        pr["labels"].each do |label|
-          if options[:bug_labels].include?(label["name"])
-            sections[:bugs] << pr
-            added_pull_requests << pr
-            added = true
-          elsif options[:enhancement_labels].include?(label["name"])
-            sections[:enhancements] << pr
-            added_pull_requests << pr
-            added = true
-          elsif options[:breaking_labels].include?(label["name"])
-            sections[:breaking] << pr
-            added_pull_requests << pr
-            added = true
-          end
-
-          break if added
-        end
-      end
-      added_pull_requests.each { |p| pull_requests.delete(p) }
-      sections
+      fetch_events_for_issues_and_pr
+      detect_actual_closed_dates(@issues + @pull_requests)
+      add_first_occurring_tag_to_prs(@sorted_tags, @pull_requests)
+      nil
     end
   end
 end

data/lib/github_changelog_generator/generator/generator_fetcher.rb

@@ -7,9 +7,7 @@ module GitHubChangelogGenerator
     # Fetch event for issues and pull requests
     # @return [Array] array of fetched issues
     def fetch_events_for_issues_and_pr
-      if options[:verbose]
-        print "Fetching events for issues and PR: 0/#{@issues.count + @pull_requests.count}\r"
-      end
+      print "Fetching events for issues and PR: 0/#{@issues.count + @pull_requests.count}\r" if options[:verbose]
 
       # Async fetching events:
       @fetcher.fetch_events_async(@issues + @pull_requests)
@@ -18,34 +16,141 @@ module GitHubChangelogGenerator
     # Async fetching of all tags dates
     def fetch_tags_dates(tags)
       print "Fetching tag dates...\r" if options[:verbose]
-      # Async fetching tags:
-      threads = []
       i = 0
-      all = tags.count
       tags.each do |tag|
-        print "                                 \r"
-        threads << Thread.new do
-          get_time_of_tag(tag)
-          print "Fetching tags dates: #{i + 1}/#{all}\r" if options[:verbose]
-          i += 1
-        end
+        get_time_of_tag(tag)
+        i += 1
       end
-      threads.each(&:join)
-      puts "Fetching tags dates: #{i}" if options[:verbose]
+      puts "Fetching tags dates: #{i}/#{tags.count}" if options[:verbose]
     end
 
     # Find correct closed dates, if issues was closed by commits
     def detect_actual_closed_dates(issues)
       print "Fetching closed dates for issues...\r" if options[:verbose]
 
-      issues.each_slice(MAX_THREAD_NUMBER) do |issues_slice|
-        threads = []
-        issues_slice.each do |issue|
-          threads << Thread.new { find_closed_date_by_commit(issue) }
+      i = 0
+      issues.each do |issue|
+        find_closed_date_by_commit(issue)
+        i += 1
+      end
+      puts "Fetching closed dates for issues: #{i}/#{issues.count}" if options[:verbose]
+    end
+
+    # Adds a key "first_occurring_tag" to each PR with a value of the oldest
+    # tag that a PR's merge commit occurs in in the git history. This should
+    # indicate the release of each PR by git's history regardless of dates and
+    # divergent branches.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs The PRs to discover the tags of.
+    # @return [Nil] No return; PRs are updated in-place.
+    def add_first_occurring_tag_to_prs(tags, prs)
+      total = prs.count
+
+      prs_left = associate_tagged_prs(tags, prs, total)
+      prs_left = associate_release_branch_prs(prs_left, total)
+      prs_left = associate_rebase_comment_prs(tags, prs_left, total) if prs_left.any?
+      # PRs in prs_left will be untagged, not in release branch, and not
+      # rebased. They should not be included in the changelog as they probably
+      # have been merged to a branch other than the release branch.
+      @pull_requests -= prs_left
+      Helper.log.info "Associating PRs with tags: #{total}/#{total}"
+    end
+
+    # Associate merged PRs by the merge SHA contained in each tag. If the
+    # merge_commit_sha is not found in any tag's history, skip association.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs The PRs to associate.
+    # @return [Array] PRs without their merge_commit_sha in a tag.
+    def associate_tagged_prs(tags, prs, total)
+      @fetcher.fetch_tag_shas_async(tags)
+
+      i = 0
+      prs.reject do |pr|
+        found = false
+        # XXX Wish I could use merge_commit_sha, but gcg doesn't currently
+        # fetch that. See
+        # https://developer.github.com/v3/pulls/#get-a-single-pull-request vs.
+        # https://developer.github.com/v3/pulls/#list-pull-requests
+        if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" })
+          # Iterate tags.reverse (oldest to newest) to find first tag of each PR.
+          if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(event["commit_id"]) })
+            pr["first_occurring_tag"] = oldest_tag["name"]
+            found = true
+            i += 1
+            print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+          end
+        else
+          # Either there were no events or no merged event. Github's api can be
+          # weird like that apparently. Check for a rebased comment before erroring.
+          no_events_pr = associate_rebase_comment_prs(tags, [pr], total)
+          raise StandardError, "No merge sha found for PR #{pr['number']} via the GitHub API" unless no_events_pr.empty?
+
+          found = true
+          i += 1
+          print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+        end
+        found
+      end
+    end
+
+    # Associate merged PRs by the HEAD of the release branch. If no
+    # --release-branch was specified, then the github default branch is used.
+    #
+    # @param [Array] prs_left PRs not associated with any tag.
+    # @param [Integer] total The total number of PRs to associate; used for verbose printing.
+    # @return [Array] PRs without their merge_commit_sha in the branch.
+    def associate_release_branch_prs(prs_left, total)
+      if prs_left.any?
+        i = total - prs_left.count
+        prs_left.reject do |pr|
+          found = false
+          if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" }) && sha_in_release_branch(event["commit_id"])
+            found = true
+            i += 1
+            print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+          end
+          found
         end
-        threads.each(&:join)
+      else
+        prs_left
+      end
+    end
+
+    # Associate merged PRs by the SHA detected in github comments of the form
+    # "rebased commit: <sha>". For use when the merge_commit_sha is not in the
+    # actual git history due to rebase.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs_left The PRs not yet associated with any tag or branch.
+    # @return [Array] PRs without rebase comments.
+    def associate_rebase_comment_prs(tags, prs_left, total)
+      i = total - prs_left.count
+      # Any remaining PRs were not found in the list of tags by their merge
+      # commit and not found in any specified release branch. Fallback to
+      # rebased commit comment.
+      @fetcher.fetch_comments_async(prs_left)
+      prs_left.reject do |pr|
+        found = false
+        if pr["comments"] && (rebased_comment = pr["comments"].reverse.find { |c| c["body"].match(%r{rebased commit: ([0-9a-f]{40})}i) })
+          rebased_sha = rebased_comment["body"].match(%r{rebased commit: ([0-9a-f]{40})}i)[1]
+          if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(rebased_sha) })
+            pr["first_occurring_tag"] = oldest_tag["name"]
+            found = true
+            i += 1
+          elsif sha_in_release_branch(rebased_sha)
+            found = true
+            i += 1
+          else
+            raise StandardError, "PR #{pr['number']} has a rebased SHA comment but that SHA was not found in the release branch or any tags"
+          end
+          print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+        else
+          puts "Warning: PR #{pr['number']} merge commit was not found in the release branch or tagged git history and no rebased SHA comment was found"
+        end
+        found
       end
-      puts "Fetching closed dates for issues: Done!" if options[:verbose]
     end
 
     # Fill :actual_date parameter of specified issue by closed date of the commit, if it was closed by commit.
@@ -74,7 +179,7 @@ module GitHubChangelogGenerator
         issue["actual_date"] = issue["closed_at"]
       else
         begin
-          commit = @fetcher.fetch_commit(event)
+          commit = @fetcher.fetch_commit(event["commit_id"])
           issue["actual_date"] = commit["commit"]["author"]["date"]
 
           # issue['actual_date'] = commit['author']['date']
@@ -84,5 +189,19 @@ module GitHubChangelogGenerator
         end
       end
     end
+
+    private
+
+    # Detect if a sha occurs in the --release-branch. Uses the github repo
+    # default branch if not specified.
+    #
+    # @param [String] sha SHA to check.
+    # @return [Boolean] True if SHA is in the branch git history.
+    def sha_in_release_branch(sha)
+      branch = @options[:release_branch] || @fetcher.default_branch
+      commits_in_branch = @fetcher.fetch_compare(@fetcher.oldest_commit["sha"], branch)
+      shas_in_branch = commits_in_branch["commits"].collect { |commit| commit["sha"] }
+      shas_in_branch.include?(sha)
+    end
   end
 end