github_changelog_generator 1.15.0.pre.alpha → 1.16.0

data/lib/github_changelog_generator/generator/generator_fetcher.rb

@@ -2,14 +2,10 @@
 
 module GitHubChangelogGenerator
   class Generator
-    MAX_THREAD_NUMBER = 25
-
     # Fetch event for issues and pull requests
     # @return [Array] array of fetched issues
     def fetch_events_for_issues_and_pr
-      if options[:verbose]
-        print "Fetching events for issues and PR: 0/#{@issues.count + @pull_requests.count}\r"
-      end
+      print "Fetching events for issues and PR: 0/#{@issues.count + @pull_requests.count}\r" if options[:verbose]
 
       # Async fetching events:
       @fetcher.fetch_events_async(@issues + @pull_requests)
@@ -18,34 +14,141 @@ module GitHubChangelogGenerator
     # Async fetching of all tags dates
     def fetch_tags_dates(tags)
       print "Fetching tag dates...\r" if options[:verbose]
-      # Async fetching tags:
-      threads = []
       i = 0
-      all = tags.count
       tags.each do |tag|
-        print "                                 \r"
-        threads << Thread.new do
-          get_time_of_tag(tag)
-          print "Fetching tags dates: #{i + 1}/#{all}\r" if options[:verbose]
-          i += 1
-        end
+        get_time_of_tag(tag)
+        i += 1
       end
-      threads.each(&:join)
-      puts "Fetching tags dates: #{i}" if options[:verbose]
+      puts "Fetching tags dates: #{i}/#{tags.count}" if options[:verbose]
     end
 
     # Find correct closed dates, if issues was closed by commits
     def detect_actual_closed_dates(issues)
       print "Fetching closed dates for issues...\r" if options[:verbose]
 
-      issues.each_slice(MAX_THREAD_NUMBER) do |issues_slice|
-        threads = []
-        issues_slice.each do |issue|
-          threads << Thread.new { find_closed_date_by_commit(issue) }
+      i = 0
+      issues.each do |issue|
+        find_closed_date_by_commit(issue)
+        i += 1
+      end
+      puts "Fetching closed dates for issues: #{i}/#{issues.count}" if options[:verbose]
+    end
+
+    # Adds a key "first_occurring_tag" to each PR with a value of the oldest
+    # tag that a PR's merge commit occurs in in the git history. This should
+    # indicate the release of each PR by git's history regardless of dates and
+    # divergent branches.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs The PRs to discover the tags of.
+    # @return [Nil] No return; PRs are updated in-place.
+    def add_first_occurring_tag_to_prs(tags, prs)
+      total = prs.count
+
+      prs_left = associate_tagged_prs(tags, prs, total)
+      prs_left = associate_release_branch_prs(prs_left, total)
+      prs_left = associate_rebase_comment_prs(tags, prs_left, total) if prs_left.any?
+      # PRs in prs_left will be untagged, not in release branch, and not
+      # rebased. They should not be included in the changelog as they probably
+      # have been merged to a branch other than the release branch.
+      @pull_requests -= prs_left
+      Helper.log.info "Associating PRs with tags: #{total}/#{total}"
+    end
+
+    # Associate merged PRs by the merge SHA contained in each tag. If the
+    # merge_commit_sha is not found in any tag's history, skip association.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs The PRs to associate.
+    # @return [Array] PRs without their merge_commit_sha in a tag.
+    def associate_tagged_prs(tags, prs, total)
+      @fetcher.fetch_tag_shas(tags)
+
+      i = 0
+      prs.reject do |pr|
+        found = false
+        # XXX Wish I could use merge_commit_sha, but gcg doesn't currently
+        # fetch that. See
+        # https://developer.github.com/v3/pulls/#get-a-single-pull-request vs.
+        # https://developer.github.com/v3/pulls/#list-pull-requests
+        if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" })
+          # Iterate tags.reverse (oldest to newest) to find first tag of each PR.
+          if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(event["commit_id"]) })
+            pr["first_occurring_tag"] = oldest_tag["name"]
+            found = true
+            i += 1
+            print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+          end
+        else
+          # Either there were no events or no merged event. Github's api can be
+          # weird like that apparently. Check for a rebased comment before erroring.
+          no_events_pr = associate_rebase_comment_prs(tags, [pr], total)
+          raise StandardError, "No merge sha found for PR #{pr['number']} via the GitHub API" unless no_events_pr.empty?
+
+          found = true
+          i += 1
+          print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
         end
-        threads.each(&:join)
+        found
+      end
+    end
+
+    # Associate merged PRs by the HEAD of the release branch. If no
+    # --release-branch was specified, then the github default branch is used.
+    #
+    # @param [Array] prs_left PRs not associated with any tag.
+    # @param [Integer] total The total number of PRs to associate; used for verbose printing.
+    # @return [Array] PRs without their merge_commit_sha in the branch.
+    def associate_release_branch_prs(prs_left, total)
+      if prs_left.any?
+        i = total - prs_left.count
+        prs_left.reject do |pr|
+          found = false
+          if pr["events"] && (event = pr["events"].find { |e| e["event"] == "merged" }) && sha_in_release_branch(event["commit_id"])
+            found = true
+            i += 1
+            print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+          end
+          found
+        end
+      else
+        prs_left
+      end
+    end
+
+    # Associate merged PRs by the SHA detected in github comments of the form
+    # "rebased commit: <sha>". For use when the merge_commit_sha is not in the
+    # actual git history due to rebase.
+    #
+    # @param [Array] tags The tags sorted by time, newest to oldest.
+    # @param [Array] prs_left The PRs not yet associated with any tag or branch.
+    # @return [Array] PRs without rebase comments.
+    def associate_rebase_comment_prs(tags, prs_left, total)
+      i = total - prs_left.count
+      # Any remaining PRs were not found in the list of tags by their merge
+      # commit and not found in any specified release branch. Fallback to
+      # rebased commit comment.
+      @fetcher.fetch_comments_async(prs_left)
+      prs_left.reject do |pr|
+        found = false
+        if pr["comments"] && (rebased_comment = pr["comments"].reverse.find { |c| c["body"].match(%r{rebased commit: ([0-9a-f]{40})}i) })
+          rebased_sha = rebased_comment["body"].match(%r{rebased commit: ([0-9a-f]{40})}i)[1]
+          if (oldest_tag = tags.reverse.find { |tag| tag["shas_in_tag"].include?(rebased_sha) })
+            pr["first_occurring_tag"] = oldest_tag["name"]
+            found = true
+            i += 1
+          elsif sha_in_release_branch(rebased_sha)
+            found = true
+            i += 1
+          else
+            raise StandardError, "PR #{pr['number']} has a rebased SHA comment but that SHA was not found in the release branch or any tags"
+          end
+          print("Associating PRs with tags: #{i}/#{total}\r") if @options[:verbose]
+        else
+          puts "Warning: PR #{pr['number']} merge commit was not found in the release branch or tagged git history and no rebased SHA comment was found"
+        end
+        found
       end
-      puts "Fetching closed dates for issues: Done!" if options[:verbose]
     end
 
     # Fill :actual_date parameter of specified issue by closed date of the commit, if it was closed by commit.
@@ -74,7 +177,7 @@ module GitHubChangelogGenerator
         issue["actual_date"] = issue["closed_at"]
       else
         begin
-          commit = @fetcher.fetch_commit(event)
+          commit = @fetcher.fetch_commit(event["commit_id"])
           issue["actual_date"] = commit["commit"]["author"]["date"]
 
           # issue['actual_date'] = commit['author']['date']
@@ -84,5 +187,18 @@ module GitHubChangelogGenerator
         end
       end
     end
+
+    private
+
+    # Detect if a sha occurs in the --release-branch. Uses the github repo
+    # default branch if not specified.
+    #
+    # @param [String] sha SHA to check.
+    # @return [Boolean] True if SHA is in the branch git history.
+    def sha_in_release_branch(sha)
+      branch = @options[:release_branch] || @fetcher.default_branch
+      shas_in_branch = @fetcher.commits_in_branch(branch)
+      shas_in_branch.include?(sha)
+    end
   end
 end

data/lib/github_changelog_generator/generator/generator_processor.rb

@@ -2,7 +2,7 @@
 
 module GitHubChangelogGenerator
   class Generator
-    # delete all labels with labels from options[:exclude_labels] array
+    # delete all issues with labels from options[:exclude_labels] array
     # @param [Array] issues
     # @return [Array] filtered array
     def exclude_issues_by_labels(issues)
@@ -14,6 +14,19 @@ module GitHubChangelogGenerator
       end
     end
 
+    # Only include issues without labels if options[:add_issues_wo_labels]
+    # @param [Array] issues
+    # @return [Array] filtered array
+    def exclude_issues_without_labels(issues)
+      return issues if issues.empty?
+      return issues if issues.first.key?("pull_request") && options[:add_pr_wo_labels]
+      return issues if !issues.first.key?("pull_request") && options[:add_issues_wo_labels]
+
+      issues.reject do |issue|
+        issue["labels"].empty?
+      end
+    end
+
     # @return [Array] filtered issues accourding milestone
     def filter_by_milestone(filtered_issues, tag_name, all_issues)
       remove_issues_in_milestones(filtered_issues)
@@ -56,6 +69,9 @@ module GitHubChangelogGenerator
         # leave issues without milestones
         if issue["milestone"].nil?
           true
+        # remove issues of open milestones if option is set
+        elsif issue["milestone"]["state"] == "open"
+          @options[:issues_of_open_milestones]
         else
           # check, that this milestone in tag list:
           @filtered_tags.find { |tag| tag["name"] == issue["milestone"]["title"] }.nil?
@@ -63,11 +79,22 @@ module GitHubChangelogGenerator
       end
     end
 
+    # Method filter issues, that belong only specified tag range
+    #
+    # @param [Array] issues issues to filter
+    # @param [Hash, Nil] newer_tag Tag to find PRs of. May be nil for unreleased section
+    # @return [Array] filtered issues
+    def filter_by_tag(issues, newer_tag = nil)
+      issues.select do |issue|
+        issue["first_occurring_tag"] == (newer_tag.nil? ? nil : newer_tag["name"])
+      end
+    end
+
     # Method filter issues, that belong only specified tag range
     # @param [Array] issues issues to filter
     # @param [Symbol] hash_key key of date value default is :actual_date
-    # @param [String] older_tag all issues before this tag date will be excluded. May be nil, if it's first tag
-    # @param [String] newer_tag all issue after this tag will be excluded. May be nil for unreleased section
+    # @param [Hash, Nil] older_tag all issues before this tag date will be excluded. May be nil, if it's first tag
+    # @param [Hash, Nil] newer_tag all issue after this tag will be excluded. May be nil for unreleased section
     # @return [Array] filtered issues
     def delete_by_time(issues, hash_key = "actual_date", older_tag = nil, newer_tag = nil)
       # in case if not tags specified - return unchanged array
@@ -97,28 +124,28 @@ module GitHubChangelogGenerator
 
     def ensure_older_tag(older_tag, newer_tag)
       return older_tag if older_tag
+
       idx = sorted_tags.index { |t| t["name"] == newer_tag["name"] }
       # skip if we are already at the oldest element
       return if idx == sorted_tags.size - 1
+
       sorted_tags[idx - 1]
     end
 
     def tag_older_new_tag?(newer_tag_time, time)
-      tag_in_range_new = if newer_tag_time.nil?
-                           true
-                         else
-                           time <= newer_tag_time
-                         end
-      tag_in_range_new
+      if newer_tag_time.nil?
+        true
+      else
+        time <= newer_tag_time
+      end
     end
 
-    def tag_newer_old_tag?(older_tag_time, t)
-      tag_in_range_old = if older_tag_time.nil?
-                           true
-                         else
-                           t > older_tag_time
-                         end
-      tag_in_range_old
+    def tag_newer_old_tag?(older_tag_time, time)
+      if older_tag_time.nil?
+        true
+      else
+        time > older_tag_time
+      end
     end
 
     # Include issues with labels, specified in :include_labels
@@ -126,37 +153,42 @@ module GitHubChangelogGenerator
     # @return [Array] filtered array of issues
     def include_issues_by_labels(issues)
       filtered_issues = filter_by_include_labels(issues)
-      filtered_issues = filter_wo_labels(filtered_issues)
-      filtered_issues
+      filter_wo_labels(filtered_issues)
     end
 
-    # @return [Array] issues without labels or empty array if add_issues_wo_labels is false
-    def filter_wo_labels(issues)
-      if options[:add_issues_wo_labels]
-        issues
-      else
-        issues.select { |issue| issue["labels"].map { |l| l["name"] }.any? }
+    # @param [Array] issues Issues & PRs to filter when without labels
+    # @return [Array] Issues & PRs without labels or empty array if
+    #                 add_issues_wo_labels or add_pr_wo_labels are false
+    def filter_wo_labels(items)
+      if items.any? && items.first.key?("pull_request")
+        return items if options[:add_pr_wo_labels]
+      elsif options[:add_issues_wo_labels]
+        return items
       end
+      # The default is to filter items without labels
+      items.select { |item| item["labels"].map { |l| l["name"] }.any? }
     end
 
+    # @todo Document this
     def filter_by_include_labels(issues)
       if options[:include_labels].nil?
         issues
       else
         issues.select do |issue|
           labels = issue["labels"].map { |l| l["name"] } & options[:include_labels]
-          labels.any?
+          labels.any? || issue["labels"].empty?
         end
       end
     end
 
     # General filtered function
     #
-    # @param [Array] all_issues
+    # @param [Array] all_issues PRs or issues
     # @return [Array] filtered issues
     def filter_array_by_labels(all_issues)
       filtered_issues = include_issues_by_labels(all_issues)
-      exclude_issues_by_labels(filtered_issues)
+      filtered_issues = exclude_issues_by_labels(filtered_issues)
+      exclude_issues_without_labels(filtered_issues)
     end
 
     # Filter issues according labels

data/lib/github_changelog_generator/generator/generator_tags.rb

@@ -4,22 +4,17 @@ module GitHubChangelogGenerator
   class Generator
     # fetch, filter tags, fetch dates and sort them in time order
     def fetch_and_filter_tags
-      detect_since_tag
-      detect_due_tag
+      since_tag
+      due_tag
 
       all_tags = @fetcher.get_all_tags
       fetch_tags_dates(all_tags) # Creates a Hash @tag_times_hash
       all_sorted_tags = sort_tags_by_date(all_tags)
 
-      @sorted_tags   = filter_excluded_tags(all_sorted_tags)
+      @sorted_tags   = filter_included_tags(all_sorted_tags)
+      @sorted_tags   = filter_excluded_tags(@sorted_tags)
       @filtered_tags = get_filtered_tags(@sorted_tags)
-
-      # Because we need to properly create compare links, we need a sorted list
-      # of all filtered tags (including the excluded ones). We'll exclude those
-      # tags from section headers inside the mapping function.
-      section_tags = get_filtered_tags(all_sorted_tags)
-
-      @tag_section_mapping = build_tag_section_mapping(section_tags, @filtered_tags)
+      @tag_section_mapping = build_tag_section_mapping(@filtered_tags, @filtered_tags)
 
       @filtered_tags
     end
@@ -35,7 +30,7 @@ module GitHubChangelogGenerator
         tag = section_tags[i]
 
         # Don't create section header for the "since" tag
-        next if @since_tag && tag["name"] == @since_tag
+        next if since_tag && tag["name"] == since_tag
 
         # Don't create a section header for the first tag in between_tags
         next if options[:between_tags] && tag == section_tags.last
@@ -83,7 +78,7 @@ module GitHubChangelogGenerator
     # @return [Array] link, name and time of the tag
     def detect_link_tag_time(newer_tag)
       # if tag is nil - set current time
-      newer_tag_time = newer_tag.nil? ? Time.new : get_time_of_tag(newer_tag)
+      newer_tag_time = newer_tag.nil? ? Time.new.getutc : get_time_of_tag(newer_tag)
 
       # if it's future release tag - set this value
       if newer_tag.nil? && options[:future_release]
@@ -98,11 +93,11 @@ module GitHubChangelogGenerator
     end
 
     # @return [Object] try to find newest tag using #Reader and :base option if specified otherwise returns nil
-    def detect_since_tag
+    def since_tag
       @since_tag ||= options.fetch(:since_tag) { version_of_first_item }
     end
 
-    def detect_due_tag
+    def due_tag
       @due_tag ||= options.fetch(:due_tag, nil)
     end
 
@@ -125,17 +120,17 @@ module GitHubChangelogGenerator
     # @return [Array] filtered tags according :since_tag option
     def filter_since_tag(all_tags)
       filtered_tags = all_tags
-      tag = detect_since_tag
+      tag = since_tag
       if tag
         if all_tags.map { |t| t["name"] }.include? tag
           idx = all_tags.index { |t| t["name"] == tag }
-          filtered_tags = if idx > 0
+          filtered_tags = if idx
                             all_tags[0..idx]
                           else
                             []
                           end
         else
-          Helper.log.warn "Warning: can't find tag #{tag}, specified with --since-tag option."
+          raise ChangelogGeneratorError, "Error: can't find tag #{tag}, specified with --since-tag option."
         end
       end
       filtered_tags
@@ -145,7 +140,7 @@ module GitHubChangelogGenerator
     # @return [Array] filtered tags according :due_tag option
     def filter_due_tag(all_tags)
       filtered_tags = all_tags
-      tag           = detect_due_tag
+      tag           = due_tag
       if tag
         if all_tags.any? && all_tags.map { |t| t["name"] }.include?(tag)
           idx = all_tags.index { |t| t["name"] == tag }
@@ -155,12 +150,23 @@ module GitHubChangelogGenerator
                             []
                           end
         else
-          Helper.log.warn "Warning: can't find tag #{tag}, specified with --due-tag option."
+          raise ChangelogGeneratorError, "Error: can't find tag #{tag}, specified with --due-tag option."
         end
       end
       filtered_tags
     end
 
+    # @param [Array] all_tags all tags
+    # @return [Array] filtered tags according to :include_tags_regex option
+    def filter_included_tags(all_tags)
+      if options[:include_tags_regex]
+        regex = Regexp.new(options[:include_tags_regex])
+        all_tags.select { |tag| regex =~ tag["name"] }
+      else
+        all_tags
+      end
+    end
+
     # @param [Array] all_tags all tags
     # @return [Array] filtered tags according :exclude_tags or :exclude_tags_regex option
     def filter_excluded_tags(all_tags)
@@ -208,9 +214,7 @@ module GitHubChangelogGenerator
     end
 
     def warn_if_tag_not_found(all_tags, tag)
-      unless all_tags.map { |t| t["name"] }.include?(tag)
-        Helper.log.warn "Warning: can't find tag #{tag}, specified with --exclude-tags option."
-      end
+      Helper.log.warn("Warning: can't find tag #{tag}, specified with --exclude-tags option.") unless all_tags.map { |t| t["name"] }.include?(tag)
     end
   end
 end