zz-export-pull-requests 0.3.6 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1bcbc5858a831bb7f50953a659eea6800e843aad238205d1f1bdd4ca45e2deb7
-  data.tar.gz: 93469b0eaa9bf4791779906e2dc6aecd4aa29d5956769ee5181430b9ee6014a6
+  metadata.gz: ec23970f15be8703f88796fd826581b0cb275913be1cbe31620659da0de5e580
+  data.tar.gz: 0caf623b5619c463baa10d45cdd3fedb61912058b980bbd3f77d2bf845e3f09e
 SHA512:
-  metadata.gz: 5e37eff10c88699a30e532c9b39d807c1187c6474922055b673d307b58543752652ddf6cd6d30dac1dcda0f56f332296b79651921b41aa79250290c8311150fd
-  data.tar.gz: 99b821c8ca3c1d5812ed8c207d1ec0707565a380dcfa6cd53870aac3feb5ec10535f528a19ed5f8efeb819208229ce58872c678fb96ffa7d9cbd3aca959a5283
+  metadata.gz: 88f774bc035801a5def78caaf335f0a26be167544c0b24a7147a2fa402edd5b06831c52e24e8b00a5a154e050b8cad05a1ca3302e9d1e403ba02a831b5c7d7df
+  data.tar.gz: 54b5aa18b655620e74642ec2607cd47047c6a81fd054262104abbc365ee72cd903cbe709bbb5835554d5445efa80c0cfafe42fe8d75d600a679bd63111ad9dd2

data/bin/epr

@@ -4,12 +4,13 @@ require "csv"
 require "optparse"
 require "time"
 require "logger"
+require 'redis'
 
 require "github_api"
 require "gitlab"
 require "bitbucket_rest_api"
 
-VERSION = "0.3.6"
+VERSION = "0.3.11"
 SERVICES = %w[github gitlab bitbucket]
 GIT_CONFIGS = %w[epr.token github.oauth-token]
 
@@ -22,6 +23,16 @@ EXPORT_PRS = "pr"
 EXPORT_PR_COMMENTS = "pr_comments"
 
 DEFAULT_BODY_LENGTH = 2 ** 32 - 1
+BITBUCKET_RATE_LIMIT = 950 # actually, it's 1000 - we play safe here
+
+# To store rate limit information for fast read & write.
+# Data will persist in Redis DB/server until we shut it down.
+# Docs: https://www.rubydoc.info/github/redis/redis-rb/master/frames
+# Read: https://www.mikeperham.com/2015/09/24/storing-data-with-redis/#databases
+$redisLogger = Logger.new(STDOUT)
+$redisLogger.info("HELLO FROM REDIS LOGGER")
+$redis = Redis.new(host: "localhost", db: 15)
+$redis.set("bb-executed", 0) if $redis.get("bb-executed").nil?
 
 def localtime(t)
   # MM/DD/YY HH:MM:SS
@@ -55,6 +66,45 @@ def lookup_token
   end
 end
 
+# extract "a3351540a2da" from
+# given "https://api.bitbucket.org/2.0/repositories/hellogold/walletfactory/diff/hellogold/walletfactory:a3351540a2da..05675a5ae8ed?path=go%2Fwalletfactory%2F.gitignore"
+def extract_hash_from_diff(user, repo, diff_link)
+  str1 = diff_link.split("..")
+  str2 = str1[0].split(":")
+  short_hash = str2[2]
+  long_hash = get_full_hash(user, repo, short_hash)
+  return long_hash
+end
+
+def get_full_hash(user, repo, short_hash)
+  sleep_if_limit_reached
+
+  begin
+    result = $bitbucket.repos.commit.get_one(user, repo, short_hash)
+    $redis.incr("bb-executed")
+    return result["hash"]
+  rescue
+    $redis.incr("bb-executed")
+    return "bb-import-commit-not-found"
+  end
+end
+
+def sleep_if_limit_reached
+  executions = $redis.get("bb-executed").to_i
+  $redisLogger.info("We have called the API for #{executions} times")
+
+  if executions >= BITBUCKET_RATE_LIMIT
+    now = Time.now
+    now_formatted = now.strftime("%I:%M:%S")
+    one_hour = 1*60*60
+    later = now + one_hour
+    later_formatted = later.strftime("%I:%M:%S")
+    $redisLogger.info("#{now_formatted} - Rate limit reached. Will pause for an hour and will resume at #{later_formatted}.")
+    sleep one_hour
+    $redis.set("bb-executed", 0)
+  end
+end
+
 def bitbucket(user, repo)
   # TODO: make sure no need to translate any states
   # https://developer.atlassian.com/bitbucket/api/2/reference/resource/repositories/%7Busername%7D/%7Brepo_slug%7D/pullrequests
@@ -73,6 +123,7 @@ def bitbucket(user, repo)
 
     loop do
       page += 1
+      sleep_if_limit_reached
 
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => page, :state => $filter.upcase)
       prs["values"].each do |pr|
@@ -90,15 +141,17 @@ def bitbucket(user, repo)
           pr["links"].html.href,
           pr.summary.raw,
           pr.summary.html,
-          !pr.source.commit.nil? ? pr.source.commit["hash"] : "",
-          !pr.destination.commit.nil? ? pr.destination.commit["hash"] : "",
+          !pr.source.commit.nil? ? get_full_hash(user, repo, pr.source.commit["hash"]) : "",
+          !pr.destination.commit.nil? ? get_full_hash(user, repo, pr.destination.commit["hash"]) : "",
           !pr.source.branch.nil? ? pr.source.branch.name : "",
           !pr.destination.branch.nil? ? pr.destination.branch.name : "",
           !pr.reason.nil? ? pr.reason : "",
-          !pr.merge_commit.nil? ? pr.merge_commit["hash"] : ""
+          !pr.merge_commit.nil? ? pr.merge_commit["hash"] : "",
+          !pr.closed_by.nil? ? pr.closed_by.display_name : ""
         ]
       end
 
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end
@@ -108,12 +161,15 @@ def bitbucket(user, repo)
     
     loop do
       pr_page += 1
+      sleep_if_limit_reached
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => pr_page, :state => $filter.upcase)
       
       prs["values"].each do |pr|
         comment_page = 0
         loop do
           comment_page += 1
+          sleep_if_limit_reached
+
           comments = $bitbucket.repos.pull_request.comments(user, repo, pr.id, :page => comment_page)
           comments["values"].each do |comment|
             rows << [
@@ -122,6 +178,7 @@ def bitbucket(user, repo)
               comment.pullrequest.id,
               comment.user? ? comment.user.display_name : no_user,
               comment.inline? ? "inline" : "normal",
+              comment.id,
               comment.content.raw,
               comment.content.html,
               localtime(comment.updated_on),
@@ -129,13 +186,17 @@ def bitbucket(user, repo)
               comment.inline? ? comment.inline.to : "",
               comment.inline? ? comment.inline.from : "",
               comment.inline? ? comment.inline.path : "",
-              comment["links"].code? ? comment["links"].code.href : ""
+              comment["links"].code? ? comment["links"].code.href : "",
+              comment.parent? ? comment.parent.id : "",
+              comment["links"].code? ? extract_hash_from_diff(user, repo, comment["links"].code.href) : ""
             ]
           end
+          $redis.incr("bb-executed")
           break unless comments["next"]
         end
       end
-
+      
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end
@@ -281,9 +342,9 @@ def export_repos(argv)
   rows = []
 
   if $export == EXPORT_PR_COMMENTS 
-    rows << %w[Repository Type PRNumber User CommentType BodyRaw BodyHTML CreatedAt IsDeleted ToLine FromLine FilePath Diff]
+    rows << %w[Repository Type PRNumber User CommentType CommentID BodyRaw BodyHTML CreatedAt IsDeleted ToLine FromLine FilePath Diff ParentID CommitHash]
   else
-    rows << %w[Repository Type # User Title State CreatedAt UpdatedAt URL BodyRaw BodyHTML SourceCommit DestinationCommit SourceBranch DestinationBranch DeclineReason MergeCommit]
+    rows << %w[Repository Type # User Title State CreatedAt UpdatedAt URL BodyRaw BodyHTML SourceCommit DestinationCommit SourceBranch DestinationBranch DeclineReason MergeCommit ClosedBy]
   end
 
   rows[-1].insert(4, "Body") if $body

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zz-export-pull-requests
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.11
 platform: ruby
 authors:
 - Zulhilmi Zainudin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-17 00:00:00.000000000 Z
+date: 2019-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: github_api
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.8
+        version: 0.1.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.8
+        version: 0.1.11
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
 description: Program to export GitHub, GitLab, or Bitbucket pull requests/merge requests
   and issues to CSV a file. This is forked version of https://github.com/sshaw/export-pull-requests
   project.