zz-export-pull-requests 0.3.7 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be7b000b9a74af6fdd565e31d64fd3f2b052e369578dbcd01876c8bf4cc88a4f
-  data.tar.gz: 178f6ca624aa7403a60dc1281bdb933215656e6ce57d29f2367e7d25e9de287d
+  metadata.gz: ad7da2494c7c8ad182157844df4558d731231afc7c3c8453b4cd0cf128e56be5
+  data.tar.gz: 9b990c531f0a9beb4c93c8d188459c15381c6f36762a373a34e00d305e7e3cb1
 SHA512:
-  metadata.gz: 64c90071b5df93136716ea5b1c5752f2658efe3e14472059dd6f465eef8a65ec5d03c82cb8958d1498e5706f1c079297ae9d9ffb3cb6260ec50f14194648d717
-  data.tar.gz: 666a29249a87ca270e211527c60d8dc1ad557c36d7de14a3f51af97f20f70fdebb16c82648bd5dbe67a22aded505daab198c0407c166ec542c4e6f9808afb74a
+  metadata.gz: 644f2764fe9702db374e77e78f382fc8da73a246a5762702ea6586a4353585db6d1e56a8c474b970e7be2594d8f556fd6dceff66793fa20803b8bac5f82427dd
+  data.tar.gz: 0bd92c548197b23b1298a7c9b09755efc8656228104970b94d8237425dcddc88ffedea58e6e5007e030e3d80e1312617eb03657efb7a1831c713331fdcaef714

data/bin/epr

@@ -4,12 +4,13 @@ require "csv"
 require "optparse"
 require "time"
 require "logger"
+require 'redis'
 
 require "github_api"
 require "gitlab"
 require "bitbucket_rest_api"
 
-VERSION = "0.3.7"
+VERSION = "0.3.12"
 SERVICES = %w[github gitlab bitbucket]
 GIT_CONFIGS = %w[epr.token github.oauth-token]
 
@@ -22,6 +23,18 @@ EXPORT_PRS = "pr"
 EXPORT_PR_COMMENTS = "pr_comments"
 
 DEFAULT_BODY_LENGTH = 2 ** 32 - 1
+BITBUCKET_RATE_LIMIT = 950 # actually, it's 1000 - we play safe here
+
+# To store rate limit information for fast read & write.
+# Data will persist in Redis DB/server until we shut it down.
+# Docs: https://www.rubydoc.info/github/redis/redis-rb/master/frames
+# Read: https://www.mikeperham.com/2015/09/24/storing-data-with-redis/#databases
+$redisLogger = Logger.new(STDOUT)
+$redisLogger.formatter = proc {|severity, datetime, progname, msg|
+  "[#{datetime.strftime('%d/%m/%Y %H:%M:%S')}] EPR-#{severity} - #{msg}\n"
+}
+$redis = Redis.new(host: "localhost", db: 15)
+$redis.set("bb-executed", 0) if $redis.get("bb-executed").nil?
 
 def localtime(t)
   # MM/DD/YY HH:MM:SS
@@ -55,9 +68,43 @@ def lookup_token
   end
 end
 
+# extract "a3351540a2da" from
+# given "https://api.bitbucket.org/2.0/repositories/hellogold/walletfactory/diff/hellogold/walletfactory:a3351540a2da..05675a5ae8ed?path=go%2Fwalletfactory%2F.gitignore"
+def extract_hash_from_diff(user, repo, diff_link)
+  str1 = diff_link.split("..")
+  str2 = str1[0].split(":")
+  short_hash = str2[2]
+  long_hash = get_full_hash(user, repo, short_hash)
+  return long_hash
+end
+
 def get_full_hash(user, repo, short_hash)
-  result = $bitbucket.repos.commit.get_one(user, repo, short_hash)
-  return result["hash"]
+  sleep_if_limit_reached
+
+  begin
+    result = $bitbucket.repos.commit.get_one(user, repo, short_hash)
+    $redis.incr("bb-executed")
+    return result["hash"]
+  rescue
+    $redis.incr("bb-executed")
+    return "bb-import-commit-not-found"
+  end
+end
+
+def sleep_if_limit_reached
+  executions = $redis.get("bb-executed").to_i
+  $redisLogger.info("We have called the API for #{executions} times")
+
+  if executions >= BITBUCKET_RATE_LIMIT
+    now = Time.now
+    now_formatted = now.strftime("%I:%M:%S")
+    one_hour = 1*60*60
+    later = now + one_hour
+    later_formatted = later.strftime("%I:%M:%S")
+    $redisLogger.info("#{now_formatted} - Rate limit reached. Will pause for an hour and will resume at #{later_formatted}.")
+    sleep one_hour
+    $redis.set("bb-executed", 0)
+  end
 end
 
 def bitbucket(user, repo)
@@ -78,6 +125,7 @@ def bitbucket(user, repo)
 
     loop do
       page += 1
+      sleep_if_limit_reached
 
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => page, :state => $filter.upcase)
       prs["values"].each do |pr|
@@ -100,10 +148,12 @@ def bitbucket(user, repo)
           !pr.source.branch.nil? ? pr.source.branch.name : "",
           !pr.destination.branch.nil? ? pr.destination.branch.name : "",
           !pr.reason.nil? ? pr.reason : "",
-          !pr.merge_commit.nil? ? pr.merge_commit["hash"] : ""
+          !pr.merge_commit.nil? ? pr.merge_commit["hash"] : "",
+          !pr.closed_by.nil? ? pr.closed_by.display_name : ""
         ]
       end
 
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end
@@ -113,12 +163,15 @@ def bitbucket(user, repo)
     
     loop do
       pr_page += 1
+      sleep_if_limit_reached
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => pr_page, :state => $filter.upcase)
       
       prs["values"].each do |pr|
         comment_page = 0
         loop do
           comment_page += 1
+          sleep_if_limit_reached
+
           comments = $bitbucket.repos.pull_request.comments(user, repo, pr.id, :page => comment_page)
           comments["values"].each do |comment|
             rows << [
@@ -127,6 +180,7 @@ def bitbucket(user, repo)
               comment.pullrequest.id,
               comment.user? ? comment.user.display_name : no_user,
               comment.inline? ? "inline" : "normal",
+              comment.id,
               comment.content.raw,
               comment.content.html,
               localtime(comment.updated_on),
@@ -134,13 +188,17 @@ def bitbucket(user, repo)
               comment.inline? ? comment.inline.to : "",
               comment.inline? ? comment.inline.from : "",
               comment.inline? ? comment.inline.path : "",
-              comment["links"].code? ? comment["links"].code.href : ""
+              comment["links"].code? ? comment["links"].code.href : "",
+              comment.parent? ? comment.parent.id : "",
+              comment["links"].code? ? extract_hash_from_diff(user, repo, comment["links"].code.href) : ""
             ]
           end
+          $redis.incr("bb-executed")
           break unless comments["next"]
         end
       end
-
+      
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end
@@ -286,9 +344,9 @@ def export_repos(argv)
   rows = []
 
   if $export == EXPORT_PR_COMMENTS 
-    rows << %w[Repository Type PRNumber User CommentType BodyRaw BodyHTML CreatedAt IsDeleted ToLine FromLine FilePath Diff]
+    rows << %w[Repository Type PRNumber User CommentType CommentID BodyRaw BodyHTML CreatedAt IsDeleted ToLine FromLine FilePath Diff ParentID CommitHash]
   else
-    rows << %w[Repository Type # User Title State CreatedAt UpdatedAt URL BodyRaw BodyHTML SourceCommit DestinationCommit SourceBranch DestinationBranch DeclineReason MergeCommit]
+    rows << %w[Repository Type # User Title State CreatedAt UpdatedAt URL BodyRaw BodyHTML SourceCommit DestinationCommit SourceBranch DestinationBranch DeclineReason MergeCommit ClosedBy]
   end
 
   rows[-1].insert(4, "Body") if $body

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zz-export-pull-requests
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.12
 platform: ruby
 authors:
 - Zulhilmi Zainudin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-19 00:00:00.000000000 Z
+date: 2019-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: github_api
@@ -66,6 +66,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
 description: Program to export GitHub, GitLab, or Bitbucket pull requests/merge requests
   and issues to CSV a file. This is forked version of https://github.com/sshaw/export-pull-requests
   project.