zz-export-pull-requests 0.3.9 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: beed724fb1aff96563cc82b34e991f10060e6ea7fe9c0387297e73b2c28bc762
-  data.tar.gz: 67c5554c4ccd7fb44d3357fee122a2afe11c527a7f6638a7bf19d7fcd730db0e
+  metadata.gz: de96b36aca31b822e84ccecae880661cf195490d51c4c2179ec728d8cff2e29a
+  data.tar.gz: a80cc917dc3f7223ad2a11012e0c8438e07192a69bb7a723b1ab3b1c403020b7
 SHA512:
-  metadata.gz: f268c94678fd04cf6c1ea572eff9a892afda085c847d83251e4349c6712a478ee65eeea3e414f508d9deb56dbe04794c0c666c357061c166ab173747792615cc
-  data.tar.gz: afaa31d97c97e9719c1692e7a14f495da21ad83e163d23c59c91e69ab4d3e972e73d7da7ea1070542062e7f689abb52a831f9baf9661da7d3828bb01e011efb3
+  metadata.gz: afa41b1d13284145982e6de17470762c997b8db2d0b392643dde5d34c57aa36bd5cf8c52fc401a1f6e892fe65ec10255abb8d075662b8fd907909180570268b4
+  data.tar.gz: 0c9ca291ab505c0e4a5cda68ebf1277cb6931c671abe0514ce6b7bd452a520a6896503fcc426c6118c643062b2c81bb309049222b3d9bf2f5401eaca983e1a21

data/bin/epr

@@ -4,12 +4,13 @@ require "csv"
 require "optparse"
 require "time"
 require "logger"
+require 'redis'
 
 require "github_api"
 require "gitlab"
 require "bitbucket_rest_api"
 
-VERSION = "0.3.9"
+VERSION = "0.3.14"
 SERVICES = %w[github gitlab bitbucket]
 GIT_CONFIGS = %w[epr.token github.oauth-token]
 
@@ -22,6 +23,18 @@ EXPORT_PRS = "pr"
 EXPORT_PR_COMMENTS = "pr_comments"
 
 DEFAULT_BODY_LENGTH = 2 ** 32 - 1
+BITBUCKET_RATE_LIMIT = 950 # actually, it's 1000 - we play safe here
+
+# To store rate limit information for fast read & write.
+# Data will persist in Redis DB/server until we shut it down.
+# Docs: https://www.rubydoc.info/github/redis/redis-rb/master/frames
+# Read: https://www.mikeperham.com/2015/09/24/storing-data-with-redis/#databases
+$redisLogger = Logger.new(STDERR)
+$redisLogger.formatter = proc {|severity, datetime, progname, msg|
+  "[#{datetime.strftime('%d/%m/%Y %H:%M:%S')}] EPR-#{severity} - #{msg}\n"
+}
+$redis = Redis.new(host: "localhost", db: 15)
+$redis.set("bb-executed", 0) if $redis.get("bb-executed").nil?
 
 def localtime(t)
   # MM/DD/YY HH:MM:SS
@@ -55,17 +68,45 @@ def lookup_token
   end
 end
 
+# extract "a3351540a2da" from
+# given "https://api.bitbucket.org/2.0/repositories/hellogold/walletfactory/diff/hellogold/walletfactory:a3351540a2da..05675a5ae8ed?path=go%2Fwalletfactory%2F.gitignore"
 def extract_hash_from_diff(user, repo, diff_link)
   str1 = diff_link.split("..")
-  str2 = str1[1].split("?")
-  short_hash = str2[0]
+  str2 = str1[0].split(":")
+  short_hash = str2[2]
   long_hash = get_full_hash(user, repo, short_hash)
   return long_hash
 end
 
 def get_full_hash(user, repo, short_hash)
-  result = $bitbucket.repos.commit.get_one(user, repo, short_hash)
-  return result["hash"]
+  sleep_if_limit_reached
+
+  begin
+    result = $bitbucket.repos.commit.get_one(user, repo, short_hash)
+    $redis.incr("bb-executed")
+    return result["hash"]
+  rescue
+    $redis.incr("bb-executed")
+    return "bb-import-commit-not-found"
+  end
+end
+
+def sleep_if_limit_reached
+  executions = $redis.get("bb-executed").to_i
+  $redisLogger.info("We have called the API for #{executions} times") if ENV['SHOW_LOGS']
+
+  if executions >= BITBUCKET_RATE_LIMIT
+    now = Time.now
+    now_formatted = now.strftime("%I:%M:%S")
+    one_hour = 1*60*60
+    later = now + one_hour
+    later_formatted = later.strftime("%I:%M:%S")
+    if ENV['SHOW_LOGS']
+      $redisLogger.info("#{now_formatted} - Rate limit reached. Will pause for an hour and will resume at #{later_formatted}.")
+    end
+    sleep one_hour
+    $redis.set("bb-executed", 0)
+  end
 end
 
 def bitbucket(user, repo)
@@ -86,6 +127,7 @@ def bitbucket(user, repo)
 
     loop do
       page += 1
+      sleep_if_limit_reached
 
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => page, :state => $filter.upcase)
       prs["values"].each do |pr|
@@ -113,6 +155,7 @@ def bitbucket(user, repo)
         ]
       end
 
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end
@@ -122,12 +165,15 @@ def bitbucket(user, repo)
     
     loop do
       pr_page += 1
+      sleep_if_limit_reached
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => pr_page, :state => $filter.upcase)
       
       prs["values"].each do |pr|
         comment_page = 0
         loop do
           comment_page += 1
+          sleep_if_limit_reached
+
           comments = $bitbucket.repos.pull_request.comments(user, repo, pr.id, :page => comment_page)
           comments["values"].each do |comment|
             rows << [
@@ -149,10 +195,12 @@ def bitbucket(user, repo)
               comment["links"].code? ? extract_hash_from_diff(user, repo, comment["links"].code.href) : ""
             ]
           end
+          $redis.incr("bb-executed")
           break unless comments["next"]
         end
       end
-
+      
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zz-export-pull-requests
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.3.14
 platform: ruby
 authors:
 - Zulhilmi Zainudin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-22 00:00:00.000000000 Z
+date: 2019-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: github_api
@@ -66,6 +66,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
 description: Program to export GitHub, GitLab, or Bitbucket pull requests/merge requests
   and issues to CSV a file. This is forked version of https://github.com/sshaw/export-pull-requests
   project.