zz-export-pull-requests 0.3.10 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6508eec69dd50e84840ef599ef8cef449bbb6c5ab1aaa4f4ee030ab48ba23926
-  data.tar.gz: cfe15da1345df4cfcdd61c06777e59b7a466efa9349a4a552a498a390382ca98
+  metadata.gz: ec23970f15be8703f88796fd826581b0cb275913be1cbe31620659da0de5e580
+  data.tar.gz: 0caf623b5619c463baa10d45cdd3fedb61912058b980bbd3f77d2bf845e3f09e
 SHA512:
-  metadata.gz: 8d46731761ebd3959d17e29b9c95888aef6c83bb44d84088ae4e6e537a55c0978c65ef838e8e0c7ae1c64c501c203263fce1fe09e305b6a482ebe165628753a4
-  data.tar.gz: c2c2f579f861d52b1e9b453ccad9e27c59e7f4c71dcf1ed291cb17d0001c68bae3881c8436198c74a104b3468d807d2959451d9cde7eaa500afaceb3ed99f6d1
+  metadata.gz: 88f774bc035801a5def78caaf335f0a26be167544c0b24a7147a2fa402edd5b06831c52e24e8b00a5a154e050b8cad05a1ca3302e9d1e403ba02a831b5c7d7df
+  data.tar.gz: 54b5aa18b655620e74642ec2607cd47047c6a81fd054262104abbc365ee72cd903cbe709bbb5835554d5445efa80c0cfafe42fe8d75d600a679bd63111ad9dd2

data/bin/epr

@@ -4,12 +4,13 @@ require "csv"
 require "optparse"
 require "time"
 require "logger"
+require 'redis'
 
 require "github_api"
 require "gitlab"
 require "bitbucket_rest_api"
 
-VERSION = "0.3.10"
+VERSION = "0.3.11"
 SERVICES = %w[github gitlab bitbucket]
 GIT_CONFIGS = %w[epr.token github.oauth-token]
 
@@ -22,6 +23,16 @@ EXPORT_PRS = "pr"
 EXPORT_PR_COMMENTS = "pr_comments"
 
 DEFAULT_BODY_LENGTH = 2 ** 32 - 1
+BITBUCKET_RATE_LIMIT = 950 # actually, it's 1000 - we play safe here
+
+# To store rate limit information for fast read & write.
+# Data will persist in Redis DB/server until we shut it down.
+# Docs: https://www.rubydoc.info/github/redis/redis-rb/master/frames
+# Read: https://www.mikeperham.com/2015/09/24/storing-data-with-redis/#databases
+$redisLogger = Logger.new(STDOUT)
+$redisLogger.info("HELLO FROM REDIS LOGGER")
+$redis = Redis.new(host: "localhost", db: 15)
+$redis.set("bb-executed", 0) if $redis.get("bb-executed").nil?
 
 def localtime(t)
   # MM/DD/YY HH:MM:SS
@@ -66,8 +77,32 @@ def extract_hash_from_diff(user, repo, diff_link)
 end
 
 def get_full_hash(user, repo, short_hash)
-  result = $bitbucket.repos.commit.get_one(user, repo, short_hash)
-  return result["hash"]
+  sleep_if_limit_reached
+
+  begin
+    result = $bitbucket.repos.commit.get_one(user, repo, short_hash)
+    $redis.incr("bb-executed")
+    return result["hash"]
+  rescue
+    $redis.incr("bb-executed")
+    return "bb-import-commit-not-found"
+  end
+end
+
+def sleep_if_limit_reached
+  executions = $redis.get("bb-executed").to_i
+  $redisLogger.info("We have called the API for #{executions} times")
+
+  if executions >= BITBUCKET_RATE_LIMIT
+    now = Time.now
+    now_formatted = now.strftime("%I:%M:%S")
+    one_hour = 1*60*60
+    later = now + one_hour
+    later_formatted = later.strftime("%I:%M:%S")
+    $redisLogger.info("#{now_formatted} - Rate limit reached. Will pause for an hour and will resume at #{later_formatted}.")
+    sleep one_hour
+    $redis.set("bb-executed", 0)
+  end
 end
 
 def bitbucket(user, repo)
@@ -88,6 +123,7 @@ def bitbucket(user, repo)
 
     loop do
       page += 1
+      sleep_if_limit_reached
 
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => page, :state => $filter.upcase)
       prs["values"].each do |pr|
@@ -115,6 +151,7 @@ def bitbucket(user, repo)
         ]
       end
 
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end
@@ -124,12 +161,15 @@ def bitbucket(user, repo)
     
     loop do
       pr_page += 1
+      sleep_if_limit_reached
       prs = $bitbucket.repos.pull_request.all(user, repo, :page => pr_page, :state => $filter.upcase)
       
       prs["values"].each do |pr|
         comment_page = 0
         loop do
           comment_page += 1
+          sleep_if_limit_reached
+
           comments = $bitbucket.repos.pull_request.comments(user, repo, pr.id, :page => comment_page)
           comments["values"].each do |comment|
             rows << [
@@ -151,10 +191,12 @@ def bitbucket(user, repo)
               comment["links"].code? ? extract_hash_from_diff(user, repo, comment["links"].code.href) : ""
             ]
           end
+          $redis.incr("bb-executed")
           break unless comments["next"]
         end
       end
-
+      
+      $redis.incr("bb-executed")
       break unless prs["next"]
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zz-export-pull-requests
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.11
 platform: ruby
 authors:
 - Zulhilmi Zainudin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-24 00:00:00.000000000 Z
+date: 2019-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: github_api
@@ -66,6 +66,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.3
 description: Program to export GitHub, GitLab, or Bitbucket pull requests/merge requests
   and issues to CSV a file. This is forked version of https://github.com/sshaw/export-pull-requests
   project.