hub_link 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca30700c01889aa81a758897b9ed2045326fc007
-  data.tar.gz: 927b8700754db70fd474635aa22cbadd5e2d68f3
+  metadata.gz: d62273454a4669abde1d3907ba432c300459c86b
+  data.tar.gz: f4241576d1e46588f13929827085ef9691c6cb31
 SHA512:
-  metadata.gz: ed4acda89cfe380cd64cf031a3be42d23340a297025fb48e625c50f720d997eefcdc031ca1cfe1e5a612caca131c7427188f0ae8e3e14ca9455c9dc6e1aaf24c
-  data.tar.gz: 8c8faa99fdb7f301f7c2a0c72d1e38bb5adc14f0efb39b6dcf8ef6d25983b1603ac2615f20c930567d2b3a0cb55253e1b7a905508f57b87a82d9dc2f7ae419d7
+  metadata.gz: 50afd523badb77a74b116530b7755b4d1c992e5bf7322e9319c3446ff8b38c3838142720ceb63371a2fcc72ef48673fb868a3b87868d37ad37498cdd0f7c4097
+  data.tar.gz: aa338de1ba9957f245c45327fa07438f2c25f41bd1474c39062b01c83c65a842ac96f3722014a3c1495a1a4797e4c93dc8ab1d9e4db154ac6091fc2b6ccc50f8

data/README.md

@@ -14,7 +14,7 @@ Add this line to your application's Gemfile:
 Then you can do:
 
 ```ruby
-stream = HubLink::Stream.new("balvig/hub_link", start_date: 3.months.ago)
+stream = HubLink::Stream.new("balvig/hub_link", since: 3.months.ago)
 
 stream.in_batches do |batch|
   batch.review_requests # => [{ digest: "a45bfa...", reviewer: "balvig", ... }]
@@ -23,8 +23,7 @@ stream.in_batches do |batch|
 end
 ```
 
-Hub Link will loop through the API in batches of 7 days at a time from
-the start date.
+Hub Link will loop through the API one page at a time.
 
 ## Contributing
 

data/hub_link.gemspec

@@ -21,7 +21,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport"
   spec.add_dependency "dotenv"
   spec.add_dependency "faraday_middleware"
-  spec.add_dependency "faraday-detailed_logger"
   spec.add_dependency "octokit"
 
   spec.add_development_dependency "bundler", "~> 1.12"

data/lib/hub_link.rb

@@ -4,5 +4,15 @@ require "hub_link/stream"
 require "hub_link/version"
 
 module HubLink
-  Configuration.new.apply
+  START = "HubLink::Started"
+  UPDATE = "HubLink::Update"
+  FINISH = "HubLink::Finished"
+
+  class << self
+    attr_accessor :config
+
+    delegate :logger, to: :config
+  end
+
+  self.config = Configuration.new
 end

data/lib/hub_link/api/pull_request.rb

@@ -7,18 +7,43 @@ require "hub_link/slicer"
 module HubLink
   module Api
     class PullRequest < SimpleDelegator
-      def self.search(filter, auto_paginate: true)
-        Octokit.auto_paginate = auto_paginate
-        Octokit.search_issues(filter).items.map { |item| new(item) }
+      EXPORT_COLUMNS = %i(
+        id
+        title
+        number
+        created_at
+        updated_at
+        closed_at
+        merged_at
+        approval_time
+        time_to_first_review
+        merge_time
+        body_size
+        additions
+        comments_count
+        review_comments_count
+        submitter
+        straight_approval?
+        labels
+        repo
+        html_url
+        state
+      )
+
+      def self.list(repo:, page:, **options)
+        Octokit.list_issues(repo, options.merge(page: page, sort: "updated", direction: "asc", state: "all")).map do |item|
+          item.repo = repo
+          new(item)
+        end.find_all(&:pull_request?)
       end
 
-      def self.oldest(repo:)
-        search("type:pr sort:updated-asc repo:#{repo}", auto_paginate: false).first
+      def pull_request?
+        respond_to?(:pull_request)
       end
 
       def submitter
         user.login
-     end
+      end
 
       def reviews
         @_reviews ||= fetch_reviews
@@ -39,7 +64,7 @@ module HubLink
       def merge_time
         return unless merged?
 
-        (closed_at - created_at).in_hours
+        (merged_at - created_at).in_hours
       end
 
       def time_to_first_review
@@ -58,26 +83,30 @@ module HubLink
         reviews.all?(&:approval?)
       end
 
-      def review_count
-        reviews.size
+      def merged_at
+        extended_data.merged_at
+      end
+
+      def comments_count
+        extended_data.comments
       end
 
-      def repo
-        extended_data.base.repo.full_name
+      def review_comments_count
+        extended_data.review_comments
       end
 
       def labels
-        @_labels ||= Octokit.labels_for_issue(repo, number).map(&:name).join(", ")
+        super.map(&:name).join(", ")
       end
 
       def to_h
-        Slicer.new(self, columns: %i(id number created_at updated_at closed_at approval_time time_to_first_review merge_time body_size additions review_count submitter straight_approval? labels repo)).to_h
+        Slicer.new(self, columns: EXPORT_COLUMNS).to_h
       end
 
       private
 
         def merged?
-          closed_at.present?
+          merged_at.present?
         end
 
         def fetch_review_requests
@@ -92,8 +121,9 @@ module HubLink
           Octokit.pull_request_reviews(repo, number).map do |data|
             data.pull_request_id = id
             data.number = number
+            data.submitter = submitter
             Review.new(data)
-          end.reject(&:invalid?)
+          end
         end
 
         def first_approval

data/lib/hub_link/api/review.rb

@@ -1,7 +1,17 @@
 module HubLink
   module Api
     class Review < SimpleDelegator
-      BOTS = %w(houndci-bot cookpad-devel)
+      EXPORT_COLUMNS = %i(
+        id
+        pull_request_id
+        submitted_at
+        reviewer
+        approval?
+        reply?
+        state
+        html_url
+        body
+      )
 
       def reviewer
         user&.login
@@ -11,8 +21,8 @@ module HubLink
         state == "APPROVED"
       end
 
-      def invalid?
-        bot?
+      def reply?
+        reviewer == submitter
       end
 
       def submitted?
@@ -26,15 +36,11 @@ module HubLink
       end
 
       def to_h
-        Slicer.new(self, columns: %i(id pull_request_id submitted_at reviewer approval? state)).to_h
+        Slicer.new(self, columns: EXPORT_COLUMNS).to_h
       end
 
       private
 
-        def bot?
-          BOTS.include?(reviewer)
-        end
-
         def draft?
           state == "PENDING"
         end

data/lib/hub_link/api_logging.rb

@@ -0,0 +1,15 @@
+module HubLink
+  class ApiLogging < Faraday::Response::Middleware
+    def call(env)
+      message = "#{env[:method].upcase} #{env[:url]}".sub("https://api.github.com/repos", "")
+      logger.info(UPDATE) { message }
+      super
+    end
+
+    private
+
+      def logger
+        HubLink.logger
+      end
+  end
+end

data/lib/hub_link/batch.rb

@@ -1,9 +1,13 @@
 module HubLink
   class Batch
-    attr_reader :query
+    attr_reader :options
 
-    def initialize(query)
-      @query = query
+    def initialize(options = {})
+      @options = options.compact
+    end
+
+    def empty?
+      results.empty?
     end
 
     def fetch(resource)
@@ -11,21 +15,45 @@ module HubLink
     end
 
     def pull_requests
-      results.map(&:to_h)
+      log "Fetching pull requests" do
+        results.map(&:to_h)
+      end
     end
 
     def reviews
-      results.flat_map(&:reviews).map(&:to_h)
+      log "Fetching reviews" do
+        results.flat_map(&:reviews).map(&:to_h)
+      end
     end
 
     def review_requests
-      results.flat_map(&:review_requests).map(&:to_h)
+      log "Fetching review requests" do
+        results.flat_map(&:review_requests).map(&:to_h)
+      end
     end
 
     private
 
       def results
-        @_results ||= Api::PullRequest.search(query)
+        @_results ||= fetch_results
+      end
+
+      def log(title, &block)
+        logger.info(START) { title }
+
+        block.call.tap do |results|
+          logger.info(FINISH) { "Found #{results.size}" }
+        end
+      end
+
+      def fetch_results
+        log "Getting page: #{options.values.join(', ')}" do
+          Api::PullRequest.list(options)
+        end
+      end
+
+      def logger
+        HubLink.logger
       end
   end
 end

data/lib/hub_link/configuration.rb

@@ -1,11 +1,17 @@
-require "faraday/detailed_logger"
 require "faraday_middleware"
 require "active_support"
+require "hub_link/simple_logger"
+require "hub_link/api_logging"
+
 
 module HubLink
   class Configuration
-    def apply
+    attr_accessor :logger
+
+    def initialize
+      self.logger = SimpleLogger.new
       Octokit.middleware = middleware
+      Octokit.auto_paginate = false
     end
 
     private
@@ -14,18 +20,11 @@ module HubLink
 
       def middleware
         Faraday::RackBuilder.new do |builder|
-          builder.response :detailed_logger, logger
+          builder.use ApiLogging
           builder.request :retry
           builder.use Octokit::Response::RaiseError
           builder.adapter Faraday.default_adapter
         end
       end
-
-      def logger
-        logger = Logger.new("hub_link.log")
-        logger.formatter = ->(_, datetime, _, msg) { "#{datetime.to_s(:db)} - #{msg}\n" }
-        logger.level = Logger::INFO
-        logger
-      end
   end
 end

data/lib/hub_link/importer.rb

@@ -1,46 +1,46 @@
 require "hub_link"
-require "hub_link/callbacks"
 require "hub_link/insert"
 
 module HubLink
   class Importer
-    delegate :callback, to: :callbacks
-
-    def self.run(*args, &block)
-      new(*args, &block).run
+    def self.run(*args)
+      new(*args).run
     end
 
-    def initialize(repo:, start_date:, resources:, &block)
+    def initialize(repo:, since:, resources:)
       @repo = repo.to_s
-      @start_date = start_date
+      @since = since
       @resources = resources
-      @callbacks = Callbacks.new(block)
     end
 
     def run
       stream.in_batches do |batch|
-        callback(:init, batch.query)
 
         resources.each do |source, target|
-          callback(:start, source)
           import batch.fetch(source), to: target
-          callback(:finish, target.count)
         end
       end
     end
 
     private
 
-      attr_reader :repo, :start_date, :resources, :callbacks
+      attr_reader :repo, :since, :resources
 
       def import(records, to:)
+        logger.info(START) { "Storing #{records.size} record(s) " }
+
         records.each do |row|
           Insert.new(row: row, target: to).run
         end
+        logger.info(FINISH) { "Total: #{to.count}" }
       end
 
       def stream
-        @_stream ||= Stream.new(repo, start_date: start_date)
+        @_stream ||= Stream.new(repo, since: since)
+      end
+
+      def logger
+        HubLink.logger
       end
   end
 end

data/lib/hub_link/simple_logger.rb

@@ -0,0 +1,10 @@
+module HubLink
+  class SimpleLogger < SimpleDelegator
+    def initialize(output = "hub_link.log")
+      logger = Logger.new(output)
+      logger.formatter = ->(_, datetime, _, msg) { "#{datetime.to_s(:db)} - #{msg}\n" }
+      __setobj__(logger)
+    end
+  end
+end
+

data/lib/hub_link/stream.rb

@@ -3,29 +3,23 @@ require "hub_link/batch"
 
 module HubLink
   class Stream
-    def initialize(repo, start_date: nil, batch_size: 7)
+    def initialize(repo, since: nil)
       @repo = repo
-      @start_date = (start_date || date_of_first_pr).to_date
-      @batch_size = batch_size
+      @since = since&.to_date
     end
 
     def in_batches(&block)
-      start_date.step(end_date, batch_size) do |date|
-        range = "#{date}..#{date + (batch_size - 1)}"
-        yield Batch.new "type:pr updated:#{range} repo:#{repo}"
+      page = 1
+      loop do
+        batch = Batch.new(repo: repo, since: since, page: page)
+        break if batch.empty?
+        yield batch
+        page += 1
       end
     end
 
     private
 
-      attr_accessor :repo, :start_date, :batch_size
-
-      def date_of_first_pr
-        Api::PullRequest.oldest(repo: repo)&.updated_at
-      end
-
-      def end_date
-        @_end_date ||= Date.tomorrow
-      end
+      attr_accessor :repo, :since
   end
 end

data/lib/hub_link/version.rb

@@ -1,3 +1,3 @@
 module HubLink
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hub_link
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Jens Balvig
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-22 00:00:00.000000000 Z
+date: 2019-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: faraday-detailed_logger
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: octokit
   requirement: !ruby/object:Gem::Requirement
@@ -157,12 +143,13 @@ files:
 - lib/hub_link/api/pull_request.rb
 - lib/hub_link/api/review.rb
 - lib/hub_link/api/review_request.rb
+- lib/hub_link/api_logging.rb
 - lib/hub_link/batch.rb
-- lib/hub_link/callbacks.rb
 - lib/hub_link/configuration.rb
 - lib/hub_link/core_ext/float.rb
 - lib/hub_link/importer.rb
 - lib/hub_link/insert.rb
+- lib/hub_link/simple_logger.rb
 - lib/hub_link/slicer.rb
 - lib/hub_link/stream.rb
 - lib/hub_link/version.rb

data/lib/hub_link/callbacks.rb

@@ -1,23 +0,0 @@
-# https://gist.github.com/nilbus/6385142
-module HubLink
-  class Callbacks
-    def initialize(block)
-      block&.call(self)
-    end
-
-    def callback(message, *args)
-      callbacks[message]&.call(*args)
-    end
-
-    def method_missing(m, *args, &block)
-      block ? callbacks[m] = block : super
-      self
-    end
-
-    private
-
-      def callbacks
-        @callbacks ||= {}
-      end
-  end
-end