hub_link 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ca30700c01889aa81a758897b9ed2045326fc007
4
- data.tar.gz: 927b8700754db70fd474635aa22cbadd5e2d68f3
3
+ metadata.gz: d62273454a4669abde1d3907ba432c300459c86b
4
+ data.tar.gz: f4241576d1e46588f13929827085ef9691c6cb31
5
5
  SHA512:
6
- metadata.gz: ed4acda89cfe380cd64cf031a3be42d23340a297025fb48e625c50f720d997eefcdc031ca1cfe1e5a612caca131c7427188f0ae8e3e14ca9455c9dc6e1aaf24c
7
- data.tar.gz: 8c8faa99fdb7f301f7c2a0c72d1e38bb5adc14f0efb39b6dcf8ef6d25983b1603ac2615f20c930567d2b3a0cb55253e1b7a905508f57b87a82d9dc2f7ae419d7
6
+ metadata.gz: 50afd523badb77a74b116530b7755b4d1c992e5bf7322e9319c3446ff8b38c3838142720ceb63371a2fcc72ef48673fb868a3b87868d37ad37498cdd0f7c4097
7
+ data.tar.gz: aa338de1ba9957f245c45327fa07438f2c25f41bd1474c39062b01c83c65a842ac96f3722014a3c1495a1a4797e4c93dc8ab1d9e4db154ac6091fc2b6ccc50f8
data/README.md CHANGED
@@ -14,7 +14,7 @@ Add this line to your application's Gemfile:
14
14
  Then you can do:
15
15
 
16
16
  ```ruby
17
- stream = HubLink::Stream.new("balvig/hub_link", start_date: 3.months.ago)
17
+ stream = HubLink::Stream.new("balvig/hub_link", since: 3.months.ago)
18
18
 
19
19
  stream.in_batches do |batch|
20
20
  batch.review_requests # => [{ digest: "a45bfa...", reviewer: "balvig", ... }]
@@ -23,8 +23,7 @@ stream.in_batches do |batch|
23
23
  end
24
24
  ```
25
25
 
26
- Hub Link will loop through the API in batches of 7 days at a time from
27
- the start date.
26
+ Hub Link will loop through the API one page at a time.
28
27
 
29
28
  ## Contributing
30
29
 
data/hub_link.gemspec CHANGED
@@ -21,7 +21,6 @@ Gem::Specification.new do |spec|
21
21
  spec.add_dependency "activesupport"
22
22
  spec.add_dependency "dotenv"
23
23
  spec.add_dependency "faraday_middleware"
24
- spec.add_dependency "faraday-detailed_logger"
25
24
  spec.add_dependency "octokit"
26
25
 
27
26
  spec.add_development_dependency "bundler", "~> 1.12"
data/lib/hub_link.rb CHANGED
@@ -4,5 +4,15 @@ require "hub_link/stream"
4
4
  require "hub_link/version"
5
5
 
6
6
  module HubLink
7
- Configuration.new.apply
7
+ START = "HubLink::Started"
8
+ UPDATE = "HubLink::Update"
9
+ FINISH = "HubLink::Finished"
10
+
11
+ class << self
12
+ attr_accessor :config
13
+
14
+ delegate :logger, to: :config
15
+ end
16
+
17
+ self.config = Configuration.new
8
18
  end
@@ -7,18 +7,43 @@ require "hub_link/slicer"
7
7
  module HubLink
8
8
  module Api
9
9
  class PullRequest < SimpleDelegator
10
- def self.search(filter, auto_paginate: true)
11
- Octokit.auto_paginate = auto_paginate
12
- Octokit.search_issues(filter).items.map { |item| new(item) }
10
+ EXPORT_COLUMNS = %i(
11
+ id
12
+ title
13
+ number
14
+ created_at
15
+ updated_at
16
+ closed_at
17
+ merged_at
18
+ approval_time
19
+ time_to_first_review
20
+ merge_time
21
+ body_size
22
+ additions
23
+ comments_count
24
+ review_comments_count
25
+ submitter
26
+ straight_approval?
27
+ labels
28
+ repo
29
+ html_url
30
+ state
31
+ )
32
+
33
+ def self.list(repo:, page:, **options)
34
+ Octokit.list_issues(repo, options.merge(page: page, sort: "updated", direction: "asc", state: "all")).map do |item|
35
+ item.repo = repo
36
+ new(item)
37
+ end.find_all(&:pull_request?)
13
38
  end
14
39
 
15
- def self.oldest(repo:)
16
- search("type:pr sort:updated-asc repo:#{repo}", auto_paginate: false).first
40
+ def pull_request?
41
+ respond_to?(:pull_request)
17
42
  end
18
43
 
19
44
  def submitter
20
45
  user.login
21
- end
46
+ end
22
47
 
23
48
  def reviews
24
49
  @_reviews ||= fetch_reviews
@@ -39,7 +64,7 @@ module HubLink
39
64
  def merge_time
40
65
  return unless merged?
41
66
 
42
- (closed_at - created_at).in_hours
67
+ (merged_at - created_at).in_hours
43
68
  end
44
69
 
45
70
  def time_to_first_review
@@ -58,26 +83,30 @@ module HubLink
58
83
  reviews.all?(&:approval?)
59
84
  end
60
85
 
61
- def review_count
62
- reviews.size
86
+ def merged_at
87
+ extended_data.merged_at
88
+ end
89
+
90
+ def comments_count
91
+ extended_data.comments
63
92
  end
64
93
 
65
- def repo
66
- extended_data.base.repo.full_name
94
+ def review_comments_count
95
+ extended_data.review_comments
67
96
  end
68
97
 
69
98
  def labels
70
- @_labels ||= Octokit.labels_for_issue(repo, number).map(&:name).join(", ")
99
+ super.map(&:name).join(", ")
71
100
  end
72
101
 
73
102
  def to_h
74
- Slicer.new(self, columns: %i(id number created_at updated_at closed_at approval_time time_to_first_review merge_time body_size additions review_count submitter straight_approval? labels repo)).to_h
103
+ Slicer.new(self, columns: EXPORT_COLUMNS).to_h
75
104
  end
76
105
 
77
106
  private
78
107
 
79
108
  def merged?
80
- closed_at.present?
109
+ merged_at.present?
81
110
  end
82
111
 
83
112
  def fetch_review_requests
@@ -92,8 +121,9 @@ module HubLink
92
121
  Octokit.pull_request_reviews(repo, number).map do |data|
93
122
  data.pull_request_id = id
94
123
  data.number = number
124
+ data.submitter = submitter
95
125
  Review.new(data)
96
- end.reject(&:invalid?)
126
+ end
97
127
  end
98
128
 
99
129
  def first_approval
@@ -1,7 +1,17 @@
1
1
  module HubLink
2
2
  module Api
3
3
  class Review < SimpleDelegator
4
- BOTS = %w(houndci-bot cookpad-devel)
4
+ EXPORT_COLUMNS = %i(
5
+ id
6
+ pull_request_id
7
+ submitted_at
8
+ reviewer
9
+ approval?
10
+ reply?
11
+ state
12
+ html_url
13
+ body
14
+ )
5
15
 
6
16
  def reviewer
7
17
  user&.login
@@ -11,8 +21,8 @@ module HubLink
11
21
  state == "APPROVED"
12
22
  end
13
23
 
14
- def invalid?
15
- bot?
24
+ def reply?
25
+ reviewer == submitter
16
26
  end
17
27
 
18
28
  def submitted?
@@ -26,15 +36,11 @@ module HubLink
26
36
  end
27
37
 
28
38
  def to_h
29
- Slicer.new(self, columns: %i(id pull_request_id submitted_at reviewer approval? state)).to_h
39
+ Slicer.new(self, columns: EXPORT_COLUMNS).to_h
30
40
  end
31
41
 
32
42
  private
33
43
 
34
- def bot?
35
- BOTS.include?(reviewer)
36
- end
37
-
38
44
  def draft?
39
45
  state == "PENDING"
40
46
  end
@@ -0,0 +1,15 @@
1
+ module HubLink
2
+ class ApiLogging < Faraday::Response::Middleware
3
+ def call(env)
4
+ message = "#{env[:method].upcase} #{env[:url]}".sub("https://api.github.com/repos", "")
5
+ logger.info(UPDATE) { message }
6
+ super
7
+ end
8
+
9
+ private
10
+
11
+ def logger
12
+ HubLink.logger
13
+ end
14
+ end
15
+ end
@@ -1,9 +1,13 @@
1
1
  module HubLink
2
2
  class Batch
3
- attr_reader :query
3
+ attr_reader :options
4
4
 
5
- def initialize(query)
6
- @query = query
5
+ def initialize(options = {})
6
+ @options = options.compact
7
+ end
8
+
9
+ def empty?
10
+ results.empty?
7
11
  end
8
12
 
9
13
  def fetch(resource)
@@ -11,21 +15,45 @@ module HubLink
11
15
  end
12
16
 
13
17
  def pull_requests
14
- results.map(&:to_h)
18
+ log "Fetching pull requests" do
19
+ results.map(&:to_h)
20
+ end
15
21
  end
16
22
 
17
23
  def reviews
18
- results.flat_map(&:reviews).map(&:to_h)
24
+ log "Fetching reviews" do
25
+ results.flat_map(&:reviews).map(&:to_h)
26
+ end
19
27
  end
20
28
 
21
29
  def review_requests
22
- results.flat_map(&:review_requests).map(&:to_h)
30
+ log "Fetching review requests" do
31
+ results.flat_map(&:review_requests).map(&:to_h)
32
+ end
23
33
  end
24
34
 
25
35
  private
26
36
 
27
37
  def results
28
- @_results ||= Api::PullRequest.search(query)
38
+ @_results ||= fetch_results
39
+ end
40
+
41
+ def log(title, &block)
42
+ logger.info(START) { title }
43
+
44
+ block.call.tap do |results|
45
+ logger.info(FINISH) { "Found #{results.size}" }
46
+ end
47
+ end
48
+
49
+ def fetch_results
50
+ log "Getting page: #{options.values.join(', ')}" do
51
+ Api::PullRequest.list(options)
52
+ end
53
+ end
54
+
55
+ def logger
56
+ HubLink.logger
29
57
  end
30
58
  end
31
59
  end
@@ -1,11 +1,17 @@
1
- require "faraday/detailed_logger"
2
1
  require "faraday_middleware"
3
2
  require "active_support"
3
+ require "hub_link/simple_logger"
4
+ require "hub_link/api_logging"
5
+
4
6
 
5
7
  module HubLink
6
8
  class Configuration
7
- def apply
9
+ attr_accessor :logger
10
+
11
+ def initialize
12
+ self.logger = SimpleLogger.new
8
13
  Octokit.middleware = middleware
14
+ Octokit.auto_paginate = false
9
15
  end
10
16
 
11
17
  private
@@ -14,18 +20,11 @@ module HubLink
14
20
 
15
21
  def middleware
16
22
  Faraday::RackBuilder.new do |builder|
17
- builder.response :detailed_logger, logger
23
+ builder.use ApiLogging
18
24
  builder.request :retry
19
25
  builder.use Octokit::Response::RaiseError
20
26
  builder.adapter Faraday.default_adapter
21
27
  end
22
28
  end
23
-
24
- def logger
25
- logger = Logger.new("hub_link.log")
26
- logger.formatter = ->(_, datetime, _, msg) { "#{datetime.to_s(:db)} - #{msg}\n" }
27
- logger.level = Logger::INFO
28
- logger
29
- end
30
29
  end
31
30
  end
@@ -1,46 +1,46 @@
1
1
  require "hub_link"
2
- require "hub_link/callbacks"
3
2
  require "hub_link/insert"
4
3
 
5
4
  module HubLink
6
5
  class Importer
7
- delegate :callback, to: :callbacks
8
-
9
- def self.run(*args, &block)
10
- new(*args, &block).run
6
+ def self.run(*args)
7
+ new(*args).run
11
8
  end
12
9
 
13
- def initialize(repo:, start_date:, resources:, &block)
10
+ def initialize(repo:, since:, resources:)
14
11
  @repo = repo.to_s
15
- @start_date = start_date
12
+ @since = since
16
13
  @resources = resources
17
- @callbacks = Callbacks.new(block)
18
14
  end
19
15
 
20
16
  def run
21
17
  stream.in_batches do |batch|
22
- callback(:init, batch.query)
23
18
 
24
19
  resources.each do |source, target|
25
- callback(:start, source)
26
20
  import batch.fetch(source), to: target
27
- callback(:finish, target.count)
28
21
  end
29
22
  end
30
23
  end
31
24
 
32
25
  private
33
26
 
34
- attr_reader :repo, :start_date, :resources, :callbacks
27
+ attr_reader :repo, :since, :resources
35
28
 
36
29
  def import(records, to:)
30
+ logger.info(START) { "Storing #{records.size} record(s) " }
31
+
37
32
  records.each do |row|
38
33
  Insert.new(row: row, target: to).run
39
34
  end
35
+ logger.info(FINISH) { "Total: #{to.count}" }
40
36
  end
41
37
 
42
38
  def stream
43
- @_stream ||= Stream.new(repo, start_date: start_date)
39
+ @_stream ||= Stream.new(repo, since: since)
40
+ end
41
+
42
+ def logger
43
+ HubLink.logger
44
44
  end
45
45
  end
46
46
  end
@@ -0,0 +1,10 @@
1
+ module HubLink
2
+ class SimpleLogger < SimpleDelegator
3
+ def initialize(output = "hub_link.log")
4
+ logger = Logger.new(output)
5
+ logger.formatter = ->(_, datetime, _, msg) { "#{datetime.to_s(:db)} - #{msg}\n" }
6
+ __setobj__(logger)
7
+ end
8
+ end
9
+ end
10
+
@@ -3,29 +3,23 @@ require "hub_link/batch"
3
3
 
4
4
  module HubLink
5
5
  class Stream
6
- def initialize(repo, start_date: nil, batch_size: 7)
6
+ def initialize(repo, since: nil)
7
7
  @repo = repo
8
- @start_date = (start_date || date_of_first_pr).to_date
9
- @batch_size = batch_size
8
+ @since = since&.to_date
10
9
  end
11
10
 
12
11
  def in_batches(&block)
13
- start_date.step(end_date, batch_size) do |date|
14
- range = "#{date}..#{date + (batch_size - 1)}"
15
- yield Batch.new "type:pr updated:#{range} repo:#{repo}"
12
+ page = 1
13
+ loop do
14
+ batch = Batch.new(repo: repo, since: since, page: page)
15
+ break if batch.empty?
16
+ yield batch
17
+ page += 1
16
18
  end
17
19
  end
18
20
 
19
21
  private
20
22
 
21
- attr_accessor :repo, :start_date, :batch_size
22
-
23
- def date_of_first_pr
24
- Api::PullRequest.oldest(repo: repo)&.updated_at
25
- end
26
-
27
- def end_date
28
- @_end_date ||= Date.tomorrow
29
- end
23
+ attr_accessor :repo, :since
30
24
  end
31
25
  end
@@ -1,3 +1,3 @@
1
1
  module HubLink
2
- VERSION = "0.12.0"
2
+ VERSION = "0.13.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hub_link
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.0
4
+ version: 0.13.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jens Balvig
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-05-22 00:00:00.000000000 Z
11
+ date: 2019-06-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -52,20 +52,6 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
- - !ruby/object:Gem::Dependency
56
- name: faraday-detailed_logger
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
59
- - - ">="
60
- - !ruby/object:Gem::Version
61
- version: '0'
62
- type: :runtime
63
- prerelease: false
64
- version_requirements: !ruby/object:Gem::Requirement
65
- requirements:
66
- - - ">="
67
- - !ruby/object:Gem::Version
68
- version: '0'
69
55
  - !ruby/object:Gem::Dependency
70
56
  name: octokit
71
57
  requirement: !ruby/object:Gem::Requirement
@@ -157,12 +143,13 @@ files:
157
143
  - lib/hub_link/api/pull_request.rb
158
144
  - lib/hub_link/api/review.rb
159
145
  - lib/hub_link/api/review_request.rb
146
+ - lib/hub_link/api_logging.rb
160
147
  - lib/hub_link/batch.rb
161
- - lib/hub_link/callbacks.rb
162
148
  - lib/hub_link/configuration.rb
163
149
  - lib/hub_link/core_ext/float.rb
164
150
  - lib/hub_link/importer.rb
165
151
  - lib/hub_link/insert.rb
152
+ - lib/hub_link/simple_logger.rb
166
153
  - lib/hub_link/slicer.rb
167
154
  - lib/hub_link/stream.rb
168
155
  - lib/hub_link/version.rb
@@ -1,23 +0,0 @@
1
- # https://gist.github.com/nilbus/6385142
2
- module HubLink
3
- class Callbacks
4
- def initialize(block)
5
- block&.call(self)
6
- end
7
-
8
- def callback(message, *args)
9
- callbacks[message]&.call(*args)
10
- end
11
-
12
- def method_missing(m, *args, &block)
13
- block ? callbacks[m] = block : super
14
- self
15
- end
16
-
17
- private
18
-
19
- def callbacks
20
- @callbacks ||= {}
21
- end
22
- end
23
- end