fbe 0.48.3 → 0.48.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/codecov.yml +2 -2
- data/.github/workflows/markdown-lint.yml +1 -1
- data/.github/workflows/typos.yml +1 -1
- data/.rubocop.yml +93 -41
- data/Gemfile +2 -1
- data/Gemfile.lock +61 -53
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/assets/bylaws/bad-branch-name-was-punished.fe.liquid +1 -1
- data/fbe.gemspec +25 -25
- data/lib/fbe/award.rb +20 -33
- data/lib/fbe/bylaws.rb +7 -7
- data/lib/fbe/conclude.rb +31 -26
- data/lib/fbe/consider.rb +1 -1
- data/lib/fbe/copy.rb +4 -4
- data/lib/fbe/delete.rb +3 -3
- data/lib/fbe/delete_one.rb +3 -3
- data/lib/fbe/enter.rb +5 -5
- data/lib/fbe/fb.rb +6 -10
- data/lib/fbe/github_graph.rb +37 -35
- data/lib/fbe/if_absent.rb +4 -4
- data/lib/fbe/issue.rb +8 -8
- data/lib/fbe/iterate.rb +54 -46
- data/lib/fbe/just_one.rb +2 -2
- data/lib/fbe/kill_if.rb +1 -1
- data/lib/fbe/middleware/formatter.rb +14 -1
- data/lib/fbe/middleware/rate_limit.rb +16 -16
- data/lib/fbe/middleware/sqlite_store.rb +53 -52
- data/lib/fbe/middleware/trace.rb +1 -5
- data/lib/fbe/octo.rb +50 -51
- data/lib/fbe/overwrite.rb +13 -13
- data/lib/fbe/pmp.rb +7 -8
- data/lib/fbe/regularly.rb +7 -7
- data/lib/fbe/repeatedly.rb +6 -6
- data/lib/fbe/sec.rb +2 -2
- data/lib/fbe/tombstone.rb +24 -16
- data/lib/fbe/unmask_repos.rb +14 -7
- data/lib/fbe/who.rb +2 -2
- data/lib/fbe.rb +2 -2
- metadata +2 -2
data/lib/fbe/issue.rb
CHANGED
|
@@ -28,15 +28,15 @@ require_relative 'octo'
|
|
|
28
28
|
# issue_fact.issue = 42 # Issue number
|
|
29
29
|
# puts Fbe.issue(issue_fact) # => "zerocracy/fbe#42"
|
|
30
30
|
def Fbe.issue(fact, options: $options, global: $global, loog: $loog)
|
|
31
|
-
raise 'The fact is nil' if fact.nil?
|
|
32
|
-
raise 'The $global is not set' if global.nil?
|
|
33
|
-
raise 'The $options is not set' if options.nil?
|
|
34
|
-
raise 'The $loog is not set' if loog.nil?
|
|
31
|
+
raise(Fbe::Error, 'The fact is nil') if fact.nil?
|
|
32
|
+
raise(Fbe::Error, 'The $global is not set') if global.nil?
|
|
33
|
+
raise(Fbe::Error, 'The $options is not set') if options.nil?
|
|
34
|
+
raise(Fbe::Error, 'The $loog is not set') if loog.nil?
|
|
35
35
|
rid = fact['repository']
|
|
36
|
-
raise "There is no 'repository' property" if rid.nil?
|
|
37
|
-
rid = rid.first.
|
|
36
|
+
raise(Fbe::Error, "There is no 'repository' property") if rid.nil?
|
|
37
|
+
rid = Integer(rid.first.to_s, 10)
|
|
38
38
|
issue = fact['issue']
|
|
39
|
-
raise "There is no 'issue' property" if issue.nil?
|
|
40
|
-
issue = issue.first.
|
|
39
|
+
raise(Fbe::Error, "There is no 'issue' property") if issue.nil?
|
|
40
|
+
issue = Integer(issue.first.to_s, 10)
|
|
41
41
|
"#{Fbe.octo(global:, options:, loog:).repo_name_by_id(rid)}##{issue}"
|
|
42
42
|
end
|
data/lib/fbe/iterate.rb
CHANGED
|
@@ -42,10 +42,10 @@ def Fbe.iterate(
|
|
|
42
42
|
fb: Fbe.fb, loog: $loog, options: $options, global: $global,
|
|
43
43
|
epoch: $epoch || Time.now, kickoff: $kickoff || Time.now, &
|
|
44
44
|
)
|
|
45
|
-
raise 'The fb is nil' if fb.nil?
|
|
46
|
-
raise 'The $global is not set' if global.nil?
|
|
47
|
-
raise 'The $options is not set' if options.nil?
|
|
48
|
-
raise 'The $loog is not set' if loog.nil?
|
|
45
|
+
raise(Fbe::Error, 'The fb is nil') if fb.nil?
|
|
46
|
+
raise(Fbe::Error, 'The $global is not set') if global.nil?
|
|
47
|
+
raise(Fbe::Error, 'The $options is not set') if options.nil?
|
|
48
|
+
raise(Fbe::Error, 'The $loog is not set') if loog.nil?
|
|
49
49
|
c = Fbe::Iterate.new(fb:, loog:, options:, global:, epoch:, kickoff:)
|
|
50
50
|
c.instance_eval(&)
|
|
51
51
|
end
|
|
@@ -98,39 +98,38 @@ class Fbe::Iterate
|
|
|
98
98
|
@label = nil
|
|
99
99
|
@since = 0
|
|
100
100
|
@query = nil
|
|
101
|
-
@
|
|
101
|
+
@sorting = nil
|
|
102
102
|
@repeats = 1
|
|
103
|
-
@
|
|
104
|
-
@
|
|
105
|
-
@
|
|
103
|
+
@quota = true
|
|
104
|
+
@lifetime = true
|
|
105
|
+
@timeout = true
|
|
106
106
|
end
|
|
107
107
|
|
|
108
|
-
# Makes the iterator
|
|
108
|
+
# Makes the iterator unaware of GitHub API quota limits.
|
|
109
109
|
#
|
|
110
|
-
# When
|
|
111
|
-
# each repository and gracefully stop when the quota is exhausted.
|
|
112
|
-
# This prevents API errors and allows for resuming later.
|
|
110
|
+
# When disabled, the iterator will not check quota status before processing
|
|
111
|
+
# each repository and will not gracefully stop when the quota is exhausted.
|
|
113
112
|
#
|
|
114
113
|
# @return [nil] Nothing is returned
|
|
115
|
-
# @example
|
|
116
|
-
# iterator.
|
|
117
|
-
# iterator.over { |repo, item| ... } # Will stop
|
|
114
|
+
# @example Disable quota awareness
|
|
115
|
+
# iterator.quota_unaware
|
|
116
|
+
# iterator.over { |repo, item| ... } # Will not stop on quota exhaustion
|
|
118
117
|
def quota_unaware
|
|
119
|
-
@
|
|
118
|
+
@quota = false
|
|
120
119
|
end
|
|
121
120
|
|
|
122
|
-
# Makes the iterator
|
|
121
|
+
# Makes the iterator unaware of lifetime limits.
|
|
123
122
|
#
|
|
124
123
|
# @return [nil] Nothing is returned
|
|
125
124
|
def lifetime_unaware
|
|
126
|
-
@
|
|
125
|
+
@lifetime = false
|
|
127
126
|
end
|
|
128
127
|
|
|
129
|
-
# Makes the iterator
|
|
128
|
+
# Makes the iterator unaware of timeout limits.
|
|
130
129
|
#
|
|
131
130
|
# @return [nil] Nothing is returned
|
|
132
131
|
def timeout_unaware
|
|
133
|
-
@
|
|
132
|
+
@timeout = false
|
|
134
133
|
end
|
|
135
134
|
|
|
136
135
|
# Sets the maximum number of iterations per repository.
|
|
@@ -144,8 +143,8 @@ class Fbe::Iterate
|
|
|
144
143
|
# @example Process up to 100 items per repository
|
|
145
144
|
# iterator.repeats(100)
|
|
146
145
|
def repeats(repeats)
|
|
147
|
-
raise 'Cannot set "repeats" to nil' if repeats.nil?
|
|
148
|
-
raise 'The "repeats" must be a positive integer' unless repeats.positive?
|
|
146
|
+
raise(Fbe::Error, 'Cannot set "repeats" to nil') if repeats.nil?
|
|
147
|
+
raise(Fbe::Error, 'The "repeats" must be a positive integer') unless repeats.positive?
|
|
149
148
|
@repeats = repeats
|
|
150
149
|
end
|
|
151
150
|
|
|
@@ -161,8 +160,8 @@ class Fbe::Iterate
|
|
|
161
160
|
# @example Query for issues after a certain ID
|
|
162
161
|
# iterator.by('(and (eq what "issue") (gt id $before) (eq repo $repository))')
|
|
163
162
|
def by(query)
|
|
164
|
-
raise 'Query is already set' unless @query.nil?
|
|
165
|
-
raise 'Cannot set query to nil' if query.nil?
|
|
163
|
+
raise(Fbe::Error, 'Query is already set') unless @query.nil?
|
|
164
|
+
raise(Fbe::Error, 'Cannot set query to nil') if query.nil?
|
|
166
165
|
@query = query
|
|
167
166
|
end
|
|
168
167
|
|
|
@@ -178,10 +177,10 @@ class Fbe::Iterate
|
|
|
178
177
|
# @example Sort issues by number
|
|
179
178
|
# iterator.sort_by('issue')
|
|
180
179
|
def sort_by(prop)
|
|
181
|
-
raise 'Sort field is already set' unless @
|
|
182
|
-
raise 'Cannot set sort field to nil' if prop.nil?
|
|
183
|
-
raise 'Sort field must be a String' unless prop.is_a?(String)
|
|
184
|
-
@
|
|
180
|
+
raise(Fbe::Error, 'Sort field is already set') unless @sorting.nil?
|
|
181
|
+
raise(Fbe::Error, 'Cannot set sort field to nil') if prop.nil?
|
|
182
|
+
raise(Fbe::Error, 'Sort field must be a String') unless prop.is_a?(String)
|
|
183
|
+
@sorting = prop
|
|
185
184
|
end
|
|
186
185
|
|
|
187
186
|
# Sets the label for tracking iteration state.
|
|
@@ -196,9 +195,11 @@ class Fbe::Iterate
|
|
|
196
195
|
# @example Set label for issue processing
|
|
197
196
|
# iterator.as('issue_processor')
|
|
198
197
|
def as(label)
|
|
199
|
-
raise 'Label is already set' unless @label.nil?
|
|
200
|
-
raise 'Cannot set "label" to nil' if label.nil?
|
|
201
|
-
|
|
198
|
+
raise(Fbe::Error, 'Label is already set') unless @label.nil?
|
|
199
|
+
raise(Fbe::Error, 'Cannot set "label" to nil') if label.nil?
|
|
200
|
+
unless label.match?(/\A[_a-z][a-zA-Z0-9_]*\z/)
|
|
201
|
+
raise(Fbe::Error, "Wrong label format '#{label}', use [_a-z][a-zA-Z0-9_]*")
|
|
202
|
+
end
|
|
202
203
|
@label = label
|
|
203
204
|
end
|
|
204
205
|
|
|
@@ -238,18 +239,19 @@ class Fbe::Iterate
|
|
|
238
239
|
# fetch_and_process_issue(repo_id, issue_number)
|
|
239
240
|
# issue_number + 1 # Return next issue number to process
|
|
240
241
|
# end
|
|
241
|
-
def over
|
|
242
|
-
raise 'Use "as" first' if @label.nil?
|
|
243
|
-
raise 'Use "by" first' if @query.nil?
|
|
242
|
+
def over # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
243
|
+
raise(Fbe::Error, 'Use "as" first') if @label.nil?
|
|
244
|
+
raise(Fbe::Error, 'Use "by" first') if @query.nil?
|
|
244
245
|
seen = {}
|
|
245
246
|
oct = Fbe.octo(loog: @loog, options: @options, global: @global)
|
|
246
247
|
return if Fbe.over?(
|
|
247
248
|
global: @global, options: @options, loog: @loog, epoch: @epoch, kickoff: @kickoff,
|
|
248
|
-
quota_aware: @
|
|
249
|
+
quota_aware: @quota, lifetime_aware: @lifetime, timeout_aware: @timeout
|
|
249
250
|
)
|
|
250
|
-
repos =
|
|
251
|
-
|
|
252
|
-
|
|
251
|
+
repos =
|
|
252
|
+
Fbe.unmask_repos(
|
|
253
|
+
loog: @loog, options: @options, global: @global, quota_aware: @quota
|
|
254
|
+
).map { |n| oct.repo_id_by_name(n) }
|
|
253
255
|
started = Time.now
|
|
254
256
|
restarted = []
|
|
255
257
|
before =
|
|
@@ -267,10 +269,10 @@ class Fbe::Iterate
|
|
|
267
269
|
end
|
|
268
270
|
starts = before.dup
|
|
269
271
|
values = {}
|
|
270
|
-
loop do
|
|
272
|
+
loop do # rubocop:disable Metrics/BlockLength
|
|
271
273
|
if Fbe.over?(
|
|
272
274
|
global: @global, options: @options, loog: @loog, epoch: @epoch, kickoff: @kickoff,
|
|
273
|
-
quota_aware: @
|
|
275
|
+
quota_aware: @quota, lifetime_aware: @lifetime, timeout_aware: @timeout
|
|
274
276
|
)
|
|
275
277
|
@loog.info("Time to stop after #{started.ago}")
|
|
276
278
|
break
|
|
@@ -278,7 +280,7 @@ class Fbe::Iterate
|
|
|
278
280
|
repos.each do |repo|
|
|
279
281
|
if Fbe.over?(
|
|
280
282
|
global: @global, options: @options, loog: @loog, epoch: @epoch, kickoff: @kickoff,
|
|
281
|
-
quota_aware: @
|
|
283
|
+
quota_aware: @quota, lifetime_aware: @lifetime, timeout_aware: @timeout
|
|
282
284
|
)
|
|
283
285
|
@loog.info("Won't check repository ##{repo}")
|
|
284
286
|
break
|
|
@@ -290,10 +292,10 @@ class Fbe::Iterate
|
|
|
290
292
|
next
|
|
291
293
|
end
|
|
292
294
|
nxt =
|
|
293
|
-
if @
|
|
295
|
+
if @sorting
|
|
294
296
|
values[repo] ||= @fb.query(@query).each(
|
|
295
297
|
@fb, before: before[repo], repository: repo
|
|
296
|
-
).
|
|
298
|
+
).filter_map { _1[@sorting]&.first }.sort.each
|
|
297
299
|
begin
|
|
298
300
|
values[repo].next
|
|
299
301
|
rescue StopIteration
|
|
@@ -306,14 +308,20 @@ class Fbe::Iterate
|
|
|
306
308
|
if nxt.nil?
|
|
307
309
|
@loog.debug("Next element after ##{before[repo]} not suggested, re-starting from ##{@since}: #{@query}")
|
|
308
310
|
restarted << repo
|
|
309
|
-
values.delete(repo) if @
|
|
311
|
+
values.delete(repo) if @sorting
|
|
310
312
|
@since
|
|
311
313
|
else
|
|
312
314
|
@loog.debug("Next is ##{nxt}, starting from it")
|
|
313
|
-
|
|
315
|
+
begin
|
|
316
|
+
yield(repo, nxt)
|
|
317
|
+
rescue Fbe::OffQuota
|
|
318
|
+
raise
|
|
319
|
+
rescue StandardError => e
|
|
320
|
+
raise(e.class, "Failure in repository ##{repo} at ##{nxt}: #{e.message}")
|
|
321
|
+
end
|
|
314
322
|
end
|
|
315
323
|
unless before[repo].is_a?(Integer)
|
|
316
|
-
raise "Iterator must return an Integer, but #{before[repo].class} was returned"
|
|
324
|
+
raise(Fbe::Error, "Iterator must return an Integer, but #{before[repo].class} was returned")
|
|
317
325
|
end
|
|
318
326
|
seen[repo] += 1
|
|
319
327
|
end
|
data/lib/fbe/just_one.rb
CHANGED
|
@@ -43,7 +43,7 @@ def Fbe.just_one(fb: Fbe.fb)
|
|
|
43
43
|
@map[k.to_sym]
|
|
44
44
|
end
|
|
45
45
|
end
|
|
46
|
-
yield
|
|
46
|
+
yield(f)
|
|
47
47
|
q = attrs.except('_id', '_time', '_version').map do |k, v|
|
|
48
48
|
vv = v.to_s
|
|
49
49
|
if v.is_a?(String)
|
|
@@ -57,6 +57,6 @@ def Fbe.just_one(fb: Fbe.fb)
|
|
|
57
57
|
before = fb.query(q).each.first
|
|
58
58
|
return before unless before.nil?
|
|
59
59
|
n = fb.insert
|
|
60
|
-
attrs.each { |k, v| n.
|
|
60
|
+
attrs.each { |k, v| n.public_send(:"#{k}=", v) }
|
|
61
61
|
n
|
|
62
62
|
end
|
data/lib/fbe/kill_if.rb
CHANGED
|
@@ -39,6 +39,19 @@ require_relative '../../fbe/middleware'
|
|
|
39
39
|
# Copyright:: Copyright (c) 2024-2026 Zerocracy
|
|
40
40
|
# License:: MIT
|
|
41
41
|
class Fbe::Middleware::Formatter < Faraday::Logging::Formatter
|
|
42
|
+
# Registers a filter that masks the credential portion of an
|
|
43
|
+
# `Authorization` header so live tokens never reach the log.
|
|
44
|
+
# Matches any auth scheme: `Authorization: "Bearer xxx"`,
|
|
45
|
+
# `Authorization: "Basic yyy"`, etc. The scheme is preserved,
|
|
46
|
+
# the credential is replaced with `[FILTERED]`.
|
|
47
|
+
#
|
|
48
|
+
# @param [Logger] logger The Faraday-supplied logger
|
|
49
|
+
# @param [Hash] options Faraday formatter options
|
|
50
|
+
def initialize(logger:, options:)
|
|
51
|
+
super
|
|
52
|
+
filter(/(Authorization:\s*"?\s*\S+\s+)[^"\s]+/i, '\1[FILTERED]')
|
|
53
|
+
end
|
|
54
|
+
|
|
42
55
|
# Captures HTTP request details for later use in error logging.
|
|
43
56
|
#
|
|
44
57
|
# @param [Hash] http Request data including method, url, headers, and body
|
|
@@ -53,7 +66,7 @@ class Fbe::Middleware::Formatter < Faraday::Logging::Formatter
|
|
|
53
66
|
# @return [void]
|
|
54
67
|
# @note Only logs when status >= 400
|
|
55
68
|
# @note Special handling for 403 JSON responses to show compact error message
|
|
56
|
-
def response(http)
|
|
69
|
+
def response(http) # rubocop:disable Metrics/AbcSize
|
|
57
70
|
return if http.status < 400
|
|
58
71
|
if http.status == 403 && http.response_headers['content-type'].start_with?('application/json')
|
|
59
72
|
warn(
|
|
@@ -29,9 +29,9 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
|
|
|
29
29
|
# @param [Object] app The next middleware in the stack
|
|
30
30
|
def initialize(app)
|
|
31
31
|
super
|
|
32
|
-
@
|
|
33
|
-
@
|
|
34
|
-
@
|
|
32
|
+
@cached = nil
|
|
33
|
+
@remaining = nil
|
|
34
|
+
@counter = 0
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
# Processes the HTTP request and handles rate limit caching.
|
|
@@ -54,14 +54,14 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
|
|
|
54
54
|
# @param [Faraday::Env] env The request environment
|
|
55
55
|
# @return [Faraday::Response] Cached or fresh response
|
|
56
56
|
def handle_rate_limit_request(env)
|
|
57
|
-
if @
|
|
57
|
+
if @cached.nil? || @counter >= 100
|
|
58
58
|
response = @app.call(env)
|
|
59
|
-
@
|
|
60
|
-
@
|
|
61
|
-
@
|
|
59
|
+
@cached = response.dup
|
|
60
|
+
@remaining = extract_remaining_count(response)
|
|
61
|
+
@counter = 0
|
|
62
62
|
response
|
|
63
63
|
else
|
|
64
|
-
response = @
|
|
64
|
+
response = @cached.dup
|
|
65
65
|
update_remaining_count(response)
|
|
66
66
|
Faraday::Response.new(response_env(env, response))
|
|
67
67
|
end
|
|
@@ -69,9 +69,9 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
|
|
|
69
69
|
|
|
70
70
|
# Tracks non-rate_limit requests and decrements counter.
|
|
71
71
|
def track_request
|
|
72
|
-
return if @
|
|
73
|
-
@
|
|
74
|
-
@
|
|
72
|
+
return if @remaining.nil?
|
|
73
|
+
@remaining -= 1 if @remaining.positive?
|
|
74
|
+
@counter += 1
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
# Extracts the remaining count from the response body.
|
|
@@ -96,8 +96,8 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
|
|
|
96
96
|
# @param [Faraday::Response] response The cached response to update
|
|
97
97
|
def update_remaining_count(response)
|
|
98
98
|
body = response.body
|
|
99
|
-
|
|
100
|
-
if
|
|
99
|
+
stringed = body.is_a?(String)
|
|
100
|
+
if stringed
|
|
101
101
|
begin
|
|
102
102
|
body = JSON.parse(body)
|
|
103
103
|
rescue JSON::ParserError
|
|
@@ -105,8 +105,8 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
|
|
|
105
105
|
end
|
|
106
106
|
end
|
|
107
107
|
return unless body.is_a?(Hash) && body['rate']
|
|
108
|
-
body['rate']['remaining'] = @
|
|
109
|
-
return unless
|
|
108
|
+
body['rate']['remaining'] = @remaining
|
|
109
|
+
return unless stringed
|
|
110
110
|
response.instance_variable_set(:@body, body.to_json)
|
|
111
111
|
end
|
|
112
112
|
|
|
@@ -117,7 +117,7 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
|
|
|
117
117
|
# @return [Hash] Response environment hash
|
|
118
118
|
def response_env(env, response)
|
|
119
119
|
headers = response.headers.dup
|
|
120
|
-
headers['x-ratelimit-remaining'] = @
|
|
120
|
+
headers['x-ratelimit-remaining'] = @remaining.to_s if @remaining
|
|
121
121
|
{
|
|
122
122
|
method: env.method,
|
|
123
123
|
url: env.url,
|
|
@@ -58,21 +58,21 @@ class Fbe::Middleware::SqliteStore
|
|
|
58
58
|
# @raise [ArgumentError] If path is nil/empty, directory doesn't exist, version is nil/empty,
|
|
59
59
|
# or ttl is not nil or not Integer or not positive
|
|
60
60
|
def initialize(path, version, loog: Loog::NULL, maxsize: '10Mb', maxvsize: '10Kb', ttl: nil, cache_min_age: nil)
|
|
61
|
-
raise
|
|
61
|
+
raise(ArgumentError, 'Database path cannot be nil or empty') if path.nil? || path.empty?
|
|
62
62
|
dir = File.dirname(path)
|
|
63
|
-
raise
|
|
64
|
-
raise
|
|
63
|
+
raise(ArgumentError, "Directory #{dir} does not exist") unless File.directory?(dir)
|
|
64
|
+
raise(ArgumentError, 'Version cannot be nil or empty') if version.nil? || version.empty?
|
|
65
65
|
@path = File.absolute_path(path)
|
|
66
66
|
@version = version
|
|
67
67
|
@loog = loog
|
|
68
|
-
@maxsize = Filesize.from(maxsize.to_s)
|
|
69
|
-
@maxvsize = Filesize.from(maxvsize.to_s)
|
|
70
|
-
raise
|
|
68
|
+
@maxsize = Integer(Filesize.from(maxsize.to_s))
|
|
69
|
+
@maxvsize = Integer(Filesize.from(maxvsize.to_s))
|
|
70
|
+
raise(ArgumentError, 'TTL can be nil or Integer > 0') if !ttl.nil? && !(ttl.is_a?(Integer) && ttl.positive?)
|
|
71
71
|
@ttl = ttl
|
|
72
72
|
if !cache_min_age.nil? && !(cache_min_age.is_a?(Integer) && cache_min_age.positive?)
|
|
73
|
-
raise
|
|
73
|
+
raise(ArgumentError, 'Cache min age can be nil or Integer > 0')
|
|
74
74
|
end
|
|
75
|
-
@
|
|
75
|
+
@minage = cache_min_age
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
# Read a value from the cache.
|
|
@@ -106,28 +106,29 @@ class Fbe::Middleware::SqliteStore
|
|
|
106
106
|
# @return [nil]
|
|
107
107
|
# @note Values larger than 10KB are not cached
|
|
108
108
|
# @note Non-GET requests and URLs with query parameters are not cached
|
|
109
|
-
def write(key, value)
|
|
109
|
+
def write(key, value) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
110
110
|
return if value.is_a?(Array) && value.any? do |vv|
|
|
111
111
|
req = JSON.parse(vv[0])
|
|
112
112
|
req['method'] != 'get'
|
|
113
113
|
end
|
|
114
|
-
if @
|
|
114
|
+
if @minage && value.is_a?(Array) && value[0].is_a?(Array) && value[0].size > 1
|
|
115
115
|
begin
|
|
116
116
|
resp = JSON.parse(value[0][1])
|
|
117
117
|
rescue TypeError, JSON::ParserError => e
|
|
118
118
|
@loog.info("Failed to parse response to rewrite the cache age: #{e.message}")
|
|
119
119
|
resp = nil
|
|
120
120
|
end
|
|
121
|
-
|
|
122
|
-
if
|
|
121
|
+
control = resp.dig('response_headers', 'cache-control') if resp.is_a?(Hash)
|
|
122
|
+
if control && !control.empty?
|
|
123
123
|
%w[max-age s-maxage].each do |key|
|
|
124
|
-
|
|
124
|
+
matched = control.scan(/#{key}=(\d+)/i).first&.first
|
|
125
|
+
age = matched.nil? ? nil : Integer(matched, 10)
|
|
125
126
|
if age
|
|
126
|
-
age = [age, @
|
|
127
|
-
|
|
127
|
+
age = [age, @minage].max
|
|
128
|
+
control = control.sub(/#{key}=(\d+)/, "#{key}=#{age}")
|
|
128
129
|
end
|
|
129
130
|
end
|
|
130
|
-
resp['response_headers']['cache-control'] =
|
|
131
|
+
resp['response_headers']['cache-control'] = control
|
|
131
132
|
value[0][1] = JSON.dump(resp)
|
|
132
133
|
end
|
|
133
134
|
end
|
|
@@ -146,10 +147,10 @@ class Fbe::Middleware::SqliteStore
|
|
|
146
147
|
# @return [void]
|
|
147
148
|
def clear
|
|
148
149
|
perform do |t|
|
|
149
|
-
t.execute
|
|
150
|
-
t.execute
|
|
150
|
+
t.execute('DELETE FROM cache;')
|
|
151
|
+
t.execute("UPDATE meta SET value = ? WHERE key = 'version';", [@version])
|
|
151
152
|
end
|
|
152
|
-
@db.execute
|
|
153
|
+
@db.execute('VACUUM;')
|
|
153
154
|
end
|
|
154
155
|
|
|
155
156
|
# Get all entries from the cache.
|
|
@@ -160,68 +161,68 @@ class Fbe::Middleware::SqliteStore
|
|
|
160
161
|
|
|
161
162
|
private
|
|
162
163
|
|
|
163
|
-
def perform(&)
|
|
164
|
+
def perform(&) # rubocop:disable Metrics/AbcSize
|
|
164
165
|
@db ||=
|
|
165
|
-
SQLite3::Database.new(@path).tap do |d|
|
|
166
|
+
SQLite3::Database.new(@path).tap do |d| # rubocop:disable Metrics/BlockLength
|
|
166
167
|
d.transaction do |t|
|
|
167
|
-
t.execute
|
|
168
|
-
t.execute
|
|
169
|
-
t.execute
|
|
170
|
-
t.execute
|
|
171
|
-
t.execute
|
|
168
|
+
t.execute('CREATE TABLE IF NOT EXISTS cache(key TEXT UNIQUE NOT NULL, value TEXT);')
|
|
169
|
+
t.execute('CREATE INDEX IF NOT EXISTS cache_key_idx ON cache(key);')
|
|
170
|
+
t.execute('CREATE TABLE IF NOT EXISTS meta(key TEXT UNIQUE NOT NULL, value TEXT);')
|
|
171
|
+
t.execute('CREATE INDEX IF NOT EXISTS meta_key_idx ON meta(key);')
|
|
172
|
+
t.execute("INSERT INTO meta(key, value) VALUES('version', ?) ON CONFLICT(key) DO NOTHING;", [@version])
|
|
172
173
|
end
|
|
173
174
|
if d.execute("SELECT 1 FROM pragma_table_info('cache') WHERE name = 'touched_at';").dig(0, 0) != 1
|
|
174
175
|
d.transaction do |t|
|
|
175
|
-
t.execute
|
|
176
|
-
t.execute
|
|
177
|
-
t.execute
|
|
178
|
-
t.execute
|
|
176
|
+
t.execute('ALTER TABLE cache ADD COLUMN touched_at TEXT;')
|
|
177
|
+
t.execute('UPDATE cache set touched_at = ?;', [Time.now.utc.iso8601])
|
|
178
|
+
t.execute('ALTER TABLE cache RENAME TO cache_old;')
|
|
179
|
+
t.execute(<<~SQL)
|
|
179
180
|
CREATE TABLE IF NOT EXISTS cache(
|
|
180
181
|
key TEXT UNIQUE NOT NULL, value TEXT, touched_at TEXT NOT NULL
|
|
181
182
|
);
|
|
182
183
|
SQL
|
|
183
|
-
t.execute
|
|
184
|
-
t.execute
|
|
185
|
-
t.execute
|
|
186
|
-
t.execute
|
|
184
|
+
t.execute('INSERT INTO cache SELECT * FROM cache_old;')
|
|
185
|
+
t.execute('DROP TABLE cache_old;')
|
|
186
|
+
t.execute('CREATE INDEX IF NOT EXISTS cache_key_idx ON cache(key);')
|
|
187
|
+
t.execute('CREATE INDEX IF NOT EXISTS cache_touched_at_idx ON cache(touched_at);')
|
|
187
188
|
end
|
|
188
|
-
d.execute
|
|
189
|
+
d.execute('VACUUM;')
|
|
189
190
|
end
|
|
190
191
|
if d.execute("SELECT 1 FROM pragma_table_info('cache') WHERE name = 'created_at';").dig(0, 0) != 1
|
|
191
192
|
d.transaction do |t|
|
|
192
|
-
t.execute
|
|
193
|
-
t.execute
|
|
194
|
-
t.execute
|
|
195
|
-
t.execute
|
|
193
|
+
t.execute('ALTER TABLE cache ADD COLUMN created_at TEXT;')
|
|
194
|
+
t.execute('UPDATE cache set created_at = ?;', [Time.now.utc.iso8601])
|
|
195
|
+
t.execute('ALTER TABLE cache RENAME TO cache_old;')
|
|
196
|
+
t.execute(<<~SQL)
|
|
196
197
|
CREATE TABLE IF NOT EXISTS cache(
|
|
197
198
|
key TEXT UNIQUE NOT NULL, value TEXT, touched_at TEXT NOT NULL, created_at TEXT NOT NULL
|
|
198
199
|
);
|
|
199
200
|
SQL
|
|
200
|
-
t.execute
|
|
201
|
-
t.execute
|
|
202
|
-
t.execute
|
|
203
|
-
t.execute
|
|
204
|
-
t.execute
|
|
201
|
+
t.execute('INSERT INTO cache SELECT * FROM cache_old;')
|
|
202
|
+
t.execute('DROP TABLE cache_old;')
|
|
203
|
+
t.execute('CREATE INDEX IF NOT EXISTS cache_key_idx ON cache(key);')
|
|
204
|
+
t.execute('CREATE INDEX IF NOT EXISTS cache_touched_at_idx ON cache(touched_at);')
|
|
205
|
+
t.execute('CREATE INDEX IF NOT EXISTS cache_created_at_idx ON cache(created_at);')
|
|
205
206
|
end
|
|
206
|
-
d.execute
|
|
207
|
+
d.execute('VACUUM;')
|
|
207
208
|
end
|
|
208
209
|
found = d.execute("SELECT value FROM meta WHERE key = 'version' LIMIT 1;").dig(0, 0)
|
|
209
210
|
if found != @version
|
|
210
211
|
@loog.info("Version mismatch in SQLite cache: stored '#{found}' != current '#{@version}', cleaning up")
|
|
211
212
|
d.transaction do |t|
|
|
212
|
-
t.execute
|
|
213
|
-
t.execute
|
|
213
|
+
t.execute('DELETE FROM cache;')
|
|
214
|
+
t.execute("UPDATE meta SET value = ? WHERE key = 'version';", [@version])
|
|
214
215
|
end
|
|
215
|
-
d.execute
|
|
216
|
+
d.execute('VACUUM;')
|
|
216
217
|
end
|
|
217
218
|
unless @ttl.nil?
|
|
218
219
|
d.transaction do |t|
|
|
219
|
-
t.execute
|
|
220
|
+
t.execute(<<~SQL, [(Time.now.utc - (@ttl * 60 * 60)).iso8601])
|
|
220
221
|
DELETE FROM cache
|
|
221
222
|
WHERE key IN (SELECT key FROM cache WHERE (created_at < ?));
|
|
222
223
|
SQL
|
|
223
224
|
end
|
|
224
|
-
d.execute
|
|
225
|
+
d.execute('VACUUM;')
|
|
225
226
|
end
|
|
226
227
|
if File.size(@path) > @maxsize
|
|
227
228
|
@loog.info(
|
|
@@ -234,14 +235,14 @@ class Fbe::Middleware::SqliteStore
|
|
|
234
235
|
FROM pragma_page_count(), pragma_freelist_count(), pragma_page_size();
|
|
235
236
|
SQL
|
|
236
237
|
d.transaction do |t|
|
|
237
|
-
t.execute
|
|
238
|
+
t.execute(<<~SQL)
|
|
238
239
|
DELETE FROM cache
|
|
239
240
|
WHERE key IN (SELECT key FROM cache ORDER BY touched_at LIMIT 50)
|
|
240
241
|
SQL
|
|
241
242
|
deleted += t.changes
|
|
242
243
|
end
|
|
243
244
|
end
|
|
244
|
-
d.execute
|
|
245
|
+
d.execute('VACUUM;')
|
|
245
246
|
@loog.info(
|
|
246
247
|
"Deleted #{deleted} old cache entries, " \
|
|
247
248
|
"new file size: #{Filesize.from(File.size(@path).to_s).pretty} bytes"
|
data/lib/fbe/middleware/trace.rb
CHANGED
|
@@ -44,11 +44,7 @@ class Fbe::Middleware::Trace < Faraday::Middleware
|
|
|
44
44
|
# @param [Faraday::Env] env The request environment
|
|
45
45
|
# @return [Faraday::Response] The response from the next middleware
|
|
46
46
|
def call(env)
|
|
47
|
-
entry = {
|
|
48
|
-
method: env.method,
|
|
49
|
-
url: env.url.to_s,
|
|
50
|
-
started_at: Time.now
|
|
51
|
-
}
|
|
47
|
+
entry = { method: env.method, url: env.url.to_s, started_at: Time.now }
|
|
52
48
|
@app.call(env).on_complete do |response_env|
|
|
53
49
|
next if !@ignores.empty? &&
|
|
54
50
|
response_env[:http_cache_trace] &&
|