fbe 0.48.5 → 0.48.6

data/lib/fbe/fb.rb

@@ -31,34 +31,37 @@ def Fbe.fb(fb: $fb, global: $global, options: $options, loog: $loog)
   raise(Fbe::Error, 'The $global is not set') if global.nil?
   raise(Fbe::Error, 'The $options is not set') if options.nil?
   raise(Fbe::Error, 'The $loog is not set') if loog.nil?
-  global[:fb] ||=
-    begin
-      rules = Dir.glob(File.join(File.join(__dir__, '../../rules'), '*.fe')).map { |f| File.read(f) }
-      fbe = Factbase::Rules.new(
-        fb,
-        "(and \n#{rules.join("\n")}\n)",
-        uid: '_id'
-      )
-      fbe =
-        Factbase::Pre.new(fbe) do |f, fbt|
-          max = fbt.query('(max _id)').one
-          f._id = (max.nil? ? 0 : max) + 1
-          f._time = Time.now
-          f._version = [Factbase::VERSION, Judges::VERSION, options.action_version].compact.join('/')
-          f._job = Integer(options.job_id.to_s, 10) if options.job_id
-        end
-      Factbase::Impatient.new(
-        Factbase::Logged.new(
-          Factbase::SyncFactbase.new(
-            Factbase::IndexedFactbase.new(
-              Factbase::CachedFactbase.new(
-                fbe
+  global[:mutex] ||= Mutex.new
+  global[:mutex].synchronize do
+    global[:fb] ||=
+      begin
+        rules = Dir.glob(File.join(File.join(__dir__, '../../rules'), '*.fe')).map { |f| File.read(f) }
+        fbe = Factbase::Rules.new(
+          fb,
+          "(and \n#{rules.join("\n")}\n)",
+          uid: '_id'
+        )
+        fbe =
+          Factbase::Pre.new(fbe) do |f, fbt|
+            max = fbt.query('(max _id)').one
+            f._id = (max.nil? ? 0 : max) + 1
+            f._time = Time.now
+            f._version = [Factbase::VERSION, Judges::VERSION, options.action_version].compact.join('/')
+            f._job = Integer(options.job_id.to_s, 10) if options.job_id
+          end
+        Factbase::Impatient.new(
+          Factbase::Logged.new(
+            Factbase::SyncFactbase.new(
+              Factbase::IndexedFactbase.new(
+                Factbase::CachedFactbase.new(
+                  fbe
+                )
               )
-            )
+            ),
+            loog
           ),
-          loog
-        ),
-        timeout: 60
-      )
-    end
+          timeout: 60
+        )
+      end
+  end
 end

data/lib/fbe/github_graph.rb

@@ -15,13 +15,16 @@ require_relative '../fbe'
 # @param [Loog] loog Logging facility
 # @return [Fbe::Graph] The instance of the class
 def Fbe.github_graph(options: $options, global: $global, loog: $loog)
-  global[:github_graph] ||=
-    if options.testing.nil?
-      Fbe::Graph.new(token: options.github_token || ENV.fetch('GITHUB_TOKEN', nil))
-    else
-      loog.debug('The connection to GitHub GraphQL API is mocked')
-      Fbe::Graph::Fake.new
-    end
+  global[:mutex] ||= Mutex.new
+  global[:mutex].synchronize do
+    global[:github_graph] ||=
+      if options.testing.nil?
+        Fbe::Graph.new(token: options.github_token || ENV.fetch('GITHUB_TOKEN', nil))
+      else
+        loog.debug('The connection to GitHub GraphQL API is mocked')
+        Fbe::Graph::Fake.new
+      end
+  end
 end
 
 # A client to GitHub GraphQL.
@@ -45,6 +48,9 @@ class Fbe::Graph # rubocop:disable Metrics/ClassLength
   #   puts result.viewer.login #=> "octocat"
   def query(qry)
     result = client.query(client.parse(qry))
+    unless result.errors.empty?
+      raise(Fbe::Error, "GitHub GraphQL query failed: #{result.errors.messages.values.flatten.join('; ')}")
+    end
     result.data
   end
 
@@ -487,11 +493,12 @@ class Fbe::Graph # rubocop:disable Metrics/ClassLength
     total = 0
     cursor = nil
     loop do
+      after = "after: \"#{cursor}\", " unless cursor.nil?
       result = query(
         <<~GRAPHQL
           {
             repository(owner: "#{owner}", name: "#{name}") {
-              releases(first: 25, after: "#{cursor}", orderBy: { field: CREATED_AT, direction: DESC }) {
+              releases(#{after}first: 25, orderBy: { field: CREATED_AT, direction: DESC }) {
                 nodes {
                   isDraft
                   publishedAt

data/lib/fbe/if_absent.rb

@@ -62,7 +62,7 @@ def Fbe.if_absent(fb: Fbe.fb, always: false)
       end
     end
   yield(f)
-  q = attrs.except('_id', '_time', '_version').map do |k, v|
+  q = attrs.except(:_id, :_time, :_version).map do |k, v|
     vv = v.to_s
     if v.is_a?(String)
       vv = "'#{vv.gsub('"', '\\\\"').gsub("'", "\\\\'")}'"

data/lib/fbe/iterate.rb

@@ -148,6 +148,24 @@ class Fbe::Iterate
     @repeats = repeats
   end
 
+  # Sets the initial value for iteration tracking.
+  #
+  # This value is used as the starting point when no marker exists in the
+  # factbase for the current iteration label. It is also used as the
+  # restart value when a query returns nil, causing the iteration for
+  # that repository to begin again from this value.
+  #
+  # @param [Integer] value The initial value for iteration tracking
+  # @return [nil] Nothing is returned
+  # @raise [RuntimeError] If value is nil or not an Integer
+  # @example Start iteration from issue number 100
+  #   iterator.since!(100)
+  def since!(value)
+    raise(Fbe::Error, 'Cannot set "since" to nil') if value.nil?
+    raise(Fbe::Error, 'The "since" must be an Integer') unless value.is_a?(Integer)
+    @since = value
+  end
+
   # Sets the query to execute for each iteration.
   #
   # The query can use two special variables:
@@ -254,19 +272,14 @@ class Fbe::Iterate
       ).map { |n| oct.repo_id_by_name(n) }
     started = Time.now
     restarted = []
-    before =
-      repos.to_h do |repo|
-        [
-          repo,
-          @fb.query(
-            "(agg (and
-              (eq what 'iterate')
-              (eq where 'github')
-              (eq repository #{repo}))
-            (first #{@label}))"
-          ).one&.first || @since
-        ]
-      end
+    markers = {}
+    @fb.query("(and (eq what 'iterate') (eq where 'github'))").each do |f|
+      r = f['repository']&.first
+      next if r.nil? || markers.key?(r)
+      v = f[@label.to_s]&.first
+      markers[r] = v unless v.nil?
+    end
+    before = repos.to_h { |repo| [repo, markers[repo] || @since] }
     starts = before.dup
     values = {}
     loop do # rubocop:disable Metrics/BlockLength
@@ -342,7 +355,7 @@ class Fbe::Iterate
        defined?(before) && !before.nil? &&
        defined?(starts) && !starts.nil?
       repos.each do |repo|
-        next if before[repo] == starts[repo] || before[repo] == @since
+        next if before[repo] == starts[repo]
         f =
           Fbe.if_absent(fb: @fb, always: true) do |n|
             n.what = 'iterate'

data/lib/fbe/just_one.rb

@@ -38,13 +38,16 @@ def Fbe.just_one(fb: Fbe.fb)
     others(map: attrs) do |*args|
       k = args[0]
       if k.end_with?('=')
-        @map[k[0..-2].to_sym] = args[1]
+        v = args[1]
+        raise(Fbe::Error, "Can't set #{k[0..-2]} to nil") if v.nil?
+        raise(Fbe::Error, "Can't set #{k[0..-2]} to empty string") if v.is_a?(String) && v.empty?
+        @map[k[0..-2].to_sym] = v
       else
         @map[k.to_sym]
       end
     end
   yield(f)
-  q = attrs.except('_id', '_time', '_version').map do |k, v|
+  q = attrs.except(:_id, :_time, :_version).map do |k, v|
     vv = v.to_s
     if v.is_a?(String)
       vv = "'#{vv.gsub('"', '\\\\"').gsub("'", "\\\\'")}'"

data/lib/fbe/kill_if.rb

@@ -19,5 +19,6 @@ def Fbe.kill_if(facts, fb: Fbe.fb, fid: '_id')
     end
     ids << f[fid].first
   end
+  return 0 if ids.empty?
   fb.query("(or #{ids.map { |id| "(eq #{fid} #{id})" }.join})").delete!
 end

data/lib/fbe/middleware/formatter.rb

@@ -68,17 +68,18 @@ class Fbe::Middleware::Formatter < Faraday::Logging::Formatter
   # @note Special handling for 403 JSON responses to show compact error message
   def response(http) # rubocop:disable Metrics/AbcSize
     return if http.status < 400
-    if http.status == 403 && http.response_headers['content-type'].start_with?('application/json')
-      warn(
-        [
-          "#{@req.method.upcase} #{apply_filters(@req.url.to_s)}",
-          '->',
-          http.status,
-          '/',
-          JSON.parse(http.response_body)['message']
-        ].join(' ')
-      )
-      return
+    if http.status == 403 && http.response_headers['content-type']&.start_with?('application/json')
+      msg =
+        begin
+          parsed = JSON.parse(http.response_body)
+          parsed['message'] if parsed.is_a?(Hash)
+        rescue JSON::ParserError, TypeError
+          nil
+        end
+      unless msg.nil?
+        warn(["#{@req.method.upcase} #{apply_filters(@req.url.to_s)}", '->', http.status, '/', msg].join(' '))
+        return
+      end
     end
     if http.status >= 500 && http.response_headers['content-type']&.start_with?('text')
       error(

data/lib/fbe/middleware/rate_limit.rb

@@ -27,11 +27,14 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
   # Initializes the rate limit middleware.
   #
   # @param [Object] app The next middleware in the stack
-  def initialize(app)
-    super
+  def initialize(app, tracker = nil)
+    super(app)
     @cached = nil
     @remaining = nil
+    @searchleft = nil
     @counter = 0
+    @lock = Mutex.new
+    tracker[:rate_limit] = self unless tracker.nil?
   end
 
   # Processes the HTTP request and handles rate limit caching.
@@ -40,10 +43,22 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
   # @return [Faraday::Response] The response from cache or the next middleware
   def call(env)
     if env.url.path == '/rate_limit'
-      handle_rate_limit_request(env)
+      @lock.synchronize { handle_rate_limit_request(env) }
     else
-      track_request
-      @app.call(env)
+      @lock.synchronize { track_request(env.url.path) }
+      @app.call(env).on_complete do |response_env|
+        @lock.synchronize { sync(response_env, env.url.path) }
+      end
+    end
+  end
+
+  # Returns the remaining requests count tracked by this middleware.
+  #
+  # @param [Symbol] resource The GitHub API resource (:core or :search)
+  # @return [Integer, nil] The remaining count, or nil when the resource is absent
+  def remaining(resource = :core)
+    @lock.synchronize do
+      resource == :search ? @searchleft : @remaining
     end
   end
 
@@ -56,22 +71,48 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
   def handle_rate_limit_request(env)
     if @cached.nil? || @counter >= 100
       response = @app.call(env)
-      @cached = response.dup
+      @cached = response
       @remaining = extract_remaining_count(response)
+      @searchleft = extract_search_remaining_count(response)
       @counter = 0
       response
     else
-      response = @cached.dup
-      update_remaining_count(response)
-      Faraday::Response.new(response_env(env, response))
+      Faraday::Response.new(response_env(env, @cached))
     end
   end
 
   # Tracks non-rate_limit requests and decrements counter.
-  def track_request
-    return if @remaining.nil?
-    @remaining -= 1 if @remaining.positive?
+  def track_request(path = nil)
     @counter += 1
+    if path&.start_with?('/search/')
+      @searchleft -= 1 if @searchleft&.positive?
+    elsif @remaining&.positive?
+      @remaining -= 1
+    end
+  end
+
+  # Syncs the internal remaining count from a real API response header.
+  #
+  # When the response was served by Faraday::HttpCache from cache
+  # (indicated by +http_cache_trace+ containing +:fresh+), the
+  # +x-ratelimit-remaining+ header is stale, so we keep our
+  # decremented count. When the API was actually contacted,
+  # we seed unknown counters from headers, but avoid raising
+  # a counter already decremented by this middleware.
+  #
+  # @param [Faraday::Env] response_env The response environment
+  def sync(response_env, path = nil)
+    return if response_env[:http_cache_trace]&.include?(:fresh)
+    headers = response_env.response_headers
+    return unless headers
+    remaining = headers['x-ratelimit-remaining']
+    return unless remaining
+    count = Integer(remaining)
+    if path&.start_with?('/search/')
+      @searchleft = @searchleft.nil? ? count : [@searchleft, count].min
+    else
+      @remaining = @remaining.nil? ? count : [@remaining, count].min
+    end
   end
 
   # Extracts the remaining count from the response body.
@@ -80,52 +121,59 @@ class Fbe::Middleware::RateLimit < Faraday::Middleware
   # @return [Integer] The remaining requests count
   def extract_remaining_count(response)
     body = response.body
-    if body.is_a?(String)
-      begin
-        body = JSON.parse(body)
-      rescue JSON::ParserError
-        return 0
-      end
-    end
-    return 0 unless body.is_a?(Hash)
-    body.dig('rate', 'remaining') || 0
+    body = JSON.parse(body) if body.is_a?(String)
+    value = body.dig('rate', 'remaining') if body.is_a?(Hash)
+    value ||= response.headers['x-ratelimit-remaining']
+    Integer(value || 0)
   end
 
-  # Updates the remaining count in the response body.
+  # Extracts the search-resource remaining count from the response body.
   #
-  # @param [Faraday::Response] response The cached response to update
-  def update_remaining_count(response)
+  # @param [Faraday::Response] response The API response
+  # @return [Integer, nil] The remaining search-API requests count
+  def extract_search_remaining_count(response)
     body = response.body
-    stringed = body.is_a?(String)
-    if stringed
-      begin
-        body = JSON.parse(body)
-      rescue JSON::ParserError
-        return
+    body = JSON.parse(body) if body.is_a?(String)
+    return nil unless body.is_a?(Hash)
+    value = body.dig('resources', 'search', 'remaining')
+    value.nil? ? nil : Integer(value)
+  end
+
+  # Builds a fresh body with the current remaining counts written in,
+  # without mutating the cached response. Uses a JSON round-trip for
+  # the deep copy so we only handle JSON-shaped data.
+  #
+  # @param [Object] original The cached response body (Hash or JSON String)
+  # @return [Object] A new body of the same type with remaining counts updated
+  def patched_body(original)
+    stringed = original.is_a?(String)
+    body =
+      if stringed
+        JSON.parse(original)
+      elsif original.is_a?(Hash)
+        JSON.parse(original.to_json)
+      else
+        return original
       end
-    end
-    return unless body.is_a?(Hash) && body['rate']
-    body['rate']['remaining'] = @remaining
-    return unless stringed
-    response.instance_variable_set(:@body, body.to_json)
+    body['rate']['remaining'] = @remaining if body['rate'] && !@remaining.nil?
+    body.dig('resources', 'core')&.[]=('remaining', @remaining) unless @remaining.nil?
+    body.dig('resources', 'search')&.[]=('remaining', @searchleft) unless @searchleft.nil?
+    stringed ? body.to_json : body
   end
 
-  # Creates a response environment for the cached response.
+  # Builds a response environment that mirrors the cached response,
+  # preserving Faraday::Env invariants by dup-ing the original env
+  # and only overriding body and rate-limit headers.
   #
   # @param [Faraday::Env] env The original request environment
   # @param [Faraday::Response] response The cached response
-  # @return [Hash] Response environment hash
+  # @return [Faraday::Env] Response env ready to wrap in Faraday::Response
   def response_env(env, response)
-    headers = response.headers.dup
-    headers['x-ratelimit-remaining'] = @remaining.to_s if @remaining
-    {
-      method: env.method,
-      url: env.url,
-      request_headers: env.request_headers,
-      request_body: env.request_body,
-      status: response.status,
-      response_headers: headers,
-      body: response.body
-    }
+    served = response.env.dup
+    served.request_headers = env.request_headers
+    served.body = patched_body(response.body)
+    served.response_headers = response.headers.dup
+    served.response_headers['x-ratelimit-remaining'] = @remaining.to_s unless @remaining.nil?
+    served
   end
 end

data/lib/fbe/middleware/sqlite_store.rb

@@ -86,8 +86,8 @@ class Fbe::Middleware::SqliteStore
     return unless value
     begin
       JSON.parse(Zlib::Inflate.inflate(value))
-    rescue Zlib::Error => e
-      @loog.info("Failed to decompress cached value for key: #{key}, error: #{e.message}, the key will be deleted")
+    rescue Zlib::Error, JSON::ParserError, TypeError => e
+      @loog.info("Failed to decode cached value for key: #{key}, error: #{e.message}, the key will be deleted")
       delete(key)
     end
   end
@@ -106,7 +106,7 @@ class Fbe::Middleware::SqliteStore
   # @return [nil]
   # @note Values larger than 10KB are not cached
   # @note Non-GET requests and URLs with query parameters are not cached
-  def write(key, value) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+  def write(key, value) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/AbcSize
     return if value.is_a?(Array) && value.any? do |vv|
       req = JSON.parse(vv[0])
       req['method'] != 'get'
@@ -129,6 +129,8 @@ class Fbe::Middleware::SqliteStore
           end
         end
         resp['response_headers']['cache-control'] = control
+        value = value.dup
+        value[0] = value[0].dup
         value[0][1] = JSON.dump(resp)
       end
     end
@@ -159,6 +161,13 @@ class Fbe::Middleware::SqliteStore
     perform { _1.execute('SELECT key, value FROM cache') }
   end
 
+  # Close the database connection explicitly.
+  # @return [nil]
+  def close
+    @db&.close
+    @db = nil
+  end
+
   private
 
   def perform(&) # rubocop:disable Metrics/AbcSize
@@ -248,7 +257,6 @@ class Fbe::Middleware::SqliteStore
             "new file size: #{Filesize.from(File.size(@path).to_s).pretty} bytes"
           )
         end
-        at_exit { @db&.close }
       end
     @db.transaction(&)
   end