permission_forge 0.14.1

data/lib/permission_forge/cache/base.rb

@@ -0,0 +1,200 @@
+module PermissionForge
+  # The permission cache backends (Base and its subclasses, config.cache_backend) and the
+  # machinery that invalidates and rebuilds cached permission hashes. The backends are public;
+  # each class says which it is (a public namespace does not make its children public).
+  #
+  # @api public
+  module Cache
+    # The interface a cache backend implements (config.cache_backend accepts a subclass):
+    # read/write/delete/clear, plus what the gem builds on them — #read_fresh, #expunge, the
+    # multi-key calls and the removal of permission entries by pattern (#clear_entries).
+    #
+    # @api public
+    class Base
+      # @return [String] put in front of every key, with a colon (config.cache_key_prefix)
+      attr_reader :prefix
+
+      def initialize(prefix: 'pf')
+        @prefix = prefix
+      end
+
+      def read(key)
+        raise NotImplementedError
+      end
+
+      # Reads `key` from the store itself. A store that keeps a copy of what a
+      # request has read (ActiveSupport's local cache) answers a repeated read
+      # from that copy — also a miss; a value another process may have written
+      # in the meantime has to be read past it. Backends without such a copy
+      # read as usual.
+      def read_fresh(key)
+        read(key)
+      end
+
+      # @return [Object] something truthy when the value was written; false or
+      #   nil when the store reports that it was not (ActiveSupport stores
+      #   answer that instead of raising when they are unreachable)
+      def write(key, value, ttl: nil)
+        raise NotImplementedError
+      end
+
+      # Whether every process has a store of its own (an in-memory store):
+      # clearing or writing it here reaches no other process.
+      #
+      # @return [Boolean]
+      def process_local?
+        false
+      end
+
+      # Writes an entry that never expires — not even under a default expiry
+      # the store applies to every entry (a Rails cache configured with
+      # expires_in). Here a write without a TTL, which is that already.
+      #
+      # @return [Object] as #write
+      def write_permanent(key, value)
+        write(key, value)
+      end
+
+      def delete(key)
+        raise NotImplementedError
+      end
+
+      # Removes `key` and says whether it is certainly gone. `delete` alone
+      # cannot say that for a store that swallows its errors: it answers false
+      # both for "there was nothing to delete" and for "the delete failed"
+      # (RailsCache#expunge deals with that). A backend that can only fail by
+      # raising is taken at its word unless its delete says false.
+      #
+      # @return [Boolean] false when the store could not confirm the removal
+      # rubocop:disable-next Naming/PredicateMethod -- removes the key; the answer is whether the store confirmed it
+      def expunge(key)
+        delete(key) != false
+      end
+
+      def clear
+        raise NotImplementedError
+      end
+
+      # Removes every permission entry of one key namespace — the base
+      # entries of every user and tenant, and their variants
+      # (UserKeys.entry_patterns) — where the store can match keys
+      # (#purge_entries), and says whether it did. Verified, not taken on
+      # trust: an entry of each key shape (UserKeys.canary_keys) is written
+      # before and read back after; a pattern that did not reach the store's
+      # keys (a store namespace the pattern does not know) leaves them, and
+      # the answer is false. False also when the backend cannot match keys,
+      # or the canaries could not be written: the caller enumerates the
+      # users instead (Services::PermissionCacheClearer).
+      #
+      # @param namespace [String] one of UserKeys.namespaces
+      # @return [Boolean]
+      def clear_entries(namespace)
+        canaries = UserKeys.canary_keys(namespace)
+        return false unless canaries.all? { |key| write(key, CANARY_ENTRY, ttl: CANARY_TTL) }
+        return false unless purge_entries(namespace)
+
+        canaries.none? { |key| read_fresh(key) }
+      ensure
+        canaries&.each { |key| delete(key) }
+      end
+
+      # What a canary entry holds, and how long it may live if left behind.
+      CANARY_ENTRY = { canary: true }.freeze
+      CANARY_TTL = 60
+
+      # Removes the entries matching UserKeys.entry_patterns in one call per
+      # pattern, where the backend can. Here it cannot.
+      #
+      # @return [Boolean] whether the backend could
+      # rubocop:disable-next Naming/PredicateMethod -- removes the entries; the answer is whether the store could
+      def purge_entries(_namespace)
+        false
+      end
+
+      # Writes several entries and says, per key, whether the store took it
+      # (#write). Here one call per key; a backend that can send them in one
+      # round trip overrides it (RailsCache). Every key is tried: one whose
+      # write raises answers false, and the error is given to the block.
+      #
+      # @param entries [Hash{String => Object}]
+      # @param ttl [Integer, nil] seconds, for every entry
+      # @yieldparam error [StandardError] what a write raised
+      # @return [Hash{String => Boolean}]
+      def write_many(entries, ttl: nil, &on_error)
+        entries.to_h { |key, value| [key, answer(on_error) { write(key, value, ttl: ttl) }] }
+      end
+
+      # Removes several keys and says, per key, whether it is certainly gone
+      # (#expunge). Here one call per key; a backend that can send them in
+      # one round trip overrides it (RailsCache). Every key is tried: one
+      # whose removal raises answers false, and the error is given to the
+      # block.
+      #
+      # @param keys [Array<String>]
+      # @yieldparam error [StandardError] what a removal raised
+      # @return [Hash{String => Boolean}]
+      def expunge_many(keys, &on_error)
+        keys.to_h { |key| [key, answer(on_error) { expunge(key) }] }
+      end
+
+      # Reads several keys from the store itself (#read_fresh) and answers,
+      # per key, the entry or nil. Here one read per key; a backend that can
+      # fetch them in one round trip overrides it (RailsCache). A key whose
+      # read raises answers nil.
+      #
+      # @param keys [Array<String>]
+      # @return [Hash{String => Object, nil}]
+      def read_many(keys)
+        keys.to_h { |key| [key, fresh_or_nil(key)] }
+      end
+
+      # Removes several keys without asking whether they are gone (#delete):
+      # for an invalidation that has failed already (its stamp was refused),
+      # where a confirmation would be paid for twice. Here one call per key;
+      # RailsCache sends them in one round trip where it can. Every key is
+      # tried; an error is given to the block.
+      #
+      # @param keys [Array<String>]
+      # @yieldparam error [StandardError] what a removal raised
+      # @return [void]
+      def delete_many(keys, &on_error)
+        keys.each { |key| answer(on_error) { delete(key) } }
+        nil
+      end
+
+      private
+
+      def fresh_or_nil(key)
+        read_fresh(key)
+      rescue StandardError
+        nil
+      end
+
+      # The store's answer as true or false; false, after the error is
+      # handed on, when the call raised.
+      def answer(on_error)
+        yield ? true : false
+      rescue StandardError => e
+        on_error&.call(e)
+        false
+      end
+
+      def prefixed_key(key)
+        "#{prefix}:#{key}"
+      end
+
+      # Stores hand back string keys (JSON) or the hash as written; the
+      # permission hash is read with symbol keys.
+      def deep_symbolize(obj)
+        case obj
+        when Hash
+          obj.to_h { |k, v| [k.to_sym, deep_symbolize(v)] }
+        when Array
+          obj.map { |v| deep_symbolize(v) }
+        else
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/permission_forge/cache/bulk_invalidation.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module PermissionForge
+  module Cache
+    # Many invalidations at once: what Invalidation does for one — a fresh
+    # stamp, then the keys, every answer checked, ATTEMPTS tries, what still
+    # fails reported (Failure) and owed (Backlog) — with one multi-key call
+    # per step (Base#write_many, Base#expunge_many) instead of one call per
+    # key. On a store that sends them together (Redis, Solid Cache) a slice
+    # of users costs two round trips instead of three to five per user.
+    #
+    # The order holds for every user: all the stamps of the call are written
+    # before any key is removed, so each user's stamp is there before that
+    # user's keys go (see Invalidation). The keys are removed even when the
+    # stamps could not be written — then plainly deleted, not confirmed
+    # (Invalidation#remove): that user has failed already, and on a store
+    # that is down a confirmation is one more failing call per key. An
+    # attempt only repeats the users the store did not confirm.
+    #
+    # Callers keep the size of one call bounded (HolderInvalidation::SLICE).
+    #
+    # @api private
+    class BulkInvalidation
+      def self.call(...) = new(...).call
+
+      # The ids of the users whose invalidation the store did not confirm.
+      # A call that raises as a whole is owed and reported user by user.
+      #
+      # @param targets [Array<Hash>] as for #initialize
+      # @return [Array]
+      def self.unconfirmed(targets)
+        call(targets).map { |target| target[:user_id] }
+      rescue StandardError => e
+        targets.each do |target|
+          Invalidation.owe(**target)
+          Failure.report(operation: :invalidate, **target.slice(:user_id, :keys), error: e)
+        end
+        targets.map { |target| target[:user_id] }
+      end
+
+      # @param targets [Array<Hash>] one per user, the arguments of an
+      #   Invalidation: user_id:, stamp_key:, keys:
+      def initialize(targets, cache: PermissionForge.cache)
+        @targets = targets
+        @cache = cache
+        @error = nil
+      end
+
+      # @return [Array<Hash>] the targets the store did not confirm (empty
+      #   when it confirmed every one)
+      def call
+        attempts = Invalidation.store_failing? ? 1 : Invalidation::ATTEMPTS
+        pending = @targets
+        attempts.times do
+          break if pending.empty?
+
+          pending = attempt(pending)
+        end
+        failed(pending, attempts) if pending.any?
+        # as Invalidation does user by user: once one is confirmed, the store
+        # answers, and what is owed (the failures just now included) is tried
+        succeeded if pending.size < @targets.size
+        pending
+      end
+
+      private
+
+      # @return [Array<Hash>] the targets still unconfirmed after this attempt
+      def attempt(targets)
+        stamped = answers { @cache.write_many(fresh_stamps(targets), ttl: PermissionForge.configuration.cache_ttl, &method(:keep_error)) }
+        confirming, failed = targets.partition { |target| stamped[target[:stamp_key]] }
+        # the keys are removed whatever became of the stamps
+        delete_keys(failed)
+        removed = expunge_keys(confirming)
+        targets.reject { |target| confirmed?(target, stamped, removed) }
+      end
+
+      # Users whose stamp was refused: deleted, no confirmation asked.
+      def delete_keys(targets)
+        answers { @cache.delete_many(keys_of(targets), &method(:keep_error)) } if targets.any?
+      end
+
+      # @return [Hash{String => Boolean}] whether each key is certainly gone
+      def expunge_keys(targets)
+        return {} if targets.empty?
+
+        answers { @cache.expunge_many(keys_of(targets), &method(:keep_error)) }
+      end
+
+      def keys_of(targets)
+        targets.flat_map { |target| target[:keys] }.uniq
+      end
+
+      # A backend whose multi-key call raises as a whole confirmed nothing.
+      def answers
+        yield
+      rescue StandardError => e
+        keep_error(e)
+        {}
+      end
+
+      # The user's stamp was written and every one of its keys removed.
+      def confirmed?(target, stamped, removed)
+        stamped[target[:stamp_key]] && target[:keys].all? { |key| removed[key] }
+      end
+
+      def fresh_stamps(targets)
+        targets.to_h { |target| [target[:stamp_key], SecureRandom.hex(8)] }
+      end
+
+      # The last error a store call raised, for the failure report.
+      def keep_error(error)
+        @error = error
+      end
+
+      # The store answers: what is owed from before can be made now.
+      def succeeded
+        Invalidation.store_answered!
+        Invalidation.retry_owed
+      end
+
+      def failed(targets, attempts)
+        Invalidation.store_failed!
+        targets.each do |target|
+          Invalidation.owe(**target)
+          Failure.report(operation: :invalidate, **target.slice(:user_id, :keys), error: @error, attempts: attempts)
+        end
+      end
+    end
+  end
+end

data/lib/permission_forge/cache/failure.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module PermissionForge
+  module Cache
+    # Makes a cache failure visible without raising: an invalidation the store
+    # did not confirm, an entry that could not be removed, or an error inside
+    # an after_commit invalidation. Raising there would make Rails skip the
+    # commit callbacks of the remaining records, so the failure is
+    #
+    #   - logged at error level,
+    #   - published as the `cache_failure.permission_forge` notification, and
+    #   - handed to `config.on_cache_failure`.
+    #
+    # A subscriber or a hook that raises is logged and does not stop the other.
+    #
+    # What a failure means: a cached permission hash that should be gone may
+    # still be in the store, and stays there until `cache_ttl`.
+    #
+    # Code that has to say whether an invalidation went through asks here
+    # (.none?), not the method it called: a host app overrides
+    # User#invalidate_permissions, and what such an override returns is its
+    # own business.
+    #
+    # @api private
+    module Failure
+      # The failures reported inside a Failure.collect block.
+      #
+      # @api private
+      class Collected
+        # Payloads kept per collect. A collect abandoned in an Enumerator
+        # stays open (the end of a run keeps the open collects), so what it
+        # keeps is bounded; past the bound it confirms nobody (fails closed).
+        MAX_PAYLOADS = 1_000
+
+        def initialize
+          @payloads = []
+          @overflowed = false
+        end
+
+        def <<(payload)
+          @payloads.size < MAX_PAYLOADS ? @payloads << payload : @overflowed = true
+          self
+        end
+
+        # @return [Array<Hash>] the payloads, in order
+        def to_a
+          @payloads.dup
+        end
+
+        # Those of `user_ids` that no failure was about — none when one was
+        # about nobody in particular (it may be about any of them).
+        #
+        # @param user_ids [Array]
+        # @return [Array]
+        def confirmed(user_ids)
+          return [] if @overflowed || failed_user_ids.include?(nil)
+
+          user_ids.reject { |user_id| failed_user_ids.include?(user_id.to_s) }
+        end
+
+        private
+
+        # As Strings (a key may come back as either); nil for a failure
+        # about nobody in particular.
+        def failed_user_ids
+          @payloads.map { |payload| payload[:user_id]&.to_s }
+        end
+      end
+
+      EVENT = 'cache_failure.permission_forge'
+      # ExecutionState slots: how many failures were reported on this thread
+      # (fiber under Rails' fiber isolation), and the open collect blocks.
+      COUNT_KEY = :pf_cache_failures
+      COLLECT_KEY = :pf_cache_failure_collectors
+      # Open collects kept per thread: more can only be abandoned ones (an
+      # Enumerator paused inside a collect), so the oldest go first.
+      MAX_OPEN_COLLECTS = 64
+
+      class << self
+        # Runs the block and says whether it got through without a failure
+        # being reported on this thread: every invalidation made inside it was
+        # confirmed by the store.
+        #
+        # @return [Boolean]
+        def none?
+          before = count
+          yield
+          count == before
+        end
+
+        # Runs the block and returns what was reported on this thread inside it.
+        # The open collects are one list per thread (fiber under fiber
+        # isolation): each one takes out its own Collected when it ends, so
+        # collects that interleave (one paused in an Enumerator) never take
+        # each other's.
+        #
+        # @return [Collected]
+        def collect
+          collected = Collected.new
+          open = ExecutionState.fetch(COLLECT_KEY) { [] }
+          open.shift while open.size >= MAX_OPEN_COLLECTS
+          open.push(collected)
+          yield
+          collected
+        ensure
+          ExecutionState[COLLECT_KEY]&.delete_if { |open| open.equal?(collected) }
+        end
+
+        # @param operation [Symbol] :invalidate, :discard or :after_commit
+        # @param details [Hash] user_id: whose cached permissions are concerned,
+        #   keys: cache keys that may still hold an entry, error: what the store
+        #   (or the callback) raised — and anything else worth knowing
+        #   (record:, attempts:)
+        # @return [void]
+        def report(operation:, **details)
+          payload = { operation: operation, user_id: nil, keys: [], error: nil, **details }
+
+          ExecutionState[COUNT_KEY] = count + 1
+          ExecutionState[COLLECT_KEY]&.each { |collected| collected << payload }
+          log(payload)
+          notify(payload)
+          call_hook(payload)
+        end
+
+        private
+
+        def count
+          ExecutionState[COUNT_KEY] || 0
+        end
+
+        def log(payload)
+          return unless defined?(Rails) && Rails.logger
+
+          Rails.logger.error("[PermissionForge] #{message(payload)} — cached permissions may be out of date until cache_ttl")
+        end
+
+        def message(payload)
+          cause = payload[:error]
+          parts = ["cache #{payload[:operation]} failed"]
+          parts << "for user #{payload[:user_id]}" if payload[:user_id]
+          parts << "(keys: #{payload[:keys].join(', ')})" if payload[:keys].any?
+          parts << "— #{cause.class}: #{cause.message}" if cause
+          parts.join(' ')
+        end
+
+        # A subscriber that raises would abort the commit callbacks this is
+        # reported from, like a hook that raises.
+        def notify(payload)
+          ActiveSupport::Notifications.instrument(EVENT, payload)
+        rescue StandardError => e
+          log_error('subscriber', e)
+        end
+
+        def call_hook(payload)
+          hook = PermissionForge.configuration.on_cache_failure
+          return unless hook
+
+          hook.call(**payload)
+        rescue StandardError => e
+          log_error('on_cache_failure hook', e)
+        end
+
+        def log_error(source, error)
+          return unless defined?(Rails) && Rails.logger
+
+          Rails.logger.error("[PermissionForge] #{EVENT} #{source} error: #{error.class}: #{error.message}")
+        end
+      end
+    end
+  end
+end

data/lib/permission_forge/cache/handled_errors.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module PermissionForge
+  module Cache
+    # The errors a store rescued during a call and handed to its error
+    # handler. Rails' Redis store rescues a Redis that cannot be reached,
+    # hands the error to its handler (the `error_handler:` option) and goes
+    # on as if nothing had been asked of it. Its answer to #write_multi is
+    # then no proof: Rails 7.1 to 8.1 answer nil, the store Rails 8.2 ships
+    # answers the same Hash whether the batch was written or not. The call
+    # of the handler is the one sign every version gives, so
+    # RailsCache#write_multi asks here whether there was one.
+    #
+    # The first time a store is watched, the handler it keeps (its
+    # @error_handler) is wrapped: the wrapper notes the error for the call
+    # being watched in this execution context, then calls the store's own
+    # handler (if it has one) with the same arguments, as the store would
+    # have. Outside a watched call it only hands the error on. A store
+    # without an @error_handler is not watched (nothing is noted).
+    #
+    # @api private
+    module HandledErrors
+      # ExecutionState slot: the errors noted for the watched call in progress.
+      KEY = :pf_store_errors
+      LOCK = Mutex.new
+      private_constant :LOCK
+
+      class << self
+        # Runs the block (a call of `store`) and returns its value with the
+        # errors the store handed its error handler meanwhile, in this
+        # execution context. An entry is nil when the handler was given no
+        # exception: a call is still noted.
+        #
+        # @return [Array(Object, Array<Exception, nil>)]
+        def during(store, &)
+          return [yield, []] unless store.instance_variable_defined?(:@error_handler)
+
+          wrap(store)
+          handled = []
+          [ExecutionState.with(KEY, handled, &), handled]
+        end
+
+        # Runs the block (a write to `store`) and says whether it was written:
+        # the store answered truthy and handed its error handler no error
+        # meanwhile (Rails 8.2's Redis store answers the same either way).
+        # Also returns the last error handed on, if any.
+        #
+        # @return [Array(Boolean, Exception, nil)]
+        def written(store, &)
+          answer, handled = during(store, &)
+          [answer ? handled.empty? : false, handled.compact.last]
+        end
+
+        private
+
+        # Puts a Handler in front of the store's own error handler, once.
+        def wrap(store)
+          return if store.instance_variable_get(:@error_handler).is_a?(Handler)
+
+          LOCK.synchronize do
+            handler = store.instance_variable_get(:@error_handler)
+            store.instance_variable_set(:@error_handler, Handler.new(handler)) unless handler.is_a?(Handler)
+          end
+        end
+      end
+
+      # Stands in for a store's error handler.
+      class Handler
+        def initialize(handler)
+          @handler = handler
+        end
+
+        # Notes the error for the watched call (if one is in progress), then
+        # hands everything to the store's own handler.
+        def call(*, **details, &)
+          ExecutionState[KEY]&.push(details[:exception])
+          @handler&.call(*, **details, &)
+        end
+      end
+    end
+  end
+end

data/lib/permission_forge/cache/holder_invalidation.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module PermissionForge
+  module Cache
+    # Invalidates the cached hashes of the holders of a changed permission
+    # record by user id, in slices of SLICE users: the same stamps and keys
+    # as User#invalidate_permissions (Cache::UserKeys), made through
+    # BulkInvalidation — one multi-key write for the stamps and one for the
+    # keys per slice — without loading a single user. A change to a set with
+    # 10,000 holders cost 30,000 store calls and 20,000 loaded records in the
+    # saving request before.
+    #
+    # Only where nothing but those store calls would happen for a user
+    # (.by_id?). Otherwise the holders are loaded and invalidated one by one
+    # through the public User#invalidate_permissions, as before
+    # (Concerns::InvalidatesInBatch):
+    #   - the user model overrides invalidate_permissions, or the key or
+    #     stamp it is built from (PermissionCacheAccessor): the host's code
+    #     has to run;
+    #   - somebody listens for permissions.invalidated (or stores it): each
+    #     event carries its user;
+    #   - the hashes are withheld in an open transaction that reads from a
+    #     snapshot (OpenTransaction.withholding): each invalidation is then
+    #     settled after the commit by the user's own instance.
+    #
+    # The processes' memos are retired as invalidate_permissions retires
+    # them: the cache version is bumped once per slice. Nothing is read: the
+    # per-context variants of a user with conditional sets are left to their
+    # TTL, unreachable once the base entry is gone (Cache::UserKeys).
+    #
+    # @api private
+    class HolderInvalidation
+      SLICE = 500
+      EVENT = 'permissions.invalidated'
+
+      # The methods of PermissionCacheAccessor whose work this does in bulk.
+      OWN_METHODS = %i[invalidate_permissions permission_cache_key pf_invalidation_stamp_key].freeze
+
+      class << self
+        def call(...) = new(...).call
+
+        # Whether the holders can be invalidated by id, here and now.
+        #
+        # @param model [Class] the host's user model
+        # @return [Boolean]
+        def by_id?(model)
+          own_methods?(model) && !PermissionForge::Audit.instrument?(EVENT) && !PermissionForge::OpenTransaction.withholding
+        end
+
+        private
+
+        # The model and its subclasses (an STI `Admin < User` that overrides
+        # them would be loaded as an Admin by the per-user path). A subclass
+        # that is not loaded yet (development) is not seen.
+        def own_methods?(model)
+          [model, *model.descendants].all? do |klass|
+            OWN_METHODS.all? do |name|
+              (klass.method_defined?(name) || klass.private_method_defined?(name)) &&
+                klass.instance_method(name).owner == PermissionForge::Concerns::PermissionCacheAccessor
+            end
+          end
+        end
+      end
+
+      # @param user_ids [Array] the holders' ids (no duplicates)
+      # @param tenant_id [Object, nil] the changed record's tenant
+      def initialize(user_ids:, tenant_id:)
+        @user_ids = user_ids
+        @tenant_id = tenant_id
+      end
+
+      # @return [Array] the ids whose invalidation the store did not confirm
+      #   (each reported through Cache::Failure, and owed)
+      def call
+        @user_ids.each_slice(SLICE).flat_map do |slice|
+          # a slice whose invalidation raised is owed and reported user by user, and the next slices still get theirs
+          BulkInvalidation.unconfirmed(UserKeys.invalidations(slice, tenant_id: @tenant_id))
+        ensure
+          # after the invalidation, whatever became of it, as invalidate_permissions does
+          PermissionForge.bump_permissions_cache_version!
+        end
+      end
+    end
+  end
+end