provenance 2.0.0

data/lib/provenance/json_safe.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+
+module Provenance
+  # Converts arbitrary Ruby values into JSON-safe primitives.
+  module JsonSafe
+    module_function
+
+    # @param value [Object]
+    # @return [Object] a String, Numeric, true, false, nil, Array or Hash with String keys
+    def call(value)
+      case value
+      when nil, true, false, Integer then value
+      when Float then value.finite? ? value : value.to_s
+      when BigDecimal then value.to_s("F")
+      when String then string(value)
+      when Symbol then value.to_s
+      when Hash then value.each_with_object({}) { |(k, v), h| h[k.to_s] = call(v) }
+      when Array, Set then value.map { |v| call(v) }
+      when Time, DateTime, ActiveSupport::TimeWithZone then value.utc.iso8601(3)
+      when Date then value.iso8601
+      when ActiveSupport::Duration then value.iso8601
+      when Numeric then value.to_s
+      else value.respond_to?(:as_json) ? call(value.as_json) : value.to_s
+      end
+    end
+
+    # @param value [String]
+    # @return [String]
+    def string(value)
+      if value.encoding == Encoding::BINARY && !(value.ascii_only? && value.valid_encoding?)
+        "[BINARY #{value.bytesize} bytes]"
+      elsif value.valid_encoding?
+        value
+      else
+        value.dup.force_encoding(Encoding::UTF_8).scrub
+      end
+    end
+  end
+end

data/lib/provenance/middleware.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Provenance
+  # Rack middleware that opens an +http+ action per request and closes it once the
+  # application has produced the response.
+  class Middleware
+    # @param app [#call]
+    def initialize(app)
+      @app = app
+    end
+
+    # @param env [Hash] Rack env
+    # @return [Array] Rack response
+    def call(env)
+      return @app.call(env) if !Provenance.enabled? || Provenance.current
+
+      action = Action.new(kind: :http, name: nil, context: Context.new(env: env))
+      Provenance.within(action) do
+        response = @app.call(env)
+        action.request["status"] = response[0].to_i
+        response
+      ensure
+        describe(action, env)
+      end
+    end
+
+    private
+
+    def describe(action, env)
+      request = ActionDispatch::Request.new(env)
+      action.request.merge!(
+        "id" => env["action_dispatch.request_id"],
+        "method" => request.request_method,
+        "path" => request.path,
+        "ip" => request.remote_ip,
+        "user_agent" => request.user_agent
+      )
+      action.request["status"] ||= nil
+      action.name ||= "#{request.request_method} #{request.path}"
+    rescue => e
+      Provenance.config.report_error(e)
+    end
+  end
+end

data/lib/provenance/model.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Provenance
+  # Opt-in model tracking: +has_provenance+ and the hooks it installs.
+  module Model
+    # Bulk operations on relations of tracked models.
+    module BulkRelation
+      # @see ActiveRecord::Relation#update_all
+      def update_all(updates)
+        Provenance::Recorder.bulk(self, :bulk_update, updates: updates) { super }
+      end
+
+      # @see ActiveRecord::Relation#delete_all
+      def delete_all
+        Provenance::Recorder.bulk(self, :bulk_delete) { super }
+      end
+
+      # @see ActiveRecord::Relation#insert_all
+      def insert_all(attributes, **options)
+        Provenance::Recorder.bulk_insert(self, attributes) { super }
+      end
+
+      # @see ActiveRecord::Relation#insert_all!
+      def insert_all!(attributes, **options)
+        Provenance::Recorder.bulk_insert(self, attributes) { super }
+      end
+
+      # @see ActiveRecord::Relation#upsert_all
+      def upsert_all(attributes, **options)
+        Provenance::Recorder.bulk_insert(self, attributes) { super }
+      end
+    end
+
+    # Bulk update through an association collection proxy.
+    module BulkCollectionProxy
+      # @see ActiveRecord::Relation#update_all
+      def update_all(updates)
+        Provenance::Recorder.bulk(self, :bulk_update, updates: updates) { super }
+      end
+    end
+
+    # Keeps bulk tracking on STI subclasses defined after +has_provenance+.
+    module Inheritance
+      # @api private
+      def inherited(subclass)
+        super
+        Provenance::Model.install_bulk(subclass)
+      end
+    end
+
+    # Class macros added to ActiveRecord::Base.
+    module ClassMethods
+      # Opts the model in to change tracking.
+      #
+      # @param only [Array<Symbol>] track only these attributes
+      # @param except [Array<Symbol>] track all attributes but these
+      # @param redact [Array<Symbol>] record these attributes as "[REDACTED]"
+      # @param associations [Array<Symbol>] collection associations producing link/unlink changes
+      # @param ignore_if [Proc, nil] +->(record) { ... }+; truthy skips the record
+      # @return [void]
+      def has_provenance(only: nil, except: nil, redact: [], associations: [], ignore_if: nil)
+        raise ArgumentError, "use either only: or except:, not both" if only && except
+
+        options = Registry::Options.new(
+          only: Array(only).map(&:to_s), except: Array(except).map(&:to_s),
+          redact: Array(redact).map(&:to_s), associations: Array(associations).map(&:to_sym),
+          ignore_if: ignore_if
+        )
+        Provenance.registry.register(self, options)
+        Provenance::Model.install(self, options)
+      end
+    end
+
+    module_function
+
+    # @api private
+    def install(model, options)
+      unless model.instance_variable_get(:@provenance_installed)
+        model.instance_variable_set(:@provenance_installed, true)
+        model.after_create { Provenance::Recorder.record(self, :create) }
+        model.after_update { Provenance::Recorder.record(self, :update) }
+        model.after_destroy { Provenance::Recorder.record(self, :destroy) }
+        model.singleton_class.prepend(Inheritance)
+        ([model] + model.descendants).each { |klass| install_bulk(klass) }
+      end
+      installed = model.instance_variable_get(:@provenance_associations) || []
+      (options.associations - installed).each { |name| install_association(model, name) }
+      model.instance_variable_set(:@provenance_associations, installed | options.associations)
+    end
+
+    # @api private
+    def install_bulk(model)
+      prepend_once(model.relation_delegate_class(ActiveRecord::Relation), BulkRelation)
+      prepend_once(model.relation_delegate_class(ActiveRecord::AssociationRelation), BulkRelation)
+      prepend_once(model.relation_delegate_class(ActiveRecord::Associations::CollectionProxy), BulkCollectionProxy)
+    end
+
+    # @api private
+    def prepend_once(klass, mod)
+      klass.prepend(mod) unless klass.ancestors.include?(mod)
+    end
+
+    # Appends link/unlink recorders to the association's after_add/after_remove
+    # callback lists.
+    #
+    # @api private
+    def install_association(model, name)
+      reflection = model.reflect_on_association(name)
+      unless reflection&.collection?
+        raise ArgumentError, "#{model.name} has no collection association #{name.inspect}"
+      end
+
+      {after_add: :link, after_remove: :unlink}.each do |callback, operation|
+        accessor = "#{callback}_for_#{name}"
+        model.class_attribute(accessor, instance_accessor: false, instance_predicate: false) unless model.respond_to?(accessor)
+        recorder = ->(_method, owner, record) { Provenance::Recorder.association(owner, name, record, operation) }
+        model.public_send(:"#{accessor}=", Array(model.public_send(accessor)) + [recorder])
+      end
+    end
+  end
+end
+
+ActiveSupport.on_load(:active_record) { extend Provenance::Model::ClassMethods }

data/lib/provenance/outbox/record.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Outbox
+    # A stored event awaiting or past delivery.
+    class Record < ActiveRecord::Base
+      # Row awaiting delivery.
+      PENDING = "pending"
+      # Row delivered to every sink.
+      DELIVERED = "delivered"
+
+      self.table_name = "provenance_outbox"
+
+      scope :pending, -> { where(status: PENDING) }
+      scope :delivered, -> { where(status: DELIVERED) }
+
+      # @return [Hash] the stored native event
+      def event
+        JSON.parse(payload)
+      end
+    end
+  end
+end

data/lib/provenance/outbox/relay.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Outbox
+    # Drains pending outbox rows in id order and delivers them to the sinks.
+    module Relay
+      # Upper bound for the retry delay, in seconds.
+      MAX_BACKOFF = 3600
+
+      module_function
+
+      # Delivers due rows batch by batch until none are left or a batch fails, then
+      # prunes old delivered rows.
+      #
+      # @param batch_size [Integer]
+      # @return [Integer] number of delivered rows
+      def run(batch_size: Provenance.config.outbox_batch_size)
+        delivered = 0
+        loop do
+          count = Record.transaction { drain(batch_size) }
+          break unless count&.positive?
+
+          delivered += count
+        end
+        prune
+        delivered
+      end
+
+      # Locks and delivers one batch. Stops at the first row that is not due yet so
+      # rows are never delivered out of order.
+      #
+      # @api private
+      # @return [Integer, nil] delivered rows, or nil when the batch failed
+      def drain(batch_size)
+        scope = Record.pending.order(:id).limit(batch_size)
+        scope = scope.lock("FOR UPDATE SKIP LOCKED") if Outbox.postgresql?
+        now = Time.current
+        rows = scope.to_a.take_while { |row| row.next_attempt_at.nil? || row.next_attempt_at <= now }
+        return 0 if rows.empty?
+
+        begin
+          Emitter.deliver(rows.map(&:event), raise_errors: true)
+        rescue => e
+          fail_rows(rows, e, now)
+          return nil
+        end
+        Record.where(id: rows.map(&:id)).update_all(status: Record::DELIVERED, delivered_at: now, last_error: nil)
+        rows.size
+      end
+
+      # @api private
+      def fail_rows(rows, error, now)
+        rows.each do |row|
+          attempts = row.attempts.to_i + 1
+          row.update_columns(
+            attempts: attempts, next_attempt_at: now + backoff(attempts),
+            last_error: "#{error.class}: #{error.message}".truncate(1000)
+          )
+        end
+        Provenance.config.report_error(error)
+      end
+
+      # @param attempts [Integer]
+      # @return [Float] seconds until the next attempt
+      def backoff(attempts)
+        base = [5 * (2**(attempts - 1)), MAX_BACKOFF].min
+        base + rand * base * 0.2
+      end
+
+      # Deletes delivered rows older than +outbox_retention+, keeping the newest row
+      # of each app as the anchor of the integrity chain.
+      #
+      # @return [Integer] deleted rows
+      def prune
+        cutoff = Time.current - Provenance.config.outbox_retention
+        anchors = Record.group(:app).maximum(:id).values
+        Record.delivered.where(delivered_at: ...cutoff).where.not(id: anchors).delete_all
+      end
+    end
+  end
+end

data/lib/provenance/outbox.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Provenance
+  # Reliable delivery: events are stored in the +provenance_outbox+ table and
+  # delivered by {RelayJob}.
+  module Outbox
+    autoload :Record, "provenance/outbox/record"
+    autoload :Relay, "provenance/outbox/relay"
+
+    # Attempts made to allocate a sequence number before giving up.
+    WRITE_ATTEMPTS = 5
+
+    module_function
+
+    # Stores a native event, adding the integrity chain when a key is configured.
+    #
+    # @param event [Hash] native event
+    # @return [Record]
+    def write(event)
+      attempts = 0
+      begin
+        attempts += 1
+        record = Record.transaction(requires_new: true) { insert(event) }
+      rescue ActiveRecord::RecordNotUnique
+        retry if attempts < WRITE_ATTEMPTS
+        raise
+      end
+      RelayJob.perform_later if Provenance.config.outbox_enqueue_relay
+      record
+    end
+
+    # @api private
+    def insert(event)
+      app = event["app"]
+      key = Provenance.config.integrity.key
+      seq = mac = nil
+      if key.present?
+        lock(app)
+        last_seq, prev = Record.where(app: app).where.not(seq: nil).order(seq: :desc).pick(:seq, :mac)
+        seq = last_seq.to_i + 1
+        mac = Integrity.mac(key, prev, event)
+        event = event.merge("integrity" => {"seq" => seq, "prev" => prev, "mac" => mac})
+      end
+      Record.create!(
+        app: app, event_id: event["id"], seq: seq, mac: mac, payload: JSON.generate(event),
+        status: Record::PENDING, attempts: 0, next_attempt_at: Time.current
+      )
+    end
+
+    # Serializes sequence allocation per app on PostgreSQL. Other databases rely on
+    # the unique (app, seq) index and a retry.
+    #
+    # @api private
+    def lock(app)
+      return unless postgresql?
+
+      Record.connection.execute(
+        Record.sanitize_sql_array(["SELECT pg_advisory_xact_lock(hashtext(?))", "provenance:#{app}"])
+      )
+    end
+
+    # @return [Boolean]
+    def postgresql?
+      Record.connection_db_config.adapter.to_s.match?(/postg/i)
+    end
+  end
+end

data/lib/provenance/railtie.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Provenance
+  # Wires Provenance into a Rails application.
+  class Railtie < Rails::Railtie
+    initializer "provenance.middleware" do |app|
+      app.middleware.insert_after ActionDispatch::RequestId, Provenance::Middleware
+    end
+
+    initializer "provenance.action_controller" do
+      ActiveSupport.on_load(:action_controller) { include Provenance::Controller }
+      ActiveSupport::Notifications.subscribe("process_action.action_controller") do |*, payload|
+        Provenance::Controller::Subscriber.call(payload)
+      end
+    end
+
+    initializer "provenance.active_job" do
+      ActiveSupport.on_load(:active_job) { include Provenance::Job }
+    end
+
+    config.after_initialize do
+      Provenance.config.validate!
+    end
+
+    rake_tasks do
+      load File.expand_path("../tasks/provenance.rake", __dir__)
+    end
+
+    generators do
+      require "generators/provenance/install/install_generator"
+    end
+  end
+end

data/lib/provenance/rake.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Provenance
+  # Opt-in rake integration: every executed task becomes a +task+ action.
+  #
+  # @example Rakefile
+  #   Provenance::Rake.install!
+  module Rake
+    # Prepended into Rake::Task.
+    module TaskExtension
+      # @api private
+      def execute(args = nil)
+        return super unless Provenance.enabled?
+
+        Provenance.track(kind: :task, name: name) { super }
+      end
+    end
+
+    # Wraps rake task execution. Idempotent.
+    #
+    # @return [void]
+    def self.install!
+      require "rake"
+      ::Rake::Task.prepend(TaskExtension) unless ::Rake::Task.ancestors.include?(TaskExtension)
+    end
+  end
+end

data/lib/provenance/recorder.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Provenance
+  # Builds entity changes from model callbacks and bulk relation calls and attaches
+  # them to the current action once their transaction commits.
+  module Recorder
+    BULK_KEY = :provenance_bulk_depth
+    private_constant :BULK_KEY
+
+    module_function
+
+    # Records a create, update or destroy of a single record.
+    #
+    # @param record [ActiveRecord::Base]
+    # @param operation [Symbol] :create, :update or :destroy
+    # @return [void]
+    def record(record, operation)
+      options = options_for(record.class)
+      return unless options
+      return if options.ignore_if&.call(record)
+
+      raw = case operation
+      when :create, :destroy then record.attributes.compact
+      when :update then record.saved_changes
+      end
+      diff = filter(record.class, raw, options)
+      return if operation == :update && diff.empty?
+
+      change = EntityChange.new(entity: record.class.name, entity_id: record.id, operation: operation, diff: diff)
+      append(record.class, change, "#{record.class.name}.#{operation}")
+    end
+
+    # Records a link or unlink through a tracked association.
+    #
+    # @param owner [ActiveRecord::Base]
+    # @param association [Symbol]
+    # @param target [ActiveRecord::Base]
+    # @param operation [Symbol] :link or :unlink
+    # @return [void]
+    def association(owner, association, target, operation)
+      options = options_for(owner.class)
+      return unless options
+      return if options.ignore_if&.call(owner)
+
+      change = EntityChange.new(
+        entity: owner.class.name, entity_id: -> { owner.id }, operation: operation, association: association,
+        target: {"type" => target.class.name, "id" => -> { target.id }}
+      )
+      append(owner.class, change, "#{owner.class.name}.#{operation}")
+    end
+
+    # Wraps update_all/delete_all on a relation of a tracked model.
+    #
+    # @param relation [ActiveRecord::Relation]
+    # @param operation [Symbol] :bulk_update or :bulk_delete
+    # @param updates [Hash, String, Array, nil] the update_all argument
+    # @yieldreturn [Integer] affected rows
+    # @return [Integer] the block's return value
+    def bulk(relation, operation, updates: nil)
+      model = relation.model
+      options = options_for(model)
+      return yield if options.nil? || bulk_depth.positive?
+
+      ids, truncated = bulk_ids(relation)
+      count = nested_bulk { yield }
+      diff = updates.is_a?(Hash) ? filter(model, updates.to_h { |k, v| [k.to_s, [nil, v]] }, options) : nil
+      change = EntityChange.new(
+        entity: model.name, operation: operation, diff: diff, count: count,
+        where: fingerprint(relation), ids: ids, truncated: truncated
+      )
+      append(model, change, "#{model.name}.#{operation}")
+      count
+    end
+
+    # Wraps insert_all/upsert_all on a tracked model.
+    #
+    # @param relation [ActiveRecord::Relation]
+    # @param attributes [Array<Hash>, Hash]
+    # @yieldreturn [ActiveRecord::Result]
+    # @return [ActiveRecord::Result] the block's return value
+    def bulk_insert(relation, attributes)
+      model = relation.model
+      options = options_for(model)
+      return yield if options.nil? || bulk_depth.positive?
+
+      result = nested_bulk { yield }
+      rows = attributes.is_a?(Hash) ? [attributes] : Array(attributes)
+      limit = Provenance.config.bulk_ids_limit
+      ids = inserted_ids(model, rows, result)
+      change = EntityChange.new(
+        entity: model.name, operation: :bulk_insert, count: rows.size,
+        ids: ids.first(limit), truncated: ids.size > limit
+      )
+      append(model, change, "#{model.name}.bulk_insert")
+      result
+    end
+
+    # Attaches a change to the current action after the surrounding transaction
+    # commits, or to an implicit +custom+ action when none is open.
+    #
+    # @param model [Class]
+    # @param change [EntityChange]
+    # @param implicit_name [String]
+    # @return [void]
+    def append(model, change, implicit_name)
+      action = Provenance.current
+      if action
+        after_commit(model) { action.add_change(change) }
+      else
+        action = Action.new(kind: :custom, name: implicit_name)
+        after_commit(model) { action.add_change(change) }
+        Emitter.finish(action)
+      end
+    end
+
+    # Runs the block when the model's innermost joinable transaction commits, or
+    # immediately when none is open. Rolled-back transactions never run it.
+    #
+    # @param model [Class]
+    # @return [void]
+    def after_commit(model, &block)
+      transaction = model.connection_pool.active_connection&.current_transaction
+      if transaction&.open? && transaction.joinable?
+        transaction.after_commit(&block)
+      else
+        block.call
+      end
+    end
+
+    # @api private
+    def options_for(model)
+      return nil unless Provenance.enabled?
+
+      Provenance.registry.options_for(model)
+    end
+
+    # @api private
+    def filter(model, attributes, options)
+      primary_keys = Array(model.primary_key).map(&:to_s)
+      diff = attributes.to_h { |k, v| [k.to_s, v] }
+        .select { |name, _| !primary_keys.include?(name) && options.track?(name) }
+      Redactor.diff(diff, options.redact + encrypted_attributes(model))
+    end
+
+    # @api private
+    def encrypted_attributes(model)
+      model.respond_to?(:encrypted_attributes) ? Array(model.encrypted_attributes).map(&:to_s) : []
+    end
+
+    # @api private
+    def bulk_ids(relation)
+      limit = Provenance.config.bulk_ids_limit
+      ids = if relation.loaded?
+        relation.records.map(&:id)
+      else
+        fetch = limit + 1
+        fetch = [relation.limit_value, fetch].min if relation.limit_value
+        relation.limit(fetch).pluck(*Array(relation.model.primary_key))
+      end
+      [ids.first(limit).map(&:to_s), ids.size > limit]
+    end
+
+    # @api private
+    def inserted_ids(model, rows, result)
+      primary_key = model.primary_key
+      return [] unless primary_key.is_a?(String)
+
+      ids = if result.respond_to?(:columns) && result.columns.include?(primary_key)
+        result.rows.map { |row| row[result.columns.index(primary_key)] }
+      else
+        rows.filter_map { |row| row[primary_key] || row[primary_key.to_sym] }
+      end
+      ids.map(&:to_s)
+    end
+
+    # Fingerprint of a relation's WHERE clause with bind values replaced by
+    # placeholders, so the same query shape yields the same fingerprint.
+    #
+    # @param relation [ActiveRecord::Relation]
+    # @return [String, nil] "sha256:<16 hex chars>"
+    def fingerprint(relation)
+      return nil if relation.where_clause.empty?
+
+      sql = relation.model.with_connection do |connection|
+        connection.visitor.compile(relation.where_clause.ast, Arel::Collectors::SQLString.new)
+      end
+      "sha256:#{Digest::SHA256.hexdigest(sql)[0, 16]}"
+    end
+
+    # @api private
+    def bulk_depth
+      ActiveSupport::IsolatedExecutionState[BULK_KEY].to_i
+    end
+
+    # @api private
+    def nested_bulk
+      state = ActiveSupport::IsolatedExecutionState
+      state[BULK_KEY] = bulk_depth + 1
+      yield
+    ensure
+      state[BULK_KEY] -= 1
+    end
+  end
+end