provenance 2.0.0

data/lib/provenance/redactor.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Provenance
+  # Replaces sensitive values with "[REDACTED]".
+  module Redactor
+    # Replacement for redacted values.
+    MASK = "[REDACTED]"
+
+    module_function
+
+    # @return [Array<Symbol, String, Regexp, Proc>] configured redactions merged with Rails filter_parameters
+    def filters
+      rails = (defined?(Rails) && Rails.respond_to?(:application) && Rails.application&.config&.filter_parameters) || []
+      Array(rails) + Array(Provenance.config.redact_attributes)
+    end
+
+    # @param name [String, Symbol]
+    # @param extra [Array<String>] exact attribute names to redact
+    # @return [Boolean]
+    def redact?(name, extra = [])
+      return true if extra.include?(name.to_s)
+
+      filtered = parameter_filter.filter(name.to_s => true)
+      filtered[name.to_s] != true
+    end
+
+    # Redacts a diff, keeping its shape.
+    #
+    # @param diff [Hash, nil]
+    # @param extra [Array<String>]
+    # @return [Hash, nil]
+    def diff(diff, extra = [])
+      return diff unless diff.is_a?(Hash)
+
+      diff.to_h do |key, value|
+        next [key, value] unless redact?(key, extra)
+
+        [key, if value.is_a?(Array)
+                value.map { |v| v.nil? ? nil : MASK }
+              else
+                MASK
+              end]
+      end
+    end
+
+    # Redacts nested hashes such as metadata.
+    #
+    # @param hash [Hash]
+    # @return [Hash]
+    def deep(hash)
+      parameter_filter.filter(hash)
+    end
+
+    # @api private
+    def parameter_filter
+      current = filters
+      cached = @parameter_filter
+      return cached.last if cached && cached.first == current
+
+      ActiveSupport::ParameterFilter.new(current, mask: MASK).tap { |f| @parameter_filter = [current, f] }
+    end
+  end
+end

data/lib/provenance/registry.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Provenance
+  # Tracking options of every model declared with +has_provenance+.
+  class Registry
+    # Options of one tracked model.
+    Options = Struct.new(:only, :except, :redact, :associations, :ignore_if) do
+      # @param attribute [String]
+      # @return [Boolean] whether the attribute belongs in diffs
+      def track?(attribute)
+        return false if Provenance.config.ignored_attributes.map(&:to_s).include?(attribute)
+        return only.include?(attribute) if only.present?
+
+        !except.include?(attribute)
+      end
+    end
+
+    def initialize
+      @models = Concurrent::Map.new
+    end
+
+    # @param model [Class]
+    # @param options [Options]
+    # @return [Options]
+    def register(model, options)
+      @models[model.name] = options
+    end
+
+    # Options for the model or its closest tracked ancestor (STI).
+    #
+    # @param model [Class]
+    # @return [Options, nil]
+    def options_for(model)
+      model.ancestors.each do |klass|
+        next unless klass.is_a?(Class) && klass.name
+
+        options = @models[klass.name]
+        return options if options
+      end
+      nil
+    end
+
+    # @return [Array<String>] names of tracked models
+    def model_names
+      @models.keys
+    end
+  end
+end

data/lib/provenance/relay_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Provenance
+  # Delivers pending outbox rows. Enqueued after every outbox write when
+  # +outbox_enqueue_relay+ is on; can also be scheduled periodically.
+  class RelayJob < ActiveJob::Base
+    skip_provenance if respond_to?(:skip_provenance)
+
+    # @param batch_size [Integer, nil]
+    # @return [Integer] delivered rows
+    def perform(batch_size: nil)
+      Outbox::Relay.run(batch_size: batch_size || Provenance.config.outbox_batch_size)
+    end
+  end
+end

data/lib/provenance/rspec.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "provenance"
+require "rspec/expectations"
+
+module Provenance
+  module Testing
+    # Block matcher for emitted events.
+    #
+    # @example
+    #   expect { patch invoice_path(invoice), params: {status: "sent"} }
+    #     .to emit_provenance_event(action: "invoices#update", outcome: "success")
+    #     .with_change(entity: "Invoice", operation: "update", diff: including(status: ["draft", "sent"]))
+    class EmitEventMatcher
+      include RSpec::Matchers::Composable
+
+      # Expected attribute name -> path in the native event.
+      PATHS = {
+        action: %w[action name], kind: %w[action kind], caused_by: %w[action caused_by],
+        outcome: %w[outcome result], error: %w[outcome error], actor: %w[actor], actor_id: %w[actor id],
+        request: %w[request], status: %w[request status], metadata: %w[metadata], app: %w[app]
+      }.freeze
+
+      # @param expected [Hash] any of the keys in {PATHS}
+      def initialize(expected)
+        unknown = expected.keys - PATHS.keys
+        raise ArgumentError, "unknown keys #{unknown.inspect}" if unknown.any?
+
+        @expected = expected
+        @changes = []
+        @count = nil
+      end
+
+      # Requires a change matching the given attributes in the same event.
+      #
+      # @param expected [Hash] entity, entity_id, operation, diff, association, count, ids
+      # @return [self]
+      def with_change(**expected)
+        @changes << expected
+        self
+      end
+
+      # Requires exactly +n+ matching events.
+      #
+      # @param n [Integer]
+      # @return [self]
+      def exactly(n)
+        @count = n
+        self
+      end
+
+      # Alias for +exactly(1)+.
+      #
+      # @return [self]
+      def once
+        exactly(1)
+      end
+
+      # @return [Boolean]
+      def supports_block_expectations?
+        true
+      end
+
+      # @return [Boolean]
+      def supports_value_expectations?
+        false
+      end
+
+      # @param block [Proc]
+      # @return [Boolean]
+      def matches?(block)
+        @events = Testing.capture(&block)
+        @matching = @events.select { |event| event_matches?(event) }
+        @count ? @matching.size == @count : @matching.any?
+      end
+
+      # @param block [Proc]
+      # @return [Boolean]
+      def does_not_match?(block)
+        raise ArgumentError, "exactly/once is not supported with not_to" if @count
+
+        @events = Testing.capture(&block)
+        @matching = @events.select { |event| event_matches?(event) }
+        @matching.empty?
+      end
+
+      # @return [String]
+      def description
+        "emit provenance event #{description_of(@expected)}" +
+          @changes.map { |c| " with change #{description_of(c)}" }.join
+      end
+
+      # @return [String]
+      def failure_message
+        "expected block to #{description}#{" #{@count} time(s)" if @count}, " \
+          "but #{@matching.size} of #{@events.size} emitted event(s) matched:\n#{dump}"
+      end
+
+      # @return [String]
+      def failure_message_when_negated
+        "expected block not to #{description}, but it emitted:\n#{dump(@matching)}"
+      end
+
+      private
+
+      def event_matches?(event)
+        @expected.all? { |key, value| values_match?(value, indifferent(event.dig(*PATHS.fetch(key)))) } &&
+          @changes.all? { |expected| event["changes"].any? { |change| change_matches?(expected, change) } }
+      end
+
+      def change_matches?(expected, change)
+        expected.all? { |key, value| values_match?(value, indifferent(change[key.to_s])) }
+      end
+
+      def indifferent(value)
+        case value
+        when Hash then value.to_h { |k, v| [k, indifferent(v)] }.with_indifferent_access
+        when Array then value.map { |v| indifferent(v) }
+        else value
+        end
+      end
+
+      def dump(events = @events)
+        events.map { |event| "  #{JSON.generate(event)}" }.join("\n")
+      end
+    end
+
+    # RSpec helpers, included into every example group.
+    module Matchers
+      # @param expected [Hash] see {EmitEventMatcher::PATHS}
+      # @return [EmitEventMatcher]
+      def emit_provenance_event(**expected)
+        EmitEventMatcher.new(expected)
+      end
+    end
+  end
+end
+
+RSpec.configure { |config| config.include Provenance::Testing::Matchers } if defined?(RSpec.configure)

data/lib/provenance/serializers/cloud_events.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Serializers
+    # CloudEvents 1.0 structured-mode JSON with the native event as +data+.
+    module CloudEvents
+      module_function
+
+      # @param event [Hash] native event
+      # @return [Array<Hash>]
+      def call(event)
+        [{
+          "specversion" => "1.0",
+          "id" => event["id"],
+          "source" => event["app"],
+          "type" => "provenance.action.#{event.dig("action", "kind")}",
+          "subject" => event.dig("action", "name"),
+          "time" => event["occurred_at"],
+          "datacontenttype" => "application/json",
+          "dataschema" => "urn:provenance:event:2",
+          "data" => event
+        }]
+      end
+    end
+  end
+end

data/lib/provenance/serializers/native.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Serializers
+    # The native "provenance/event@2" format, as built by {Event.build}.
+    module Native
+      module_function
+
+      # @param event [Hash] native event
+      # @return [Array<Hash>]
+      def call(event)
+        [event]
+      end
+    end
+  end
+end

data/lib/provenance/serializers/ocsf.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Serializers
+    # OCSF 1.x: one API Activity (class_uid 6003) event per action plus one Entity
+    # Management (class_uid 3004) event per entity change. The README documents the
+    # field mapping.
+    module OCSF
+      # OCSF schema version the events declare.
+      VERSION = "1.3.0"
+      # API Activity class.
+      API_ACTIVITY = {class_uid: 6003, category_uid: 6, class_name: "API Activity", category_name: "Application Activity"}.freeze
+      # Entity Management class.
+      ENTITY_MANAGEMENT = {class_uid: 3004, category_uid: 3, class_name: "Entity Management", category_name: "Identity & Access Management"}.freeze
+      # activity_id by HTTP method for API Activity.
+      HTTP_ACTIVITIES = {"POST" => 1, "GET" => 2, "HEAD" => 2, "PUT" => 3, "PATCH" => 3, "DELETE" => 4}.freeze
+      # activity_id by change operation for Entity Management.
+      CHANGE_ACTIVITIES = {
+        "create" => 1, "bulk_insert" => 1, "update" => 3, "bulk_update" => 3, "destroy" => 4, "bulk_delete" => 4
+      }.freeze
+      # Activity names for ids used above.
+      ACTIVITY_NAMES = {1 => "Create", 2 => "Read", 3 => "Update", 4 => "Delete", 99 => "Other"}.freeze
+      # status_id, status and severity_id by outcome.
+      OUTCOMES = {
+        "success" => [1, "Success", 1],
+        "failure" => [2, "Failure", 2],
+        "denied" => [2, "Failure", 3]
+      }.freeze
+
+      module_function
+
+      # @param event [Hash] native event
+      # @return [Array<Hash>] the API Activity event followed by Entity Management events
+      def call(event)
+        [api_activity(event)] + event["changes"].each_with_index.map { |change, i| entity_management(event, change, i) }
+      end
+
+      # @param event [Hash]
+      # @return [Hash]
+      def api_activity(event)
+        request = event["request"] || {}
+        activity_id = HTTP_ACTIVITIES.fetch(request["method"].to_s.upcase) { activity_from_changes(event["changes"]) }
+        result = base(event, API_ACTIVITY, activity_id, event["id"])
+        result["api"] = {
+          "operation" => event.dig("action", "name"),
+          "request" => {"uid" => request["id"] || event["id"]},
+          "response" => response(event)
+        }.compact
+        result["src_endpoint"] = {"ip" => request["ip"]}.compact
+        if event["request"]
+          result["http_request"] = {
+            "http_method" => request["method"], "url" => {"path" => request["path"]},
+            "user_agent" => request["user_agent"], "uid" => request["id"]
+          }.compact
+          result["http_response"] = {"code" => request["status"]}.compact
+        end
+        result["resources"] = event["changes"].map { |c| {"type" => c["entity"], "uid" => c["entity_id"]}.compact }
+        result
+      end
+
+      # @param event [Hash]
+      # @param change [Hash]
+      # @param index [Integer]
+      # @return [Hash]
+      def entity_management(event, change, index)
+        activity_id = CHANGE_ACTIVITIES.fetch(change["operation"], 99)
+        result = base(event, ENTITY_MANAGEMENT, activity_id, "#{event["id"]}/#{index}")
+        result["activity_name"] = change["operation"].capitalize if activity_id == 99
+        result["entity"] = {
+          "name" => change["entity"], "type" => change["entity"], "uid" => change["entity_id"], "data" => change["diff"]
+        }.compact
+        result["unmapped"]["change"] = change.except("entity", "entity_id", "diff")
+        result
+      end
+
+      # @api private
+      def base(event, klass, activity_id, uid)
+        status_id, status, severity_id = OUTCOMES.fetch(event.dig("outcome", "result"), [99, "Other", 1])
+        {
+          "category_uid" => klass[:category_uid], "category_name" => klass[:category_name],
+          "class_uid" => klass[:class_uid], "class_name" => klass[:class_name],
+          "activity_id" => activity_id, "activity_name" => ACTIVITY_NAMES[activity_id],
+          "type_uid" => (klass[:class_uid] * 100) + activity_id,
+          "time" => (Time.iso8601(event["occurred_at"]).to_r * 1000).to_i,
+          "severity_id" => severity_id,
+          "status_id" => status_id, "status" => status,
+          "status_detail" => event.dig("outcome", "result"),
+          "metadata" => {
+            "version" => VERSION, "uid" => uid, "correlation_uid" => event["id"], "log_name" => event["app"],
+            "product" => {"name" => "Provenance", "vendor_name" => "Provenance", "version" => Provenance::VERSION}
+          },
+          "actor" => actor(event["actor"], event["app"]),
+          "unmapped" => {
+            "action" => event["action"], "metadata" => event["metadata"],
+            "impersonator" => event.dig("actor", "impersonator"), "integrity" => event["integrity"]
+          }.compact
+        }
+      end
+
+      # @api private
+      def actor(actor, app)
+        result = {"app_name" => app}
+        if actor
+          result["user"] = {
+            "uid" => actor["id"], "name" => actor["display"], "type" => actor["type"],
+            "groups" => Array(actor["roles"]).map { |role| {"name" => role} }
+          }.compact
+        end
+        result
+      end
+
+      # @api private
+      def response(event)
+        error = event.dig("outcome", "error")
+        response = {"code" => event.dig("request", "status")}
+        response.merge!("error" => error["class"], "error_message" => error["message"]) if error
+        response.compact.presence
+      end
+
+      # @api private
+      def activity_from_changes(changes)
+        ids = changes.map { |c| CHANGE_ACTIVITIES.fetch(c["operation"], 99) }.uniq
+        (ids.size == 1) ? ids.first : 99
+      end
+    end
+  end
+end

data/lib/provenance/serializers.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "provenance/serializers/native"
+require "provenance/serializers/ocsf"
+require "provenance/serializers/cloud_events"
+
+module Provenance
+  # Converts native events into the configured wire format.
+  module Serializers
+    module_function
+
+    # @param format [Symbol] :native, :ocsf or :cloudevents
+    # @return [#call] serializer returning an Array of serialized events per native event
+    def for(format)
+      case format.to_sym
+      when :native then Native
+      when :ocsf then OCSF
+      when :cloudevents then CloudEvents
+      else raise ConfigurationError, "unknown format #{format.inspect}"
+      end
+    end
+
+    # @param events [Array<Hash>] native events
+    # @param format [Symbol]
+    # @return [Array<Hash>] serialized events; OCSF may produce several per native event
+    def serialize_all(events, format)
+      serializer = self.for(format)
+      events.flat_map { |event| serializer.call(event) }
+    end
+  end
+end

data/lib/provenance/sinks/http.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Provenance
+  module Sinks
+    # POSTs each batch as a JSON array. Any 2xx response is a success; other
+    # responses and network errors are retried with exponential backoff and jitter.
+    class HTTP
+      # Errors treated as retryable network failures.
+      NETWORK_ERRORS = [
+        IOError, SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError, Net::ProtocolError
+      ].freeze
+
+      # @return [URI::HTTP]
+      attr_reader :uri
+
+      # @param url [String]
+      # @param headers [Hash{String => String}]
+      # @param open_timeout [Numeric] seconds
+      # @param read_timeout [Numeric] seconds
+      # @param retries [Integer] retries after the first attempt
+      # @param backoff [Numeric] base delay in seconds, doubled per retry
+      # @param max_backoff [Numeric] delay cap in seconds
+      # @param sleeper [#call] receives the delay; replaceable in tests
+      def initialize(url:, headers: {}, open_timeout: 2, read_timeout: 5, retries: 3, backoff: 0.5,
+        max_backoff: 30, sleeper: ->(seconds) { sleep(seconds) })
+        @uri = URI.parse(url.to_s)
+        raise ConfigurationError, "http sink needs an http(s) url" unless @uri.is_a?(URI::HTTP)
+
+        @headers = {"Content-Type" => "application/json"}.merge(headers.transform_keys(&:to_s))
+        @open_timeout = open_timeout
+        @read_timeout = read_timeout
+        @retries = retries
+        @backoff = backoff
+        @max_backoff = max_backoff
+        @sleeper = sleeper
+      end
+
+      # @param batch [Array<Hash>]
+      # @return [void]
+      # @raise [DeliveryError] when every attempt failed
+      def deliver(batch)
+        body = JSON.generate(batch)
+        attempt = 0
+        begin
+          post(body)
+        rescue DeliveryError, *NETWORK_ERRORS => e
+          raise DeliveryError, "#{uri.host}: #{e.message}" if attempt >= @retries
+
+          @sleeper.call(delay(attempt))
+          attempt += 1
+          retry
+        end
+      end
+
+      # @param attempt [Integer] zero-based retry number
+      # @return [Float] seconds to wait before the retry
+      def delay(attempt)
+        base = [@backoff * (2**attempt), @max_backoff].min
+        base / 2.0 + rand * base / 2.0
+      end
+
+      private
+
+      def post(body)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = @open_timeout
+        http.read_timeout = @read_timeout
+        request = Net::HTTP::Post.new(uri.request_uri, @headers)
+        request.body = body
+        response = http.request(request)
+        return response if response.code.to_i.between?(200, 299)
+
+        raise DeliveryError, "HTTP #{response.code}"
+      end
+    end
+  end
+end

data/lib/provenance/sinks/io.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Sinks
+    # Writes one JSON line per event to any IO.
+    class IO
+      # @param io [#write]
+      def initialize(io = $stdout)
+        @io = io
+        @lock = Mutex.new
+      end
+
+      # @param batch [Array<Hash>]
+      # @return [void]
+      def deliver(batch)
+        lines = batch.map { |event| "#{JSON.generate(event)}\n" }.join
+        @lock.synchronize do
+          @io.write(lines)
+          @io.flush if @io.respond_to?(:flush)
+        end
+      end
+    end
+  end
+end

data/lib/provenance/sinks/kafka.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+begin
+  require "rdkafka"
+rescue LoadError
+  raise LoadError, "the :kafka sink needs the rdkafka gem; add it to your Gemfile"
+end
+
+module Provenance
+  module Sinks
+    # Produces one Kafka message per event, keyed by event id. Optional: requires
+    # the +rdkafka+ gem.
+    #
+    # @example
+    #   require "provenance/sinks/kafka"
+    #   c.sink :kafka, topic: "audit", config: {"bootstrap.servers": "kafka:9092"}
+    class Kafka
+      # @param topic [String]
+      # @param config [Hash] rdkafka producer configuration
+      # @param producer [Object, nil] an existing rdkafka producer
+      # @param timeout [Numeric] seconds to wait for delivery reports
+      def initialize(topic:, config: {}, producer: nil, timeout: 10)
+        @topic = topic
+        @producer = producer || Rdkafka::Config.new(config).producer
+        @timeout = timeout
+      end
+
+      # @param batch [Array<Hash>]
+      # @return [void]
+      def deliver(batch)
+        handles = batch.map do |event|
+          @producer.produce(topic: @topic, payload: JSON.generate(event), key: (event["id"] || event.dig("metadata", "uid")).to_s)
+        end
+        handles.each { |handle| handle.wait(max_wait_timeout: @timeout) }
+      end
+
+      # Closes the underlying producer.
+      #
+      # @return [void]
+      def close
+        @producer.close
+      end
+    end
+  end
+end

data/lib/provenance/sinks/logger.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Sinks
+    # Writes one JSON line per event to a logger (Rails.logger by default).
+    class Logger
+      # @param logger [::Logger, nil] defaults to the configured Provenance logger
+      # @param level [Symbol] log level
+      def initialize(logger: nil, level: :info)
+        @logger = logger
+        @level = level
+      end
+
+      # @param batch [Array<Hash>]
+      # @return [void]
+      def deliver(batch)
+        logger = @logger || Provenance.config.logger
+        batch.each { |event| logger.public_send(@level, JSON.generate(event)) }
+      end
+    end
+  end
+end

data/lib/provenance/sinks/memory.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Provenance
+  module Sinks
+    # Keeps delivered events in memory. Intended for tests.
+    class Memory
+      def initialize
+        @events = []
+        @lock = Mutex.new
+      end
+
+      # @param batch [Array<Hash>]
+      # @return [void]
+      def deliver(batch)
+        @lock.synchronize { @events.concat(batch) }
+      end
+
+      # @return [Array<Hash>] a copy of the delivered events
+      def events
+        @lock.synchronize { @events.dup }
+      end
+
+      # @return [void]
+      def clear
+        @lock.synchronize { @events.clear }
+      end
+    end
+  end
+end