whodunit-chronicles 0.1.0.pre

data/lib/whodunit/chronicles/change_event.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Represents a database change event in a common format
+    #
+    # This class normalizes database changes from different sources
+    # (PostgreSQL WAL, MariaDB binlog, etc.) into a consistent format
+    # for processing by audit systems.
+    class ChangeEvent
+      # Supported database actions
+      ACTIONS = %w[INSERT UPDATE DELETE].freeze
+
+      attr_reader :table_name, :schema_name, :action, :primary_key, :old_data, :new_data,
+        :timestamp, :transaction_id, :sequence_number, :metadata
+
+      # Initialize a new change event
+      #
+      # @param table_name [String] The name of the table that changed
+      # @param action [String] The type of change (INSERT, UPDATE, DELETE)
+      # @param primary_key [Hash] The primary key values for the changed row
+      # @param old_data [Hash, nil] The row data before the change (nil for INSERT)
+      # @param new_data [Hash, nil] The row data after the change (nil for DELETE)
+      # @param timestamp [Time] When the change occurred
+      # @param schema_name [String] The schema name (optional, defaults to 'public')
+      # @param transaction_id [String, Integer] Database transaction identifier
+      # @param sequence_number [Integer] Sequence number within the transaction
+      # @param metadata [Hash] Additional adapter-specific metadata
+      def initialize(
+        table_name:,
+        action:,
+        primary_key:,
+        old_data: nil,
+        new_data: nil,
+        timestamp: Time.now,
+        schema_name: 'public',
+        transaction_id: nil,
+        sequence_number: nil,
+        metadata: {}
+      )
+        @table_name = table_name.to_s
+        @schema_name = schema_name.to_s
+        @action = validate_action(action.to_s.upcase)
+        @primary_key = primary_key || {}
+        @old_data = old_data
+        @new_data = new_data
+        @timestamp = timestamp
+        @transaction_id = transaction_id
+        @sequence_number = sequence_number
+        @metadata = metadata || {}
+
+        validate_data_consistency
+      end
+
+      # Get the qualified table name (schema.table)
+      #
+      # @return [String]
+      def qualified_table_name
+        "#{schema_name}.#{table_name}"
+      end
+
+      # Check if this is a create event
+      #
+      # @return [Boolean]
+      def create?
+        action == 'INSERT'
+      end
+
+      # Check if this is an update event
+      #
+      # @return [Boolean]
+      def update?
+        action == 'UPDATE'
+      end
+
+      # Check if this is a delete event
+      #
+      # @return [Boolean]
+      def delete?
+        action == 'DELETE'
+      end
+
+      # Get the changed columns for UPDATE events
+      #
+      # @return [Array<String>] Array of column names that changed
+      def changed_columns
+        return [] unless update? && old_data && new_data
+
+        old_data.keys.reject { |key| old_data[key] == new_data[key] }
+      end
+
+      # Get a hash of changes in [old_value, new_value] format
+      #
+      # @return [Hash] Hash of column_name => [old_value, new_value]
+      def changes
+        return {} unless update? && old_data && new_data
+
+        changed_columns.each_with_object({}) do |column, changes_hash|
+          changes_hash[column] = [old_data[column], new_data[column]]
+        end
+      end
+
+      # Get the current data for this event
+      #
+      # @return [Hash] The new_data for INSERT/UPDATE, old_data for DELETE
+      def current_data
+        case action
+        when 'INSERT', 'UPDATE'
+          new_data
+        when 'DELETE'
+          old_data
+        end
+      end
+
+      # Get all available data for this event
+      #
+      # @return [Hash] Combined old and new data
+      def all_data
+        (old_data || {}).merge(new_data || {})
+      end
+
+      # Convert to hash representation
+      #
+      # @return [Hash]
+      def to_h
+        {
+          table_name: table_name,
+          schema_name: schema_name,
+          qualified_table_name: qualified_table_name,
+          action: action,
+          primary_key: primary_key,
+          old_data: old_data,
+          new_data: new_data,
+          current_data: current_data,
+          changes: changes,
+          changed_columns: changed_columns,
+          timestamp: timestamp,
+          transaction_id: transaction_id,
+          sequence_number: sequence_number,
+          metadata: metadata,
+        }
+      end
+
+      # String representation
+      #
+      # @return [String]
+      def to_s
+        pk_str = primary_key.map { |k, v| "#{k}=#{v}" }.join(', ')
+        "#{action} #{qualified_table_name}(#{pk_str}) at #{timestamp}"
+      end
+
+      # Detailed string representation
+      #
+      # @return [String]
+      def inspect
+        "#<#{self.class.name} #{self}>"
+      end
+
+      # Compare events for equality
+      #
+      # @param other [ChangeEvent]
+      # @return [Boolean]
+      def ==(other)
+        return false unless other.is_a?(ChangeEvent)
+
+        table_name == other.table_name &&
+          schema_name == other.schema_name &&
+          action == other.action &&
+          primary_key == other.primary_key &&
+          old_data == other.old_data &&
+          new_data == other.new_data &&
+          timestamp == other.timestamp &&
+          transaction_id == other.transaction_id &&
+          sequence_number == other.sequence_number
+      end
+
+      private
+
+      def validate_action(action)
+        unless ACTIONS.include?(action)
+          raise ArgumentError, "Invalid action: #{action}. Must be one of: #{ACTIONS.join(', ')}"
+        end
+
+        action
+      end
+
+      def validate_data_consistency
+        case action
+        when 'INSERT'
+          raise ArgumentError, 'INSERT events must have new_data' if new_data.nil? || new_data.empty?
+          raise ArgumentError, 'INSERT events should not have old_data' unless old_data.nil?
+        when 'UPDATE'
+          raise ArgumentError, 'UPDATE events must have both old_data and new_data' if old_data.nil? || new_data.nil?
+        when 'DELETE'
+          raise ArgumentError, 'DELETE events must have old_data' if old_data.nil? || old_data.empty?
+          raise ArgumentError, 'DELETE events should not have new_data' unless new_data.nil?
+        end
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/configuration.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Configuration management for Chronicles
+    #
+    # Provides a centralized configuration system with sensible defaults
+    # and validation for all Chronicles settings.
+    class Configuration
+      attr_accessor :database_url, :audit_database_url, :adapter, :publication_name,
+        :replication_slot_name, :batch_size, :max_retry_attempts, :retry_delay,
+        :logger, :table_filter, :schema_filter
+
+      def initialize
+        @database_url = ENV.fetch('DATABASE_URL', nil)
+        @audit_database_url = ENV.fetch('AUDIT_DATABASE_URL', nil)
+        @adapter = :postgresql
+        @publication_name = 'whodunit_audit'
+        @replication_slot_name = 'whodunit_audit_slot'
+        @batch_size = 100
+        @max_retry_attempts = 3
+        @retry_delay = 5
+        @logger = Dry::Logger.new
+        @table_filter = nil
+        @schema_filter = nil
+      end
+
+      # Validate configuration settings
+      #
+      # @raise [ConfigurationError] if configuration is invalid
+      def validate!
+        raise ConfigurationError, 'database_url is required' if database_url.nil?
+        raise ConfigurationError, 'adapter must be :postgresql' unless adapter == :postgresql
+        raise ConfigurationError, 'batch_size must be positive' unless batch_size.positive?
+        raise ConfigurationError, 'max_retry_attempts must be positive' unless max_retry_attempts.positive?
+        raise ConfigurationError, 'retry_delay must be positive' unless retry_delay.positive?
+
+        validate_publication_name!
+        validate_slot_name!
+      end
+
+      # Check if a table should be audited based on filters
+      #
+      # @param table_name [String] The table name to check
+      # @param schema_name [String] The schema name to check
+      # @return [Boolean] true if the table should be audited
+      def audit_table?(table_name, schema_name = 'public')
+        return false if filtered_by_schema?(schema_name)
+        return false if filtered_by_table?(table_name)
+
+        true
+      end
+
+      private
+
+      def validate_publication_name!
+        return if /\A[a-zA-Z_][a-zA-Z0-9_]*\z/.match?(publication_name)
+
+        raise ConfigurationError, 'publication_name must be a valid PostgreSQL identifier'
+      end
+
+      def validate_slot_name!
+        return if /\A[a-zA-Z_][a-zA-Z0-9_]*\z/.match?(replication_slot_name)
+
+        raise ConfigurationError, 'replication_slot_name must be a valid PostgreSQL identifier'
+      end
+
+      def filtered_by_schema?(schema_name)
+        return false unless schema_filter
+
+        case schema_filter
+        when Array
+          !schema_filter.include?(schema_name)
+        when String, Symbol
+          schema_name != schema_filter.to_s
+        when Proc
+          !schema_filter.call(schema_name)
+        else
+          false
+        end
+      end
+
+      def filtered_by_table?(table_name)
+        return false unless table_filter
+
+        case table_filter
+        when Array
+          !table_filter.include?(table_name)
+        when String, Symbol
+          table_name != table_filter.to_s
+        when Regexp
+          !table_filter.match?(table_name)
+        when Proc
+          !table_filter.call(table_name)
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/service.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+
+module Whodunit
+  module Chronicles
+    # Main service orchestrator for audit streaming
+    #
+    # Coordinates the stream adapter and audit processor to provide
+    # a complete audit streaming solution with error handling and monitoring.
+    class Service
+      attr_reader :adapter, :processor, :logger, :executor
+
+      def initialize(
+        adapter: nil,
+        processor: nil,
+        logger: Chronicles.logger
+      )
+        @adapter = adapter || build_adapter
+        @processor = processor || AuditProcessor.new(logger: logger)
+        @logger = logger
+        @executor = Concurrent::ThreadPoolExecutor.new(
+          min_threads: 1,
+          max_threads: 4,
+          max_queue: 100,
+          fallback_policy: :caller_runs,
+        )
+        @running = false
+        @retry_count = 0
+      end
+
+      # Start the audit streaming service
+      #
+      # @return [self]
+      def start
+        return self if running?
+
+        log(:info, 'Starting Chronicles audit streaming service')
+
+        validate_setup!
+        test_connections!
+
+        @running = true
+        @retry_count = 0
+
+        start_streaming_with_retry
+
+        log(:info, 'Chronicles audit streaming service started successfully')
+        self
+      rescue StandardError => e
+        log(:error, 'Failed to start service', error: e.message)
+        @running = false
+        raise
+      end
+
+      # Stop the audit streaming service
+      #
+      # @return [void]
+      def stop
+        return unless running?
+
+        log(:info, 'Stopping Chronicles audit streaming service')
+        @running = false
+
+        adapter.stop_streaming if adapter.streaming?
+        @executor.shutdown
+        @executor.wait_for_termination(timeout: 30)
+
+        processor.close
+        log(:info, 'Chronicles audit streaming service stopped')
+      end
+
+      # Check if service is running
+      #
+      # @return [Boolean]
+      def running?
+        @running
+      end
+
+      # Get service status information
+      #
+      # @return [Hash]
+      def status
+        {
+          running: running?,
+          adapter_streaming: adapter.streaming?,
+          adapter_position: adapter.current_position,
+          retry_count: @retry_count,
+          executor_status: {
+            active_count: @executor.active_count,
+            completed_task_count: @executor.completed_task_count,
+            queue_length: @executor.queue_length,
+          },
+        }
+      end
+
+      # Set up the audit streaming infrastructure
+      #
+      # @return [void]
+      def setup!
+        log(:info, 'Setting up audit streaming infrastructure')
+        adapter.setup
+        log(:info, 'Audit streaming infrastructure setup completed')
+      end
+
+      # Tear down the audit streaming infrastructure
+      #
+      # @return [void]
+      def teardown!
+        log(:info, 'Tearing down audit streaming infrastructure')
+        stop if running?
+        adapter.teardown
+        log(:info, 'Audit streaming infrastructure teardown completed')
+      end
+
+      private
+
+      def build_adapter
+        case Chronicles.config.adapter
+        when :postgresql
+          Adapters::PostgreSQL.new(logger: logger)
+        else
+          raise ConfigurationError, "Unsupported adapter: #{Chronicles.config.adapter}"
+        end
+      end
+
+      def validate_setup!
+        Chronicles.config.validate!
+
+        return if adapter.test_connection
+
+        raise AdapterError, 'Failed to connect to source database'
+      end
+
+      def test_connections!
+        adapter.test_connection
+        # Test audit processor connection by creating a dummy connection
+        processor.send(:ensure_audit_connection)
+      rescue StandardError => e
+        raise AdapterError, "Connection test failed: #{e.message}"
+      end
+
+      def start_streaming_with_retry
+        @executor.post do
+          loop do
+            break unless running?
+
+            begin
+              adapter.start_streaming do |change_event|
+                process_change_event(change_event)
+              end
+            rescue StandardError => e
+              handle_streaming_error(e)
+              break unless should_retry?
+            end
+          end
+        end
+      end
+
+      def process_change_event(change_event)
+        return unless change_event
+        return unless should_audit_table?(change_event)
+
+        log(:debug, 'Processing change event',
+          table: change_event.qualified_table_name,
+          action: change_event.action
+        )
+
+        processor.process(change_event)
+      rescue StandardError => e
+        log(:error, 'Failed to process change event',
+          error: e.message,
+          event: change_event.to_s
+        )
+      end
+
+      def should_audit_table?(change_event)
+        Chronicles.config.audit_table?(
+          change_event.table_name,
+          change_event.schema_name,
+        )
+      end
+
+      def handle_streaming_error(error)
+        @retry_count += 1
+        log(:error, 'Streaming error occurred',
+          error: error.message,
+          retry_count: @retry_count,
+          max_retries: Chronicles.config.max_retry_attempts
+        )
+
+        # Wait before retry
+        sleep(Chronicles.config.retry_delay) if should_retry?
+      end
+
+      def should_retry?
+        running? && @retry_count < Chronicles.config.max_retry_attempts
+      end
+
+      def log(level, message, context = {})
+        logger.public_send(level, message, service: 'Chronicles::Service', **context)
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/stream_adapter.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Abstract base class for database streaming adapters
+    #
+    # Defines the interface that all database-specific adapters must implement
+    # for streaming database changes into audit events.
+    class StreamAdapter
+      attr_reader :logger
+
+      def initialize(logger: Chronicles.logger)
+        @logger = logger
+        @running = false
+        @position = nil
+      end
+
+      # Start streaming database changes
+      #
+      # @param block [Proc] Block to call for each change event
+      # @return [void]
+      # @raise [NotImplementedError] Must be implemented by subclasses
+      def start_streaming(&)
+        raise NotImplementedError, "#{self.class} must implement #start_streaming"
+      end
+
+      # Stop streaming database changes
+      #
+      # @return [void]
+      # @raise [NotImplementedError] Must be implemented by subclasses
+      def stop_streaming
+        raise NotImplementedError, "#{self.class} must implement #stop_streaming"
+      end
+
+      # Get current replication position
+      #
+      # @return [String, nil] Current position or nil if not available
+      # @raise [NotImplementedError] Must be implemented by subclasses
+      def current_position
+        raise NotImplementedError, "#{self.class} must implement #current_position"
+      end
+
+      # Check if adapter is currently streaming
+      #
+      # @return [Boolean]
+      def streaming?
+        @running
+      end
+
+      # Set up the database for streaming (create publications, slots, etc.)
+      #
+      # @return [void]
+      # @raise [NotImplementedError] Must be implemented by subclasses
+      def setup
+        raise NotImplementedError, "#{self.class} must implement #setup"
+      end
+
+      # Tear down streaming setup (remove publications, slots, etc.)
+      #
+      # @return [void]
+      # @raise [NotImplementedError] Must be implemented by subclasses
+      def teardown
+        raise NotImplementedError, "#{self.class} must implement #teardown"
+      end
+
+      # Test connection to the database
+      #
+      # @return [Boolean] true if connection is successful
+      # @raise [NotImplementedError] Must be implemented by subclasses
+      def test_connection
+        raise NotImplementedError, "#{self.class} must implement #test_connection"
+      end
+
+      protected
+
+      attr_writer :running, :position
+
+      # Log a message with context
+      #
+      # @param level [Symbol] Log level (:info, :warn, :error, etc.)
+      # @param message [String] Log message
+      # @param context [Hash] Additional context
+      def log(level, message, context = {})
+        logger.public_send(level, message,
+          adapter: self.class.name.split('::').last,
+          position: current_position,
+          **context)
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    VERSION = '0.1.0.pre'
+  end
+end

data/lib/whodunit/chronicles.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'concurrent-ruby'
+require 'dry/configurable'
+require 'dry/logger'
+
+require_relative 'chronicles/version'
+require_relative 'chronicles/configuration'
+require_relative 'chronicles/change_event'
+require_relative 'chronicles/stream_adapter'
+require_relative 'chronicles/audit_processor'
+require_relative 'chronicles/service'
+
+# Adapters
+require_relative 'chronicles/adapters/postgresql'
+
+module Whodunit
+  # Chronicles - The complete historical record of `whodunit did what?` data
+  #
+  # While Whodunit tracks who made changes, Chronicles captures what changed
+  # by streaming database events into comprehensive audit trails with zero
+  # Rails application overhead.
+  module Chronicles
+    extend Dry::Configurable
+
+    # Configuration settings
+    setting :logger, default: Dry::Logger.new
+    setting :database_url, default: ENV.fetch('DATABASE_URL', nil)
+    setting :audit_database_url, default: ENV.fetch('AUDIT_DATABASE_URL', nil)
+    setting :adapter, default: :postgresql
+    setting :publication_name, default: 'whodunit_audit'
+    setting :replication_slot_name, default: 'whodunit_audit_slot'
+    setting :batch_size, default: 100
+    setting :max_retry_attempts, default: 3
+    setting :retry_delay, default: 5
+
+    class Error < StandardError; end
+    class ConfigurationError < Error; end
+    class AdapterError < Error; end
+    class ReplicationError < Error; end
+
+    # Configure Chronicles
+    #
+    # @example
+    #   Whodunit::Chronicles.configure do |config|
+    #     config.database_url = "postgresql://localhost/myapp"
+    #     config.audit_database_url = "postgresql://localhost/myapp_audit"
+    #     config.adapter = :postgresql
+    #   end
+    def self.configure
+      yield(config) if block_given?
+      config
+    end
+
+    # Get the configured logger
+    #
+    # @return [Dry::Logger]
+    def self.logger
+      config.logger
+    end
+
+    # Start the audit streaming service
+    #
+    # @return [Service]
+    def self.start
+      Service.new.start
+    end
+  end
+end

data/lib/whodunit-chronicles.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+# Main gem entry point
+require_relative 'whodunit/chronicles'