whodunit-chronicles 0.1.0 → 0.3.0

data/docker-compose.yml

@@ -0,0 +1,138 @@
+name: whodunit-chronicles
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Local development + CI test databases
+#
+# Quickstart:
+#   docker compose up -d
+#   bundle exec rake test
+#
+# Stop + wipe volumes:
+#   docker compose down -v
+# ──────────────────────────────────────────────────────────────────────────────
+
+services:
+  # ── PostgreSQL (logical replication enabled) ─────────────────────────────
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER:     chronicles
+      POSTGRES_PASSWORD: chronicles
+      POSTGRES_DB:       chronicles_test
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
+    command: >
+      postgres
+        -c wal_level=logical
+        -c max_replication_slots=10
+        -c max_wal_senders=10
+        -c log_replication_commands=on
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+      - ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U chronicles -d chronicles_test"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  # Separate audit database (mirrors production split)
+  postgres_audit:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER:     chronicles
+      POSTGRES_PASSWORD: chronicles
+      POSTGRES_DB:       chronicles_audit_test
+    ports:
+      - "${POSTGRES_AUDIT_PORT:-5433}:5432"
+    volumes:
+      - pg_audit_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U chronicles -d chronicles_audit_test"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  # ── MySQL (binary logging enabled) ───────────────────────────────────────
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: chronicles_root
+      MYSQL_USER:          chronicles
+      MYSQL_PASSWORD:      chronicles
+      MYSQL_DATABASE:      chronicles_test
+    ports:
+      - "3306:3306"
+    command: >
+      mysqld
+        --server-id=1
+        --log-bin=mysql-bin
+        --binlog-format=ROW
+        --binlog-row-image=FULL
+        --expire-logs-days=1
+        --gtid-mode=ON
+        --enforce-gtid-consistency=ON
+    volumes:
+      - mysql_data:/var/lib/mysql
+      - ./docker/mysql/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "chronicles", "-pchronicles"]
+      interval: 5s
+      timeout: 5s
+      retries: 15
+      start_period: 30s
+
+  # Separate audit database for MySQL
+  mysql_audit:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: chronicles_root
+      MYSQL_USER:          chronicles
+      MYSQL_PASSWORD:      chronicles
+      MYSQL_DATABASE:      chronicles_audit_test
+    ports:
+      - "3307:3306"
+    volumes:
+      - mysql_audit_data:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "chronicles", "-pchronicles"]
+      interval: 5s
+      timeout: 5s
+      retries: 15
+      start_period: 30s
+
+  # ── MariaDB (binary logging enabled) ─────────────────────────────────────
+  mariadb:
+    image: mariadb:11
+    environment:
+      MARIADB_ROOT_PASSWORD: chronicles_root
+      MARIADB_USER:          chronicles
+      MARIADB_PASSWORD:      chronicles
+      MARIADB_DATABASE:      chronicles_test
+    ports:
+      - "3308:3306"
+    command: >
+      mariadbd
+        --server-id=2
+        --log-bin=mariadb-bin
+        --binlog-format=ROW
+        --binlog-row-image=FULL
+        --expire-logs-days=1
+    volumes:
+      - mariadb_data:/var/lib/mysql
+      - ./docker/mysql/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      interval: 5s
+      timeout: 5s
+      retries: 15
+      start_period: 30s
+
+volumes:
+  pg_data:
+  pg_audit_data:
+  mysql_data:
+  mysql_audit_data:
+  mariadb_data:

data/lib/whodunit/chronicles/adapter_loader.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Lazily loads the correct database adapter gem at runtime.
+    #
+    # This avoids forcing all users to install both `pg` and `trilogy`
+    # regardless of which database they actually use.
+    #
+    # @example
+    #   adapter = AdapterLoader.load(:postgresql)
+    #   adapter = AdapterLoader.load(:mysql)
+    #   adapter = AdapterLoader.load(:mariadb)
+    module AdapterLoader
+      # Map of adapter type symbols to their required gem and class path.
+      ADAPTER_REGISTRY = {
+        postgresql: {
+          gem: 'pg',
+          require: 'whodunit/chronicles/adapters/postgresql',
+          class: 'Whodunit::Chronicles::Adapters::PostgreSQL',
+          hint: "Add `gem 'pg', '~> 1.5'` to your Gemfile.",
+        },
+        mysql: {
+          gem: 'trilogy',
+          require: 'whodunit/chronicles/adapters/mysql',
+          class: 'Whodunit::Chronicles::Adapters::MySQL',
+          hint: "Add `gem 'trilogy', '~> 2.9'` to your Gemfile.",
+        },
+        mariadb: {
+          gem: 'trilogy',
+          require: 'whodunit/chronicles/adapters/mysql',
+          class: 'Whodunit::Chronicles::Adapters::MySQL',
+          hint: "Add `gem 'trilogy', '~> 2.9'` to your Gemfile.",
+        },
+      }.freeze
+
+      # Load and instantiate an adapter by type.
+      #
+      # @param type [Symbol] one of :postgresql, :mysql, :mariadb
+      # @param options [Hash] options forwarded to the adapter constructor
+      # @return [Adapters::Base] the instantiated adapter
+      # @raise [Whodunit::Chronicles::ConfigurationError] for unknown adapter types
+      # @raise [Whodunit::Chronicles::AdapterLoadError] when the required gem is missing
+      def self.load(type, **)
+        config = ADAPTER_REGISTRY[type.to_sym]
+
+        unless config
+          known = ADAPTER_REGISTRY.keys.map(&:inspect).join(', ')
+          raise ConfigurationError,
+            "Unknown adapter type #{type.inspect}. Known adapters: #{known}"
+        end
+
+        load_gem!(config)
+        require config[:require]
+        Object.const_get(config[:class]).new(**)
+      rescue LoadError => e
+        raise AdapterLoadError,
+          "Could not load the '#{config[:gem]}' gem required for the " \
+          "#{type} adapter.\n#{config[:hint]}\nOriginal error: #{e.message}"
+      end
+
+      # @api private
+      def self.load_gem!(config)
+        require config[:gem]
+      end
+      private_class_method :load_gem!
+    end
+  end
+end

data/lib/whodunit/chronicles/adapters/mysql.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+require 'trilogy'
+require 'uri'
+
+module Whodunit
+  module Chronicles
+    module Adapters
+      # MySQL/MariaDB binary log streaming adapter
+      #
+      # Uses MySQL's binary log replication to stream database changes
+      # without impacting application performance.
+      class MySQL < Chronicles::StreamAdapter
+        DEFAULT_SERVER_ID = 1001
+
+        attr_reader :connection, :database_url, :server_id, :binlog_file, :binlog_position
+
+        def initialize(
+          database_url: Chronicles.config.database_url,
+          server_id: DEFAULT_SERVER_ID,
+          logger: Chronicles.logger
+        )
+          super(logger: logger)
+          @database_url = database_url
+          @server_id = server_id
+          @connection = nil
+          @binlog_file = nil
+          @binlog_position = nil
+          @binlog_checksum = true
+        end
+
+        # Start streaming binary log changes
+        def start_streaming(&)
+          raise ArgumentError, 'Block required for processing events' unless block_given?
+
+          log(:info, 'Starting MySQL binary log streaming')
+
+          establish_connection
+          ensure_setup
+
+          self.running = true
+          fetch_current_position
+
+          log(:info, 'Starting replication from position',
+            file: @binlog_file, position: @binlog_position)
+
+          begin
+            stream_binlog_events(&)
+          rescue StandardError => e
+            log(:error, 'Streaming error', error: e.message, backtrace: e.backtrace.first(5))
+            raise ReplicationError, "Failed to stream changes: #{e.message}"
+          ensure
+            self.running = false
+          end
+        end
+
+        # Stop streaming
+        def stop_streaming
+          log(:info, 'Stopping MySQL binary log streaming')
+          self.running = false
+          close_connection
+        end
+
+        # Get current replication position
+        def current_position
+          return "#{@binlog_file}:#{@binlog_position}" if @binlog_file && @binlog_position
+
+          fetch_current_position
+          "#{@binlog_file}:#{@binlog_position}"
+        end
+
+        # Set up binary log replication
+        def setup
+          log(:info, 'Setting up MySQL binary log replication')
+
+          establish_connection
+          validate_binlog_format
+          validate_server_id
+          enable_binlog_checksum
+
+          log(:info, 'MySQL setup completed successfully')
+        end
+
+        # Remove binary log replication setup (minimal cleanup needed)
+        def teardown
+          log(:info, 'Tearing down MySQL binary log replication')
+          close_connection
+          log(:info, 'MySQL teardown completed')
+        end
+
+        # Test database connection
+        def test_connection
+          establish_connection
+          result = @connection.query('SELECT @@hostname, @@version, @@server_id')
+          info = result.first
+
+          log(:info, 'Connection test successful',
+            hostname: info['@@hostname'],
+            version: info['@@version'],
+            server_id: info['@@server_id'])
+
+          true
+        rescue StandardError => e
+          log(:error, 'Connection test failed', error: e.message)
+          false
+        end
+
+        private
+
+        def establish_connection
+          return if @connection&.ping
+
+          parsed_url = parse_database_url(@database_url)
+
+          @connection = Trilogy.new(
+            host: parsed_url[:host],
+            port: parsed_url[:port] || 3306,
+            username: parsed_url[:username],
+            password: parsed_url[:password],
+            database: parsed_url[:database],
+            ssl: parsed_url[:ssl],
+          )
+
+          log(:debug, 'Established MySQL connection',
+            host: parsed_url[:host],
+            database: parsed_url[:database])
+        rescue StandardError => e
+          log(:error, 'Failed to establish connection', error: e.message)
+          raise AdapterLoadError, "Connection failed: #{e.message}"
+        end
+
+        def close_connection
+          @connection&.close
+          @connection = nil
+        end
+
+        def parse_database_url(url)
+          uri = URI.parse(url)
+          {
+            host: uri.host,
+            port: uri.port,
+            username: uri.user,
+            password: uri.password,
+            database: uri.path&.sub('/', ''),
+            ssl: uri.query&.include?('ssl=true'),
+          }
+        end
+
+        def ensure_setup
+          validate_binlog_format
+          validate_server_id
+        end
+
+        def validate_binlog_format
+          result = @connection.query('SELECT @@binlog_format')
+          format = result.first['@@binlog_format']
+
+          unless %w[ROW MIXED].include?(format)
+            raise ReplicationError,
+              "Binary log format must be ROW or MIXED, currently: #{format}"
+          end
+
+          log(:debug, 'Binary log format validated', format: format)
+        end
+
+        def validate_server_id
+          result = @connection.query('SELECT @@server_id')
+          current_server_id = result.first['@@server_id'].to_i
+
+          if current_server_id == @server_id
+            raise ReplicationError,
+              "Server ID conflict: #{@server_id} is already in use"
+          end
+
+          log(:debug, 'Server ID validated',
+            current: current_server_id,
+            replication: @server_id)
+        end
+
+        def enable_binlog_checksum
+          @connection.query('SET @master_binlog_checksum = @@global.binlog_checksum')
+          log(:debug, 'Binary log checksum enabled')
+        end
+
+        def fetch_current_position
+          result = @connection.query('SHOW MASTER STATUS')
+          status = result.first
+
+          raise ReplicationError, 'Unable to fetch master status - binary logging may be disabled' unless status
+
+          @binlog_file = status['File']
+          @binlog_position = status['Position']
+          log(:debug, 'Fetched master position',
+            file: @binlog_file,
+            position: @binlog_position)
+        end
+
+        def stream_binlog_events(&)
+          # Register as replica server
+          register_replica_server
+
+          # Request binary log dump
+          request_binlog_dump
+
+          # Process binary log events
+          process_binlog_stream(&)
+        rescue StandardError => e
+          log(:error, 'Binary log streaming error', error: e.message)
+          raise
+        end
+
+        def register_replica_server
+          # This would typically use COM_REGISTER_SLAVE MySQL protocol command
+          # For now, we'll use a simplified approach
+          log(:debug, 'Registering as replica server', server_id: @server_id)
+
+          # NOTE: Full implementation would require low-level MySQL protocol handling
+          # This is a placeholder for the binary log streaming setup
+        end
+
+        def request_binlog_dump
+          log(:debug, 'Requesting binary log dump',
+            file: @binlog_file,
+            position: @binlog_position)
+
+          # This would use COM_BINLOG_DUMP MySQL protocol command
+          # Full implementation requires binary protocol handling
+        end
+
+        def process_binlog_stream(&)
+          # This would process the binary log event stream
+          # Each event would be parsed and converted to a ChangeEvent
+
+          log(:info, 'Processing binary log stream (placeholder implementation)')
+
+          # Placeholder: In a real implementation, this would:
+          # 1. Read binary log events from the stream
+          # 2. Parse event headers and data
+          # 3. Convert to ChangeEvent objects
+          # 4. Yield each event to the block
+
+          # For now, we'll simulate with a warning
+          log(:warn, 'MySQL binary log streaming requires full protocol implementation')
+
+          # Yield a placeholder change event to demonstrate the interface
+          change_event = ChangeEvent.new(
+            table_name: 'example_table',
+            action: 'INSERT',
+            primary_key: { id: 1 },
+            new_data: { id: 1, name: 'test' },
+            old_data: nil,
+            timestamp: Time.now,
+            metadata: { position: current_position },
+          )
+
+          yield(change_event) if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/adapters/postgresql.rb

@@ -9,7 +9,7 @@ module Whodunit
       #
       # Uses PostgreSQL's logical replication functionality to stream
       # database changes via WAL decoding without impacting application performance.
-      class PostgreSQL < StreamAdapter
+      class PostgreSQL < Chronicles::StreamAdapter
         DEFAULT_PLUGIN = 'pgoutput'
 
         attr_reader :connection, :replication_connection, :publication_name, :slot_name

data/lib/whodunit/chronicles/change_event.rb

@@ -95,8 +95,8 @@ module Whodunit
       def changes
         return {} unless update? && old_data && new_data
 
-        changed_columns.each_with_object({}) do |column, changes_hash|
-          changes_hash[column] = [old_data[column], new_data[column]]
+        changed_columns.to_h do |column|
+          [column, [old_data[column], new_data[column]]]
         end
       end
 

data/lib/whodunit/chronicles/composite_processor.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Fans out a single change event to multiple processors in sequence.
+    #
+    # Use this to build pipelines — e.g. simultaneously storing audit records,
+    # streaming to Grafana, and triggering alerts — without coupling any single
+    # processor to the others.
+    #
+    # Each processor runs independently. If one raises, the error is logged and
+    # the remaining processors still execute (fail-open by default). Set
+    # `fail_fast: true` to instead halt the chain on the first error.
+    #
+    # @example Basic pipeline
+    #   service = Whodunit::Chronicles::Service.new(
+    #     adapter: adapter,
+    #     processor: Whodunit::Chronicles::CompositeProcessor.new([
+    #       AuditStoreProcessor.new,
+    #       AlertingProcessor.new,
+    #       GrafanaProcessor.new
+    #     ])
+    #   )
+    #
+    # @example Halt on first error
+    #   CompositeProcessor.new(processors, fail_fast: true)
+    #
+    class CompositeProcessor
+      # @param processors [Array<#process>] ordered list of processors to invoke
+      # @param fail_fast [Boolean] when true, halt the chain on the first error
+      # @param logger [Logger, nil] optional logger; defaults to Chronicles logger
+      def initialize(processors, fail_fast: false, logger: nil)
+        raise ArgumentError, 'processors must be an Array' unless processors.is_a?(Array)
+        raise ArgumentError, 'processors cannot be empty' if processors.empty?
+
+        @processors = processors
+        @fail_fast  = fail_fast
+        @logger     = logger || Whodunit::Chronicles.logger
+      end
+
+      # Process a change event through every processor in the chain.
+      #
+      # @param change_event [ChangeEvent] the event to process
+      # @return [void]
+      # @raise [ProcessingError] only when fail_fast is true and a child raises
+      def process(change_event)
+        errors = []
+
+        @processors.each do |processor|
+          processor.process(change_event)
+        rescue StandardError => e
+          raise ProcessingError, "#{processor.class} failed: #{e.message}" if @fail_fast
+
+          @logger.error { "CompositeProcessor: #{processor.class} raised #{e.class}: #{e.message}" }
+          errors << e
+        end
+
+        return if errors.empty?
+
+        @logger.warn do
+          "CompositeProcessor: #{errors.size} processor(s) failed for #{change_event.table_name}##{change_event.action}"
+        end
+      end
+
+      # @return [Integer] the number of processors in the chain
+      def size
+        @processors.size
+      end
+
+      # @return [Array<Class>] the processor classes in chain order
+      def processor_classes
+        @processors.map(&:class)
+      end
+
+      # Append a processor to the end of the chain.
+      #
+      # @param processor [#process] the processor to add
+      # @return [self]
+      def add(processor)
+        @processors << processor
+        self
+      end
+      alias << add
+    end
+  end
+end

data/lib/whodunit/chronicles/configuration.rb

@@ -30,21 +30,20 @@ module Whodunit
       # @raise [ConfigurationError] if configuration is invalid
       def validate!
         raise ConfigurationError, 'database_url is required' if database_url.nil?
-        raise ConfigurationError, 'adapter must be :postgresql' unless adapter == :postgresql
+        raise ConfigurationError, 'adapter must be :postgresql or :mysql' unless %i[postgresql mysql].include?(adapter)
         raise ConfigurationError, 'batch_size must be positive' unless batch_size.positive?
         raise ConfigurationError, 'max_retry_attempts must be positive' unless max_retry_attempts.positive?
         raise ConfigurationError, 'retry_delay must be positive' unless retry_delay.positive?
 
-        validate_publication_name!
-        validate_slot_name!
+        validate_adapter_specific_settings!
       end
 
-      # Check if a table should be audited based on filters
+      # Check if a table should be chronicled based on filters
       #
       # @param table_name [String] The table name to check
       # @param schema_name [String] The schema name to check
-      # @return [Boolean] true if the table should be audited
-      def audit_table?(table_name, schema_name = 'public')
+      # @return [Boolean] true if the table should be chronicled
+      def chronicle_table?(table_name, schema_name = 'public')
         return false if filtered_by_schema?(schema_name)
         return false if filtered_by_table?(table_name)
 
@@ -53,18 +52,30 @@ module Whodunit
 
       private
 
-      def validate_publication_name!
-        return if /\A[a-zA-Z_][a-zA-Z0-9_]*\z/.match?(publication_name)
-
-        raise ConfigurationError, 'publication_name must be a valid PostgreSQL identifier'
+      def validate_adapter_specific_settings!
+        case adapter
+        when :postgresql
+          validate_postgresql_settings!
+        when :mysql
+          validate_mysql_settings!
+        end
       end
 
-      def validate_slot_name!
-        return if /\A[a-zA-Z_][a-zA-Z0-9_]*\z/.match?(replication_slot_name)
+      def validate_postgresql_settings!
+        if publication_name && !/\A[a-zA-Z_][a-zA-Z0-9_]*\z/.match?(publication_name)
+          raise ConfigurationError, 'publication_name must be a valid PostgreSQL identifier'
+        end
+
+        return unless replication_slot_name && !/\A[a-zA-Z_][a-zA-Z0-9_]*\z/.match?(replication_slot_name)
 
         raise ConfigurationError, 'replication_slot_name must be a valid PostgreSQL identifier'
       end
 
+      def validate_mysql_settings!
+        # MySQL-specific validations can be added here in the future
+        # For now, MySQL settings are less restrictive
+      end
+
       def filtered_by_schema?(schema_name)
         return false unless schema_filter