whodunit-chronicles 0.1.0 → 0.3.0

data/lib/whodunit/chronicles/connection.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Whodunit
+  module Chronicles
+    # Handles database connections for chronicles processing
+    #
+    # Provides adapter-agnostic connection management for both PostgreSQL and MySQL
+    module Connection
+      private
+
+      def create_connection
+        audit_url = @audit_database_url || Chronicles.config.database_url
+
+        case detect_database_type(audit_url)
+        when :postgresql
+          require 'pg'
+          PG.connect(audit_url)
+        when :mysql
+          require 'trilogy'
+          parsed = parse_mysql_url(audit_url)
+          Trilogy.new(
+            host: parsed[:host],
+            port: parsed[:port] || 3306,
+            username: parsed[:username],
+            password: parsed[:password],
+            database: parsed[:database],
+            ssl: parsed[:ssl],
+          )
+        else
+          raise ConfigurationError, 'Unsupported database type for connection'
+        end
+      end
+
+      def detect_database_type(url)
+        return Chronicles.config.adapter unless url
+        return :postgresql if url.start_with?('postgres://', 'postgresql://')
+        return :mysql if url.start_with?('mysql://', 'mysql2://')
+
+        # Fallback to configured adapter
+        Chronicles.config.adapter
+      end
+
+      def parse_mysql_url(url)
+        return {} if url.nil? || url.empty?
+
+        uri = URI.parse(url)
+        {
+          host: uri.host,
+          port: uri.port,
+          username: uri.user,
+          password: uri.password,
+          database: uri.path&.sub('/', ''),
+          ssl: uri.query&.include?('ssl=true'),
+        }
+      end
+
+      def connection_active?
+        case detect_database_type(@audit_database_url || Chronicles.config.database_url)
+        when :postgresql
+          @connection && !@connection.finished?
+        when :mysql
+          @connection&.ping
+        else
+          false
+        end
+      end
+
+      def setup_connection_specifics
+        case detect_database_type(@audit_database_url || Chronicles.config.database_url)
+        when :postgresql
+          @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)
+        when :mysql
+          # MySQL/Trilogy doesn't need special setup
+        end
+      end
+
+      def ensure_connection
+        return if @connection && connection_active?
+
+        @connection = create_connection
+        setup_connection_specifics
+        ensure_table_exists
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/errors.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Base error class for all whodunit-chronicles errors.
+    #
+    # Rescuing this class catches every error raised by the gem:
+    #
+    #   begin
+    #     Whodunit::Chronicles.service.start
+    #   rescue Whodunit::Chronicles::Error => e
+    #     logger.error("Chronicles failure: #{e.message}")
+    #   end
+    class Error < StandardError; end
+
+    # Raised when the gem is configured with invalid or missing settings.
+    #
+    # @example
+    #   Whodunit::Chronicles.configure do |c|
+    #     c.adapter = :unknown_db   # => raises ConfigurationError
+    #   end
+    class ConfigurationError < Error; end
+
+    # Raised when a required database driver gem is not installed.
+    #
+    # @example message
+    #   "Could not load the 'pg' gem required for the postgresql adapter.
+    #    Add `gem 'pg', '~> 1.5'` to your Gemfile."
+    class AdapterLoadError < Error; end
+
+    # Raised when a streaming or replication connection fails.
+    class ConnectionError < Error; end
+
+    # Raised when processing a change event fails.
+    class ProcessingError < Error; end
+
+    # Raised when writing an audit record to the audit database fails.
+    class PersistenceError < Error; end
+
+    # Raised when replication related error occurs
+    class ReplicationError < Error; end
+  end
+end

data/lib/whodunit/chronicles/persistence.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Handles record persistence for different database adapters
+    #
+    # Provides adapter-specific SQL for inserting chronicle records
+    module Persistence
+      private
+
+      def persist_record(record)
+        db_type = detect_database_type(@audit_database_url || Chronicles.config.database_url)
+
+        case db_type
+        when :postgresql
+          persist_record_postgresql(record)
+        when :mysql
+          persist_record_mysql(record)
+        end
+      end
+
+      def persist_record_postgresql(record)
+        sql = <<~SQL
+          INSERT INTO whodunit_chronicles_audits (
+            table_name, schema_name, record_id, action, old_data, new_data, changes,
+            user_id, user_type, transaction_id, sequence_number, occurred_at, created_at, metadata
+          ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
+          RETURNING id
+        SQL
+
+        params = build_record_params(record)
+        result = @connection.exec_params(sql, params)
+        record[:id] = result.first['id'].to_i
+        result.clear
+
+        record
+      end
+
+      def persist_record_mysql(record)
+        sql = <<~SQL
+          INSERT INTO whodunit_chronicles_audits (
+            table_name, schema_name, record_id, action, old_data, new_data, changes,
+            user_id, user_type, transaction_id, sequence_number, occurred_at, created_at, metadata
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        SQL
+
+        params = build_record_params(record)
+        @connection.execute(sql, *params)
+        record[:id] = @connection.last_insert_id
+
+        record
+      end
+
+      def persist_records_batch(records)
+        return records if records.empty?
+
+        db_type = detect_database_type(@audit_database_url || Chronicles.config.database_url)
+
+        case db_type
+        when :postgresql
+          persist_records_batch_postgresql(records)
+        when :mysql
+          persist_records_batch_mysql(records)
+        end
+      end
+
+      def persist_records_batch_postgresql(records)
+        # Use multi-row INSERT for better performance
+        values_clauses = []
+        all_params = []
+        param_index = 1
+
+        records.each do |record|
+          param_positions = (param_index..(param_index + 13)).map { |i| "$#{i}" }.join(', ')
+          values_clauses << "(#{param_positions})"
+          all_params.concat(build_record_params(record))
+          param_index += 14
+        end
+
+        sql = <<~SQL
+          INSERT INTO whodunit_chronicles_audits (
+            table_name, schema_name, record_id, action, old_data, new_data, changes,
+            user_id, user_type, transaction_id, sequence_number, occurred_at, created_at, metadata
+          ) VALUES #{values_clauses.join(', ')}
+          RETURNING id
+        SQL
+
+        result = @connection.exec_params(sql, all_params)
+
+        # Set IDs on the records
+        result.each_with_index do |row, index|
+          records[index][:id] = row['id'].to_i
+        end
+
+        result.clear
+        records
+      end
+
+      def persist_records_batch_mysql(records)
+        # For MySQL, we'll use individual inserts in a transaction for simplicity
+        # A more optimized version could use VALUES() with multiple rows
+        records.each do |record|
+          persist_record_mysql(record)
+        end
+
+        records
+      end
+
+      def build_record_params(record)
+        [
+          record[:table_name],
+          record[:schema_name],
+          record[:record_id].to_json,
+          record[:action],
+          record[:old_data]&.to_json,
+          record[:new_data]&.to_json,
+          record[:changes].to_json,
+          record[:user_id],
+          record[:user_type],
+          record[:transaction_id],
+          record[:sequence_number],
+          record[:occurred_at],
+          record[:created_at],
+          record[:metadata].to_json,
+        ]
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/processor.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Processes database change events and creates chronicle records
+    #
+    # Transforms ChangeEvent objects into structured chronicle records
+    # with complete object serialization and metadata.
+    class Processor
+      include Connection
+      include Table
+      include Persistence
+
+      attr_reader :logger, :connection
+
+      def initialize(
+        audit_database_url: Chronicles.config.audit_database_url,
+        logger: Chronicles.logger
+      )
+        @audit_database_url = audit_database_url
+        @logger = logger
+        @connection = nil
+      end
+
+      # Process a change event and create chronicle record
+      #
+      # @param change_event [ChangeEvent] The database change to chronicle
+      # @return [Hash] The created chronicle record
+      def process(change_event)
+        ensure_connection
+
+        record = build_record(change_event)
+        persist_record(record)
+
+        log(:debug, 'Processed change event',
+          table: change_event.qualified_table_name,
+          action: change_event.action,
+          id: record[:id])
+
+        record
+      rescue StandardError => e
+        log(:error, 'Failed to process change event',
+          error: e.message,
+          event: change_event.to_s)
+        raise
+      end
+
+      # Process multiple change events in a batch
+      #
+      # @param change_events [Array<ChangeEvent>] Array of change events
+      # @return [Array<Hash>] Array of created chronicle records
+      def process_batch(change_events)
+        return [] if change_events.empty?
+
+        ensure_connection
+
+        records = change_events.map { |event| build_record(event) }
+        persist_records_batch(records)
+
+        log(:info, 'Processed batch of change events', count: change_events.size)
+
+        records
+      rescue StandardError => e
+        log(:error, 'Failed to process batch',
+          error: e.message,
+          count: change_events.size)
+        raise
+      end
+
+      # Close database connection
+      def close
+        @connection&.close
+        @connection = nil
+      end
+
+      private
+
+      def build_record(change_event)
+        user_info = extract_user_info(change_event)
+
+        {
+          id: nil, # Will be set by database
+          table_name: change_event.table_name,
+          schema_name: change_event.schema_name,
+          record_id: change_event.primary_key,
+          action: change_event.action,
+          old_data: change_event.old_data,
+          new_data: change_event.new_data,
+          changes: change_event.changes,
+          user_id: user_info[:user_id],
+          user_type: user_info[:user_type],
+          transaction_id: change_event.transaction_id,
+          sequence_number: change_event.sequence_number,
+          occurred_at: change_event.timestamp,
+          created_at: Time.now,
+          metadata: build_metadata(change_event),
+        }
+      end
+
+      def extract_user_info(change_event)
+        data = change_event.current_data || {}
+
+        # Look for Whodunit user attribution fields
+        user_id = data['creator_id'] || data['updater_id'] || data['deleter_id']
+
+        {
+          user_id: user_id,
+          user_type: user_id ? 'User' : nil,
+        }
+      end
+
+      def build_metadata(change_event)
+        {
+          table_schema: change_event.schema_name,
+          qualified_table_name: change_event.qualified_table_name,
+          changed_columns: change_event.changed_columns,
+          adapter_metadata: change_event.metadata,
+          chronicles_version: Chronicles::VERSION,
+        }
+      end
+
+      def log(level, message, context = {})
+        logger.public_send(level, message, processor: 'Processor', **context)
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/service.rb

@@ -4,10 +4,10 @@ require 'concurrent-ruby'
 
 module Whodunit
   module Chronicles
-    # Main service orchestrator for audit streaming
+    # Main service orchestrator for chronicle streaming
     #
-    # Coordinates the stream adapter and audit processor to provide
-    # a complete audit streaming solution with error handling and monitoring.
+    # Coordinates the stream adapter and processor to provide
+    # a complete chronicle streaming solution with error handling and monitoring.
     class Service
       attr_reader :adapter, :processor, :logger, :executor
 
@@ -17,7 +17,7 @@ module Whodunit
         logger: Chronicles.logger
       )
         @adapter = adapter || build_adapter
-        @processor = processor || AuditProcessor.new(logger: logger)
+        @processor = processor || Processor.new(logger: logger)
         @logger = logger
         @executor = Concurrent::ThreadPoolExecutor.new(
           min_threads: 1,
@@ -29,13 +29,13 @@ module Whodunit
         @retry_count = 0
       end
 
-      # Start the audit streaming service
+      # Start the chronicle streaming service
       #
       # @return [self]
       def start
         return self if running?
 
-        log(:info, 'Starting Chronicles audit streaming service')
+        log(:info, 'Starting Chronicles streaming service')
 
         validate_setup!
         test_connections!
@@ -45,7 +45,7 @@ module Whodunit
 
         start_streaming_with_retry
 
-        log(:info, 'Chronicles audit streaming service started successfully')
+        log(:info, 'Chronicles streaming service started successfully')
         self
       rescue StandardError => e
         log(:error, 'Failed to start service', error: e.message)
@@ -53,13 +53,13 @@ module Whodunit
         raise
       end
 
-      # Stop the audit streaming service
+      # Stop the chronicle streaming service
       #
       # @return [void]
       def stop
         return unless running?
 
-        log(:info, 'Stopping Chronicles audit streaming service')
+        log(:info, 'Stopping Chronicles streaming service')
         @running = false
 
         adapter.stop_streaming if adapter.streaming?
@@ -67,7 +67,7 @@ module Whodunit
         @executor.wait_for_termination(timeout: 30)
 
         processor.close
-        log(:info, 'Chronicles audit streaming service stopped')
+        log(:info, 'Chronicles streaming service stopped')
       end
 
       # Check if service is running
@@ -94,23 +94,23 @@ module Whodunit
         }
       end
 
-      # Set up the audit streaming infrastructure
+      # Set up the chronicle streaming infrastructure
       #
       # @return [void]
       def setup!
-        log(:info, 'Setting up audit streaming infrastructure')
+        log(:info, 'Setting up chronicle streaming infrastructure')
         adapter.setup
-        log(:info, 'Audit streaming infrastructure setup completed')
+        log(:info, 'Chronicle streaming infrastructure setup completed')
       end
 
-      # Tear down the audit streaming infrastructure
+      # Tear down the chronicle streaming infrastructure
       #
       # @return [void]
       def teardown!
-        log(:info, 'Tearing down audit streaming infrastructure')
+        log(:info, 'Tearing down chronicle streaming infrastructure')
         stop if running?
         adapter.teardown
-        log(:info, 'Audit streaming infrastructure teardown completed')
+        log(:info, 'Chronicle streaming infrastructure teardown completed')
       end
 
       private
@@ -118,7 +118,9 @@ module Whodunit
       def build_adapter
         case Chronicles.config.adapter
         when :postgresql
-          Adapters::PostgreSQL.new(logger: logger)
+          Whodunit::Chronicles::AdapterLoader.load(:postgresql, logger:)
+        when :mysql
+          Whodunit::Chronicles::AdapterLoader.load(:mysql, logger:)
         else
           raise ConfigurationError, "Unsupported adapter: #{Chronicles.config.adapter}"
         end
@@ -129,15 +131,15 @@ module Whodunit
 
         return if adapter.test_connection
 
-        raise AdapterError, 'Failed to connect to source database'
+        raise AdapterLoadError, 'Failed to connect to source database'
       end
 
       def test_connections!
         adapter.test_connection
-        # Test audit processor connection by creating a dummy connection
-        processor.send(:ensure_audit_connection)
+        # Test processor connection by creating a dummy connection
+        processor.send(:ensure_connection)
       rescue StandardError => e
-        raise AdapterError, "Connection test failed: #{e.message}"
+        raise AdapterLoadError, "Connection test failed: #{e.message}"
       end
 
       def start_streaming_with_retry
@@ -159,7 +161,7 @@ module Whodunit
 
       def process_change_event(change_event)
         return unless change_event
-        return unless should_audit_table?(change_event)
+        return unless should_chronicle_table?(change_event)
 
         log(:debug, 'Processing change event',
           table: change_event.qualified_table_name,
@@ -174,8 +176,8 @@ module Whodunit
         )
       end
 
-      def should_audit_table?(change_event)
-        Chronicles.config.audit_table?(
+      def should_chronicle_table?(change_event)
+        Chronicles.config.chronicle_table?(
           change_event.table_name,
           change_event.schema_name,
         )

data/lib/whodunit/chronicles/table.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Whodunit
+  module Chronicles
+    # Handles table creation for different database adapters
+    #
+    # Provides adapter-specific SQL for creating chronicles tables
+    module Table
+      private
+
+      def ensure_table_exists
+        db_type = detect_database_type(@audit_database_url || Chronicles.config.database_url)
+
+        case db_type
+        when :postgresql
+          create_postgresql_table
+        when :mysql
+          create_mysql_table
+        end
+      end
+
+      def create_postgresql_table
+        create_sql = <<~SQL
+          CREATE TABLE IF NOT EXISTS whodunit_chronicles_audits (
+            id BIGSERIAL PRIMARY KEY,
+            table_name TEXT NOT NULL,
+            schema_name TEXT NOT NULL DEFAULT 'public',
+            record_id JSONB,
+            action TEXT NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
+            old_data JSONB,
+            new_data JSONB,
+            changes JSONB,
+            user_id BIGINT,
+            user_type TEXT,
+            transaction_id TEXT,
+            sequence_number INTEGER,
+            occurred_at TIMESTAMP WITH TIME ZONE NOT NULL,
+            created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+            metadata JSONB DEFAULT '{}'::jsonb,
+          #{'  '}
+            -- Indexes for performance
+            CONSTRAINT valid_data_for_action CHECK (
+              (action = 'INSERT' AND old_data IS NULL AND new_data IS NOT NULL) OR
+              (action = 'UPDATE' AND old_data IS NOT NULL AND new_data IS NOT NULL) OR#{'  '}
+              (action = 'DELETE' AND old_data IS NOT NULL AND new_data IS NULL)
+            )
+          );
+
+          -- Performance indexes
+          CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chronicles_audits_table_record#{' '}
+            ON whodunit_chronicles_audits (table_name, (record_id->>'id'));
+
+          CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chronicles_audits_occurred_at#{' '}
+            ON whodunit_chronicles_audits (occurred_at DESC);
+
+          CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chronicles_audits_user#{' '}
+            ON whodunit_chronicles_audits (user_id, user_type);
+
+          CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chronicles_audits_action#{' '}
+            ON whodunit_chronicles_audits (action);
+
+          -- GIN index for JSONB columns
+          CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chronicles_audits_record_id_gin#{' '}
+            ON whodunit_chronicles_audits USING GIN (record_id);
+
+          CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_chronicles_audits_changes_gin#{' '}
+            ON whodunit_chronicles_audits USING GIN (changes);
+        SQL
+
+        @connection.exec(create_sql)
+      rescue PG::Error => e
+        # Ignore "already exists" errors from CONCURRENTLY
+        raise unless e.message.include?('already exists')
+      end
+
+      def create_mysql_table
+        create_sql = <<~SQL
+          CREATE TABLE IF NOT EXISTS whodunit_chronicles_audits (
+            id BIGINT AUTO_INCREMENT PRIMARY KEY,
+            table_name TEXT NOT NULL,
+            schema_name TEXT NOT NULL DEFAULT 'public',
+            record_id JSON,
+            action TEXT NOT NULL CHECK (action IN ('INSERT', 'UPDATE', 'DELETE')),
+            old_data JSON,
+            new_data JSON,
+            changes JSON,
+            user_id BIGINT,
+            user_type TEXT,
+            transaction_id TEXT,
+            sequence_number INTEGER,
+            occurred_at TIMESTAMP NOT NULL,
+            created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+            metadata JSON DEFAULT (JSON_OBJECT()),
+          #{'  '}
+            -- Constraint for data integrity
+            CONSTRAINT valid_data_for_action CHECK (
+              (action = 'INSERT' AND old_data IS NULL AND new_data IS NOT NULL) OR
+              (action = 'UPDATE' AND old_data IS NOT NULL AND new_data IS NOT NULL) OR
+              (action = 'DELETE' AND old_data IS NOT NULL AND new_data IS NULL)
+            ),
+          #{'  '}
+            -- Performance indexes
+            INDEX idx_chronicles_audits_table_record (table_name(255), (JSON_UNQUOTE(JSON_EXTRACT(record_id, '$.id')))),
+            INDEX idx_chronicles_audits_occurred_at (occurred_at DESC),
+            INDEX idx_chronicles_audits_user (user_id, user_type(255)),
+            INDEX idx_chronicles_audits_action (action(50))
+          );
+        SQL
+
+        @connection.query(create_sql)
+      rescue StandardError => e
+        # Ignore "already exists" errors
+        unless e.message.include?('already exists') ||
+               (e.message.include?('Table') && e.message.include?('already exists'))
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/whodunit/chronicles/version.rb

@@ -2,6 +2,6 @@
 
 module Whodunit
   module Chronicles
-    VERSION = '0.1.0'
+    VERSION = '0.3.0'
   end
 end