flu-rails 1.2.0

data/lib/flu-rails/core_ext.rb

@@ -0,0 +1,30 @@
+require_relative "active_record_extender"
+require_relative "action_controller_extender"
+
+module Flu
+  class CoreExt
+    def self.extend_model_classes(event_factory, event_publisher)
+      ActiveRecordExtender.extend_models(event_factory, event_publisher)
+    end
+
+    def self.extend_controller_classes(event_factory, event_publisher, logger)
+      ActionControllerExtender.extend_controllers(event_factory, event_publisher, logger)
+    end
+
+    def self.flu_tracker_request_id=(value)
+      Thread.current[:flu_tracker_request_id] = value
+    end
+
+    def self.flu_tracker_request_id
+      Thread.current[:flu_tracker_request_id]
+    end
+
+    def self.flu_tracker_request_entity_metadata=(value)
+      Thread.current[:flu_tracker_request_entity_metadata] = value
+    end
+
+    def self.flu_tracker_request_entity_metadata
+      Thread.current[:flu_tracker_request_entity_metadata]
+    end
+  end
+end

data/lib/flu-rails/dummy/in_memory_event_publisher.rb

@@ -0,0 +1,40 @@
+module Flu
+  module Dummy
+    class InMemoryEventPublisher < Flu::EventPublisher
+      
+      attr_reader :published_events_by_routing_key, :ordered_published_event_routing_keys
+  
+      def initialize(configuration)
+        @logger                               = configuration.logger
+        @configuration                        = configuration
+        @published_events_by_routing_key      = {}
+        @ordered_published_event_routing_keys = []
+      end
+  
+      def publish(event, persistent=true)
+        routing_key                                    = event.to_routing_key
+        published_events_by_routing_key[routing_key] ||= []
+        published_events_by_routing_key[routing_key].push(event)
+        @ordered_published_event_routing_keys.push(routing_key)
+      end
+  
+      def connect
+      end
+  
+      def events_count
+        @published_events_by_routing_key.map do | key, value |
+          value.size
+        end.sum
+      end
+  
+      def fetch_events(routing_key)
+        @published_events_by_routing_key[routing_key]
+      end
+  
+      def clear
+        @published_events_by_routing_key      = {}
+        @ordered_published_event_routing_keys = []
+      end
+    end
+  end
+end

data/lib/flu-rails/event.rb

@@ -0,0 +1,106 @@
+require "json"
+
+module Flu
+  class Event
+
+    attr_reader :data
+
+    def initialize(uuid, emitter, kind, name, data)
+      raise ArgumentError, "uuid must not be nil"              if uuid.nil?
+      raise ArgumentError, "emitter must not be nil nor empty" if emitter.nil? || emitter.length == 0
+      raise ArgumentError, "kind must not be nil nor empty"    if kind.nil?    || kind.length == 0
+      raise ArgumentError, "name must not be nil nor empty"    if name.nil?    || name.length == 0
+
+      @meta = {
+        id:        uuid,
+        name:      name,
+        emitter:   emitter,
+        timestamp: Time.now.utc,
+        kind:      kind,
+        status:    :new
+      }
+      @data = data || {}
+    end
+
+    def to_routing_key
+      "#{@meta[:status]}.#{@meta[:emitter]}.#{@meta[:kind]}.#{@meta[:name]}"
+    end
+
+    def to_json(options=nil)
+      JSON.dump({
+        meta: @meta,
+        data: map_complex_object(@data)
+      })
+    end
+
+    def id
+      @meta[:id]
+    end
+
+    def timestamp=(new_timestamp)
+      @meta[:timestamp] = new_timestamp
+    end
+
+    def mark_as_replayed
+      @meta[:status] = :replayed
+    end
+
+    def emitter
+      @meta[:emitter]
+    end
+
+    def timestamp
+      @meta[:timestamp]
+    end
+
+    def kind
+      @meta[:kind]
+    end
+
+    def name
+      @meta[:name]
+    end
+
+    def status
+      @meta[:status]
+    end
+
+    private
+
+    def map_complex_object(object)
+      if object.is_a?(Array)
+        map_array(object)
+      elsif object.is_a?(Hash)
+        map_hash(object)
+      elsif object.is_a?(ActionDispatch::Http::UploadedFile)
+        map_file(object)
+      else
+        object
+      end
+    end
+
+    def map_array(object)
+      array = []
+      object.each do |value|
+        array.push(map_complex_object(value))
+      end
+      array
+    end
+
+    def map_hash(object)
+      hash = {}
+      object.each do |key, value|
+        hash[key] = map_complex_object(value)
+      end
+      hash
+    end
+
+    def map_file(object)
+      {
+        "file_name":    object.original_filename,
+        "content_type": object.content_type
+      }
+    end
+  end
+end
+

data/lib/flu-rails/event_factory.rb

@@ -0,0 +1,117 @@
+require "securerandom"
+require_relative "event"
+
+module Flu
+  class EventFactory
+    def initialize(configuration)
+      @logger                         = configuration.logger
+      @configuration                  = configuration
+      @emitter                        = configuration.application_name
+      @default_ignored_model_changes  = configuration.default_ignored_model_changes.map(&:to_s)
+      @default_ignored_request_params = configuration.default_ignored_request_params
+    end
+
+    def build_request_event(data)
+      raise ArgumentError, "data must not be nil"              if data.nil?
+      raise ArgumentError, "data must have an action_name"     if !data.has_key?(:action_name) || data[:action_name].empty?
+      raise ArgumentError, "data must have an controller_name" if !data.has_key?(:controller_name) || data[:controller_name].empty?
+      name  = "request to #{data[:action_name]} #{data[:controller_name]}"
+      event = build_event(name, :request, data)
+      @logger.debug("Track action: #{JSON.pretty_generate(event)}")
+      event
+    end
+
+    def build_entity_change_event(data)
+      raise ArgumentError, "data must not be nil"          if data.nil?
+      raise ArgumentError, "data must have changes"        if !data.has_key?(:changes) || data[:changes].empty?
+      raise ArgumentError, "data must have an action_name" if !data.has_key?(:action_name) || data[:action_name].empty?
+      raise ArgumentError, "data must have an entity_name" if !data.has_key?(:entity_name) || data[:entity_name].empty?
+      name  = "#{data[:action_name]} #{data[:entity_name]}"
+      event = build_event(name, :entity_change, data)
+      @logger.debug("Track change: " + JSON.pretty_generate(event))
+      event
+    end
+
+    def build_manual_event(name, data)
+      raise ArgumentError, "data must not be nil"          if data.nil?
+      raise ArgumentError, "data must be a hash"           if !data.is_a?(Hash)
+      raise ArgumentError, "name must not be nil or empty" if name.nil? || name.empty?
+      event = build_event(name.to_s, :manual, data)
+      @logger.debug("Track manual: " + JSON.pretty_generate(event))
+      event
+    end
+
+    def build_event(name, kind, data)
+      original_emitter  = @emitter
+      overriden_emitter = data[:overriden_emitter]&.strip&.delete(".")
+      final_emitter     = overriden_emitter.blank? ? original_emitter : overriden_emitter
+      Event.new(SecureRandom.uuid, final_emitter, kind, name, deep_camelize(data))
+    end
+
+    def create_data_from_entity_changes(action_name, entity, request_id, request_entity_metadata, changes, user_metadata_lambda, association_columns, ignored_model_changes, flu_overriden_emitter_lambda)
+      {
+        entity_id:         entity.id,
+        entity_name:       entity.class.name.underscore,
+        overriden_emitter: flu_overriden_emitter_lambda ? entity.instance_exec(&flu_overriden_emitter_lambda) : nil,
+        request_id:        request_id,
+        request_metadata:  request_entity_metadata.nil? ? {} : request_entity_metadata,
+        action_name:       action_name,
+        changes:           changes.except(*ignored_model_changes).except(*@default_ignored_model_changes),
+        user_metadata:     user_metadata_lambda ? entity.instance_exec(&user_metadata_lambda) : {},
+        associations:      extract_associations_from(entity, association_columns)
+      }
+    end
+
+    def create_data_from_request(request_id, params, request, response, request_start_time, ignored_request_params)
+      {
+        request_id:      request_id,
+        controller_name: params[:controller],
+        action_name:     params[:action],
+        path:            request.original_fullpath,
+        response_code:   response.status,
+        user_agent:      request.user_agent,
+        duration:        Time.zone.now - request_start_time,
+        params:          params.except(*ignored_request_params).except(*@default_ignored_request_params).to_h
+      }
+    end
+
+    private
+
+    def deep_camelize(value)
+      case value
+      when Array
+        value.map { |v| deep_camelize v }
+      when Hash
+        value.reduce({}) do |camelized_hash, (k,v)|
+          camelized_hash[camelize(sanitize(k.to_s), false)] = deep_camelize v
+          camelized_hash
+        end
+      else
+        sanitize(value)
+      end
+    end
+
+    def camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
+      if first_letter_in_uppercase
+        lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+      else
+        (lower_case_and_underscored_word[0] || "") + camelize(lower_case_and_underscored_word)[1..-1]
+      end
+    end
+
+    def extract_associations_from(entity, association_columns)
+      association_columns.reduce({}) do |associations, column_name|
+        associations[column_name] = entity[column_name]
+        associations
+      end
+    end
+
+    def sanitize(value)
+      if value.respond_to?(:encode)
+        value.encode("UTF-8", invalid: :replace, undef: :replace).delete("\u0000")
+      else
+        value
+      end
+    end
+  end
+end

data/lib/flu-rails/event_publisher.rb

@@ -0,0 +1,51 @@
+require "bunny"
+require_relative "event"
+
+
+module Flu
+  class EventPublisher
+    def initialize(configuration)
+      @logger        = configuration.logger
+      @configuration = configuration
+    end
+
+    def publish(event, persistent=true)
+      routing_key = event.to_routing_key
+      @logger.debug("Publishing event with id '#{event.id}' with routing key: #{routing_key}")
+      @exchange.publish(event.to_json, routing_key: routing_key, persistent: persistent)
+      @logger.debug("Event published.")
+    end
+
+    def connect
+      connected = false
+      while !connected
+        begin
+          connect_to_exchange
+          connected = true
+        rescue Bunny::TCPConnectionFailedForAllHosts
+          @logger.warn("RabbitMQ connection failed, try again in 1 second.")
+          sleep 1
+        end
+      end
+    end
+
+    private
+
+    def connect_to_exchange
+      options = {
+        host:     @configuration.rabbitmq_host,
+        port:     @configuration.rabbitmq_port&.to_i,
+        user:     @configuration.rabbitmq_user,
+        password: @configuration.rabbitmq_password,
+        automatically_recover: true
+      }.merge(@configuration.bunny_options || {})
+      
+      @connection = Bunny.new(options)
+      @connection.start
+      @channel  = @connection.create_channel
+      @exchange = @channel.send(:topic,
+                                @configuration.rabbitmq_exchange_name,
+                                durable: @configuration.rabbitmq_exchange_durable)
+    end
+  end
+end

data/lib/flu-rails/queue_repository.rb

@@ -0,0 +1,40 @@
+require "rabbitmq/http/client"
+
+module Flu
+  class QueueRepository
+    def initialize(configuration)
+      @logger            = configuration.logger
+      @configuration     = configuration
+      @management_client = create_management_client(configuration)
+    end
+
+    def find_all
+      @management_client.list_queues
+    end
+
+    def find_queue(name)
+      @management_client.queue_info("/", name)
+    end
+
+    def find_bindings_for_queue(name)
+      @management_client.list_queue_bindings("/", name)
+    end
+
+    def purge_queue(name)
+      @management_client.purge_queue("/", name)
+    end
+
+    def delete_queue(name)
+      @management_client.delete_queue("/", name)
+    end
+
+    private
+
+    def create_management_client(configuration)
+      rabbitmq_url = "#{configuration.rabbitmq_management_scheme || "http"}://#{configuration.rabbitmq_host}:#{configuration.rabbitmq_management_port}/"
+      RabbitMQ::HTTP::Client.new(rabbitmq_url,
+                                 username: configuration.rabbitmq_user,
+                                 password: configuration.rabbitmq_password)
+    end
+  end
+end

data/lib/flu-rails/railtie.rb

@@ -0,0 +1,14 @@
+module Flu
+  class Railtie < Rails::Railtie
+    railtie_name :flu
+
+    config.to_prepare do
+      Flu.init
+      Flu.start
+    end
+
+    config.after_initialize do
+      Flu.start
+    end
+  end
+end

data/lib/flu-rails/util.rb

@@ -0,0 +1,71 @@
+module Flu
+  module Util
+    class ExportService
+      def export_existing_entities_to_events(event_publisher, event_factory, entity_types=nil)
+        entity_types ||= find_all_tracked_entity_types(entity_types)
+        check_that_all_entity_types_have_created_at(entity_types)
+
+        Flu.logger.level             = Logger::WARN
+        total_number_of_entity_types = entity_types.size
+        current_entity_type_index    = 0
+        entity_types.each do | entity_type |
+          entities                     = entity_type.all
+          total_number_of_entities     = entities.size
+          current_entity_index         = 0
+          current_entity_type_index   += 1
+          association_columns          = entity_type.flu_association_columns
+          user_metadata_lambda         = entity_type.flu_user_metadata_lambdas[:create]
+          ignored_model_changes        = entity_type.flu_ignored_model_changes
+
+          entities.each do |entity|
+            print "\r"      unless current_entity_index == 0
+            current_entity_index += 1
+            print "#{entity_type} (#{current_entity_type_index}/#{total_number_of_entity_types}) : #{current_entity_index}/#{total_number_of_entities}"
+            data            = extract_data_from(entity, event_factory, user_metadata_lambda, association_columns, ignored_model_changes)
+            event           = event_factory.build_entity_change_event(data)
+            event.timestamp = entity.created_at unless entity.created_at.nil?
+            event_publisher.publish(event)
+            print "\n"      if current_entity_index == total_number_of_entities
+          end
+        end
+      end
+
+      private
+
+      def check_that_all_entity_types_have_created_at(entity_types)
+        entity_types_without_timestamp = entity_types.select do | entity_type |
+          !entity_type.attribute_names.include?("created_at")
+        end
+        if (entity_types_without_timestamp.size > 0)
+          raise "#{entity_types_without_timestamp.size} entities do not have 'created_at': #{entity_types_without_timestamp.map {|type| type.name}}"
+        end
+      end
+
+      def find_all_entity_types
+        eager_loaded = Rails.application.eager_load!
+        Zeitwerk::Loader.eager_load_all if !eager_loaded && defined?(Zeitwerk)
+        ActiveRecord::Base.descendants
+      end
+
+      def find_all_tracked_entity_types(entity_types)
+        (entity_types || find_all_entity_types).select do | entity_type |
+          ActiveRecord::Base.connection.table_exists?(entity_type.table_name) && entity_type.flu_is_tracked
+        end
+      end
+
+      def extract_data_from(entity, event_factory, user_metadata_lambda, association_columns, ignored_model_changes)
+        changes = create_changes_from_existing(entity)
+        event_factory.create_data_from_entity_changes(:create, entity, nil, nil, changes, user_metadata_lambda, association_columns, ignored_model_changes, nil)
+      end
+
+      def create_changes_from_existing(entity)
+        changes = entity.attribute_names.inject({}) do | result, attribute_name |
+          attribute_value             = entity.attributes[attribute_name]
+          result[attribute_name.to_s] = [nil, attribute_value] unless attribute_value.nil?
+          result
+        end
+        changes.except(:created_at, :updated_at)
+      end
+    end
+  end
+end

data/lib/flu-rails/version.rb

@@ -0,0 +1,3 @@
+module Flu
+  VERSION = "1.2.0"
+end

data/lib/flu-rails.rb

@@ -0,0 +1,88 @@
+require "logger"
+require "json"
+require_relative "flu-rails/version"
+require_relative "flu-rails/event"
+require_relative "flu-rails/event_factory"
+require_relative "flu-rails/queue_repository"
+require_relative "flu-rails/configuration"
+require_relative "flu-rails/core_ext"
+require_relative "flu-rails/event_publisher"
+require_relative "flu-rails/util"
+require_relative "flu-rails/dummy/in_memory_event_publisher"
+require_relative "flu-rails/railtie" if defined?(Rails)
+
+module Flu
+  def self.configure
+    yield @configuration ||= Flu::Configuration.new
+  end
+
+  def self.config
+    @configuration
+  end
+
+  def self.logger
+    @logger
+  end
+
+  def self.event_factory
+    @event_factory
+  end
+
+  def self.event_publisher
+    @event_publisher
+  end
+
+  def self.init
+    raise "configuration.application_name must not be nil" if @configuration.application_name.nil?
+    @logger          = @configuration.logger
+    @event_factory   = Flu::EventFactory.new(@configuration)
+    @event_publisher = create_event_publisher(@configuration)
+    extend_models_and_controllers
+  end
+
+  def self.create_event_publisher(configuration)
+    if is_testing_environment?
+      logger.info("Loading Flu with a dummy event publisher (this will not connect any exchange)")
+      require_relative "flu-rails/dummy/in_memory_event_publisher"
+      Flu::Dummy::InMemoryEventPublisher.new(@configuration)
+    else
+      Flu::EventPublisher.new(@configuration)
+    end
+  end
+
+  def self.is_testing_environment?
+    defined?(Rails) && config.development_environments.include?(Rails.env)
+  end
+
+  def self.extend_models_and_controllers
+    Flu::CoreExt.extend_model_classes(@event_factory, @event_publisher)
+    Flu::CoreExt.extend_controller_classes(@event_factory, @event_publisher, @logger)
+  end
+
+  def self.start
+    @event_publisher.connect if config.auto_connect_to_exchange
+  end
+
+  def self.load_configuration
+    configure do |config|
+      config.development_environments       = []
+      config.rejected_user_agents           = []
+      config.logger                         = ::Logger.new(STDOUT)
+      config.rabbitmq_host                  = "localhost"
+      config.rabbitmq_port                  = 5672
+      config.rabbitmq_management_scheme     = "http"
+      config.rabbitmq_management_port       = 15672
+      config.rabbitmq_user                  = ""
+      config.rabbitmq_password              = ""
+      config.rabbitmq_exchange_name         = "events"
+      config.rabbitmq_exchange_durable      = true
+      config.auto_connect_to_exchange       = true
+      config.default_ignored_model_changes  = [:password, :password_confirmation, :created_at, :updated_at]
+      config.default_ignored_request_params = [:password, :password_confirmation, :controller, :action]
+      config.application_name               = defined?(Rails) ? Rails.application.class.module_parent_name : nil
+      config.bunny_options                  = {}
+    end
+  end
+
+  load_configuration
+end