behavior_analytics 0.1.0 → 2.0.0

data/lib/behavior_analytics/storage/kafka_adapter.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+begin
+  require "ruby-kafka"
+rescue LoadError
+  raise LoadError, "ruby-kafka gem is required for KafkaAdapter. Please add 'ruby-kafka' to your Gemfile."
+end
+
+module BehaviorAnalytics
+  module Storage
+    class KafkaAdapter < Adapter
+      def initialize(kafka: nil, topic: "behavior_events", producer: nil)
+        @kafka = kafka || Kafka.new(seed_brokers: ["localhost:9092"])
+        @topic = topic
+        @producer = producer || @kafka.producer
+      end
+
+      def save_events(events)
+        return if events.empty?
+
+        events.each do |event|
+          event_hash = event.is_a?(Hash) ? event : event.to_h
+          key = event_hash[:tenant_id] || "default"
+          value = serialize_event(event_hash)
+          
+          @producer.produce(value, topic: @topic, key: key)
+        end
+        
+        @producer.deliver_messages
+      rescue StandardError => e
+        raise Error, "Failed to save events to Kafka: #{e.message}"
+      end
+
+      def events_for_context(context, options = {})
+        context.validate!
+        
+        # Kafka is primarily for streaming, so we need a consumer
+        # This is a simplified version - in production you'd use a proper consumer group
+        consumer = @kafka.consumer(group_id: "behavior_analytics_#{context.tenant_id}")
+        consumer.subscribe(@topic)
+        
+        events = []
+        timeout = options[:timeout] || 5
+        
+        begin
+          consumer.each_message(max_wait_time: timeout) do |message|
+            event = deserialize_event(message.value)
+            
+            if matches_context?(event, context, options)
+              events << event
+              break if options[:limit] && events.size >= options[:limit]
+            end
+          end
+        rescue Kafka::Error
+          # Timeout or other Kafka errors
+        ensure
+          consumer.stop
+        end
+        
+        events
+      end
+
+      def delete_old_events(before_date)
+        # Kafka doesn't support deletion of old messages directly
+        # Messages are retained based on retention policy
+        # This is a no-op for Kafka
+      end
+
+      def event_count(context, options = {})
+        events_for_context(context, options).count
+      end
+
+      def unique_users(context, options = {})
+        events = events_for_context(context, options)
+        events.map { |e| e[:user_id] }.compact.uniq.count
+      end
+
+      private
+
+      def serialize_event(event_hash)
+        require "json"
+        JSON.generate(event_hash)
+      end
+
+      def deserialize_event(data)
+        require "json"
+        JSON.parse(data, symbolize_names: true)
+      end
+
+      def matches_context?(event, context, options)
+        return false unless event[:tenant_id] == context.tenant_id
+        return false if context.user_id && event[:user_id] != context.user_id
+        return false if context.user_type && event[:user_type] != context.user_type
+        return false if options[:event_name] && event[:event_name] != options[:event_name]
+        return false if options[:event_type] && event[:event_type] != options[:event_type]
+        
+        if options[:since]
+          event_time = Time.parse(event[:created_at].to_s)
+          return false if event_time < options[:since]
+        end
+        
+        if options[:until]
+          event_time = Time.parse(event[:created_at].to_s)
+          return false if event_time > options[:until]
+        end
+        
+        true
+      end
+    end
+  end
+end
+

data/lib/behavior_analytics/storage/redis_adapter.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+begin
+  require "redis"
+rescue LoadError
+  raise LoadError, "Redis gem is required for RedisAdapter. Please add 'redis' to your Gemfile."
+end
+
+module BehaviorAnalytics
+  module Storage
+    class RedisAdapter < Adapter
+      def initialize(redis: nil, key_prefix: "behavior_analytics", ttl: nil)
+        @redis = redis || Redis.new
+        @key_prefix = key_prefix
+        @ttl = ttl
+      end
+
+      def save_events(events)
+        return if events.empty?
+
+        events.each do |event|
+          event_hash = event.is_a?(Hash) ? event : event.to_h
+          key = event_key(event_hash)
+          @redis.setex(key, @ttl || 86400, serialize_event(event_hash))
+          
+          # Add to index sets
+          index_event(event_hash)
+        end
+      rescue StandardError => e
+        raise Error, "Failed to save events to Redis: #{e.message}"
+      end
+
+      def events_for_context(context, options = {})
+        context.validate!
+
+        # Get event IDs from index
+        event_ids = find_event_ids(context, options)
+        
+        # Fetch events
+        events = event_ids.map do |id|
+          deserialize_event(@redis.get("#{@key_prefix}:event:#{id}"))
+        end.compact
+
+        # Apply filters that can't be done in Redis
+        events = filter_events(events, context, options)
+        
+        # Apply limit and ordering
+        events = sort_events(events, options[:order_by]) if options[:order_by]
+        events = events.first(options[:limit]) if options[:limit]
+
+        events
+      end
+
+      def delete_old_events(before_date)
+        # Redis TTL handles expiration, but we can also scan and delete
+        pattern = "#{@key_prefix}:event:*"
+        @redis.scan_each(match: pattern) do |key|
+          event = deserialize_event(@redis.get(key))
+          if event && event[:created_at] && Time.parse(event[:created_at].to_s) < before_date
+            @redis.del(key)
+            remove_from_indexes(event)
+          end
+        end
+      end
+
+      def event_count(context, options = {})
+        context.validate!
+        find_event_ids(context, options).count
+      end
+
+      def unique_users(context, options = {})
+        context.validate!
+        event_ids = find_event_ids(context, options)
+        events = event_ids.map { |id| deserialize_event(@redis.get("#{@key_prefix}:event:#{id}")) }.compact
+        events.map { |e| e[:user_id] }.compact.uniq.count
+      end
+
+      private
+
+      def event_key(event_hash)
+        id = event_hash[:id] || SecureRandom.uuid
+        "#{@key_prefix}:event:#{id}"
+      end
+
+      def serialize_event(event_hash)
+        require "json"
+        JSON.generate(event_hash)
+      end
+
+      def deserialize_event(data)
+        return nil unless data
+        require "json"
+        JSON.parse(data, symbolize_names: true)
+      end
+
+      def index_event(event_hash)
+        tenant_id = event_hash[:tenant_id]
+        user_id = event_hash[:user_id]
+        event_type = event_hash[:event_type]
+        
+        @redis.sadd("#{@key_prefix}:tenant:#{tenant_id}", event_hash[:id])
+        @redis.sadd("#{@key_prefix}:user:#{user_id}", event_hash[:id]) if user_id
+        @redis.sadd("#{@key_prefix}:type:#{event_type}", event_hash[:id]) if event_type
+      end
+
+      def remove_from_indexes(event_hash)
+        tenant_id = event_hash[:tenant_id]
+        user_id = event_hash[:user_id]
+        event_type = event_hash[:event_type]
+        
+        @redis.srem("#{@key_prefix}:tenant:#{tenant_id}", event_hash[:id])
+        @redis.srem("#{@key_prefix}:user:#{user_id}", event_hash[:id]) if user_id
+        @redis.srem("#{@key_prefix}:type:#{event_type}", event_hash[:id]) if event_type
+      end
+
+      def find_event_ids(context, options)
+        # Start with tenant index
+        ids = @redis.smembers("#{@key_prefix}:tenant:#{context.tenant_id}").to_a
+        
+        # Intersect with user index if specified
+        if context.user_id
+          user_ids = @redis.smembers("#{@key_prefix}:user:#{context.user_id}").to_a
+          ids = ids & user_ids
+        end
+        
+        # Intersect with event type if specified
+        if options[:event_type]
+          type_ids = @redis.smembers("#{@key_prefix}:type:#{options[:event_type]}").to_a
+          ids = ids & type_ids
+        end
+        
+        ids
+      end
+
+      def filter_events(events, context, options)
+        events.select do |event|
+          matches = true
+          
+          matches &&= event[:user_type] == context.user_type if context.user_type
+          matches &&= event[:event_name] == options[:event_name] if options[:event_name]
+          
+          if options[:since]
+            matches &&= Time.parse(event[:created_at].to_s) >= options[:since]
+          end
+          
+          if options[:until]
+            matches &&= Time.parse(event[:created_at].to_s) <= options[:until]
+          end
+          
+          if options[:metadata_filters]
+            options[:metadata_filters].each do |key, value|
+              metadata = event[:metadata] || {}
+              matches &&= (metadata[key.to_sym] == value || metadata[key.to_s] == value)
+            end
+          end
+          
+          matches
+        end
+      end
+
+      def sort_events(events, order_by)
+        field = order_by[:field]
+        direction = order_by[:direction] || :desc
+        
+        events.sort do |a, b|
+          a_val = a[field.to_sym] || a[field.to_s]
+          b_val = b[field.to_sym] || b[field.to_s]
+          comparison = (a_val <=> b_val) || 0
+          direction == :desc ? -comparison : comparison
+        end
+      end
+    end
+  end
+end
+

data/lib/behavior_analytics/streaming/event_stream.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module BehaviorAnalytics
+  module Streaming
+    class EventStream
+      attr_reader :subscribers, :filters
+
+      def initialize
+        @subscribers = []
+        @filters = []
+        @mutex = Mutex.new
+      end
+
+      def subscribe(filter: nil, &block)
+        @mutex.synchronize do
+          @subscribers << { filter: filter, callback: block }
+        end
+        self
+      end
+
+      def publish(event)
+        @mutex.synchronize do
+          @subscribers.each do |subscriber|
+            if should_deliver?(event, subscriber[:filter])
+              begin
+                subscriber[:callback].call(event)
+              rescue StandardError => e
+                handle_subscriber_error(e, event, subscriber)
+              end
+            end
+          end
+        end
+      end
+
+      def unsubscribe_all
+        @mutex.synchronize do
+          @subscribers.clear
+        end
+      end
+
+      private
+
+      def should_deliver?(event, filter)
+        return true unless filter
+
+        case filter
+        when Proc
+          filter.call(event)
+        when Hash
+          filter.all? { |key, value| event_matches?(event, key, value) }
+        when Symbol, String
+          event[:event_type] == filter || event[:event_type].to_s == filter.to_s
+        else
+          true
+        end
+      end
+
+      def event_matches?(event, key, value)
+        event_value = event[key.to_sym] || event[key.to_s] || get_metadata_value(event, key.to_s)
+        event_value == value || event_value.to_s == value.to_s
+      end
+
+      def get_metadata_value(event, key)
+        metadata = event[:metadata] || event["metadata"] || {}
+        metadata[key.to_sym] || metadata[key.to_s] || metadata[key]
+      end
+
+      def handle_subscriber_error(error, event, subscriber)
+        # Log error but don't stop other subscribers
+        if defined?(Rails) && Rails.logger
+          Rails.logger.error("BehaviorAnalytics: Subscriber error: #{error.message}")
+        end
+      end
+    end
+  end
+end
+

data/lib/behavior_analytics/throttling/limiter.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module BehaviorAnalytics
+  module Throttling
+    class Limiter
+      def initialize(options = {})
+        @per_tenant_limits = options[:per_tenant] || {}
+        @per_user_limits = options[:per_user] || {}
+        @global_limit = options[:global]
+        @window_size = options[:window_size] || 60 # seconds
+        @counters = {}
+        @mutex = Mutex.new
+      end
+
+      def check_limit(context, event = nil)
+        return { allowed: true } unless should_check_limits?
+
+        @mutex.synchronize do
+          # Check global limit
+          if @global_limit && !check_global_limit
+            return { allowed: false, reason: "global_limit_exceeded" }
+          end
+
+          # Check per-tenant limit
+          if context.tenant_id && @per_tenant_limits[context.tenant_id]
+            limit = @per_tenant_limits[context.tenant_id]
+            if !check_limit_for_key("tenant:#{context.tenant_id}", limit)
+              return { allowed: false, reason: "tenant_limit_exceeded", tenant_id: context.tenant_id }
+            end
+          end
+
+          # Check per-user limit
+          if context.user_id && @per_user_limits[context.user_id]
+            limit = @per_user_limits[context.user_id]
+            if !check_limit_for_key("user:#{context.user_id}", limit)
+              return { allowed: false, reason: "user_limit_exceeded", user_id: context.user_id }
+            end
+          end
+
+          { allowed: true }
+        end
+      end
+
+      def record_event(context)
+        return unless should_check_limits?
+
+        @mutex.synchronize do
+          increment_counter("global") if @global_limit
+          increment_counter("tenant:#{context.tenant_id}") if context.tenant_id && @per_tenant_limits[context.tenant_id]
+          increment_counter("user:#{context.user_id}") if context.user_id && @per_user_limits[context.user_id]
+        end
+      end
+
+      def reset_counters
+        @mutex.synchronize do
+          @counters.clear
+        end
+      end
+
+      private
+
+      def should_check_limits?
+        @global_limit || !@per_tenant_limits.empty? || !@per_user_limits.empty?
+      end
+
+      def check_global_limit
+        check_limit_for_key("global", @global_limit)
+      end
+
+      def check_limit_for_key(key, limit)
+        counter = get_counter(key)
+        counter < limit
+      end
+
+      def get_counter(key)
+        counter = @counters[key]
+        
+        # Reset counter if window expired
+        if counter && counter[:expires_at] < Time.now
+          @counters.delete(key)
+          counter = nil
+        end
+
+        counter ||= { count: 0, expires_at: Time.now + @window_size }
+        @counters[key] = counter
+        
+        counter[:count]
+      end
+
+      def increment_counter(key)
+        counter = get_counter(key)
+        @counters[key][:count] += 1
+      end
+    end
+  end
+end
+

data/lib/behavior_analytics/tracker.rb

@@ -5,13 +5,22 @@ require "thread"
 
 module BehaviorAnalytics
   class Tracker
-    attr_reader :storage_adapter, :batch_size, :flush_interval, :context_resolver
+    attr_reader :storage_adapter, :batch_size, :flush_interval, :context_resolver, :async_processor, :event_stream
 
     def initialize(options = {})
       @storage_adapter = options[:storage_adapter] || BehaviorAnalytics.configuration.storage_adapter || Storage::InMemoryAdapter.new
       @batch_size = options[:batch_size] || BehaviorAnalytics.configuration.batch_size
       @flush_interval = options[:flush_interval] || BehaviorAnalytics.configuration.flush_interval
       @context_resolver = options[:context_resolver] || BehaviorAnalytics.configuration.context_resolver
+      @async_processor = options[:async_processor] || BehaviorAnalytics.configuration.async_processor
+      @use_async = options.fetch(:use_async, BehaviorAnalytics.configuration.use_async || false)
+      @event_stream = options[:event_stream] || BehaviorAnalytics.configuration.event_stream || Streaming::EventStream.new
+      @hooks_manager = options[:hooks_manager] || BehaviorAnalytics.configuration.hooks_manager || Hooks::Manager.new
+      @sampling_strategy = options[:sampling_strategy] || BehaviorAnalytics.configuration.sampling_strategy
+      @rate_limiter = options[:rate_limiter] || BehaviorAnalytics.configuration.rate_limiter
+      @schema_validator = options[:schema_validator] || BehaviorAnalytics.configuration.schema_validator
+      @metrics = options[:metrics] || BehaviorAnalytics.configuration.metrics || Observability::Metrics.new
+      @tracer = options[:tracer] || BehaviorAnalytics.configuration.tracer
 
       @buffer = []
       @mutex = Mutex.new
@@ -23,7 +32,50 @@ module BehaviorAnalytics
       context = normalize_context(context)
       context.validate!
 
-      event = Event.new(
+      # Check rate limiting
+      if @rate_limiter
+        limit_check = @rate_limiter.check_limit(context)
+        unless limit_check[:allowed]
+          raise Error, "Rate limit exceeded: #{limit_check[:reason]}"
+        end
+      end
+
+      # Check sampling
+      if @sampling_strategy
+        event_data = {
+          tenant_id: context.tenant_id,
+          user_id: context.user_id,
+          event_name: event_name,
+          event_type: event_type
+        }
+        unless @sampling_strategy.should_sample?(event_data, context)
+          return # Skip this event
+        end
+      end
+
+      # Build event data
+      event_data = {
+        tenant_id: context.tenant_id,
+        user_id: context.user_id,
+        user_type: context.user_type,
+        event_name: event_name,
+        event_type: event_type,
+        metadata: metadata.merge(context.filters),
+        session_id: options[:session_id],
+        ip: options[:ip],
+        user_agent: options[:user_agent],
+        duration_ms: options[:duration_ms]
+      }
+
+      # Validate schema if validator is configured
+      if @schema_validator
+        validation_result = @schema_validator.validate(event_data)
+        unless validation_result[:valid]
+          raise Error, "Event validation failed: #{validation_result[:errors].join(', ')}"
+        end
+      end
+
+      event = Event.new(event_data)
         tenant_id: context.tenant_id,
         user_id: context.user_id,
         user_type: context.user_type,
@@ -36,7 +88,56 @@ module BehaviorAnalytics
         duration_ms: options[:duration_ms]
       )
 
-      add_to_buffer(event)
+      # Execute before_track hooks
+      begin
+        @hooks_manager.execute_before_track(event.to_h, context.to_h)
+      rescue StandardError => e
+        @hooks_manager.execute_on_error(e, event.to_h, context.to_h)
+        raise if BehaviorAnalytics.configuration.raise_on_hook_error
+      end
+
+      begin
+        # Start tracing if enabled
+        span = @tracer&.start_span("track_event", tags: {
+          event_name: event_name,
+          event_type: event_type.to_s,
+          tenant_id: context.tenant_id
+        })
+
+        add_to_buffer(event)
+        
+        # Record metrics
+        @metrics.increment_counter("events.tracked", tags: {
+          event_type: event_type.to_s,
+          tenant_id: context.tenant_id.to_s
+        })
+        
+        # Debug logging
+        if BehaviorAnalytics.configuration.debug_mode
+          BehaviorAnalytics.configuration.debug("Event tracked: #{event_name}", context: context.to_h)
+        end
+        
+        # Publish to event stream
+        @event_stream.publish(event.to_h) if @event_stream
+        
+        # Execute after_track hooks
+        @hooks_manager.execute_after_track(event.to_h, context.to_h)
+        
+        # Record event for rate limiting
+        @rate_limiter.record_event(context) if @rate_limiter
+
+        # Finish tracing
+        @tracer&.finish_span(span[:id]) if span
+      rescue StandardError => e
+        # Record error metrics
+        @metrics.increment_counter("events.errors", tags: {
+          event_type: event_type.to_s,
+          tenant_id: context.tenant_id.to_s
+        })
+        
+        @hooks_manager.execute_on_error(e, event.to_h, context.to_h)
+        raise
+      end
     end
 
     def track_api_call(context:, method:, path:, status_code:, duration_ms: nil, **options)
@@ -76,7 +177,24 @@ module BehaviorAnalytics
 
       return if events_to_flush.empty?
 
-      @storage_adapter.save_events(events_to_flush)
+      start_time = Time.now
+      
+      begin
+        if @use_async && @async_processor
+          @async_processor.process_async(events_to_flush)
+        else
+          @storage_adapter.save_events(events_to_flush)
+        end
+        
+        # Record flush metrics
+        duration_ms = ((Time.now - start_time) * 1000).to_i
+        @metrics.record_histogram("flush.duration_ms", duration_ms)
+        @metrics.increment_counter("flush.count", value: events_to_flush.size)
+      rescue StandardError => e
+        @metrics.increment_counter("flush.errors")
+        raise
+      end
+      
       restart_flush_timer
     end
 
@@ -88,6 +206,14 @@ module BehaviorAnalytics
       Query.new(@storage_adapter)
     end
 
+    def subscribe_to_stream(filter: nil, &block)
+      @event_stream.subscribe(filter: filter, &block)
+    end
+
+    def inspector
+      @inspector ||= Debug::Inspector.new(self)
+    end
+
     private
 
     def normalize_context(context)

data/lib/behavior_analytics/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BehaviorAnalytics
-  VERSION = "0.1.0"
+  VERSION = "2.0.0"
 end