amplitude-experiment 1.1.4 → 1.2.0

data/lib/amplitude/plugin.rb

@@ -0,0 +1,131 @@
+require 'securerandom'
+module AmplitudeAnalytics
+  # Plugin
+  class Plugin
+    attr_reader :plugin_type
+
+    def initialize(plugin_type)
+      @plugin_type = plugin_type
+    end
+
+    def setup(client)
+      # Setup plugins with client instance parameter
+    end
+
+    def execute(event)
+      # Process event with plugin instance
+    end
+  end
+
+  # EventPlugin
+  class EventPlugin < Plugin
+    def execute(event)
+      track(event)
+    end
+
+    def track(event)
+      event
+    end
+  end
+
+  # DestinationPlugin
+  class DestinationPlugin < EventPlugin
+    attr_reader :timeline
+
+    def initialize
+      super(PluginType::DESTINATION)
+      @timeline = Timeline.new
+    end
+
+    def setup(client)
+      @timeline.setup(client)
+    end
+
+    def add(plugin)
+      @timeline.add(plugin)
+      self
+    end
+
+    def remove(plugin)
+      @timeline.remove(plugin)
+      self
+    end
+
+    def execute(event)
+      event = @timeline.process(event)
+      super(event)
+    end
+
+    def shutdown
+      @timeline.shutdown
+    end
+  end
+
+  # AmplitudeDestinationPlugin
+  class AmplitudeDestinationPlugin < DestinationPlugin
+    attr_reader :workers
+
+    def initialize
+      super
+      @workers = Workers.new
+      @storage = nil
+      @configuration = nil
+    end
+
+    def setup(client)
+      @configuration = client.configuration
+      @storage = client.configuration.storage
+      @workers.setup(client.configuration, @storage)
+      @storage.setup(client.configuration, @workers)
+    end
+
+    def verify_event(event)
+      return false unless event.is_a?(BaseEvent) && event.event_type && (event.user_id || event.device_id)
+
+      true
+    end
+
+    def execute(event)
+      event = @timeline.process(event)
+      raise InvalidEventError, 'Invalid event.' unless verify_event(event)
+
+      @storage.push(event)
+    end
+
+    def flush
+      @workers.flush
+    end
+
+    def shutdown
+      @timeline.shutdown
+      @workers.stop
+    end
+  end
+
+  # ContextPlugin
+  class ContextPlugin < Plugin
+    attr_accessor :configuration
+
+    def initialize
+      super(PluginType::BEFORE)
+      @context_string = "#{SDK_LIBRARY}/#{SDK_VERSION}"
+      @configuration = nil
+    end
+
+    def setup(client)
+      @configuration = client.configuration
+    end
+
+    def apply_context_data(event)
+      event.library = @context_string
+    end
+
+    def execute(event)
+      event.time ||= AmplitudeAnalytics.current_milliseconds
+      event.insert_id ||= SecureRandom.uuid
+      event.ingestion_metadata ||= @configuration.ingestion_metadata if @configuration.ingestion_metadata
+      apply_context_data(event)
+      event
+    end
+  end
+end

data/lib/amplitude/processor.rb

@@ -0,0 +1,100 @@
+module AmplitudeAnalytics
+  # ResponseProcessor
+  class ResponseProcessor
+    def initialize
+      @configuration = nil
+      @storage = nil
+    end
+
+    def setup(configuration, storage)
+      @configuration = configuration
+      @storage = storage
+    end
+
+    def process_response(res, events)
+      case res.status
+      when HttpStatus::SUCCESS
+        callback(events, res.code, 'Event sent successfully.')
+        log(events, res.code, 'Event sent successfully.')
+      when HttpStatus::TIMEOUT, HttpStatus::FAILED
+        push_to_storage(events, 0, res)
+      when HttpStatus::PAYLOAD_TOO_LARGE
+        if events.length == 1
+          callback(events, res.code, res.error)
+          log(events, res.code, res.error)
+        else
+          @configuration.increase_flush_divider
+          push_to_storage(events, 0, res)
+        end
+      when HttpStatus::INVALID_REQUEST
+        raise InvalidAPIKeyError, res.error if res.error.start_with?('Invalid API key:')
+
+        if res.missing_field
+          callback(events, res.code, "Request missing required field #{res.missing_field}")
+          log(events, res.code, "Request missing required field #{res.missing_field}")
+        else
+          invalid_index_set = res.invalid_or_silenced_index
+          events_for_retry = []
+          events_for_callback = []
+          events.each_with_index do |event, index|
+            if invalid_index_set.include?(index)
+              events_for_callback << event
+            else
+              events_for_retry << event
+            end
+          end
+          callback(events_for_callback, res.code, res.error)
+          log(events_for_callback, res.code, res.error)
+          push_to_storage(events_for_retry, 0, res)
+        end
+      when HttpStatus::TOO_MANY_REQUESTS
+        events_for_callback = []
+        events_for_retry_delay = []
+        events_for_retry = []
+        events.each_with_index do |event, index|
+          if res.throttled_events&.include?(index)
+            if res.exceed_daily_quota(event)
+              events_for_callback << event
+            else
+              events_for_retry_delay << event
+            end
+          else
+            events_for_retry << event
+          end
+        end
+        callback(events_for_callback, res.code, 'Exceeded daily quota')
+        push_to_storage(events_for_retry_delay, 30_000, res)
+        push_to_storage(events_for_retry, 0, res)
+      else
+        callback(events, res.code, res.error || 'Unknown error')
+        log(events, res.code, res.error || 'Unknown error')
+      end
+    end
+
+    def push_to_storage(events, delay, res)
+      events.each do |event|
+        event.retry += 1
+        success, message = @storage.push(event, delay)
+        unless success
+          callback([event], res.code, message)
+          log([event], res.code, message)
+        end
+      end
+    end
+
+    def callback(events, code, message)
+      events.each do |event|
+        @configuration.callback.call(event, code, message) if @configuration.callback.respond_to?(:call)
+        event.callback(code, message)
+      rescue StandardError => e
+        @configuration.logger.exception("Error callback for event #{event}: #{e.message}")
+      end
+    end
+
+    def log(events, code, message)
+      events.each do |event|
+        @configuration.logger.info("#{message}, response code: #{code}, event: #{event}")
+      end
+    end
+  end
+end

data/lib/amplitude/storage.rb

@@ -0,0 +1,146 @@
+require 'monitor'
+module AmplitudeAnalytics
+  # Storage
+  class Storage
+    def push(_event, _delay = 0)
+      raise NotImplementedError, 'push method must be implemented in subclasses'
+    end
+
+    def pull(_batch_size)
+      raise NotImplementedError, 'pull method must be implemented in subclasses'
+    end
+
+    def pull_all
+      raise NotImplementedError, 'pull_all method must be implemented in subclasses'
+    end
+  end
+
+  # StorageProvider class
+  class StorageProvider
+    def storage
+      raise NotImplementedError, 'get_storage method must be implemented in subclasses'
+    end
+  end
+
+  # InMemoryStorage class
+  class InMemoryStorage < Storage
+    attr_reader :total_events, :ready_queue, :workers, :buffer_data, :monitor
+
+    def initialize
+      super
+      @total_events = 0
+      @buffer_data = []
+      @ready_queue = []
+      @monitor = Monitor.new
+      @buffer_lock_cv = @monitor.new_cond
+      @configuration = nil
+      @workers = nil
+    end
+
+    def lock
+      @buffer_lock_cv
+    end
+
+    def max_retry
+      @configuration.flush_max_retries
+    end
+
+    def wait_time
+      if @ready_queue.any?
+        0
+      elsif @buffer_data.any?
+        [@buffer_data[0][0] - AmplitudeAnalytics.current_milliseconds, @configuration.flush_interval_millis].min
+      else
+        @configuration.flush_interval_millis
+      end
+    end
+
+    def setup(configuration, workers)
+      @configuration = configuration
+      @workers = workers
+    end
+
+    def push(event, delay = 0)
+      return false, 'Destination buffer full. Retry temporarily disabled' if event.retry && @total_events >= MAX_BUFFER_CAPACITY
+
+      return false, "Event reached max retry times #{max_retry}." if event.retry >= max_retry
+
+      total_delay = delay + retry_delay(event.retry)
+      insert_event(total_delay, event)
+      @workers.start
+      [true, nil]
+    end
+
+    def pull(batch_size)
+      current_time = AmplitudeAnalytics.current_milliseconds
+      @monitor.synchronize do
+        result = @ready_queue.shift(batch_size)
+        index = 0
+        while index < @buffer_data.length && index < batch_size - result.length &&
+              current_time >= @buffer_data[index][0]
+          event = @buffer_data[index][1]
+          result << event
+          index += 1
+        end
+        @buffer_data.slice!(0, index)
+        @total_events -= result.length
+        result
+      end
+    end
+
+    def pull_all
+      @monitor.synchronize do
+        result = @ready_queue + @buffer_data.map { |element| element[1] }
+        @buffer_data.clear
+        @ready_queue.clear
+        @total_events = 0
+        result
+      end
+    end
+
+    def insert_event(total_delay, event)
+      current_time = AmplitudeAnalytics.current_milliseconds
+      @monitor.synchronize do
+        @ready_queue << @buffer_data.shift[1] while @buffer_data.any? && @buffer_data[0][0] <= current_time
+
+        if total_delay == 0
+          @ready_queue << event
+        else
+          time_stamp = current_time + total_delay
+          left = 0
+          right = @buffer_data.length - 1
+          while left <= right
+            mid = (left + right) / 2
+            if @buffer_data[mid][0] > time_stamp
+              right = mid - 1
+            else
+              left = mid + 1
+            end
+          end
+          @buffer_data.insert(left, [time_stamp, event])
+        end
+
+        @total_events += 1
+
+        lock.signal if @ready_queue.length >= @configuration.flush_queue_size
+      end
+    end
+
+    def retry_delay(ret)
+      if ret > max_retry
+        3200
+      elsif ret <= 0
+        0
+      else
+        100 * (2**((ret - 1) / 2))
+      end
+    end
+  end
+
+  # InMemoryStorageProvider class
+  class InMemoryStorageProvider < StorageProvider
+    def storage
+      InMemoryStorage.new
+    end
+  end
+end

data/lib/amplitude/timeline.rb

@@ -0,0 +1,98 @@
+require_relative 'constants'
+
+module AmplitudeAnalytics
+  # Timeline
+  class Timeline
+    attr_accessor :configuration
+    attr_reader :plugins
+
+    def initialize(configuration = nil)
+      @locks = {
+        PluginType::BEFORE => Mutex.new,
+        PluginType::ENRICHMENT => Mutex.new,
+        PluginType::DESTINATION => Mutex.new
+      }
+      @plugins = {
+        PluginType::BEFORE => [],
+        PluginType::ENRICHMENT => [],
+        PluginType::DESTINATION => []
+      }
+      @configuration = configuration
+    end
+
+    def logger
+      @configuration&.logger
+      Logger.new($stdout, progname: LOGGER_NAME)
+    end
+
+    def setup(client)
+      @configuration = client.configuration
+    end
+
+    def add(plugin)
+      @locks[plugin.plugin_type].synchronize do
+        @plugins[plugin.plugin_type] << plugin
+      end
+    end
+
+    def remove(plugin)
+      @locks.each_key do |plugin_type|
+        @locks[plugin_type].synchronize do
+          @plugins[plugin_type].reject! { |p| p == plugin }
+        end
+      end
+    end
+
+    def flush
+      destination_futures = []
+      @locks[PluginType::DESTINATION].synchronize do
+        @plugins[PluginType::DESTINATION].each do |destination|
+          destination_futures << destination.flush
+        rescue StandardError
+          logger.exception('Error for flush events')
+        end
+      end
+      destination_futures
+    end
+
+    def process(event)
+      if @configuration&.opt_out
+        logger.info('Skipped event for opt out config')
+        return event
+      end
+
+      before_result = apply_plugins(PluginType::BEFORE, event)
+      enrich_result = apply_plugins(PluginType::ENRICHMENT, before_result)
+      apply_plugins(PluginType::DESTINATION, enrich_result)
+      enrich_result
+    end
+
+    def apply_plugins(plugin_type, event)
+      result = event
+      @locks[plugin_type].synchronize do
+        @plugins[plugin_type].each do |plugin|
+          break unless result
+
+          begin
+            if plugin.plugin_type == PluginType::DESTINATION
+              plugin.execute(Marshal.load(Marshal.dump(result)))
+            else
+              result = plugin.execute(result)
+            end
+          rescue InvalidEventError
+            logger.error("Invalid event body #{event}")
+          rescue StandardError
+            logger.error("Error for apply #{PluginType.name(plugin_type)} plugin for event #{event}")
+          end
+        end
+      end
+      result
+    end
+
+    def shutdown
+      @locks[PluginType::DESTINATION].synchronize do
+        @plugins[PluginType::DESTINATION].each(&:shutdown)
+      end
+    end
+  end
+end

data/lib/amplitude/utils.rb

@@ -0,0 +1,29 @@
+require 'logger'
+require 'time'
+
+# Amplitude
+module AmplitudeAnalytics
+  def self.logger
+    @logger ||= Logger.new($stdout, progname: LOGGER_NAME)
+  end
+
+  def self.current_milliseconds
+    (Time.now.to_f * 1000).to_i
+  end
+
+  def self.truncate(obj)
+    case obj
+    when Hash
+      if obj.length > MAX_PROPERTY_KEYS
+        logger.error("Too many properties. #{MAX_PROPERTY_KEYS} maximum.")
+        return {}
+      end
+      obj.each { |key, value| obj[key] = truncate(value) }
+    when Array
+      obj.map! { |element| truncate(element) }
+    when String
+      obj = obj[0, MAX_STRING_LENGTH]
+    end
+    obj
+  end
+end

data/lib/amplitude/workers.rb

@@ -0,0 +1,101 @@
+require 'json'
+require 'concurrent'
+
+module AmplitudeAnalytics
+  # Workers
+  class Workers
+    attr_reader :is_active, :is_started, :storage, :configuration, :threads_pool, :consumer_lock, :response_processor, :http_client
+
+    def initialize
+      @threads_pool = Concurrent::ThreadPoolExecutor.new(max_threads: 16)
+      @is_active = true
+      @consumer_lock = Mutex.new
+      @is_started = false
+      @configuration = nil
+      @storage = nil
+      @response_processor = ResponseProcessor.new
+      @http_client = HttpClient.new
+    end
+
+    def setup(configuration, storage)
+      @configuration = configuration
+      @storage = storage
+      @response_processor = ResponseProcessor.new
+      @response_processor.setup(configuration, storage)
+    end
+
+    def start
+      @consumer_lock.synchronize do
+        unless @is_started
+          @is_started = true
+          Thread.new { buffer_consumer }
+        end
+      end
+    end
+
+    def stop
+      flush
+      @is_active = false
+      @is_started = true
+      @threads_pool.shutdown
+    end
+
+    def flush
+      events = @storage.pull_all unless @storage.nil?
+      Concurrent::Future.execute do
+        send(events) if events && !events.empty?
+      end
+    end
+
+    def send(events)
+      url = @configuration.server_url
+      payload = get_payload(events)
+      res = @http_client.post(url, payload)
+      begin
+        @response_processor.process_response(res, events)
+      rescue InvalidAPIKeyError
+        @configuration.logger.error('Invalid API Key')
+      end
+    end
+
+    def get_payload(events)
+      payload_body = {
+        'api_key' => @configuration.api_key,
+        'events' => []
+      }
+
+      events.each do |event|
+        event_body = event.event_body
+        payload_body['events'] << event_body if event_body
+      end
+      payload_body['options'] = @configuration.options if @configuration.options
+      JSON.dump(payload_body).encode('utf-8')
+    end
+
+    def buffer_consumer
+      if @is_active
+        @storage.monitor.synchronize do
+          @storage.lock.wait(@configuration.flush_interval_millis.to_f / 1000)
+
+          loop do
+            break unless @storage.total_events.positive?
+
+            events = @storage.pull(@configuration.flush_queue_size)
+            if events
+              @threads_pool.post { send(events) }
+            else
+              wait_time = @storage.wait_time.to_f / 1000
+              @storage.lock.wait(wait_time) if wait_time > 0
+            end
+          end
+        end
+      end
+    rescue StandardError => e
+      @configuration.logger.error("Consumer thread error: #{e}")
+    ensure
+      @consumer_lock.synchronize do
+        @is_started = false
+      end
+    end
+  end
+end

data/lib/amplitude-experiment.rb

@@ -9,6 +9,12 @@ require 'experiment/remote/client'
 require 'experiment/local/client'
 require 'experiment/local/config'
 require 'experiment/local/fetcher'
+require 'experiment/local/assignment/assignment'
+require 'experiment/local/assignment/assignment_filter'
+require 'experiment/local/assignment/assignment_service'
+require 'experiment/local/assignment/assignment_config'
+require 'experiment/util/lru_cache'
+require 'experiment/util/hash'
 
 # Amplitude Experiment Module
 module AmplitudeExperiment

data/lib/amplitude.rb

@@ -0,0 +1,12 @@
+require 'amplitude/client'
+require 'amplitude/config'
+require 'amplitude/constants'
+require 'amplitude/event'
+require 'amplitude/exception'
+require 'amplitude/http_client'
+require 'amplitude/plugin'
+require 'amplitude/processor'
+require 'amplitude/storage'
+require 'amplitude/timeline'
+require 'amplitude/utils'
+require 'amplitude/workers'

data/lib/experiment/local/assignment/assignment.rb

@@ -0,0 +1,21 @@
+module AmplitudeExperiment
+  DAY_MILLIS = 86_400_000
+  # Assignment
+  class Assignment
+    attr_accessor :user, :results, :timestamp
+
+    def initialize(user, results)
+      @user = user
+      @results = results
+      @timestamp = (Time.now.to_f * 1000).to_i
+    end
+
+    def canonicalize
+      sb = "#{@user&.user_id&.strip} #{@user&.device_id&.strip} "
+      results.sort.to_h.each do |key, value|
+        sb += "#{key.strip} #{value['variant']&.fetch('key', '')&.strip} "
+      end
+      sb
+    end
+  end
+end

data/lib/experiment/local/assignment/assignment_config.rb

@@ -0,0 +1,12 @@
+module AmplitudeExperiment
+  # AssignmentConfig
+  class AssignmentConfig < AmplitudeAnalytics::Config
+    attr_accessor :api_key, :cache_capacity
+
+    def initialize(api_key, cache_capacity = 65_536, **kwargs)
+      super(**kwargs)
+      @api_key = api_key
+      @cache_capacity = cache_capacity
+    end
+  end
+end

data/lib/experiment/local/assignment/assignment_filter.rb

@@ -0,0 +1,15 @@
+module AmplitudeExperiment
+  # AssignmentFilter
+  class AssignmentFilter
+    def initialize(size, ttl_millis = DAY_MILLIS)
+      @cache = LRUCache.new(size, ttl_millis)
+    end
+
+    def should_track(assignment)
+      canonical_assignment = assignment.canonicalize
+      track = @cache.get(canonical_assignment).nil?
+      @cache.put(canonical_assignment, 0) if track
+      track
+    end
+  end
+end