prefab-cloud-ruby 0

data/lib/prefab/logger_client.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+module Prefab
+  class LoggerClient < ::Logger
+    SEP = '.'
+    BASE_KEY = 'log-level'
+    UNKNOWN_PATH = 'unknown.'
+    LOG_TAGS = 'log.tags'
+    REQ_TAGS = 'req.tags'
+
+    LOG_LEVEL_LOOKUPS = {
+      PrefabProto::LogLevel::NOT_SET_LOG_LEVEL => ::Logger::DEBUG,
+      PrefabProto::LogLevel::TRACE => ::Logger::DEBUG,
+      PrefabProto::LogLevel::DEBUG => ::Logger::DEBUG,
+      PrefabProto::LogLevel::INFO => ::Logger::INFO,
+      PrefabProto::LogLevel::WARN => ::Logger::WARN,
+      PrefabProto::LogLevel::ERROR => ::Logger::ERROR,
+      PrefabProto::LogLevel::FATAL => ::Logger::FATAL
+    }.freeze
+
+    def self.instance
+      @@shared_instance ||= LoggerClient.new($stdout)
+    end
+
+    def initialize(logdev, log_path_aggregator: nil, formatter: Options::DEFAULT_LOG_FORMATTER, prefix: nil)
+      super(logdev)
+      self.formatter = Prefab::Logging::FormatterBase.new(formatter_proc: formatter, logger_client: self)
+      @config_client = BootstrappingConfigClient.new
+      @silences = Concurrent::Map.new(initial_capacity: 2)
+      @recurse_check = Concurrent::Map.new(initial_capacity: 2)
+      @prefix = "#{prefix}#{prefix && '.'}"
+
+      @context_keys_map = Concurrent::Map.new(initial_capacity: 4)
+
+      @log_path_aggregator = log_path_aggregator
+      @@shared_instance = self
+    end
+
+    def add_context_keys(*keys)
+      context_keys.merge(keys)
+    end
+
+    def with_context_keys(*keys)
+      context_keys.merge(keys)
+      yield
+    ensure
+      context_keys.subtract(keys)
+    end
+
+    def internal_logger(path = nil)
+      InternalLogger.new(path, self)
+    end
+
+    def context_keys
+      @context_keys_map.fetch_or_store(local_log_id, Concurrent::Set.new)
+    end
+
+    # InternalLoggers Will Call This
+    def add_internal(severity, message, progname, loc, log_context = {}, &block)
+      path_loc = get_loc_path(loc)
+      path = @prefix + path_loc
+
+      log(message, path, progname, severity, log_context, &block)
+    end
+
+    def log_internal(severity, message, path, log_context = {}, &block)
+      return if @recurse_check[local_log_id]
+      @recurse_check[local_log_id] = true
+      begin
+        log(message, path, nil, severity, log_context, &block)
+      ensure
+        @recurse_check[local_log_id] = false
+      end
+    end
+
+    def log(message, path, progname, severity, log_context = {})
+      severity ||= ::Logger::UNKNOWN
+
+      return true if !should_log? severity, path
+      return true if @logdev.nil? || @silences[local_log_id]
+
+      progname = @progname if progname.nil?
+
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+          progname = @progname
+        end
+      end
+
+      @logdev.write(
+        format_message(format_severity(severity), Time.now, progname, message, path, stringify_keys(log_context.merge(fetch_context_for_context_keys)))
+      )
+      true
+    end
+
+    def should_log?(severity, path)
+      @log_path_aggregator&.push(path, severity)
+      severity >= level_of(path)
+    end
+
+    def debug(progname = nil, **log_context, &block)
+      add_internal(DEBUG, nil, progname, caller_locations(1, 1)[0], log_context, &block)
+    end
+
+    def info(progname = nil, **log_context, &block)
+      add_internal(INFO, nil, progname, caller_locations(1, 1)[0], log_context, &block)
+    end
+
+    def warn(progname = nil, **log_context, &block)
+      add_internal(WARN, nil, progname, caller_locations(1, 1)[0], log_context, &block)
+    end
+
+    def error(progname = nil, **log_context, &block)
+      add_internal(ERROR, nil, progname, caller_locations(1, 1)[0], log_context, &block)
+    end
+
+    def fatal(progname = nil, **log_context, &block)
+      add_internal(FATAL, nil, progname, caller_locations(1, 1)[0], log_context, &block)
+    end
+
+    def debug?
+      true
+    end
+
+    def info?
+      true
+    end
+
+    def warn?
+      true
+    end
+
+    def error?
+      true
+    end
+
+    def fatal?
+      true
+    end
+
+    def level
+      DEBUG
+    end
+
+    def tagged(*tags)
+      to_add = tags.flatten.compact
+      if block_given?
+        new_log_tags = current_tags
+        new_log_tags += to_add unless to_add.empty?
+        Prefab::Context.with_merged_context({ "log" => { "tags" => new_log_tags } }) do
+          with_context_keys LOG_TAGS do
+            yield self
+          end
+        end
+      else
+        new_log_tags = Prefab::Context.current.get(REQ_TAGS) || []
+        new_log_tags += to_add unless to_add.empty?
+        add_context_keys REQ_TAGS
+        Prefab::Context.current.set("req", {"tags": new_log_tags})
+        self
+      end
+    end
+
+    def current_tags
+      Prefab::Context.current.get(LOG_TAGS) || []
+    end
+
+    def flush
+      Prefab::Context.current.set("req", {"tags": nil})
+      super if defined?(super)
+    end
+
+    def config_client=(config_client)
+      @config_client = config_client
+    end
+
+    def local_log_id
+      Thread.current.__id__
+    end
+
+    def silence
+      @silences[local_log_id] = true
+      yield self
+    ensure
+      @silences[local_log_id] = false
+    end
+
+    private
+
+    NO_DEFAULT = nil
+
+    def stringify_keys(hash)
+      Hash[hash.map { |k, v| [k.to_s, v] }]
+    end
+
+    def fetch_context_for_context_keys
+      context = Prefab::Context.current.to_h
+      Hash[context_keys.map do |key|
+        [key, context.dig(*key.split("."))]
+      end]
+    end
+
+    # Find the closest match to 'log_level.path' in config
+    def level_of(path)
+      closest_log_level_match = nil
+
+      path.split(SEP).each_with_object([BASE_KEY]) do |n, memo|
+        memo << n
+        val = @config_client.get(memo.join(SEP), NO_DEFAULT)
+        closest_log_level_match = val unless val.nil?
+      end
+
+      if closest_log_level_match.nil?
+        # get the top-level setting or default to WARN
+        closest_log_level_match = @config_client.get(BASE_KEY, :WARN)
+      end
+
+      closest_log_level_match_int = PrefabProto::LogLevel.resolve(closest_log_level_match)
+      LOG_LEVEL_LOOKUPS[closest_log_level_match_int]
+    end
+
+    def get_loc_path(loc)
+      loc_path = loc.absolute_path || loc.to_s
+      get_path(loc_path, loc.base_label)
+    end
+
+    # sanitize & clean the path of the caller so the key
+    # looks like log_level.app.models.user
+    def get_path(absolute_path, base_label)
+      path = (absolute_path || UNKNOWN_PATH).dup
+      path.slice! Dir.pwd
+      path.gsub!(%r{(.*)?(?=/lib)}im, '') # replace everything before first lib
+
+      path = path.gsub('/', SEP).gsub(/.rb.*/, '') + SEP + base_label
+      path.slice! '.lib'
+      path.slice! SEP
+      path
+    end
+
+    def format_message(severity, datetime, progname, msg, path = nil, log_context = {})
+      formatter = (@formatter || @default_formatter)
+      compact_context = log_context.reject{ |_, v| v.nil? || ((v.is_a? Array) && v.empty?) }
+      @formatter.call_proc(
+        severity: severity,
+        datetime: datetime,
+        progname: progname,
+        path: path,
+        message: msg,
+        log_context: compact_context
+      )
+    end
+  end
+
+  # StubConfigClient to be used while config client initializes
+  # since it may log
+  class BootstrappingConfigClient
+    def get(_key, default = nil, _properties = {})
+      ENV['PREFAB_LOG_CLIENT_BOOTSTRAP_LOG_LEVEL'] ? ENV['PREFAB_LOG_CLIENT_BOOTSTRAP_LOG_LEVEL'].upcase.to_sym : default
+    end
+  end
+end

data/lib/prefab/murmer3.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class Murmur3
+  ## MurmurHash3 was written by Austin Appleby, and is placed in the public
+  ## domain. The author hereby disclaims copyright to this source code.
+
+  MASK32 = 0xffffffff
+
+  def self.murmur3_32_rotl(x, r)
+    ((x << r) | (x >> (32 - r))) & MASK32
+  end
+
+  def self.murmur3_32_fmix(h)
+    h &= MASK32
+    h ^= h >> 16
+    h = (h * 0x85ebca6b) & MASK32
+    h ^= h >> 13
+    h = (h * 0xc2b2ae35) & MASK32
+    h ^ (h >> 16)
+  end
+
+  def self.murmur3_32__mmix(k1)
+    k1 = (k1 * 0xcc9e2d51) & MASK32
+    k1 = murmur3_32_rotl(k1, 15)
+    (k1 * 0x1b873593) & MASK32
+  end
+
+  def self.murmur3_32(str, seed = 0)
+    h1 = seed
+    numbers = str.unpack('V*C*')
+    tailn = str.length % 4
+    tail = numbers.slice!(numbers.size - tailn, tailn)
+    for k1 in numbers
+      h1 ^= murmur3_32__mmix(k1)
+      h1 = murmur3_32_rotl(h1, 13)
+      h1 = (h1 * 5 + 0xe6546b64) & MASK32
+    end
+
+    unless tail.empty?
+      k1 = 0
+      tail.reverse_each do |c1|
+        k1 = (k1 << 8) | c1
+      end
+      h1 ^= murmur3_32__mmix(k1)
+    end
+
+    h1 ^= str.length
+    murmur3_32_fmix(h1)
+  end
+end

data/lib/prefab/options.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module Prefab
+  # This class contains all the options that can be passed to the Prefab client.
+  class Options
+    attr_reader :api_key
+    attr_reader :logdev
+    attr_reader :log_prefix
+    attr_reader :log_formatter
+    attr_reader :namespace
+    attr_reader :prefab_api_url
+    attr_reader :on_no_default
+    attr_reader :initialization_timeout_sec
+    attr_reader :on_init_failure
+    attr_reader :prefab_config_override_dir
+    attr_reader :prefab_config_classpath_dir
+    attr_reader :prefab_envs
+    attr_reader :collect_sync_interval
+    attr_reader :use_local_cache
+    attr_reader :datafile
+    attr_reader :disable_action_controller_logging
+    attr_accessor :is_fork
+
+    DEFAULT_LOG_FORMATTER = proc { |data|
+      severity = data[:severity]
+      datetime = data[:datetime]
+      progname = data[:progname]
+      path = data[:path]
+      msg = data[:message]
+      log_context = data[:log_context]
+
+      progname = (progname.nil? || progname.empty?) ? path : "#{progname}: #{path}"
+
+      formatted_log_context = log_context.sort.map do |k, v|
+        "#{k}=#{v}"
+      end.join(" ")
+      "#{severity.ljust(5)} #{datetime}:#{' ' if progname}#{progname} #{msg}#{log_context.any? ? " " + formatted_log_context : ""}\n"
+    }
+
+    JSON_LOG_FORMATTER = proc { |data|
+      log_context = data.delete(:log_context)
+      data.merge(log_context).compact.to_json << "\n"
+    }
+
+    COMPACT_LOG_FORMATTER = proc { |data|
+      severity = data[:severity]
+      msg = data[:message]
+      log_context = data[:log_context]
+      log_context["path"] = data[:path] || ""
+
+      formatted_log_context = log_context.sort.map do |k, v|
+        "#{k}=#{v}"
+      end.join(" ")
+      "#{severity.ljust(5)} #{msg&.strip} #{formatted_log_context}\n"
+    }
+
+    module ON_INITIALIZATION_FAILURE
+      RAISE = :raise
+      RETURN = :return
+    end
+
+    module ON_NO_DEFAULT
+      RAISE = :raise
+      RETURN_NIL = :return_nil
+    end
+
+    module DATASOURCES
+      ALL = :all
+      LOCAL_ONLY = :local_only
+    end
+
+    DEFAULT_MAX_PATHS = 1_000
+    DEFAULT_MAX_KEYS = 100_000
+    DEFAULT_MAX_EXAMPLE_CONTEXTS = 100_000
+    DEFAULT_MAX_EVAL_SUMMARIES = 100_000
+
+    private def init(
+      api_key: ENV['PREFAB_API_KEY'],
+      logdev: $stdout,
+      namespace: '',
+      log_formatter: DEFAULT_LOG_FORMATTER,
+      log_prefix: nil,
+      prefab_api_url: ENV['PREFAB_API_URL'] || 'https://api.prefab.cloud',
+      on_no_default: ON_NO_DEFAULT::RAISE, # options :raise, :warn_and_return_nil,
+      initialization_timeout_sec: 10, # how long to wait before on_init_failure
+      on_init_failure: ON_INITIALIZATION_FAILURE::RAISE,
+      prefab_datasources: ENV['PREFAB_DATASOURCES'] == 'LOCAL_ONLY' ? DATASOURCES::LOCAL_ONLY : DATASOURCES::ALL,
+      prefab_config_override_dir: Dir.home,
+      prefab_config_classpath_dir: '.', # where to load local overrides
+      prefab_envs: ENV['PREFAB_ENVS'].nil? ? [] : ENV['PREFAB_ENVS'].split(','),
+      collect_logger_counts: true,
+      collect_max_paths: DEFAULT_MAX_PATHS,
+      collect_sync_interval: nil,
+      context_upload_mode: :periodic_example, # :periodic_example, :shape_only, :none
+      context_max_size: DEFAULT_MAX_EVAL_SUMMARIES,
+      collect_evaluation_summaries: true,
+      collect_max_evaluation_summaries: DEFAULT_MAX_EVAL_SUMMARIES,
+      allow_telemetry_in_local_mode: false,
+      x_datafile: ENV['PREFAB_DATAFILE'],
+      x_use_local_cache: false,
+      disable_action_controller_logging: false
+    )
+      @api_key = api_key
+      @logdev = logdev
+      @namespace = namespace
+      @log_formatter = log_formatter
+      @log_prefix = log_prefix
+      @prefab_api_url = remove_trailing_slash(prefab_api_url)
+      @on_no_default = on_no_default
+      @initialization_timeout_sec = initialization_timeout_sec
+      @on_init_failure = on_init_failure
+      @prefab_datasources = prefab_datasources
+      @datafile = x_datafile
+      @prefab_config_classpath_dir = prefab_config_classpath_dir
+      @prefab_config_override_dir = prefab_config_override_dir
+      @prefab_envs = Array(prefab_envs)
+      @collect_logger_counts = collect_logger_counts
+      @collect_max_paths = collect_max_paths
+      @collect_sync_interval = collect_sync_interval
+      @collect_evaluation_summaries = collect_evaluation_summaries
+      @collect_max_evaluation_summaries = collect_max_evaluation_summaries
+      @allow_telemetry_in_local_mode = allow_telemetry_in_local_mode
+      @use_local_cache = x_use_local_cache
+      @disable_action_controller_logging = disable_action_controller_logging
+      @is_fork = false
+
+      # defaults that may be overridden by context_upload_mode
+      @collect_shapes = false
+      @collect_max_shapes = 0
+      @collect_example_contexts = false
+      @collect_max_example_contexts = 0
+
+      case context_upload_mode
+      when :none
+        # do nothing
+      when :periodic_example
+        @collect_example_contexts = true
+        @collect_max_example_contexts = context_max_size
+      when :shape_only
+        @collect_shapes = true
+        @collect_max_shapes = context_max_size
+      else
+        raise "Unknown context_upload_mode #{context_upload_mode}. Please provide :periodic_example, :shape_only, or :none."
+      end
+    end
+
+    def initialize(options = {})
+      init(**options)
+    end
+
+    def local_only?
+      @prefab_datasources == DATASOURCES::LOCAL_ONLY
+    end
+
+    def datafile?
+      !@datafile.nil?
+    end
+
+    def collect_max_paths
+      return 0 unless telemetry_allowed?(@collect_logger_counts)
+
+      @collect_max_paths
+    end
+
+    def collect_max_shapes
+      return 0 unless telemetry_allowed?(@collect_shapes)
+
+      @collect_max_shapes
+    end
+
+    def collect_max_example_contexts
+      return 0 unless telemetry_allowed?(@collect_example_contexts)
+
+      @collect_max_example_contexts
+    end
+
+    def collect_max_evaluation_summaries
+      return 0 unless telemetry_allowed?(@collect_evaluation_summaries)
+
+      @collect_max_evaluation_summaries
+    end
+
+    # https://api.prefab.cloud -> https://api-prefab-cloud.global.ssl.fastly.net
+    def url_for_api_cdn
+      ENV['PREFAB_CDN_URL'] || "#{@prefab_api_url.gsub(/\./, '-')}.global.ssl.fastly.net"
+    end
+
+    def api_key_id
+      @api_key&.split("-")&.first
+    end
+
+    def for_fork
+      clone = self.clone
+      clone.is_fork = true
+      clone
+    end
+
+    private
+
+    def telemetry_allowed?(option)
+      option && (!local_only? || @allow_telemetry_in_local_mode)
+    end
+
+    def remove_trailing_slash(url)
+      url.end_with?('/') ? url[0..-2] : url
+    end
+  end
+end

data/lib/prefab/periodic_sync.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Prefab
+  module PeriodicSync
+    LOG = Prefab::InternalLogger.new("periodsync")
+    def sync
+      return if @data.size.zero?
+
+      LOG.debug "Syncing #{@data.size} items"
+
+      start_at_was = @start_at
+      @start_at = Prefab::TimeHelpers.now_in_ms
+
+      flush(prepare_data, start_at_was)
+    end
+
+    def prepare_data
+      to_ship = @data.dup
+      @data.clear
+
+      on_prepare_data
+
+      to_ship
+    end
+
+    def on_prepare_data
+      # noop -- override as you wish
+    end
+
+    def post(url, data)
+      @client.post(url, data)
+    end
+
+    def start_periodic_sync(sync_interval)
+      @start_at = Prefab::TimeHelpers.now_in_ms
+
+      @sync_interval = calculate_sync_interval(sync_interval)
+
+      Thread.new do
+        LOG.debug "Initialized #{@name} instance_hash=#{@client.instance_hash}"
+
+        loop do
+          sleep @sync_interval.call
+          sync
+        end
+      end
+    end
+
+    def pool
+      @pool ||= Concurrent::ThreadPoolExecutor.new(
+        fallback_policy: :discard,
+        max_queue: 5,
+        max_threads: 4,
+        min_threads: 1,
+        name: @name
+      )
+    end
+
+    private
+
+    def calculate_sync_interval(sync_interval)
+      if sync_interval.is_a?(Numeric)
+        proc { sync_interval }
+      else
+        sync_interval || ExponentialBackoff.new(initial_delay: 8, max_delay: 60 * 5)
+      end
+    end
+  end
+end

data/lib/prefab/prefab.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Prefab
+  @@lock = Concurrent::ReadWriteLock.new
+
+  def self.init(options = Prefab::Options.new)
+    unless @singleton.nil?
+      Prefab::LoggerClient.instance.warn 'Prefab already initialized.'
+      return @singleton
+    end
+
+    @@lock.with_write_lock {
+      @singleton = Prefab::Client.new(options)
+    }
+  end
+
+  def self.fork
+    ensure_initialized
+    @@lock.with_write_lock {
+      @singleton = @singleton.fork
+    }
+  end
+
+  def self.set_rails_loggers
+    ensure_initialized
+    @singleton.set_rails_loggers
+  end
+
+  def self.get(key, properties = NO_DEFAULT_PROVIDED)
+    ensure_initialized key
+    @singleton.get(key, properties)
+  end
+
+  def self.enabled?(feature_name, jit_context = NO_DEFAULT_PROVIDED)
+    ensure_initialized feature_name
+    @singleton.enabled?(feature_name, jit_context)
+  end
+
+  def self.with_context(properties, &block)
+    ensure_initialized
+    @singleton.with_context(properties, &block)
+  end
+
+  def self.instance
+    ensure_initialized
+    @singleton
+  end
+
+  private
+
+  def self.ensure_initialized(key = nil)
+    if not defined? @singleton or @singleton.nil?
+      raise Prefab::Errors::UninitializedError.new(key)
+    end
+  end
+end

data/lib/prefab/rate_limit_cache.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Prefab
+  # A key-based rate limiter that considers a key to be fresh if it has been
+  # seen within the last `duration` seconds.
+  #
+  # This is used to rate limit the number of times we send a given context
+  # to the server.
+  #
+  # Because expected usage is to immediately `set` on a `fresh?` miss, we do
+  # not prune the data structure on `fresh?` calls. Instead, we manually invoke
+  # `prune` periodically from the cache consumer.
+  class RateLimitCache
+    attr_reader :data
+
+    def initialize(duration)
+      @data = Concurrent::Map.new
+      @duration = duration
+    end
+
+    def fresh?(key)
+      timestamp = @data[key]
+
+      return false unless timestamp
+      return false if Time.now.utc.to_i - timestamp > @duration
+
+      true
+    end
+
+    def set(key)
+      @data[key] = Time.now.utc.to_i
+    end
+
+    def prune
+      now = Time.now.utc.to_i
+      @data.each_pair do |key, (timestamp, _)|
+        @data.delete(key) if now - timestamp > @duration
+      end
+    end
+  end
+end