event_meter 0.1.0

data/exe/event_meter

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "event_meter/cli"
+
+exit EventMeter::CLI.call(ARGV)

data/lib/event_meter/auto_cleanup.rb

@@ -0,0 +1,93 @@
+require "time"
+
+module EventMeter
+  class AutoCleanup
+    WATERMARK_KEY_SUFFIX = "auto_cleanup:history:last_run"
+
+    attr_reader :configuration, :rollup_storage
+
+    def initialize(configuration:, rollup_storage:)
+      @configuration = configuration
+      @rollup_storage = rollup_storage
+    end
+
+    def run
+      return false unless configuration.auto_cleanup_history
+      return false unless rollup_storage.respond_to?(:cleanup_history)
+
+      with_cleanup_lock { cleanup_if_due }
+    rescue StandardError => error
+      report_error(error)
+      false
+    end
+
+    private
+
+    def with_cleanup_lock
+      return yield unless rollup_storage.respond_to?(:with_lock)
+
+      result = false
+      locked = rollup_storage.with_lock(ttl: configuration.lock_ttl) do
+        result = yield
+      end
+
+      locked ? result : false
+    end
+
+    def cleanup_if_due
+      return false unless cleanup_due?
+
+      result = rollup_storage.cleanup_history(
+        before: cleanup_before,
+        events: nil,
+        interval_state: true
+      )
+      write_watermark
+      result
+    end
+
+    def cleanup_due?
+      last_cleanup_at.nil? ||
+        last_cleanup_at <= current_time - configuration.cleanup_history_interval
+    end
+
+    def last_cleanup_at
+      return unless rollup_storage.respond_to?(:cleanup_watermark)
+
+      parse_time(rollup_storage.cleanup_watermark(watermark_key))
+    end
+
+    def write_watermark
+      return unless rollup_storage.respond_to?(:write_cleanup_watermark)
+
+      rollup_storage.write_cleanup_watermark(watermark_key, current_time.iso8601(6))
+    end
+
+    def cleanup_before
+      current_time - configuration.cleanup_history_retention
+    end
+
+    def watermark_key
+      [configuration.namespace, WATERMARK_KEY_SUFFIX].join(":")
+    end
+
+    def current_time
+      Time.now.utc
+    end
+
+    def parse_time(value)
+      return if value.nil?
+
+      Time.parse(value.to_s).utc
+    rescue ArgumentError, TypeError, RangeError
+      nil
+    end
+
+    def report_error(error)
+      handler = configuration.auto_cleanup_error_handler
+      handler&.call(error)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/event_meter/cli.rb

@@ -0,0 +1,124 @@
+require "optparse"
+
+require_relative "../event_meter"
+
+module EventMeter
+  class CLI
+    def self.call(argv, out: $stdout, err: $stderr)
+      new(argv, out: out, err: err).call
+    end
+
+    def initialize(argv, out:, err:)
+      @argv = argv.dup
+      @out = out
+      @err = err
+    end
+
+    def call
+      case argv.shift
+      when "postgres"
+        postgres
+      when "help", "-h", "--help", nil
+        out.puts usage
+        0
+      else
+        err.puts usage
+        1
+      end
+    rescue OptionParser::ParseError, ArgumentError => error
+      err.puts error.message
+      1
+    end
+
+    private
+
+    attr_reader :argv, :out, :err
+
+    def postgres
+      case argv.shift
+      when "install"
+        postgres_install
+      when "schema"
+        postgres_schema
+      when "help", "-h", "--help", nil
+        out.puts postgres_usage
+        0
+      else
+        err.puts postgres_usage
+        1
+      end
+    end
+
+    def postgres_install
+      options = postgres_options
+      parse_postgres_options!(options)
+
+      url = options.fetch(:url) || ENV["DATABASE_URL"]
+      raise ArgumentError, "missing database URL; pass --url or set DATABASE_URL" if url.to_s.strip.empty?
+
+      require_pg!
+
+      connection = PG.connect(url)
+      EventMeter::Stores::Rollup::Postgres.install!(
+        connection: connection,
+        table_prefix: options.fetch(:table_prefix)
+      )
+      out.puts "Installed EventMeter PostgreSQL tables with prefix #{options.fetch(:table_prefix)}"
+      0
+    ensure
+      connection&.close
+    end
+
+    def postgres_schema
+      options = postgres_options
+      parse_postgres_options!(options)
+
+      out.puts EventMeter::Stores::Rollup::Postgres.schema_sql(
+        table_prefix: options.fetch(:table_prefix)
+      )
+      0
+    end
+
+    def postgres_options
+      {
+        table_prefix: "event_meter",
+        url: nil
+      }
+    end
+
+    def parse_postgres_options!(options)
+      parser = OptionParser.new do |parser_config|
+        parser_config.on("--table-prefix PREFIX", "PostgreSQL table prefix") do |value|
+          options[:table_prefix] = value
+        end
+        parser_config.on("--url URL", "PostgreSQL connection URL") do |value|
+          options[:url] = value
+        end
+      end
+      parser.parse!(argv)
+      raise OptionParser::InvalidArgument, argv.join(" ") unless argv.empty?
+    end
+
+    def require_pg!
+      require "pg"
+    rescue LoadError
+      raise ArgumentError, "pg is required for postgres install; add gem \"pg\" to your app"
+    end
+
+    def usage
+      <<~TEXT
+        Usage:
+          event_meter postgres schema [--table-prefix PREFIX]
+          event_meter postgres install [--url DATABASE_URL] [--table-prefix PREFIX]
+      TEXT
+    end
+
+    def postgres_usage
+      <<~TEXT
+        Usage:
+          event_meter postgres schema [--table-prefix PREFIX]
+          event_meter postgres install [--url DATABASE_URL] [--table-prefix PREFIX]
+      TEXT
+    end
+  end
+end

data/lib/event_meter/configuration.rb

@@ -0,0 +1,244 @@
+require "monitor"
+
+module EventMeter
+  class Configuration
+    DEFAULT_NAMESPACE = "event_meter:v1"
+    DEFAULT_REDIS_READ_LIMIT = nil
+    DEFAULT_ROLLUP_TTL = 31 * 24 * 60 * 60
+    DEFAULT_LOCK_TTL = 30
+    DEFAULT_AUTO_CLEANUP_HISTORY = false
+    DEFAULT_CLEANUP_HISTORY_RETENTION = DEFAULT_ROLLUP_TTL
+    DEFAULT_CLEANUP_HISTORY_INTERVAL = 60 * 60
+    DEFAULT_SUMMARY_KEY_LIMIT = 10_000
+    DEFAULT_AUTO_CLEANUP_ERROR_HANDLER = lambda do |error|
+      warn "EventMeter auto cleanup failed: #{error.class}: #{error.message}"
+    end
+
+    attr_reader :namespace, :redis, :redis_read_limit, :rollup_ttl, :lock_ttl,
+      :auto_cleanup_history, :cleanup_history_retention, :cleanup_history_interval,
+      :summary_key_limit, :auto_cleanup_error_handler
+
+    def initialize
+      @configuration_lock = Monitor.new
+      @namespace = DEFAULT_NAMESPACE
+      @redis_read_limit = DEFAULT_REDIS_READ_LIMIT
+      @rollup_ttl = DEFAULT_ROLLUP_TTL
+      @lock_ttl = DEFAULT_LOCK_TTL
+      @auto_cleanup_history = DEFAULT_AUTO_CLEANUP_HISTORY
+      @cleanup_history_retention = DEFAULT_CLEANUP_HISTORY_RETENTION
+      @cleanup_history_interval = DEFAULT_CLEANUP_HISTORY_INTERVAL
+      @summary_key_limit = DEFAULT_SUMMARY_KEY_LIMIT
+      @auto_cleanup_error_handler = DEFAULT_AUTO_CLEANUP_ERROR_HANDLER
+    end
+
+    def namespace=(value)
+      namespace = value.to_s
+      raise ArgumentError, "namespace cannot be blank" if namespace.strip.empty?
+
+      synchronize do
+        @namespace = namespace
+        reset_default_storages
+      end
+    end
+
+    def redis=(client_or_factory)
+      synchronize do
+        @redis = client_or_factory
+        reset_redis_client_cache
+        reset_default_storages
+      end
+    end
+
+    def redis_read_limit=(value)
+      redis_read_limit = positive_integer_or_nil(value, "redis_read_limit")
+
+      synchronize do
+        @redis_read_limit = redis_read_limit
+        reset_default_storages
+      end
+    end
+
+    def rollup_ttl=(value)
+      rollup_ttl = positive_integer(value, "rollup_ttl")
+
+      synchronize do
+        @rollup_ttl = rollup_ttl
+        reset_default_rollup_storage
+      end
+    end
+
+    def lock_ttl=(value)
+      @lock_ttl = positive_integer(value, "lock_ttl")
+    end
+
+    def auto_cleanup_history=(value)
+      @auto_cleanup_history = boolean(value, "auto_cleanup_history")
+    end
+
+    def cleanup_history_retention=(value)
+      @cleanup_history_retention = positive_integer(value, "cleanup_history_retention")
+    end
+
+    def cleanup_history_interval=(value)
+      @cleanup_history_interval = positive_integer(value, "cleanup_history_interval")
+    end
+
+    def summary_key_limit=(value)
+      @summary_key_limit = positive_integer_or_nil(value, "summary_key_limit")
+    end
+
+    def auto_cleanup_error_handler=(handler)
+      unless handler.nil? || handler.respond_to?(:call)
+        raise ArgumentError, "auto_cleanup_error_handler must respond to call"
+      end
+
+      @auto_cleanup_error_handler = handler
+    end
+
+    def stream_storage
+      synchronize do
+        reset_redis_clients_if_stale
+        @stream_storage ||= default_stream_storage
+      end
+    end
+
+    def stream_storage=(storage)
+      raise ArgumentError, "stream_storage cannot be nil" if storage.nil?
+
+      synchronize do
+        @stream_storage = storage
+        @stream_storage_default = false
+      end
+    end
+
+    def rollup_storage
+      synchronize do
+        reset_redis_clients_if_stale
+        @rollup_storage ||= default_rollup_storage
+      end
+    end
+
+    def rollup_storage=(storage)
+      raise ArgumentError, "rollup_storage cannot be nil" if storage.nil?
+
+      synchronize do
+        @rollup_storage = storage
+        @rollup_storage_default = false
+      end
+    end
+
+    private
+
+    def synchronize(&block)
+      @configuration_lock.synchronize(&block)
+    end
+
+    def default_stream_storage
+      return missing_storage!("stream_storage") unless redis
+
+      @stream_storage_default = true
+      Stores::Stream::Redis.new(
+        redis: redis_client(:stream),
+        lock_redis: redis_client(:stream_lock),
+        namespace: namespace,
+        redis_read_limit: redis_read_limit
+      )
+    end
+
+    def default_rollup_storage
+      return missing_storage!("rollup_storage") unless redis
+
+      @rollup_storage_default = true
+      Stores::Rollup::Redis.new(
+        redis: redis_client(:rollup),
+        lock_redis: redis_client(:rollup_lock),
+        namespace: namespace,
+        rollup_ttl: rollup_ttl
+      )
+    end
+
+    def reset_default_storages
+      @stream_storage = nil if @stream_storage_default
+      reset_default_rollup_storage
+    end
+
+    def reset_default_rollup_storage
+      @rollup_storage = nil if @rollup_storage_default
+    end
+
+    def redis_client(purpose = :default)
+      synchronize do
+        reset_redis_clients_if_stale
+        initialize_redis_client_cache
+        @redis_clients[purpose] ||= begin
+          client = redis.is_a?(Proc) ? redis.call : redis
+          raise ConfigurationError, "redis client cannot be nil" if client.nil?
+          raise ConfigurationError, "redis client cannot be closed" if redis_client_closed?(client)
+
+          client
+        end
+      end
+    end
+
+    def reset_redis_clients_if_stale
+      return unless @redis_clients
+      return unless redis_client_cache_stale?
+
+      reset_redis_client_cache
+      reset_default_storages
+    end
+
+    def reset_redis_client_cache
+      @redis_clients = nil
+      @redis_client_pid = nil
+    end
+
+    def initialize_redis_client_cache
+      return if @redis_clients
+
+      @redis_clients = {}
+      @redis_client_pid = process_id
+    end
+
+    def redis_client_cache_stale?
+      @redis_client_pid != process_id || @redis_clients.any? do |_purpose, client|
+        redis_client_closed?(client)
+      end
+    end
+
+    def process_id
+      Process.pid
+    end
+
+    def redis_client_closed?(client)
+      client.respond_to?(:closed?) && client.closed?
+    rescue StandardError
+      false
+    end
+
+    def missing_storage!(name)
+      raise ConfigurationError, "configure #{name}, or set config.redis to use Redis storage"
+    end
+
+    def positive_integer(value, name)
+      integer = Integer(value)
+      return integer if integer.positive?
+
+      raise ArgumentError, "#{name} must be positive"
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "#{name} must be positive"
+    end
+
+    def positive_integer_or_nil(value, name)
+      return nil if value.nil?
+
+      positive_integer(value, name)
+    end
+
+    def boolean(value, name)
+      return value if value == true || value == false
+
+      raise ArgumentError, "#{name} must be true or false"
+    end
+  end
+end

data/lib/event_meter/errors.rb

@@ -0,0 +1,9 @@
+module EventMeter
+  class Error < StandardError; end
+  class UnsupportedQueryError < Error; end
+  class AlreadyRecordedError < Error; end
+  class ConfigurationError < Error; end
+  class DefinitionChangedError < Error; end
+  class DefinitionNotFoundError < Error; end
+  class LockLostError < Error; end
+end

data/lib/event_meter/event.rb

@@ -0,0 +1,180 @@
+require "time"
+
+require_relative "errors"
+require_relative "event_payload"
+require_relative "hash_input"
+require_relative "write_result"
+
+module EventMeter
+  class Event
+    INVALID_EVENT_NAME = "event_meter.invalid"
+
+    attr_reader :name, :started_at, :finished_at, :status, :error
+
+    def self.start(name, attributes, keyword_attributes, started_at:)
+      new(name, attributes, keyword_attributes, started_at: started_at)
+    end
+
+    def self.failed(error)
+      new(INVALID_EVENT_NAME, error: error)
+    end
+
+    def initialize(name, attributes = nil, keyword_attributes = {}, started_at: nil, error: nil)
+      @started_at = nil
+      @finished_at = nil
+      @status = nil
+      @recorded = false
+      @error = nil
+
+      if error
+        @name = INVALID_EVENT_NAME
+        @base_attributes = {}
+        @error = error
+        return
+      end
+
+      @started_at = started_at&.utc
+      @name = normalize_name(name)
+      @base_attributes = normalize_start_attributes(attributes, keyword_attributes)
+    rescue StandardError => error
+      @name = fallback_name(name)
+      @base_attributes = {}
+      @error = error
+    end
+
+    def success(attributes = {})
+      finish("success") { attributes }
+    end
+
+    def skip(reason_or_attributes = nil, attributes = {})
+      finish("skipped") do
+        normalize_reason_attributes(reason_or_attributes, attributes, :skip_reason)
+      end
+    end
+
+    def failure(error_or_attributes = nil, attributes = {})
+      finish("failure") do
+        normalize_error_attributes(error_or_attributes, attributes)
+      end
+    end
+
+    def error?
+      !error.nil?
+    end
+
+    private
+
+    def finish(status)
+      if recorded?
+        return WriteResult.failed(
+          payload: nil,
+          error: AlreadyRecordedError.new("event has already been recorded")
+        )
+      end
+
+      @started_at ||= current_time
+      @status = status
+      @finished_at = current_time
+
+      return WriteResult.failed(payload: nil, error: error) if error?
+
+      payload = nil
+      payload_hash = nil
+      attributes = yield
+
+      payload = EventPayload.build(
+        name,
+        params: record_attributes(attributes),
+        status: status,
+        started_at: started_at,
+        duration_ms: duration_ms
+      )
+      payload_hash = payload.to_h
+
+      EventMeter.stream_storage.append(payload)
+      @recorded = true
+      WriteResult.recorded(payload_hash)
+    rescue StandardError => error
+      WriteResult.failed(payload: payload_hash, error: error)
+    end
+
+    def record_attributes(attributes)
+      @base_attributes.merge(normalize_attributes(attributes))
+    end
+
+    def duration_ms
+      [((finished_at.to_f - started_at.to_f) * 1000).round, 0].max
+    end
+
+    def normalize_reason_attributes(reason_or_attributes, attributes, key)
+      attributes = normalize_attributes(attributes)
+
+      case reason_or_attributes
+      when nil
+        attributes
+      when Hash
+        normalize_attributes(reason_or_attributes).merge(attributes)
+      else
+        { key => reason_or_attributes.to_s }.merge(attributes)
+      end
+    end
+
+    def normalize_error_attributes(error_or_attributes, attributes)
+      attributes = normalize_attributes(attributes)
+
+      case error_or_attributes
+      when nil
+        attributes
+      when Hash
+        normalize_attributes(error_or_attributes).merge(attributes)
+      else
+        {
+          error_class: error_or_attributes.class.name,
+          error_message: error_message(error_or_attributes)
+        }.merge(attributes)
+      end
+    end
+
+    def normalize_attributes(attributes)
+      HashInput.coerce(attributes, "event attributes")
+    end
+
+    def normalize_start_attributes(attributes, keyword_attributes)
+      normalize_attributes(attributes).merge(normalize_attributes(keyword_attributes))
+    end
+
+    def normalize_name(value)
+      name = value.to_s
+      raise ArgumentError, "event name cannot be blank" if name.strip.empty?
+
+      name
+    end
+
+    def fallback_name(value)
+      name = safe_string(value)
+      return INVALID_EVENT_NAME if name.strip.empty?
+
+      name
+    end
+
+    def safe_string(value)
+      value.to_s
+    rescue StandardError
+      INVALID_EVENT_NAME
+    end
+
+    def error_message(error)
+      return error.message if error.respond_to?(:message)
+
+      error.to_s
+    end
+
+    def recorded?
+      @recorded
+    end
+
+    def current_time
+      Time.now.utc
+    end
+  end
+end