event_meter 0.1.0

data/lib/event_meter/report_definition.rb

@@ -0,0 +1,184 @@
+require "digest"
+require "json"
+
+require_relative "hash_input"
+
+module EventMeter
+  class ReportDefinition
+    Index = Struct.new(:params, keyword_init: true) do
+      def matches?(by)
+        params == ReportDefinition.normalize_params(by.keys)
+      end
+
+      def key_for(values)
+        IndexKey.build(params, values)
+      end
+
+      def to_h
+        params.map(&:to_s)
+      end
+    end
+
+    Interval = Struct.new(:param, :group_by, keyword_init: true) do
+      def initialize(param:, group_by: [])
+        super(
+          param: ReportDefinition.normalize_param(param),
+          group_by: ReportDefinition.normalize_params(group_by)
+        )
+      end
+
+      def group_index
+        Index.new(params: group_by)
+      end
+
+      def to_h
+        {
+          "param" => param.to_s,
+          "group_by" => group_by.map(&:to_s)
+        }
+      end
+    end
+
+    BuiltIndex = Struct.new(:index, :key, keyword_init: true)
+
+    attr_reader :name, :version, :indexes, :intervals
+
+    def self.build(name, version:)
+      definition = new(name: name, version: version)
+      yield definition if block_given?
+      definition
+    end
+
+    def self.from_h(hash)
+      hash = HashInput.coerce(hash, "report definition")
+
+      new(name: hash.fetch("name"), version: hash.fetch("version")).tap do |definition|
+        definition.send(:replace_indexes, hash.fetch("indexes", []))
+        definition.send(:replace_intervals, hash.fetch("intervals", []))
+        definition.send(:validate_fingerprint!, hash.fetch("fingerprint"))
+      end
+    end
+
+    def initialize(name:, version:)
+      @name = normalize_name(name)
+      @version = normalize_version(version)
+      @indexes = [Index.new(params: [])]
+      @intervals = []
+    end
+
+    def index_by(*params)
+      normalized = self.class.normalize_params(params)
+      return self if indexes.any? { |index| index.params == normalized }
+
+      indexes << Index.new(params: normalized)
+      self
+    end
+
+    def measure_interval_by(param, group_by: [])
+      interval = Interval.new(param: param, group_by: group_by)
+      intervals << interval unless intervals.any? { |existing| existing == interval }
+      index_by(*interval.group_by) unless interval.group_by.empty?
+      self
+    end
+
+    def indexes_for(payload)
+      indexes.filter_map do |index|
+        next unless index.params.all? { |param| self.class.indexable_value?(payload.params, param) }
+
+        BuiltIndex.new(index: index, key: index.key_for(payload.params))
+      end
+    end
+
+    def index_for!(by)
+      normalized_by = normalize_by(by)
+      index = indexes.find { |candidate| candidate.matches?(normalized_by) }
+
+      unless index
+        raise UnsupportedQueryError, "no index configured for #{name} v#{version} by #{normalized_by.keys.inspect}"
+      end
+
+      BuiltIndex.new(index: index, key: index.key_for(normalized_by))
+    end
+
+    def fingerprint
+      Digest::SHA256.hexdigest(JSON.generate(canonical_h))
+    end
+
+    def to_h
+      canonical_h.merge("fingerprint" => fingerprint)
+    end
+
+    def self.normalize_params(params)
+      params = params.first if params.is_a?(Array) && params.length == 1 && params.first.is_a?(Array)
+      Array(params).map { |param| normalize_param(param) }.uniq.sort
+    end
+
+    def self.normalize_param(param)
+      return param.to_sym if param.respond_to?(:to_sym) && !param.to_s.strip.empty?
+
+      raise ArgumentError, "report params must be strings or symbols"
+    end
+
+    def self.indexable_value?(values, param)
+      values.key?(param.to_s) && !values[param.to_s].nil?
+    end
+
+    private
+
+    def replace_indexes(raw_indexes)
+      @indexes = []
+      Array(raw_indexes).each { |params| index_by(*Array(params)) }
+      index_by if @indexes.empty?
+    end
+
+    def replace_intervals(raw_intervals)
+      @intervals = []
+      Array(raw_intervals).each do |raw_interval|
+        interval = HashInput.coerce(raw_interval, "interval")
+        measure_interval_by(
+          interval.fetch("param"),
+          group_by: interval.fetch("group_by", [])
+        )
+      end
+    end
+
+    def validate_fingerprint!(stored_fingerprint)
+      return if stored_fingerprint == fingerprint
+
+      raise DefinitionChangedError, "#{name} v#{version} definition fingerprint does not match stored metadata"
+    end
+
+    def canonical_h
+      {
+        "name" => name,
+        "version" => version,
+        "indexes" => indexes.map(&:to_h).sort,
+        "intervals" => intervals.map(&:to_h).sort_by { |interval| [interval.fetch("param"), interval.fetch("group_by")] }
+      }
+    end
+
+    def normalize_name(value)
+      name = value.to_s
+      raise ArgumentError, "report name cannot be blank" if name.strip.empty?
+
+      name
+    end
+
+    def normalize_version(value)
+      version = Integer(value)
+      return version if version.positive?
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "report version must be a positive integer"
+    end
+
+    def normalize_by(by)
+      HashInput.coerce(by, "by").each_with_object({}) do |(key, value), hash|
+        raise ArgumentError, "by values cannot be nil" if value.nil?
+
+        hash[self.class.normalize_param(key)] = value
+      end
+    end
+  end
+end

data/lib/event_meter/reports.rb

@@ -0,0 +1,143 @@
+require "time"
+
+module EventMeter
+  class Reports
+    DEFAULT_SERIES_SECONDS = 3600
+
+    def initialize(configuration:, rollup_storage:)
+      @configuration = configuration
+      @rollup_storage = rollup_storage
+    end
+
+    def summary(name, version:, from: nil, to: nil, by: {})
+      raise ArgumentError, "pass both from: and to:, or neither" if from.nil? != to.nil?
+      from = time_value(from) if from
+      to = time_value(to) if to
+      validate_window!(from, to) if from && to
+
+      definition = report_definition(name, version)
+      index = definition.index_for!(by)
+      keys, seconds = summary_keys(definition, from: from, to: to, index: index)
+
+      Rollup.combine(@rollup_storage.hgetall_many(keys)).to_h(seconds: seconds)
+    end
+
+    def series(name, version:, from: nil, to: nil, every: :minute, by: {})
+      every = TimeBuckets.normalize(every)
+      to = to ? time_value(to) : TimeBuckets.time(Time.now.utc, every) + TimeBuckets.seconds(every)
+      from = time_value(from) if from
+      from ||= to - DEFAULT_SERIES_SECONDS
+      validate_window!(from, to)
+
+      definition = report_definition(name, version)
+      index = definition.index_for!(by)
+      buckets = TimeBuckets.between(from, to, every)
+      keys = buckets.map { |bucket| rollup_key(definition, every, bucket, index) }
+      raw_rollups = @rollup_storage.hgetall_many(keys)
+      seconds = TimeBuckets.seconds(every)
+
+      buckets.zip(raw_rollups).map do |bucket, raw|
+        Rollup.from_hash(raw).to_h(seconds: seconds).merge(bucket: bucket.iso8601)
+      end
+    end
+
+    def compare(name, version:, before:, after:, by: {})
+      before_from, before_to = comparison_window(before)
+      after_from, after_to = comparison_window(after)
+
+      {
+        before: summary(name, version: version, from: before_from, to: before_to, by: by),
+        after: summary(name, version: version, from: after_from, to: after_to, by: by)
+      }
+    end
+
+    private
+
+    def summary_keys(definition, from:, to:, index:)
+      if from && to
+        buckets = TimeBuckets.between(from, to, :minute)
+        keys = buckets.map { |bucket| rollup_key(definition, :minute, bucket, index) }
+        seconds = buckets.length * TimeBuckets.seconds(:minute)
+
+        [keys, seconds]
+      else
+        pattern = Keys.rollup_pattern(
+          namespace: @configuration.namespace,
+          name: definition.name,
+          version: definition.version,
+          every: :hour,
+          index: index
+        )
+
+        keys = keys_matching(pattern)
+        validate_summary_key_count!(keys)
+        [keys, nil]
+      end
+    end
+
+    def keys_matching(pattern)
+      limit = @configuration.summary_key_limit
+      storage_key_limit = limit && limit + 1
+      method = @rollup_storage.method(:keys_matching)
+
+      if method.parameters.any? { |kind, name| kind == :keyrest || name == :limit }
+        @rollup_storage.keys_matching(pattern, limit: storage_key_limit)
+      else
+        @rollup_storage.keys_matching(pattern)
+      end
+    end
+
+    def validate_summary_key_count!(keys)
+      limit = @configuration.summary_key_limit
+      return unless limit && keys.length > limit
+
+      raise ArgumentError, "summary without a time window matched more than #{limit} rollup buckets; pass from: and to:"
+    end
+
+    def rollup_key(definition, every, bucket, index)
+      Keys.rollup(
+        namespace: @configuration.namespace,
+        name: definition.name,
+        version: definition.version,
+        every: every,
+        bucket: bucket,
+        index: index
+      )
+    end
+
+    def report_definition(name, version)
+      stored = @rollup_storage.report_definition(name: name, version: version)
+      raise DefinitionNotFoundError, "no definition stored for #{name} v#{version}" unless stored
+
+      ReportDefinition.from_h(stored)
+    end
+
+    def validate_window!(from, to)
+      return if to > from
+
+      raise ArgumentError, "to must be after from"
+    end
+
+    def comparison_window(window)
+      unless window.respond_to?(:begin) && window.respond_to?(:end)
+        raise ArgumentError, "comparison windows must be ranges"
+      end
+
+      from = window.begin
+      to = window.end
+      raise ArgumentError, "comparison windows must have a start and end" if from.nil? || to.nil?
+
+      [from, to]
+    end
+
+    def time_value(value)
+      return value.utc if value.respond_to?(:utc)
+      raise ArgumentError unless value.respond_to?(:to_str)
+
+      Time.parse(value.to_str).utc
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "time must be a Time or parseable time string"
+    end
+
+  end
+end

data/lib/event_meter/rollup.rb

@@ -0,0 +1,148 @@
+require "time"
+
+module EventMeter
+  class Rollup
+    MIN_FIELDS = %w[duration_ms_min interval_ms_min started_at_ms_min].freeze
+    MAX_FIELDS = %w[duration_ms_max interval_ms_max started_at_ms_max].freeze
+    MAX_RENDERABLE_TIMESTAMP_MS = 253_402_300_799_999 # 9999-12-31T23:59:59.999Z
+
+    attr_reader :fields
+
+    def self.from_hash(hash)
+      new(hash || {})
+    end
+
+    def self.combine(raw_rollups)
+      raw_rollups.reduce(new) do |combined, raw|
+        combined.merge!(from_hash(raw))
+      end
+    end
+
+    def self.min_field?(field)
+      MIN_FIELDS.include?(field.to_s)
+    end
+
+    def self.max_field?(field)
+      MAX_FIELDS.include?(field.to_s)
+    end
+
+    def initialize(fields = {})
+      @fields = {}
+      fields.each { |key, value| @fields[key.to_s] = numeric_value(value) }
+    end
+
+    def increment(field, value = 1)
+      fields[field.to_s] = fields.fetch(field.to_s, 0) + numeric_value(value)
+    end
+
+    def add_duration(duration_ms)
+      return if duration_ms.nil?
+
+      add_metric("duration_ms", duration_ms)
+    end
+
+    def add_interval(interval_ms)
+      return if interval_ms.nil?
+
+      add_metric("interval_ms", interval_ms)
+    end
+
+    def add_started_at(started_ms)
+      return if started_ms.nil?
+
+      value = numeric_value(started_ms)
+      fields["started_at_ms_min"] = [fields["started_at_ms_min"], value].compact.min
+      fields["started_at_ms_max"] = [fields["started_at_ms_max"], value].compact.max
+    end
+
+    def merge!(other)
+      other.fields.each do |field, value|
+        if self.class.min_field?(field)
+          fields[field] = [fields[field], value].compact.min
+        elsif self.class.max_field?(field)
+          fields[field] = [fields[field], value].compact.max
+        else
+          increment(field, value)
+        end
+      end
+
+      self
+    end
+
+    def to_h(seconds: nil)
+      count = fields.fetch("count", 0)
+      duration_count = fields.fetch("duration_ms_count", 0)
+      interval_count = fields.fetch("interval_ms_count", 0)
+      started_at_min_ms = fields["started_at_ms_min"]
+      started_at_max_ms = fields["started_at_ms_max"]
+      rate_window_seconds = seconds || inferred_rate_window_seconds(started_at_min_ms, started_at_max_ms)
+
+      {
+        count: count,
+        success_count: fields.fetch("success_count", 0),
+        failure_count: fields.fetch("failure_count", 0),
+        skipped_count: fields.fetch("skipped_count", 0),
+        started_at_min: iso_time(started_at_min_ms),
+        started_at_max: iso_time(started_at_max_ms),
+        rate_window_seconds: rate_window_seconds,
+        per_second: rate(count, rate_window_seconds),
+        per_minute: rate(count * 60, rate_window_seconds),
+        duration_ms_count: duration_count,
+        duration_ms_sum: fields.fetch("duration_ms_sum", 0),
+        duration_ms_avg: average(fields.fetch("duration_ms_sum", 0), duration_count),
+        duration_ms_min: fields["duration_ms_min"],
+        duration_ms_max: fields["duration_ms_max"],
+        interval_ms_count: interval_count,
+        interval_ms_sum: fields.fetch("interval_ms_sum", 0),
+        interval_ms_avg: average(fields.fetch("interval_ms_sum", 0), interval_count),
+        interval_ms_min: fields["interval_ms_min"],
+        interval_ms_max: fields["interval_ms_max"]
+      }.delete_if { |_key, value| value.nil? }
+    end
+
+    private
+
+    def inferred_rate_window_seconds(started_at_min_ms, started_at_max_ms)
+      return nil unless started_at_min_ms && started_at_max_ms
+
+      (started_at_max_ms - started_at_min_ms) / 1000.0
+    end
+
+    def rate(count, seconds)
+      return nil unless seconds&.positive?
+
+      count / seconds.to_f
+    end
+
+    def average(sum, count)
+      return nil unless count.positive?
+
+      sum / count.to_f
+    end
+
+    def iso_time(milliseconds)
+      return nil unless milliseconds
+      return nil if milliseconds.abs > MAX_RENDERABLE_TIMESTAMP_MS
+
+      Time.at(milliseconds / 1000.0).utc.iso8601(6)
+    rescue ArgumentError, RangeError
+      nil
+    end
+
+    def add_metric(prefix, value)
+      value = numeric_value(value)
+      return if value.negative?
+
+      increment("#{prefix}_count")
+      increment("#{prefix}_sum", value)
+      fields["#{prefix}_min"] = [fields["#{prefix}_min"], value].compact.min
+      fields["#{prefix}_max"] = [fields["#{prefix}_max"], value].compact.max
+    end
+
+    def numeric_value(value)
+      Integer(value)
+    rescue ArgumentError, TypeError, RangeError
+      0
+    end
+  end
+end

data/lib/event_meter/stores/cleanup_helpers.rb

@@ -0,0 +1,76 @@
+require "time"
+
+require_relative "../index_key"
+
+module EventMeter
+  module Stores
+    module CleanupHelpers
+      private
+
+      def event_filter(events)
+        events = Array(events).compact
+
+        events.empty? ? nil : events.map { |event| IndexKey.escape(event) }
+      end
+
+      def rollup_key_old?(key, before, filter)
+        prefix = "#{namespace}:rollup:"
+        return false unless key.start_with?(prefix)
+
+        event_name, _version, every, bucket_id = key.delete_prefix(prefix).split(":", 5)
+        return false unless event_name && every && bucket_id
+        return false if filter && !filter.include?(event_name)
+
+        bucket_end_time(every, bucket_id) <= before
+      rescue ArgumentError, TypeError
+        false
+      end
+
+      def state_key_old?(key, before_ms, filter, value)
+        prefix = "#{namespace}:state:"
+        return false unless key.start_with?(prefix)
+
+        event_name = key.delete_prefix(prefix).split(":", 3).first
+        return false if filter && !filter.include?(event_name)
+
+        Integer(value) < before_ms
+      rescue ArgumentError, TypeError, RangeError
+        true
+      end
+
+      def bucket_end_time(every, bucket_id)
+        case every
+        when "minute"
+          minute_bucket_time(bucket_id) + 60
+        when "hour"
+          hour_bucket_time(bucket_id) + 3600
+        else
+          raise ArgumentError, "unsupported rollup bucket: #{every.inspect}"
+        end
+      end
+
+      def minute_bucket_time(bucket_id)
+        raise ArgumentError, "malformed minute bucket" unless bucket_id.to_s.match?(/\A\d{12}\z/)
+
+        Time.utc(
+          bucket_id[0, 4].to_i,
+          bucket_id[4, 2].to_i,
+          bucket_id[6, 2].to_i,
+          bucket_id[8, 2].to_i,
+          bucket_id[10, 2].to_i
+        )
+      end
+
+      def hour_bucket_time(bucket_id)
+        raise ArgumentError, "malformed hour bucket" unless bucket_id.to_s.match?(/\A\d{10}\z/)
+
+        Time.utc(
+          bucket_id[0, 4].to_i,
+          bucket_id[4, 2].to_i,
+          bucket_id[6, 2].to_i,
+          bucket_id[8, 2].to_i
+        )
+      end
+    end
+  end
+end

data/lib/event_meter/stores/file_helpers.rb

@@ -0,0 +1,47 @@
+require "fileutils"
+require "json"
+require "securerandom"
+
+module EventMeter
+  module Stores
+    module FileHelpers
+      private
+
+      def atomic_write_json(path, value)
+        atomic_write(path, JSON.generate(value))
+      end
+
+      def normalize_file_store_path(value, name: "path")
+        path = value.to_s
+        raise ArgumentError, "#{name} cannot be blank" if path.strip.empty?
+
+        ::File.expand_path(path)
+      end
+
+      def atomic_write(path, contents)
+        temporary_path = temporary_path_for(path)
+
+        ::File.open(temporary_path, ::File::WRONLY | ::File::CREAT | ::File::TRUNC, 0o600) do |file|
+          file.write(contents)
+          file.flush
+          file.fsync
+        end
+
+        ::File.rename(temporary_path, path)
+        fsync_directory(::File.dirname(path))
+      ensure
+        FileUtils.rm_f(temporary_path) if temporary_path && ::File.exist?(temporary_path)
+      end
+
+      def fsync_directory(path)
+        ::File.open(path, ::File::RDONLY) { |directory| directory.fsync }
+      rescue SystemCallError
+        nil
+      end
+
+      def temporary_path_for(path)
+        "#{path}.#{Process.pid}.#{Thread.current.object_id}.#{SecureRandom.hex(4)}.tmp"
+      end
+    end
+  end
+end

data/lib/event_meter/stores/lock_refresher.rb

@@ -0,0 +1,75 @@
+require_relative "../errors"
+
+module EventMeter
+  module Stores
+    class LockRefresher
+      JOIN_TIMEOUT = 1.0
+
+      def initialize(interval:, refresh:, failure_message:, thread_name:)
+        @interval = interval
+        @refresh = refresh
+        @failure_message = failure_message
+        @thread_name = thread_name
+        @mutex = Mutex.new
+        @condition = ConditionVariable.new
+        @stopped = false
+      end
+
+      def start(owner: Thread.current)
+        @owner = owner
+        @thread = Thread.new { run }
+        self
+      end
+
+      def stop
+        @mutex.synchronize do
+          @stopped = true
+          @condition.broadcast
+        end
+
+        @thread&.join(JOIN_TIMEOUT)
+      end
+
+      private
+
+      def run
+        Thread.current.report_on_exception = false
+        Thread.current.name = @thread_name if Thread.current.respond_to?(:name=)
+
+        loop do
+          break if stopped_after_wait?
+
+          refreshed = @refresh.call
+          raise LockLostError, @failure_message unless refreshed
+        end
+      rescue LockLostError => error
+        notify_owner(error)
+      rescue StandardError => error
+        notify_owner(LockLostError.new("#{@failure_message}: #{error.class}: #{error.message}"))
+      end
+
+      def stopped_after_wait?
+        @mutex.synchronize do
+          @condition.wait(@mutex, @interval) unless @stopped
+          @stopped
+        end
+      end
+
+      def notify_owner(error)
+        owner = @mutex.synchronize do
+          already_stopped = @stopped
+          @stopped = true
+          @condition.broadcast
+          already_stopped ? nil : @owner
+        end
+
+        # Losing the refresh means another process can acquire this lock while
+        # the owner still writes. Interrupt the owner immediately instead of
+        # letting split-brain processing continue until the block returns.
+        owner&.raise(error)
+      rescue ThreadError
+        nil
+      end
+    end
+  end
+end

data/lib/event_meter/stores/namespace.rb

@@ -0,0 +1,14 @@
+module EventMeter
+  module Stores
+    module Namespace
+      private
+
+      def normalize_namespace(value)
+        namespace = value.to_s
+        raise ArgumentError, "namespace cannot be blank" if namespace.strip.empty?
+
+        namespace
+      end
+    end
+  end
+end