event_meter 0.1.0

data/lib/event_meter/event_payload.rb

@@ -0,0 +1,103 @@
+require "json"
+require "time"
+
+require_relative "hash_input"
+
+module EventMeter
+  class EventPayload
+    attr_reader :name, :status, :started_at, :duration_ms, :params
+
+    def self.build(name, params:, status: nil, started_at: nil, duration_ms: nil)
+      new(
+        name: name,
+        params: params,
+        status: status,
+        started_at: started_at,
+        duration_ms: duration_ms
+      )
+    end
+
+    def self.load(hash)
+      hash = stringify(hash)
+
+      new(
+        name: hash.fetch("name"),
+        params: hash.fetch("params", {}),
+        status: hash["status"],
+        started_at: hash.fetch("started_at"),
+        duration_ms: hash["duration_ms"]
+      )
+    end
+
+    def initialize(name:, params:, status:, started_at:, duration_ms:)
+      now = Time.now.utc
+
+      @name = normalize_name(name)
+      @status = normalize_status(status || "success")
+      @started_at = parse_time(started_at || now)
+      @duration_ms = integer_or_nil(duration_ms)
+      @params = stringify(params)
+    end
+
+    def to_h
+      {
+        "name" => name,
+        "status" => status,
+        "started_at" => started_at.iso8601(6),
+        "duration_ms" => duration_ms,
+        "params" => params
+      }.delete_if { |_key, value| value.nil? || value == {} }
+    end
+
+    def to_json(*args)
+      to_h.to_json(*args)
+    end
+
+    def started_ms
+      (started_at.to_f * 1000).to_i
+    end
+
+    private
+
+    def self.stringify(hash)
+      HashInput.coerce(hash, "event params").each_with_object({}) do |(key, value), result|
+        result[key.to_s] = value
+      end
+    end
+
+    def stringify(hash)
+      self.class.stringify(hash)
+    end
+
+    def parse_time(value)
+      return value.utc if value.respond_to?(:utc)
+      raise ArgumentError unless value.respond_to?(:to_str)
+
+      Time.parse(value.to_str).utc
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "started_at must be a Time or parseable time string"
+    end
+
+    def normalize_name(value)
+      name = value.to_s
+      raise ArgumentError, "event name cannot be blank" if name.strip.empty?
+
+      name
+    end
+
+    def normalize_status(value)
+      status = value.to_s
+      raise ArgumentError, "event status cannot be blank" if status.strip.empty?
+
+      status
+    end
+
+    def integer_or_nil(value)
+      return nil if value.nil?
+
+      [Integer(value), 0].max
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "duration_ms must be an integer"
+    end
+  end
+end

data/lib/event_meter/hash_input.rb

@@ -0,0 +1,20 @@
+module EventMeter
+  module HashInput
+    module_function
+
+    def coerce(value, label)
+      return {} if value.nil?
+
+      unless value.respond_to?(:to_h)
+        raise TypeError, "#{label} must respond to to_h"
+      end
+
+      hash = value.to_h
+      unless hash.is_a?(Hash)
+        raise TypeError, "#{label}#to_h must return a Hash"
+      end
+
+      hash.dup
+    end
+  end
+end

data/lib/event_meter/index_key.rb

@@ -0,0 +1,19 @@
+require "cgi/escape"
+
+module EventMeter
+  module IndexKey
+    module_function
+
+    def build(params, values)
+      return "all" if params.empty?
+
+      params.map do |param|
+        "#{escape(param)}=#{escape(values.fetch(param) { values.fetch(param.to_s) })}"
+      end.join("|")
+    end
+
+    def escape(value)
+      CGI.escape(value.to_s)
+    end
+  end
+end

data/lib/event_meter/keys.rb

@@ -0,0 +1,63 @@
+module EventMeter
+  module Keys
+    module_function
+
+    def rollup(namespace:, name:, version:, every:, bucket:, index:)
+      [
+        namespace,
+        "rollup",
+        event_name(name),
+        version_key(version),
+        every,
+        TimeBuckets.id(bucket, every),
+        index.key
+      ].join(":")
+    end
+
+    def rollup_pattern(namespace:, name:, version:, every:, index:)
+      [
+        namespace,
+        "rollup",
+        event_name(name),
+        version_key(version),
+        every,
+        "*",
+        index.key
+      ].join(":")
+    end
+
+    def interval_state(namespace:, name:, version:, definition:, value:)
+      [
+        namespace,
+        "state",
+        event_name(name),
+        version_key(version),
+        "interval",
+        IndexKey.escape(definition.param),
+        IndexKey.escape(value)
+      ].join(":")
+    end
+
+    def definition(namespace:, name:, version:)
+      [namespace, "definition", event_name(name), version_key(version)].join(":")
+    end
+
+    def processed(namespace:, name:, version:, id:)
+      [
+        namespace,
+        "processed",
+        event_name(name),
+        version_key(version),
+        IndexKey.escape(id)
+      ].join(":")
+    end
+
+    def event_name(name)
+      IndexKey.escape(name)
+    end
+
+    def version_key(version)
+      PathName.version(version)
+    end
+  end
+end

data/lib/event_meter/path_name.rb

@@ -0,0 +1,37 @@
+require "digest"
+
+module EventMeter
+  module PathName
+    WINDOWS_RESERVED_NAMES = %w[
+      con prn aux nul
+      com1 com2 com3 com4 com5 com6 com7 com8 com9
+      lpt1 lpt2 lpt3 lpt4 lpt5 lpt6 lpt7 lpt8 lpt9
+    ].freeze
+
+    module_function
+
+    def event(name)
+      original = name.to_s
+      readable = original
+        .downcase
+        .gsub(/[^a-z0-9_-]+/, "-")
+        .gsub(/-+/, "-")
+        .gsub(/\A-|-+\z/, "")
+
+      readable = "event" if readable.empty?
+      readable = "event-#{readable}" if WINDOWS_RESERVED_NAMES.include?(readable)
+      readable = readable[0, 80].gsub(/-+\z/, "")
+
+      "#{readable}-#{Digest::SHA256.hexdigest(original)[0, 16]}"
+    end
+
+    def version(version)
+      version = Integer(version)
+      raise ArgumentError if version <= 0
+
+      "v#{version}"
+    rescue ArgumentError, TypeError, RangeError
+      raise ArgumentError, "report version must be a positive integer"
+    end
+  end
+end

data/lib/event_meter/processor.rb

@@ -0,0 +1,305 @@
+module EventMeter
+  class Processor
+    ROLLUP_BUCKETS = %i[minute hour].freeze
+
+    Result = Struct.new(
+      :event_name,
+      :version,
+      :processed,
+      :skipped_already_processed,
+      :malformed,
+      :complete,
+      :locked,
+      keyword_init: true
+    ) do
+      def self.empty(definition)
+        new(
+          event_name: definition.name,
+          version: definition.version,
+          processed: 0,
+          skipped_already_processed: 0,
+          malformed: 0,
+          complete: true,
+          locked: false
+        )
+      end
+
+      def self.locked(definition)
+        new(
+          event_name: definition.name,
+          version: definition.version,
+          processed: 0,
+          skipped_already_processed: 0,
+          malformed: 0,
+          complete: false,
+          locked: true
+        )
+      end
+
+      def self.processed(definition, processed:, skipped_already_processed:, malformed:, complete:)
+        new(
+          event_name: definition.name,
+          version: definition.version,
+          processed: processed,
+          skipped_already_processed: skipped_already_processed,
+          malformed: malformed,
+          complete: complete,
+          locked: false
+        )
+      end
+
+      def to_h
+        {
+          event_name: event_name,
+          version: version,
+          processed: processed,
+          skipped_already_processed: skipped_already_processed,
+          malformed: malformed,
+          complete: complete,
+          locked: locked
+        }
+      end
+    end
+
+    def initialize(configuration:, report_definition:, stream_storage:, rollup_storage:)
+      @configuration = configuration
+      @report_definition = report_definition
+      @stream_storage = stream_storage
+      @rollup_storage = rollup_storage_for(report_definition, rollup_storage)
+    end
+
+    def process
+      with_stream_lock do
+        process_with_rollup_lock
+      end
+    end
+
+    private
+
+    def with_stream_lock
+      return yield unless @stream_storage.respond_to?(:with_lock)
+
+      result = nil
+      lock_acquired = @stream_storage.with_lock(ttl: @configuration.lock_ttl) do
+        result = yield
+      end
+
+      return result if lock_acquired
+
+      Result.locked(@report_definition)
+    end
+
+    def process_with_rollup_lock
+      # Counts and durations are merge-safe and use processed-entry retry
+      # guards. Only interval metrics need the rollup-wide lock because they
+      # advance shared "previous start time" state.
+      return process_unlocked unless @report_definition.intervals.any?
+
+      result = nil
+      lock_acquired = @rollup_storage.with_lock(ttl: @configuration.lock_ttl) do
+        result = process_unlocked
+      end
+
+      return result if lock_acquired
+
+      Result.locked(@report_definition)
+    end
+
+    def rollup_storage_for(definition, storage)
+      return storage if scoped_for?(definition, storage)
+      return storage.for_report(name: definition.name, version: definition.version) if storage.respond_to?(:for_report)
+
+      storage
+    end
+
+    def scoped_for?(definition, storage)
+      return false unless storage.respond_to?(:report_name) && storage.respond_to?(:version)
+
+      storage.report_name.to_s == definition.name.to_s && storage.version.to_i == definition.version.to_i
+    end
+
+    def process_unlocked
+      @rollup_storage.ensure_definition(@report_definition)
+
+      entries = @stream_storage.read(name: @report_definition.name)
+      return Result.empty(@report_definition) if entries.empty?
+
+      entry_ids = entries.map(&:first)
+      process_entries(entries, entry_ids)
+    rescue StandardError
+      release_stream if entries
+      raise
+    end
+
+    def process_entries(entries, entry_ids)
+      pending_entries = unprocessed_entries(entries, entry_ids)
+      skipped_count = entries.length - pending_entries.length
+      batch = build_batch(pending_entries)
+
+      @rollup_storage.apply(batch) unless batch.empty?
+      stream_deleted = !!@stream_storage.delete
+      @rollup_storage.forget_processed_ids(entry_ids) if stream_deleted && @rollup_storage.respond_to?(:forget_processed_ids)
+
+      Result.processed(
+        @report_definition,
+        processed: pending_entries.length,
+        skipped_already_processed: skipped_count,
+        malformed: batch.malformed,
+        complete: stream_deleted
+      )
+    end
+
+    def release_stream
+      @stream_storage.release if @stream_storage.respond_to?(:release)
+    rescue StandardError
+      nil
+    end
+
+    def unprocessed_entries(entries, entry_ids)
+      processed_ids = @rollup_storage.processed_ids(entry_ids)
+      processed_lookup = processed_ids.to_h { |id| [id, true] }
+      seen = {}
+
+      entries.reject do |id, _raw_payload|
+        duplicate = seen.key?(id)
+        seen[id] = true
+
+        duplicate || processed_lookup.key?(id)
+      end
+    end
+
+    def build_batch(entries)
+      Batch.new.tap do |batch|
+        entries.each do |id, raw_payload|
+          payload = load_payload(raw_payload)
+          unless payload
+            batch.entry_ids << id
+            batch.malformed += 1
+            next
+          end
+
+          if payload.name == @report_definition.name
+            record_event(batch, payload)
+            record_intervals(batch, payload)
+          else
+            batch.malformed += 1
+          end
+
+          batch.entry_ids << id
+        end
+      end
+    end
+
+    def load_payload(raw_payload)
+      EventPayload.load(raw_payload)
+    rescue ArgumentError, KeyError, TypeError
+      nil
+    end
+
+    def record_event(batch, payload)
+      @report_definition.indexes_for(payload).each do |index|
+        add_event_rollup(batch, payload, index)
+      end
+    end
+
+    def add_event_rollup(batch, payload, index)
+      ROLLUP_BUCKETS.each do |every|
+        rollup = rollup_for(batch, payload, every, index)
+        rollup.increment("count")
+        rollup.increment("#{payload.status}_count")
+        rollup.add_started_at(payload.started_ms)
+        rollup.add_duration(payload.duration_ms)
+      end
+    end
+
+    def record_intervals(batch, payload)
+      state_updates = {}
+
+      @report_definition.intervals.each do |definition|
+        value = payload.params[definition.param.to_s]
+        next if value.nil?
+        next unless definition.group_by.all? { |param| ReportDefinition.indexable_value?(payload.params, param) }
+
+        state_key = interval_state_key(payload.name, definition, value)
+        previous_ms = previous_interval_ms(batch, state_key)
+        current_ms = payload.started_ms
+        next if previous_ms && current_ms <= previous_ms
+
+        state_updates[state_key] = current_ms
+        next unless previous_ms
+
+        index = definition.group_index.key_for(payload.params)
+        built_index = ReportDefinition::BuiltIndex.new(index: definition.group_index, key: index)
+        interval_ms = current_ms - previous_ms
+
+        ROLLUP_BUCKETS.each do |every|
+          rollup_for(batch, payload, every, built_index).add_interval(interval_ms)
+        end
+      end
+
+      state_updates.each do |state_key, current_ms|
+        batch.interval_state[state_key] = current_ms
+        batch.state_updates[state_key] = current_ms
+      end
+    end
+
+    def previous_interval_ms(batch, state_key)
+      batch.interval_state.fetch(state_key) do
+        stored_value = @rollup_storage.get(state_key)
+        batch.interval_state[state_key] = interval_timestamp(stored_value)
+      end
+    end
+
+    def interval_timestamp(value)
+      return nil if value.nil?
+
+      timestamp = Integer(value)
+      timestamp >= 0 ? timestamp : nil
+    rescue ArgumentError, TypeError, RangeError
+      nil
+    end
+
+    def rollup_for(batch, payload, every, index)
+      bucket = TimeBuckets.time(payload.started_at, every)
+      batch.rollups[rollup_key(payload.name, every, bucket, index)]
+    end
+
+    def rollup_key(name, every, bucket, index)
+      Keys.rollup(
+        namespace: @configuration.namespace,
+        name: name,
+        version: @report_definition.version,
+        every: every,
+        bucket: bucket,
+        index: index
+      )
+    end
+
+    def interval_state_key(name, definition, value)
+      Keys.interval_state(
+        namespace: @configuration.namespace,
+        name: name,
+        version: @report_definition.version,
+        definition: definition,
+        value: value
+      )
+    end
+
+    class Batch
+      attr_reader :rollups, :state_updates, :entry_ids, :interval_state
+      attr_accessor :malformed
+
+      def initialize
+        @rollups = Hash.new { |hash, key| hash[key] = Rollup.new }
+        @state_updates = {}
+        @entry_ids = []
+        @interval_state = {}
+        @malformed = 0
+      end
+
+      def empty?
+        entry_ids.empty?
+      end
+    end
+  end
+end

data/lib/event_meter/rails.rb

@@ -0,0 +1,79 @@
+require_relative "../event_meter"
+require_relative "stores/rollup/active_record_postgres"
+
+module EventMeter
+  module Rails
+    module_function
+
+    def configure(namespace:, stream_path:, stream_storage: :file, rollup_storage: :postgres,
+      table_prefix: "event_meter", connection_class: nil, stream_sync: :flush,
+      auto_cleanup_history: false, cleanup_history_retention: nil,
+      cleanup_history_interval: nil, summary_key_limit: nil, logger: nil)
+
+      connection_class ||= Stores::Rollup::ActiveRecordPostgres.default_connection_class
+
+      EventMeter.configure do |config|
+        config.namespace = namespace
+        config.stream_storage = build_stream_storage(
+          stream_storage,
+          path: stream_path,
+          sync: stream_sync
+        )
+        config.rollup_storage = build_rollup_storage(
+          rollup_storage,
+          connection_class: connection_class,
+          namespace: namespace,
+          table_prefix: table_prefix
+        )
+        config.auto_cleanup_history = auto_cleanup_history
+        config.cleanup_history_retention = cleanup_history_retention if cleanup_history_retention
+        config.cleanup_history_interval = cleanup_history_interval if cleanup_history_interval
+        config.summary_key_limit = summary_key_limit if summary_key_limit
+        config.auto_cleanup_error_handler = auto_cleanup_error_handler(logger) if logger
+      end
+    end
+
+    def migration_sql(table_prefix: "event_meter")
+      Stores::Rollup::Postgres.schema_sql(table_prefix: table_prefix)
+    end
+
+    def install_postgres!(connection_class: nil, table_prefix: "event_meter")
+      Stores::Rollup::ActiveRecordPostgres.install!(
+        connection_class: connection_class || Stores::Rollup::ActiveRecordPostgres.default_connection_class,
+        table_prefix: table_prefix
+      )
+    end
+
+    def auto_cleanup_error_handler(logger)
+      lambda do |error|
+        logger.warn "EventMeter auto cleanup failed: #{error.class}: #{error.message}"
+      end
+    end
+
+    def build_stream_storage(storage, path:, sync:)
+      validate_storage!(storage, expected: :file, name: "stream_storage")
+
+      Stores::Stream::File.new(path: path, sync: sync)
+    end
+
+    def build_rollup_storage(storage, connection_class:, namespace:, table_prefix:)
+      validate_storage!(storage, expected: :postgres, name: "rollup_storage")
+
+      Stores::Rollup::ActiveRecordPostgres.new(
+        connection_class: connection_class,
+        namespace: namespace,
+        table_prefix: table_prefix
+      )
+    end
+
+    def validate_storage!(storage, expected:, name:)
+      return if storage_key(storage) == expected
+
+      raise ArgumentError, "#{name} must be #{expected.inspect}"
+    end
+
+    def storage_key(storage)
+      storage.respond_to?(:to_sym) ? storage.to_sym : storage
+    end
+  end
+end