event_meter 0.1.0

data/lib/event_meter/stores/rollup/redis.rb

@@ -0,0 +1,349 @@
+require "json"
+require "time"
+
+require_relative "../../index_key"
+require_relative "../../rollup"
+require_relative "../cleanup_helpers"
+require_relative "../namespace"
+require_relative "../redis_lock"
+
+module EventMeter
+  module Stores
+    module Rollup
+      class Redis
+        include CleanupHelpers
+        include Namespace
+        include RedisLock
+
+        DEFAULT_ROLLUP_TTL = 31 * 24 * 60 * 60
+
+        MIN_FIELD_SCRIPT = <<~LUA
+          local current = redis.call("hget", KEYS[1], ARGV[1])
+          local current_number = tonumber(current)
+          local value = tonumber(ARGV[2])
+
+          if value == nil then
+            return redis.error_reply("ERR event_meter rollup min value must be numeric")
+          end
+
+          if current == false or current_number == nil or current_number > value then
+            redis.call("hset", KEYS[1], ARGV[1], ARGV[2])
+          end
+
+          return 1
+        LUA
+        MAX_FIELD_SCRIPT = <<~LUA
+          local current = redis.call("hget", KEYS[1], ARGV[1])
+          local current_number = tonumber(current)
+          local value = tonumber(ARGV[2])
+
+          if value == nil then
+            return redis.error_reply("ERR event_meter rollup max value must be numeric")
+          end
+
+          if current == false or current_number == nil or current_number < value then
+            redis.call("hset", KEYS[1], ARGV[1], ARGV[2])
+          end
+
+          return 1
+        LUA
+        SET_MAX_SCRIPT = <<~LUA
+          local current = redis.call("get", KEYS[1])
+          local current_number = tonumber(current)
+          local value = tonumber(ARGV[1])
+
+          if value == nil then
+            return redis.error_reply("ERR event_meter state value must be numeric")
+          end
+
+          if current == false or current_number == nil or current_number < value then
+            redis.call("set", KEYS[1], ARGV[1])
+          end
+
+          return 1
+        LUA
+
+        attr_reader :redis, :lock_redis, :namespace, :report_name, :version, :lock_scope, :rollup_ttl
+
+        def initialize(redis:, namespace:, lock_redis: nil, report_name: nil, version: nil, lock_scope: nil, rollup_ttl: DEFAULT_ROLLUP_TTL)
+          @redis = redis
+          @lock_redis = lock_redis || redis
+          @namespace = normalize_namespace(namespace)
+          @report_name = report_name&.to_s
+          @version = version&.to_i
+          @lock_scope = lock_scope
+          @rollup_ttl = positive_integer(rollup_ttl, "rollup_ttl")
+        end
+
+        def for_report(name:, version:)
+          name = name.to_s
+          version = version.to_i
+
+          self.class.new(
+            redis: redis,
+            lock_redis: lock_redis,
+            namespace: namespace,
+            report_name: name,
+            version: version,
+            lock_scope: "#{Keys.event_name(name)}:#{Keys.version_key(version)}",
+            rollup_ttl: rollup_ttl
+          )
+        end
+
+        def ensure_definition(definition)
+          key = definition_key(definition.name, definition.version)
+          payload = JSON.generate(definition.to_h)
+          stored = redis.get(key)
+
+          if stored
+            ensure_same_definition!(stored, definition)
+          else
+            redis.set(key, payload, nx: true)
+            ensure_same_definition!(redis.get(key), definition)
+          end
+        end
+
+        def report_definition(name:, version:)
+          payload = redis.get(definition_key(name, version))
+          payload && JSON.parse(payload)
+        rescue JSON::ParserError, TypeError
+          nil
+        end
+
+        def processed_ids(ids)
+          ensure_scoped!
+          return [] if ids.empty?
+
+          values = redis.pipelined do |pipe|
+            ids.each { |id| pipe.get(processed_key(id)) }
+          end
+
+          ids.zip(values).filter_map { |id, value| id if value }
+        end
+
+        def forget_processed_ids(ids)
+          ensure_scoped!
+          keys = ids.map { |id| processed_key(id) }
+          delete_keys(keys)
+        end
+
+        def apply(batch)
+          ensure_scoped!
+
+          redis.multi do |transaction|
+            batch.rollups.each { |key, rollup| apply_rollup(transaction, key, rollup) }
+
+            batch.state_updates.each do |key, value|
+              transaction.eval(SET_MAX_SCRIPT, keys: [key], argv: [value])
+              transaction.expire(key, rollup_ttl)
+            end
+
+            processed_at = Time.now.utc.iso8601(6)
+            batch.entry_ids.each do |id|
+              key = processed_key(id)
+              transaction.set(key, processed_at)
+              transaction.expire(key, rollup_ttl)
+            end
+          end
+        end
+
+        def hgetall_many(keys)
+          return [] if keys.empty?
+
+          redis.pipelined do |pipe|
+            keys.each { |key| pipe.hgetall(key) }
+          end
+        end
+
+        def keys_matching(pattern, limit: nil)
+          limit = positive_integer(limit, "limit") if limit
+          keys = []
+
+          redis.scan_each(match: namespace_glob(pattern)) do |key|
+            next unless key_matches?(key, pattern)
+
+            keys << key
+            break if limit && keys.length >= limit
+          end
+
+          keys.sort
+        end
+
+        def get(key)
+          redis.get(key)
+        end
+
+        def cleanup_watermark(key)
+          redis.get(key)
+        end
+
+        def write_cleanup_watermark(key, value)
+          redis.set(key, value)
+        end
+
+        def with_lock(ttl:)
+          with_redis_lock(lock_key, ttl: ttl) { yield }
+        end
+
+        def cleanup_history(before:, events:, interval_state:)
+          filter = event_filter(events)
+
+          {
+            rollup_keys_deleted: cleanup_rollups(before, filter),
+            interval_state_keys_deleted: interval_state ? cleanup_interval_state(before, filter) : 0,
+            processed_entries_deleted: cleanup_processed_entries(before, filter)
+          }
+        end
+
+        private
+
+        def apply_rollup(transaction, key, rollup)
+          rollup.fields.each do |field, value|
+            if EventMeter::Rollup.min_field?(field)
+              transaction.eval(MIN_FIELD_SCRIPT, keys: [key], argv: [field, value])
+            elsif EventMeter::Rollup.max_field?(field)
+              transaction.eval(MAX_FIELD_SCRIPT, keys: [key], argv: [field, value])
+            else
+              transaction.hincrby(key, field, value)
+            end
+          end
+
+          transaction.expire(key, rollup_ttl)
+        end
+
+        def positive_integer(value, name)
+          integer = Integer(value)
+          return integer if integer.positive?
+
+          raise ArgumentError, "#{name} must be positive"
+        rescue ArgumentError, TypeError, RangeError
+          raise ArgumentError, "#{name} must be positive"
+        end
+
+        def lock_key
+          [namespace, "process_lock", lock_scope].compact.join(":")
+        end
+
+        def processed_key(id)
+          Keys.processed(
+            namespace: namespace,
+            name: report_name,
+            version: version,
+            id: id
+          )
+        end
+
+        def ensure_scoped!
+          return if report_name && version&.positive?
+
+          raise ConfigurationError, "redis rollup storage must be scoped with for_report"
+        end
+
+        def definition_key(name, version)
+          Keys.definition(namespace: namespace, name: name, version: version)
+        end
+
+        def ensure_same_definition!(stored, definition)
+          stored_definition = ReportDefinition.from_h(JSON.parse(stored))
+          return if stored_definition.fingerprint == definition.fingerprint
+
+          raise DefinitionChangedError, "#{definition.name} v#{definition.version} changed; bump version"
+        rescue JSON::ParserError, TypeError
+          raise DefinitionChangedError, "#{definition.name} v#{definition.version} stored definition is invalid"
+        end
+
+        def cleanup_rollups(before, event_filter)
+          keys = scan_keys(namespace_glob("#{namespace}:rollup:*")).select do |key|
+            rollup_key_old?(key, before, event_filter)
+          end
+          delete_keys(keys)
+          keys.length
+        end
+
+        def cleanup_interval_state(before, event_filter)
+          before_ms = (before.to_f * 1000).to_i
+          keys = scan_keys(namespace_glob("#{namespace}:state:*")).select do |key|
+            state_key_matches_event?(key, event_filter)
+          end
+          return 0 if keys.empty?
+
+          values = redis.pipelined { |pipe| keys.each { |key| pipe.get(key) } }
+          expired = keys.zip(values).filter_map do |key, value|
+            key if state_key_old?(key, before_ms, event_filter, value)
+          end
+
+          delete_keys(expired)
+          expired.length
+        end
+
+        def cleanup_processed_entries(before, event_filter)
+          keys = scan_keys(namespace_glob("#{namespace}:processed:*")).select do |key|
+            processed_key_matches_event?(key, event_filter)
+          end
+          return 0 if keys.empty?
+
+          values = redis.pipelined { |pipe| keys.each { |key| pipe.get(key) } }
+          expired = keys.zip(values).filter_map do |key, value|
+            key if processed_entry_old?(value, before)
+          end
+
+          delete_keys(expired)
+          expired.length
+        end
+
+        def processed_key_matches_event?(key, event_filter)
+          prefix = "#{namespace}:processed:"
+          return false unless key.start_with?(prefix)
+
+          event_name = key.delete_prefix(prefix).split(":", 2).first
+          event_name && (!event_filter || event_filter.include?(event_name))
+        end
+
+        def state_key_matches_event?(key, event_filter)
+          prefix = "#{namespace}:state:"
+          return false unless key.start_with?(prefix)
+
+          event_name = key.delete_prefix(prefix).split(":", 3).first
+          event_name && (!event_filter || event_filter.include?(event_name))
+        end
+
+        def processed_entry_old?(timestamp, before)
+          Time.parse(timestamp).utc < before
+        rescue ArgumentError, TypeError, RangeError
+          true
+        end
+
+        def scan_keys(pattern)
+          keys = []
+          redis.scan_each(match: pattern) { |key| keys << key }
+          keys
+        end
+
+        def delete_keys(keys)
+          keys.each_slice(500) do |slice|
+            redis.del(*slice)
+          end
+        end
+
+        def namespace_glob(pattern)
+          prefix = "#{namespace}:"
+          return pattern unless pattern.start_with?(prefix)
+
+          "#{glob_escape(namespace)}:#{pattern.delete_prefix(prefix)}"
+        end
+
+        def glob_escape(value)
+          value.to_s.gsub(/[\\*\?\[\]]/) { |character| "\\#{character}" }
+        end
+
+        def key_matches?(key, pattern)
+          prefix = "#{namespace}:"
+          return ::File.fnmatch?(pattern, key) unless pattern.start_with?(prefix)
+          return false unless key.start_with?(prefix)
+
+          ::File.fnmatch?(pattern.delete_prefix(prefix), key.delete_prefix(prefix))
+        end
+      end
+    end
+  end
+end

data/lib/event_meter/stores/stream/file.rb

@@ -0,0 +1,98 @@
+require "time_bucket_stream"
+
+require_relative "../file_helpers"
+
+module EventMeter
+  module Stores
+    module Stream
+      class File
+        include FileHelpers
+
+        SYNC_MODES = %i[none flush fsync].freeze
+
+        attr_reader :path, :stream_options, :sync
+
+        def initialize(path:, sync: :flush, **stream_options)
+          @path = normalize_file_store_path(path)
+          @sync = sync.respond_to?(:to_sym) ? sync.to_sym : sync
+          @stream_options = stream_options
+          validate_sync!
+          @streams = {}
+          @read_ids = []
+        end
+
+        def append(payload)
+          hash = payload.to_h
+          stream_for(hash.fetch("name")).append(hash)
+        end
+
+        def read(name:)
+          release
+          @read_name = name.to_s
+          @batch = stream_for(name).read
+          @read_ids = @batch.entries.map(&:first)
+          @batch.entries
+        end
+
+        def delete
+          return false unless @batch
+
+          @batch.delete
+          deleted_claims?
+        ensure
+          @read_ids = []
+          @read_name = nil
+          @batch = nil
+        end
+
+        def release
+          @batch&.release
+        ensure
+          @read_ids = []
+          @read_name = nil
+          @batch = nil
+        end
+
+        def close
+          release
+          @streams.each_value(&:close)
+        end
+
+        private
+
+        def validate_sync!
+          return if SYNC_MODES.include?(sync)
+
+          raise ArgumentError, "unsupported file sync mode: #{sync.inspect}"
+        end
+
+        def stream_for(name)
+          @streams[name.to_s] ||= TimeBucketStream.new(
+            path: ::File.join(path, "streams", PathName.event(name)),
+            sync: sync,
+            **stream_options
+          )
+        end
+
+        def deleted_claims?
+          log_names = @read_ids.filter_map { |id| stream_file_name(id) }.uniq
+          return true if log_names.empty?
+
+          log_names.none? do |log_name|
+            ::File.exist?(::File.join(stream_path_for(@read_name), "processing", log_name))
+          end
+        end
+
+        def stream_file_name(entry_id)
+          value = entry_id.to_s
+          separator = value.index(":")
+          value[0...separator] if separator&.positive?
+        end
+
+        def stream_path_for(name)
+          ::File.join(path, "streams", PathName.event(name))
+        end
+      end
+    end
+  end
+end

data/lib/event_meter/stores/stream/redis.rb

@@ -0,0 +1,79 @@
+require "json"
+
+require_relative "../namespace"
+require_relative "../redis_lock"
+
+module EventMeter
+  module Stores
+    module Stream
+      class Redis
+        include Namespace
+        include RedisLock
+
+        attr_reader :redis, :lock_redis, :namespace, :redis_read_limit
+
+        def initialize(redis:, namespace:, lock_redis: nil, redis_read_limit: nil)
+          @redis = redis
+          @lock_redis = lock_redis || redis
+          @namespace = normalize_namespace(namespace)
+          @redis_read_limit = normalize_redis_read_limit(redis_read_limit)
+          @read_ids = []
+        end
+
+        def append(payload)
+          hash = payload.to_h
+          redis.xadd(stream_key(hash.fetch("name")), { "payload" => hash.to_json })
+        end
+
+        def read(name:)
+          range_options = redis_read_limit ? { count: redis_read_limit } : {}
+          @read_key = stream_key(name)
+          rows = redis.xrange(@read_key, "-", "+", **range_options)
+          @read_ids = rows.map(&:first)
+
+          rows.map do |id, fields|
+            [id, parse_payload(fields)]
+          end
+        end
+
+        def delete
+          redis.xdel(@read_key, *@read_ids) if @read_key && !@read_ids.empty?
+        ensure
+          @read_ids = []
+          @read_key = nil
+        end
+
+        def with_lock(ttl:)
+          with_redis_lock(lock_key, ttl: ttl) { yield }
+        end
+
+        private
+
+        def stream_key(name)
+          [namespace, "stream", Keys.event_name(name)].join(":")
+        end
+
+        def lock_key
+          [namespace, "stream_lock"].join(":")
+        end
+
+        def parse_payload(fields)
+          JSON.parse(fields.fetch("payload"))
+        rescue JSON::ParserError, KeyError, TypeError
+          nil
+        end
+
+        def normalize_redis_read_limit(limit)
+          return nil if limit.nil?
+
+          limit = Integer(limit)
+          return limit if limit.positive?
+
+          raise ArgumentError, "redis_read_limit must be positive"
+        rescue ArgumentError, TypeError, RangeError
+          raise ArgumentError, "redis_read_limit must be positive"
+        end
+      end
+    end
+  end
+end

data/lib/event_meter/time_buckets.rb

@@ -0,0 +1,56 @@
+module EventMeter
+  module TimeBuckets
+    SIZES = {
+      minute: 60,
+      hour: 3600
+    }.freeze
+
+    module_function
+
+    def id(time, size)
+      time = time.utc
+
+      case normalize(size)
+      when :minute
+        time.strftime("%Y%m%d%H%M")
+      when :hour
+        time.strftime("%Y%m%d%H")
+      end
+    end
+
+    def time(value, size)
+      value = value.utc
+
+      case normalize(size)
+      when :minute
+        Time.utc(value.year, value.month, value.day, value.hour, value.min)
+      when :hour
+        Time.utc(value.year, value.month, value.day, value.hour)
+      end
+    end
+
+    def seconds(size)
+      SIZES.fetch(normalize(size))
+    end
+
+    def between(from, to, size)
+      step = seconds(size)
+      current = time(from, size)
+      buckets = []
+
+      while current < to
+        buckets << current
+        current += step
+      end
+
+      buckets
+    end
+
+    def normalize(size)
+      size = size.to_sym if size.respond_to?(:to_sym)
+      return size if SIZES.key?(size)
+
+      raise ArgumentError, "unsupported bucket size: #{size.inspect}"
+    end
+  end
+end

data/lib/event_meter/version.rb

@@ -0,0 +1,3 @@
+module EventMeter
+  VERSION = "0.1.0"
+end

data/lib/event_meter/write_result.rb

@@ -0,0 +1,26 @@
+module EventMeter
+  class WriteResult
+    attr_reader :payload, :error
+
+    def self.recorded(payload)
+      new(payload: payload, error: nil)
+    end
+
+    def self.failed(payload:, error:)
+      new(payload: payload, error: error)
+    end
+
+    def initialize(payload:, error:)
+      @payload = payload
+      @error = error
+    end
+
+    def recorded?
+      error.nil?
+    end
+
+    def error?
+      !recorded?
+    end
+  end
+end