mqkv 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52ec1386119f7a8371f41f5942360c248f6b9e9c3a4efaf67a80fac35d1a2d27
-  data.tar.gz: 1863f574c1fa148399abb666248cec1e2f31b2e9b5c4da52b686e9c2ce84b7bf
+  metadata.gz: d737d046ae643ce3d316df07cc167f11f9c141b2d6f58f504557edcb3c4384c8
+  data.tar.gz: 574cf60004e1150b02959944c6fb38797ae14ecdca48351d7813a87441c465d4
 SHA512:
-  metadata.gz: 761d510fea8ee47f91116b7c11f61435f47095cd92d8805603065d5d19723d3027111ed1ee89c4c5785ce2fb9c70cb8766fe55508f5cf3b8d48f563ea5d0021d
-  data.tar.gz: 877451290430c1b5e268881c6193f239619a0f9c4c506b3f6697c70939404f4ef8fea80561e4dc634f48d81c0167f6ba85680ca40fba748ea06d38763f46996c
+  metadata.gz: 6880875f67dc0e324073475cba672c60c8e747ff6da11289b39ff41d5a2c20b4c772a39fa114ffef00ad9d0c5aadcbad438d385ae3d6f1fce37c408086122e9c
+  data.tar.gz: 168e701a4272b63237cc2a59755fb89fe4e28f94828a171632655a355dd9cc7581d931b7431bfefd79221bd403c08de54374534b3c4abc457a0d29e0c3f6a434

data/lib/mqkv/store.rb

@@ -1,46 +1,111 @@
 # frozen_string_literal: true
 
 require "amqp-client"
+require "logger"
 require "set"
 
 module MQKV
   class Store
     WatchHandle = Data.define(:channel, :consumer_tag)
 
-    def initialize(url, prefix: "mqkv", read_timeout: 0.5)
+    CacheEntry = Data.define(:value, :expires_at) do
+      def current_value
+        return nil if value.nil?
+        return nil if expires_at && Process.clock_gettime(Process::CLOCK_REALTIME) >= expires_at
+
+        value
+      end
+    end
+
+    TOMBSTONE_HEADER = "__mqkv_deleted__"
+    EXPIRES_HEADER = "__mqkv_expires_at__"
+
+    # Stream consumers must ack to advance the broker's flow-control
+    # window: with manual acks and a prefetch of N, delivery stalls
+    # after N outstanding messages. Acking every half-window keeps
+    # deliveries flowing while still batching (multiple: true).
+    CONSUME_PREFETCH = 256
+    CONSUME_ACK_BATCH = CONSUME_PREFETCH / 2
+
+    def initialize(url, prefix: "mqkv", read_timeout: 0.5, connect_timeout: nil, confirm: true, cache_watchers: true, logger: nil)
       @url = url
       @prefix = prefix
       @read_timeout = read_timeout
+      @connect_timeout = connect_timeout
+      @confirm = confirm
+      @cache_watchers_enabled = cache_watchers
+      @logger = logger
       @mutex = Mutex.new
       @connection = nil
       @declared_streams = Set.new
+      @cache = nil
+      @cache_mutex = Mutex.new
+      @cache_watchers = {}
     end
 
-    def set(key, value)
+    def set(key, value, ttl: nil)
       name = queue_name(key)
-      ensure_stream(name)
-      connection.with_channel do |ch|
-        ch.basic_publish_confirm(value.to_s, exchange: "", routing_key: name)
+      max_age = ttl ? "#{ttl.ceil}s" : nil
+      ensure_stream(name, max_age: max_age)
+      expires_at = ttl ? Process.clock_gettime(Process::CLOCK_REALTIME) + ttl : nil
+      headers = expires_at ? { EXPIRES_HEADER => expires_at } : nil
+      publish(name, value.to_s, headers: headers)
+      log(:debug) { "at=set key=#{key} queue=#{name}" }
+      if @cache
+        @cache_mutex.synchronize { @cache[key] = CacheEntry.new(value: value.to_s, expires_at: expires_at) }
+        log(:debug) { "at=set key=#{key} cache=updated" }
+        start_cache_watcher(key)
       end
       nil
     end
 
     def get(key)
-      messages = consume_stream(queue_name(key), offset: "last")
-      return nil if messages.empty?
+      if @cache
+        @cache_mutex.synchronize do
+          if @cache.key?(key)
+            log(:debug) { "at=get key=#{key} source=cache" }
+            return @cache[key].current_value
+          end
+        end
+      end
+      log(:debug) { "at=get key=#{key} source=stream" }
+      resolve_current(consume_stream(queue_name(key), offset: "last"))
+    end
 
-      last = messages.last
-      return nil if tombstone?(last)
+    # Cache-only read. Returns the cached value, or nil if the key has
+    # been tombstoned or isn't in the cache at all. Never falls through
+    # to the stream — intended for callers that want to decouple read
+    # latency from the broker (e.g. an HTTP handler that should show a
+    # placeholder when the cache isn't ready yet rather than block on
+    # AMQP). Use `cached?(key)` to distinguish "tombstoned" from
+    # "not cached".
+    def cached_get(key)
+      return nil unless @cache
 
-      last.body
+      @cache_mutex.synchronize do
+        return @cache[key].current_value if @cache.key?(key)
+      end
+      nil
+    end
+
+    # Whether the key is present in the in-memory cache. A tombstoned
+    # key still counts as cached (cached_get returns nil for it).
+    # Returns false if no preload has happened yet (no cache exists).
+    def cached?(key)
+      return false unless @cache
+
+      @cache_mutex.synchronize { @cache.key?(key) }
     end
 
     def delete(key)
       name = queue_name(key)
       ensure_stream(name)
-      connection.with_channel do |ch|
-        ch.basic_publish_confirm("", exchange: "", routing_key: name,
-                                 headers: { "__mqkv_deleted__" => true })
+      publish_tombstone(name)
+      log(:debug) { "at=delete key=#{key} queue=#{name}" }
+      if @cache
+        @cache_mutex.synchronize { @cache[key] = CacheEntry.new(value: nil, expires_at: nil) }
+        log(:debug) { "at=delete key=#{key} cache=tombstoned" }
+        start_cache_watcher(key)
       end
       nil
     end
@@ -50,16 +115,52 @@ module MQKV
     end
 
     def history(key, limit: 10)
-      messages = consume_stream(queue_name(key), offset: "first")
       values = []
-      messages.each do |msg|
+      consume_stream(queue_name(key), offset: "first") do |msg|
         if tombstone?(msg)
           values.clear
         else
           values << msg.body
+          # Only the last `limit` values can ever be returned, so drop
+          # older ones as we scan — memory stays O(limit) instead of
+          # holding every body in the stream at once.
+          values.shift if values.size > limit
+        end
+      end
+      values
+    end
+
+    def preload(*keys, max_messages: 10_000)
+      @cache_mutex.synchronize { @cache ||= {} }
+      keys.each do |key|
+        # Only the last message decides the cache entry; stream the
+        # scan so a long history never sits in memory all at once.
+        last = nil
+        count = consume_stream(queue_name(key), offset: "first", max_messages: max_messages) do |msg|
+          last = msg
         end
+        entry = last ? msg_to_cache_entry(last) : CacheEntry.new(value: nil, expires_at: nil)
+        @cache_mutex.synchronize { @cache[key] = entry }
+        log(:debug) { "at=preload key=#{key} messages=#{count} value=#{entry.current_value.nil? ? "nil" : "present"}" }
+        start_cache_watcher(key)
+      end
+    end
+
+    # Like `preload`, but reads from `x-stream-offset: last` so only the
+    # current value is consumed (plus anything appended during the brief
+    # `read_timeout` window). Use this when keys live in streams with long
+    # histories and callers only care about the latest value — `preload`'s
+    # `offset: "first"` would otherwise drain up to `max_messages` of
+    # accumulated history per key on every boot.
+    def preload_latest(*keys)
+      @cache_mutex.synchronize { @cache ||= {} }
+      keys.each do |key|
+        messages = consume_stream(queue_name(key), offset: "last")
+        entry = resolve_entry(messages)
+        @cache_mutex.synchronize { @cache[key] = entry }
+        log(:debug) { "at=preload_latest key=#{key} value=#{entry.current_value.nil? ? "nil" : "present"}" }
+        start_cache_watcher(key)
       end
-      values.last(limit)
     end
 
     def watch(key, &block)
@@ -82,6 +183,7 @@ module MQKV
     end
 
     def purge!
+      stop_cache_watchers
       conn = @mutex.synchronize { @connection }
       return unless conn && !conn.closed?
 
@@ -93,6 +195,8 @@ module MQKV
     end
 
     def close
+      log(:info) { "at=close" }
+      stop_cache_watchers
       @mutex.synchronize do
         @connection&.close
         @connection = nil
@@ -106,32 +210,67 @@ module MQKV
       @mutex.synchronize do
         return @connection if @connection && !@connection.closed?
 
-        @connection = AMQP::Client.new(@url).connect
+        log(:info) { "at=connect url=#{sanitized_url}" }
+        @connection = AMQP::Client.new(@url, **client_options).connect
         @declared_streams.clear
         @connection
       end
     end
 
+    def client_options
+      @connect_timeout ? { connect_timeout: @connect_timeout } : {}
+    end
+
     def queue_name(key)
       "#{@prefix}.#{key}"
     end
 
-    def ensure_stream(name)
+    def ensure_stream(name, max_age: nil)
       already_declared = @mutex.synchronize { @declared_streams.include?(name) }
       return if already_declared
 
+      args = { "x-queue-type" => "stream" }
+      args["x-max-age"] = max_age if max_age
+
       connection.with_channel do |ch|
-        ch.queue_declare(name, durable: true, arguments: { "x-queue-type" => "stream" })
+        ch.queue_declare(name, durable: true, arguments: args)
       end
       @mutex.synchronize { @declared_streams.add(name) }
+      log(:debug) { "at=ensure_stream queue=#{name} status=declared" }
     end
 
-    def consume_stream(name, offset:)
+    def publish(name, body, **properties)
+      properties.compact!
+      connection.with_channel do |ch|
+        if @confirm
+          ch.basic_publish_confirm(body, exchange: "", routing_key: name, **properties)
+        else
+          ch.basic_publish(body, exchange: "", routing_key: name, **properties)
+        end
+      end
+    end
+
+    def publish_tombstone(name)
+      publish(name, "", headers: { TOMBSTONE_HEADER => true })
+    end
+
+    # Drains the stream from `offset` until it goes quiet for
+    # `read_timeout`. With a block, each message is yielded as it
+    # arrives and the message count is returned — nothing is retained,
+    # so callers control their own memory. Without a block, all
+    # messages are collected and returned (only suitable for short
+    # reads like offset=last). Acks are issued in batches as the scan
+    # progresses; a single ack at the end would stall delivery once
+    # the prefetch window fills and silently truncate longer streams.
+    def consume_stream(name, offset:, max_messages: 0)
       ensure_stream(name)
       ch = connection.channel
       begin
-        ch.basic_qos(256)
-        collected = []
+        ch.basic_qos(CONSUME_PREFETCH)
+        collected = block_given? ? nil : []
+        count = 0
+        unacked = 0
+        last = nil
         q = ::Queue.new
 
         consume_ok = ch.basic_consume(name, no_ack: false,
@@ -144,19 +283,115 @@ module MQKV
           msg = q.pop(timeout: @read_timeout)
           break if msg.nil?
 
-          collected << msg
+          count += 1
+          last = msg
+          if collected
+            collected << msg
+          else
+            yield msg
+          end
+
+          unacked += 1
+          if unacked >= CONSUME_ACK_BATCH
+            ch.basic_ack(msg.delivery_tag, multiple: true)
+            unacked = 0
+          end
+          break if max_messages > 0 && count >= max_messages
         end
 
-        ch.basic_ack(collected.last.delivery_tag, multiple: true) if collected.any?
+        ch.basic_ack(last.delivery_tag, multiple: true) if last && unacked > 0
         ch.basic_cancel(consume_ok.consumer_tag)
-        collected
+        collected || count
       ensure
         ch.close
       end
     end
 
+    def resolve_entry(messages)
+      return CacheEntry.new(value: nil, expires_at: nil) if messages.empty?
+
+      last = messages.last
+      return CacheEntry.new(value: nil, expires_at: nil) if tombstone?(last)
+
+      CacheEntry.new(value: last.body, expires_at: msg_expires_at(last))
+    end
+
+    def resolve_current(messages)
+      resolve_entry(messages).current_value
+    end
+
     def tombstone?(msg)
-      msg.properties&.headers&.fetch("__mqkv_deleted__", false) == true
+      msg.properties&.headers&.fetch(TOMBSTONE_HEADER, false) == true
+    end
+
+    def msg_expires_at(msg)
+      msg.properties&.headers&.fetch(EXPIRES_HEADER, nil)
+    end
+
+    def msg_to_cache_entry(msg)
+      if tombstone?(msg)
+        CacheEntry.new(value: nil, expires_at: nil)
+      else
+        CacheEntry.new(value: msg.body, expires_at: msg_expires_at(msg))
+      end
+    end
+
+    def start_cache_watcher(key)
+      return unless @cache_watchers_enabled
+
+      @cache_mutex.synchronize { return if @cache_watchers.key?(key) }
+
+      name = queue_name(key)
+      ensure_stream(name)
+      ch = connection.channel
+      ch.basic_qos(256)
+      consume_ok = ch.basic_consume(name, no_ack: false,
+                                    arguments: { "x-stream-offset" => "next" },
+                                    worker_threads: 1) do |msg|
+        msg.ack
+        entry = msg_to_cache_entry(msg)
+        @cache_mutex.synchronize { @cache[key] = entry }
+        log(:debug) { "at=cache_watcher key=#{key} value=#{entry.current_value.nil? ? "nil" : "present"}" }
+      end
+      handle = WatchHandle.new(channel: ch, consumer_tag: consume_ok.consumer_tag)
+
+      duplicate = @cache_mutex.synchronize do
+        if @cache_watchers.key?(key)
+          true
+        else
+          @cache_watchers[key] = handle
+          false
+        end
+      end
+
+      if duplicate
+        unwatch(handle)
+      else
+        log(:debug) { "at=cache_watcher key=#{key} status=started" }
+      end
+    end
+
+    def stop_cache_watchers
+      watchers = @cache_mutex.synchronize do
+        result = @cache_watchers.values
+        @cache_watchers.clear
+        @cache = nil
+        result
+      end
+      log(:debug) { "at=stop_cache_watchers count=#{watchers.size}" } if watchers.any?
+      watchers.each do |handle|
+        unwatch(handle)
+      rescue StandardError
+        nil
+      end
+    end
+
+    def sanitized_url
+      @url.sub(%r{//[^@]+@}, "//")
+    end
+
+    def log(level, &block)
+      @logger&.send(level, &block)
     end
   end
 end

data/lib/mqkv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MQKV
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mqkv
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - mqkv contributors
@@ -23,6 +23,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Uses AMQP stream queues (RabbitMQ 3.9+ / LavinMQ) as a key-value store.
   Each key maps to a dedicated stream queue; the latest message is the current value.
 executables: []
@@ -50,7 +64,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.7
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Key-value store backed by AMQP stream queues
 test_files: []