mqkv 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52ec1386119f7a8371f41f5942360c248f6b9e9c3a4efaf67a80fac35d1a2d27
-  data.tar.gz: 1863f574c1fa148399abb666248cec1e2f31b2e9b5c4da52b686e9c2ce84b7bf
+  metadata.gz: c4110775ffdb60a5ede4fadbdb553beec726b337d7a06c7feb97b4c367cf0b56
+  data.tar.gz: e4fd4b75b0d80fe6c70af944ed94507c40e92c610bd55f68259761680de7b61b
 SHA512:
-  metadata.gz: 761d510fea8ee47f91116b7c11f61435f47095cd92d8805603065d5d19723d3027111ed1ee89c4c5785ce2fb9c70cb8766fe55508f5cf3b8d48f563ea5d0021d
-  data.tar.gz: 877451290430c1b5e268881c6193f239619a0f9c4c506b3f6697c70939404f4ef8fea80561e4dc634f48d81c0167f6ba85680ca40fba748ea06d38763f46996c
+  metadata.gz: eed925c0d43599e3d8dc283ca70befc5bb79e10b72f9a5727eaa33dba93e3509e50304d6aa6cb428fe7cb6e40dff13dc97b75c01ab682b6e0c4ce4e6ab60f998
+  data.tar.gz: 5a15102dbaad4203e0b8db1fa8bf22cd9f240ee88c20edf963e6a85f2ef89e6422b7a5d7d1cf25ae529edeb2dd3e6c05e092ba8bbf306ffdcc34c09f0df8b102

data/lib/mqkv/store.rb

@@ -1,46 +1,77 @@
 # frozen_string_literal: true
 
 require "amqp-client"
+require "logger"
 require "set"
 
 module MQKV
   class Store
     WatchHandle = Data.define(:channel, :consumer_tag)
 
-    def initialize(url, prefix: "mqkv", read_timeout: 0.5)
+    CacheEntry = Data.define(:value, :expires_at) do
+      def current_value
+        return nil if value.nil?
+        return nil if expires_at && Process.clock_gettime(Process::CLOCK_REALTIME) >= expires_at
+
+        value
+      end
+    end
+
+    TOMBSTONE_HEADER = "__mqkv_deleted__"
+    EXPIRES_HEADER = "__mqkv_expires_at__"
+
+    def initialize(url, prefix: "mqkv", read_timeout: 0.5, confirm: true, logger: nil)
       @url = url
       @prefix = prefix
       @read_timeout = read_timeout
+      @confirm = confirm
+      @logger = logger
       @mutex = Mutex.new
       @connection = nil
       @declared_streams = Set.new
+      @cache = nil
+      @cache_mutex = Mutex.new
+      @cache_watchers = {}
     end
 
-    def set(key, value)
+    def set(key, value, ttl: nil)
       name = queue_name(key)
-      ensure_stream(name)
-      connection.with_channel do |ch|
-        ch.basic_publish_confirm(value.to_s, exchange: "", routing_key: name)
+      max_age = ttl ? "#{ttl.ceil}s" : nil
+      ensure_stream(name, max_age: max_age)
+      expires_at = ttl ? Process.clock_gettime(Process::CLOCK_REALTIME) + ttl : nil
+      headers = expires_at ? { EXPIRES_HEADER => expires_at } : nil
+      publish(name, value.to_s, headers: headers)
+      log(:debug) { "at=set key=#{key} queue=#{name}" }
+      if @cache
+        @cache_mutex.synchronize { @cache[key] = CacheEntry.new(value: value.to_s, expires_at: expires_at) }
+        log(:debug) { "at=set key=#{key} cache=updated" }
+        start_cache_watcher(key)
       end
       nil
     end
 
     def get(key)
-      messages = consume_stream(queue_name(key), offset: "last")
-      return nil if messages.empty?
-
-      last = messages.last
-      return nil if tombstone?(last)
-
-      last.body
+      if @cache
+        @cache_mutex.synchronize do
+          if @cache.key?(key)
+            log(:debug) { "at=get key=#{key} source=cache" }
+            return @cache[key].current_value
+          end
+        end
+      end
+      log(:debug) { "at=get key=#{key} source=stream" }
+      resolve_current(consume_stream(queue_name(key), offset: "last"))
     end
 
     def delete(key)
       name = queue_name(key)
       ensure_stream(name)
-      connection.with_channel do |ch|
-        ch.basic_publish_confirm("", exchange: "", routing_key: name,
-                                 headers: { "__mqkv_deleted__" => true })
+      publish_tombstone(name)
+      log(:debug) { "at=delete key=#{key} queue=#{name}" }
+      if @cache
+        @cache_mutex.synchronize { @cache[key] = CacheEntry.new(value: nil, expires_at: nil) }
+        log(:debug) { "at=delete key=#{key} cache=tombstoned" }
+        start_cache_watcher(key)
       end
       nil
     end
@@ -62,6 +93,17 @@ module MQKV
       values.last(limit)
     end
 
+    def preload(*keys, max_messages: 10_000)
+      @cache_mutex.synchronize { @cache ||= {} }
+      keys.each do |key|
+        messages = consume_stream(queue_name(key), offset: "first", max_messages: max_messages)
+        entry = resolve_entry(messages)
+        @cache_mutex.synchronize { @cache[key] = entry }
+        log(:debug) { "at=preload key=#{key} messages=#{messages.size} value=#{entry.current_value.nil? ? "nil" : "present"}" }
+        start_cache_watcher(key)
+      end
+    end
+
     def watch(key, &block)
       name = queue_name(key)
       ensure_stream(name)
@@ -82,6 +124,7 @@ module MQKV
     end
 
     def purge!
+      stop_cache_watchers
       conn = @mutex.synchronize { @connection }
       return unless conn && !conn.closed?
 
@@ -93,6 +136,8 @@ module MQKV
     end
 
     def close
+      log(:info) { "at=close" }
+      stop_cache_watchers
       @mutex.synchronize do
         @connection&.close
         @connection = nil
@@ -106,6 +151,7 @@ module MQKV
       @mutex.synchronize do
         return @connection if @connection && !@connection.closed?
 
+        log(:info) { "at=connect url=#{sanitized_url}" }
         @connection = AMQP::Client.new(@url).connect
         @declared_streams.clear
         @connection
@@ -116,17 +162,36 @@ module MQKV
       "#{@prefix}.#{key}"
     end
 
-    def ensure_stream(name)
+    def ensure_stream(name, max_age: nil)
       already_declared = @mutex.synchronize { @declared_streams.include?(name) }
       return if already_declared
 
+      args = { "x-queue-type" => "stream" }
+      args["x-max-age"] = max_age if max_age
+
       connection.with_channel do |ch|
-        ch.queue_declare(name, durable: true, arguments: { "x-queue-type" => "stream" })
+        ch.queue_declare(name, durable: true, arguments: args)
       end
       @mutex.synchronize { @declared_streams.add(name) }
+      log(:debug) { "at=ensure_stream queue=#{name} status=declared" }
+    end
+
+    def publish(name, body, **properties)
+      properties.compact!
+      connection.with_channel do |ch|
+        if @confirm
+          ch.basic_publish_confirm(body, exchange: "", routing_key: name, **properties)
+        else
+          ch.basic_publish(body, exchange: "", routing_key: name, **properties)
+        end
+      end
+    end
+
+    def publish_tombstone(name)
+      publish(name, "", headers: { TOMBSTONE_HEADER => true })
     end
 
-    def consume_stream(name, offset:)
+    def consume_stream(name, offset:, max_messages: 0)
       ensure_stream(name)
       ch = connection.channel
       begin
@@ -145,6 +210,7 @@ module MQKV
           break if msg.nil?
 
           collected << msg
+          break if max_messages > 0 && collected.size >= max_messages
         end
 
         ch.basic_ack(collected.last.delivery_tag, multiple: true) if collected.any?
@@ -155,8 +221,89 @@ module MQKV
       end
     end
 
+    def resolve_entry(messages)
+      return CacheEntry.new(value: nil, expires_at: nil) if messages.empty?
+
+      last = messages.last
+      return CacheEntry.new(value: nil, expires_at: nil) if tombstone?(last)
+
+      CacheEntry.new(value: last.body, expires_at: msg_expires_at(last))
+    end
+
+    def resolve_current(messages)
+      resolve_entry(messages).current_value
+    end
+
     def tombstone?(msg)
-      msg.properties&.headers&.fetch("__mqkv_deleted__", false) == true
+      msg.properties&.headers&.fetch(TOMBSTONE_HEADER, false) == true
+    end
+
+    def msg_expires_at(msg)
+      msg.properties&.headers&.fetch(EXPIRES_HEADER, nil)
+    end
+
+    def msg_to_cache_entry(msg)
+      if tombstone?(msg)
+        CacheEntry.new(value: nil, expires_at: nil)
+      else
+        CacheEntry.new(value: msg.body, expires_at: msg_expires_at(msg))
+      end
+    end
+
+    def start_cache_watcher(key)
+      @cache_mutex.synchronize { return if @cache_watchers.key?(key) }
+
+      name = queue_name(key)
+      ensure_stream(name)
+      ch = connection.channel
+      ch.basic_qos(256)
+      consume_ok = ch.basic_consume(name, no_ack: false,
+                                    arguments: { "x-stream-offset" => "next" },
+                                    worker_threads: 1) do |msg|
+        msg.ack
+        entry = msg_to_cache_entry(msg)
+        @cache_mutex.synchronize { @cache[key] = entry }
+        log(:debug) { "at=cache_watcher key=#{key} value=#{entry.current_value.nil? ? "nil" : "present"}" }
+      end
+      handle = WatchHandle.new(channel: ch, consumer_tag: consume_ok.consumer_tag)
+
+      duplicate = @cache_mutex.synchronize do
+        if @cache_watchers.key?(key)
+          true
+        else
+          @cache_watchers[key] = handle
+          false
+        end
+      end
+
+      if duplicate
+        unwatch(handle)
+      else
+        log(:debug) { "at=cache_watcher key=#{key} status=started" }
+      end
+    end
+
+    def stop_cache_watchers
+      watchers = @cache_mutex.synchronize do
+        result = @cache_watchers.values
+        @cache_watchers.clear
+        @cache = nil
+        result
+      end
+      log(:debug) { "at=stop_cache_watchers count=#{watchers.size}" } if watchers.any?
+      watchers.each do |handle|
+        unwatch(handle)
+      rescue StandardError
+        nil
+      end
+    end
+
+    def sanitized_url
+      @url.sub(%r{//[^@]+@}, "//")
+    end
+
+    def log(level, &block)
+      @logger&.send(level, &block)
     end
   end
 end

data/lib/mqkv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MQKV
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mqkv
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - mqkv contributors
@@ -23,6 +23,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Uses AMQP stream queues (RabbitMQ 3.9+ / LavinMQ) as a key-value store.
   Each key maps to a dedicated stream queue; the latest message is the current value.
 executables: []
@@ -50,7 +64,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.7
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Key-value store backed by AMQP stream queues
 test_files: []