prosody 0.5.0-aarch64-linux → 0.6.0-aarch64-linux

data/examples/keyed_state.rb

@@ -3,7 +3,7 @@
 require "prosody"
 require "logger"
 
-# Keyed-state example: per-key value/map/deque collections that survive across
+# Keyed-state example: per-key value/map/set/deque collections that survive across
 # events. Definitions are declared once and reused for both registration (on the
 # client) and binding (inside the handler). Every state op yields the fiber,
 # never the thread.
@@ -12,6 +12,7 @@ require "logger"
 # keyed_state.rbs for payload and state types checked by Steep.
 CART = Prosody.value("cart", ttl: 30 * 24 * 3600)      # ValueState
 TOTALS = Prosody.map("totals")                          # keys are always String
+SEEN = Prosody.set("seen", ttl: 7 * 24 * 3600)          # String members, no payload
 BACKLOG = Prosody.message_deque("backlog", capacity: 100) # bounded window of messages
 
 class KeyedStateHandler < Prosody::EventHandler
@@ -19,6 +20,7 @@ class KeyedStateHandler < Prosody::EventHandler
     puts "Excise #{message.key}"
     context.state(CART).clear
     context.state(TOTALS).clear
+    context.state(SEEN).clear
     context.state(BACKLOG).clear
     nil
   end
@@ -29,6 +31,10 @@ class KeyedStateHandler < Prosody::EventHandler
 
   def on_message(context, message)
     payload = message.payload
+    seen = context.state(SEEN)
+    return if seen.include?(payload["order_id"]) # skip an order seen before
+    seen << payload["order_id"]
+
     cart = context.state(CART)             # bound for this attempt only
     current = cart.get || {"items" => []}  # Hash, or nil when absent
     cart.set(current.merge("items" => current["items"] + [payload["order_id"]]))
@@ -37,6 +43,8 @@ class KeyedStateHandler < Prosody::EventHandler
     totals.set(message.key, payload["total"])
     # Steep infers key as String and total as Integer from TOTALS's RBS type.
     totals.each_pair { |key, total| @logger.info(format_total(key, total)) }
+    # Query keywords narrow a traversal: here, the first ten keys after this one.
+    totals.each_key(after: message.key, limit: 10) { |key| @logger.info("next key: #{key}") }
 
     backlog = context.state(BACKLOG)
     backlog.push(message)                  # stores the full Prosody::Message
@@ -63,7 +71,7 @@ if __FILE__ == $PROGRAM_NAME
     mock: true,
     group_id: "keyed-state-example",
     subscribed_topics: "orders",
-    state_collections: [CART, TOTALS, BACKLOG]
+    state_collections: [CART, TOTALS, SEEN, BACKLOG]
   )
   client.subscribe(KeyedStateHandler.new(logger: Logger.new($stdout)))
   client.shutdown

data/examples/keyed_state.rbs

@@ -3,6 +3,7 @@ type keyed_state_order_event = { "order_id" => String, "total" => Integer }
 
 CART: Prosody::_ValueDefinition[keyed_state_cart]
 TOTALS: Prosody::_MapDefinition[Integer]
+SEEN: Prosody::_SetDefinition
 BACKLOG: Prosody::_MessageDequeDefinition[keyed_state_order_event]
 
 class KeyedStateHandler < Prosody::EventHandler[keyed_state_order_event]

data/lib/prosody/3.2/prosody.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/prosody/3.2/prosody.so
-size: 35858144 bytes
-sha256: 82bf4030a8a814044822b2ed39fc8ceeb38f73ec8084593f46fe2a235c240132
+size: 37148304 bytes
+sha256: 59593e6c5f8d787d1ef88b0da37e4ea9e89362f2dee9a71f68c2b331a066e5d7

data/lib/prosody/3.3/prosody.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/prosody/3.3/prosody.so
-size: 35841368 bytes
-sha256: c6a206ba1cab3f02dab0057d38a73393d06e7c723edc38ddbb2914ddc3e9a289
+size: 37160880 bytes
+sha256: b80346fb441d4f421b17cfa6cb1d12fdf0aa5f9cd34a81ae86537dd8ad0e1b92

data/lib/prosody/3.4/prosody.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/prosody/3.4/prosody.so
-size: 35849680 bytes
-sha256: 2baed460b6cb7fc66d07ae87ef568d6517a5232aa48f74d289c48a1b63604229
+size: 37160744 bytes
+sha256: a37eb1bfbf9826d6bc07d86a05211a7c7580f8bba9bb19413883badc0c11482c

data/lib/prosody/client.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Prosody
+  # Ruby-side lifecycle helpers for the native {Prosody::Client}.
+  class Client
+    # Creates a client, yields it, and shuts it down when the block exits.
+    # The client shuts down also when the block raises. Repeated shutdown
+    # calls wait for the same operation, so the block can also call
+    # {#shutdown}.
+    #
+    # @param config [Hash, Configuration] Client configuration
+    # @yieldparam client [Client] the new client
+    # @return [Object] the value of the block
+    # @raise [ArgumentError] if no block is given or the configuration is invalid
+    # @raise [RuntimeError] if client initialization or shutdown fails
+    #
+    # @example
+    #   Prosody::Client.open(bootstrap_servers: "localhost:9092") do |client|
+    #     client.send_message("my-topic", "key", {"hello" => "world"})
+    #   end
+    def self.open(config)
+      raise ArgumentError, "Prosody::Client.open requires a block" unless block_given?
+
+      client = new(config)
+      begin
+        yield client
+      ensure
+        client.shutdown
+      end
+    end
+  end
+end

data/lib/prosody/configuration.rb

@@ -124,14 +124,18 @@ module Prosody
     # Interval between offset commit operations (in seconds).
     config_param :commit_interval, converter: ->(v) { duration_converter(v) }
 
+    # Interval between librdkafka statistics reports (in seconds).
+    # Env: PROSODY_STATISTICS_INTERVAL. Default: 5 seconds.
+    config_param :statistics_interval, converter: ->(v) { duration_converter(v) }
+
     # Base delay for retry operations (in seconds).
     config_param :retry_base, converter: ->(v) { duration_converter(v) }
 
     # Maximum delay between retries (in seconds).
     config_param :max_retry_delay, converter: ->(v) { duration_converter(v) }
 
-    # Global shared cache capacity across all partitions for message deduplication.
-    # Must be at least 1. Default: 8192.
+    # Capacity of the producer idempotence cache and of the consumer
+    # deduplication cache. Must be at least 1. Default: 8192.
     config_param :idempotence_cache_size, converter: ->(v) { Integer(v) }
 
     # Version string for cache-busting deduplication hashes. Changing this
@@ -303,7 +307,7 @@ module Prosody
 
     # Keyed-state collections to register before subscribe.
     #
-    # Accepts an array of StateDefinition objects (from Prosody.value/map/deque
+    # Accepts an array of StateDefinition objects (from Prosody.value/map/set/deque
     # and their message_* siblings) or already-serialized registration hashes.
     # Duplicate names within the set are rejected by the native layer.
     config_param :state_collections,
@@ -312,13 +316,15 @@ module Prosody
         list.map { |d| d.respond_to?(:to_state_config) ? d.to_state_config : d }
       }
 
-    # Subsystem under which published JSON collections are advertised.
+    # Subsystem under which published JSON and set collections are advertised.
     # Uses PROSODY_SUBSYSTEM when omitted. Published collections require it.
     config_param :subsystem, converter: lambda(&:to_s)
 
-    # Disk workspace for the local keyed-state cache. Each live client
-    # needs its own directory. Falls back to the
-    # PROSODY_STATE_CACHE_DIR environment variable. Must not be an empty string.
+    # Directory for the local keyed-state caches. Each consumer opens its
+    # cache in a new subdirectory and removes it when the consumer stops, so
+    # clients can share the directory. Falls back to the
+    # PROSODY_STATE_CACHE_DIR environment variable, then to
+    # <temp>/prosody/keyed-state. Must not be an empty string.
     config_param :state_cache_dir, converter: lambda(&:to_s)
 
     # Capacity of the owning keyed-state cache, such as "64 MiB". Uses
@@ -326,6 +332,13 @@ module Prosody
     # selects its default.
     config_param :state_owned_cache_size, converter: lambda(&:to_s)
 
+    # Bytes of in-memory writes the local keyed-state cache holds for each
+    # assigned partition before it flushes them to disk, such as "16 MiB".
+    # Memory use scales with the number of assigned partitions. Uses
+    # PROSODY_STATE_MEMTABLE_SIZE when omitted. Otherwise, the engine uses
+    # its default of 64 MiB.
+    config_param :state_memtable_size, converter: lambda(&:to_s)
+
     # Capacity of the published-state read-through cache, such as "1 MiB".
     # Uses PROSODY_STATE_READ_CACHE_SIZE when omitted. It then uses the owned
     # cache size when set, or 1 MiB when both sizes are unset.
@@ -336,11 +349,6 @@ module Prosody
     config_param :state_read_cache,
       converter: ->(v) { (v == true || v == false) ? v : Float(v) }
 
-    # Delay in whole seconds between staging a provisional cell and the
-    # keyed-state recovery sweep. Every registered TTL must strictly exceed this.
-    # Must be a whole number of seconds >= 1 (validated natively).
-    config_param :state_recovery_delay, converter: ->(v) { duration_converter(v) }
-
     # Operation mode of the client.
     #
     # Valid values:

data/lib/prosody/demand.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Prosody
+  # The demand a handler call serves, from +context.demand+.
+  #
+  # +kind+ is +:normal+ for a normal delivery and +:failure+ for a retry after
+  # a failure. +retries+ is the number of retries: 0 for a normal delivery and
+  # 1 on the first retry. After Prosody defers an event, the count starts
+  # again at 1. The count is an estimate. Keep an exact attempt count in
+  # keyed state if the handler needs one.
+  #
+  # @example Alert only on a retry
+  #   def on_message(context, message)
+  #     alert(message.key) if context.demand.failure?
+  #   end
+  Demand = Data.define(:kind, :retries) do
+    # Whether this is a normal delivery.
+    #
+    # @return [Boolean]
+    def normal? = kind == :normal
+
+    # Whether this is a retry after a failure.
+    #
+    # @return [Boolean]
+    def failure? = kind == :failure
+  end
+end

data/lib/prosody/native_stubs/client.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+# Native stubs for {Prosody::Client}: sending and subscribing.
+
+module Prosody
+  # Main client for interacting with the Prosody messaging system.
+  # Provides methods for sending messages and subscribing to Kafka topics.
+  #
+  # @see ext/prosody/src/client/mod.rs for implementation
+  class Client
+    # Creates a new Prosody client with the given configuration.
+    #
+    # @param config [Hash, Configuration] Client configuration
+    # @return [Client] A new client instance
+    # @raise [ArgumentError] If the configuration is invalid
+    # @raise [RuntimeError] If client initialization fails
+    #
+    # @example Creating a client with a Configuration object
+    #   config = Prosody::Configuration.new do |c|
+    #     c.bootstrap_servers = "localhost:9092"
+    #     c.group_id = "my-consumer-group"
+    #   end
+    #   client = Prosody::Client.new(config)
+    #
+    # @example Creating a client with a hash
+    #   client = Prosody::Client.new(
+    #     bootstrap_servers: "localhost:9092",
+    #     group_id: "my-consumer-group"
+    #   )
+    def self.new(config)
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Returns the current state of the consumer.
+    #
+    # The consumer can be in one of four states:
+    # - `:shut_down` - The client is shut down
+    # - `:unconfigured` - The consumer has not been configured yet
+    # - `:configured` - The consumer is configured but not running
+    # - `:running` - The consumer is actively consuming messages
+    #
+    # @return [Symbol] The current consumer state
+    def consumer_state
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Returns the number of Kafka partitions currently assigned to this consumer.
+    #
+    # This method can be used to monitor the consumer's workload and ensure
+    # proper load balancing across multiple consumer instances.
+    #
+    # @return [Integer] The number of assigned partitions
+    def assigned_partition_count
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Checks if the consumer is stalled.
+    #
+    # A stalled consumer is one that has stopped processing messages due to
+    # errors or reaching processing limits. This can be used to detect unhealthy
+    # consumers that need attention.
+    #
+    # @return [Boolean] true if the consumer is stalled, false otherwise
+    def stalled?
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Sends a message to the specified Kafka topic.
+    #
+    # @param topic [String] The destination topic name
+    # @param key [String] The message key for partitioning
+    # @param payload [Prosody::json_value] The JSON-compatible message payload
+    # @return [void]
+    # @raise [RuntimeError] If the message cannot be sent
+    #
+    # @example Sending a simple message
+    #   client.send_message("my-topic", "user-123", {
+    #     "event" => "login", "timestamp" => Time.now.to_i
+    #   })
+    def send_message(topic, key, payload)
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Sends an excise record to the specified Kafka topic.
+    #
+    # @param topic [String] The destination topic name
+    # @param key [String] The message key for partitioning
+    # @return [void]
+    # @raise [RuntimeError] If the excise record cannot be sent
+    def excise(topic, key)
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Subscribes to Kafka topics using the provided handler.
+    # The handler must implement `on_message`, `on_excise`, and `on_timer`.
+    #
+    # @param handler [EventHandler] A handler object that processes messages
+    # @return [void]
+    # @raise [ArgumentError] If a required handler method is missing
+    # @raise [RuntimeError] If subscription fails
+    #
+    # @example Subscribing with a handler
+    #   class MyHandler < Prosody::EventHandler
+    #     def on_message(context, message)
+    #       puts "Received message: #{message.payload}"
+    #     end
+    #
+    #     def on_excise(_context, message)
+    #       puts "Excised key: #{message.key}"
+    #     end
+    #
+    #     def on_timer(_context, _timer)
+    #     end
+    #   end
+    #
+    #   client.subscribe(MyHandler.new)
+    def subscribe(handler)
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Unsubscribes from all topics, stopping message processing.
+    #
+    # This method gracefully shuts down the consumer, completing any in-flight
+    # messages before stopping.
+    #
+    # @return [void]
+    # @raise [RuntimeError] If unsubscription fails
+    #
+    def unsubscribe
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Shuts down the consumer and all client services.
+    #
+    # @return [void]
+    # @raise [RuntimeError] If shutdown fails
+    #
+    # @example Shutting down a client
+    #   client.shutdown
+    def shutdown
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Returns the configured source system identifier.
+    #
+    # The source system is used to identify the originating service or
+    # component in produced messages, enabling loop detection.
+    #
+    # @return [String] The source system identifier
+    #
+    # @example Getting the source system
+    #   puts client.source_system  # => "my-service"
+    def source_system
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+  end
+end

data/lib/prosody/native_stubs/context.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+# Native stubs for {Prosody::Context}: event metadata, cancellation, and timer
+# scheduling.
+
+module Prosody
+  # Represents the context of a Kafka message, providing metadata and control
+  # capabilities for message handling.
+  #
+  # Instances of this class are created by the native code and passed to your
+  # EventHandler's #on_message method.
+  #
+  # @see ext/prosody/src/handler/context/mod.rs for implementation
+  class Context
+    # @private
+    def initialize
+      raise NotImplementedError, "This class is implemented natively in Rust"
+    end
+
+    # Checks if cancellation has been requested.
+    #
+    # This method can be called within message handlers to detect when the
+    # handler should exit. Cancellation includes message-level cancellation
+    # (e.g., handler timeout) and partition shutdown. During shutdown,
+    # cancellation is delayed until near the end of the shutdown timeout to
+    # allow in-flight work to complete.
+    #
+    # @return [Boolean] true if cancellation has been requested, false otherwise
+    #
+    # @example Checking for cancellation in a loop
+    #   def on_message(context, message)
+    #     items = message.payload["items"]
+    #     items.each do |item|
+    #       return if context.should_cancel?
+    #       process_item(item)
+    #     end
+    #   end
+    def should_cancel?
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Returns the demand this handler call serves.
+    #
+    # A normal delivery has kind +:normal+ and 0 retries. A retry after a
+    # failure has kind +:failure+ and a retry count that is 1 on the first
+    # retry. The count is an estimate; see {Prosody::Demand}.
+    #
+    # @return [Prosody::Demand]
+    #
+    # @example Reading the retry count
+    #   def on_message(context, message)
+    #     logger.warn("retry #{context.demand.retries}") if context.demand.failure?
+    #   end
+    def demand
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Blocks until cancellation is signaled.
+    #
+    # Cancellation includes message-level cancellation (e.g., handler timeout)
+    # and partition shutdown. During shutdown, cancellation is delayed until near
+    # the end of the shutdown timeout to allow in-flight work to complete.
+    # This method is useful for long-running handlers that need to wait for
+    # external events while remaining responsive to cancellation.
+    #
+    # @return [void]
+    #
+    # @example Waiting for cancellation
+    #   def on_message(context, message)
+    #     # Do some work, then wait for cancellation
+    #     context.on_cancel
+    #   end
+    def on_cancel
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Schedules a timer to fire at the specified time.
+    #
+    # Timers allow you to delay execution or implement timeout behavior within
+    # your message handlers. When a timer fires, your handler's #on_timer method
+    # will be called with the timer object.
+    #
+    # @param time [Time] When the timer should fire
+    # @return [void]
+    # @raise [ArgumentError] If the time is invalid or outside the supported range (1970-2106)
+    # @raise [RuntimeError] If timer scheduling fails
+    #
+    # @example Scheduling a delayed action
+    #   def on_message(context, message)
+    #     # Schedule a timer to fire in 30 seconds
+    #     context.schedule(Time.now + 30)
+    #   end
+    #
+    #   def on_timer(context, timer)
+    #     puts "Timer fired for key: #{timer.key}"
+    #   end
+    def schedule(time)
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Clears all scheduled timers and schedules a new one at the specified time.
+    #
+    # This is equivalent to calling clear_scheduled followed by schedule, but
+    # performed atomically.
+    #
+    # @param time [Time] When the new timer should fire
+    # @return [void]
+    # @raise [ArgumentError] If the time is invalid or outside the supported range
+    # @raise [RuntimeError] If timer operations fail
+    #
+    # @example Replacing all timers with a new one
+    #   def on_message(context, message)
+    #     # Clear any existing timers and schedule a new one
+    #     context.clear_and_schedule(Time.now + 60)
+    #   end
+    def clear_and_schedule(time)
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Unschedules a timer that was scheduled for the specified time.
+    #
+    # If multiple timers were scheduled for the same time, this will remove one
+    # of them. If no timer exists for the specified time, this method does nothing.
+    #
+    # @param time [Time] The time for which to unschedule the timer
+    # @return [void]
+    # @raise [ArgumentError] If the time is invalid
+    # @raise [RuntimeError] If timer unscheduling fails
+    #
+    # @example Canceling a specific timer
+    #   def on_message(context, message)
+    #     timer_time = Time.now + 30
+    #     context.schedule(timer_time)
+    #
+    #     # Later, cancel that specific timer
+    #     context.unschedule(timer_time)
+    #   end
+    def unschedule(time)
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Clears all scheduled timers.
+    #
+    # After calling this method, no timers will be scheduled to fire for this
+    # message context.
+    #
+    # @return [void]
+    # @raise [RuntimeError] If clearing timers fails
+    #
+    # @example Canceling all timers
+    #   def on_message(context, message)
+    #     context.clear_scheduled
+    #   end
+    def clear_scheduled
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+
+    # Returns all currently scheduled timer times.
+    #
+    # The returned array contains Time objects representing when each scheduled
+    # timer will fire. The array may be empty if no timers are scheduled.
+    #
+    # @return [Array<Time>] Array of scheduled timer times
+    # @raise [RuntimeError] If retrieving scheduled times fails
+    #
+    # @example Checking scheduled timers
+    #   def on_message(context, message)
+    #     scheduled_times = context.scheduled
+    #     puts "#{scheduled_times.length} timers scheduled"
+    #     scheduled_times.each do |time|
+    #       puts "Timer will fire at: #{time}"
+    #     end
+    #   end
+    def scheduled
+      raise NotImplementedError, "This method is implemented natively in Rust"
+    end
+  end
+end