prosody 0.1.1

data/lib/prosody/processor.rb

@@ -0,0 +1,321 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/barrier"
+require "opentelemetry-api"
+require "prosody/version"
+
+module Prosody
+  # Provides a mechanism for canceling asynchronous tasks.
+  #
+  # This class implements a simple cancellation mechanism using a Ruby Queue,
+  # allowing tasks to be safely canceled while they're in progress. Each token
+  # maintains its own queue for signaling cancellation.
+  class CancellationToken
+    # Creates a new cancellation token with an internal queue for signaling.
+    def initialize
+      @queue = Queue.new
+    end
+
+    # Signals that the associated task should be canceled.
+    #
+    # This method pushes a cancellation signal to the internal queue, which will
+    # wake up any threads waiting on #wait.
+    def cancel
+      @queue.push(:cancel)
+    end
+
+    # Blocks until cancellation is requested.
+    #
+    # This method blocks the current thread until the token is canceled by
+    # another thread calling #cancel.
+    #
+    # @return [Boolean] Always returns true after cancellation is received
+    def wait
+      @queue.pop
+      true
+    end
+  end
+
+  # Contains command classes for the AsyncTaskProcessor's command queue.
+  #
+  # This module implements a command pattern for communication with the processor
+  # thread, allowing for type-safe message passing between threads.
+  module Commands
+    # Base class for all processor commands.
+    #
+    # All commands sent to the AsyncTaskProcessor must inherit from this class
+    # for proper type identification.
+    class Command; end
+
+    # Command to execute a task with the given parameters.
+    #
+    # This command encapsulates all the information needed to execute an
+    # asynchronous task in the Ruby runtime.
+    class Execute < Command
+      # Task identifier for logging and debugging
+      attr_reader :task_id
+
+      # OpenTelemetry context carrier for trace propagation
+      attr_reader :carrier
+
+      # Structured event context for error reporting
+      attr_reader :event_context
+
+      # The block of code to execute
+      attr_reader :block
+
+      # Callback to invoke when execution completes or fails
+      attr_reader :callback
+
+      # Cancellation token for this task
+      attr_reader :token
+
+      # Creates a new execute command with all required parameters.
+      #
+      # @param task_id [String] Unique identifier for the task
+      # @param carrier [Hash] OpenTelemetry context carrier with trace information
+      # @param event_context [Hash] Structured event fields for error reporting
+      # @param block [Proc] The code to execute
+      # @param callback [Proc] Called with (success, result) when complete
+      # @param token [CancellationToken] Token that can be used to cancel execution
+      def initialize(task_id, carrier, event_context, block, callback, token)
+        @task_id = task_id
+        @carrier = carrier
+        @event_context = event_context
+        @block = block
+        @callback = callback
+        @token = token
+      end
+    end
+
+    # Command that signals the processor to shut down.
+    #
+    # When received, the processor will complete all in-flight tasks before
+    # shutting down.
+    class Shutdown < Command; end
+  end
+
+  # Processes asynchronous tasks in a dedicated thread with OpenTelemetry tracing.
+  #
+  # This processor manages a dedicated Ruby thread that executes tasks asynchronously
+  # with proper OpenTelemetry context propagation. It provides:
+  #
+  # - Task submission and execution in an isolated thread
+  # - Cancellation support for in-flight tasks
+  # - Context propagation for distributed tracing
+  # - Graceful shutdown with task completion
+  class AsyncTaskProcessor
+    # Creates a new processor with the given logger.
+    #
+    # @param logger [Logger] Logger for diagnostic messages (defaults to Prosody.logger)
+    def initialize(logger = Prosody.logger)
+      @logger = logger
+      @command_queue = Queue.new
+      @processing_thread = nil
+      @tracer = nil
+    end
+
+    # Starts the processor by launching a dedicated thread.
+    #
+    # The OpenTelemetry tracer is initialized in the processing thread
+    # to avoid crossing thread boundaries. Does nothing if the processor
+    # is already running.
+    def start
+      return if running?
+
+      @logger.debug("Starting async task processor")
+      @processing_thread = Thread.new do
+        # Initialize the tracer in the processing thread to keep
+        # OpenTelemetry context within the same thread
+        @tracer = OpenTelemetry.tracer_provider.tracer(
+          "Prosody::AsyncTaskProcessor",
+          Prosody::VERSION
+        )
+        process_commands
+      end
+    end
+
+    # Gracefully stops the processor.
+    #
+    # Tasks in progress will complete before the processor fully shuts down.
+    # Does nothing if the processor is already stopped.
+    def stop
+      return unless running?
+
+      @logger.debug("Stopping async task processor")
+      @command_queue.push(Commands::Shutdown.new)
+    end
+
+    # Submits a task for asynchronous execution.
+    #
+    # @param task_id [String] Unique identifier for the task
+    # @param carrier [Hash] OpenTelemetry context carrier for tracing
+    # @param event_context [Hash] Structured event fields for error reporting
+    # @param callback [Proc] Called with (success, result) when task completes
+    # @yield The block to execute asynchronously
+    # @return [CancellationToken] Token that can be used to cancel the task
+    def submit(task_id, carrier, event_context, callback, &task_block)
+      token = CancellationToken.new
+      @command_queue.push(
+        Commands::Execute.new(task_id, carrier, event_context.transform_keys(&:to_sym), task_block, callback, token)
+      )
+      token
+    end
+
+    private
+
+    # Checks if the processor thread is running.
+    #
+    # @return [Boolean] true if the processor is running, false otherwise
+    def running?
+      @processing_thread&.alive?
+    end
+
+    # Main processing loop for the async thread.
+    #
+    # Uses the async gem to handle concurrent task execution and tracks
+    # active tasks with a barrier for clean shutdown.
+    def process_commands
+      Async do
+        # Barrier tracks all running tasks for clean shutdown
+        barrier = Async::Barrier.new
+
+        loop do
+          command = @command_queue.pop
+
+          case command
+          when Commands::Execute
+            handle_execute(command, barrier)
+          when Commands::Shutdown
+            @logger.debug("Received shutdown command")
+            # Wait for all tasks to complete before shutting down
+            barrier.wait
+            break
+          else
+            @logger.warn("Unknown command type: #{command.class}")
+          end
+        end
+      end
+    rescue => e
+      @logger.error("Error in process_commands: #{e.message}")
+      @logger.error(e.backtrace.join("\n"))
+    end
+
+    # Handles execution of a task with proper context propagation and error handling.
+    #
+    # @param command [Commands::Execute] The command containing task details
+    # @param barrier [Async::Barrier] Barrier for tracking active tasks
+    def handle_execute(command, barrier)
+      task_id = command.task_id
+      carrier = command.carrier
+      event_context = command.event_context
+      token = command.token
+      callback = command.callback
+      task_block = command.block
+
+      # Extract parent context from the incoming carrier for distributed tracing
+      parent_ctx = OpenTelemetry.propagation.extract(carrier)
+
+      # Create the dispatch span as a child of the extracted context, then
+      # capture the resulting context so it can be explicitly restored inside
+      # the worker fiber. The span is owned by run_with_cancellation, which
+      # finishes it in ensure.
+      dispatch_ctx = OpenTelemetry::Context.with_current(parent_ctx) do
+        span = @tracer.start_span("async_dispatch", kind: :consumer)
+        OpenTelemetry::Trace.with_span(span) { OpenTelemetry::Context.current }
+      end
+
+      @logger.debug("Executing task #{task_id}")
+
+      begin
+        barrier.async do
+          run_with_cancellation(task_id, token, task_block, callback, dispatch_ctx, event_context)
+        end
+      rescue => e
+        # If we failed to enqueue, finish the span here since run_with_cancellation
+        # will never take ownership.
+        OpenTelemetry::Trace.current_span(dispatch_ctx).finish
+        raise e
+      end
+    end
+
+    # Executes a task with proper cancellation support.
+    #
+    # Spawns a worker task and a cancellation watcher within a barrier. When
+    # cancellation is signaled, the worker receives Async::Stop (similar to
+    # Python's asyncio.CancelledError). The worker can catch Async::Stop for
+    # cleanup. The barrier ensures both tasks are cleaned up when the block
+    # exits, even if an unexpected error occurs.
+    #
+    # @param task_id [String] The task identifier for logging
+    # @param token [CancellationToken] The token to monitor for cancellation
+    # @param task_block [Proc] The work to execute
+    # @param callback [Proc] The callback to notify of completion or error
+    # @param dispatch_ctx [OpenTelemetry::Context] Context with async_dispatch span active
+    def run_with_cancellation(task_id, token, task_block, callback, dispatch_ctx, event_context)
+      span = OpenTelemetry::Trace.current_span(dispatch_ctx)
+      # Use a barrier to ensure both tasks are cleaned up when block exits
+      barrier = Async::Barrier.new
+
+      # Spawn worker task - handles its own result reporting
+      worker_task = barrier.async do |task|
+        task.annotate("Worker for task #{task_id}")
+        # Async fibers do not inherit fiber-local OTel context from their parent,
+        # so we must explicitly restore dispatch_ctx so user spans are children
+        # of async_dispatch.
+        OpenTelemetry::Context.with_current(dispatch_ctx) do
+          result = task_block.call
+          if callback.call(true, result)
+            @logger.debug("Task #{task_id} completed successfully")
+          end
+        rescue Async::Stop
+          # Task was cancelled - report via callback
+          if callback.call(false, RuntimeError.new("Task cancelled"))
+            @logger.debug("Task #{task_id} was cancelled")
+          end
+        rescue => e
+          Prosody::SentryIntegration.capture_exception(e, event_context.merge(task_id: task_id))
+          if callback.call(false, e)
+            @logger.error("Error executing task #{task_id}: #{e.message}")
+            span.record_exception(e)
+            span.status = OpenTelemetry::Trace::Status.error(e.to_s)
+          end
+        ensure
+          # Always signal the cancellation watcher to stop waiting
+          token.cancel
+        end
+      end
+
+      # Spawn cancellation watcher - bridges the thread-boundary cancellation signal
+      # into the fiber scheduler. The CancellationToken is a one-shot channel: the
+      # Rust bridge pushes a signal from its thread, and this fiber pops it and
+      # translates it into Async::Stop on the worker. We can't call worker_task.stop
+      # directly from the Rust thread since Async task control must happen on the
+      # scheduler thread.
+      barrier.async do |task|
+        task.annotate("Cancellation watcher for task #{task_id}")
+        begin
+          token.wait
+          worker_task.stop
+        rescue => e
+          @logger.debug("Cancellation watcher error: #{e.message}")
+          span.record_exception(e)
+          span.status = OpenTelemetry::Trace::Status.error(e.to_s)
+        end
+      end
+
+      # Wait for worker to complete (normally, via Async::Stop, or with error)
+      worker_task.wait
+    ensure
+      # Stop any remaining tasks (primarily the cancellation watcher).
+      # Finish the span last so it covers the full execution, and is guaranteed
+      # to close even if barrier.stop raises.
+      begin
+        barrier&.stop
+      ensure
+        span.finish
+      end
+    end
+  end
+end

data/lib/prosody/sentry.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Prosody
+  module SentryIntegration
+    def self.enabled?
+      if ENV["SENTRY_DSN"] && !defined?(::Sentry)
+        unless @warned_missing_gem
+          @warned_missing_gem = true
+          Prosody.logger.error("SENTRY_DSN is set but sentry-ruby is not installed. Add `gem 'sentry-ruby'` to your Gemfile.")
+        end
+        return false
+      end
+
+      return false unless defined?(::Sentry)
+
+      unless ::Sentry.initialized?
+        return false unless ENV["SENTRY_DSN"]
+
+        ::Sentry.init { |c| c.dsn = ENV["SENTRY_DSN"] }
+      end
+
+      ::Sentry.initialized?
+    end
+
+    def self.capture_exception(exception, context = {})
+      return unless enabled?
+
+      ::Sentry.with_scope do |scope|
+        scope.set_context("prosody", context)
+        event_type = context[:event_type]
+        scope.set_tag("prosody.event_type", event_type.to_s) if event_type
+        ::Sentry.capture_exception(exception)
+      end
+    end
+  end
+end

data/lib/prosody/version.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Prosody
+  # Current version of the Prosody library.
+  #
+  # This version number follows semantic versioning and is used by the
+  # gem system to identify the library version. It should be updated
+  # according to semver guidelines when making releases.
+  VERSION = "0.1.1"
+end

data/lib/prosody.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# The Prosody gem provides a Ruby interface for the Prosody event processing system.
+# It implements a high-level client for working with Kafka message streams, with
+# support for both producing and consuming messages in an idiomatic Ruby way.
+#
+# This library wraps a native Rust implementation for high performance while
+# providing a comfortable Ruby API, with features including:
+# - Configuration with Ruby-friendly syntax
+# - Handler classes for processing messages
+# - Async/non-blocking processing
+# - OpenTelemetry integration for distributed tracing
+# - Automatic error classification and retry logic
+
+require "async"
+require "logger"
+require_relative "prosody/version"
+require_relative "prosody/configuration"
+require_relative "prosody/handler"
+require_relative "prosody/processor"
+require_relative "prosody/sentry"
+require_relative "prosody/native_stubs" if defined?(Prosody::Client)
+
+module Prosody
+  def self.logger
+    @logger ||= Logger.new($stdout).tap { |l| l.level = Logger::INFO }
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+end
+
+# Attempt to load the native extension specific to the current Ruby version first,
+# falling back to the generic version if not available. This allows for optimized
+# builds targeting specific Ruby versions.
+begin
+  ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
+  require_relative "prosody/#{ruby_version}/prosody"
+rescue LoadError
+  require_relative "prosody/prosody"
+end

data/release-please-config.json

@@ -0,0 +1,10 @@
+{
+  "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json",
+  "packages": {
+    ".": {
+      "release-type": "ruby",
+      "package-name": "prosody",
+      "version-file": "lib/prosody/version.rb"
+    }
+  }
+}

data/sig/configuration.rbs

@@ -0,0 +1,252 @@
+module Prosody
+  # Configuration for the Prosody messaging client with type validation
+  class Configuration
+    private
+
+    # Internal configuration store
+    @config: Hash[Symbol, untyped]
+
+    public
+
+    # Define a typed configuration parameter with optional conversion and default
+    #
+    # @param name Configuration parameter name
+    # @param converter Function to validate and convert input values
+    # @param default Value returned when parameter is unset
+    def self.config_param: (Symbol | String name, ?converter: ^(untyped) -> untyped, ?default: untyped) -> void
+
+    # Convert duration values to floating point seconds
+    #
+    # @param v Duration value to convert
+    # @raise ArgumentError when value is not a valid duration
+    def self.duration_converter: (Numeric | untyped) -> Float
+
+    # Create a new configuration with initial values
+    #
+    # @param kwargs Initial configuration values
+    def initialize: (?Hash[Symbol, untyped] kwargs) ?{ (Configuration) -> void } -> void
+
+    # Kafka bootstrap server addresses
+    def bootstrap_servers: () -> Array[String]?
+    def bootstrap_servers=: (String | _ToA[String]) -> Array[String]
+
+    # Kafka topics to subscribe to
+    def subscribed_topics: () -> Array[String]?
+    def subscribed_topics=: (String | _ToA[String]) -> Array[String]
+
+    # Event types the consumer is allowed to process
+    def allowed_events: () -> Array[String]?
+    def allowed_events=: (String | _ToA[String]) -> Array[String]
+
+    # Use mock Kafka implementation for testing
+    def mock: () -> bool?
+    def mock=: (untyped) -> bool
+
+    # Maximum time to wait for a send operation (seconds)
+    def send_timeout: () -> Float?
+    def send_timeout=: (Numeric) -> Float
+
+    # Time threshold for detecting stalled consumers (seconds)
+    def stall_threshold: () -> Float?
+    def stall_threshold=: (Numeric) -> Float
+
+    # Maximum time to wait for clean shutdown (seconds)
+    def shutdown_timeout: () -> Float?
+    def shutdown_timeout=: (Numeric) -> Float
+
+    # Interval between Kafka poll operations (seconds)
+    def poll_interval: () -> Float?
+    def poll_interval=: (Numeric) -> Float
+
+    # Interval between offset commit operations (seconds)
+    def commit_interval: () -> Float?
+    def commit_interval=: (Numeric) -> Float
+
+    # Base delay for retry operations (seconds)
+    def retry_base: () -> Float?
+    def retry_base=: (Numeric) -> Float
+
+    # Maximum delay between retries (seconds)
+    def max_retry_delay: () -> Float?
+    def max_retry_delay=: (Numeric) -> Float
+
+    # Size of the cache used for message idempotence
+    def idempotence_cache_size: () -> Integer?
+    def idempotence_cache_size=: (Integer | _ToInt) -> Integer
+
+    # Version string for cache-busting deduplication hashes
+    def idempotence_version: () -> String?
+    def idempotence_version=: (_ToS) -> String
+
+    # TTL for deduplication records in Cassandra (in seconds)
+    def idempotence_ttl: () -> Float?
+    def idempotence_ttl=: (Numeric) -> Float
+
+    # Maximum number of concurrent message processing tasks
+    def max_concurrency: () -> Integer?
+    def max_concurrency=: (Integer | _ToInt) -> Integer
+
+    # Maximum messages to process before committing offsets
+    def max_uncommitted: () -> Integer?
+    def max_uncommitted=: (Integer | _ToInt) -> Integer
+
+    # Maximum number of retry attempts
+    def max_retries: () -> Integer?
+    def max_retries=: (Integer | _ToInt) -> Integer
+
+    # Kafka consumer group ID
+    def group_id: () -> String?
+    def group_id=: (_ToS) -> String
+
+    # Identifier for the system producing messages
+    def source_system: () -> String?
+    def source_system=: (_ToS) -> String
+
+    # Topic to send failed messages to
+    def failure_topic: () -> String?
+    def failure_topic=: (_ToS) -> String
+
+    # Operation mode: :pipeline, :low_latency, or :best_effort
+    def mode: () -> Symbol?
+    def mode=: (Symbol | String) -> Symbol
+
+    # Health probe port configuration (port number, false, or nil)
+    def probe_port: () -> (Integer | false | nil)
+    def probe_port=: (Integer | Symbol | String | false | nil) -> (Integer | false | nil)
+
+    # Cassandra nodes (hostnames or IPs)
+    def cassandra_nodes: () -> Array[String]?
+    def cassandra_nodes=: (String | _ToA[String]) -> Array[String]
+
+    # Cassandra keyspace name
+    def cassandra_keyspace: () -> String?
+    def cassandra_keyspace=: (_ToS) -> String
+
+    # Cassandra datacenter for query routing
+    def cassandra_datacenter: () -> String?
+    def cassandra_datacenter=: (_ToS) -> String
+
+    # Cassandra rack for topology-aware routing
+    def cassandra_rack: () -> String?
+    def cassandra_rack=: (_ToS) -> String
+
+    # Cassandra authentication user
+    def cassandra_user: () -> String?
+    def cassandra_user=: (_ToS) -> String
+
+    # Cassandra authentication password
+    def cassandra_password: () -> String?
+    def cassandra_password=: (_ToS) -> String
+
+    # Cassandra data retention period (seconds)
+    def cassandra_retention: () -> Float?
+    def cassandra_retention=: (Numeric) -> Float
+
+    # Timer slab partitioning duration (seconds)
+    def slab_size: () -> Float?
+    def slab_size=: (Numeric) -> Float
+
+    # Scheduler: failure/retry task weight (0.0 to 1.0)
+    def scheduler_failure_weight: () -> Float?
+    def scheduler_failure_weight=: (Numeric) -> Float
+
+    # Scheduler: max wait for urgency boost (seconds)
+    def scheduler_max_wait: () -> Float?
+    def scheduler_max_wait=: (Numeric) -> Float
+
+    # Scheduler: wait time priority weight
+    def scheduler_wait_weight: () -> Float?
+    def scheduler_wait_weight=: (Numeric) -> Float
+
+    # Scheduler: cache size for per-key virtual time
+    def scheduler_cache_size: () -> Integer?
+    def scheduler_cache_size=: (Integer | _ToInt) -> Integer
+
+    # Monopolization detection enabled
+    def monopolization_enabled: () -> bool?
+    def monopolization_enabled=: (untyped) -> bool
+
+    # Monopolization detection threshold (0.0 to 1.0)
+    def monopolization_threshold: () -> Float?
+    def monopolization_threshold=: (Numeric) -> Float
+
+    # Monopolization detection window (seconds)
+    def monopolization_window: () -> Float?
+    def monopolization_window=: (Numeric) -> Float
+
+    # Monopolization cache size
+    def monopolization_cache_size: () -> Integer?
+    def monopolization_cache_size=: (Integer | _ToInt) -> Integer
+
+    # Deferral enabled
+    def defer_enabled: () -> bool?
+    def defer_enabled=: (untyped) -> bool
+
+    # Defer base backoff delay (seconds)
+    def defer_base: () -> Float?
+    def defer_base=: (Numeric) -> Float
+
+    # Defer maximum delay (seconds)
+    def defer_max_delay: () -> Float?
+    def defer_max_delay=: (Numeric) -> Float
+
+    # Defer failure threshold (0.0 to 1.0)
+    def defer_failure_threshold: () -> Float?
+    def defer_failure_threshold=: (Numeric) -> Float
+
+    # Defer failure window (seconds)
+    def defer_failure_window: () -> Float?
+    def defer_failure_window=: (Numeric) -> Float
+
+    # Defer cache size
+    def defer_cache_size: () -> Integer?
+    def defer_cache_size=: (Integer | _ToInt) -> Integer
+
+    # Defer seek timeout (seconds)
+    def defer_seek_timeout: () -> Float?
+    def defer_seek_timeout=: (Numeric) -> Float
+
+    # Defer discard threshold (messages)
+    def defer_discard_threshold: () -> Integer?
+    def defer_discard_threshold=: (Integer | _ToInt) -> Integer
+
+    # Handler execution timeout (seconds)
+    def timeout: () -> Float?
+    def timeout=: (Numeric) -> Float
+
+    # Kafka topic to produce internal telemetry events to
+    def telemetry_topic: () -> String?
+    def telemetry_topic=: (_ToS) -> String
+
+    # Whether the telemetry emitter is enabled
+    def telemetry_enabled: () -> bool?
+    def telemetry_enabled=: (untyped) -> bool
+
+    # Span linking for message execution spans ('child' or 'follows_from')
+    def message_spans: () -> String?
+    def message_spans=: (_ToS) -> String
+
+    # Span linking for timer execution spans ('child' or 'follows_from')
+    def timer_spans: () -> String?
+    def timer_spans=: (_ToS) -> String
+
+    # Convert configuration to a hash with non-nil values only
+    def to_hash: () -> Hash[Symbol, untyped]
+  end
+
+  # Helper interface for objects convertible to array
+  interface _ToA[T]
+    def to_a: () -> Array[T]
+  end
+
+  # Helper interface for objects convertible to string
+  interface _ToS
+    def to_s: () -> String
+  end
+
+  # Helper interface for objects convertible to integer
+  interface _ToInt
+    def to_int: () -> Integer
+  end
+
+end