pulsar-ruby 0.1.0.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f69966a02c965ad73caf5c5f74c457f51b4d815f0d0b67286eb0e16038723740
+  data.tar.gz: aa38828340d83fc09f8817936c9c50e4a2b16a7e76fda19c404e2ce0b926264d
+SHA512:
+  metadata.gz: 2f0412268a94dacd003510f572f3b6c5ebe1609a95c165dadd907344eb436568603a8526e8786db0e4520a4dffc9770b1fea3d1c6b8dc52906948e1dad9c549f
+  data.tar.gz: 9b9871caad93a69d35c381098e8866c9804e001c53d10a787f7be094f084de4ad2175bcfc9d45f1f2fe58e5e514979df4d4e9eceea3324c45be0c3afd43a5deb

data/LICENSE

@@ -0,0 +1,204 @@
+Copyright 2026 Franky Lau
+
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,198 @@
+# pulsar-ruby
+
+A pure Ruby client for [Apache Pulsar](https://pulsar.apache.org/).
+
+`pulsar-ruby` implements the Pulsar binary protocol in Ruby (no C++ bindings or native
+extensions). The goal is a small, idiomatic API for producing and consuming messages on a
+local or self-hosted broker, with room to grow into TLS, authentication, batching, and
+other features over time.
+
+**Status:** MVP implementation. The public API, protocol definitions, frame
+codec, plaintext TCP transport, connection reader, producer/consumer protocol
+paths, close cleanup, and conservative reconnect are in place. Supported
+integration scenarios are verified against Pulsar standalone.
+
+## Requirements
+
+- Ruby >= 3.0
+
+## Installation
+
+The gem is not published on [RubyGems](https://rubygems.org/) yet. Install from GitHub:
+
+```ruby
+# Gemfile
+gem "pulsar-ruby", github: "killalau/pulsar-ruby"
+```
+
+Or clone the repository and use Bundler locally:
+
+```bash
+git clone https://github.com/killalau/pulsar-ruby.git
+cd pulsar-ruby
+bundle install
+```
+
+## Quickstart
+
+Start a local standalone broker:
+
+```bash
+docker compose up -d pulsar
+```
+
+Produce, receive, and acknowledge one message:
+
+```ruby
+require "pulsar"
+
+Pulsar::Client.open("pulsar://localhost:6650") do |client|
+  producer = client.producer(
+    topic: "persistent://public/default/orders",
+    max_pending_messages: 1000
+  )
+
+  consumer = client.consumer(
+    topic: "persistent://public/default/orders",
+    subscription: "orders-ruby"
+  )
+
+  message_id = producer.send("created", timeout: 5)
+  message = consumer.receive(timeout: 5)
+
+  puts message.payload
+  consumer.ack(message)
+end
+```
+
+`producer.send`, `consumer.receive`, and `consumer.ack` are wired to the
+plaintext Pulsar binary protocol path for non-partitioned topics.
+
+## Supported MVP features
+
+- Plaintext `pulsar://` broker connections.
+- Binary protocol connect handshake.
+- Topic lookup before producer or consumer creation.
+- Non-partitioned topic producers.
+- Non-partitioned topic consumers with one subscription.
+- Single, unbatched string or byte payload messages.
+- Message properties, key, event time, and structured message IDs.
+- Individual acknowledgement.
+- Consumer flow permits.
+- Producer pending-send limits.
+- Operation and receive timeouts.
+- Idempotent producer, consumer, and client close.
+- Conservative reconnect after a dropped connection.
+
+## Deferred features
+
+- TLS.
+- Authentication.
+- Partitioned topics.
+- Reader API.
+- Multi-topic consumers.
+- Subscription type options beyond the current default broker behavior.
+- Batching.
+- Compression.
+- Chunking.
+- Schemas beyond raw byte/string payloads.
+- Negative ack and delayed reconsume.
+- Dead letter and retry topics.
+- Transactions.
+- Interceptors.
+- Admin API.
+- Metrics and tracing.
+
+## Timeouts and reconnect
+
+`Pulsar::Client.new` accepts `operation_timeout:` and `connection_timeout:`.
+`producer.send` and `consumer.receive` also accept `timeout:` for the current
+operation.
+
+Reconnect is intentionally conservative. If the socket drops, in-flight
+operations fail with `Pulsar::ConnectionError`. Existing producer and consumer
+objects remain usable; the next operation lazily opens a replacement connection
+and recreates broker-side producer or consumer state. The failed in-flight
+operation is not silently retried.
+
+See [docs/design/reconnect-policy.md](docs/design/reconnect-policy.md) for the
+design rationale.
+
+## Development
+
+```bash
+bundle install
+bundle exec rake verify
+```
+
+`bundle exec rake verify` runs RuboCop and the normal RSpec suite.
+
+Run the local Pulsar standalone integration spec:
+
+```bash
+docker compose up -d pulsar
+bundle exec rake spec:integration
+```
+
+Build the local gem package:
+
+```bash
+gem build pulsar-ruby.gemspec
+```
+
+Run the installed-gem smoke test against local Pulsar standalone:
+
+```bash
+docker compose up -d pulsar
+bundle exec rake smoke:local
+```
+
+`smoke:local` builds the gem, installs that built artifact into a temporary gem
+home, requires `pulsar`, and performs one produce/consume/ack round trip.
+
+### Git hooks
+
+This repository includes a versioned pre-push hook in `.githooks/pre-push`.
+Enable it for this checkout with:
+
+```bash
+git config core.hooksPath .githooks
+```
+
+After that, `git push` runs `bundle exec rake verify` before sending commits.
+The hook does not run standalone integration specs because they require a local
+broker and are slower than the regular push guard.
+
+### Project layout
+
+| Path | Purpose |
+|------|---------|
+| `lib/pulsar/` | Public API (`Client`, `Producer`, `Consumer`, `Message`, …) |
+| `lib/pulsar/internal/` | Thread runtime, promises, bounded queues (not public API) |
+| `docs/` | Research, design, and implementation planning |
+| `script/` | Local release and smoke-test scripts |
+| `spec/` | RSpec tests |
+
+See [docs/overview.md](docs/overview.md) for the documentation index.
+
+## MVP scope
+
+The first release targets plaintext `pulsar://` connections, single-topic producer and
+consumer workflows, unbatched messages, acknowledgements, flow control, and basic
+reconnect behavior. TLS, authentication, partitioned topics, batching, compression, and
+admin APIs are explicitly out of scope for the MVP.
+
+Details: [docs/research/mvp-scope.md](docs/research/mvp-scope.md).
+
+## Why another Ruby client?
+
+Ruby is not a first-class official Pulsar client today. Existing gems are unmaintained,
+incomplete, or wrap native code. This project aims for a maintained, pure-Ruby
+implementation with a clear public API and testable layers.
+
+Background: [docs/research/ruby-client-landscape.md](docs/research/ruby-client-landscape.md).
+
+## License
+
+Licensed under the [Apache License, Version 2.0](LICENSE), aligned with the Apache
+Pulsar ecosystem. The software is provided as-is, without warranty.

data/lib/pulsar/client.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'uri'
+
+module Pulsar
+  # Entry point for creating producers and consumers against a Pulsar broker.
+  class Client
+    DEFAULT_OPERATION_TIMEOUT = 30
+    DEFAULT_CONNECTION_TIMEOUT = 10
+
+    attr_reader :service_url, :operation_timeout, :connection_timeout, :logger
+
+    def self.open(service_url, **options)
+      client = new(service_url, **options)
+      return client unless block_given?
+
+      begin
+        yield client
+      ensure
+        client.close
+      end
+    end
+
+    def initialize(service_url, operation_timeout: DEFAULT_OPERATION_TIMEOUT,
+                   connection_timeout: DEFAULT_CONNECTION_TIMEOUT, logger: nil, **options)
+      validate_unsupported_options!(options)
+
+      @service_url = normalize_service_url(service_url)
+      @service_uri = URI(@service_url)
+      @operation_timeout = operation_timeout
+      @connection_timeout = connection_timeout
+      @logger = logger
+      @producers = Set.new
+      @consumers = Set.new
+      @closed = false
+      @producer_id = 0
+      @consumer_id = 0
+    end
+
+    def producer(topic:, max_pending_messages: 1000, **_options)
+      ensure_open!
+      lookup_topic(topic)
+
+      impl = Internal::ProducerImpl.create(
+        connection_provider: -> { connection },
+        topic: topic,
+        producer_id: next_producer_id,
+        operation_timeout: operation_timeout,
+        max_pending_messages: max_pending_messages
+      )
+      Producer.new(topic: topic, impl: impl).tap { |producer| @producers.add(producer) }
+    end
+
+    def consumer(topic:, subscription:, **_options)
+      ensure_open!
+      lookup_topic(topic)
+
+      impl = Internal::ConsumerImpl.create(
+        connection_provider: -> { connection },
+        topic: topic,
+        subscription: subscription,
+        consumer_id: next_consumer_id,
+        operation_timeout: operation_timeout,
+        receiver_queue_size: 1000
+      )
+      Consumer.new(topic: topic, subscription: subscription, impl: impl).tap { |consumer| @consumers.add(consumer) }
+    end
+
+    def close
+      return if closed?
+
+      @closed = true
+      close_all(@producers)
+      close_all(@consumers)
+      @connection&.close
+      nil
+    end
+
+    def closed?
+      @closed
+    end
+
+    private
+
+    def close_all(resources)
+      resources.each(&:close)
+      resources.clear
+    end
+
+    def connection
+      @connection&.close if @connection && !@connection.connected?
+      @connection = nil if @connection && !@connection.connected?
+      @connection ||= Internal::Connection.connect(
+        host: @service_uri.host,
+        port: @service_uri.port || 6650,
+        connection_timeout: connection_timeout,
+        operation_timeout: operation_timeout,
+        client_version: "pulsar-ruby/#{VERSION}"
+      )
+    end
+
+    def lookup_topic(topic)
+      Internal::LookupService.new(connection: connection, operation_timeout: operation_timeout).lookup(topic)
+    end
+
+    def next_producer_id
+      @producer_id += 1
+    end
+
+    def next_consumer_id
+      @consumer_id += 1
+    end
+
+    def ensure_open!
+      raise ClosedError, 'client is closed' if closed?
+    end
+
+    def normalize_service_url(service_url)
+      uri = URI(String(service_url))
+      raise ConfigurationError, 'service URL must use pulsar:// scheme' unless uri.scheme == 'pulsar'
+      raise ConfigurationError, 'service URL must include a host' if uri.host.nil? || uri.host.empty?
+
+      uri.to_s
+    rescue URI::InvalidURIError => e
+      raise ConfigurationError, "invalid service URL: #{e.message}"
+    end
+
+    def validate_unsupported_options!(options)
+      unsupported = options.keys & %i[authentication tls]
+      return if unsupported.empty?
+
+      raise ConfigurationError, "unsupported MVP option(s): #{unsupported.join(', ')}"
+    end
+  end
+end

data/lib/pulsar/consumer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Pulsar
+  # Public consumer API for receiving and acknowledging messages.
+  class Consumer
+    attr_reader :topic, :subscription
+
+    def initialize(topic:, subscription:, impl: nil)
+      @topic = String(topic)
+      @subscription = String(subscription)
+      @impl = impl
+      @closed = false
+    end
+
+    def receive(timeout: nil)
+      ensure_open!
+      raise UnsupportedFeatureError, 'consumer receive is not implemented yet' unless @impl
+
+      @impl.receive(timeout: timeout)
+    end
+
+    def ack(message_or_message_id)
+      ensure_open!
+      raise UnsupportedFeatureError, 'consumer ack is not implemented yet' unless @impl
+
+      @impl.ack(message_or_message_id)
+    end
+
+    def close
+      return if closed?
+
+      @impl&.close
+      @closed = true
+      nil
+    end
+
+    def closed?
+      @closed
+    end
+
+    private
+
+    def ensure_open!
+      raise ClosedError, 'consumer is closed' if closed?
+    end
+  end
+end

data/lib/pulsar/errors.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Pulsar
+  class Error < StandardError; end
+
+  class TimeoutError < Error; end
+  class ConnectionError < Error; end
+  class ClosedError < Error; end
+  class AuthenticationError < Error; end
+  class AuthorizationError < Error; end
+  class TopicNotFoundError < Error; end
+  class ProducerBusyError < Error; end
+  class ConsumerBusyError < Error; end
+  class BrokerError < Error; end
+  class ProtocolError < Error; end
+  class ConfigurationError < Error; end
+  class UnsupportedFeatureError < Error; end
+end

data/lib/pulsar/internal/bounded_queue.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Pulsar
+  module Internal
+    # Sized queue wrapper with close and timeout behavior.
+    class BoundedQueue
+      def initialize(capacity:)
+        raise ArgumentError, 'capacity must be positive' unless capacity.positive?
+
+        @queue = SizedQueue.new(capacity)
+        @mutex = Mutex.new
+        @closed = false
+      end
+
+      def push(value, timeout: nil)
+        ensure_open!
+
+        if timeout
+          push_with_timeout(value, timeout)
+        else
+          push_until_available(value)
+        end
+
+        nil
+      end
+
+      def pop(timeout:)
+        ensure_open!
+
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+
+        loop do
+          ensure_open!
+          return @queue.pop(true)
+        rescue ThreadError
+          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          raise TimeoutError, 'operation timed out' if remaining <= 0
+
+          sleep([remaining, 0.001].min)
+        end
+      end
+
+      def close
+        @mutex.synchronize { @closed = true }
+        nil
+      end
+
+      private
+
+      def push_with_timeout(value, timeout)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+
+        loop do
+          ensure_open!
+          return @queue.push(value, true)
+        rescue ThreadError
+          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          raise TimeoutError, 'operation timed out' if remaining <= 0
+
+          sleep([remaining, 0.001].min)
+        end
+      end
+
+      def push_until_available(value)
+        loop do
+          ensure_open!
+          return @queue.push(value, true)
+        rescue ThreadError
+          sleep 0.001
+        end
+      end
+
+      def ensure_open!
+        raise ClosedError, 'queue is closed' if @mutex.synchronize { @closed }
+      end
+    end
+  end
+end