eventhub-processor2 1.27.2 → 1.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c00f385100ce56b24b64a527d935dc4820ce5446c41361792d5f00b7345940d
-  data.tar.gz: f8d3a05880d7af80858c39b000325ec4e3789551961d941cf0511fd051ec6d58
+  metadata.gz: 712c5d8a0d59c49a1efe698824cca79e0d229894db4673cfc6eaed9d8080bc57
+  data.tar.gz: c3107bdb2f39ab2e79faa2bcd6cd2230c0c4fa07b31a067efd9cba07fe51540b
 SHA512:
-  metadata.gz: d20c763b7e403e2946a83af3cba688ebbd2ef92199332ce7aa2e22e6c3bd9bc8c288fea4dcace16c2a1d504fb2c8d643a6d3eb54ea419bc482d75675155c5c70
-  data.tar.gz: fb3830c2b68e056d995b8c33742cdd46f216788094cce20f25d5e176a74f86facdc52c9b2da9eeb7b9561d7b16cce8683187e1eccb14c6d29ed1d0b820195a74
+  metadata.gz: 8ff4cb3ad79b36661685556048fd9006f899c1deb7dcc80e00c83f15a253e2d5bd6e60d5697a83827ad1adcefe22dd2c54d3de7ee4eeeb0e9ce3126e7f3c345d
+  data.tar.gz: e486e53d7a81179645b3e2a0be64bca1e6604215a16bdb5e1c5a08114ed2b7893339bf89ae48c9c34241bf516c49fa0bded503673b2fa15934b08c9bf20f2ee4

data/.gitignore

@@ -2,7 +2,8 @@
 coverage
 .rspec_status
 logs
-example/data
+soak/data
+soak/logs
 pkg
 tmp
 checksums

data/.tool-versions

@@ -1 +1 @@
-ruby 4.0.2
+ruby 4.0.4

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog of EventHub::Processor2
 
+# 1.28.0 / 2026-05-18
+
+**Reliability**
+
+* Survive broker restarts. `automatically_recover: true` + recovery callbacks, so transient disconnects no longer crash the Celluloid actor or exhaust the supervisor's restart budget. Fixes the dispatcher's crash-loop into permanent death after a broker bounce.
+* Survive silent consumer-thread death. New `channel#on_uncaught_exception` / `consumer#on_cancellation` hooks on `ActorListenerAmqp` escalate to an actor restart, but only for non-recoverable errors - transient `Bunny::NetworkFailure` / `ConnectionClosedError` / `TCPConnectionFailed` / `Timeout::Error` / `IOError` and mid-disconnect cancellations are left to Bunny's recovery (no spurious 15s restart sleep).
+* `ActorWatchdog` tolerates transient broker errors; only escalates on 3 consecutive cycles of persistent queue absence.
+* Tolerant `cleanup` in `ActorListenerAmqp`, `ActorPublisher`, `ActorHeartbeat` - bunny-3 raises on `close` of a torn-down session no longer crash the finalizer.
+* **Patch Celluloid 0.18 / Ruby 3.x incompatibility.** Celluloid's `Internals::Logger.crash` mutates a frozen string literal, which raises `FrozenError` before our exception handler runs - the actor thread then dies silently, the supervisor never sees the exit, and the listener becomes a zombie. Symptom: SIGHUP-triggered restart looks like it works but the listener stops consuming. Patched via `Module#prepend` in `lib/eventhub/patches/celluloid_logger.rb`.
+
+**Throughput**
+
+* `ActorPublisher` and `ActorHeartbeat` now reuse a single channel per actor instead of opening one per message. Eliminates the RabbitMQ `Channel is stopping with N pending publisher confirms` warning and lifts publisher throughput from ~hundreds to ~7k msg/s in local tests.
+* New `rake test:performance` task: regression gate at 5k msg/s on a reused channel (opt-in, tagged `:performance`).
+
+**Tracing**
+
+* `correlation_id` now survives the cross-actor hop on publish. `Processor2#publish` captures `CorrelationId.current` in the caller's thread before handing off to the publisher actor, so the AMQP header is preserved end-to-end.
+
+**Security**
+
+* Sensitive-value redaction in the rendered config page now walks nested hashes and arrays. Previously `server.credentials.password` and `connections[].token` leaked through; now redacted at any depth.
+
+**Observability**
+
+* `Celluloid.exception_handler` log line includes the dying actor's class name.
+* Log broker `connection.blocked` / `connection.unblocked` events.
+
+**Test harness**
+
+* New `soak/` reliability harness (publisher / router / receiver / crasher) with `make soak` Makefile target. Adaptive drain, real-vs-in-flight orphan classification, configurable chaos length. Validated with 2h chaos run: 0 real orphans.
+
 # 1.27.2 / 2026-04-08
 
 * Fix publish return value leaking Bunny::Exchange object back to callers, causing unintended re-publishing of garbage messages via `handle_payload`

data/Makefile

@@ -0,0 +1,115 @@
+# Convenience targets for running and soak-testing the chaos harness in soak/.
+# The gem itself is built via `rake` (see Rakefile).
+
+SHELL          := /bin/bash
+SOAK_DIR       := soak
+DATA_DIR       := $(SOAK_DIR)/data
+LOG_DIR        := $(SOAK_DIR)/logs
+SOAK_MINUTES     ?= 10
+SOAK_DRAIN_POLL_S ?= 5
+SOAK_DRAIN_MAX_S  ?= 600
+
+.PHONY: help
+help:
+	@echo "Targets:"
+	@echo "  make soak-start            start publisher, router, receiver, crasher in background"
+	@echo "  make soak-stop             stop everything (SIGINT, clean shutdown)"
+	@echo "  make soak-clean            stop and wipe $(DATA_DIR)/ + $(LOG_DIR)/"
+	@echo "  make soak                  run reliability soak for SOAK_MINUTES (default 10) min"
+	@echo ""
+	@echo "Soak env overrides:"
+	@echo "  SOAK_MINUTES=N             length of the chaos phase (default 10)"
+	@echo "  SOAK_DRAIN_POLL_S=N        drain poll interval in seconds (default 5)"
+	@echo "  SOAK_DRAIN_MAX_S=N         drain hard cap in seconds (default 600 = 10 min)"
+	@echo ""
+	@echo "Publisher env overrides (forwarded to publisher.rb):"
+	@echo "  PAUSE_BETWEEN_WORK=F       seconds between publishes (default 0.05)"
+	@echo "  PUBLISH_MAX_ATTEMPTS=N     publish retry attempts on transient errors (default 8)"
+	@echo "  PUBLISH_RETRY_DELAY_S=F    seconds between publish retries (default 1)"
+
+.PHONY: soak-start
+soak-start: soak-stop
+	@mkdir -p $(DATA_DIR)
+	@echo "==> starting receiver"
+	@cd $(SOAK_DIR) && nohup ruby receiver.rb  > /dev/null 2>&1 &
+	@echo "==> starting router"
+	@cd $(SOAK_DIR) && nohup ruby router.rb    > /dev/null 2>&1 &
+	@sleep 3
+	@echo "==> starting publisher"
+	@cd $(SOAK_DIR) && nohup ruby publisher.rb > /dev/null 2>&1 &
+	@sleep 2
+	@echo "==> starting crasher"
+	@cd $(SOAK_DIR) && nohup ruby crasher.rb   > /dev/null 2>&1 &
+	@echo "==> all processes started; tail logs in $(SOAK_DIR)/logs/ruby/"
+
+.PHONY: soak-stop
+soak-stop:
+	@pkill -INT -f "ruby (publisher|router|receiver|crasher)\.rb" 2>/dev/null || true
+	@sleep 2
+
+.PHONY: soak-clean
+soak-clean: soak-stop
+	@rm -rf $(DATA_DIR) $(LOG_DIR)
+	@mkdir -p $(DATA_DIR) $(LOG_DIR)/ruby
+	@echo "==> cleaned $(DATA_DIR) and $(LOG_DIR)"
+
+# Soak: run the chaos loop for SOAK_MINUTES, then SIGKILL the publisher
+# (skipping its cleanup so we keep an honest snapshot of any in-flight
+# files), give router+receiver SOAK_DRAIN_S seconds to drain the queue,
+# then count remaining files. Anything left is an orphan.
+.PHONY: soak
+soak:
+	@started=$$(date +%s); started_human=$$(date '+%Y-%m-%d %H:%M:%S %Z'); \
+	  chaos_ends_at=$$(date -r $$(($$started + $(SOAK_MINUTES) * 60)) '+%H:%M:%S' 2>/dev/null \
+	                   || date -d @"$$(($$started + $(SOAK_MINUTES) * 60))" '+%H:%M:%S' 2>/dev/null); \
+	  echo "==> soak: $(SOAK_MINUTES) min chaos + adaptive drain (cap $(SOAK_DRAIN_MAX_S)s)"; \
+	  echo "    started:           $$started_human"; \
+	  echo "    chaos ends ~       $$chaos_ends_at"; \
+	  $(MAKE) --no-print-directory soak-clean; \
+	  mkdir -p $(DATA_DIR); \
+	  ( cd $(SOAK_DIR) && nohup ruby receiver.rb  > /dev/null 2>&1 & ); \
+	  ( cd $(SOAK_DIR) && nohup ruby router.rb    > /dev/null 2>&1 & ); \
+	  sleep 3; \
+	  ( cd $(SOAK_DIR) && nohup ruby publisher.rb > /dev/null 2>&1 & ); \
+	  sleep 2; \
+	  ( cd $(SOAK_DIR) && nohup ruby crasher.rb   > /dev/null 2>&1 & ); \
+	  echo "==> chaos phase running for $$(($(SOAK_MINUTES) * 60))s..."; \
+	  sleep $$(($(SOAK_MINUTES) * 60)); \
+	  echo "==> stopping crasher (SIGINT)"; \
+	  pkill -INT -f "ruby crasher\.rb"   2>/dev/null || true; \
+	  echo "==> SIGKILL publisher to skip its cleanup and freeze the snapshot"; \
+	  pkill -KILL -f "ruby publisher\.rb" 2>/dev/null || true; \
+	  sleep 1; \
+	  drain_started=$$(date +%s); prev=-1; deadline=$$(($$drain_started + $(SOAK_DRAIN_MAX_S))); \
+	  echo "==> draining (poll every $(SOAK_DRAIN_POLL_S)s, cap $(SOAK_DRAIN_MAX_S)s)"; \
+	  echo "    (orphans = files with no matching store.json entry; in-flight = SIGKILL race, not a failure)"; \
+	  while :; do \
+	    now=$$(date +%s); \
+	    read real in_flight <<< $$($(SOAK_DIR)/check_orphans.rb $(DATA_DIR)); \
+	    elapsed_drain=$$(($$now - $$drain_started)); \
+	    printf "    [%4ds] real=%d in_flight=%d\n" $$elapsed_drain $$real $$in_flight; \
+	    if [ "$$real" = "0" ]; then break; fi; \
+	    if [ $$now -ge $$deadline ]; then echo "    drain cap reached, giving up"; break; fi; \
+	    if [ "$$real" = "$$prev" ]; then echo "    no progress in last interval, giving up"; break; fi; \
+	    prev=$$real; \
+	    sleep $(SOAK_DRAIN_POLL_S); \
+	  done; \
+	  finished=$$(date +%s); finished_human=$$(date '+%Y-%m-%d %H:%M:%S %Z'); \
+	  elapsed=$$(($$finished - $$started)); \
+	  read real in_flight <<< $$($(SOAK_DIR)/check_orphans.rb $(DATA_DIR)); \
+	  echo ""; \
+	  echo "==> soak result"; \
+	  echo "    started:               $$started_human"; \
+	  echo "    finished:              $$finished_human"; \
+	  echo "    elapsed total:         $${elapsed}s"; \
+	  echo "    drain time:            $$(($$finished - $$drain_started))s"; \
+	  echo "    real orphans:          $$real  (pipeline loss)"; \
+	  echo "    in-flight at SIGKILL:  $$in_flight  (expected residual; cleaned on next publisher start)"; \
+	  $(MAKE) --no-print-directory soak-stop; \
+	  if [ "$$real" = "0" ]; then \
+	    echo "==> PASS"; \
+	  else \
+	    echo "==> FAIL: $$real real orphan(s) remain after drain"; \
+	    $(SOAK_DIR)/check_orphans.rb $(DATA_DIR) --list-orphans >/dev/null; \
+	    exit 1; \
+	  fi

data/README.md

@@ -463,7 +463,7 @@ end
 
 ### Configuration
 
-Displays the active configuration as an HTML table. Sensitive values (passwords, tokens, keys) are automatically redacted.
+Displays the active configuration as an HTML table. Sensitive values (passwords, tokens, keys) are automatically redacted at any depth — keys matching the sensitive list are masked whether they appear at the top level, inside nested hashes, or inside hashes nested in arrays.
 
 ```
 GET {base_path}/docs/configuration
@@ -547,6 +547,35 @@ end
 
 To install this gem onto your local machine, run `bundle exec rake install`.
 
+### Reliability soak harness
+
+A multi-process chaos harness lives in `soak/` (publisher, router, receiver, crasher). It validates end-to-end reliability under sustained broker restarts and `SIGHUP`-triggered listener restarts. A run is considered passing when zero "real orphan" files remain (files whose UUID is not in the publisher's transaction store — i.e. messages whose delivery was claimed by the publisher but never made it through the pipeline).
+
+```
+  # 10-minute chaos + adaptive drain (default), prints PASS/FAIL
+  make soak
+
+  # Longer runs
+  SOAK_MINUTES=60 make soak
+  SOAK_MINUTES=120 SOAK_DRAIN_MAX_S=1200 make soak
+
+  # See all targets and env knobs
+  make help
+```
+
+See `soak/README.md` for the full description.
+
+### Throughput baseline
+
+`rake test:performance` runs a publisher-throughput regression spec against a real local RabbitMQ. It is excluded from `rake spec` (tagged `:performance`) so it does not run on every CI build.
+
+```
+  bundle exec rake test:performance
+  PERF_FLOOR=3000 bundle exec rake test:performance   # lower the floor for slower hosts
+```
+
+Default floor is 5,000 msg/s. Typical local result is ~7,000 msg/s.
+
 ## Publishing
 
 This project uses [Trusted Publishing](https://guides.rubygems.org/trusted-publishing/) to securely publish gems to RubyGems.org via GitHub Actions. To release a new version:

data/Rakefile

@@ -4,6 +4,15 @@ require "standard/rake"
 
 RSpec::Core::RakeTask.new(:spec) do |t|
   t.verbose = false
+  t.rspec_opts = "--tag ~performance"
+end
+
+namespace :test do
+  desc "Run throughput baseline against real RabbitMQ (skipped by `rake spec`)"
+  RSpec::Core::RakeTask.new(:performance) do |t|
+    t.verbose = false
+    t.rspec_opts = "--tag performance --format documentation"
+  end
 end
 
 desc "Initialize or reset rabbitmq docker container (run before rspec)"

data/example/README.md

@@ -1,40 +1,20 @@
-## Example Application
+## Example processor
 
-### Description
+`example.rb` is a minimal `EventHub::Processor2` subclass used to demo and
+visually verify the gem's built-in HTTP endpoints (`/heartbeat`, `/version`,
+`/docs`, `/changelog`, `/configuration`). It's intentionally tiny - just enough
+to start a processor against the local RabbitMQ container.
 
-Example folder contains a series of applications in order to test reliability and performance of processor2 gem.
-* publisher.rb - Creates a unique file and a message
-* router.rb - routes messges via queues
-* receiver.rb - receives message and does final processing
-* crasher.rb - restarts message broker or sends signals to other processes
+For the chaos / reliability test harness (publisher, router, receiver,
+crasher), see [`../soak/`](../soak/) and the `make soak` target in the
+project root.
 
-### How does it work?
+### Run
 
-A message is passed throuhg the following components.
-publisher.rb => [example.outbound] => router.rb => [example.inbound] => receiver.rb
+```bash
+cd example
+bundle exec ruby example.rb
+```
 
-1. publisher.rb generates a unique ID, creates a json message with the ID as payload, save the message in a file, and passes the message to example.outbound queue.
-
-2. router.rb receives the message and passes it to exmaple.outbound queue
-
-3. receiver.rb gets the message and deletes the file with the given ID
-
-### Goal
-What ever happens to these components (restarted, killed and restarted, stopped and started, message broker killed, stopped and started) if you do a graceful shutdown at the end there should be no message in the /data folder (except store.json).
-
-Graceful shutdown with CTRL-C or TERM signal to pid
-* Stop producer.rb. Leave the other components running until all messages in example.* queues are gone.
-* Stop remaining components
-* Check ./example/data folder
-
-
-### How to use?
-* Make sure docker container (process-rabbitmq) is running (see [readme](../docker/README.md))
-* Start one or more router with: bundle exec ruby router.rb
-* Start one or more receiver with: bundle exec ruby receier.rb
-* Start one publisher with: bundle exec ruby publisher.rb
-* Start one crasher with: bundle exec ruby crasher.rb (or do this manually)
-
-### Note
-* Publisher has a simple transaction store implemented to deal with issues between file creation and file publishing. At the end of the publisher process in the cleanup method pending transaction get processed and coresponding files get deleted.
-* Watch for huge log files!
+Then visit `http://localhost:8083/svc/example/docs` (or whatever `http.port`
+is configured in `config/example.json`).

data/lib/eventhub/actor_heartbeat.rb

@@ -9,6 +9,8 @@ module EventHub
 
     def initialize(processor_instance)
       @processor_instance = processor_instance
+      @connection = nil
+      @channel = nil
       async.start
     end
 
@@ -28,21 +30,49 @@ module EventHub
 
     def cleanup
       EventHub.logger.info("Heartbeat is cleaning up...")
-      publish(heartbeat(action: "stopped"))
-      EventHub.logger.info("Heartbeat has sent a [stopped] beat")
+      begin
+        publish(heartbeat(action: "stopped"))
+        EventHub.logger.info("Heartbeat has sent a [stopped] beat")
+      rescue => ex
+        EventHub.logger.warn("Heartbeat cleanup publish: ignoring #{ex.class}: #{ex.message}")
+      end
+      begin
+        @channel&.close
+      rescue => ex
+        EventHub.logger.warn("Heartbeat cleanup channel: ignoring #{ex.class}: #{ex.message}")
+      end
+      begin
+        @connection&.close
+      rescue => ex
+        EventHub.logger.warn("Heartbeat cleanup connection: ignoring #{ex.class}: #{ex.message}")
+      end
     end
 
     private
 
     def publish(message)
-      connection = create_bunny_connection
-      connection.start
-      channel = connection.create_channel
-      channel.confirm_select(tracking: true)
-      exchange = channel.direct(EventHub::EH_X_INBOUND, durable: true)
+      ensure_channel
+      exchange = @channel.direct(EventHub::EH_X_INBOUND, durable: true)
       exchange.publish(message, persistent: true)
-    ensure
-      connection&.close
+    rescue Bunny::NetworkFailure, Bunny::ChannelAlreadyClosed => e
+      EventHub.logger.warn("Heartbeat channel dropped: #{e.class}: #{e.message}")
+      begin
+        @channel&.close
+      rescue
+        nil
+      end
+      @channel = nil
+      raise
+    end
+
+    def ensure_channel
+      unless @connection
+        @connection = create_bunny_connection
+        @connection.start
+      end
+      return if @channel&.open?
+      @channel = @connection.create_channel
+      @channel.confirm_select(tracking: true)
     end
 
     def heartbeat(args = {action: "running"})

data/lib/eventhub/actor_listener_amqp.rb

@@ -39,6 +39,37 @@ module EventHub
     def listen(args = {})
       with_listen(args) do |connection, channel, consumer, queue, queue_name|
         EventHub.logger.info("Listening to queue [#{queue_name}]")
+
+        # log broker-initiated connection state changes
+        connection.on_blocked { |reason| EventHub.logger.warn("Broker blocked connection: #{reason}") }
+        connection.on_unblocked { EventHub.logger.info("Broker unblocked connection") }
+
+        # Only escalate to an actor restart for exceptions Bunny will NOT
+        # recover from. Transient network exceptions are handled by Bunny's
+        # automatic recovery; an actor restart in that window races recovery
+        # and incurs an avoidable 15s before_restart sleep without consumption.
+        channel.on_uncaught_exception do |ex, _consumer|
+          if recoverable_bunny_error?(ex)
+            EventHub.logger.warn("Consumer thread raised recoverable #{ex.class}: #{ex.message} - leaving recovery to Bunny")
+          else
+            EventHub.logger.error("Consumer thread raised non-recoverable #{ex.class}: #{ex.message} - restarting listener")
+            Celluloid::Actor[:actor_listener_amqp]&.async&.restart
+          end
+        end
+
+        # Broker may cancel a consumer (queue deleted, HA failover, policy change).
+        # If the connection is still open, this is a real broker-side cancel and
+        # we must restart. If the connection is closed/recovering, Bunny will
+        # re-register the consumer itself on reconnect; do not race it.
+        consumer.on_cancellation do
+          if connection.open?
+            EventHub.logger.error("Consumer for [#{queue_name}] cancelled by broker - restarting listener")
+            Celluloid::Actor[:actor_listener_amqp]&.async&.restart
+          else
+            EventHub.logger.warn("Consumer for [#{queue_name}] cancelled during disconnect - leaving recovery to Bunny")
+          end
+        end
+
         consumer.on_delivery do |delivery_info, metadata, payload|
           CorrelationId.with(metadata[:correlation_id]) do
             EventHub.logger.info("#{queue_name}: [#{delivery_info.delivery_tag}]" \
@@ -70,9 +101,10 @@ module EventHub
 
     def with_listen(args = {}, &block)
       connection = create_bunny_connection
-      connection.start
       queue_name = args[:queue_name]
+      # store FIRST so cleanup can find a partially-started session
       @connections[queue_name] = connection
+      connection.start
       channel = connection.create_channel
       channel.prefetch(1)
       queue = channel.queue(queue_name, durable: true)
@@ -88,6 +120,7 @@ module EventHub
     def handle_payload(args = {})
       response_messages = []
       connection = args[:connection]
+      correlation_id = args[:correlation_id] || CorrelationId.current
 
       # convert to EventHub message
       message = EventHub::Message.from_json(args[:payload])
@@ -123,9 +156,12 @@ module EventHub
         end
       end
 
+      # use possibly-updated execution_id fallback from above
+      correlation_id ||= CorrelationId.current
+
       Array(response_messages).each do |message|
         next unless message.is_a?(EventHub::Message)
-        publish(message: message.to_json, connection: connection)
+        publish(message: message.to_json, connection: connection, correlation_id: correlation_id)
       end
     end
 
@@ -136,15 +172,33 @@ module EventHub
 
     def cleanup
       EventHub.logger.info("Listener amqp is cleaning up...")
-      # close all open connections
+      # close all open connections; bunny-3 can raise on a torn-down session
       return unless @connections
       @connections.values.each do |connection|
         connection&.close
+      rescue => ex
+        EventHub.logger.warn("Listener cleanup: ignoring #{ex.class}: #{ex.message}")
       end
     end
 
     def publish(args)
       @actor_publisher.publish(args)
     end
+
+    # Exceptions that Bunny's network recovery handles transparently. If one of
+    # these bubbles into `on_uncaught_exception`, the right move is to let the
+    # in-flight recovery complete rather than racing it with an actor restart.
+    RECOVERABLE_BUNNY_ERRORS = [
+      Bunny::NetworkFailure,
+      Bunny::ConnectionClosedError,
+      Bunny::TCPConnectionFailed,
+      Bunny::TCPConnectionFailedForAllHosts,
+      Timeout::Error,
+      IOError
+    ].freeze
+
+    def recoverable_bunny_error?(ex)
+      RECOVERABLE_BUNNY_ERRORS.any? { |klass| ex.is_a?(klass) }
+    end
   end
 end

data/lib/eventhub/actor_publisher.rb

@@ -10,23 +10,17 @@ module EventHub
     def initialize
       EventHub.logger.info("Publisher is starting...")
       @connection = nil
+      @channel = nil
     end
 
     def publish(args = {})
-      # keep connection once established
-      unless @connection
-        @connection = create_bunny_connection
-        @connection.start
-      end
+      ensure_channel
 
       message = args[:message]
       return if message.nil?
 
       exchange_name = args[:exchange_name] || EH_X_INBOUND
-
-      channel = @connection.create_channel
-      channel.confirm_select(tracking: true)
-      exchange = channel.direct(exchange_name, durable: true)
+      exchange = @channel.direct(exchange_name, durable: true)
 
       publish_options = {persistent: true}
       correlation_id = args[:correlation_id] || CorrelationId.current
@@ -34,13 +28,53 @@ module EventHub
 
       exchange.publish(message, publish_options)
       nil
-    ensure
-      channel&.close
+    rescue Bunny::NetworkFailure, Bunny::ChannelAlreadyClosed => e
+      # broker-side close - drop the channel so next publish reopens it
+      EventHub.logger.warn("Publisher channel dropped: #{e.class}: #{e.message}")
+      begin
+        @channel&.close
+      rescue
+        nil
+      end
+      @channel = nil
+      raise
     end
 
     def cleanup
       EventHub.logger.info("Publisher is cleaning up...")
-      @connection&.close
+      begin
+        @channel&.close
+      rescue => ex
+        EventHub.logger.warn("Publisher cleanup channel: ignoring #{ex.class}: #{ex.message}")
+      end
+      begin
+        @connection&.close
+      rescue => ex
+        EventHub.logger.warn("Publisher cleanup connection: ignoring #{ex.class}: #{ex.message}")
+      end
+    end
+
+    private
+
+    def ensure_channel
+      unless @connection
+        @connection = create_bunny_connection
+        @connection.start
+      end
+      return if @channel&.open?
+
+      attempts = 0
+      begin
+        @channel = @connection.create_channel
+        @channel.confirm_select(tracking: true)
+      rescue Bunny::NetworkFailure, Bunny::ChannelAlreadyClosed
+        attempts += 1
+        if attempts < 3
+          sleep 1
+          retry
+        end
+        raise
+      end
     end
   end
 end

data/lib/eventhub/actor_watchdog.rb

@@ -7,9 +7,13 @@ module EventHub
 
     finalizer :cleanup
 
+    # number of consecutive failed cycles before we raise to force a restart
+    MISSING_QUEUE_THRESHOLD = 3
+
     def initialize
       cycle = Configuration.processor[:watchdog_cycle_in_s]
       EventHub.logger.info("Watchdog is starting [cycle: #{cycle}s]...")
+      @consecutive_failures = 0
       async.start
     end
 
@@ -30,14 +34,28 @@ module EventHub
       connection = create_bunny_connection
       connection.start
 
-      EventHub::Configuration.processor[:listener_queues].each do |queue_name|
-        unless connection.queue_exists?(queue_name)
-          EventHub.logger.warn("Queue [#{queue_name}] is missing")
-          raise "Queue [#{queue_name}] is missing"
+      missing = EventHub::Configuration.processor[:listener_queues].reject do |queue_name|
+        connection.queue_exists?(queue_name)
+      end
+
+      if missing.empty?
+        @consecutive_failures = 0
+      else
+        @consecutive_failures += 1
+        EventHub.logger.warn("Watchdog: queue(s) missing #{missing.inspect} (#{@consecutive_failures}/#{MISSING_QUEUE_THRESHOLD})")
+        if @consecutive_failures >= MISSING_QUEUE_THRESHOLD
+          raise "Queue(s) missing for #{@consecutive_failures} consecutive cycles: #{missing.inspect}"
         end
       end
+    rescue Bunny::NetworkFailure, Bunny::TCPConnectionFailed, Timeout::Error => ex
+      # transient broker problems are auto-recovered by Bunny; don't fight it
+      EventHub.logger.warn("Watchdog: transient broker error #{ex.class}: #{ex.message} - skipping cycle")
     ensure
-      connection&.close
+      begin
+        connection&.close
+      rescue
+        nil
+      end
     end
   end
 end

data/lib/eventhub/base.rb

@@ -28,6 +28,32 @@ require_relative "actor_listener_amqp"
 require_relative "actor_listener_http"
 require_relative "docs_renderer"
 require_relative "processor2"
+require_relative "patches/celluloid_logger"
+
+module EventHub
+  # Format a Celluloid actor exception with the dying actor's class name
+  # so post-mortem analysis can identify which actor died.
+  #
+  # Important: inside an actor's crash flow, `Celluloid.current_actor`
+  # returns the Proxy::Cell, whose `.class` goes through method_missing /
+  # the mailbox - which can hang on a dying actor. Read the raw actor
+  # object out of the thread-local instead and walk to the subject class
+  # via instance variables (no proxy round-trips).
+  def self.format_celluloid_exception(ex)
+    actor_name = begin
+      actor = Thread.current[:celluloid_actor]
+      if actor
+        behavior = actor.instance_variable_get(:@behavior)
+        subject = behavior&.instance_variable_get(:@subject)
+        subject&.class&.name
+      end
+    rescue
+      nil
+    end
+    prefix = actor_name ? "[#{actor_name}] " : ""
+    "#{prefix}Exception occured: #{ex.class}: #{ex.message}"
+  end
+end
 
 Celluloid.logger = nil
-Celluloid.exception_handler { |ex| EventHub.logger.error "Exception occured: #{ex}" }
+Celluloid.exception_handler { |ex| EventHub.logger.error(EventHub.format_celluloid_exception(ex)) }

data/lib/eventhub/docs_renderer.rb

@@ -162,7 +162,9 @@ module EventHub
     def config_to_html_table(hash, depth = 0, prefix = "")
       rows = hash.map do |key, value|
         full_key = prefix.empty? ? key.to_s : "#{prefix}.#{key}"
-        if depth == 0 && value.is_a?(Hash) && !value.empty?
+        if sensitive_key?(key) && !value.nil? && !(value.respond_to?(:empty?) && value.empty?)
+          "<tr><td class=\"config-key\">#{ERB::Util.html_escape(full_key)}</td><td>#{redacted_html}</td></tr>"
+        elsif depth == 0 && value.is_a?(Hash) && !value.empty?
           "<tr class=\"is-section is-section-top\"><td colspan=\"2\"><strong>#{ERB::Util.html_escape(full_key)}</strong></td></tr>\n" \
           "#{config_to_html_table(value, 1, full_key)}"
         elsif value.is_a?(Hash) && value.empty?
@@ -178,9 +180,7 @@ module EventHub
         elsif value.is_a?(Array)
           format_array_rows(full_key, key, value, depth)
         else
-          display_value = if sensitive_key?(key)
-            "<span class=\"redacted\">***</span>"
-          elsif value.nil? || value.to_s.strip.empty?
+          display_value = if value.nil? || value.to_s.strip.empty?
             "<span class=\"not-set\">(not set)</span>"
           else
             ERB::Util.html_escape(value.to_s)
@@ -200,7 +200,7 @@ module EventHub
       return "<tr><td class=\"config-key\">#{ERB::Util.html_escape(full_key)}</td><td><span class=\"not-set\">(empty)</span></td></tr>" if array.empty?
 
       if sensitive_key?(key)
-        return "<tr><td class=\"config-key\">#{ERB::Util.html_escape(full_key)}</td><td><span class=\"redacted\">***</span></td></tr>"
+        return "<tr><td class=\"config-key\">#{ERB::Util.html_escape(full_key)}</td><td>#{redacted_html}</td></tr>"
       end
 
       inner = array.map { |item| format_array_item(item) }.join("\n")
@@ -210,7 +210,7 @@ module EventHub
     def format_array_item(item)
       if item.is_a?(Hash)
         rows = item.map do |k, v|
-          value = format_nested_value(v)
+          value = sensitive_key?(k) ? redacted_html : format_nested_value(v)
           "<tr><td>#{ERB::Util.html_escape(k)}</td><td>#{value}</td></tr>"
         end.join
         "<li><table class=\"table is-bordered is-narrow config-subtable\">#{rows}</table></li>"
@@ -225,7 +225,8 @@ module EventHub
     def format_nested_value(value)
       if value.is_a?(Hash)
         rows = value.map do |k, v|
-          "<tr><td>#{ERB::Util.html_escape(k)}</td><td>#{format_nested_value(v)}</td></tr>"
+          inner = sensitive_key?(k) ? redacted_html : format_nested_value(v)
+          "<tr><td>#{ERB::Util.html_escape(k)}</td><td>#{inner}</td></tr>"
         end.join
         "<table class=\"table is-bordered is-narrow config-subtable\">#{rows}</table>"
       elsif value.is_a?(Array)
@@ -238,6 +239,10 @@ module EventHub
       end
     end
 
+    def redacted_html
+      "<span class=\"redacted\">***</span>"
+    end
+
     def compact_hash?(hash)
       hash.values.all? do |v|
         if v.is_a?(Hash)

data/lib/eventhub/helper.rb

@@ -18,6 +18,11 @@ module EventHub
     end
 
     def create_bunny_connection
+      connection_string, connection_properties = bunny_connection_options
+      Bunny.new(connection_string, connection_properties)
+    end
+
+    def bunny_connection_options
       server = EventHub::Configuration.server
 
       protocol = "amqp"
@@ -31,8 +36,21 @@ module EventHub
         connection_properties[:logger] = Logger.new(File::NULL)
       end
 
-      # we don't need it since reactors can deal with it
-      connection_properties[:automatically_recover] = false
+      # Bunny's network recovery: re-establish connection, re-open channels,
+      # re-declare queues, and re-register consumers transparently after a
+      # broker disconnect (heartbeat miss, broker restart, LB drop).
+      connection_properties[:automatically_recover] = true
+      connection_properties[:network_recovery_interval] = 5
+      connection_properties[:recovery_attempts] = nil
+      connection_properties[:continuation_timeout] = 15_000
+
+      # Belt-and-suspenders: if recovery_attempts is ever capped, escalate to
+      # a Celluloid actor restart instead of going silent.
+      connection_properties[:recovery_attempts_exhausted] = lambda do
+        EventHub.logger.error("Bunny recovery attempts exhausted - actor will restart")
+        actor = Celluloid::Actor[:actor_listener_amqp]
+        actor&.async&.restart
+      end
 
       # do we do tls?
       if server[:tls]
@@ -45,8 +63,7 @@ module EventHub
       end
 
       connection_string = "#{protocol}://#{server[:host]}:#{server[:port]}"
-
-      Bunny.new(connection_string, connection_properties)
+      [connection_string, connection_properties]
     end
 
     # Formats stamp into UTC format

data/lib/eventhub/patches/celluloid_logger.rb

@@ -0,0 +1,51 @@
+# EventHub patches for upstream gems.
+#
+# Celluloid 0.18 (the last released version, ~2016) is incompatible with
+# Ruby 3.x frozen-string-literal defaults. Its `Internals::Logger.crash`
+# mutates string literals like:
+#
+#     def crash(string, exception)
+#       if Celluloid.log_actor_crashes
+#         string << "\n" << format_exception(exception)   # FrozenError under Ruby 3.x
+#         error string
+#       end
+#       @exception_handlers.each { |h| h.call(exception) }
+#     end
+#
+# The `string << ...` raises FrozenError BEFORE the registered exception
+# handlers fire. The actor thread then dies silently:
+#   * no exit event is sent to the supervisor (no restart),
+#   * no exit event is sent to linked sub-actors (they stay alive as zombies),
+#   * no error is logged anywhere.
+#
+# Externally the symptom is: an actor whose method raises (e.g. our
+# `ActorListenerAmqp#restart` raising "Listener amqp is restarting...") appears
+# to be entering the raise but never actually dies, and the listener never
+# gets restarted. We hit this in 1.28.0 testing: SIGHUP looked like it
+# worked (Configuration reloaded, async.restart enqueued, restart entered)
+# but the listener silently became a zombie.
+#
+# Upstream fix is unlikely - Celluloid is unmaintained. We prepend a corrected
+# `crash` that defrosts the input string before mutating it. Behavior is
+# otherwise identical to the original.
+module EventHub
+  module Patches
+    module CelluloidLoggerCrash
+      def crash(string, exception)
+        message = +String(string)
+        if Celluloid.log_actor_crashes
+          message << "\n" << format_exception(exception)
+          error message
+        end
+
+        @exception_handlers.each do |handler|
+          handler.call(exception)
+        rescue => ex
+          error(+"EXCEPTION HANDLER CRASHED:\n" << format_exception(ex))
+        end
+      end
+    end
+  end
+end
+
+Celluloid::Internals::Logger.singleton_class.prepend(EventHub::Patches::CelluloidLoggerCrash)

data/lib/eventhub/processor2.rb

@@ -61,6 +61,11 @@ module EventHub
     # pass message as string like: '{ "header": ... , "body": { .. }}'
     # and optionally exchange_name: 'your exchange name'
     def publish(args = {})
+      # capture caller-thread thread-local before the cross-actor hop;
+      # CorrelationId.current is nil inside the publisher actor's thread
+      if CorrelationId.current && !args[:correlation_id]
+        args = args.merge(correlation_id: CorrelationId.current)
+      end
       Celluloid::Actor[:actor_listener_amqp].publish(args)
     rescue => error
       EventHub.logger.error("Unexpected exeption while publish: #{error}")

data/lib/eventhub/version.rb

@@ -1,3 +1,3 @@
 module EventHub
-  VERSION = "1.27.2".freeze
+  VERSION = "1.28.0".freeze
 end

data/soak/README.md

@@ -0,0 +1,72 @@
+## Soak / reliability harness
+
+This folder contains a multi-process chaos test for the `eventhub-processor2`
+gem. Four programs cooperate so that any reliability gap (lost messages,
+zombie consumers, stuck publisher, ...) shows up as an orphan file in
+`soak/data/`.
+
+* `publisher.rb` - creates a unique file and a message, publishes to `example.outbound`
+* `router.rb` - listens on `example.outbound`, re-publishes to `example.inbound`
+* `receiver.rb` - listens on `example.inbound`, deletes the file with the given id
+* `crasher.rb` - randomly restarts RabbitMQ or sends SIGHUP to router/receiver
+
+```
+publisher => [example.outbound] => router => [example.inbound] => receiver
+```
+
+### Goal
+
+No matter what the crasher does, after a graceful shutdown and a drain
+period `soak/data/` should contain only `store.json` (and that should
+be `{}`).
+
+### Quick start (via Makefile)
+
+The project root has a `Makefile` that wraps everything:
+
+```bash
+make soak                              # 10-minute default soak, prints PASS/FAIL
+SOAK_MINUTES=30 make soak              # longer
+make soak-start                        # start all four manually
+make soak-stop                         # SIGINT them all
+make soak-clean                        # stop + wipe data/
+make help                              # list targets and env knobs
+```
+
+The `soak` target runs the chaos loop, then `SIGKILL`s the publisher
+(skipping its cleanup) so any in-flight file stays on disk as an honest
+snapshot. After draining, it counts files in `data/` excluding `store.json`
+and exits 0 if empty, 1 with the first 5 orphan ids if not.
+
+### Manual start
+
+Make sure the RabbitMQ container is running (see [docker readme](../docker/README.md)):
+
+```bash
+cd soak
+bundle exec ruby receiver.rb   # in its own terminal
+bundle exec ruby router.rb     # in its own terminal
+bundle exec ruby publisher.rb  # in its own terminal
+bundle exec ruby crasher.rb    # optional, in its own terminal
+```
+
+Graceful shutdown order: stop `publisher.rb` first, let `router`/`receiver`
+drain the queues, then stop the rest. Check `soak/data/` afterwards.
+
+### Publisher knobs (env overridable)
+
+* `PAUSE_BETWEEN_WORK=F` - seconds between publishes (default `0.05` ~ 20 msg/s)
+* `PUBLISH_MAX_ATTEMPTS=N` - publish retry attempts on transient Bunny errors (default `8`)
+* `PUBLISH_RETRY_DELAY_S=F` - seconds between publish retries (default `1`)
+
+The publisher uses `wait_for_confirms` (synchronous publisher confirms) and
+retries `Bunny::NetworkFailure` / `Bunny::ChannelAlreadyClosed` / `Timeout::Error`
+to bridge Bunny's channel-recovery window after a broker restart.
+
+### Notes
+
+* The publisher's `TransactionStore.cleanup` runs on graceful shutdown and
+  deletes any pending file in `store.json`. That's why the `make soak`
+  target uses `SIGKILL` for the publisher at the end - you want to see what
+  was actually in flight at the moment the chaos phase ended.
+* Watch for huge log files in `logs/ruby/` during long runs.

data/soak/check_orphans.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# Distinguish real orphans from in-flight-at-SIGKILL artifacts.
+#
+# After the publisher is SIGKILLed, three states can exist in data/:
+#   * file + UUID in store.json   - publisher was mid-transaction; expected residual.
+#                                    A real restart would cleanup via TransactionStore.
+#   * file + UUID NOT in store    - publisher confirmed delivery but receiver never
+#                                    deleted the file. This is a real pipeline loss.
+#   * no file + UUID in store     - delivery completed despite SIGKILL; harmless.
+#
+# We treat only the middle case as a soak FAILURE.
+#
+# Usage: soak/check_orphans.rb <data_dir>
+# Output (single line): "<real_orphan_count> <in_flight_count>"
+
+require "json"
+
+data_dir = ARGV[0] || "soak/data"
+store_path = File.join(data_dir, "store.json")
+in_flight = begin
+  JSON.parse(File.read(store_path))
+rescue
+  {}
+end.keys
+
+files = Dir.glob(File.join(data_dir, "*.json"))
+  .map { |f| File.basename(f, ".json") }
+  .reject { |id| id == "store" }
+
+real_orphans = files - in_flight
+
+puts "#{real_orphans.size} #{in_flight.size}"
+
+if ARGV.include?("--list-orphans") && !real_orphans.empty?
+  warn "first 5 real orphan ids:"
+  real_orphans.first(5).each { |id| warn "      #{id}" }
+end

data/{example → soak}/publisher.rb

@@ -9,10 +9,16 @@ require_relative "../lib/eventhub/sleeper"
 SIGNALS_FOR_TERMINATION = [:INT, :TERM, :QUIT]
 SIGNALS_FOR_RELOAD_CONFIG = [:HUP]
 ALL_SIGNALS = SIGNALS_FOR_TERMINATION + SIGNALS_FOR_RELOAD_CONFIG
-PAUSE_BETWEEN_WORK = 0.05 # default is 0.05
+
+# Tunables (override via env). Defaults are sized to bridge a typical
+# Bunny channel-recovery window (~network_recovery_interval = 5s) plus
+# reconnect, so a broker restart doesn't leak in-flight publishes.
+PAUSE_BETWEEN_WORK = Float(ENV.fetch("PAUSE_BETWEEN_WORK", "0.05"))
+PUBLISH_MAX_ATTEMPTS = Integer(ENV.fetch("PUBLISH_MAX_ATTEMPTS", "8"))
+PUBLISH_RETRY_DELAY_S = Float(ENV.fetch("PUBLISH_RETRY_DELAY_S", "1"))
 
 Celluloid.logger = nil
-Celluloid.exception_handler { |ex| Publisher.logger.error "Exception occured: #{ex}}" }
+Celluloid.exception_handler { |ex| Publisher.logger.error "Exception occured: #{ex.class}: #{ex.message}" }
 Celluloid.boot
 
 # Publisher module
@@ -110,14 +116,24 @@ module Publisher
         sleep PAUSE_BETWEEN_WORK
       end
     ensure
-      @connection&.close
+      begin
+        @connection&.close
+      rescue => ex
+        Publisher.logger.warn("Worker connection close: ignoring #{ex.class}: #{ex.message}")
+      end
     end
 
     private
 
     def connect
       @connection = Bunny.new(vhost: "event_hub",
-        automatic_recovery: false,
+        # match the gem's recovery defaults so Bunny re-opens the channel
+        # and re-declares the exchange after a broker restart, without
+        # crashing the Worker actor.
+        automatically_recover: true,
+        network_recovery_interval: 5,
+        recovery_attempts: nil,
+        continuation_timeout: 15_000,
         logger: Logger.new(File::NULL))
       @connection.start
       @channel = @connection.create_channel
@@ -131,14 +147,35 @@ module Publisher
       file_name = "data/#{id}.json"
       data = {body: {id: id}}.to_json
 
-      # start transaction...
+      # start transaction (durable on disk via store.json)
       Celluloid::Actor[:transaction_store].start(id)
       File.write(file_name, data)
       Publisher.logger.info("[#{id}] - Message/File created")
 
-      @exchange.publish(data, persistent: true)
+      # Bridge Bunny's channel-recovery window: when the broker is bouncing,
+      # the channel needs ~network_recovery_interval seconds to reopen and
+      # publishes during that window raise ChannelAlreadyClosed. Retry a few
+      # times with backoff so the message survives a broker hiccup. After
+      # PUBLISH_MAX_ATTEMPTS, leave it pending on disk for an external sweep.
+      attempts = 0
+      begin
+        attempts += 1
+        @exchange.publish(data, persistent: true)
+        unless @channel.wait_for_confirms
+          Publisher.logger.warn("[#{id}] - broker nacked; leaving pending")
+          return
+        end
+      rescue Bunny::NetworkFailure, Bunny::ChannelAlreadyClosed, Timeout::Error => ex
+        if attempts < PUBLISH_MAX_ATTEMPTS
+          sleep PUBLISH_RETRY_DELAY_S
+          retry
+        end
+        Publisher.logger.warn("[#{id}] - publish failed after #{attempts} attempts (#{ex.class}: #{ex.message}); leaving pending")
+        return
+      end
+
       Celluloid::Actor[:transaction_store]&.stop(id)
-      Publisher&.logger&.info("[#{id}] - Message sent")
+      Publisher.logger.info("[#{id}] - Message sent")
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eventhub-processor2
 version: !ruby/object:Gem::Version
-  version: 1.27.2
+  version: 1.28.0
 platform: ruby
 authors:
 - Steiner, Thomas
@@ -179,6 +179,7 @@ files:
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
+- Makefile
 - README.md
 - Rakefile
 - bin/console
@@ -190,16 +191,9 @@ files:
 - docker/rabbitmq.config
 - docker/reset
 - eventhub-processor2.gemspec
-- example/CHANGELOG.md
 - example/README.md
 - example/config/example.json
-- example/config/receiver.json
-- example/config/router.json
-- example/crasher.rb
 - example/example.rb
-- example/publisher.rb
-- example/receiver.rb
-- example/router.rb
 - lib/eventhub/actor_heartbeat.rb
 - lib/eventhub/actor_listener_amqp.rb
 - lib/eventhub/actor_listener_http.rb
@@ -220,11 +214,21 @@ files:
 - lib/eventhub/helper.rb
 - lib/eventhub/logger.rb
 - lib/eventhub/message.rb
+- lib/eventhub/patches/celluloid_logger.rb
 - lib/eventhub/processor2.rb
 - lib/eventhub/sleeper.rb
 - lib/eventhub/statistics.rb
 - lib/eventhub/templates/layout.erb
 - lib/eventhub/version.rb
+- soak/CHANGELOG.md
+- soak/README.md
+- soak/check_orphans.rb
+- soak/config/receiver.json
+- soak/config/router.json
+- soak/crasher.rb
+- soak/publisher.rb
+- soak/receiver.rb
+- soak/router.rb
 homepage: https://github.com/thomis/eventhub-processor2
 licenses:
 - MIT
@@ -243,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Next generation gem to build ruby based eventhub processor
 test_files: []