sneakers-queue-migrator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fc0b131523f71a4bd19d4d5d0275fbe7e47814e55ad51b0d927b2672ed7578cb
+  data.tar.gz: 3ee9adb466b69696575011fd7a4a192cac88f9370226b33566cb07a084a637f8
+SHA512:
+  metadata.gz: 663174974230df88b4c71ed747c682c5329347d5c815087fd0c59cfa93d05c2e05d8dd196d1345a05b2c4e787850513fc40e727e5a0a7ba61dd63ed0d63677d1
+  data.tar.gz: b4023d485167c76211c06ba6765c550589fceaae77e1c6ed47d91a22d6d228a31b673168d06429a8fbc2b207e94f8845b06243b13386a71e3dc607403058f724

data/.rubocop.yml

@@ -0,0 +1,39 @@
+AllCops:
+  TargetRubyVersion: 3.1
+  SuggestExtensions: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Style/Documentation:
+  Enabled: false
+
+Style/MultilineBlockChain:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in sneakers-queue-migrator.gemspec
+gemspec
+
+gem "rake", ">= 12.0"
+
+gem "minitest", "~> 5.16"
+
+gem "rubocop", "~> 1.21"
+
+gem "sneakers", "~> 2.12"

data/Gemfile.lock

@@ -0,0 +1,76 @@
+PATH
+  remote: .
+  specs:
+    sneakers-queue-migrator (0.1.0)
+      bunny (~> 2.0)
+      json (~> 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    amq-protocol (2.3.4)
+    ast (2.4.3)
+    bunny (2.24.0)
+      amq-protocol (~> 2.3)
+      sorted_set (~> 1, >= 1.0.2)
+    concurrent-ruby (1.3.5)
+    json (2.10.2)
+    language_server-protocol (3.17.0.4)
+    lint_roller (1.1.0)
+    minitest (5.16.3)
+    parallel (1.26.3)
+    parser (3.3.7.4)
+      ast (~> 2.4.1)
+      racc
+    prism (1.4.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (12.3.3)
+    rbtree (0.4.6)
+    regexp_parser (2.10.0)
+    rubocop (1.75.2)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.44.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    serverengine (2.1.1)
+      sigdump (~> 0.2.2)
+    set (1.1.2)
+    sigdump (0.2.5)
+    sneakers (2.12.0)
+      bunny (~> 2.14)
+      concurrent-ruby (~> 1.0)
+      rake (~> 12.3)
+      serverengine (~> 2.1.0)
+      thor
+    sorted_set (1.0.3)
+      rbtree
+      set (~> 1.0)
+    thor (1.3.2)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  minitest (~> 5.16)
+  rake (>= 12.0)
+  rubocop (~> 1.21)
+  sneakers (~> 2.12)
+  sneakers-queue-migrator!
+
+BUNDLED WITH
+   2.6.3

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# Sneakers Queue Migrator
+
+A Ruby gem to safely migrate RabbitMQ queues when queue arguments change, with dynamic Sneakers subscriber discovery, queue argument comparison, and safe message shoveling.
+
+## Features
+- Discovers all Sneakers subscribers dynamically
+- Compares current and desired queue arguments using the RabbitMQ Management API
+- Safely migrates messages between queues using Bunny
+- Exposes a class-based API for integration
+- Optionally provides a Rake task for Rails apps
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sneakers-queue-migrator'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install sneakers-queue-migrator
+
+## Usage
+
+You need the sneakers or kicks gem for this to work
+
+```ruby
+require 'sneakers/migrator'
+
+# Example usage (API subject to change)
+Sneakers::Migrator.migrate!(
+    amqp_url: 'amqp://user:password@rabbitmq:5672/',
+    amqp_api_url: 'http://user:password@rabbitmq:15672',
+    subscriber_paths: ['app/workers/**/*.rb']
+)
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests.
+
+## Contributing
+Bug reports and pull requests are welcome!
+
+## License
+MIT

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+Minitest::TestTask.create
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3.8'
+services:
+  rabbitmq:
+    image: rabbitmq:4.1-management
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    environment:
+      RABBITMQ_DEFAULT_USER: guest
+      RABBITMQ_DEFAULT_PASS: guest
+      RABBITMQ_DEFAULT_VHOST: test_vhost
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "ping"]
+      interval: 5s
+      timeout: 5s
+      retries: 10

data/lib/sneakers/migrator/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sneakers
+  module Migrator
+    VERSION = "0.1.0"
+  end
+end

data/lib/sneakers/migrator.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require_relative "migrator/version"
+require "bunny"
+require "json"
+require "net/http"
+require "uri"
+
+module Sneakers
+  # Do not define a module named Queue here to avoid conflict with Sneakers::Queue (which is a class, not a module)
+  module Migrator
+    class Error < StandardError; end
+
+    class << self
+      # Main migration entrypoint
+      def migrate!(amqp_url:, amqp_api_url:, subscriber_paths: [], logger: nil)
+        logger ||= $stdout
+        # Try to require sneakers or kicks
+        begin
+          require "sneakers"
+        rescue LoadError
+          begin
+            require "kicks"
+          rescue LoadError
+            raise Error, "You must have either the 'sneakers' or 'kicks' gem installed to use subscriber discovery."
+          end
+        end
+        # Load all subscribers
+        subscriber_paths.each do |path|
+          Dir[File.join(path, "*.rb")].each { |f| require f }
+        end
+        # Find all worker classes for the loaded framework
+        subscribers = ObjectSpace.each_object(Class).select do |klass|
+          klass.included_modules.include?(::Sneakers::Worker)
+        rescue StandardError
+          false
+        end
+        queues = subscribers.map do |klass|
+          opts = klass.instance_variable_get(:@queue_opts) || {}
+          name = klass.instance_variable_get(:@queue_name) ||
+                 (klass.const_defined?(:QUEUE_NAME) ? klass.const_get(:QUEUE_NAME) : nil) ||
+                 klass.name.gsub("::", "_").gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
+          {
+            name: name,
+            exchange: opts[:exchange] || "domain_events",
+            routing_keys: opts[:routing_key] || [],
+            arguments: opts[:arguments] || {},
+            durable: opts.fetch(:durable, true)
+          }.tap { |q| q[:arguments]["x-queue-type"] ||= "classic" }
+        end.uniq { |q| q[:name] }
+
+        conn = Bunny.new(amqp_url)
+        conn.start
+        ch = conn.create_channel
+
+        # Remove unused api_uri and fix variable usage
+        rabbitmq_api_user = URI.parse(amqp_api_url).user
+        rabbitmq_api_pass = URI.parse(amqp_api_url).password
+        all_queues = fetch_all_queues(amqp_api_url, rabbitmq_api_user, rabbitmq_api_pass)
+
+        queues.each do |q|
+          queue_info = all_queues.find { |info| info["name"] == q[:name] }
+          tmp_queue_name = "#{q[:name]}_tmp"
+          if queue_info.nil?
+            logger.puts "Migrator: Queue #{q[:name]} does not exist, creating..."
+            ch.queue(q[:name], durable: q[:durable], arguments: q[:arguments])
+            next
+          end
+          current_args = (queue_info["arguments"] || {}).transform_keys(&:to_s)
+          desired_args = (q[:arguments] || {}).transform_keys(&:to_s)
+          logger.puts "Migrator: Checking queue: #{q[:name]}"
+          logger.puts "Migrator: Current args: #{current_args.inspect}"
+          logger.puts "Migrator: Desired args: #{desired_args.inspect}"
+          if queue_args_match?({ arguments: current_args }, desired_args)
+            logger.puts "Migrator: Queue #{q[:name]} arguments match, nothing to do."
+            next
+          end
+          logger.puts "Migrator: Queue #{q[:name]} arguments differ, migrating..."
+
+          # --- Fetch and store bindings (exchange/routing_key pairs) ---
+          bindings = fetch_queue_bindings(amqp_api_url, rabbitmq_api_user, rabbitmq_api_pass, queue_info)
+          logger.puts "Migrator: Found bindings: #{bindings.inspect}"
+
+          ch.queue(tmp_queue_name, durable: queue_info["durable"], arguments: desired_args)
+          Array(q[:routing_keys]).each do |rk|
+            next if rk.nil? || rk.empty?
+
+            # bind the tmp queue first so new messages go somewhere
+            ch.queue(tmp_queue_name).bind(q[:exchange], routing_key: rk)
+            # unbind the original queue from the exchange
+            logger.puts "Migrator: Unbinding queue #{q[:name]} from exchange #{q[:exchange]} with routing key #{rk}"
+            ch.queue(q[:name], passive: true).unbind(q[:exchange], routing_key: rk)
+          end
+          count = 0
+          queue = ch.queue(q[:name], passive: true)
+          loop do
+            delivery_info, headers, payload = queue.pop
+            break unless payload
+
+            logger.puts "Migrator: Moving message to #{tmp_queue_name}: payload=#{payload.inspect}, headers=#{headers.inspect}, delivery_info=#{delivery_info.inspect}"
+            safe_basic_publish(ch, payload, tmp_queue_name, headers)
+            count += 1
+          end
+          logger.puts "Migrator: Moved #{count} messages to #{tmp_queue_name}"
+          ch.queue(q[:name], passive: true).delete
+
+          # create new queue with desired arguments
+          logger.puts "Migrator: Recreating queue #{q[:name]} with new arguments: #{q[:arguments].inspect}"
+          ch.queue(q[:name], durable: q[:durable], arguments: q[:arguments])
+
+          # --- Restore bindings ---
+          bindings.each do |binding|
+            # Only restore if it's a binding from an exchange (not from a queue)
+            next unless binding["source"] && !binding["source"].empty?
+
+            ch.queue(q[:name]).bind(binding["source"], routing_key: binding["routing_key"])
+            logger.puts "Migrator: Restored binding: exchange=#{binding["source"]}, routing_key=#{binding["routing_key"]}"
+            # now drop the tmp queue binding
+            ch.queue(tmp_queue_name).unbind(binding["source"], routing_key: binding["routing_key"])
+          rescue StandardError => e
+            logger.puts "Migrator: Failed to restore binding: #{e.class}: #{e.message}"
+          end
+
+          count = 0
+          tmp_queue = ch.queue(tmp_queue_name, passive: true)
+          loop do
+            delivery_info, headers, payload = tmp_queue.pop
+            break unless payload
+
+            logger.puts "Migrator: Moving message back to #{q[:name]}: payload=#{payload.inspect}, headers=#{headers.inspect}, delivery_info=#{delivery_info.inspect}"
+            safe_basic_publish(ch, payload, q[:name], headers)
+            count += 1
+          end
+          logger.puts "Migrator: Moved #{count} messages to #{q[:name]}"
+          ch.queue(tmp_queue_name, passive: true).delete
+          logger.puts "Migrator: Migration for #{q[:name]} complete."
+        end
+        ch.close
+        conn.close
+      end
+
+      # Fetches all bindings for a queue using the RabbitMQ Management API
+      def fetch_queue_bindings(api_url, user, pass, queue_info)
+        vhost = queue_info["vhost"] || "/"
+        queue_name = queue_info["name"]
+        uri = URI.parse(api_url)
+        scheme = uri.scheme || "http"
+        host = uri.host
+        port = uri.port || 15_672
+        vhost_enc = URI.encode_www_form_component(vhost)
+        queue_enc = URI.encode_www_form_component(queue_name)
+        url = URI.parse("#{scheme}://#{host}:#{port}/api/queues/#{vhost_enc}/#{queue_enc}/bindings")
+        http = Net::HTTP.new(url.hostname, url.port)
+        http.use_ssl = (scheme == "https")
+        req = Net::HTTP::Get.new(url)
+        req.basic_auth(user, pass)
+        res = http.request(req)
+        return [] unless res.is_a?(Net::HTTPSuccess)
+
+        JSON.parse(res.body)
+      end
+
+      private
+
+      def fetch_all_queues(api_url, user, pass)
+        uri = URI.parse(api_url)
+        scheme = uri.scheme || "http"
+        host = uri.host
+        port = uri.port || 15_672
+        url = URI.parse("#{scheme}://#{host}:#{port}/api/queues")
+        http = Net::HTTP.new(url.hostname, url.port)
+        http.use_ssl = (scheme == "https")
+        req = Net::HTTP::Get.new(url)
+        req.basic_auth(user, pass)
+        res = http.request(req)
+        return [] unless res.is_a?(Net::HTTPSuccess)
+
+        JSON.parse(res.body)
+      end
+
+      def queue_args_match?(queue_info, desired_args)
+        (desired_args.to_a - queue_info[:arguments].to_a).empty? &&
+          (queue_info[:arguments].to_a - desired_args.to_a).empty?
+      end
+
+      def safe_basic_publish(channel, payload, queue_name, headers)
+        # Bunny's queue.pop returns [payload, properties, delivery_info]
+        # The payload is always the message body as a string or nil
+        # The properties hash contains all AMQP properties
+        amqp_property_keys = %i[routing_key persistent mandatory timestamp expiration type reply_to content_type content_encoding correlation_id priority message_id user_id app_id]
+        # Normalize headers to a Hash if it's an Array (Bunny can return either)
+        props = case headers
+                when Hash then headers
+                when Array then headers.to_h
+                else {}
+                end
+        sanitized_properties = props.select { |k, _| amqp_property_keys.include?(k.to_sym) }
+        # Always pass the payload as-is (should be a string or nil)
+        queue = channel.queue(queue_name, passive: true)
+        if sanitized_properties.any?
+          queue.publish(payload, sanitized_properties)
+        else
+          queue.publish(payload)
+        end
+      end
+    end
+  end
+end

data/sneakers-queue-migrator.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "lib/sneakers/migrator/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "sneakers-queue-migrator"
+  spec.version = Sneakers::Migrator::VERSION
+  spec.authors = ["Mike Gane"]
+  spec.email = ["mike.gane@nexusmods.com"]
+
+  spec.summary = "Ruby sneakers queue migrator for RabbitMQ when you need to change queue arguments."
+  spec.description = "A migrator that handles changing RabbitMQ queue arguments for Sneakers subscribers, ensuring smooth transitions without data loss."
+  spec.homepage = "https://gitlab.nexdev.uk/nexus-mods/public/sneakers-queue-migrator"
+  spec.required_ruby_version = ">= 3.1.0"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) || f.start_with?(
+        "bin/",
+        "test/",
+        "spec/",
+        "features/",
+        ".git",
+        ".circleci",
+        "appveyor"
+      )
+    end
+  end
+
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "bunny", "~> 2.0"
+  spec.add_dependency "json", "~> 2.0"
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: sneakers-queue-migrator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Mike Gane
+bindir: exe
+cert_chain: []
+date: 2025-07-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: A migrator that handles changing RabbitMQ queue arguments for Sneakers
+  subscribers, ensuring smooth transitions without data loss.
+email:
+- mike.gane@nexusmods.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- docker-compose.yml
+- lib/sneakers/migrator.rb
+- lib/sneakers/migrator/version.rb
+- sneakers-queue-migrator.gemspec
+homepage: https://gitlab.nexdev.uk/nexus-mods/public/sneakers-queue-migrator
+licenses: []
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.3
+specification_version: 4
+summary: Ruby sneakers queue migrator for RabbitMQ when you need to change queue arguments.
+test_files: []