sneakers-queue-migrator 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f911a0cec356d8603bbe2b50353493d9699ff706d48791f823bdd49361386b88
-  data.tar.gz: 559e63d51d1619eb0f2b1b729facef58eafc66727a650f378fd321d9d57f2b17
+  metadata.gz: 07b21432931960e830cbb956ab0e53406657147d9674f495ef74237f114e6fc3
+  data.tar.gz: aa60dd0d5714f56410240567c0a63da1fa937eefd6d3ad706bdb8a6b35cd2bdc
 SHA512:
-  metadata.gz: db62556ef5ab33c39279ba327d4b8ce8638648a574d0716e7dcb20a71d13450940e4345796ee62421c55572c9938ad1bc5970368f76cba57d764016a7d1f4d34
-  data.tar.gz: 6cbced20f4da4e6b45f7c4cb9d3b302aabec96bbd15cc9d9b405927e719bde2803aeaf650a576118617e980e6818bdbb7ebe0941e882d397f136c5eaf98ece8d
+  metadata.gz: af7f10c370532ceb2f76d908bd86fc5d5743cb1c6b491de59ef2eb3fd6749b19ccd6da1a44368c398f51281ce6440e81abd3aa6fb5d3a2f7110de84d5507b99a
+  data.tar.gz: 772f61353f22feea63646cebe38eaf2c6f0c2864a0041b74f54a6eba88cbf01a2c569780b3ac379899c61b6d8137253187e1376f02bfeeac53bf0a707f9241af

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sneakers-queue-migrator (0.1.3)
+    sneakers-queue-migrator (0.1.5)
       bunny (~> 2.0)
       json (~> 2.0)
 

data/README.md

@@ -35,10 +35,16 @@ require 'sneakers/migrator'
 Sneakers::Migrator.migrate!(
     amqp_url: 'amqp://user:password@rabbitmq:5672/',
     amqp_api_url: 'http://user:password@rabbitmq:15672',
-    subscriber_paths: ['app/workers']
+    subscriber_paths: ['app/workers'],
+    load_with_regex: false
 )
 ```
 
+`amqp_url` - regular bunny connection string
+`amqp_api_url` - url WITH PORT for the management interface
+`subscriber_paths` - array of folders to scan for subscribers
+`load_with_regex` - `false` requires the files and reflects the arguments, `true` parses the from_queue block with regex
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests.

data/lib/sneakers/migrator/version.rb

@@ -2,6 +2,6 @@
 
 module Sneakers
   module Migrator
-    VERSION = "0.1.3"
+    VERSION = "0.1.5"
   end
 end

data/lib/sneakers/migrator.rb

@@ -13,19 +13,38 @@ module Sneakers
 
     class << self
       # Main migration entrypoint
-      def migrate!(amqp_url:, amqp_api_url:, subscriber_paths: [], logger: nil)
+      def migrate!(amqp_url:, amqp_api_url:, subscriber_paths: [], logger: nil, load_with_regex: false)
         logger ||= $stdout
-        # Try to require sneakers or kicks
-        begin
-          require "sneakers"
-        rescue LoadError
-          begin
-            require "kicks"
-          rescue LoadError
-            raise Error, "You must have either the 'sneakers' or 'kicks' gem installed to use subscriber discovery."
+        # Optionally parse subscribers with regex/static analysis
+        queues = if load_with_regex
+                   parse_queues_from_files(subscriber_paths, logger)
+                 else
+                   load_files_to_find_queues(subscriber_paths, logger)
+                 end
+
+        conn = Bunny.new(amqp_url)
+        conn.start
+        ch = conn.create_channel
+
+        # Remove unused api_uri and fix variable usage
+        rabbitmq_api_user = URI.parse(amqp_api_url).user
+        rabbitmq_api_pass = URI.parse(amqp_api_url).password
+        all_queues = fetch_all_queues(amqp_api_url, rabbitmq_api_user, rabbitmq_api_pass)
+
+        queues.each do |q|
+          existing_queue_info = all_queues.find { |info| info["name"] == q[:name] }
+          if existing_queue_info.nil?
+            logger.puts "Migrator: Queue #{q[:name]} does not exist, creating..."
+            ch.queue(q[:name], durable: q[:durable], arguments: q[:arguments])
+            next
           end
+          migrate_queue!(q, ch, amqp_api_url, rabbitmq_api_user, rabbitmq_api_pass, logger, existing_queue_info)
         end
+      end
 
+      private
+
+      def load_files_to_find_queues(subscriber_paths, logger)
         # Load all subscribers (recursively)
         subscriber_paths.each do |path|
           Dir[File.join(path, "**", "*.rb")].sort.each do |f|
@@ -51,10 +70,10 @@ module Sneakers
         logger.puts "Migrator: Found #{subscribers.size} subscribers:"
         subscribers.each { |s| logger.puts "  - #{s.name}" }
 
-        queues = subscribers.map do |klass|
+        subscribers.map do |klass|
           opts = klass.instance_variable_get(:@queue_opts) || {}
-          name = klass.instance_variable_get(:@queue_name) ||
-                 (klass.const_defined?(:QUEUE_NAME) ? klass.const_get(:QUEUE_NAME) : nil) ||
+          name = klass.instance_variable_get(:@queue_name) || \
+                 (klass.const_defined?(:QUEUE_NAME) ? klass.const_get(:QUEUE_NAME) : nil) || \
                  klass.name.gsub("::", "_").gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
           {
             name: name,
@@ -64,28 +83,205 @@ module Sneakers
             durable: opts.fetch(:durable, true)
           }.tap { |q| q[:arguments]["x-queue-type"] ||= "classic" }
         end.uniq { |q| q[:name] }
+      end
 
-        conn = Bunny.new(amqp_url)
-        conn.start
-        ch = conn.create_channel
+      # Robustly parse all from_queue calls in .rb files under subscriber_paths
+      def parse_queues_from_files(subscriber_paths, logger)
+        require "ripper"
+        require "yaml"
+        queues = []
+        seen = {}
+        subscriber_paths.each do |path|
+          Dir[File.join(path, "**", "*.rb")].sort.each do |file|
+            logger.puts "Migrator: Parsing subscriber file: #{file}"
+            src = File.read(file)
+            # Use a robust regex to find from_queue calls (multi-line, with hash)
+            # Improved: allow multi-line, nested, and indented argument hashes
+            # Custom scan to robustly extract the full options hash or argument list (handles nested braces)
+            # Use a more robust approach: for each from_queue, extract the full argument list (multi-line, nested)
+            # Multi-line robust extraction for from_queue calls
+            src_lines = src.lines
+            src_lines.each_with_index do |line, idx|
+              # Match from_queue with or without parentheses, any quote style, and allow whitespace
+              next unless line =~ /from_queue\s*(\(|\s)+(['"])([^'"]+)\2\s*,?/
 
-        # Remove unused api_uri and fix variable usage
-        rabbitmq_api_user = URI.parse(amqp_api_url).user
-        rabbitmq_api_pass = URI.parse(amqp_api_url).password
-        all_queues = fetch_all_queues(amqp_api_url, rabbitmq_api_user, rabbitmq_api_pass)
+              queue_name = ::Regexp.last_match(3)
+              # Multi-line argument collection: collect all subsequent lines that are more indented than the from_queue line
+              from_indent = line[/^\s*/].size
+              opts_lines = []
+              j = idx + 1
+              while j < src_lines.size
+                l = src_lines[j]
+                # Stop if line is less indented or blank or looks like a new method/class/def
+                break if l.strip.empty? || l[/^\s*/].size <= from_indent || l =~ /^\s*(def |class |module )/
 
-        queues.each do |q|
-          existing_queue_info = all_queues.find { |info| info["name"] == q[:name] }
-          if existing_queue_info.nil?
-            logger.puts "Migrator: Queue #{q[:name]} does not exist, creating..."
-            ch.queue(q[:name], durable: q[:durable], arguments: q[:arguments])
-            next
+                opts_lines << l
+                j += 1
+              end
+              opts_str = opts_lines.join(" ").strip
+              puts "Migrator: Parsing queue: #{queue_name} with options: #{opts_str}"
+              opts_hash = parse_options_hash_for_file(opts_str)
+              puts "Migrator: Parsed options for #{queue_name}: #{opts_hash.inspect}"
+              routing_keys = opts_hash[:routing_key]
+              routing_keys = [routing_keys] if routing_keys && !routing_keys.is_a?(Array)
+              arguments = opts_hash[:arguments] || {}
+              # Always stringify keys deeply for arguments
+              arguments = stringify_keys_deep(arguments)
+              # Add "x-queue-type" if not present as string
+              arguments["x-queue-type"] ||= "classic"
+              # Remove any symbol keys that duplicate string keys
+              arguments.delete_if { |k, _| k.is_a?(Symbol) && arguments.key?(k.to_s) }
+
+              exchange_options = opts_hash[:exchange_options] || {}
+              exchange_options = stringify_keys_deep(exchange_options)
+              exchange_options.delete_if { |k, _| k.is_a?(Symbol) && exchange_options.key?(k.to_s) }
+
+              q = {
+                name: queue_name,
+                exchange: opts_hash[:exchange],
+                exchange_options: exchange_options,
+                routing_keys: routing_keys || [],
+                arguments: arguments,
+                durable: opts_hash.key?(:durable) ? opts_hash[:durable] : true
+              }
+              unless seen[q[:name]]
+                queues << q
+                seen[q[:name]] = true
+              end
+              queues
+            end
           end
-          migrate_queue!(q, ch, amqp_api_url, rabbitmq_api_user, rabbitmq_api_pass, logger, existing_queue_info)
         end
+        puts "Migrator: Found #{queues.size} queues to migrate:"
+        queues.each { |q| puts "  - #{q[:name]} (exchange: #{q[:exchange]}, routing_keys: #{Array(q[:routing_keys]).join(", ")}, arguments: #{q[:arguments].inspect})" }
+        queues
       end
 
-      private
+      # Recursively convert all hash keys to strings
+      def stringify_keys_deep(obj)
+        case obj
+        when Hash
+          obj.each_with_object({}) { |(k, v), h| h[k.to_s] = stringify_keys_deep(v) }
+        when Array
+          obj.map { |v| stringify_keys_deep(v) }
+        else
+          obj
+        end
+      end
+
+      # Parse a Ruby hash from a string for file-based queue declarations (robust for multi-line, symbol, string keys, nested hashes)
+      def parse_options_hash_for_file(str)
+        return {} unless str
+
+        # Remove comments and excessive whitespace
+        code = str.gsub(/#.*$/, "").gsub(/\n/, " ").gsub(/\s+/, " ")
+        # Convert Ruby %w[ ... ] and %w( ... ) to array syntax for YAML
+        code = code.gsub(/%w[\[(](.*?)[\])]/) do
+          words = ::Regexp.last_match(1).strip.split(/\s+/)
+          "[" + words.map { |w| "\"#{w}\"" }.join(", ") + "]" # rubocop:disable Style/StringConcatenation
+        end
+        # Convert Ruby hashrockets and symbol keys to YAML style, and handle symbol values and numbers with underscores
+        yaml_str = code
+                   .gsub(/:(\w+)\s*=>/, '"\\1":') # :foo =>
+                   .gsub(/(\w+):/, '"\\1":')      # foo:
+                   .gsub(/=>/, ":") # fallback for any =>
+                   .gsub(/:([\w\d_]+)/, '"\\1"') # symbol values :foo => "foo"
+                   .gsub(/([\d_]+)(?=,|\s|\})/) { |m| m.delete("_") } # numbers with underscores
+        # Ensure the string is wrapped in braces for YAML parsing
+        yaml_str = "{#{yaml_str}}" unless yaml_str.strip.start_with?("{") && yaml_str.strip.end_with?("}")
+        begin
+          # Try to parse as YAML (single argument for compatibility)
+          hash = YAML.safe_load(yaml_str) || {}
+          # Symbolize keys recursively and dedupe symbol/string keys
+          dedupe_symbol_and_string_keys_deep(symbolize_keys_deep(hash))
+        rescue StandardError => e
+          warn "Migrator: parse_options_hash_for_file YAML parse error: #{e.class}: #{e.message}"
+          # Fallback: try to extract common keys manually
+          out = {}
+          out[:exchange] = ::Regexp.last_match(1) if str =~ /(?:["']?exchange["']?\s*(?:=>|:)\s*["']?([^"'\s,}]+)["']?)/
+          out[:routing_key] = ::Regexp.last_match(1) if str =~ /(?:["']?routing_key["']?\s*(?:=>|:)\s*["']?([\w\-:]+)["']?)/
+          if str =~ /arguments:\s*\{([^}]*)\}/m
+            begin
+              args = YAML.safe_load("{#{::Regexp.last_match(1)}}") || {}
+              out[:arguments] = dedupe_symbol_and_string_keys_deep(symbolize_keys_deep(args))
+            rescue StandardError => e2
+              warn "Migrator: parse_options_hash_for_file arguments parse error: #{e2.class}: #{e2.message}"
+            end
+          end
+          if str =~ /exchange_options:\s*\{([^}]*)\}/m
+            begin
+              ex_opts = YAML.safe_load("{#{::Regexp.last_match(1)}}") || {}
+              out[:exchange_options] = dedupe_symbol_and_string_keys_deep(symbolize_keys_deep(ex_opts))
+            rescue StandardError => e3
+              warn "Migrator: parse_options_hash_for_file exchange_options parse error: #{e3.class}: #{e3.message}"
+            end
+          end
+          out
+        end
+      end
+
+      # Remove string keys if the same key exists as a symbol, recursively
+      def dedupe_symbol_and_string_keys_deep(obj)
+        case obj
+        when Hash
+          # Remove string keys if symbol key exists
+          symbol_keys = obj.keys.select { |k| k.is_a?(Symbol) }
+          obj.reject { |k, _| k.is_a?(String) && symbol_keys.include?(k.to_sym) }
+             .transform_values { |v| dedupe_symbol_and_string_keys_deep(v) }
+        when Array
+          obj.map { |v| dedupe_symbol_and_string_keys_deep(v) }
+        else
+          obj
+        end
+      end
+
+      # Recursively symbolize keys in a hash
+      def symbolize_keys_deep(obj)
+        case obj
+        when Hash
+          obj.each_with_object({}) { |(k, v), h| h[k.to_sym] = symbolize_keys_deep(v) }
+        when Array
+          obj.map { |v| symbolize_keys_deep(v) }
+        else
+          obj
+        end
+      end
+
+      # Parse a Ruby hash from a string (robust for multi-line, symbol, and string keys)
+      # This is used for parsing options from rabbitmq management API strings
+      def parse_options_hash(str)
+        return {} unless str
+
+        # Remove newlines and excessive whitespace for easier parsing
+        compact = str.gsub(/\n/, " ").gsub(/\s+/, " ")
+        # Convert Ruby symbols and hashrockets to YAML style
+        yaml_str = compact
+                   .gsub(/:(\w+)\s*=>/, '"\\1":') # :foo =>
+                   .gsub(/(\w+):/, '"\\1":')      # foo:
+                   .gsub(/=>/, ":") # fallback for any =>
+        begin
+          # Try to parse as YAML (single argument for compatibility)
+          YAML.safe_load("{#{yaml_str}}") || {}
+        rescue StandardError => e
+          warn "Migrator: parse_options_hash YAML parse error: #{e.class}: #{e.message}"
+          # Fallback: try to extract exchange and routing_key robustly (symbol or string keys, colon or hashrocket, any quotes)
+          out = {}
+          # Match exchange: "exchange" => "test", :exchange => 'test', exchange: "test", etc.
+          out[:exchange] = ::Regexp.last_match(1) if str =~ /(?:["']?exchange["']?\s*(?:=>|:)\s*["']?([^"'\s,}]+)["']?)/
+          # Match routing_key: "routing_key" => "foo", :routing_key => 'foo', routing_key: "foo", etc.
+          out[:routing_key] = ::Regexp.last_match(1) if str =~ /(?:["']?routing_key["']?\s*(?:=>|:)\s*["']?([\w\-:]+)["']?)/
+          if str =~ /arguments:\s*\{([^}]*)\}/m
+            # Try to parse arguments as a hash
+            begin
+              args = YAML.safe_load("{#{::Regexp.last_match(1)}}") || {}
+              out[:arguments] = args
+            rescue StandardError => e2
+              warn "Migrator: parse_options_hash arguments parse error: #{e2.class}: #{e2.message}"
+            end
+          end
+          out
+        end
+      end
 
       def migrate_queue!(q, ch, amqp_api_url, rabbitmq_api_user, rabbitmq_api_pass, logger, existing_queue_info)
         tmp_queue_name = "#{q[:name]}_tmp"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: sneakers-queue-migrator
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Mike Gane
 bindir: exe
 cert_chain: []
-date: 2025-07-03 00:00:00.000000000 Z
+date: 2025-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny