postburner 1.0.0.pre.5 → 1.0.0.pre.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38e2982c4478f99aa0d19c58ac7455cdf82ff1c3707a154d1dbd0c3a1546ceb8
-  data.tar.gz: 0d17bb9da882d8e1cf2cce7c1cddd508ddbc7531c037dd56b7e0c6d34bd40f5b
+  metadata.gz: 1d37676412b662bd876f2d2d09ac57e09876e5f0d8a7186c37f4f71f3d40ad89
+  data.tar.gz: 7c9e1badc1628b5cab1edaa164e449d7a926e65198009773a702a3b6f49888cd
 SHA512:
-  metadata.gz: 28421eb19f5e288e43e9d455364906dabe0ee3ab024a230efb9aca19842ac3677c4b602788d8f0f7c67253f64e1b382d12a4f23756ff01ef2e43a5b51d478d55
-  data.tar.gz: f13d5452e5394c3e6d8b9468d8863344f35049972fbe5eac6c20cbede12ddc7e9eb2a17538bccb626a7a00a744c567d33e59558f5e7309ab3ecf775bea3db5b9
+  metadata.gz: b7bafbb9a871ce492a3f36537b237bbcb0a38e5291065384d2d2c2003bde2ebd078e9f2fd5455023bd9d81b38701a391b09d4dbeb336bfb7759e8c550497e5e9
+  data.tar.gz: 9515f40b1fe1e1782dd4a9fc6fd0c5e78e6a30c6678f656d6534eb2344360ec2c03ca745a0aedf7e3e3ec676a1d9edd092d8ff0db5b50966d24ade078496d440

data/README.md

@@ -1333,7 +1333,25 @@ job.attempts # Array of attempt timestamps
 
 ## Beanstalkd Integration
 
-Direct access to Beanstalkd for advanced operations:
+Postburner uses [Beaneater](https://github.com/beanstalkd/beaneater) as the Ruby client for Beanstalkd. You can access the underlying Beaneater connection directly for advanced operations.
+
+### Connection Methods
+
+```ruby
+# Get a cached Beaneater connection (returns Beaneater instance)
+conn = Postburner.connection
+conn.tubes.to_a  # List all tubes
+conn.stats       # Server statistics
+
+# Block form - yields connection, recommended for one-off operations
+Postburner.connected do |conn|
+  conn.tubes.to_a  # List all tubes
+  conn.tubes['postburner.production.critical'].stats
+  conn.tubes['postburner.production.critical'].kick(10)  # Kick 10 buried jobs
+end
+```
+
+### Job-Level Access
 
 ```ruby
 # Get Beanstalkd job ID
@@ -1342,16 +1360,114 @@ job.bkid  # => 12345
 # Access Beaneater job object
 job.bk.stats
 # => {"id"=>12345, "tube"=>"critical", "state"=>"ready", ...}
+```
 
-# Connection management
+### Beaneater API
+
+The connection object is a standard Beaneater instance. See the [Beaneater documentation](https://github.com/beanstalkd/beaneater) for full API details:
+
+```ruby
 Postburner.connected do |conn|
-  conn.tubes.to_a  # List all tubes
-  conn.tubes['postburner.production.critical'].stats
-  conn.tubes['postburner.production.critical'].kick(10)  # Kick 10 buried jobs
+  # Tubes
+  conn.tubes.to_a                    # List all tube names
+  conn.tubes['my-tube'].stats        # Tube statistics
+  conn.tubes['my-tube'].peek(:ready) # Peek at next ready job
+  conn.tubes['my-tube'].kick(10)     # Kick 10 buried jobs
+
+  # Server
+  conn.stats                         # Server statistics
+
+  # Jobs
+  conn.jobs.find(12345)              # Find job by ID
 end
 ```
 
 
+### Tube Statistics and Management
+
+Postburner provides methods to inspect and manage Beanstalkd tubes:
+
+**View tube statistics:**
+
+```ruby
+# View all tubes on the Beanstalkd server
+stats = Postburner.stats
+# => {
+#   tubes: [
+#     { name: "postburner.production.default", ready: 10, delayed: 5, buried: 0, reserved: 2, total: 17 },
+#     { name: "postburner.production.critical", ready: 0, delayed: 0, buried: 0, reserved: 1, total: 1 }
+#   ],
+#   totals: { ready: 10, delayed: 5, buried: 0, reserved: 3, total: 18 }
+# }
+
+# View specific tubes only
+stats = Postburner.stats(['postburner.production.critical'])
+# => { tubes: [...], totals: {...} }
+```
+
+**Clear jobs from tubes:**
+
+For safety, `clear_jobs!` requires you to explicitly specify which tubes to clear. This prevents accidentally clearing tubes from other applications sharing the same Beanstalkd server.
+
+```ruby
+# Collect stats only (no clearing)
+result = Postburner.clear_jobs!
+# => { tubes: [...], totals: {...}, cleared: false }
+
+# Clear specific tubes (must be in config/postburner.yml)
+result = Postburner.clear_jobs!(['postburner.production.default'])
+# => { tubes: [...], totals: {...}, cleared: true }
+
+# Pretty-print JSON output
+Postburner.clear_jobs!(['postburner.production.default'], silent: false)
+# Outputs formatted JSON to stdout
+
+# Silent mode (no output, just return data)
+result = Postburner.clear_jobs!(['postburner.production.default'], silent: true)
+```
+
+**Safety validation:**
+
+Only tubes defined in your loaded configuration can be cleared. This prevents mistakes in multi-tenant Beanstalkd environments:
+
+```ruby
+# Error: trying to clear tube not in config
+Postburner.clear_jobs!(['postburner.production.other-app'])
+# => ArgumentError: Cannot clear tubes not in configuration.
+#    Invalid tubes: postburner.production.other-app
+#    Configured tubes: postburner.production.default, postburner.production.critical
+```
+
+**Shortcut using watched_tube_names:**
+
+Clear all configured tubes at once:
+
+```ruby
+# Get all tubes from current configuration
+watched_tubes = Postburner.watched_tube_names
+# => ["postburner.production.default", "postburner.production.critical", "postburner.production.mailers"]
+
+# Clear all configured tubes
+Postburner.clear_jobs!(watched_tubes, silent: true)
+# or
+Postburner.clear_jobs!(Postburner.watched_tube_names, silent: true)
+```
+
+**Low-level Connection API:**
+
+For programmatic use without output formatting, use `Connection#clear_tubes!`:
+
+```ruby
+Postburner.connected do |conn|
+  # Returns data only (no puts)
+  result = conn.clear_tubes!(Postburner.watched_tube_names)
+  # => { tubes: [...], totals: {...}, cleared: true }
+
+  # Same validation - must be in configuration
+  result = conn.clear_tubes!(['postburner.production.default'])
+end
+```
+
 ## Web UI
 
 Mount the inspection interface:
@@ -1472,6 +1588,15 @@ There is a CLAUDE.md file for guidance when using Claude Code. Please use it or
 
 We encourage AI tools, but do not vibe, as the code must look like it was written by a human. Code that contains AI agent idioms will be rejected. Code that doesn't follow the project conventions will be rejected.
 
+
+### Testing
+
+```bash
+bundle install
+bundle exec rails test # must have beanstalkd on 11300 by default
+bundle exec rails app:postburner:work # if you want to run the worker
+```
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/app/concerns/postburner/insertion.rb

@@ -159,9 +159,8 @@ module Postburner
       #debugger
 
       # Response must be a hash with an :id key (value can be nil)
-      # Backburner returns symbol keys
       unless response.is_a?(Hash) && response.key?(:id)
-        raise MalformedResponse, "Missing :id key in response: #{response.inspect}"
+        raise Postburner::Job::MalformedResponse, "Missing :id key in response: #{response.inspect}"
       end
 
       persist_metadata!(bkid: response[:id])

data/lib/postburner/configuration.rb

@@ -10,37 +10,48 @@ module Postburner
   # @example Programmatic configuration
   #   Postburner.configure do |config|
   #     config.beanstalk_url = 'beanstalk://localhost:11300'
-  #     config.worker_type = :threads_on_fork
   #     config.logger = Rails.logger
+  #     config.worker_config = { name: 'default', queues: ['default'], forks: 2, threads: 10 }
   #   end
   #
   # @example Loading from YAML
-  #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production')
+  #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production', 'imports')
   #
   class Configuration
-    attr_accessor :beanstalk_url, :logger, :queues, :default_queue, :default_priority, :default_ttr, :default_threads, :default_forks, :default_gc_limit
+    # Global settings
+    attr_accessor :beanstalk_url, :logger, :default_queue, :default_priority, :default_ttr
+
+    # Worker-specific settings (loaded for a single worker)
+    attr_accessor :worker_config
 
     # @param options [Hash] Configuration options
     # @option options [String] :beanstalk_url Beanstalkd URL (default: ENV['BEANSTALK_URL'] or localhost)
     # @option options [Logger] :logger Logger instance (default: Rails.logger)
-    # @option options [Hash] :queues Queue configurations
     # @option options [String] :default_queue Default queue name (default: 'default')
     # @option options [Integer] :default_priority Default job priority (default: 65536, lower = higher priority)
     # @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
-    # @option options [Integer] :default_threads Default thread count per fork (default: 1)
-    # @option options [Integer] :default_forks Default fork count (default: 0, single process)
-    # @option options [Integer] :default_gc_limit Default GC limit for worker restarts (default: nil, no limit)
+    # @option options [Hash] :worker_config Worker configuration hash with keys:
+    #   - :name [String] Worker name
+    #   - :queues [Array<String>] Queue/tube names to process
+    #   - :forks [Integer] Number of forked processes (0 = single process mode)
+    #   - :threads [Integer] Number of threads per fork
+    #   - :gc_limit [Integer, nil] Jobs to process before restart (nil = unlimited)
+    #   - :timeout [Integer] Reserve command timeout in seconds (1-10, default: 3)
     #
     def initialize(options = {})
       @beanstalk_url = options[:beanstalk_url] || ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300'
       @logger = options[:logger] || (defined?(Rails) ? Rails.logger : Logger.new(STDOUT))
-      @queues = options[:queues] || { 'default' => {} }
       @default_queue = options[:default_queue] || 'default'
       @default_priority = options[:default_priority] || 65536
       @default_ttr = options[:default_ttr] || 300
-      @default_threads = options[:default_threads] || 1
-      @default_forks = options[:default_forks] || 0
-      @default_gc_limit = options[:default_gc_limit]
+      @worker_config = options[:worker_config] || {
+        name: 'default',
+        queues: ['default'],
+        forks: 0,
+        threads: 1,
+        gc_limit: nil,
+        timeout: 3
+      }
     end
 
     # Loads configuration from a YAML file.
@@ -68,30 +79,31 @@ module Postburner
     #     beanstalk_url: beanstalk://localhost:11300
     #     default_priority: 131072 # change default priority from 65536 to 131072
     #
-    #   production: # <- environment config, i.e. defaults, NOT worker config
+    #   production: # <- environment config, i.e. defaults
     #     <<: *default
     #     default_forks: 2
     #     default_threads: 10
     #     default_gc_limit: 5000
     #     default_ttr: 300
-    #     workers: # <- worker config, i.e. overrides, NOT environment config
-    #       imports: # <- worker "group" name
+    #     workers: # <- worker configs
+    #       imports: # <- worker name
+    #         timeout: 3       # Reserve timeout in seconds (1-10, default: 3)
+    #                          # Lower values enable faster graceful shutdowns
     #         forks: 4         # Overrides default_forks
     #         threads: 1       # Overrides default_threads
     #         gc_limit: 500    # Overrides default_gc_limit
-    #         # ttr: 60        # Use default from production, i.e. 300 because not set
     #         queues:
     #           - imports
     #           - data_processing
     #
     def self.load_yaml(path, env = 'development', worker_name = nil)
       yaml = YAML.load_file(path, aliases: true)
-      # env_defaults = top-level environment config (development:, production:, etc.)
-      env_defaults = yaml[env.to_s] || yaml[env.to_sym]
+      # env_config = top-level environment config (development:, production:, etc.)
+      env_config = yaml[env.to_s] || yaml[env.to_sym]
 
-      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_defaults
+      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_config
 
-      workers = env_defaults['workers']
+      workers = env_config['workers']
       raise ArgumentError, "No 'workers:' section found in #{path} for environment '#{env}'" unless workers
 
       # Auto-select single worker or validate worker_name
@@ -99,63 +111,54 @@ module Postburner
         if workers.size == 1
           worker_name = workers.keys.first
         else
-          raise ArgumentError, "Configuration has multiple workers, but --worker not specified\nAvailable workers: #{workers.keys.join(', ')}\nUsage: bin/postburner --worker <name>"
+          raise ArgumentError, <<~ERROR
+            Configuration has multiple workers, but --worker not specified
+            Available workers: #{workers.keys.join(', ')}
+            Usage: bin/postburner --worker <name>
+          ERROR
         end
       else
         unless workers.key?(worker_name)
-          raise ArgumentError, "Worker '#{worker_name}' not found in #{path}\nAvailable workers: #{workers.keys.join(', ')}"
+          raise ArgumentError, <<~ERROR
+            Worker '#{worker_name}' not found in #{path}
+            Available workers: #{workers.keys.join(', ')}
+          ERROR
         end
       end
 
-      # worker_config = specific worker configuration (workers: imports:)
-      worker_config = workers[worker_name]
-
-      # Convert queue array to hash format (queues no longer have per-queue config)
-      queue_list = worker_config['queues'] || []
-      queues_hash = {}
-      queue_list.each do |queue_name|
-        queues_hash[queue_name] = {}  # Empty hash - queues run in worker pool
-      end
+      # worker_yaml = specific worker configuration from YAML (workers: imports:)
+      worker_yaml = workers[worker_name]
+
+      # Build worker_config hash - worker-level overrides env-level defaults
+      worker_config = {
+        name: worker_name,
+        queues: worker_yaml['queues'] || ['default'],
+        forks: worker_yaml['forks'] || env_config['default_forks'] || 0,
+        threads: worker_yaml['threads'] || env_config['default_threads'] || 1,
+        gc_limit: worker_yaml['gc_limit'] || env_config['default_gc_limit'],
+        timeout: worker_yaml['timeout'] || 3
+      }
 
-      # Cascade: worker-level overrides env-level defaults
-      # Worker uses: forks, threads, gc_limit, ttr, priority (NO default_ prefix)
-      # Env uses: default_forks, default_threads, etc. (WITH default_ prefix)
       options = {
-        beanstalk_url: env_defaults['beanstalk_url'],
-        queues: queues_hash,
-        default_queue: worker_config['default_queue'] || env_defaults['default_queue'],
-        default_priority: worker_config['priority'] || env_defaults['default_priority'],
-        default_ttr: worker_config['ttr'] || env_defaults['default_ttr'],
-        default_threads: worker_config['threads'] || env_defaults['default_threads'],
-        default_forks: worker_config['forks'] || env_defaults['default_forks'],
-        default_gc_limit: worker_config['gc_limit'] || env_defaults['default_gc_limit']
+        beanstalk_url: env_config['beanstalk_url'],
+        default_queue: env_config['default_queue'],
+        default_priority: env_config['default_priority'],
+        default_ttr: env_config['default_ttr'],
+        worker_config: worker_config
       }
 
       new(options)
     end
 
-    # Returns queue configuration for a specific queue name.
-    #
-    # @param queue_name [String, Symbol] Name of the queue
-    #
-    # @return [Hash] Queue configuration with threads, gc_limit, etc.
-    #
-    # @example
-    #   config.queue_config('critical')  # => { threads: 1, gc_limit: 100 }
-    #
-    def queue_config(queue_name)
-      @queues[queue_name.to_s] || @queues[queue_name.to_sym] || {}
-    end
-
-    # Returns array of all configured queue names.
+    # Returns array of queue names from worker config.
     #
     # @return [Array<String>] Queue names
     #
     # @example
-    #   config.queue_names  # => ['default', 'critical', 'mailers']
+    #   config.queue_names  # => ['imports', 'data_processing']
     #
     def queue_names
-      @queues.keys.map(&:to_s)
+      @worker_config[:queues].map(&:to_s)
     end
 
     # Expands queue name to full tube name with environment prefix.
@@ -188,7 +191,6 @@ module Postburner
       ].compact.join('.')
     end
 
-
     # Returns array of expanded tube names with environment prefix.
     #
     # @param env [String, Symbol, nil] Environment name (defaults to Rails.env or 'development')
@@ -196,7 +198,8 @@ module Postburner
     # @return [Array<String>] Array of expanded tube names
     #
     # @example
-    #   config.expanded_tube_names('production')  # => ['postburner.production.default', 'postburner.production.critical']
+    #   config.expanded_tube_names('production')  # => ['postburner.production.imports', 'postburner.production.data_processing']
+    #
     def expanded_tube_names(env = nil)
       queue_names.map { |q| expand_tube_name(q, env) }
     end
@@ -220,7 +223,7 @@ module Postburner
   # @example
   #   Postburner.configure do |config|
   #     config.beanstalk_url = 'beanstalk://localhost:11300'
-  #     config.worker_type = :threads_on_fork
+  #     config.worker_config = { name: 'default', queues: ['default'], forks: 2, threads: 10 }
   #   end
   #
   def self.configure

data/lib/postburner/connection.rb

@@ -88,6 +88,68 @@ module Postburner
       @pool = nil
     end
 
+    # Clears jobs from specified tubes or collects stats for all tubes.
+    #
+    # Low-level method that returns data only (no output to stdout).
+    # Delegates to Postburner.stats for collecting statistics.
+    # For user-facing output, use Postburner.clear_jobs! instead.
+    #
+    # SAFETY: Only allows clearing tubes that are defined in the loaded
+    # configuration (watched_tube_names). This prevents accidentally clearing
+    # tubes from other applications or environments.
+    #
+    # @param tube_names [Array<String>, nil] Array of tube names to clear, or nil to only collect stats
+    #
+    # @return [Hash] Statistics and results with keys:
+    #   - tubes: Array of hashes with per-tube stats
+    #   - totals: Hash with aggregated counts across all tubes
+    #   - cleared: Boolean indicating if tubes were actually cleared
+    #
+    # @raise [ArgumentError] if tube_names contains tubes not in watched_tube_names
+    #
+    # @example Collect stats only (no clearing)
+    #   result = conn.clear_tubes!
+    #   result[:totals][:total]  # => 42
+    #
+    # @example Clear configured tubes only
+    #   result = conn.clear_tubes!(Postburner.watched_tube_names)
+    #   result[:cleared]  # => true
+    #
+    # @example Invalid tube raises error
+    #   conn.clear_tubes!(['random-tube'])
+    #   # => ArgumentError: Cannot clear tubes not in configuration
+    #
+    def clear_tubes!(tube_names = nil)
+      ensure_connected!
+
+      # Validate that tubes to clear are in the loaded configuration
+      if tube_names&.any?
+        watched = Postburner.watched_tube_names
+        invalid_tubes = tube_names - watched
+
+        if invalid_tubes.any?
+          raise ArgumentError, <<~ERROR
+            Cannot clear tubes not in configuration.
+            Invalid tubes: #{invalid_tubes.join(', ')}
+            Configured tubes: #{watched.join(', ')}
+          ERROR
+        end
+      end
+
+      # Get stats using Postburner.stats
+      result = Postburner.stats(tube_names)
+      result[:cleared] = tube_names&.any? ? true : false
+
+      # Actually clear if tube names were provided and validated
+      if tube_names&.any?
+        tube_names.each do |tube_name|
+          tubes[tube_name].clear
+        end
+      end
+
+      result
+    end
+
     private
 
     # Establishes connection to Beanstalkd.

data/lib/postburner/runner.rb

@@ -87,7 +87,8 @@ module Postburner
       config.logger.info "[Postburner] Environment: #{options[:env]}"
       config.logger.info "[Postburner] Worker: #{options[:worker] || '(auto-selected)'}" if options[:worker] || options[:queues].nil?
       config.logger.info "[Postburner] Queues: #{config.queue_names.join(', ')}"
-      config.logger.info "[Postburner] Defaults: forks=#{config.default_forks}, threads=#{config.default_threads}, gc_limit=#{config.default_gc_limit || 'none'}"
+      wc = config.worker_config
+      config.logger.info "[Postburner] Worker config: forks=#{wc[:forks]}, threads=#{wc[:threads]}, gc_limit=#{wc[:gc_limit] || 'none'}, timeout=#{wc[:timeout]}s"
     end
 
     # Returns root directory for config file resolution.

data/lib/postburner/time_helpers.rb

@@ -64,8 +64,10 @@ module Postburner
     #
     def travel_to(time, &block)
       unless defined?(ActiveSupport::Testing::TimeHelpers)
-        raise "ActiveSupport::Testing::TimeHelpers not available. " \
-              "Postburner::TimeHelpers requires Rails testing helpers for time travel."
+        raise <<~ERROR
+          ActiveSupport::Testing::TimeHelpers not available.
+          Postburner::TimeHelpers requires Rails testing helpers for time travel.
+        ERROR
       end
 
       helper = Object.new.extend(ActiveSupport::Testing::TimeHelpers)

data/lib/postburner/version.rb

@@ -1,3 +1,3 @@
 module Postburner
-  VERSION = '1.0.0.pre.5'
+  VERSION = '1.0.0.pre.7'
 end

data/lib/postburner/workers/worker.rb

@@ -4,7 +4,7 @@ require 'concurrent'
 
 module Postburner
   module Workers
-    # Puma-style worker with configurable forks and threads per queue.
+    # Puma-style worker with configurable forks and threads.
     #
     # This is the universal Postburner worker that scales from development
     # to production using forks and threads configuration. Just like Puma:
@@ -16,27 +16,18 @@ module Postburner
     # ### Single Process Mode (forks: 0)
     # ```
     # Main Process
-    # ├─ Queue 'default' Thread Pool (10 threads)
-    # ├─ Queue 'critical' Thread Pool (1 thread)
-    # └─ Queue 'mailers' Thread Pool (5 threads)
+    # └─ Thread Pool (N threads watching all queues)
     # ```
     #
     # ### Multi-Process Mode (forks: 1+)
     # ```
     # Parent Process
-    # ├─ Fork 0 (queue: critical)
-    # │   └─ Thread 1
-    # ├─ Fork 0 (queue: default)
-    # │   ├─ Thread 1-10
-    # ├─ Fork 1 (queue: default)  # Puma-style: multiple forks of same queue
-    # │   ├─ Thread 1-10
-    # ├─ Fork 2 (queue: default)
-    # │   └─ Thread 1-10
-    # ├─ Fork 3 (queue: default)
-    # │   └─ Thread 1-10
-    # │   └─ Total: 40 concurrent jobs for 'default' (4 forks × 10 threads)
-    # └─ Fork 0 (queue: mailers)
-    #     └─ Thread 1-5
+    # ├─ Fork 0
+    # │   └─ Thread Pool (N threads watching all queues)
+    # ├─ Fork 1
+    # │   └─ Thread Pool (N threads watching all queues)
+    # └─ Fork 2
+    #     └─ Thread Pool (N threads watching all queues)
     # ```
     #
     # ## Scaling Strategy
@@ -60,38 +51,25 @@ module Postburner
     # forks: 4
     # threads: 10
     # ```
-    # 4 processes × 10 threads = 40 concurrent jobs per queue
+    # 4 processes × 10 threads = 40 concurrent jobs
     #
     # ## Configuration
     #
     # @example Development (single-threaded)
-    #   development:  # <- environment config, i.e. defaults
-    #     default_forks: 0
-    #     default_threads: 1
-    #     workers:  # <- worker config, i.e. overrides
+    #   development:
+    #     workers:
     #       default:
+    #         forks: 0
+    #         threads: 1
     #         queues:
     #           - default
     #           - mailers
     #
-    # @example Staging (multi-threaded, single process)
-    #   staging:  # <- environment config, i.e. defaults
-    #     default_forks: 0
-    #     default_threads: 10
-    #     default_gc_limit: 5000
-    #     workers:  # <- worker config, i.e. overrides
-    #       default:
-    #         queues:
-    #           - critical
-    #           - default
-    #           - mailers
-    #
-    # @example Production (Puma-style: forks × threads with worker overrides)
-    #   production:  # <- environment config, i.e. defaults
+    # @example Production (Puma-style: forks × threads)
+    #   production:
     #     default_forks: 2
     #     default_threads: 10
-    #     default_gc_limit: 5000
-    #     workers:  # <- worker config, i.e. overrides
+    #     workers:
     #       default:
     #         forks: 4        # Overrides default_forks
     #         threads: 10     # Overrides default_threads
@@ -109,12 +87,13 @@ module Postburner
       # @return [void]
       #
       def start
-        logger.info "[Postburner::Worker] Starting..."
+        logger.info "[Postburner::Worker] Starting worker '#{worker_config[:name]}'..."
         logger.info "[Postburner::Worker] Queues: #{config.queue_names.join(', ')}"
+        logger.info "[Postburner::Worker] Config: #{worker_config[:forks]} forks, #{worker_config[:threads]} threads, gc_limit: #{worker_config[:gc_limit] || 'unlimited'}, timeout: #{worker_config[:timeout]}s"
         logger.info "[Postburner] #{config.beanstalk_url} watching tubes: #{config.expanded_tube_names.join(', ')}"
 
         # Detect mode based on fork configuration
-        if using_forks?
+        if worker_config[:forks] > 0
           start_forked_mode
         else
           start_single_process_mode
@@ -123,21 +102,17 @@ module Postburner
 
       private
 
-      # Checks if any queue is configured to use forks.
+      # Returns the worker configuration hash.
       #
-      # @return [Boolean] true if any queue has forks > 0
+      # @return [Hash] Worker config with :name, :queues, :forks, :threads, :gc_limit, :timeout
       #
-      def using_forks?
-        config.queue_names.any? do |queue_name|
-          queue_config = config.queue_config(queue_name)
-          fork_count = queue_config['forks'] || queue_config[:forks] || config.default_forks
-          fork_count > 0
-        end
+      def worker_config
+        config.worker_config
       end
 
       # Starts worker in single-process mode (forks: 0).
       #
-      # Creates thread pools for each queue in the main process.
+      # Creates a thread pool that watches all configured queues.
       # Suitable for development and moderate concurrency needs.
       #
       # @return [void]
@@ -147,12 +122,17 @@ module Postburner
 
         # Track total jobs processed across all threads
         @jobs_processed = Concurrent::AtomicFixnum.new(0)
-        @gc_limit = config.default_gc_limit
+        @gc_limit = worker_config[:gc_limit]
+
+        # Create thread pool
+        thread_count = worker_config[:threads]
+        @pool = Concurrent::FixedThreadPool.new(thread_count)
 
-        # Create thread pools for each queue
-        @pools = {}
-        config.queue_names.each do |queue_name|
-          spawn_queue_threads(queue_name)
+        # Spawn worker threads
+        thread_count.times do
+          @pool.post do
+            process_jobs
+          end
         end
 
         # Monitor for shutdown or GC limit
@@ -160,9 +140,10 @@ module Postburner
           sleep 0.5
         end
 
-        # Shutdown pools gracefully
+        # Shutdown pool gracefully
         logger.info "[Postburner::Worker] Shutting down..."
-        shutdown_pools
+        @pool.shutdown
+        @pool.wait_for_termination(30)
 
         if @gc_limit && @jobs_processed.value >= @gc_limit
           logger.info "[Postburner::Worker] Reached GC limit (#{@jobs_processed.value} jobs), exiting for restart..."
@@ -172,49 +153,24 @@ module Postburner
         end
       end
 
-      # Spawns thread pool for a specific queue in single-process mode.
-      #
-      # @param queue_name [String] Name of the queue to process
-      #
-      # @return [void]
-      #
-      def spawn_queue_threads(queue_name)
-        queue_config = config.queue_config(queue_name)
-        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
-
-        logger.info "[Postburner::Worker] Queue '#{queue_name}': #{thread_count} threads"
-
-        # Create thread pool
-        pool = Concurrent::FixedThreadPool.new(thread_count)
-        @pools[queue_name] = pool
-
-        # Spawn worker threads
-        thread_count.times do
-          pool.post do
-            process_jobs_in_single_process(queue_name)
-          end
-        end
-      end
-
-      # Processes jobs in a single thread (single-process mode).
+      # Processes jobs in a single thread.
       #
       # Each thread has its own Beanstalkd connection and reserves jobs
-      # from the specified queue.
-      #
-      # @param queue_name [String] Name of the queue to process
+      # from all configured queues.
       #
       # @return [void]
       #
-      def process_jobs_in_single_process(queue_name)
+      def process_jobs
         connection = Postburner::Connection.new
+        timeout = worker_config[:timeout]
 
-        # Watch only this queue
-        watch_queues(connection, queue_name)
+        # Watch all configured queues
+        watch_queues(connection, *config.queue_names)
 
         until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
           begin
-            # Reserve with short timeout
-            job = connection.beanstalk.tubes.reserve(timeout: 1)
+            # Reserve with configured timeout
+            job = connection.beanstalk.tubes.reserve(timeout: timeout)
 
             if job
               logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} reserved job #{job.id}"
@@ -238,34 +194,22 @@ module Postburner
         connection&.close rescue nil
       end
 
-      # Gracefully shuts down all thread pools (single-process mode).
-      #
-      # @return [void]
-      #
-      def shutdown_pools
-        @pools.each do |queue_name, pool|
-          pool.shutdown
-          pool.wait_for_termination(30)
-          logger.info "[Postburner::Worker] Queue '#{queue_name}' shutdown complete"
-        end
-      end
-
       # Starts worker in forked mode (forks: 1+).
       #
-      # Forks multiple child processes for each queue, each running
-      # a thread pool. Parent process monitors children and restarts them when they exit.
+      # Forks multiple child processes, each running a thread pool.
+      # Parent process monitors children and restarts them when they exit.
       #
       # @return [void]
       #
       def start_forked_mode
-        logger.info "[Postburner::Worker] Mode: Multi-process (forks: 1+)"
+        logger.info "[Postburner::Worker] Mode: Multi-process (#{worker_config[:forks]} forks)"
 
-        # Track children: { pid => { queue: 'name', fork_num: 0 } }
+        # Track children: { pid => fork_num }
         @children = {}
 
-        # Spawn configured number of forks for each queue
-        config.queue_names.each do |queue_name|
-          spawn_queue_workers(queue_name)
+        # Spawn configured number of forks
+        worker_config[:forks].times do |fork_num|
+          spawn_fork(fork_num)
         end
 
         # Parent process monitors children
@@ -274,18 +218,16 @@ module Postburner
             pid, status = Process.wait2(-1, Process::WNOHANG)
 
             if pid
-              child_info = @children.delete(pid)
+              fork_num = @children.delete(pid)
               exit_code = status.exitstatus
-              queue_name = child_info[:queue]
-              fork_num = child_info[:fork_num]
 
               if exit_code == 99
                 # GC restart - this is normal
-                logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} reached GC limit, restarting..."
-                spawn_queue_worker(queue_name, fork_num) unless shutdown?
+                logger.info "[Postburner::Worker] Fork #{fork_num} reached GC limit, restarting..."
+                spawn_fork(fork_num) unless shutdown?
               else
-                logger.error "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} exited unexpectedly (code: #{exit_code})"
-                spawn_queue_worker(queue_name, fork_num) unless shutdown?
+                logger.error "[Postburner::Worker] Fork #{fork_num} exited unexpectedly (code: #{exit_code})"
+                spawn_fork(fork_num) unless shutdown?
               end
             end
 
@@ -305,61 +247,36 @@ module Postburner
         logger.info "[Postburner::Worker] Shutdown complete"
       end
 
-      # Spawns all forked worker processes for a specific queue.
-      #
-      # @param queue_name [String] Name of the queue to process
-      #
-      # @return [void]
-      #
-      def spawn_queue_workers(queue_name)
-        queue_config = config.queue_config(queue_name)
-        fork_count = queue_config['forks'] || queue_config[:forks] || config.default_forks
-        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
-
-        # Skip if this queue has 0 forks (shouldn't happen in forked mode, but be defensive)
-        return if fork_count == 0
-
-        total_concurrency = fork_count * thread_count
-        logger.info "[Postburner::Worker] Queue '#{queue_name}': #{fork_count} forks × #{thread_count} threads = #{total_concurrency} total concurrency"
-
-        fork_count.times do |fork_num|
-          spawn_queue_worker(queue_name, fork_num)
-        end
-      end
-
-      # Spawns a single forked worker process for a specific queue.
+      # Spawns a single forked worker process.
       #
-      # @param queue_name [String] Name of the queue to process
       # @param fork_num [Integer] Fork number (0-indexed)
       #
       # @return [void]
       #
-      def spawn_queue_worker(queue_name, fork_num)
+      def spawn_fork(fork_num)
         pid = fork do
           # Child process
-          run_queue_worker(queue_name, fork_num)
+          run_fork(fork_num)
         end
 
-        @children[pid] = { queue: queue_name, fork_num: fork_num }
-        logger.info "[Postburner::Worker] Spawned worker for queue '#{queue_name}' fork #{fork_num} (pid: #{pid})"
+        @children[pid] = fork_num
+        logger.info "[Postburner::Worker] Spawned fork #{fork_num} (pid: #{pid})"
       end
 
-      # Runs the thread pool worker for a specific queue fork.
+      # Runs the thread pool worker in a forked process.
       #
       # This runs in the child process. Creates a thread pool and processes
       # jobs until GC limit is reached or shutdown is requested.
       #
-      # @param queue_name [String] Name of the queue to process
       # @param fork_num [Integer] Fork number (for logging)
       #
       # @return [void]
       #
-      def run_queue_worker(queue_name, fork_num)
-        queue_config = config.queue_config(queue_name)
-        thread_count = queue_config['threads'] || queue_config[:threads] || config.default_threads
-        gc_limit = queue_config['gc_limit'] || queue_config[:gc_limit] || config.default_gc_limit
+      def run_fork(fork_num)
+        thread_count = worker_config[:threads]
+        gc_limit = worker_config[:gc_limit]
 
-        logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
+        logger.info "[Postburner::Worker] Fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
 
         # Track jobs processed in this fork
         jobs_processed = Concurrent::AtomicFixnum.new(0)
@@ -370,7 +287,7 @@ module Postburner
         # Each thread needs its own Beanstalkd connection
         thread_count.times do
           pool.post do
-            process_jobs_in_fork(queue_name, fork_num, jobs_processed, gc_limit)
+            process_jobs_in_fork(fork_num, jobs_processed, gc_limit)
           end
         end
 
@@ -384,43 +301,43 @@ module Postburner
         pool.wait_for_termination(30)
 
         if gc_limit && jobs_processed.value >= gc_limit
-          logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} reached GC limit (#{jobs_processed.value} jobs), exiting for restart..."
+          logger.info "[Postburner::Worker] Fork #{fork_num} reached GC limit (#{jobs_processed.value} jobs), exiting for restart..."
           exit 99  # Special exit code for GC restart
         else
-          logger.info "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} shutting down gracefully..."
+          logger.info "[Postburner::Worker] Fork #{fork_num} shutting down gracefully..."
           exit 0
         end
       rescue => e
-        logger.error "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} error: #{e.message}"
+        logger.error "[Postburner::Worker] Fork #{fork_num} error: #{e.message}"
         logger.error e.backtrace.join("\n")
         exit 1
       end
 
-      # Processes jobs in a single thread within a fork (forked mode).
+      # Processes jobs in a single thread within a fork.
       #
       # Each thread has its own Beanstalkd connection and reserves jobs
-      # from the specified queue.
+      # from all configured queues.
       #
-      # @param queue_name [String] Name of the queue to process
       # @param fork_num [Integer] Fork number (for logging)
       # @param jobs_processed [Concurrent::AtomicFixnum] Shared counter of jobs processed
       # @param gc_limit [Integer, nil] Maximum jobs before triggering GC restart (nil = unlimited)
       #
       # @return [void]
       #
-      def process_jobs_in_fork(queue_name, fork_num, jobs_processed, gc_limit)
+      def process_jobs_in_fork(fork_num, jobs_processed, gc_limit)
         connection = Postburner::Connection.new
+        timeout = worker_config[:timeout]
 
-        # Watch only this queue
-        watch_queues(connection, queue_name)
+        # Watch all configured queues
+        watch_queues(connection, *config.queue_names)
 
         until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
           begin
-            # Reserve with short timeout
-            job = connection.beanstalk.tubes.reserve(timeout: 1)
+            # Reserve with configured timeout
+            job = connection.beanstalk.tubes.reserve(timeout: timeout)
 
             if job
-              logger.debug "[Postburner::Worker] Queue '#{queue_name}' fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
+              logger.debug "[Postburner::Worker] Fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
               execute_job(job)
               jobs_processed.increment
             end

data/lib/postburner.rb

@@ -375,25 +375,50 @@ module Postburner
     end
   end
 
-  # Removes all jobs from all tubes (not yet implemented).
+  # Clears jobs from specified tubes or shows stats for all tubes.
   #
-  # This is a destructive operation intended for development/testing cleanup.
-  # Requires confirmation string "CONFIRM" to prevent accidental execution.
+  # High-level method with formatted output. Delegates to Connection#clear_tubes!
+  # for the actual work, then pretty-prints the results.
   #
-  # @param confirm [String] Must be exactly "CONFIRM" to execute
+  # SAFETY: Only allows clearing tubes that are defined in the loaded
+  # configuration. This prevents accidentally clearing tubes from other
+  # applications or environments sharing the same Beanstalkd server.
   #
-  # @return [void]
+  # @param tube_names [Array<String>, nil] Array of tube names to clear, or nil to only show stats
+  # @param silent [Boolean] If true, suppress output to stdout (default: false)
   #
-  # @example
-  #   Postburner.remove_all!("CONFIRM")
+  # @return [Hash] Statistics and results (see Connection#clear_tubes!)
+  #
+  # @raise [ArgumentError] if tube_names contains tubes not in watched_tube_names
+  #
+  # @example Show stats only (no clearing) - SAFE
+  #   Postburner.clear_jobs!
+  #   # Shows stats for ALL tubes on Beanstalkd, but doesn't clear anything
+  #
+  # @example Clear watched tubes only - SAFE
+  #   Postburner.clear_jobs!(Postburner.watched_tube_names)
+  #   # Only clears tubes defined in your config
   #
-  # @note Currently a no-op - implementation pending
-  # @todo Implement job removal from all tubes
+  # @example Trying to clear unconfigured tube - RAISES ERROR
+  #   Postburner.clear_jobs!(['some-other-app-tube'])
+  #   # => ArgumentError: Cannot clear tubes not in configuration
   #
-  def self.remove_all!(confirm)
-    return unless confirm == "CONFIRM"
+  # @example Silent mode (programmatic use)
+  #   result = Postburner.clear_jobs!(Postburner.watched_tube_names, silent: true)
+  #   result[:totals][:total]  # => 42
+  #
+  # @see Connection#clear_tubes!
+  #
+  def self.clear_jobs!(tube_names = nil, silent: false)
+    require 'json'
+
+    result = connection.clear_tubes!(tube_names)
+
+    unless silent
+      puts JSON.pretty_generate(result)
+    end
 
-    # TODO
+    result
   end
 
   # Returns array of watched tube names with environment prefix.
@@ -423,38 +448,82 @@ module Postburner
     @__watched_tubes ||= watched_tube_names.map { |tube_name| connection.tubes[tube_name] }
   end
 
-  # Returns statistics and introspection data about Beanstalkd and configured queues.
+  # Returns detailed statistics about Beanstalkd tubes.
   #
-  # Provides Beaneater tube instances for configured tubes and all tubes that exist
-  # on the Beanstalkd server. Tube instances support introspection methods:
+  # Collects job counts (ready, delayed, buried, reserved) for each tube
+  # and provides aggregate totals across all tubes.
   #
-  # - tube.name - Tube name
-  # - tube.stats - Tube statistics hash (current-jobs-ready, current-jobs-buried, etc.)
-  # - tube.peek_ready - Next ready job
-  # - tube.peek_delayed - Next delayed job
-  # - tube.peek_buried - Next buried job
-  # - tube.kick(n) - Kick n buried jobs back to ready
-  # - tube.pause(delay) - Pause tube for delay seconds
-  # - tube.clear - Delete all jobs in tube
+  # @param tube_names [Array<String>, nil] Specific tube names to inspect, or nil for all tubes
   #
-  # @return [Hash] Statistics hash with the following keys:
-  #   - watched_tubes: Array of configured/watched Beaneater::Tube instances
-  #   - tubes: Array of all Beaneater::Tube instances on the server
+  # @return [Hash] Statistics hash with keys:
+  #   - tubes: Array of hashes with per-tube stats (name, ready, delayed, buried, reserved, total)
+  #   - totals: Hash with aggregated counts across all tubes
   #
   # @raise [Beaneater::NotConnected] if connection to Beanstalkd fails
   #
-  # @example
+  # @example Get stats for all tubes
   #   stats = Postburner.stats
-  #   stats[:watched_tubes].each { |tube| puts "#{tube.name}: #{tube.stats}" }
-  #   stats[:tubes].first.peek_ready
+  #   stats[:totals][:total]  # => 42
+  #   stats[:tubes].first[:name]  # => "default"
+  #
+  # @example Get stats for specific tubes
+  #   stats = Postburner.stats(Postburner.watched_tube_names)
+  #   stats[:tubes].size  # => 3
   #
-  def self.stats
+  def self.stats(tube_names = nil)
     connected do |conn|
-      {
-        watched_tubes: self.watched_tubes,
-        # Get all tube instances that exist on Beanstalkd
-        tubes: conn.beanstalk.tubes.all
+      # Get tubes to inspect
+      tubes_to_inspect = if tube_names&.any?
+        tube_names.map { |name| conn.tubes[name] }
+      else
+        conn.beanstalk.tubes.all
+      end
+
+      result = {
+        tubes: [],
+        totals: {
+          ready: 0,
+          delayed: 0,
+          buried: 0,
+          reserved: 0,
+          total: 0
+        }
       }
+
+      # Collect stats from each tube
+      tubes_to_inspect.each do |tube|
+        begin
+          stats = tube.stats
+          # Beaneater returns a StatStruct; access the underlying hash
+          stats_hash = stats.instance_variable_get(:@hash) || {}
+
+          tube_data = {
+            name: tube.name,
+            ready: stats_hash['current-jobs-ready'] || 0,
+            delayed: stats_hash['current-jobs-delayed'] || 0,
+            buried: stats_hash['current-jobs-buried'] || 0,
+            reserved: stats_hash['current-jobs-reserved'] || 0,
+            total: (stats_hash['current-jobs-ready'] || 0) +
+                   (stats_hash['current-jobs-delayed'] || 0) +
+                   (stats_hash['current-jobs-buried'] || 0) +
+                   (stats_hash['current-jobs-reserved'] || 0)
+          }
+        rescue Beaneater::NotFoundError
+          # Tube doesn't exist yet, skip it
+          next
+        end
+
+        result[:tubes] << tube_data
+
+        # Aggregate totals
+        result[:totals][:ready] += tube_data[:ready]
+        result[:totals][:delayed] += tube_data[:delayed]
+        result[:totals][:buried] += tube_data[:buried]
+        result[:totals][:reserved] += tube_data[:reserved]
+        result[:totals][:total] += tube_data[:total]
+      end
+
+      result
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: postburner
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.5
+  version: 1.0.0.pre.7
 platform: ruby
 authors:
 - Matt Smith