sumologic-query 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b7bbf8f091fa6d3c6e213a34c1ba72952c43aa995814834379c30a051d3f4f5
-  data.tar.gz: 4e7bc3f985d01543a5af7ce89d7746944107d4fd99ecf4367c276a5e550dbdca
+  metadata.gz: 120fb7b7d2ce1d70d3dbb921224e068ace399125dcd7cfab98633d4e58d7514a
+  data.tar.gz: 4e256d592e59a754baf942f07b81a63c4c2cce95f0d2d671cfff320d211c7d86
 SHA512:
-  metadata.gz: 4f34b22f610d453649fff4926c7dfdff0fd2d5ff18a2e5fa6994cf1f41635990c28ace805fb2508520a1f016899c99f71ba31be2d5b0771a8b30d9e781f310a2
-  data.tar.gz: 0a12a47f8ddfeda0bab912f2011a033b121e01f3c63b19d9fa80e7cfa0a7df32a7277784a92a88ca988819a1f42c717c61dbbae79d5bab154161d9cff5f571a8
+  metadata.gz: c18535243d40611c287544902924646314323aa3b01df87b9e6758c9af62afedb610753c9e27d0fec3a338f1c911fe5fca0245140c6acc18c1e099136aab4c18
+  data.tar.gz: b88b9887a0886962844737bd3087e9282f9fb2402e619de5ec919853727bc59f6e6c5baeeefff82cd63509c35bb9d7c36b3e626aef75aaa75e90f296a8ae0c78

data/CHANGELOG.md

@@ -1,15 +1,20 @@
-# Changelog
+## [1.2.2](https://github.com/patrick204nqh/sumologic-query/compare/v1.2.1...v1.2.2) (2025-11-15)
 
-All notable changes to this project are documented in [GitHub Releases](https://github.com/patrick204nqh/sumologic-query/releases).
 
-This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Releases
+## [1.2.1](https://github.com/patrick204nqh/sumologic-query/compare/v1.2.0...v1.2.1) (2025-11-14)
+
+
+
+# [1.2.0](https://github.com/patrick204nqh/sumologic-query/compare/v1.1.2...v1.2.0) (2025-11-14)
+
+
+
+## [1.1.2](https://github.com/patrick204nqh/sumologic-query/compare/v1.1.1...v1.1.2) (2025-11-14)
+
+
+
+## [1.1.1](https://github.com/patrick204nqh/sumologic-query/compare/v1.1.0...v1.1.1) (2025-11-14)
 
-- [v1.1.0](https://github.com/patrick204nqh/sumologic-query/releases/tag/v1.1.0) - Latest
-- [v1.0.0](https://github.com/patrick204nqh/sumologic-query/releases/tag/v1.0.0) - Initial release
 
----
 
-**Note:** Release notes are automatically generated from commit messages and pull requests.
-See [GitHub Releases](https://github.com/patrick204nqh/sumologic-query/releases) for detailed changelogs.

data/README.md

@@ -86,7 +86,8 @@ sumo-query search --query "YOUR_QUERY" \
   --to "END_TIME" \
   [--output FILE] \
   [--limit N] \
-  [--time-zone TZ]
+  [--time-zone TZ] \
+  [--interactive]
 ```
 
 **Required options:**
@@ -95,11 +96,50 @@ sumo-query search --query "YOUR_QUERY" \
 - `-t, --to TIME` - End time (ISO 8601 format)
 
 **Optional options:**
+- `-i, --interactive` - Launch interactive browser with FZF
 - `-z, --time-zone TZ` - Time zone (default: UTC)
 - `-l, --limit N` - Limit number of messages
 - `-o, --output FILE` - Save to file (default: stdout)
 - `-d, --debug` - Enable debug output
 
+### Interactive Mode 🚀
+
+Explore your logs interactively with a powerful FZF-based interface:
+
+```bash
+# Launch interactive mode
+sumo-query search --query 'error' \
+  --from '2025-11-13T14:00:00' \
+  --to '2025-11-13T15:00:00' \
+  --interactive
+
+# Or use the shorthand
+sumo-query search -q 'error' -f '2025-11-13T14:00:00' -t '2025-11-13T15:00:00' -i
+```
+
+**Features:**
+- 🔍 Fuzzy search across all message fields
+- 👁️ Live preview with full JSON details
+- 🎨 Color-coded log levels (ERROR, WARN, INFO)
+- ⌨️ Keyboard shortcuts for quick actions
+- 📋 Multi-select and batch operations
+- 💾 Export selected messages
+
+**Keybindings:**
+- `Enter` - Toggle selection (mark/unmark message)
+- `Tab` - Open current message in pager (copyable view)
+- `Ctrl-S` - Save selected messages to `sumo-selected.txt` and exit
+- `Ctrl-Y` - Copy selected messages to clipboard and exit
+- `Ctrl-E` - Export selected messages to `sumo-export.jsonl` and exit
+- `Ctrl-A` - Select all messages
+- `Ctrl-D` - Deselect all messages
+- `Ctrl-/` - Toggle preview pane
+- `Ctrl-Q` - Quit without saving
+
+**Requirements:**
+- Install FZF: `brew install fzf` (macOS) or `apt-get install fzf` (Linux)
+- See: https://github.com/junegunn/fzf#installation
+
 ### List Collectors
 
 ```bash
@@ -215,7 +255,7 @@ Query execution time depends on data volume:
 - **[Quick Reference (tldr)](docs/tldr.md)** - Concise command examples in tldr format
 - **[Query Examples](examples/queries.md)** - Common query patterns and use cases
 - **[API Reference](docs/api-reference.md)** - Complete Ruby library documentation
-- **[Architecture](docs/architecture.md)** - How the tool works internally
+- **[Architecture](docs/architecture/)** - System design and architecture decisions
 - **[Troubleshooting](docs/troubleshooting.md)** - Common issues and solutions
 
 ## Development

data/lib/sumologic/cli.rb

@@ -26,12 +26,18 @@ module Sumologic
         sumo-query search --query '"connection timeout"' \\
           --from '2025-11-13T14:00:00' --to '2025-11-13T15:00:00' \\
           --limit 100
+
+        # Interactive mode with FZF
+        sumo-query search --query 'error' \\
+          --from '2025-11-13T14:00:00' --to '2025-11-13T15:00:00' \\
+          --interactive
     DESC
     option :query, type: :string, required: true, aliases: '-q', desc: 'Search query'
     option :from, type: :string, required: true, aliases: '-f', desc: 'Start time (ISO 8601)'
     option :to, type: :string, required: true, aliases: '-t', desc: 'End time (ISO 8601)'
     option :time_zone, type: :string, default: 'UTC', aliases: '-z', desc: 'Time zone'
-    option :limit, type: :numeric, aliases: '-l', desc: 'Limit number of messages'
+    option :limit, type: :numeric, aliases: '-l', desc: 'Maximum messages to return'
+    option :interactive, type: :boolean, aliases: '-i', desc: 'Launch interactive browser (requires fzf)'
     def search
       $DEBUG = true if options[:debug]
 
@@ -39,9 +45,17 @@ module Sumologic
 
       log_search_info
       results = execute_search(client)
-      output_search_results(results)
 
-      warn "\nMessage count: #{results.size}"
+      warn '=' * 60
+      warn "Results: #{results.size} messages"
+      warn '=' * 60
+      $stderr.puts
+
+      if options[:interactive]
+        launch_interactive_mode(results)
+      else
+        output_search_results(results)
+      end
     end
 
     desc 'list-collectors', 'List all Sumo Logic collectors'
@@ -178,9 +192,14 @@ module Sumologic
     end
 
     def log_search_info
-      warn "Querying Sumo Logic: #{options[:from]} to #{options[:to]}"
+      warn '=' * 60
+      warn 'Sumo Logic Search Query'
+      warn '=' * 60
+      warn "Time Range: #{options[:from]} to #{options[:to]}"
       warn "Query: #{options[:query]}"
-      warn 'This may take 1-3 minutes depending on data volume...'
+      warn "Limit: #{options[:limit] || 'unlimited'}"
+      warn '-' * 60
+      warn 'Creating search job...'
       $stderr.puts
     end
 
@@ -204,5 +223,24 @@ module Sumologic
         messages: results
       )
     end
+
+    def launch_interactive_mode(results)
+      require_relative 'interactive'
+
+      # Format results for interactive mode
+      formatted_results = {
+        'query' => options[:query],
+        'from' => options[:from],
+        'to' => options[:to],
+        'time_zone' => options[:time_zone],
+        'message_count' => results.size,
+        'messages' => results
+      }
+
+      Sumologic::Interactive.launch(formatted_results)
+    rescue Sumologic::Interactive::Error => e
+      error e.message
+      exit 1
+    end
   end
 end

data/lib/sumologic/client.rb

@@ -28,6 +28,8 @@ module Sumologic
 
     # Search logs with query
     # Returns array of messages
+    #
+    # @param limit [Integer, nil] Maximum number of messages to return (stops fetching after limit)
     def search(query:, from_time:, to_time:, time_zone: 'UTC', limit: nil)
       @search.execute(
         query: query,

data/lib/sumologic/configuration.rb

@@ -15,8 +15,8 @@ module Sumologic
       @deployment = ENV['SUMO_DEPLOYMENT'] || 'us2'
 
       # Search job polling
-      @initial_poll_interval = 5 # seconds - start fast for small queries
-      @max_poll_interval = 20 # seconds - slow down for large queries
+      @initial_poll_interval = 2 # seconds - aggressive polling for faster response
+      @max_poll_interval = 15 # seconds - slow down for large queries
       @poll_backoff_factor = 1.5 # increase interval by 50% each time
 
       # Timeouts and limits

data/lib/sumologic/http/client.rb

@@ -3,27 +3,36 @@
 require 'net/http'
 require 'json'
 require 'uri'
+require_relative 'connection_pool'
 
 module Sumologic
   module Http
     # Handles HTTP communication with Sumo Logic API
     # Responsibilities: request execution, error handling, SSL configuration
+    # Uses connection pooling for thread-safe parallel requests
     class Client
-      READ_TIMEOUT = 60
-      OPEN_TIMEOUT = 10
-
       def initialize(base_url:, authenticator:)
         @base_url = base_url
         @authenticator = authenticator
+        @connection_pool = ConnectionPool.new(base_url: base_url, max_connections: 10)
       end
 
       # Execute HTTP request with error handling
+      # Uses connection pool for thread-safe parallel execution
       def request(method:, path:, body: nil, query_params: nil)
         uri = build_uri(path, query_params)
         request = build_request(method, uri, body)
 
         response = execute_request(uri, request)
         handle_response(response)
+      rescue Errno::ECONNRESET, Errno::EPIPE, EOFError, Net::HTTPBadResponse => e
+        # Connection error - raise for retry at higher level
+        raise Error, "Connection error: #{e.message}"
+      end
+
+      # Close all connections in the pool
+      def close_all_connections
+        @connection_pool.close_all
       end
 
       private
@@ -55,12 +64,9 @@ module Sumologic
       end
 
       def execute_request(uri, request)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = true
-        http.read_timeout = READ_TIMEOUT
-        http.open_timeout = OPEN_TIMEOUT
-
-        http.request(request)
+        @connection_pool.with_connection(uri) do |http|
+          http.request(request)
+        end
       end
 
       def handle_response(response)

data/lib/sumologic/http/connection_pool.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Http
+    # Thread-safe connection pool for HTTP clients
+    # Allows multiple threads to have their own connections
+    class ConnectionPool
+      READ_TIMEOUT = 60
+      OPEN_TIMEOUT = 10
+
+      def initialize(base_url:, max_connections: 10)
+        @base_url = base_url
+        @max_connections = max_connections
+        @pool = []
+        @mutex = Mutex.new
+      end
+
+      # Get a connection from the pool (or create new one)
+      def with_connection(uri)
+        connection = acquire_connection(uri)
+        yield connection
+      ensure
+        release_connection(connection) if connection
+      end
+
+      # Close all connections in the pool
+      def close_all
+        @mutex.synchronize do
+          @pool.each do |conn|
+            conn[:http].finish if conn[:http].started?
+          rescue StandardError => e
+            warn "Error closing connection: #{e.message}"
+          end
+          @pool.clear
+        end
+      end
+
+      private
+
+      def acquire_connection(uri)
+        @mutex.synchronize do
+          # Try to find an available connection for this host
+          connection = find_available_connection(uri)
+          return connection[:http] if connection
+
+          # Create new connection if under limit
+          if @pool.size < @max_connections
+            http = create_connection(uri)
+            @pool << { http: http, in_use: true, host: uri.host, port: uri.port }
+            return http
+          end
+
+          # Wait and retry if pool is full
+          nil
+        end || create_temporary_connection(uri)
+      end
+
+      def find_available_connection(uri)
+        connection = @pool.find do |conn|
+          !conn[:in_use] &&
+            conn[:host] == uri.host &&
+            conn[:port] == uri.port &&
+            conn[:http].started?
+        rescue StandardError
+          # Connection is invalid
+          @pool.delete(conn)
+          nil
+        end
+
+        connection[:in_use] = true if connection
+        connection
+      end
+
+      def release_connection(http)
+        @mutex.synchronize do
+          connection = @pool.find { |conn| conn[:http] == http }
+          connection[:in_use] = false if connection
+        end
+      end
+
+      def create_connection(uri)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.read_timeout = READ_TIMEOUT
+        http.open_timeout = OPEN_TIMEOUT
+        http.keep_alive_timeout = 30
+        http.start
+        http
+      end
+
+      def create_temporary_connection(uri)
+        # Fallback: create a temporary connection if pool is exhausted
+        create_connection(uri)
+      end
+    end
+  end
+end

data/lib/sumologic/interactive/fzf_viewer.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'tempfile'
+require 'time'
+require 'open3'
+require 'shellwords'
+
+module Sumologic
+  module Interactive
+    class FzfViewer
+      DELIMITER = '||'
+
+      def initialize(results)
+        @results = results
+        @messages = results['messages'] || []
+      end
+
+      def run
+        return if @messages.empty?
+
+        Dir.mktmpdir('sumo-interactive') do |tmpdir|
+          input_file = File.join(tmpdir, 'input.txt')
+          preview_file = File.join(tmpdir, 'preview.jsonl')
+
+          prepare_data(input_file, preview_file)
+          selected = execute_fzf(input_file, preview_file)
+          handle_selection(selected) unless selected.empty?
+        end
+      end
+
+      private
+
+      def prepare_data(input_file, preview_file)
+        # Write data lines only (no header in file - handled by FZF --header)
+        File.open(input_file, 'w') do |f|
+          @messages.each do |msg|
+            f.puts format_line(msg)
+          end
+        end
+
+        # Write JSONL for preview (one JSON per line, line numbers match input)
+        File.open(preview_file, 'w') do |f|
+          @messages.each do |msg|
+            f.puts JSON.generate(msg['map'])
+          end
+        end
+      end
+
+      def format_line(msg)
+        map = msg['map']
+
+        time = format_time(map['_messagetime'])
+        level = format_level(map['level'] || map['severity'] || 'INFO')
+        source = truncate(map['_sourceCategory'] || '-', 25)
+        message = truncate(sanitize(map['_raw'] || map['message'] || ''), 80)
+
+        # No index in display - use FZF line number instead
+        "#{time} #{level} #{source.ljust(25)} #{message}"
+      end
+
+      def format_time(timestamp_ms)
+        return 'N/A' unless timestamp_ms
+
+        Time.at(timestamp_ms.to_i / 1000).strftime('%H:%M:%S')
+      end
+
+      def format_level(level)
+        level_str = level.to_s.upcase.ljust(7)
+
+        case level_str.strip
+        when 'ERROR', 'FATAL', 'CRITICAL'
+          "\e[31m#{level_str}\e[0m"  # Red
+        when 'WARN', 'WARNING'
+          "\e[33m#{level_str}\e[0m"  # Yellow
+        when 'INFO'
+          "\e[36m#{level_str}\e[0m"  # Cyan
+        when 'DEBUG', 'TRACE'
+          "\e[90m#{level_str}\e[0m"  # Gray
+        else
+          level_str
+        end
+      end
+
+      def sanitize(text)
+        text.to_s.gsub(/[\n\r\t]/, ' ').squeeze(' ')
+      end
+
+      def truncate(text, length)
+        text = text.to_s
+        text.length > length ? "#{text[0...(length - 3)]}..." : text
+      end
+
+      def colorize_json(data)
+        JSON.pretty_generate(data)
+      end
+
+      def execute_fzf(input_path, preview_path)
+        fzf_args = build_fzf_args(input_path, preview_path)
+
+        # Use IO.popen with array to avoid shell escaping issues
+        result = IO.popen(fzf_args, 'r+') do |io|
+          File.readlines(input_path).each { |line| io.puts line }
+          io.close_write
+          io.read
+        end
+
+        result.strip
+      end
+
+      def build_fzf_args(input_path, preview_path)
+        preview_cmd = build_preview_command(preview_path)
+        view_cmd = build_view_command(preview_path)
+        header_text = build_header_text
+
+        [
+          'fzf',
+          '--ansi',
+          '--multi',
+          "--header=#{header_text}",
+          "--preview=#{preview_cmd}",
+          '--preview-window=right:60%:wrap:follow',
+          '--bind=enter:toggle',
+          "--bind=tab:execute(#{view_cmd})",
+          '--bind=ctrl-a:select-all',
+          '--bind=ctrl-d:deselect-all',
+          '--bind=ctrl-s:execute-silent(echo {+} > sumo-selected.txt)+abort',
+          '--bind=ctrl-y:execute-silent(echo {+} | pbcopy || echo {+} | xclip -selection clipboard 2>/dev/null)+abort',
+          '--bind=ctrl-e:execute-silent(echo {+} > sumo-export.jsonl)+abort',
+          '--bind=ctrl-/:toggle-preview',
+          "--bind=ctrl-r:reload(cat #{input_path})",
+          '--bind=ctrl-q:abort',
+          '--height=100%'
+        ]
+      end
+
+      def build_view_command(preview_path)
+        # FZF {n} is 0-indexed! Add 1 to get sed line number (1-indexed)
+        'LINE=$(({n} + 1)); ' \
+          "sed -n \"$LINE\"p #{Shellwords.escape(preview_path)} | jq -C . | less -R"
+      end
+
+      def build_preview_command(preview_path)
+        # FZF {n} is 0-indexed! Add 1 to get JSONL line number (1-indexed)
+        escaped_path = Shellwords.escape(preview_path)
+        calc = "LINE=$(({n} + 1)); TOTAL=$(wc -l < #{escaped_path}); "
+        display = 'echo "Message $LINE of $TOTAL"; echo ""; '
+        extract = "sed -n \"$LINE\"p #{escaped_path}"
+
+        calc + display + "#{extract} | jq -C . || #{extract}"
+      end
+
+      def build_header_text
+        query = @results['query'] || 'N/A'
+        count = @messages.size
+        sources = @messages.map { |m| m['map']['_sourceCategory'] }.compact.uniq.size
+
+        # Column headers
+        columns = "#{pad('TIME', 8)} #{pad('LEVEL', 7)} #{pad('SOURCE', 25)} MESSAGE"
+        # Info and keys on second line
+        info = "#{count} msgs | #{sources} sources | Query: #{truncate(query, 40)}"
+        keys = 'Enter=select Tab=view Ctrl-S=save Ctrl-Y=copy Ctrl-E=export Ctrl-Q=quit'
+
+        "#{columns}\n#{info} | #{keys}"
+      end
+
+      def pad(text, width)
+        text.ljust(width)
+      end
+
+      def handle_selection(selected)
+        # Selected contains the actual display lines (no index field)
+        # We don't show them since user already saw in FZF
+        # The keybindings (Ctrl-S, Ctrl-Y, Ctrl-E) handle the export
+        return if selected.empty?
+
+        puts "\n#{'═' * 80}"
+        puts '📋 Exited interactive mode'
+        puts '═' * 80
+        puts "\n💡 Your selected messages were:"
+        puts '   • Saved to file (if you pressed Ctrl-S)'
+        puts '   • Copied to clipboard (if you pressed Ctrl-Y)'
+        puts '   • Exported to JSONL (if you pressed Ctrl-E)'
+      end
+    end
+  end
+end

data/lib/sumologic/interactive.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Interactive
+    class Error < StandardError; end
+
+    class << self
+      def launch(results)
+        raise Error, fzf_install_message unless fzf_available?
+
+        require_relative 'interactive/fzf_viewer'
+        FzfViewer.new(results).run
+      end
+
+      private
+
+      def fzf_available?
+        system('which fzf > /dev/null 2>&1')
+      end
+
+      def fzf_install_message
+        <<~MSG
+
+          ╔════════════════════════════════════════════════════════════╗
+          ║  Interactive mode requires FZF to be installed             ║
+          ╚════════════════════════════════════════════════════════════╝
+
+          📦 Install FZF:
+
+             macOS:    brew install fzf
+             Ubuntu:   sudo apt-get install fzf
+             Fedora:   sudo dnf install fzf
+             Arch:     sudo pacman -S fzf
+
+          🔗 Or visit: https://github.com/junegunn/fzf#installation
+
+          After installing, run your command again with -i flag.
+        MSG
+      end
+    end
+  end
+end

data/lib/sumologic/metadata/collector_source_fetcher.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative '../utils/worker'
+
+module Sumologic
+  module Metadata
+    # Fetches sources from multiple collectors efficiently
+    # Uses Worker utility for concurrent fetching
+    class CollectorSourceFetcher
+      def initialize
+        @worker = Utils::Worker.new
+      end
+
+      # Fetch sources for collectors concurrently
+      # Returns array of results with collector info and sources
+      def fetch_all(collectors, &block)
+        @worker.execute(collectors, callbacks: {
+                          start: ->(workers, total) { log_start(workers, total) },
+                          progress: ->(done, total) { log_progress(done, total) },
+                          finish: ->(results, duration) { log_finish(results.size, duration) }
+                        }, &block)
+      end
+
+      private
+
+      def log_start(workers, total)
+        warn "  Created #{workers} workers for #{total} collectors" if ENV['SUMO_DEBUG'] || $DEBUG
+      end
+
+      def log_progress(done, total)
+        return unless ENV['SUMO_DEBUG'] || $DEBUG
+
+        warn "  Progress: #{done}/#{total} collectors processed" if (done % 10).zero? || done == total
+      end
+
+      def log_finish(count, duration)
+        warn "  Fetched sources from #{count} collectors in #{duration.round(2)}s" if ENV['SUMO_DEBUG'] || $DEBUG
+      end
+    end
+  end
+end

data/lib/sumologic/metadata/source.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'parallel_fetcher'
+require_relative 'collector_source_fetcher'
 
 module Sumologic
   module Metadata
@@ -9,7 +9,7 @@ module Sumologic
       def initialize(http_client:, collector_client:)
         @http = http_client
         @collector_client = collector_client
-        @parallel_fetcher = ParallelFetcher.new(max_threads: 10)
+        @fetcher = CollectorSourceFetcher.new
       end
 
       # List sources for a specific collector
@@ -36,7 +36,7 @@ module Sumologic
 
         log_info "Fetching sources for #{active_collectors.size} active collectors in parallel..."
 
-        result = @parallel_fetcher.fetch_all(active_collectors) do |collector|
+        result = @fetcher.fetch_all(active_collectors) do |collector|
           fetch_collector_sources(collector)
         end
 

data/lib/sumologic/search/job.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'message_fetcher'
+
 module Sumologic
   module Search
     # Manages search job lifecycle: create, poll, fetch, delete
@@ -8,7 +10,7 @@ module Sumologic
         @http = http_client
         @config = config
         @poller = Poller.new(http_client: http_client, config: config)
-        @paginator = Paginator.new(http_client: http_client, config: config)
+        @message_fetcher = MessageFetcher.new(http_client: http_client, config: config)
       end
 
       # Execute a complete search workflow
@@ -16,7 +18,7 @@ module Sumologic
       def execute(query:, from_time:, to_time:, time_zone: 'UTC', limit: nil)
         job_id = create(query, from_time, to_time, time_zone)
         @poller.poll(job_id)
-        messages = @paginator.fetch_all(job_id, limit: limit)
+        messages = @message_fetcher.fetch_all(job_id, limit: limit)
         delete(job_id)
         messages
       rescue StandardError => e
@@ -57,7 +59,12 @@ module Sumologic
       end
 
       def log_info(message)
-        warn "[Sumologic::Search::Job] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
+        # Always show job creation (with ID) for user reference
+        if message.start_with?('Created search job:')
+          warn "  #{message}"
+        elsif ENV['SUMO_DEBUG'] || $DEBUG
+          warn "[Sumologic::Search::Job] #{message}"
+        end
       end
 
       def log_error(message)

data/lib/sumologic/search/message_fetcher.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative '../utils/worker'
+
+module Sumologic
+  module Search
+    # Fetches search messages with automatic pagination
+    # Uses Worker utility for concurrent page fetching when beneficial
+    class MessageFetcher
+      PAGE_SIZE = 10_000
+
+      def initialize(http_client:, config:)
+        @http = http_client
+        @config = config
+        @worker = Utils::Worker.new
+      end
+
+      # Fetch all messages for a job with automatic pagination
+      # Single page: fetches directly
+      # Multiple pages: uses Worker for concurrent fetching
+      def fetch_all(job_id, limit: nil)
+        # Fetch first page to check size
+        first_batch_limit = calculate_batch_limit(limit, 0)
+        return [] if first_batch_limit <= 0
+
+        first_batch = fetch_page(job_id, 0, first_batch_limit)
+        return [] if first_batch.empty?
+
+        # Single page result? Return immediately
+        return first_batch if first_batch.size < first_batch_limit || (limit && first_batch.size >= limit)
+
+        # Multi-page result: calculate remaining pages and fetch in parallel
+        fetch_all_pages(job_id, first_batch, limit)
+      end
+
+      private
+
+      def fetch_all_pages(job_id, first_batch, limit)
+        messages = first_batch.dup
+        offset = first_batch.size
+
+        # Calculate remaining pages to fetch
+        pages = calculate_remaining_pages(job_id, offset, limit)
+        return messages if pages.empty?
+
+        total_pages = pages.size + 1 # +1 for first page already fetched
+
+        # Fetch remaining pages in parallel using Worker with progress callbacks
+        additional_messages = @worker.execute(pages, callbacks: {
+                                                start: lambda { |workers, _total|
+                                                  warn "  Created #{workers} workers for #{total_pages} pages"
+                                                },
+                                                progress: lambda { |done, _total|
+                                                  warn "  Progress: #{done + 1}/#{total_pages} pages fetched"
+                                                },
+                                                finish: lambda { |_results, duration|
+                                                  warn "  All workers completed in #{duration.round(2)}s"
+                                                }
+                                              }) do |page|
+          fetch_page(page[:job_id], page[:offset], page[:limit])
+        end
+
+        # Flatten and combine results
+        additional_messages.each { |batch| messages.concat(batch) }
+
+        # Respect limit if specified
+        limit ? messages.first(limit) : messages
+      end
+
+      def calculate_remaining_pages(job_id, offset, limit)
+        pages = []
+        total_fetched = offset
+
+        loop do
+          batch_limit = calculate_batch_limit(limit, total_fetched)
+          break if batch_limit <= 0
+
+          pages << { job_id: job_id, offset: offset, limit: batch_limit }
+          total_fetched += batch_limit
+          offset += batch_limit
+
+          # Stop estimating if we've planned enough
+          break if pages.size >= 9 # First page + 9 more = 10 parallel fetches
+          break if limit && total_fetched >= limit
+        end
+
+        pages
+      end
+
+      def calculate_batch_limit(user_limit, total_fetched)
+        return PAGE_SIZE unless user_limit
+
+        remaining = user_limit - total_fetched
+        [PAGE_SIZE, remaining].min
+      end
+
+      def fetch_page(job_id, offset, limit)
+        data = @http.request(
+          method: :get,
+          path: "/search/jobs/#{job_id}/messages",
+          query_params: { offset: offset, limit: limit }
+        )
+
+        messages = data['messages'] || []
+        log_progress(messages.size, offset) if messages.any?
+        messages
+      end
+
+      def log_progress(batch_size, offset)
+        total = offset + batch_size
+        warn "  Fetched #{batch_size} messages (total: #{total})"
+
+        # Detailed info in debug mode
+        log_debug "  [Offset: #{offset}, batch: #{batch_size}]" if ENV['SUMO_DEBUG'] || $DEBUG
+      end
+
+      def log_debug(message)
+        warn "[Sumologic::Search::MessageFetcher] #{message}"
+      end
+    end
+  end
+end

data/lib/sumologic/search/poller.rb

@@ -63,18 +63,24 @@ module Sumologic
       end
 
       def log_poll_status(state, data, interval, count)
-        msg_count = data['messageCount']
-        rec_count = data['recordCount']
-        log_info "Job state: #{state} (#{msg_count} messages, #{rec_count} records) " \
-                 "[interval: #{interval}s, poll: #{count}]"
+        msg_count = data['messageCount'] || 0
+        rec_count = data['recordCount'] || 0
+
+        # Always show progress to user (not just in debug mode)
+        warn "  Status: #{state} | Messages: #{msg_count} | Records: #{rec_count}"
+
+        # Detailed info in debug mode
+        log_debug "  [Poll #{count + 1}, interval: #{interval}s]"
       end
 
-      def log_completion(start_time, poll_count)
+      def log_completion(start_time, _poll_count)
         elapsed = Time.now - start_time
-        log_info "Job completed in #{elapsed.round(1)} seconds after #{poll_count + 1} polls"
+        warn "Search job completed in #{elapsed.round(1)}s"
+        warn 'Fetching messages...'
+        $stderr.puts
       end
 
-      def log_info(message)
+      def log_debug(message)
         warn "[Sumologic::Search::Poller] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
       end
     end

data/lib/sumologic/utils/worker.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Utils
+    # Generic worker pool for parallel execution of tasks
+    # Uses Queue + Mutex pattern for thread-safe concurrent processing
+    #
+    # This utility abstracts the parallel execution pattern used across the codebase
+    # (metadata fetching, search pagination, etc.) into a reusable component.
+    #
+    # Example:
+    #   worker = Worker.new
+    #   results = worker.execute(items) do |item|
+    #     fetch_data(item)
+    #   end
+    class Worker
+      MAX_THREADS = 10
+
+      # Execute work items using a thread pool
+      # Returns array of results from the block execution
+      #
+      # @param items [Array] Work items to process
+      # @param callbacks [Hash] Optional callbacks for progress tracking:
+      #   - :start => ->(worker_count, total_items) { }
+      #   - :progress => ->(completed_count, total_items) { }
+      #   - :finish => ->(results, duration) { }
+      # @yield [item] Block to execute for each item
+      # @return [Array] Results from block executions (nil results are filtered out)
+      def execute(items, callbacks: {}, &block)
+        return [] if items.empty?
+
+        start_time = Time.now
+        context = {
+          result: [],
+          completed: { count: 0 },
+          mutex: Mutex.new,
+          total_items: items.size,
+          callbacks: callbacks
+        }
+
+        queue = create_work_queue(items)
+        worker_count = [MAX_THREADS, queue.size].min
+
+        # Callback: start
+        callbacks[:start]&.call(worker_count, items.size)
+
+        threads = create_workers(queue, context, &block)
+
+        threads.each(&:join)
+
+        # Callback: finish
+        duration = Time.now - start_time
+        callbacks[:finish]&.call(context[:result], duration)
+
+        context[:result]
+      end
+
+      private
+
+      def create_work_queue(items)
+        queue = Queue.new
+        items.each { |item| queue << item }
+        queue
+      end
+
+      def create_workers(queue, context, &block)
+        worker_count = [MAX_THREADS, queue.size].min
+
+        Array.new(worker_count) do
+          Thread.new { process_queue(queue, context, &block) }
+        end
+      end
+
+      def process_queue(queue, context, &block)
+        until queue.empty?
+          item = pop_safely(queue)
+          break unless item
+
+          process_item(item, context[:result], context[:mutex], &block)
+
+          # Callback: progress (thread-safe)
+          next unless context[:callbacks][:progress]
+
+          context[:mutex].synchronize do
+            context[:completed][:count] += 1
+            context[:callbacks][:progress].call(context[:completed][:count], context[:total_items])
+          end
+        end
+      end
+
+      def pop_safely(queue)
+        queue.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def process_item(item, result, mutex, &block)
+        item_result = block.call(item)
+
+        mutex.synchronize do
+          result << item_result if item_result
+        end
+      end
+    end
+  end
+end

data/lib/sumologic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sumologic
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
 end

data/lib/sumologic.rb

@@ -20,13 +20,17 @@ require_relative 'sumologic/configuration'
 require_relative 'sumologic/http/authenticator'
 require_relative 'sumologic/http/client'
 
+# Load utilities
+require_relative 'sumologic/utils/worker'
+
 # Load search domain
 require_relative 'sumologic/search/poller'
-require_relative 'sumologic/search/paginator'
+require_relative 'sumologic/search/message_fetcher'
 require_relative 'sumologic/search/job'
 
 # Load metadata domain
 require_relative 'sumologic/metadata/collector'
+require_relative 'sumologic/metadata/collector_source_fetcher'
 require_relative 'sumologic/metadata/source'
 
 # Load main client (facade)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sumologic-query
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - patrick204nqh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-11-14 00:00:00.000000000 Z
+date: 2025-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -101,12 +101,16 @@ files:
 - lib/sumologic/configuration.rb
 - lib/sumologic/http/authenticator.rb
 - lib/sumologic/http/client.rb
+- lib/sumologic/http/connection_pool.rb
+- lib/sumologic/interactive.rb
+- lib/sumologic/interactive/fzf_viewer.rb
 - lib/sumologic/metadata/collector.rb
-- lib/sumologic/metadata/parallel_fetcher.rb
+- lib/sumologic/metadata/collector_source_fetcher.rb
 - lib/sumologic/metadata/source.rb
 - lib/sumologic/search/job.rb
-- lib/sumologic/search/paginator.rb
+- lib/sumologic/search/message_fetcher.rb
 - lib/sumologic/search/poller.rb
+- lib/sumologic/utils/worker.rb
 - lib/sumologic/version.rb
 homepage: https://github.com/patrick204nqh/sumologic-query
 licenses:

data/lib/sumologic/metadata/parallel_fetcher.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-module Sumologic
-  module Metadata
-    # Handles parallel fetching of sources from multiple collectors
-    class ParallelFetcher
-      def initialize(max_threads: 10)
-        @max_threads = max_threads
-      end
-
-      # Fetch sources for collectors in parallel
-      # Returns array of results with collector info and sources
-      def fetch_all(collectors, &block)
-        result = []
-        mutex = Mutex.new
-        queue = create_work_queue(collectors)
-        threads = create_workers(queue, result, mutex, &block)
-
-        threads.each(&:join)
-        result
-      end
-
-      private
-
-      def create_work_queue(collectors)
-        queue = Queue.new
-        collectors.each { |collector| queue << collector }
-        queue
-      end
-
-      def create_workers(queue, result, mutex, &block)
-        worker_count = [@max_threads, queue.size].min
-
-        Array.new(worker_count) do
-          Thread.new { process_queue(queue, result, mutex, &block) }
-        end
-      end
-
-      def process_queue(queue, result, mutex, &block)
-        until queue.empty?
-          collector = pop_safely(queue)
-          break unless collector
-
-          process_collector(collector, result, mutex, &block)
-        end
-      end
-
-      def pop_safely(queue)
-        queue.pop(true)
-      rescue ThreadError
-        nil
-      end
-
-      def process_collector(collector, result, mutex, &block)
-        collector_result = block.call(collector)
-
-        mutex.synchronize do
-          result << collector_result if collector_result
-        end
-      end
-    end
-  end
-end

data/lib/sumologic/search/paginator.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-module Sumologic
-  module Search
-    # Handles paginated fetching of search job messages
-    class Paginator
-      def initialize(http_client:, config:)
-        @http = http_client
-        @config = config
-      end
-
-      # Fetch all messages for a job with automatic pagination
-      # Returns array of message objects
-      def fetch_all(job_id, limit: nil)
-        messages = []
-        offset = 0
-        total_fetched = 0
-
-        loop do
-          batch_limit = calculate_batch_limit(limit, total_fetched)
-          break if batch_limit <= 0
-
-          batch = fetch_batch(job_id, offset, batch_limit)
-          messages.concat(batch)
-          total_fetched += batch.size
-
-          log_progress(batch.size, total_fetched)
-
-          break if batch.size < batch_limit # No more messages
-          break if limit && total_fetched >= limit
-
-          offset += batch.size
-        end
-
-        messages
-      end
-
-      private
-
-      def calculate_batch_limit(user_limit, total_fetched)
-        if user_limit
-          [@config.max_messages_per_request, user_limit - total_fetched].min
-        else
-          @config.max_messages_per_request
-        end
-      end
-
-      def fetch_batch(job_id, offset, limit)
-        data = @http.request(
-          method: :get,
-          path: "/search/jobs/#{job_id}/messages",
-          query_params: { offset: offset, limit: limit }
-        )
-
-        data['messages'] || []
-      end
-
-      def log_progress(batch_size, total)
-        log_info "Fetched #{batch_size} messages (total: #{total})"
-      end
-
-      def log_info(message)
-        warn "[Sumologic::Search::Paginator] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
-      end
-    end
-  end
-end