sumologic-query 1.2.1 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 292e0931b6826a0cffd5b28dceac1cdd1b34fcade1112dce48cf067e9e798884
-  data.tar.gz: 9ad954fc938daf22716343d7525b290c1b29b4cdc916915404b0e9315821a62a
+  metadata.gz: ee3064549af4cc950fcd9ff873bd9f1bb9940b4e6078bc45d9086973fbab1162
+  data.tar.gz: c16964c3af7afd22e6ebdb878bc00d22cb4b6ab55f6b4a6946c0db435856e128
 SHA512:
-  metadata.gz: 39f7ce4434a5cd6706d3469e98ed264514be13923b13b7265d6912254df60369af5954d2103babdcb394a24dda6ff94e288caff551e651b13411524b9cdff9e8
-  data.tar.gz: 8c718fa5ff299ff50bdfa94636c45e4e1b9fa2f7f3120fe4325b6a4069d18cce61d6f324c0917ab5eacf37fc61b3662c5740055710adc2403b2a238ad13ddffa
+  metadata.gz: 2a8c9c6b61000afc3d3ba204215b054332c7efc001bb1fc0a3a349e161b2ac9f9f36b7794ca561b0564b6b3b56245bf8ed9f630a7fe57fdba086ddba4f94f7ce
+  data.tar.gz: 6466d1a6180686719d614022fc908db2a2a8949594192c41bdf87e1a8c4c3d6d93bf754e4ef00e4479467f15c2eee5b4f240086afc200ef78bfad1786444b427

data/CHANGELOG.md

@@ -1,15 +1,20 @@
-# Changelog
+## [1.2.2](https://github.com/patrick204nqh/sumologic-query/compare/v1.2.1...v1.2.2) (2025-11-15)
 
-All notable changes to this project are documented in [GitHub Releases](https://github.com/patrick204nqh/sumologic-query/releases).
 
-This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Releases
+## [1.2.1](https://github.com/patrick204nqh/sumologic-query/compare/v1.2.0...v1.2.1) (2025-11-14)
+
+
+
+# [1.2.0](https://github.com/patrick204nqh/sumologic-query/compare/v1.1.2...v1.2.0) (2025-11-14)
+
+
+
+## [1.1.2](https://github.com/patrick204nqh/sumologic-query/compare/v1.1.1...v1.1.2) (2025-11-14)
+
+
+
+## [1.1.1](https://github.com/patrick204nqh/sumologic-query/compare/v1.1.0...v1.1.1) (2025-11-14)
 
-- [v1.1.0](https://github.com/patrick204nqh/sumologic-query/releases/tag/v1.1.0) - Latest
-- [v1.0.0](https://github.com/patrick204nqh/sumologic-query/releases/tag/v1.0.0) - Initial release
 
----
 
-**Note:** Release notes are automatically generated from commit messages and pull requests.
-See [GitHub Releases](https://github.com/patrick204nqh/sumologic-query/releases) for detailed changelogs.

data/README.md

@@ -86,7 +86,8 @@ sumo-query search --query "YOUR_QUERY" \
   --to "END_TIME" \
   [--output FILE] \
   [--limit N] \
-  [--time-zone TZ]
+  [--time-zone TZ] \
+  [--interactive]
 ```
 
 **Required options:**
@@ -95,11 +96,50 @@ sumo-query search --query "YOUR_QUERY" \
 - `-t, --to TIME` - End time (ISO 8601 format)
 
 **Optional options:**
+- `-i, --interactive` - Launch interactive browser with FZF
 - `-z, --time-zone TZ` - Time zone (default: UTC)
 - `-l, --limit N` - Limit number of messages
 - `-o, --output FILE` - Save to file (default: stdout)
 - `-d, --debug` - Enable debug output
 
+### Interactive Mode 🚀
+
+Explore your logs interactively with a powerful FZF-based interface:
+
+```bash
+# Launch interactive mode
+sumo-query search --query 'error' \
+  --from '2025-11-13T14:00:00' \
+  --to '2025-11-13T15:00:00' \
+  --interactive
+
+# Or use the shorthand
+sumo-query search -q 'error' -f '2025-11-13T14:00:00' -t '2025-11-13T15:00:00' -i
+```
+
+**Features:**
+- 🔍 Fuzzy search across all message fields
+- 👁️ Live preview with full JSON details
+- 🎨 Color-coded log levels (ERROR, WARN, INFO)
+- ⌨️ Keyboard shortcuts for quick actions
+- 📋 Multi-select and batch operations
+- 💾 Export selected messages
+
+**Keybindings:**
+- `Enter` - Toggle selection (mark/unmark message)
+- `Tab` - Open current message in pager (copyable view)
+- `Ctrl-S` - Save selected messages to `sumo-selected.txt` and exit
+- `Ctrl-Y` - Copy selected messages to clipboard and exit
+- `Ctrl-E` - Export selected messages to `sumo-export.jsonl` and exit
+- `Ctrl-A` - Select all messages
+- `Ctrl-D` - Deselect all messages
+- `Ctrl-/` - Toggle preview pane
+- `Ctrl-Q` - Quit without saving
+
+**Requirements:**
+- Install FZF: `brew install fzf` (macOS) or `apt-get install fzf` (Linux)
+- See: https://github.com/junegunn/fzf#installation
+
 ### List Collectors
 
 ```bash
@@ -215,7 +255,7 @@ Query execution time depends on data volume:
 - **[Quick Reference (tldr)](docs/tldr.md)** - Concise command examples in tldr format
 - **[Query Examples](examples/queries.md)** - Common query patterns and use cases
 - **[API Reference](docs/api-reference.md)** - Complete Ruby library documentation
-- **[Architecture](docs/architecture.md)** - How the tool works internally
+- **[Architecture](docs/architecture/)** - System design and architecture decisions
 - **[Troubleshooting](docs/troubleshooting.md)** - Common issues and solutions
 
 ## Development

data/lib/sumologic/cli.rb

@@ -26,12 +26,18 @@ module Sumologic
         sumo-query search --query '"connection timeout"' \\
           --from '2025-11-13T14:00:00' --to '2025-11-13T15:00:00' \\
           --limit 100
+
+        # Interactive mode with FZF
+        sumo-query search --query 'error' \\
+          --from '2025-11-13T14:00:00' --to '2025-11-13T15:00:00' \\
+          --interactive
     DESC
     option :query, type: :string, required: true, aliases: '-q', desc: 'Search query'
     option :from, type: :string, required: true, aliases: '-f', desc: 'Start time (ISO 8601)'
     option :to, type: :string, required: true, aliases: '-t', desc: 'End time (ISO 8601)'
     option :time_zone, type: :string, default: 'UTC', aliases: '-z', desc: 'Time zone'
-    option :limit, type: :numeric, aliases: '-l', desc: 'Limit number of messages'
+    option :limit, type: :numeric, aliases: '-l', desc: 'Maximum messages to return'
+    option :interactive, type: :boolean, aliases: '-i', desc: 'Launch interactive browser (requires fzf)'
     def search
       $DEBUG = true if options[:debug]
 
@@ -39,9 +45,17 @@ module Sumologic
 
       log_search_info
       results = execute_search(client)
-      output_search_results(results)
 
-      warn "\nMessage count: #{results.size}"
+      warn '=' * 60
+      warn "Results: #{results.size} messages"
+      warn '=' * 60
+      $stderr.puts
+
+      if options[:interactive]
+        launch_interactive_mode(results)
+      else
+        output_search_results(results)
+      end
     end
 
     desc 'list-collectors', 'List all Sumo Logic collectors'
@@ -178,9 +192,14 @@ module Sumologic
     end
 
     def log_search_info
-      warn "Querying Sumo Logic: #{options[:from]} to #{options[:to]}"
+      warn '=' * 60
+      warn 'Sumo Logic Search Query'
+      warn '=' * 60
+      warn "Time Range: #{options[:from]} to #{options[:to]}"
       warn "Query: #{options[:query]}"
-      warn 'This may take 1-3 minutes depending on data volume...'
+      warn "Limit: #{options[:limit] || 'unlimited'}"
+      warn '-' * 60
+      warn 'Creating search job...'
       $stderr.puts
     end
 
@@ -204,5 +223,24 @@ module Sumologic
         messages: results
       )
     end
+
+    def launch_interactive_mode(results)
+      require_relative 'interactive'
+
+      # Format results for interactive mode
+      formatted_results = {
+        'query' => options[:query],
+        'from' => options[:from],
+        'to' => options[:to],
+        'time_zone' => options[:time_zone],
+        'message_count' => results.size,
+        'messages' => results
+      }
+
+      Sumologic::Interactive.launch(formatted_results)
+    rescue Sumologic::Interactive::Error => e
+      error e.message
+      exit 1
+    end
   end
 end

data/lib/sumologic/client.rb

@@ -28,6 +28,8 @@ module Sumologic
 
     # Search logs with query
     # Returns array of messages
+    #
+    # @param limit [Integer, nil] Maximum number of messages to return (stops fetching after limit)
     def search(query:, from_time:, to_time:, time_zone: 'UTC', limit: nil)
       @search.execute(
         query: query,
@@ -38,25 +40,6 @@ module Sumologic
       )
     end
 
-    # Search logs with streaming interface
-    # Returns an Enumerator that yields messages one at a time
-    # More memory efficient for large result sets
-    #
-    # Example:
-    #   client.search_stream(query: 'error', from_time: ..., to_time: ...).each do |message|
-    #     puts message['map']['message']
-    #   end
-    def search_stream(query:, from_time:, to_time:, time_zone: 'UTC', limit: nil)
-      job_id = @search.create_and_wait(
-        query: query,
-        from_time: from_time,
-        to_time: to_time,
-        time_zone: time_zone
-      )
-
-      @search.stream_messages(job_id, limit: limit)
-    end
-
     # List all collectors
     # Returns array of collector objects
     def list_collectors

data/lib/sumologic/configuration.rb

@@ -4,7 +4,7 @@ module Sumologic
   # Centralized configuration for Sumo Logic client
   class Configuration
     attr_accessor :access_id, :access_key, :deployment, :timeout, :initial_poll_interval, :max_poll_interval,
-                  :poll_backoff_factor, :max_messages_per_request, :enable_parallel_pagination
+                  :poll_backoff_factor, :max_messages_per_request
 
     API_VERSION = 'v1'
 
@@ -15,18 +15,13 @@ module Sumologic
       @deployment = ENV['SUMO_DEPLOYMENT'] || 'us2'
 
       # Search job polling
-      @initial_poll_interval = 5 # seconds - start fast for small queries
-      @max_poll_interval = 20 # seconds - slow down for large queries
+      @initial_poll_interval = 2 # seconds - aggressive polling for faster response
+      @max_poll_interval = 15 # seconds - slow down for large queries
       @poll_backoff_factor = 1.5 # increase interval by 50% each time
 
       # Timeouts and limits
       @timeout = 300 # seconds (5 minutes)
       @max_messages_per_request = 10_000
-
-      # Performance options
-      # Parallel pagination enabled by default for better performance
-      # Uses connection pooling for thread-safe concurrent requests
-      @enable_parallel_pagination = true
     end
 
     def base_url

data/lib/sumologic/interactive/fzf_viewer.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'tempfile'
+require 'time'
+require 'open3'
+require 'shellwords'
+
+module Sumologic
+  module Interactive
+    class FzfViewer
+      DELIMITER = '||'
+
+      def initialize(results)
+        @results = results
+        @messages = results['messages'] || []
+      end
+
+      def run
+        return if @messages.empty?
+
+        Dir.mktmpdir('sumo-interactive') do |tmpdir|
+          input_file = File.join(tmpdir, 'input.txt')
+          preview_file = File.join(tmpdir, 'preview.jsonl')
+
+          prepare_data(input_file, preview_file)
+          selected = execute_fzf(input_file, preview_file)
+          handle_selection(selected) unless selected.empty?
+        end
+      end
+
+      private
+
+      def prepare_data(input_file, preview_file)
+        # Write data lines only (no header in file - handled by FZF --header)
+        File.open(input_file, 'w') do |f|
+          @messages.each do |msg|
+            f.puts format_line(msg)
+          end
+        end
+
+        # Write JSONL for preview (one JSON per line, line numbers match input)
+        File.open(preview_file, 'w') do |f|
+          @messages.each do |msg|
+            f.puts JSON.generate(msg['map'])
+          end
+        end
+      end
+
+      def format_line(msg)
+        map = msg['map']
+
+        time = format_time(map['_messagetime'])
+        level = format_level(map['level'] || map['severity'] || 'INFO')
+        source = truncate(map['_sourceCategory'] || '-', 25)
+        message = truncate(sanitize(map['_raw'] || map['message'] || ''), 80)
+
+        # No index in display - use FZF line number instead
+        "#{time} #{level} #{source.ljust(25)} #{message}"
+      end
+
+      def format_time(timestamp_ms)
+        return 'N/A' unless timestamp_ms
+
+        Time.at(timestamp_ms.to_i / 1000).strftime('%H:%M:%S')
+      end
+
+      def format_level(level)
+        level_str = level.to_s.upcase.ljust(7)
+
+        case level_str.strip
+        when 'ERROR', 'FATAL', 'CRITICAL'
+          "\e[31m#{level_str}\e[0m"  # Red
+        when 'WARN', 'WARNING'
+          "\e[33m#{level_str}\e[0m"  # Yellow
+        when 'INFO'
+          "\e[36m#{level_str}\e[0m"  # Cyan
+        when 'DEBUG', 'TRACE'
+          "\e[90m#{level_str}\e[0m"  # Gray
+        else
+          level_str
+        end
+      end
+
+      def sanitize(text)
+        text.to_s.gsub(/[\n\r\t]/, ' ').squeeze(' ')
+      end
+
+      def truncate(text, length)
+        text = text.to_s
+        text.length > length ? "#{text[0...(length - 3)]}..." : text
+      end
+
+      def colorize_json(data)
+        JSON.pretty_generate(data)
+      end
+
+      def execute_fzf(input_path, preview_path)
+        fzf_args = build_fzf_args(input_path, preview_path)
+
+        # Use IO.popen with array to avoid shell escaping issues
+        result = IO.popen(fzf_args, 'r+') do |io|
+          File.readlines(input_path).each { |line| io.puts line }
+          io.close_write
+          io.read
+        end
+
+        result.strip
+      end
+
+      def build_fzf_args(input_path, preview_path)
+        preview_cmd = build_preview_command(preview_path)
+        view_cmd = build_view_command(preview_path)
+        header_text = build_header_text
+
+        [
+          'fzf',
+          '--ansi',
+          '--multi',
+          "--header=#{header_text}",
+          "--preview=#{preview_cmd}",
+          '--preview-window=right:60%:wrap:follow',
+          '--bind=enter:toggle',
+          "--bind=tab:execute(#{view_cmd})",
+          '--bind=ctrl-a:select-all',
+          '--bind=ctrl-d:deselect-all',
+          '--bind=ctrl-s:execute-silent(echo {+} > sumo-selected.txt)+abort',
+          '--bind=ctrl-y:execute-silent(echo {+} | pbcopy || echo {+} | xclip -selection clipboard 2>/dev/null)+abort',
+          '--bind=ctrl-e:execute-silent(echo {+} > sumo-export.jsonl)+abort',
+          '--bind=ctrl-/:toggle-preview',
+          "--bind=ctrl-r:reload(cat #{input_path})",
+          '--bind=ctrl-q:abort',
+          '--height=100%'
+        ]
+      end
+
+      def build_view_command(preview_path)
+        # FZF {n} is 0-indexed! Add 1 to get sed line number (1-indexed)
+        'LINE=$(({n} + 1)); ' \
+          "sed -n \"$LINE\"p #{Shellwords.escape(preview_path)} | jq -C . | less -R"
+      end
+
+      def build_preview_command(preview_path)
+        # FZF {n} is 0-indexed! Add 1 to get JSONL line number (1-indexed)
+        escaped_path = Shellwords.escape(preview_path)
+        calc = "LINE=$(({n} + 1)); TOTAL=$(wc -l < #{escaped_path}); "
+        display = 'echo "Message $LINE of $TOTAL"; echo ""; '
+        extract = "sed -n \"$LINE\"p #{escaped_path}"
+
+        calc + display + "#{extract} | jq -C . || #{extract}"
+      end
+
+      def build_header_text
+        query = @results['query'] || 'N/A'
+        count = @messages.size
+        sources = @messages.map { |m| m['map']['_sourceCategory'] }.compact.uniq.size
+
+        # Column headers
+        columns = "#{pad('TIME', 8)} #{pad('LEVEL', 7)} #{pad('SOURCE', 25)} MESSAGE"
+        # Info and keys on second line
+        info = "#{count} msgs | #{sources} sources | Query: #{truncate(query, 40)}"
+        keys = 'Enter=select Tab=view Ctrl-S=save Ctrl-Y=copy Ctrl-E=export Ctrl-Q=quit'
+
+        "#{columns}\n#{info} | #{keys}"
+      end
+
+      def pad(text, width)
+        text.ljust(width)
+      end
+
+      def handle_selection(selected)
+        # Selected contains the actual display lines (no index field)
+        # We don't show them since user already saw in FZF
+        # The keybindings (Ctrl-S, Ctrl-Y, Ctrl-E) handle the export
+        return if selected.empty?
+
+        puts "\n#{'═' * 80}"
+        puts '📋 Exited interactive mode'
+        puts '═' * 80
+        puts "\n💡 Your selected messages were:"
+        puts '   • Saved to file (if you pressed Ctrl-S)'
+        puts '   • Copied to clipboard (if you pressed Ctrl-Y)'
+        puts '   • Exported to JSONL (if you pressed Ctrl-E)'
+      end
+    end
+  end
+end

data/lib/sumologic/interactive.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Interactive
+    class Error < StandardError; end
+
+    class << self
+      def launch(results)
+        raise Error, fzf_install_message unless fzf_available?
+
+        require_relative 'interactive/fzf_viewer'
+        FzfViewer.new(results).run
+      end
+
+      private
+
+      def fzf_available?
+        system('which fzf > /dev/null 2>&1')
+      end
+
+      def fzf_install_message
+        <<~MSG
+
+          ╔════════════════════════════════════════════════════════════╗
+          ║  Interactive mode requires FZF to be installed             ║
+          ╚════════════════════════════════════════════════════════════╝
+
+          📦 Install FZF:
+
+             macOS:    brew install fzf
+             Ubuntu:   sudo apt-get install fzf
+             Fedora:   sudo dnf install fzf
+             Arch:     sudo pacman -S fzf
+
+          🔗 Or visit: https://github.com/junegunn/fzf#installation
+
+          After installing, run your command again with -i flag.
+        MSG
+      end
+    end
+  end
+end

data/lib/sumologic/metadata/collector_source_fetcher.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative '../utils/worker'
+
+module Sumologic
+  module Metadata
+    # Fetches sources from multiple collectors efficiently
+    # Uses Worker utility for concurrent fetching
+    class CollectorSourceFetcher
+      def initialize
+        @worker = Utils::Worker.new
+      end
+
+      # Fetch sources for collectors concurrently
+      # Returns array of results with collector info and sources
+      def fetch_all(collectors, &block)
+        @worker.execute(collectors, callbacks: {
+                          start: ->(workers, total) { log_start(workers, total) },
+                          progress: ->(done, total) { log_progress(done, total) },
+                          finish: ->(results, duration) { log_finish(results.size, duration) }
+                        }, &block)
+      end
+
+      private
+
+      def log_start(workers, total)
+        warn "  Created #{workers} workers for #{total} collectors" if ENV['SUMO_DEBUG'] || $DEBUG
+      end
+
+      def log_progress(done, total)
+        return unless ENV['SUMO_DEBUG'] || $DEBUG
+
+        warn "  Progress: #{done}/#{total} collectors processed" if (done % 10).zero? || done == total
+      end
+
+      def log_finish(count, duration)
+        warn "  Fetched sources from #{count} collectors in #{duration.round(2)}s" if ENV['SUMO_DEBUG'] || $DEBUG
+      end
+    end
+  end
+end

data/lib/sumologic/metadata/source.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'parallel_fetcher'
+require_relative 'collector_source_fetcher'
 
 module Sumologic
   module Metadata
@@ -9,7 +9,7 @@ module Sumologic
       def initialize(http_client:, collector_client:)
         @http = http_client
         @collector_client = collector_client
-        @parallel_fetcher = ParallelFetcher.new(max_threads: 10)
+        @fetcher = CollectorSourceFetcher.new
       end
 
       # List sources for a specific collector
@@ -36,7 +36,7 @@ module Sumologic
 
         log_info "Fetching sources for #{active_collectors.size} active collectors in parallel..."
 
-        result = @parallel_fetcher.fetch_all(active_collectors) do |collector|
+        result = @fetcher.fetch_all(active_collectors) do |collector|
           fetch_collector_sources(collector)
         end
 

data/lib/sumologic/search/job.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'stream'
+require_relative 'message_fetcher'
 
 module Sumologic
   module Search
@@ -10,8 +10,7 @@ module Sumologic
         @http = http_client
         @config = config
         @poller = Poller.new(http_client: http_client, config: config)
-        @paginator = Paginator.new(http_client: http_client, config: config)
-        @stream = Stream.new(paginator: @paginator)
+        @message_fetcher = MessageFetcher.new(http_client: http_client, config: config)
       end
 
       # Execute a complete search workflow
@@ -19,7 +18,7 @@ module Sumologic
       def execute(query:, from_time:, to_time:, time_zone: 'UTC', limit: nil)
         job_id = create(query, from_time, to_time, time_zone)
         @poller.poll(job_id)
-        messages = @paginator.fetch_all(job_id, limit: limit)
+        messages = @message_fetcher.fetch_all(job_id, limit: limit)
         delete(job_id)
         messages
       rescue StandardError => e
@@ -27,22 +26,6 @@ module Sumologic
         raise Error, "Search failed: #{e.message}"
       end
 
-      # Create job and wait for completion
-      # Returns job_id for use with streaming
-      def create_and_wait(query:, from_time:, to_time:, time_zone: 'UTC')
-        job_id = create(query, from_time, to_time, time_zone)
-        @poller.poll(job_id)
-        job_id
-      end
-
-      # Stream messages from a completed job
-      # Returns an Enumerator
-      def stream_messages(job_id, limit: nil)
-        @stream.each(job_id, limit: limit)
-      ensure
-        delete(job_id)
-      end
-
       private
 
       def create(query, from_time, to_time, time_zone)
@@ -76,7 +59,12 @@ module Sumologic
       end
 
       def log_info(message)
-        warn "[Sumologic::Search::Job] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
+        # Always show job creation (with ID) for user reference
+        if message.start_with?('Created search job:')
+          warn "  #{message}"
+        elsif ENV['SUMO_DEBUG'] || $DEBUG
+          warn "[Sumologic::Search::Job] #{message}"
+        end
       end
 
       def log_error(message)

data/lib/sumologic/search/message_fetcher.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative '../utils/worker'
+
+module Sumologic
+  module Search
+    # Fetches search messages with automatic pagination
+    # Uses Worker utility for concurrent page fetching when beneficial
+    class MessageFetcher
+      PAGE_SIZE = 10_000
+
+      def initialize(http_client:, config:)
+        @http = http_client
+        @config = config
+        @worker = Utils::Worker.new
+      end
+
+      # Fetch all messages for a job with automatic pagination
+      # Single page: fetches directly
+      # Multiple pages: uses Worker for concurrent fetching
+      def fetch_all(job_id, limit: nil)
+        # Fetch first page to check size
+        first_batch_limit = calculate_batch_limit(limit, 0)
+        return [] if first_batch_limit <= 0
+
+        first_batch = fetch_page(job_id, 0, first_batch_limit)
+        return [] if first_batch.empty?
+
+        # Single page result? Return immediately
+        return first_batch if first_batch.size < first_batch_limit || (limit && first_batch.size >= limit)
+
+        # Multi-page result: calculate remaining pages and fetch in parallel
+        fetch_all_pages(job_id, first_batch, limit)
+      end
+
+      private
+
+      def fetch_all_pages(job_id, first_batch, limit)
+        messages = first_batch.dup
+        offset = first_batch.size
+
+        # Calculate remaining pages to fetch
+        pages = calculate_remaining_pages(job_id, offset, limit)
+        return messages if pages.empty?
+
+        total_pages = pages.size + 1 # +1 for first page already fetched
+
+        # Fetch remaining pages in parallel using Worker with progress callbacks
+        additional_messages = @worker.execute(pages, callbacks: {
+                                                start: lambda { |workers, _total|
+                                                  warn "  Created #{workers} workers for #{total_pages} pages"
+                                                },
+                                                progress: lambda { |done, _total|
+                                                  warn "  Progress: #{done + 1}/#{total_pages} pages fetched"
+                                                },
+                                                finish: lambda { |_results, duration|
+                                                  warn "  All workers completed in #{duration.round(2)}s"
+                                                }
+                                              }) do |page|
+          fetch_page(page[:job_id], page[:offset], page[:limit])
+        end
+
+        # Flatten and combine results
+        additional_messages.each { |batch| messages.concat(batch) }
+
+        # Respect limit if specified
+        limit ? messages.first(limit) : messages
+      end
+
+      def calculate_remaining_pages(job_id, offset, limit)
+        pages = []
+        total_fetched = offset
+
+        loop do
+          batch_limit = calculate_batch_limit(limit, total_fetched)
+          break if batch_limit <= 0
+
+          pages << { job_id: job_id, offset: offset, limit: batch_limit }
+          total_fetched += batch_limit
+          offset += batch_limit
+
+          # Stop estimating if we've planned enough
+          break if pages.size >= 9 # First page + 9 more = 10 parallel fetches
+          break if limit && total_fetched >= limit
+        end
+
+        pages
+      end
+
+      def calculate_batch_limit(user_limit, total_fetched)
+        return PAGE_SIZE unless user_limit
+
+        remaining = user_limit - total_fetched
+        [PAGE_SIZE, remaining].min
+      end
+
+      def fetch_page(job_id, offset, limit)
+        data = @http.request(
+          method: :get,
+          path: "/search/jobs/#{job_id}/messages",
+          query_params: { offset: offset, limit: limit }
+        )
+
+        messages = data['messages'] || []
+        log_progress(messages.size, offset) if messages.any?
+        messages
+      end
+
+      def log_progress(batch_size, offset)
+        total = offset + batch_size
+        warn "  Fetched #{batch_size} messages (total: #{total})"
+
+        # Detailed info in debug mode
+        log_debug "  [Offset: #{offset}, batch: #{batch_size}]" if ENV['SUMO_DEBUG'] || $DEBUG
+      end
+
+      def log_debug(message)
+        warn "[Sumologic::Search::MessageFetcher] #{message}"
+      end
+    end
+  end
+end

data/lib/sumologic/search/poller.rb

@@ -63,18 +63,24 @@ module Sumologic
       end
 
       def log_poll_status(state, data, interval, count)
-        msg_count = data['messageCount']
-        rec_count = data['recordCount']
-        log_info "Job state: #{state} (#{msg_count} messages, #{rec_count} records) " \
-                 "[interval: #{interval}s, poll: #{count}]"
+        msg_count = data['messageCount'] || 0
+        rec_count = data['recordCount'] || 0
+
+        # Always show progress to user (not just in debug mode)
+        warn "  Status: #{state} | Messages: #{msg_count} | Records: #{rec_count}"
+
+        # Detailed info in debug mode
+        log_debug "  [Poll #{count + 1}, interval: #{interval}s]"
       end
 
-      def log_completion(start_time, poll_count)
+      def log_completion(start_time, _poll_count)
         elapsed = Time.now - start_time
-        log_info "Job completed in #{elapsed.round(1)} seconds after #{poll_count + 1} polls"
+        warn "Search job completed in #{elapsed.round(1)}s"
+        warn 'Fetching messages...'
+        $stderr.puts
       end
 
-      def log_info(message)
+      def log_debug(message)
         warn "[Sumologic::Search::Poller] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
       end
     end

data/lib/sumologic/utils/worker.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Utils
+    # Generic worker pool for parallel execution of tasks
+    # Uses Queue + Mutex pattern for thread-safe concurrent processing
+    #
+    # This utility abstracts the parallel execution pattern used across the codebase
+    # (metadata fetching, search pagination, etc.) into a reusable component.
+    #
+    # Example:
+    #   worker = Worker.new
+    #   results = worker.execute(items) do |item|
+    #     fetch_data(item)
+    #   end
+    class Worker
+      MAX_THREADS = 10
+
+      # Execute work items using a thread pool
+      # Returns array of results from the block execution
+      #
+      # @param items [Array] Work items to process
+      # @param callbacks [Hash] Optional callbacks for progress tracking:
+      #   - :start => ->(worker_count, total_items) { }
+      #   - :progress => ->(completed_count, total_items) { }
+      #   - :finish => ->(results, duration) { }
+      # @yield [item] Block to execute for each item
+      # @return [Array] Results from block executions (nil results are filtered out)
+      def execute(items, callbacks: {}, &block)
+        return [] if items.empty?
+
+        start_time = Time.now
+        context = {
+          result: [],
+          completed: { count: 0 },
+          mutex: Mutex.new,
+          total_items: items.size,
+          callbacks: callbacks
+        }
+
+        queue = create_work_queue(items)
+        worker_count = [MAX_THREADS, queue.size].min
+
+        # Callback: start
+        callbacks[:start]&.call(worker_count, items.size)
+
+        threads = create_workers(queue, context, &block)
+
+        threads.each(&:join)
+
+        # Callback: finish
+        duration = Time.now - start_time
+        callbacks[:finish]&.call(context[:result], duration)
+
+        context[:result]
+      end
+
+      private
+
+      def create_work_queue(items)
+        queue = Queue.new
+        items.each { |item| queue << item }
+        queue
+      end
+
+      def create_workers(queue, context, &block)
+        worker_count = [MAX_THREADS, queue.size].min
+
+        Array.new(worker_count) do
+          Thread.new { process_queue(queue, context, &block) }
+        end
+      end
+
+      def process_queue(queue, context, &block)
+        until queue.empty?
+          item = pop_safely(queue)
+          break unless item
+
+          process_item(item, context[:result], context[:mutex], &block)
+
+          # Callback: progress (thread-safe)
+          next unless context[:callbacks][:progress]
+
+          context[:mutex].synchronize do
+            context[:completed][:count] += 1
+            context[:callbacks][:progress].call(context[:completed][:count], context[:total_items])
+          end
+        end
+      end
+
+      def pop_safely(queue)
+        queue.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def process_item(item, result, mutex, &block)
+        item_result = block.call(item)
+
+        mutex.synchronize do
+          result << item_result if item_result
+        end
+      end
+    end
+  end
+end

data/lib/sumologic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sumologic
-  VERSION = '1.2.1'
+  VERSION = '1.3.1'
 end

data/lib/sumologic.rb

@@ -20,13 +20,17 @@ require_relative 'sumologic/configuration'
 require_relative 'sumologic/http/authenticator'
 require_relative 'sumologic/http/client'
 
+# Load utilities
+require_relative 'sumologic/utils/worker'
+
 # Load search domain
 require_relative 'sumologic/search/poller'
-require_relative 'sumologic/search/paginator'
+require_relative 'sumologic/search/message_fetcher'
 require_relative 'sumologic/search/job'
 
 # Load metadata domain
 require_relative 'sumologic/metadata/collector'
+require_relative 'sumologic/metadata/collector_source_fetcher'
 require_relative 'sumologic/metadata/source'
 
 # Load main client (facade)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sumologic-query
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.1
 platform: ruby
 authors:
 - patrick204nqh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-11-14 00:00:00.000000000 Z
+date: 2025-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -102,13 +102,15 @@ files:
 - lib/sumologic/http/authenticator.rb
 - lib/sumologic/http/client.rb
 - lib/sumologic/http/connection_pool.rb
+- lib/sumologic/interactive.rb
+- lib/sumologic/interactive/fzf_viewer.rb
 - lib/sumologic/metadata/collector.rb
-- lib/sumologic/metadata/parallel_fetcher.rb
+- lib/sumologic/metadata/collector_source_fetcher.rb
 - lib/sumologic/metadata/source.rb
 - lib/sumologic/search/job.rb
-- lib/sumologic/search/paginator.rb
+- lib/sumologic/search/message_fetcher.rb
 - lib/sumologic/search/poller.rb
-- lib/sumologic/search/stream.rb
+- lib/sumologic/utils/worker.rb
 - lib/sumologic/version.rb
 homepage: https://github.com/patrick204nqh/sumologic-query
 licenses:

data/lib/sumologic/metadata/parallel_fetcher.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-module Sumologic
-  module Metadata
-    # Handles parallel fetching of sources from multiple collectors
-    class ParallelFetcher
-      def initialize(max_threads: 10)
-        @max_threads = max_threads
-      end
-
-      # Fetch sources for collectors in parallel
-      # Returns array of results with collector info and sources
-      def fetch_all(collectors, &block)
-        result = []
-        mutex = Mutex.new
-        queue = create_work_queue(collectors)
-        threads = create_workers(queue, result, mutex, &block)
-
-        threads.each(&:join)
-        result
-      end
-
-      private
-
-      def create_work_queue(collectors)
-        queue = Queue.new
-        collectors.each { |collector| queue << collector }
-        queue
-      end
-
-      def create_workers(queue, result, mutex, &block)
-        worker_count = [@max_threads, queue.size].min
-
-        Array.new(worker_count) do
-          Thread.new { process_queue(queue, result, mutex, &block) }
-        end
-      end
-
-      def process_queue(queue, result, mutex, &block)
-        until queue.empty?
-          collector = pop_safely(queue)
-          break unless collector
-
-          process_collector(collector, result, mutex, &block)
-        end
-      end
-
-      def pop_safely(queue)
-        queue.pop(true)
-      rescue ThreadError
-        nil
-      end
-
-      def process_collector(collector, result, mutex, &block)
-        collector_result = block.call(collector)
-
-        mutex.synchronize do
-          result << collector_result if collector_result
-        end
-      end
-    end
-  end
-end

data/lib/sumologic/search/paginator.rb

@@ -1,170 +0,0 @@
-# frozen_string_literal: true
-
-module Sumologic
-  module Search
-    # Handles paginated fetching of search job messages
-    # Supports both sequential and parallel pagination
-    class Paginator
-      # Number of pages to fetch in parallel
-      PARALLEL_BATCH_SIZE = 5
-
-      def initialize(http_client:, config:)
-        @http = http_client
-        @config = config
-      end
-
-      # Fetch all messages for a job with automatic pagination
-      # Uses parallel fetching for better performance on large result sets (if enabled)
-      # Returns array of message objects
-      def fetch_all(job_id, limit: nil)
-        # Check if parallel pagination is enabled and appropriate
-        if should_use_parallel?(limit)
-          fetch_parallel(job_id, limit: limit)
-        else
-          fetch_sequential(job_id, limit: limit)
-        end
-      end
-
-      private
-
-      # Check if we should use parallel fetching
-      def should_use_parallel?(limit)
-        return false unless @config.enable_parallel_pagination
-
-        # Only use parallel for large result sets (over 20K messages / 2 pages)
-        !limit || limit >= @config.max_messages_per_request * 2
-      end
-
-      # Sequential fetching (original implementation)
-      def fetch_sequential(job_id, limit: nil)
-        messages = []
-        offset = 0
-        total_fetched = 0
-
-        loop do
-          batch_limit = calculate_batch_limit(limit, total_fetched)
-          break if batch_limit <= 0
-
-          batch = fetch_batch(job_id, offset, batch_limit)
-          messages.concat(batch)
-          total_fetched += batch.size
-
-          log_progress(batch.size, total_fetched)
-
-          break if batch.size < batch_limit # No more messages
-          break if limit && total_fetched >= limit
-
-          offset += batch.size
-        end
-
-        messages
-      end
-
-      # Parallel fetching for large result sets
-      def fetch_parallel(job_id, limit: nil)
-        messages = []
-        total_fetched = 0
-
-        loop do
-          pages_to_fetch = calculate_parallel_pages(limit, total_fetched)
-          break if pages_to_fetch.empty?
-
-          batches = fetch_batches_parallel(job_id, pages_to_fetch)
-          total_fetched = process_batches(batches, messages, total_fetched)
-
-          break if done_fetching?(batches, limit, total_fetched)
-        end
-
-        messages
-      end
-
-      # Process fetched batches and update counters
-      def process_batches(batches, messages, total_fetched)
-        batches.each do |batch|
-          messages.concat(batch[:messages])
-          total_fetched += batch[:messages].size
-        end
-
-        log_progress(batches.sum { |b| b[:messages].size }, total_fetched)
-        total_fetched
-      end
-
-      # Check if we're done fetching messages
-      def done_fetching?(batches, limit, total_fetched)
-        last_batch = batches.last
-        return true if last_batch[:messages].size < last_batch[:limit]
-        return true if limit && total_fetched >= limit
-
-        false
-      end
-
-      # Calculate which pages to fetch in parallel
-      def calculate_parallel_pages(limit, total_fetched)
-        pages = []
-        offset = total_fetched
-
-        PARALLEL_BATCH_SIZE.times do
-          batch_limit = calculate_batch_limit(limit, offset)
-          break if batch_limit <= 0
-
-          pages << { offset: offset, limit: batch_limit }
-          offset += batch_limit
-
-          break if limit && offset >= limit
-        end
-
-        pages
-      end
-
-      # Fetch multiple batches in parallel
-      def fetch_batches_parallel(job_id, pages)
-        results = []
-        mutex = Mutex.new
-        threads = pages.map do |page|
-          Thread.new do
-            batch_messages = fetch_batch(job_id, page[:offset], page[:limit])
-
-            mutex.synchronize do
-              results << {
-                offset: page[:offset],
-                limit: page[:limit],
-                messages: batch_messages
-              }
-            end
-          end
-        end
-
-        threads.each(&:join)
-
-        # Sort by offset to maintain order
-        results.sort_by { |r| r[:offset] }
-      end
-
-      def calculate_batch_limit(user_limit, total_fetched)
-        if user_limit
-          [@config.max_messages_per_request, user_limit - total_fetched].min
-        else
-          @config.max_messages_per_request
-        end
-      end
-
-      def fetch_batch(job_id, offset, limit)
-        data = @http.request(
-          method: :get,
-          path: "/search/jobs/#{job_id}/messages",
-          query_params: { offset: offset, limit: limit }
-        )
-
-        data['messages'] || []
-      end
-
-      def log_progress(batch_size, total)
-        log_info "Fetched #{batch_size} messages (total: #{total})"
-      end
-
-      def log_info(message)
-        warn "[Sumologic::Search::Paginator] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
-      end
-    end
-  end
-end

data/lib/sumologic/search/stream.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-module Sumologic
-  module Search
-    # Provides streaming interface for search results
-    # Returns an Enumerator that yields messages as they are fetched
-    # Reduces memory usage by not loading all results at once
-    class Stream
-      def initialize(paginator:)
-        @paginator = paginator
-      end
-
-      # Create an enumerator that streams messages from a job
-      # Yields messages one at a time as pages are fetched
-      def each(job_id, limit: nil, &block)
-        return enum_for(:each, job_id, limit: limit) unless block_given?
-
-        stream_messages(job_id, limit: limit, &block)
-      end
-
-      private
-
-      def stream_messages(job_id, limit: nil)
-        offset = 0
-        total_yielded = 0
-
-        loop do
-          batch_limit = calculate_batch_limit(limit, total_yielded)
-          break if batch_limit <= 0
-
-          batch = fetch_batch(job_id, offset, batch_limit)
-          break if batch.empty?
-
-          total_yielded = yield_batch_messages(batch, total_yielded, limit, &Proc.new)
-
-          break if done_streaming?(batch, batch_limit, limit, total_yielded)
-
-          offset += batch.size
-        end
-      end
-
-      # Yield messages from batch and return updated count
-      def yield_batch_messages(batch, total_yielded, limit)
-        batch.each do |message|
-          yield message
-          total_yielded += 1
-          break if limit_reached?(limit, total_yielded)
-        end
-        total_yielded
-      end
-
-      # Check if we've reached the limit
-      def limit_reached?(limit, total_yielded)
-        limit && total_yielded >= limit
-      end
-
-      # Check if we're done streaming
-      def done_streaming?(batch, batch_limit, limit, total_yielded)
-        return true if batch.size < batch_limit # No more messages
-        return true if limit_reached?(limit, total_yielded)
-
-        false
-      end
-
-      def calculate_batch_limit(user_limit, total_yielded)
-        page_size = @paginator.instance_variable_get(:@config).max_messages_per_request
-
-        if user_limit
-          [page_size, user_limit - total_yielded].min
-        else
-          page_size
-        end
-      end
-
-      def fetch_batch(job_id, offset, limit)
-        @paginator.send(:fetch_batch, job_id, offset, limit)
-      end
-    end
-  end
-end