sumologic-query 1.3.3 → 1.3.5

data/lib/sumologic/search/record_fetcher.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require_relative '../utils/worker'
+
+module Sumologic
+  module Search
+    # Fetches aggregation records (count by, group by results) with automatic pagination
+    # Uses Worker utility for concurrent page fetching when beneficial
+    class RecordFetcher
+      PAGE_SIZE = 10_000
+
+      def initialize(http_client:, config:)
+        @http = http_client
+        @config = config
+        @worker = Utils::Worker.new(
+          max_threads: @config.max_workers,
+          request_delay: @config.request_delay
+        )
+      end
+
+      # Fetch all records for a job with automatic pagination
+      # Used for aggregation queries (count by, group by, etc.)
+      # Single page: fetches directly
+      # Multiple pages: uses Worker for concurrent fetching
+      def fetch_all(job_id, limit: nil)
+        # Fetch first page to check size
+        first_batch_limit = calculate_batch_limit(limit, 0)
+        return [] if first_batch_limit <= 0
+
+        first_batch = fetch_page(job_id, 0, first_batch_limit)
+        return [] if first_batch.empty?
+
+        # Single page result? Return immediately
+        return first_batch if first_batch.size < first_batch_limit || (limit && first_batch.size >= limit)
+
+        # Multi-page result: calculate remaining pages and fetch in parallel
+        fetch_all_pages(job_id, first_batch, limit)
+      end
+
+      private
+
+      def fetch_all_pages(job_id, first_batch, limit)
+        records = first_batch.dup
+        offset = first_batch.size
+
+        # Calculate remaining pages to fetch
+        pages = calculate_remaining_pages(job_id, offset, limit)
+        return records if pages.empty?
+
+        total_pages = pages.size + 1 # +1 for first page already fetched
+
+        # Fetch remaining pages in parallel using Worker with progress callbacks
+        additional_records = @worker.execute(pages, callbacks: {
+                                               start: lambda { |workers, _total|
+                                                 warn "  Created #{workers} workers for #{total_pages} pages"
+                                               },
+                                               progress: lambda { |done, _total|
+                                                 warn "  Progress: #{done + 1}/#{total_pages} pages fetched"
+                                               },
+                                               finish: lambda { |_results, duration|
+                                                 warn "  All workers completed in #{duration.round(2)}s"
+                                               }
+                                             }) do |page|
+          fetch_page(page[:job_id], page[:offset], page[:limit])
+        end
+
+        # Flatten and combine results
+        additional_records.each { |batch| records.concat(batch) }
+
+        # Respect limit if specified
+        records = records.take(limit) if limit
+
+        records
+      end
+
+      def calculate_remaining_pages(job_id, offset, limit)
+        pages = []
+        total_fetched = offset
+
+        loop do
+          batch_limit = calculate_batch_limit(limit, total_fetched)
+          break if batch_limit <= 0
+
+          pages << { job_id: job_id, offset: offset, limit: batch_limit }
+          total_fetched += batch_limit
+          offset += batch_limit
+
+          # Stop estimating if we've planned enough
+          break if pages.size >= 9 # First page + 9 more = 10 parallel fetches
+          break if limit && total_fetched >= limit
+        end
+
+        pages
+      end
+
+      def calculate_batch_limit(user_limit, total_fetched)
+        return PAGE_SIZE unless user_limit
+
+        remaining = user_limit - total_fetched
+        [PAGE_SIZE, remaining].min
+      end
+
+      def fetch_page(job_id, offset, limit)
+        data = @http.request(
+          method: :get,
+          path: "/search/jobs/#{job_id}/records",
+          query_params: { offset: offset, limit: limit }
+        )
+
+        # Records endpoint returns 'records' not 'messages'
+        records = data['records'] || []
+        log_progress(records.size, offset) if records.any?
+        records
+      end
+
+      def log_progress(batch_size, offset)
+        return unless ENV['SUMO_DEBUG'] || $DEBUG
+
+        total_fetched = offset + batch_size
+        warn "[Sumologic::Search::RecordFetcher]   [Offset: #{offset}, batch: #{batch_size}]"
+        warn "  Fetched #{batch_size} records (total: #{total_fetched})" if offset.zero?
+      end
+    end
+  end
+end

data/lib/sumologic/utils/time_parser.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module Sumologic
+  module Utils
+    # Parses various time formats into ISO 8601 strings for the Sumo Logic API
+    # Supports:
+    # - 'now' - current time
+    # - Relative times: '-30s', '-5m', '-2h', '-7d', '-1w', '-1M'
+    # - Unix timestamps: '1700000000' or 1700000000
+    # - ISO 8601: '2025-11-13T14:00:00'
+    class TimeParser
+      # Time unit multipliers in seconds
+      UNITS = {
+        's' => 1,           # seconds
+        'm' => 60,          # minutes
+        'h' => 3600,        # hours
+        'd' => 86_400,       # days
+        'w' => 604_800,      # weeks (7 days)
+        'M' => 2_592_000 # months (30 days approximation)
+      }.freeze
+
+      RELATIVE_TIME_REGEX = /^([+-])(\d+)([smhdwM])$/.freeze
+
+      class ParseError < StandardError; end
+
+      # Parse a time string into ISO 8601 format
+      # @param time_str [String, Integer] Time string or Unix timestamp
+      # @param _timezone [String] IANA timezone name (default: 'UTC') - Reserved for future use
+      # @return [String] ISO 8601 formatted time string
+      def self.parse(time_str, _timezone: 'UTC')
+        return parse_now if time_str.to_s.downcase == 'now'
+
+        # Try relative time format (e.g., '-30m', '+1h')
+        if time_str.is_a?(String) && (match = time_str.match(RELATIVE_TIME_REGEX))
+          return parse_relative_time(match)
+        end
+
+        # Try Unix timestamp (integer or numeric string)
+        return parse_unix_timestamp(time_str) if unix_timestamp?(time_str)
+
+        # Try ISO 8601 format
+        begin
+          # Parse in UTC context to avoid local timezone conversion
+          parsed = Time.parse(time_str.to_s)
+          # If the input doesn't have timezone info, treat it as UTC
+          parsed = parsed.getutc unless time_str.to_s.match?(/Z|[+-]\d{2}:?\d{2}$/)
+          format_time(parsed)
+        rescue ArgumentError
+          raise ParseError,
+                "Invalid time format: '#{time_str}'. " \
+                "Supported formats: 'now', relative (e.g., '-30m'), Unix timestamp, or ISO 8601"
+        end
+      end
+
+      # Parse timezone string to standard format
+      # Accepts IANA names, offset formats, or common abbreviations
+      # @param timezone_str [String] Timezone string
+      # @return [String] Standardized timezone string
+      def self.parse_timezone(timezone_str)
+        return 'UTC' if timezone_str.nil? || timezone_str.empty?
+
+        # Handle offset formats like "+00:00", "-05:00", "+0000"
+        if timezone_str.match?(/^[+-]\d{2}:?\d{2}$/)
+          # Normalize to format with colon
+          normalized = timezone_str.sub(/^([+-]\d{2})(\d{2})$/, '\1:\2')
+          return normalized
+        end
+
+        # Map common abbreviations to IANA names
+        timezone_map = {
+          # US timezones
+          'EST' => 'America/New_York',
+          'EDT' => 'America/New_York',
+          'CST' => 'America/Chicago',
+          'CDT' => 'America/Chicago',
+          'MST' => 'America/Denver',
+          'MDT' => 'America/Denver',
+          'PST' => 'America/Los_Angeles',
+          'PDT' => 'America/Los_Angeles',
+          # Australian timezones
+          'AEST' => 'Australia/Sydney',      # Australian Eastern Standard Time
+          'AEDT' => 'Australia/Sydney',      # Australian Eastern Daylight Time
+          'ACST' => 'Australia/Adelaide',    # Australian Central Standard Time
+          'ACDT' => 'Australia/Adelaide',    # Australian Central Daylight Time
+          'AWST' => 'Australia/Perth',       # Australian Western Standard Time
+          'AWDT' => 'Australia/Perth'        # Australian Western Daylight Time (rarely used)
+        }
+
+        timezone_map[timezone_str.upcase] || timezone_str
+      end
+
+      private_class_method def self.parse_now
+        format_time(Time.now)
+      end
+
+      private_class_method def self.parse_relative_time(match)
+        sign, amount, unit = match.captures
+        amount = amount.to_i
+        amount = -amount if sign == '-'
+
+        seconds_delta = amount * UNITS[unit]
+        target_time = Time.now + seconds_delta
+
+        format_time(target_time)
+      end
+
+      private_class_method def self.parse_unix_timestamp(timestamp)
+        timestamp_int = timestamp.to_i
+
+        # Handle millisecond timestamps (13 digits) - convert to seconds
+        timestamp_int /= 1000 if timestamp.to_s.length == 13
+
+        # Validate reasonable range (between year 2000 and 2100)
+        min_timestamp = 946_684_800 # 2000-01-01
+        max_timestamp = 4_102_444_800 # 2100-01-01
+
+        unless timestamp_int.between?(min_timestamp, max_timestamp)
+          raise ParseError, "Unix timestamp out of reasonable range: #{timestamp}"
+        end
+
+        time = Time.at(timestamp_int).utc
+        format_time(time)
+      end
+
+      private_class_method def self.unix_timestamp?(value)
+        # Check if it's an integer or a string that looks like a Unix timestamp
+        # Unix timestamps are typically 10 digits (seconds) or 13 digits (milliseconds)
+        return true if value.is_a?(Integer) && value.to_s.length.between?(10, 13)
+
+        if value.is_a?(String)
+          # Must be all digits, and between 10-13 characters
+          return value.match?(/^\d{10,13}$/)
+        end
+
+        false
+      end
+
+      private_class_method def self.format_time(time)
+        # Format as ISO 8601 without timezone suffix
+        # Sumo Logic API expects format like "2025-11-13T14:00:00"
+        time.utc.strftime('%Y-%m-%dT%H:%M:%S')
+      end
+    end
+  end
+end

data/lib/sumologic/utils/worker.rb

@@ -9,12 +9,23 @@ module Sumologic
     # (metadata fetching, search pagination, etc.) into a reusable component.
     #
     # Example:
-    #   worker = Worker.new
+    #   worker = Worker.new(max_threads: 3, request_delay: 0.2)
     #   results = worker.execute(items) do |item|
     #     fetch_data(item)
     #   end
     class Worker
-      MAX_THREADS = 10
+      DEFAULT_MAX_THREADS = 10
+      DEFAULT_REQUEST_DELAY = 0.0
+
+      attr_reader :max_threads, :request_delay
+
+      # Initialize worker pool
+      # @param max_threads [Integer] Maximum number of concurrent threads
+      # @param request_delay [Float] Delay in seconds between requests (for rate limiting)
+      def initialize(max_threads: DEFAULT_MAX_THREADS, request_delay: DEFAULT_REQUEST_DELAY)
+        @max_threads = max_threads
+        @request_delay = request_delay
+      end
 
       # Execute work items using a thread pool
       # Returns array of results from the block execution
@@ -39,7 +50,7 @@ module Sumologic
         }
 
         queue = create_work_queue(items)
-        worker_count = [MAX_THREADS, queue.size].min
+        worker_count = [@max_threads, queue.size].min
 
         # Callback: start
         callbacks[:start]&.call(worker_count, items.size)
@@ -64,7 +75,7 @@ module Sumologic
       end
 
       def create_workers(queue, context, &block)
-        worker_count = [MAX_THREADS, queue.size].min
+        worker_count = [@max_threads, queue.size].min
 
         Array.new(worker_count) do
           Thread.new { process_queue(queue, context, &block) }
@@ -76,6 +87,9 @@ module Sumologic
           item = pop_safely(queue)
           break unless item
 
+          # Add delay before processing to avoid rate limits
+          sleep(@request_delay) if @request_delay.positive?
+
           process_item(item, context[:result], context[:mutex], &block)
 
           # Callback: progress (thread-safe)

data/lib/sumologic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sumologic
-  VERSION = '1.3.3'
+  VERSION = '1.3.5'
 end

data/lib/sumologic.rb

@@ -26,12 +26,16 @@ require_relative 'sumologic/utils/worker'
 # Load search domain
 require_relative 'sumologic/search/poller'
 require_relative 'sumologic/search/message_fetcher'
+require_relative 'sumologic/search/record_fetcher'
 require_relative 'sumologic/search/job'
 
 # Load metadata domain
+require_relative 'sumologic/metadata/loggable'
+require_relative 'sumologic/metadata/models'
 require_relative 'sumologic/metadata/collector'
 require_relative 'sumologic/metadata/collector_source_fetcher'
 require_relative 'sumologic/metadata/source'
+require_relative 'sumologic/metadata/dynamic_source_discovery'
 
 # Load main client (facade)
 require_relative 'sumologic/client'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sumologic-query
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.5
 platform: ruby
 authors:
 - patrick204nqh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-11-17 00:00:00.000000000 Z
+date: 2025-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -98,6 +98,7 @@ files:
 - lib/sumologic.rb
 - lib/sumologic/cli.rb
 - lib/sumologic/cli/commands/base_command.rb
+- lib/sumologic/cli/commands/discover_sources_command.rb
 - lib/sumologic/cli/commands/list_collectors_command.rb
 - lib/sumologic/cli/commands/list_sources_command.rb
 - lib/sumologic/cli/commands/search_command.rb
@@ -119,10 +120,15 @@ files:
 - lib/sumologic/interactive/fzf_viewer/searchable_builder.rb
 - lib/sumologic/metadata/collector.rb
 - lib/sumologic/metadata/collector_source_fetcher.rb
+- lib/sumologic/metadata/dynamic_source_discovery.rb
+- lib/sumologic/metadata/loggable.rb
+- lib/sumologic/metadata/models.rb
 - lib/sumologic/metadata/source.rb
 - lib/sumologic/search/job.rb
 - lib/sumologic/search/message_fetcher.rb
 - lib/sumologic/search/poller.rb
+- lib/sumologic/search/record_fetcher.rb
+- lib/sumologic/utils/time_parser.rb
 - lib/sumologic/utils/worker.rb
 - lib/sumologic/version.rb
 homepage: https://github.com/patrick204nqh/sumologic-query