sumologic-query 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 94a41b01c95f9975b2caa3fd6e35ed59f3a0a50ea1bef0a382984db8b28a2df1
-  data.tar.gz: 2a2af32a87f880dfbea771d08e1ba0aff16015a14da80742ee90f8bcaa8f653c
+  metadata.gz: 7b7bbf8f091fa6d3c6e213a34c1ba72952c43aa995814834379c30a051d3f4f5
+  data.tar.gz: 4e7bc3f985d01543a5af7ce89d7746944107d4fd99ecf4367c276a5e550dbdca
 SHA512:
-  metadata.gz: 42e019ecf001cf27fe9c4a6fb09ab3830c94e4d198b150ea9a34854a4cb77c72b0d769b61cce671dc3fb548b2d5bf6f7b8df97fcea08175bde3897f205ff8d24
-  data.tar.gz: 5f83a1f886d1b2d3e995be899b9b093bfb03738b9626ca5a5c7c2c06fecf88da037d017fe894b5c0ad014ec3a9a5bdabc0173eeb748b6e07bb013452e1cfd833
+  metadata.gz: 4f34b22f610d453649fff4926c7dfdff0fd2d5ff18a2e5fa6994cf1f41635990c28ace805fb2508520a1f016899c99f71ba31be2d5b0771a8b30d9e781f310a2
+  data.tar.gz: 0a12a47f8ddfeda0bab912f2011a033b121e01f3c63b19d9fa80e7cfa0a7df32a7277784a92a88ca988819a1f42c717c61dbbae79d5bab154161d9cff5f571a8

data/README.md

@@ -1,19 +1,20 @@
 # Sumo Logic Query Tool
 
-> A lightweight Ruby CLI for querying Sumo Logic logs quickly. Simple, fast, read-only access to your logs.
+> A lightweight Ruby CLI for querying Sumo Logic logs and metadata. Simple, fast, read-only access to your logs.
 
 [![Gem Version](https://badge.fury.io/rb/sumologic-query.svg)](https://badge.fury.io/rb/sumologic-query)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 ## Why This Tool?
 
-- **Zero dependencies**: Uses only Ruby stdlib - no external gems required
+- **Minimal dependencies**: Uses only Ruby stdlib + Thor for CLI
 - **Fast queries**: Efficient polling and automatic pagination
 - **Simple interface**: Just query, get results, done
 - **Read-only**: No write operations, perfect for safe log access
-- **Lightweight**: ~300 lines of code total
+- **Modular architecture**: Clean separation of concerns (HTTP, Search, Metadata)
+- **Metadata support**: List collectors and sources alongside log queries
 
-All existing Ruby Sumo Logic gems are unmaintained (2-9 years dormant). This tool provides a fresh, minimal approach focused solely on querying logs.
+All existing Ruby Sumo Logic gems are unmaintained (2-9 years dormant). This tool provides a fresh, minimal approach focused on querying logs and metadata.
 
 ## Installation
 
@@ -48,7 +49,7 @@ Export your Sumo Logic API credentials:
 ```bash
 export SUMO_ACCESS_ID="your_access_id"
 export SUMO_ACCESS_KEY="your_access_key"
-export SUMO_DEPLOYMENT="us2"  # Optional: us1, us2 (default), eu, au
+export SUMO_DEPLOYMENT="us2"  # Optional: us1, us2 (default), eu, au, etc.
 ```
 
 **Getting credentials:**
@@ -60,18 +61,27 @@ export SUMO_DEPLOYMENT="us2"  # Optional: us1, us2 (default), eu, au
 ### 2. Run Your First Query
 
 ```bash
-sumo-query --query 'error' \
+# Search logs
+sumo-query search --query 'error' \
   --from '2025-11-13T14:00:00' \
   --to '2025-11-13T15:00:00' \
   --limit 10
+
+# List collectors
+sumo-query collectors
+
+# List sources
+sumo-query sources
 ```
 
 ## Usage
 
-### Basic Command Structure
+The CLI provides three main commands:
+
+### Search Logs
 
 ```bash
-sumo-query --query "YOUR_QUERY" \
+sumo-query search --query "YOUR_QUERY" \
   --from "START_TIME" \
   --to "END_TIME" \
   [--output FILE] \
@@ -79,87 +89,85 @@ sumo-query --query "YOUR_QUERY" \
   [--time-zone TZ]
 ```
 
-### Required Options
-
+**Required options:**
 - `-q, --query QUERY` - Sumo Logic query string
-- `-f, --from TIME` - Start time in ISO 8601 format
-- `-t, --to TIME` - End time in ISO 8601 format
-
-### Optional Options
+- `-f, --from TIME` - Start time (ISO 8601 format)
+- `-t, --to TIME` - End time (ISO 8601 format)
 
+**Optional options:**
 - `-z, --time-zone TZ` - Time zone (default: UTC)
 - `-l, --limit N` - Limit number of messages
-- `-o, --output FILE` - Save results to file (default: stdout)
+- `-o, --output FILE` - Save to file (default: stdout)
 - `-d, --debug` - Enable debug output
-- `-h, --help` - Show help message
-- `-v, --version` - Show version
 
-## Common Query Patterns
-
-### Error Analysis
+### List Collectors
 
 ```bash
-# Find all errors in a time window
-sumo-query --query 'error' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00' \
-  --output errors.json
-
-# Error timeline with 5-minute buckets
-sumo-query --query 'error | timeslice 5m | count by _timeslice' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
+sumo-query collectors [--output FILE]
 ```
 
-### Text Search
+Lists all collectors in your account with status and metadata.
 
-```bash
-# Search for specific text
-sumo-query --query '"connection timeout"' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
+### List Sources
 
-# Case-insensitive search
-sumo-query --query 'timeout OR failure OR exception' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
+```bash
+sumo-query sources [--output FILE]
 ```
 
-### Filtering by Source
+Lists all sources from active collectors.
 
-```bash
-# Filter by source category
-sumo-query --query '_sourceCategory=prod/api error' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
+**See [examples/queries.md](examples/queries.md) for more query patterns and examples.**
 
-# Multiple sources
-sumo-query --query '(_sourceCategory=prod/api OR _sourceCategory=prod/web) AND error' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
-```
+## Ruby Library Usage
 
-### Aggregation Queries
+Use the library directly in your Ruby code:
 
-```bash
-# Count by field
-sumo-query --query '* | count by status_code' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
+```ruby
+require 'sumologic'
 
-# Top 10 slowest requests
-sumo-query --query 'duration_ms > 1000 | sort by duration_ms desc | limit 10' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
+# Initialize client
+client = Sumologic::Client.new(
+  access_id: ENV['SUMO_ACCESS_ID'],
+  access_key: ENV['SUMO_ACCESS_KEY'],
+  deployment: 'us2'
+)
+
+# Search logs
+results = client.search(
+  query: 'error',
+  from_time: '2025-11-13T14:00:00',
+  to_time: '2025-11-13T15:00:00',
+  time_zone: 'UTC',
+  limit: 1000
+)
+
+results.each do |message|
+  puts message['map']['message']
+end
+
+# List collectors
+collectors = client.list_collectors
+
+# List all sources
+sources = client.list_all_sources
 ```
 
-### Parsing and Field Extraction
+**See [docs/api-reference.md](docs/api-reference.md) for complete API documentation.**
+
+## Time Formats
+
+Use ISO 8601 format for timestamps:
 
 ```bash
-# Parse specific fields
-sumo-query --query '* | parse "user_id=* " as user_id | count by user_id' \
-  --from '2025-11-13T14:00:00' \
-  --to '2025-11-13T15:00:00'
+# UTC timestamps (default)
+--from "2025-11-13T14:30:00" --to "2025-11-13T15:00:00"
+
+# With timezone
+--from "2025-11-13T14:30:00" --time-zone "America/New_York"
+
+# Using shell helpers
+--from "$(date -u -v-1H '+%Y-%m-%dT%H:%M:%S')"  # 1 hour ago
+--to "$(date -u '+%Y-%m-%dT%H:%M:%S')"          # now
 ```
 
 ## Output Format
@@ -186,157 +194,54 @@ Results are returned as JSON:
 }
 ```
 
-## Time Formats
-
-Use ISO 8601 format for timestamps:
-
-```bash
-# UTC timestamps (default)
---from "2025-11-13T14:30:00"
---to "2025-11-13T15:00:00"
-
-# With timezone
---from "2025-11-13T14:30:00" --time-zone "America/New_York"
-
-# Alternative: Relative times (in your shell)
---from "$(date -u -v-1H '+%Y-%m-%dT%H:%M:%S')"  # 1 hour ago
---to "$(date -u '+%Y-%m-%dT%H:%M:%S')"         # now
-```
-
 ## Performance
 
 Query execution time depends on data volume:
 
-- **Small queries** (<10K messages): ~30-60 seconds
-- **Medium queries** (10K-100K): ~1-2 minutes
-- **Large queries** (100K+): ~2-5 minutes
+| Messages | Typical Time |
+|----------|--------------|
+| < 10K | 30-60 seconds |
+| 10K-100K | 1-2 minutes |
+| 100K+ | 2-5 minutes |
 
-Default timeout: 5 minutes
-
-To improve performance:
+**Tips for faster queries:**
 - Narrow your time range
-- Add specific `_sourceCategory` filters
+- Add `_sourceCategory` filters
 - Use `--limit` to cap results
-- Use aggregation queries instead of fetching raw messages
-
-## Troubleshooting
+- Use aggregation queries instead of raw messages
 
-### Authentication Error
+## Documentation
 
-```
-Error: SUMO_ACCESS_ID not set
-```
-
-**Solution**: Export your credentials:
-```bash
-export SUMO_ACCESS_ID="your_access_id"
-export SUMO_ACCESS_KEY="your_access_key"
-```
-
-### Timeout Error
-
-```
-Timeout Error: Search job timed out after 300 seconds
-```
-
-**Solutions**:
-- Reduce time range
-- Add more specific filters (`_sourceCategory`, `_sourceName`)
-- Use `--limit` to cap results
-- Consider using aggregation instead of raw messages
-
-### Empty Results
-
-```json
-{
-  "message_count": 0,
-  "messages": []
-}
-```
-
-**Check**:
-- Time range matches your expected data
-- Query syntax is valid (test in Sumo Logic UI first)
-- Source categories are correct
-- Time zone is correct (default is UTC)
-
-### Rate Limit Error
-
-```
-HTTP 429: Rate limit exceeded
-```
-
-**Solution**: Wait 1-2 minutes between queries. Sumo Logic enforces rate limits per account.
+- **[Quick Reference (tldr)](docs/tldr.md)** - Concise command examples in tldr format
+- **[Query Examples](examples/queries.md)** - Common query patterns and use cases
+- **[API Reference](docs/api-reference.md)** - Complete Ruby library documentation
+- **[Architecture](docs/architecture.md)** - How the tool works internally
+- **[Troubleshooting](docs/troubleshooting.md)** - Common issues and solutions
 
 ## Development
 
-### Running Tests
+See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, testing, and contribution guidelines.
+
+Quick start:
 
 ```bash
+# Clone and install
+git clone https://github.com/patrick204nqh/sumologic-query.git
+cd sumologic-query
 bundle install
-bundle exec rake spec
-```
 
-### Code Quality
+# Run tests
+bundle exec rspec
 
-```bash
+# Run linter
 bundle exec rubocop
-bundle exec rubocop -A  # Auto-fix issues
-```
 
-### Running Locally
-
-```bash
-# Without installing
-bundle exec bin/sumo-query --query "error" \
-  --from "2025-11-13T14:00:00" \
-  --to "2025-11-13T15:00:00"
-
-# With debug output
-SUMO_DEBUG=1 bundle exec bin/sumo-query --query "error" \
+# Test locally
+bundle exec bin/sumo-query search --query "error" \
   --from "2025-11-13T14:00:00" \
   --to "2025-11-13T15:00:00"
 ```
 
-## How It Works
-
-This tool uses the Sumo Logic Search Job API:
-
-1. **Create Job** - POST to `/api/v1/search/jobs` with your query
-2. **Poll Status** - GET `/api/v1/search/jobs/:id` every 20 seconds until complete
-3. **Fetch Messages** - GET `/api/v1/search/jobs/:id/messages` (automatically paginated)
-4. **Clean Up** - DELETE `/api/v1/search/jobs/:id`
-
-All steps are handled automatically. You just provide the query and get results.
-
-## API Reference
-
-### Ruby Library Usage
-
-You can also use the library directly in your Ruby code:
-
-```ruby
-require 'sumologic'
-
-client = Sumologic::Client.new(
-  access_id: 'your_access_id',
-  access_key: 'your_access_key',
-  deployment: 'us2'
-)
-
-results = client.search(
-  query: 'error',
-  from_time: '2025-11-13T14:00:00',
-  to_time: '2025-11-13T15:00:00',
-  time_zone: 'UTC',
-  limit: 1000
-)
-
-results.each do |message|
-  puts message['map']['message']
-end
-```
-
 ## Contributing
 
 Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for details.
@@ -351,17 +256,17 @@ Contributions are welcome! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for det
 
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
 
+## Support
+
+- **Issues**: [GitHub Issues](https://github.com/patrick204nqh/sumologic-query/issues)
+- **Discussions**: [GitHub Discussions](https://github.com/patrick204nqh/sumologic-query/discussions)
+- **Documentation**: [docs/](docs/)
+
 ## Resources
 
 - **Sumo Logic API Docs**: https://help.sumologic.com/docs/api/search-job/
 - **Query Language**: https://help.sumologic.com/docs/search/
 - **Bug Reports**: https://github.com/patrick204nqh/sumologic-query/issues
-- **Feature Requests**: https://github.com/patrick204nqh/sumologic-query/issues
-
-## Support
-
-- **Issues**: [GitHub Issues](https://github.com/patrick204nqh/sumologic-query/issues)
-- **Discussions**: [GitHub Discussions](https://github.com/patrick204nqh/sumologic-query/discussions)
 
 ---
 

data/lib/sumologic/cli.rb

@@ -89,6 +89,12 @@ module Sumologic
       end
     end
 
+    desc 'version', 'Show version information'
+    def version
+      puts "sumo-query version #{Sumologic::VERSION}"
+    end
+    map %w[-v --version] => :version
+
     default_task :search
 
     private

data/lib/sumologic/metadata/parallel_fetcher.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Metadata
+    # Handles parallel fetching of sources from multiple collectors
+    class ParallelFetcher
+      def initialize(max_threads: 10)
+        @max_threads = max_threads
+      end
+
+      # Fetch sources for collectors in parallel
+      # Returns array of results with collector info and sources
+      def fetch_all(collectors, &block)
+        result = []
+        mutex = Mutex.new
+        queue = create_work_queue(collectors)
+        threads = create_workers(queue, result, mutex, &block)
+
+        threads.each(&:join)
+        result
+      end
+
+      private
+
+      def create_work_queue(collectors)
+        queue = Queue.new
+        collectors.each { |collector| queue << collector }
+        queue
+      end
+
+      def create_workers(queue, result, mutex, &block)
+        worker_count = [@max_threads, queue.size].min
+
+        Array.new(worker_count) do
+          Thread.new { process_queue(queue, result, mutex, &block) }
+        end
+      end
+
+      def process_queue(queue, result, mutex, &block)
+        until queue.empty?
+          collector = pop_safely(queue)
+          break unless collector
+
+          process_collector(collector, result, mutex, &block)
+        end
+      end
+
+      def pop_safely(queue)
+        queue.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def process_collector(collector, result, mutex, &block)
+        collector_result = block.call(collector)
+
+        mutex.synchronize do
+          result << collector_result if collector_result
+        end
+      end
+    end
+  end
+end

data/lib/sumologic/metadata/source.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'parallel_fetcher'
+
 module Sumologic
   module Metadata
     # Handles source metadata operations
@@ -7,6 +9,7 @@ module Sumologic
       def initialize(http_client:, collector_client:)
         @http = http_client
         @collector_client = collector_client
+        @parallel_fetcher = ParallelFetcher.new(max_threads: 10)
       end
 
       # List sources for a specific collector
@@ -26,30 +29,15 @@ module Sumologic
 
       # List all sources from all collectors
       # Returns array of hashes with collector info and their sources
+      # Uses parallel fetching with thread pool for better performance
       def list_all
         collectors = @collector_client.list
-        result = []
-
-        collectors.each do |collector|
-          next unless collector['alive'] # Skip offline collectors
-
-          collector_id = collector['id']
-          collector_name = collector['name']
+        active_collectors = collectors.select { |c| c['alive'] }
 
-          log_info "Fetching sources for collector: #{collector_name} (#{collector_id})"
+        log_info "Fetching sources for #{active_collectors.size} active collectors in parallel..."
 
-          sources = list(collector_id: collector_id)
-
-          result << {
-            'collector' => {
-              'id' => collector_id,
-              'name' => collector_name,
-              'collectorType' => collector['collectorType']
-            },
-            'sources' => sources
-          }
-        rescue StandardError => e
-          log_error "Failed to fetch sources for collector #{collector_name}: #{e.message}"
+        result = @parallel_fetcher.fetch_all(active_collectors) do |collector|
+          fetch_collector_sources(collector)
         end
 
         log_info "Total: #{result.size} collectors with sources"
@@ -60,6 +48,27 @@ module Sumologic
 
       private
 
+      # Fetch sources for a single collector
+      def fetch_collector_sources(collector)
+        collector_id = collector['id']
+        collector_name = collector['name']
+
+        log_info "Fetching sources for collector: #{collector_name} (#{collector_id})"
+        sources = list(collector_id: collector_id)
+
+        {
+          'collector' => {
+            'id' => collector_id,
+            'name' => collector_name,
+            'collectorType' => collector['collectorType']
+          },
+          'sources' => sources
+        }
+      rescue StandardError => e
+        log_error "Failed to fetch sources for collector #{collector_name}: #{e.message}"
+        nil
+      end
+
       def log_info(message)
         warn "[Sumologic::Metadata::Source] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
       end

data/lib/sumologic/search/poller.rb

@@ -11,6 +11,7 @@ module Sumologic
 
       # Poll until job completes or times out
       # Returns final job status data
+      # Starts polling immediately, then applies exponential backoff
       def poll(job_id)
         start_time = Time.now
         interval = @config.initial_poll_interval
@@ -32,6 +33,7 @@ module Sumologic
             raise Error, "Search job #{state.downcase}"
           end
 
+          # Sleep after checking status (not before first check)
           sleep interval
           poll_count += 1
           interval = calculate_next_interval(interval)

data/lib/sumologic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sumologic
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sumologic-query
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - patrick204nqh
@@ -102,6 +102,7 @@ files:
 - lib/sumologic/http/authenticator.rb
 - lib/sumologic/http/client.rb
 - lib/sumologic/metadata/collector.rb
+- lib/sumologic/metadata/parallel_fetcher.rb
 - lib/sumologic/metadata/source.rb
 - lib/sumologic/search/job.rb
 - lib/sumologic/search/paginator.rb