sumologic-query 1.3.4 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +85 -270
- data/lib/sumologic/cli/commands/discover_sources_command.rb +59 -0
- data/lib/sumologic/cli.rb +40 -0
- data/lib/sumologic/client.rb +23 -1
- data/lib/sumologic/configuration.rb +5 -1
- data/lib/sumologic/metadata/collector.rb +5 -6
- data/lib/sumologic/metadata/collector_source_fetcher.rb +7 -3
- data/lib/sumologic/metadata/dynamic_source_discovery.rb +155 -0
- data/lib/sumologic/metadata/loggable.rb +32 -0
- data/lib/sumologic/metadata/models.rb +108 -0
- data/lib/sumologic/metadata/source.rb +13 -18
- data/lib/sumologic/search/job.rb +17 -1
- data/lib/sumologic/search/message_fetcher.rb +4 -1
- data/lib/sumologic/search/record_fetcher.rb +125 -0
- data/lib/sumologic/utils/worker.rb +18 -4
- data/lib/sumologic/version.rb +1 -1
- data/lib/sumologic.rb +4 -0
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 83e78d35f43a7e12f98e3ed11d9d4b0ddd7fe4cf567c281fa4729c8873c583a0
|
|
4
|
+
data.tar.gz: 4cc242bc523b5ce0cf709ec9f9aceb44cb1fe5911582983ae80e3a49d00a175d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7b9d63d998ff14e1b9db86dc982dd84b508f642c01a62883b7bdce997ec21aed3097477956776319f01b68da2e8ed967b713894ed8002353ab99ee9fd72bad85
|
|
7
|
+
data.tar.gz: '084c1c7db713d88f88979941d8b6a261f48c4bc533f3ce2ac04cb97ea3bae5fb0762889611dbfffd790740e0c7928acffb9e7df144c6e6d7b0fa129128d49e23'
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,23 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
|
|
|
6
6
|
|
|
7
7
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
8
8
|
and release notes are automatically generated from commit messages.
|
|
9
|
+
## [1.3.5](https://github.com/patrick204nqh/sumologic-query/compare/v1.3.4...v1.3.5) (2025-11-19)
|
|
10
|
+
|
|
11
|
+
### 🎉 New Features
|
|
12
|
+
|
|
13
|
+
- add discover-sources command for dynamic source discovery from logs
|
|
14
|
+
- implement rate limiting configuration and enhance documentation for querying options
|
|
15
|
+
|
|
16
|
+
### 🔧 Refactoring
|
|
17
|
+
|
|
18
|
+
- improve source discovery logic and enhance debugging output
|
|
19
|
+
|
|
20
|
+
### 📚 Documentation
|
|
21
|
+
|
|
22
|
+
- update tldr.md with enhanced search options and new commands for querying logs
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
9
26
|
## [1.3.4](https://github.com/patrick204nqh/sumologic-query/compare/v1.3.3...v1.3.4) (2025-11-19)
|
|
10
27
|
|
|
11
28
|
### 🎉 New Features
|
data/README.md
CHANGED
|
@@ -6,351 +6,166 @@
|
|
|
6
6
|
[](https://rubygems.org/gems/sumologic-query)
|
|
7
7
|
[](https://opensource.org/licenses/MIT)
|
|
8
8
|
|
|
9
|
-
##
|
|
9
|
+
## Features
|
|
10
10
|
|
|
11
|
-
- **
|
|
12
|
-
- **
|
|
13
|
-
- **
|
|
14
|
-
- **
|
|
15
|
-
- **
|
|
16
|
-
- **
|
|
17
|
-
- **Read-only**: No write operations, perfect for safe log access
|
|
18
|
-
- **Modular architecture**: Clean separation of concerns (HTTP, Search, Metadata)
|
|
19
|
-
- **Metadata support**: List collectors and sources alongside log queries
|
|
20
|
-
|
|
21
|
-
All existing Ruby Sumo Logic gems are unmaintained (2-9 years dormant). This tool provides a fresh, minimal approach focused on querying logs and metadata.
|
|
11
|
+
- **Simple time parsing** - Use `-1h`, `-30m`, `now` instead of timestamps
|
|
12
|
+
- **Dynamic source discovery** - Find CloudWatch/ECS/Lambda sources from logs
|
|
13
|
+
- **Interactive mode** - Explore logs with FZF fuzzy search
|
|
14
|
+
- **Timezone support** - US, Australian, and IANA formats
|
|
15
|
+
- **Fast & efficient** - Smart polling and pagination
|
|
16
|
+
- **Read-only** - Safe log access with no write operations
|
|
22
17
|
|
|
23
18
|
## Installation
|
|
24
19
|
|
|
25
|
-
### Via RubyGems
|
|
26
|
-
|
|
27
20
|
```bash
|
|
21
|
+
# Via RubyGems
|
|
28
22
|
gem install sumologic-query
|
|
29
|
-
```
|
|
30
23
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
```bash
|
|
24
|
+
# Via Homebrew
|
|
34
25
|
brew tap patrick204nqh/tap
|
|
35
26
|
brew install sumologic-query
|
|
36
27
|
```
|
|
37
28
|
|
|
38
|
-
### From Source
|
|
39
|
-
|
|
40
|
-
```bash
|
|
41
|
-
git clone https://github.com/patrick204nqh/sumologic-query.git
|
|
42
|
-
cd sumologic-query
|
|
43
|
-
bundle install
|
|
44
|
-
bundle exec rake install
|
|
45
|
-
```
|
|
46
|
-
|
|
47
29
|
## Quick Start
|
|
48
30
|
|
|
49
|
-
### 1. Set
|
|
50
|
-
|
|
51
|
-
Export your Sumo Logic API credentials:
|
|
31
|
+
### 1. Set Credentials
|
|
52
32
|
|
|
53
33
|
```bash
|
|
54
34
|
export SUMO_ACCESS_ID="your_access_id"
|
|
55
35
|
export SUMO_ACCESS_KEY="your_access_key"
|
|
56
|
-
export SUMO_DEPLOYMENT="us2" # Optional: us1, us2 (default), eu, au
|
|
36
|
+
export SUMO_DEPLOYMENT="us2" # Optional: us1, us2 (default), eu, au
|
|
57
37
|
```
|
|
58
38
|
|
|
59
|
-
**
|
|
60
|
-
1. Log in to Sumo Logic
|
|
61
|
-
2. Go to **Administration → Security → Access Keys**
|
|
62
|
-
3. Create a new access key or use existing
|
|
63
|
-
4. Copy the Access ID and Access Key
|
|
39
|
+
Get credentials: Sumo Logic → **Administration → Security → Access Keys**
|
|
64
40
|
|
|
65
|
-
### 2. Run
|
|
41
|
+
### 2. Run Queries
|
|
66
42
|
|
|
67
43
|
```bash
|
|
68
|
-
# Search logs
|
|
69
|
-
sumo-query search
|
|
70
|
-
|
|
71
|
-
# Search logs from last 30 minutes
|
|
72
|
-
sumo-query search --query 'error' --from '-30m' --to 'now'
|
|
44
|
+
# Search logs
|
|
45
|
+
sumo-query search -q 'error' -f '-1h' -t 'now' --limit 100
|
|
73
46
|
|
|
74
|
-
#
|
|
75
|
-
sumo-query
|
|
76
|
-
--from '2025-11-13T14:00:00' \
|
|
77
|
-
--to '2025-11-13T15:00:00' \
|
|
78
|
-
--limit 10
|
|
47
|
+
# Discover dynamic sources (CloudWatch/ECS/Lambda)
|
|
48
|
+
sumo-query discover-sources
|
|
79
49
|
|
|
80
|
-
# List collectors
|
|
50
|
+
# List collectors and sources
|
|
81
51
|
sumo-query collectors
|
|
82
|
-
|
|
83
|
-
# List sources
|
|
84
52
|
sumo-query sources
|
|
85
53
|
```
|
|
86
54
|
|
|
87
|
-
##
|
|
55
|
+
## Commands
|
|
88
56
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
### Search Logs
|
|
57
|
+
### 1. Search Logs
|
|
92
58
|
|
|
93
59
|
```bash
|
|
94
|
-
sumo-query search
|
|
95
|
-
--from "START_TIME" \
|
|
96
|
-
--to "END_TIME" \
|
|
97
|
-
[--output FILE] \
|
|
98
|
-
[--limit N] \
|
|
99
|
-
[--time-zone TZ] \
|
|
100
|
-
[--interactive]
|
|
60
|
+
sumo-query search -q "YOUR_QUERY" -f "START" -t "END" [OPTIONS]
|
|
101
61
|
```
|
|
102
62
|
|
|
103
|
-
**
|
|
104
|
-
- `-q, --query
|
|
105
|
-
- `-f, --from
|
|
106
|
-
- `-t, --to
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
- `-
|
|
110
|
-
- `-
|
|
111
|
-
- `-
|
|
112
|
-
- `-o, --output FILE` - Save to file (default: stdout)
|
|
113
|
-
- `-d, --debug` - Enable debug output
|
|
63
|
+
**Options:**
|
|
64
|
+
- `-q, --query` - Query string (required)
|
|
65
|
+
- `-f, --from` - Start time (required, e.g., `-1h`, `2025-11-19T14:00:00`)
|
|
66
|
+
- `-t, --to` - End time (required, e.g., `now`)
|
|
67
|
+
- `-z, --time-zone` - Timezone (default: UTC)
|
|
68
|
+
- `-l, --limit` - Max messages to return
|
|
69
|
+
- `-o, --output` - Save to file
|
|
70
|
+
- `-i, --interactive` - Launch FZF browser
|
|
71
|
+
- `-d, --debug` - Debug output
|
|
114
72
|
|
|
115
|
-
|
|
73
|
+
**Interactive Mode** (`-i`): FZF-based browser with fuzzy search, preview, and multi-select. Requires `fzf` ([install](https://github.com/junegunn/fzf#installation)).
|
|
116
74
|
|
|
117
|
-
|
|
75
|
+
### 2. Discover Dynamic Sources
|
|
118
76
|
|
|
119
77
|
```bash
|
|
120
|
-
|
|
121
|
-
|
|
78
|
+
sumo-query discover-sources [OPTIONS]
|
|
79
|
+
```
|
|
122
80
|
|
|
123
|
-
|
|
124
|
-
sumo-query search -q 'error' -f '-30m' -t 'now' -i
|
|
81
|
+
Finds dynamic source names from log data (CloudWatch, ECS, Lambda streams).
|
|
125
82
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
83
|
+
**Options:**
|
|
84
|
+
- `-f, --from` - Start time (default: `-24h`)
|
|
85
|
+
- `-t, --to` - End time (default: `now`)
|
|
86
|
+
- `--filter` - Filter query (e.g., `_sourceCategory=*ecs*`)
|
|
87
|
+
- `-z, --time-zone` - Timezone (default: UTC)
|
|
88
|
+
- `-o, --output` - Save to file
|
|
129
89
|
|
|
130
|
-
**
|
|
131
|
-
- 🔍 Fuzzy search across all message fields
|
|
132
|
-
- 👁️ Live preview with full JSON details
|
|
133
|
-
- 🎨 Color-coded log levels (ERROR, WARN, INFO)
|
|
134
|
-
- ⌨️ Keyboard shortcuts for quick actions
|
|
135
|
-
- 📋 Multi-select and batch operations
|
|
136
|
-
- 💾 Export selected messages
|
|
137
|
-
|
|
138
|
-
**Keybindings:**
|
|
139
|
-
- `Enter` - Toggle selection (mark/unmark message)
|
|
140
|
-
- `Tab` - Open current message in pager (copyable view)
|
|
141
|
-
- `Ctrl-S` - Save selected messages to `sumo-selected.txt` and exit
|
|
142
|
-
- `Ctrl-Y` - Copy selected messages to clipboard and exit
|
|
143
|
-
- `Ctrl-E` - Export selected messages to `sumo-export.jsonl` and exit
|
|
144
|
-
- `Ctrl-A` - Select all messages
|
|
145
|
-
- `Ctrl-D` - Deselect all messages
|
|
146
|
-
- `Ctrl-/` - Toggle preview pane
|
|
147
|
-
- `Ctrl-Q` - Quit without saving
|
|
148
|
-
|
|
149
|
-
**Requirements:**
|
|
150
|
-
- Install FZF: `brew install fzf` (macOS) or `apt-get install fzf` (Linux)
|
|
151
|
-
- See: https://github.com/junegunn/fzf#installation
|
|
152
|
-
|
|
153
|
-
### Time Format Examples
|
|
154
|
-
|
|
155
|
-
Combine relative times with timezones for powerful queries:
|
|
90
|
+
**Examples:**
|
|
156
91
|
|
|
157
92
|
```bash
|
|
158
|
-
#
|
|
159
|
-
sumo-query
|
|
93
|
+
# Discover all sources from last 24 hours
|
|
94
|
+
sumo-query discover-sources
|
|
160
95
|
|
|
161
|
-
#
|
|
162
|
-
sumo-query
|
|
96
|
+
# Filter to ECS only
|
|
97
|
+
sumo-query discover-sources --filter '_sourceCategory=*ecs*'
|
|
163
98
|
|
|
164
|
-
# Last 7 days
|
|
165
|
-
sumo-query
|
|
99
|
+
# Last 7 days, save to file
|
|
100
|
+
sumo-query discover-sources -f '-7d' -o sources.json
|
|
101
|
+
```
|
|
166
102
|
|
|
167
|
-
|
|
168
|
-
sumo-query search -q 'error' -f '-24h' -t '2025-11-19T12:00:00'
|
|
103
|
+
### 3. List Collectors & Sources
|
|
169
104
|
|
|
170
|
-
|
|
171
|
-
|
|
105
|
+
```bash
|
|
106
|
+
# List collectors
|
|
107
|
+
sumo-query collectors [-o FILE]
|
|
108
|
+
|
|
109
|
+
# List static sources
|
|
110
|
+
sumo-query sources [-o FILE]
|
|
172
111
|
```
|
|
173
112
|
|
|
174
|
-
|
|
113
|
+
## Time Formats
|
|
175
114
|
|
|
176
115
|
```bash
|
|
177
|
-
|
|
178
|
-
|
|
116
|
+
# Relative (recommended)
|
|
117
|
+
-1h, -30m, -7d, now
|
|
179
118
|
|
|
180
|
-
|
|
119
|
+
# ISO 8601
|
|
120
|
+
2025-11-19T14:00:00
|
|
181
121
|
|
|
182
|
-
|
|
122
|
+
# Unix timestamp
|
|
123
|
+
1700000000
|
|
183
124
|
|
|
184
|
-
|
|
185
|
-
|
|
125
|
+
# Timezones
|
|
126
|
+
UTC, AEST, EST, America/New_York, Australia/Sydney, +10:00
|
|
186
127
|
```
|
|
187
128
|
|
|
188
|
-
|
|
189
|
-
|
|
129
|
+
See [examples/queries.md](examples/queries.md) for comprehensive query patterns.
|
|
190
130
|
|
|
191
|
-
## Ruby Library
|
|
131
|
+
## Ruby Library
|
|
192
132
|
|
|
193
133
|
```ruby
|
|
194
134
|
require 'sumologic'
|
|
195
135
|
|
|
196
|
-
# Initialize client
|
|
197
136
|
client = Sumologic::Client.new(
|
|
198
137
|
access_id: ENV['SUMO_ACCESS_ID'],
|
|
199
|
-
access_key: ENV['SUMO_ACCESS_KEY']
|
|
200
|
-
deployment: 'us2'
|
|
201
|
-
)
|
|
202
|
-
|
|
203
|
-
# Search logs
|
|
204
|
-
results = client.search(
|
|
205
|
-
query: 'error',
|
|
206
|
-
from_time: '2025-11-13T14:00:00',
|
|
207
|
-
to_time: '2025-11-13T15:00:00',
|
|
208
|
-
time_zone: 'UTC',
|
|
209
|
-
limit: 1000
|
|
138
|
+
access_key: ENV['SUMO_ACCESS_KEY']
|
|
210
139
|
)
|
|
211
140
|
|
|
212
|
-
#
|
|
213
|
-
|
|
214
|
-
sources = client.list_all_sources
|
|
215
|
-
```
|
|
141
|
+
# Search
|
|
142
|
+
client.search(query: 'error', from_time: '-1h', to_time: 'now')
|
|
216
143
|
|
|
217
|
-
|
|
144
|
+
# Discover sources
|
|
145
|
+
client.discover_dynamic_sources(from_time: '-24h', to_time: 'now')
|
|
218
146
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
# Parse relative times and timezones
|
|
223
|
-
from_time = Sumologic::Utils::TimeParser.parse('-1h')
|
|
224
|
-
timezone = Sumologic::Utils::TimeParser.parse_timezone('AEST')
|
|
147
|
+
# Metadata
|
|
148
|
+
client.list_collectors
|
|
149
|
+
client.list_all_sources
|
|
225
150
|
```
|
|
226
151
|
|
|
227
|
-
|
|
228
|
-
## Time Formats
|
|
229
|
-
|
|
230
|
-
Multiple time formats are supported:
|
|
231
|
-
|
|
232
|
-
```bash
|
|
233
|
-
# Relative time (easiest!)
|
|
234
|
-
sumo-query search -q 'error' -f '-1h' -t 'now'
|
|
235
|
-
sumo-query search -q 'error' -f '-30m' -t 'now'
|
|
236
|
-
|
|
237
|
-
# ISO 8601
|
|
238
|
-
sumo-query search -q 'error' -f '2025-11-13T14:00:00' -t '2025-11-13T15:00:00'
|
|
239
|
-
|
|
240
|
-
# Unix timestamps
|
|
241
|
-
sumo-query search -q 'error' -f '1700000000' -t 'now'
|
|
242
|
-
|
|
243
|
-
# With timezones
|
|
244
|
-
sumo-query search -q 'error' -f '-1h' -t 'now' -z 'AEST'
|
|
245
|
-
sumo-query search -q 'error' -f '-1h' -t 'now' -z 'America/New_York'
|
|
246
|
-
```
|
|
247
|
-
|
|
248
|
-
**Supported time units:** `s`, `m`, `h`, `d`, `w`, `M`, `now`
|
|
249
|
-
|
|
250
|
-
**Supported timezones:** IANA names (`UTC`, `America/New_York`, `Australia/Sydney`), US abbreviations (`EST`, `PST`), Australian abbreviations (`AEST`, `ACST`, `AWST`), UTC offsets (`+10:00`)
|
|
251
|
-
|
|
252
|
-
See [examples/time-formats.md](examples/time-formats.md) for comprehensive examples.
|
|
253
|
-
|
|
254
|
-
## Output Format
|
|
255
|
-
|
|
256
|
-
Results are returned as JSON:
|
|
257
|
-
|
|
258
|
-
```json
|
|
259
|
-
{
|
|
260
|
-
"query": "error",
|
|
261
|
-
"from": "2025-11-13T14:00:00",
|
|
262
|
-
"to": "2025-11-13T15:00:00",
|
|
263
|
-
"time_zone": "UTC",
|
|
264
|
-
"message_count": 42,
|
|
265
|
-
"messages": [
|
|
266
|
-
{
|
|
267
|
-
"map": {
|
|
268
|
-
"_messagetime": "1731506400123",
|
|
269
|
-
"_sourceCategory": "prod/api",
|
|
270
|
-
"_sourceName": "api-server-01",
|
|
271
|
-
"message": "Error processing request: timeout"
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
]
|
|
275
|
-
}
|
|
276
|
-
```
|
|
277
|
-
|
|
278
|
-
## Performance
|
|
279
|
-
|
|
280
|
-
Query execution time depends on data volume:
|
|
281
|
-
|
|
282
|
-
| Messages | Typical Time |
|
|
283
|
-
|----------|--------------|
|
|
284
|
-
| < 10K | 30-60 seconds |
|
|
285
|
-
| 10K-100K | 1-2 minutes |
|
|
286
|
-
| 100K+ | 2-5 minutes |
|
|
287
|
-
|
|
288
|
-
**Tips for faster queries:**
|
|
289
|
-
- Narrow your time range
|
|
290
|
-
- Add `_sourceCategory` filters
|
|
291
|
-
- Use `--limit` to cap results
|
|
292
|
-
- Use aggregation queries instead of raw messages
|
|
293
|
-
|
|
294
152
|
## Documentation
|
|
295
153
|
|
|
296
|
-
-
|
|
297
|
-
-
|
|
298
|
-
-
|
|
299
|
-
-
|
|
300
|
-
|
|
301
|
-
## Development
|
|
302
|
-
|
|
303
|
-
See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, testing, and contribution guidelines.
|
|
304
|
-
|
|
305
|
-
Quick start:
|
|
306
|
-
|
|
307
|
-
```bash
|
|
308
|
-
# Clone and install
|
|
309
|
-
git clone https://github.com/patrick204nqh/sumologic-query.git
|
|
310
|
-
cd sumologic-query
|
|
311
|
-
bundle install
|
|
312
|
-
|
|
313
|
-
# Run tests (73+ specs including time parser tests)
|
|
314
|
-
bundle exec rspec
|
|
315
|
-
|
|
316
|
-
# Run linter
|
|
317
|
-
bundle exec rubocop
|
|
318
|
-
|
|
319
|
-
# Test locally with new time formats
|
|
320
|
-
bundle exec bin/sumo-query search --query "error" \
|
|
321
|
-
--from "-1h" --to "now"
|
|
322
|
-
|
|
323
|
-
# Test with timezone support
|
|
324
|
-
bundle exec bin/sumo-query search --query "error" \
|
|
325
|
-
--from "-30m" --to "now" --time-zone "AEST"
|
|
326
|
-
```
|
|
154
|
+
- [Query Examples](examples/queries.md) - Query patterns and examples
|
|
155
|
+
- [Quick Reference](docs/tldr.md) - Command cheat sheet
|
|
156
|
+
- [Rate Limiting](docs/rate-limiting.md) - Performance tuning
|
|
157
|
+
- [Architecture](docs/architecture/) - Design decisions
|
|
327
158
|
|
|
328
159
|
## Contributing
|
|
329
160
|
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
1. Fork the repository
|
|
333
|
-
2. Create your feature branch (`git checkout -b feature/amazing-feature`)
|
|
334
|
-
3. Commit your changes (`git commit -m 'Add amazing feature'`)
|
|
335
|
-
4. Push to the branch (`git push origin feature/amazing-feature`)
|
|
336
|
-
5. Open a Pull Request
|
|
161
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
|
|
337
162
|
|
|
338
163
|
## License
|
|
339
164
|
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
## Support
|
|
343
|
-
|
|
344
|
-
- **Issues**: [GitHub Issues](https://github.com/patrick204nqh/sumologic-query/issues)
|
|
345
|
-
- **Discussions**: [GitHub Discussions](https://github.com/patrick204nqh/sumologic-query/discussions)
|
|
346
|
-
- **Documentation**: [docs/](docs/)
|
|
347
|
-
|
|
348
|
-
## Resources
|
|
349
|
-
|
|
350
|
-
- **Sumo Logic API Docs**: https://help.sumologic.com/docs/api/search-job/
|
|
351
|
-
- **Query Language**: https://help.sumologic.com/docs/search/
|
|
352
|
-
- **Bug Reports**: https://github.com/patrick204nqh/sumologic-query/issues
|
|
165
|
+
MIT License - see [LICENSE](LICENSE) file.
|
|
353
166
|
|
|
354
|
-
|
|
167
|
+
## Links
|
|
355
168
|
|
|
356
|
-
|
|
169
|
+
- [Issues](https://github.com/patrick204nqh/sumologic-query/issues)
|
|
170
|
+
- [Sumo Logic API Docs](https://help.sumologic.com/docs/api/search-job/)
|
|
171
|
+
- [Query Language](https://help.sumologic.com/docs/search/)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'base_command'
|
|
4
|
+
require_relative '../../utils/time_parser'
|
|
5
|
+
|
|
6
|
+
module Sumologic
|
|
7
|
+
class CLI < Thor
|
|
8
|
+
module Commands
|
|
9
|
+
# Handles the discover-sources command execution
|
|
10
|
+
class DiscoverSourcesCommand < BaseCommand
|
|
11
|
+
def execute
|
|
12
|
+
parse_time_options
|
|
13
|
+
log_discovery_info
|
|
14
|
+
results = perform_discovery
|
|
15
|
+
|
|
16
|
+
output_json(results)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def parse_time_options
|
|
22
|
+
# Parse time formats and store both original and parsed values
|
|
23
|
+
@original_from = options[:from]
|
|
24
|
+
@original_to = options[:to]
|
|
25
|
+
@parsed_from = Utils::TimeParser.parse(options[:from])
|
|
26
|
+
@parsed_to = Utils::TimeParser.parse(options[:to])
|
|
27
|
+
@parsed_timezone = Utils::TimeParser.parse_timezone(options[:time_zone])
|
|
28
|
+
rescue Utils::TimeParser::ParseError => e
|
|
29
|
+
warn "Error parsing time: #{e.message}"
|
|
30
|
+
exit 1
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def log_discovery_info
|
|
34
|
+
warn '=' * 60
|
|
35
|
+
warn 'Discovering Dynamic Source Names'
|
|
36
|
+
warn '=' * 60
|
|
37
|
+
warn "Time Range: #{@original_from} to #{@original_to}"
|
|
38
|
+
if @original_from != @parsed_from || @original_to != @parsed_to
|
|
39
|
+
warn " (Parsed: #{@parsed_from} to #{@parsed_to})"
|
|
40
|
+
end
|
|
41
|
+
warn "Time Zone: #{@parsed_timezone}"
|
|
42
|
+
warn "Filter: #{options[:filter] || 'none (all sources)'}"
|
|
43
|
+
warn '-' * 60
|
|
44
|
+
warn 'Running aggregation query to discover sources...'
|
|
45
|
+
$stderr.puts
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def perform_discovery
|
|
49
|
+
client.discover_dynamic_sources(
|
|
50
|
+
from_time: @parsed_from,
|
|
51
|
+
to_time: @parsed_to,
|
|
52
|
+
time_zone: @parsed_timezone,
|
|
53
|
+
filter: options[:filter]
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
data/lib/sumologic/cli.rb
CHANGED
|
@@ -5,6 +5,7 @@ require 'json'
|
|
|
5
5
|
require_relative 'cli/commands/search_command'
|
|
6
6
|
require_relative 'cli/commands/list_collectors_command'
|
|
7
7
|
require_relative 'cli/commands/list_sources_command'
|
|
8
|
+
require_relative 'cli/commands/discover_sources_command'
|
|
8
9
|
|
|
9
10
|
module Sumologic
|
|
10
11
|
# Thor-based CLI for Sumo Logic query tool
|
|
@@ -91,6 +92,45 @@ module Sumologic
|
|
|
91
92
|
Commands::ListSourcesCommand.new(options, create_client).execute
|
|
92
93
|
end
|
|
93
94
|
|
|
95
|
+
desc 'discover-sources', 'Discover dynamic source names from logs'
|
|
96
|
+
long_desc <<~DESC
|
|
97
|
+
Discover dynamic source names by querying actual log data.
|
|
98
|
+
Useful for CloudWatch/ECS sources that use dynamic _sourceName values.
|
|
99
|
+
|
|
100
|
+
Time Formats:
|
|
101
|
+
--from and --to support multiple formats:
|
|
102
|
+
• 'now' - current time
|
|
103
|
+
• Relative: '-30s', '-5m', '-2h', '-7d', '-1w', '-1M' (sec/min/hour/day/week/month)
|
|
104
|
+
• Unix timestamp: '1700000000' (seconds since epoch)
|
|
105
|
+
• ISO 8601: '2025-11-13T14:00:00'
|
|
106
|
+
|
|
107
|
+
Examples:
|
|
108
|
+
# Discover all sources from last 24 hours (default)
|
|
109
|
+
sumo-query discover-sources
|
|
110
|
+
|
|
111
|
+
# Discover sources from last 7 days
|
|
112
|
+
sumo-query discover-sources --from '-7d' --to 'now'
|
|
113
|
+
|
|
114
|
+
# Filter by source category (ECS only)
|
|
115
|
+
sumo-query discover-sources --filter '_sourceCategory=*ecs*'
|
|
116
|
+
|
|
117
|
+
# Discover CloudWatch sources
|
|
118
|
+
sumo-query discover-sources --filter '_sourceCategory=*cloudwatch*'
|
|
119
|
+
|
|
120
|
+
# Save to file
|
|
121
|
+
sumo-query discover-sources --output discovered-sources.json
|
|
122
|
+
DESC
|
|
123
|
+
option :from, type: :string, default: '-24h', aliases: '-f',
|
|
124
|
+
desc: 'Start time (default: -24h)'
|
|
125
|
+
option :to, type: :string, default: 'now', aliases: '-t',
|
|
126
|
+
desc: 'End time (default: now)'
|
|
127
|
+
option :time_zone, type: :string, default: 'UTC', aliases: '-z',
|
|
128
|
+
desc: 'Time zone (UTC, EST, AEST, +00:00, America/New_York, Australia/Sydney)'
|
|
129
|
+
option :filter, type: :string, desc: 'Optional filter query (e.g., _sourceCategory=*ecs*)'
|
|
130
|
+
def discover_sources
|
|
131
|
+
Commands::DiscoverSourcesCommand.new(options, create_client).execute
|
|
132
|
+
end
|
|
133
|
+
|
|
94
134
|
desc 'version', 'Show version information'
|
|
95
135
|
def version
|
|
96
136
|
puts "sumo-query version #{Sumologic::VERSION}"
|
data/lib/sumologic/client.rb
CHANGED
|
@@ -23,7 +23,12 @@ module Sumologic
|
|
|
23
23
|
# Initialize domain components
|
|
24
24
|
@search = Search::Job.new(http_client: @http, config: @config)
|
|
25
25
|
@collector = Metadata::Collector.new(http_client: @http)
|
|
26
|
-
@source = Metadata::Source.new(http_client: @http, collector_client: @collector)
|
|
26
|
+
@source = Metadata::Source.new(http_client: @http, collector_client: @collector, config: @config)
|
|
27
|
+
@dynamic_source_discovery = Metadata::DynamicSourceDiscovery.new(
|
|
28
|
+
http_client: @http,
|
|
29
|
+
search_job: @search,
|
|
30
|
+
config: @config
|
|
31
|
+
)
|
|
27
32
|
end
|
|
28
33
|
|
|
29
34
|
# Search logs with query
|
|
@@ -57,5 +62,22 @@ module Sumologic
|
|
|
57
62
|
def list_all_sources
|
|
58
63
|
@source.list_all
|
|
59
64
|
end
|
|
65
|
+
|
|
66
|
+
# Discover dynamic source names from actual log data
|
|
67
|
+
# Useful for CloudWatch/ECS sources with dynamic _sourceName values
|
|
68
|
+
# Returns hash with ALL unique source names found, with message counts
|
|
69
|
+
#
|
|
70
|
+
# @param from_time [String] Start time (ISO 8601, unix timestamp, or relative)
|
|
71
|
+
# @param to_time [String] End time
|
|
72
|
+
# @param time_zone [String] Time zone (default: UTC)
|
|
73
|
+
# @param filter [String, nil] Optional filter query to scope results
|
|
74
|
+
def discover_dynamic_sources(from_time:, to_time:, time_zone: 'UTC', filter: nil)
|
|
75
|
+
@dynamic_source_discovery.discover(
|
|
76
|
+
from_time: from_time,
|
|
77
|
+
to_time: to_time,
|
|
78
|
+
time_zone: time_zone,
|
|
79
|
+
filter: filter
|
|
80
|
+
)
|
|
81
|
+
end
|
|
60
82
|
end
|
|
61
83
|
end
|
|
@@ -4,7 +4,7 @@ module Sumologic
|
|
|
4
4
|
# Centralized configuration for Sumo Logic client
|
|
5
5
|
class Configuration
|
|
6
6
|
attr_accessor :access_id, :access_key, :deployment, :timeout, :initial_poll_interval, :max_poll_interval,
|
|
7
|
-
:poll_backoff_factor, :max_messages_per_request
|
|
7
|
+
:poll_backoff_factor, :max_messages_per_request, :max_workers, :request_delay
|
|
8
8
|
|
|
9
9
|
API_VERSION = 'v1'
|
|
10
10
|
|
|
@@ -22,6 +22,10 @@ module Sumologic
|
|
|
22
22
|
# Timeouts and limits
|
|
23
23
|
@timeout = 300 # seconds (5 minutes)
|
|
24
24
|
@max_messages_per_request = 10_000
|
|
25
|
+
|
|
26
|
+
# Rate limiting (default: 5 workers, 250ms delay)
|
|
27
|
+
@max_workers = ENV.fetch('SUMO_MAX_WORKERS', '5').to_i
|
|
28
|
+
@request_delay = ENV.fetch('SUMO_REQUEST_DELAY', '0.25').to_f
|
|
25
29
|
end
|
|
26
30
|
|
|
27
31
|
def base_url
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'loggable'
|
|
4
|
+
require_relative 'models'
|
|
5
|
+
|
|
3
6
|
module Sumologic
|
|
4
7
|
module Metadata
|
|
5
8
|
# Handles collector metadata operations
|
|
6
9
|
class Collector
|
|
10
|
+
include Loggable
|
|
11
|
+
|
|
7
12
|
def initialize(http_client:)
|
|
8
13
|
@http = http_client
|
|
9
14
|
end
|
|
@@ -22,12 +27,6 @@ module Sumologic
|
|
|
22
27
|
rescue StandardError => e
|
|
23
28
|
raise Error, "Failed to list collectors: #{e.message}"
|
|
24
29
|
end
|
|
25
|
-
|
|
26
|
-
private
|
|
27
|
-
|
|
28
|
-
def log_info(message)
|
|
29
|
-
warn "[Sumologic::Metadata::Collector] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
|
|
30
|
-
end
|
|
31
30
|
end
|
|
32
31
|
end
|
|
33
32
|
end
|