solid_observer 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/README.md +100 -25
- data/app/assets/javascripts/solid_observer/live_poll.js +3 -1
- data/app/controllers/solid_observer/application_controller.rb +1 -0
- data/app/controllers/solid_observer/cable_dashboard_controller.rb +52 -0
- data/app/controllers/solid_observer/cable_operations_controller.rb +16 -0
- data/app/controllers/solid_observer/cache_dashboard_controller.rb +33 -40
- data/app/controllers/solid_observer/dashboard_controller.rb +21 -13
- data/app/controllers/solid_observer/traces_controller.rb +17 -0
- data/app/helpers/solid_observer/application_helper.rb +145 -3
- data/app/models/solid_observer/cable_event.rb +13 -0
- data/app/models/solid_observer/cable_metric.rb +12 -0
- data/app/models/solid_observer/cache_metric.rb +1 -2
- data/app/models/solid_observer/storage_info.rb +1 -1
- data/app/views/layouts/solid_observer/application.html.erb +35 -9
- data/app/views/solid_observer/cable_dashboard/_charts.html.erb +31 -0
- data/app/views/solid_observer/cable_dashboard/_recent_events.html.erb +39 -0
- data/app/views/solid_observer/cable_dashboard/_summary.html.erb +34 -0
- data/app/views/solid_observer/cable_dashboard/index.html.erb +118 -0
- data/app/views/solid_observer/cache_dashboard/_recent_events.html.erb +6 -1
- data/app/views/solid_observer/dashboard/_component_cards.html.erb +18 -0
- data/app/views/solid_observer/dashboard/_health.html.erb +6 -0
- data/app/views/solid_observer/dashboard/_home.html.erb +10 -0
- data/app/views/solid_observer/dashboard/_queue.html.erb +138 -0
- data/app/views/solid_observer/dashboard/_queue_table.html.erb +1 -0
- data/app/views/solid_observer/dashboard/_unified_feed.html.erb +38 -0
- data/app/views/solid_observer/dashboard/index.html.erb +3 -139
- data/app/views/solid_observer/events/index.html.erb +7 -1
- data/app/views/solid_observer/jobs/index.html.erb +1 -0
- data/app/views/solid_observer/storages/show.html.erb +29 -3
- data/app/views/solid_observer/traces/show.html.erb +80 -0
- data/config/routes.rb +3 -0
- data/db/migrate/20260612000001_add_event_type_recorded_at_index_to_cache_events.rb +21 -0
- data/db/migrate/20260619000001_create_solid_observer_cable_events.rb +22 -0
- data/db/migrate/20260619000002_create_solid_observer_cable_metrics.rb +17 -0
- data/db/migrate/20260630000001_add_correlation_id_to_cache_events.rb +8 -0
- data/db/migrate/20260630000002_add_correlation_id_to_cable_events.rb +8 -0
- data/lib/generators/solid_observer/install_generator.rb +8 -1
- data/lib/generators/solid_observer/templates/initializer.rb.tt +24 -5
- data/lib/solid_observer/base_event.rb +1 -1
- data/lib/solid_observer/base_metric.rb +1 -1
- data/lib/solid_observer/base_record.rb +8 -0
- data/lib/solid_observer/cable_event_buffer.rb +28 -0
- data/lib/solid_observer/cable_metric_buffer.rb +230 -0
- data/lib/solid_observer/cable_subscriber.rb +57 -0
- data/lib/solid_observer/cache_event_buffer.rb +11 -36
- data/lib/solid_observer/cache_metric_buffer.rb +229 -0
- data/lib/solid_observer/chart_buffer.rb +84 -27
- data/lib/solid_observer/cli/health.rb +49 -0
- data/lib/solid_observer/cli/trace.rb +106 -0
- data/lib/solid_observer/configuration.rb +47 -7
- data/lib/solid_observer/correlated.rb +23 -0
- data/lib/solid_observer/engine.rb +46 -28
- data/lib/solid_observer/event_buffer_core.rb +218 -0
- data/lib/solid_observer/queries/trace_query.rb +151 -0
- data/lib/solid_observer/queue_event_buffer.rb +9 -201
- data/lib/solid_observer/services/cable_operations.rb +74 -0
- data/lib/solid_observer/services/cable_stats.rb +385 -0
- data/lib/solid_observer/services/cache_stats.rb +35 -18
- data/lib/solid_observer/services/cleanup_storage.rb +82 -47
- data/lib/solid_observer/services/flush_cable_event_buffer.rb +54 -0
- data/lib/solid_observer/services/flush_cable_metrics.rb +54 -0
- data/lib/solid_observer/services/flush_cache_metrics.rb +56 -0
- data/lib/solid_observer/services/health_score.rb +85 -0
- data/lib/solid_observer/services/record_cable_event.rb +115 -0
- data/lib/solid_observer/services/record_cable_metric.rb +73 -0
- data/lib/solid_observer/services/record_cache_event.rb +24 -0
- data/lib/solid_observer/services/record_cache_metric.rb +13 -21
- data/lib/solid_observer/services/storage_info_snapshot.rb +103 -15
- data/lib/solid_observer/services/unified_feed.rb +101 -0
- data/lib/solid_observer/version.rb +1 -1
- data/lib/solid_observer.rb +42 -11
- data/lib/tasks/solid_observer.rake +97 -21
- metadata +41 -6
- data/app/assets/stylesheets/solid_observer/application.css +0 -18
- data/bin/console +0 -11
- data/bin/quality_gate +0 -95
- data/bin/setup +0 -8
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7c363ea187ff44b2a28184384cfbdee1d570c6e8fd981abceded3844819a2338
|
|
4
|
+
data.tar.gz: ff5bb32bb53ad65ea688365a5b8666aa34552580d3d78856f8e542fcf53b6d4c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cbac07f62a04d4ea7ae4c8c2c1200f04c07f400f94ca5775785380853cd63f56eccdc154b1debe55438fc60482d54be55199265c299eae43577e1943b60abd1e
|
|
7
|
+
data.tar.gz: '018f433df069c72a8a12ced3434e4b05f5a26eb6209bb33af0e5c29b3aca2df1961359692e250f247769cb0e9bab322b946dc208ca265688fb4ae0316de24dd9'
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,33 @@
|
|
|
1
|
+
## [0.6.0] - 2026-08-04
|
|
2
|
+
|
|
3
|
+
Headline: **Cross-component Tracing, Health Scoring & Unified Dashboard** — trace correlation across Queue/Cache/Cable, unified home dashboard, health scoring service, and CLI health & trace commands.
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- **Correlation ID propagation** — automatic correlation ID generation and propagation across ActiveJob (`Correlated.stamp`), SolidCache, and SolidCable events; configurable `correlation_id_generator`.
|
|
7
|
+
- **Cross-component trace service & CLI** — `Queries::TraceQuery` for fetching cross-component events by `correlation_id`; `rails solid_observer:trace[CORRELATION_ID,LIMIT]` CLI command.
|
|
8
|
+
- **Trace Web UI** — Web interface at `/solid_observer/traces/:correlation_id` displaying component-tagged event timelines across Queue, Cache, and Cable.
|
|
9
|
+
- **Unified health scoring service** — `Services::HealthScore` auto-calculates overall host stability using worst-of component statuses (Queue, Cache, Cable) with zero configuration required.
|
|
10
|
+
- **CLI health command** — `rails solid_observer:health` CLI command displaying aggregate and per-component health status.
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
- **Combined / Unified Dashboard** — Engine root (`GET /solid_observer/`) now renders the unified Home dashboard (overall health banner, per-component status cards, and merged multi-component activity feed). Dedicated Queue overview moved to `/solid_observer/queue`.
|
|
14
|
+
|
|
15
|
+
### Upgrade & Database Migration
|
|
16
|
+
- **Required DB Migration** — Running `bin/rails solid_observer:install:migrations && bin/rails db:migrate` is required when upgrading to v0.6.0. Adds `correlation_id` columns to `solid_observer_cache_events` and `solid_observer_cable_events` tables.
|
|
17
|
+
|
|
18
|
+
## [0.5.0] - 2026-06-24
|
|
19
|
+
|
|
20
|
+
Headline: **Solid Cable observability** — optional Cable telemetry, dashboard, storage health, and guarded trim controls.
|
|
21
|
+
|
|
22
|
+
### Added
|
|
23
|
+
- **Cable telemetry foundation** (SO-089, SO-090) — `observe_cable` config gate with `solid_cable_available?`/`solid_cable_enabled?` predicates; `CableSubscriber` hooks into `ActiveSupport::Notifications` broadcast events; event/metric buffers reuse `EventBufferCore`; `CableEvent`/`CableMetric` models with two migrations; `cable_sampling_rate` default 0.1; broadcasting names stored as `Digest::SHA256.hexdigest` (PII boundary).
|
|
24
|
+
- **Cable storage health** (SO-091) — Storages page aggregates Solid Cable message storage and SolidObserver Cable telemetry rows with optional fallback states.
|
|
25
|
+
- **Cable dashboard** (SO-092) — `/solid_observer/cable_dashboard` with summary cards, broadcast/rejection trends, stability indicator (hybrid: event + backlog signals), recent events. Three configurable thresholds: `cable_rejection_threshold` (0.05), `cable_backlog_threshold` (0.10), `cable_error_threshold` (0.0). Sidebar navigation entry.
|
|
26
|
+
- **Guarded Cable trim** (SO-093) — UI button for ≤1,000 trimmable messages; `solid_observer:cable:trim` Rake task for larger backlogs. No clear-all.
|
|
27
|
+
|
|
28
|
+
### Changed
|
|
29
|
+
- **Self-contained SQLite generator config** (SO-094) — install generator produces a `solid_observer_queue` block with explicit `adapter: sqlite3`, `pool`, `timeout`, `database`, and `migrations_paths: db/solid_observer_migrate` — no `<<: *default` YAML merge. Post-install warning printed for PostgreSQL/MySQL hosts. Cable telemetry label standardised to `Cable telemetry`.
|
|
30
|
+
|
|
1
31
|
## [0.4.0] - 2026-06-03
|
|
2
32
|
|
|
3
33
|
Headline: **Solid Cache observability** — multi-component foundation, cache dashboard, operational controls, and shared storage health.
|
data/README.md
CHANGED
|
@@ -7,39 +7,43 @@
|
|
|
7
7
|
</p>
|
|
8
8
|
|
|
9
9
|
<p align="center">
|
|
10
|
-
<a href="https://github.com/bart-oz/solid_observer/releases"><img src="https://img.shields.io/badge/version-0.
|
|
10
|
+
<a href="https://github.com/bart-oz/solid_observer/releases"><img src="https://img.shields.io/badge/version-0.6.0-blue.svg" alt="Version"></a>
|
|
11
11
|
<a href="LICENSE.txt"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="License"></a>
|
|
12
12
|
<a href="https://github.com/bart-oz/solid_observer/actions"><img src="https://img.shields.io/badge/tests-passing-brightgreen.svg" alt="Tests"></a>
|
|
13
|
-
<a href="https://github.com/bart-oz/solid_observer/actions"><img src="https://img.shields.io/badge/coverage-96
|
|
14
|
-
</p>
|
|
13
|
+
<a href="https://github.com/bart-oz/solid_observer/actions"><img src="https://img.shields.io/badge/coverage-97.96%25-brightgreen.svg" alt="Coverage"></a>
|
|
15
14
|
|
|
16
15
|
---
|
|
17
16
|
<p align="center">
|
|
18
17
|
<a href=".github/assets/dash_1.png"><img src=".github/assets/dash_1.png" alt="SolidObserver dashboard overview" width="700"></a>
|
|
19
18
|
</p>
|
|
20
19
|
|
|
21
|
-
SolidObserver is a production-grade observability solution for Rails 8's Solid Stack. v0.
|
|
20
|
+
SolidObserver is a production-grade observability solution for Rails 8's Solid Stack. v0.6.0 covers **Solid Queue**, **Solid Cache**, and **Solid Cable** with a unified Web UI dashboard, health scoring (`Services::HealthScore`), CLI tools, metrics collection, and cross-component trace correlation by `correlation_id`.
|
|
22
21
|
|
|
23
|
-
## Features (v0.
|
|
22
|
+
## Features (v0.6.0)
|
|
24
23
|
|
|
25
|
-
| | Solid Queue | Solid Cache |
|
|
26
|
-
|
|
27
|
-
| **Web UI Dashboard** | Queue stats, jobs browser, events log | Hit rate, ops/sec, error rate, avg duration |
|
|
28
|
-
| **
|
|
29
|
-
| **
|
|
30
|
-
| **
|
|
31
|
-
| **
|
|
32
|
-
| **
|
|
33
|
-
| **
|
|
34
|
-
| **
|
|
24
|
+
| | Solid Queue | Solid Cache | Solid Cable |
|
|
25
|
+
|---|---|---|---|
|
|
26
|
+
| **Web UI Dashboard** | Queue stats, jobs browser, events log | Hit rate, ops/sec, error rate, avg duration | Broadcasts, rejection rate, trends |
|
|
27
|
+
| **Unified Home** | Overall health banner, component cards, merged activity feed at `/solid_observer` |
|
|
28
|
+
| **Health Scoring** | Worst-of aggregate stability (`Services::HealthScore`), config-free |
|
|
29
|
+
| **Cross-Component Trace** | Timeline by `correlation_id` across Queue, Cache, Cable (`/solid_observer/traces/:id`) |
|
|
30
|
+
| **Storage footprint** | DB size, event counts | SolidCache table size, row counts | Message count + backlogs |
|
|
31
|
+
| **Activity trends** | Sparklines (Performed, Ready, Failed) | Activity trend sparklines | Broadcast/rejection sparklines |
|
|
32
|
+
| **Stability indicator** | Stable / Degraded / Critical badge | Stability pill badge | Stable / Degraded / Critical (hybrid) |
|
|
33
|
+
| **Operational controls** | Retry / discard failed jobs | Prune expired entries, clear all entries | Trim expired messages |
|
|
34
|
+
| **CLI tools** | status, health, trace, jobs:list/show/retry/discard | cache:prune, cache:clear | cable:trim |
|
|
35
|
+
| **Privacy** | Job arguments excluded from persisted events | Keys and values **never** shown | Broadcasting names hashed (SHA256) |
|
|
36
|
+
| **Operating modes** | Real-time (no DB) or persistence (full history) | Persistence mode only | Persistence mode only |
|
|
35
37
|
|
|
36
|
-
Additional: 🔗
|
|
38
|
+
Additional: 🔗 Cross-component trace lookup by correlation_id · 🩺 Config-free aggregate health score · 🏠 Unified Home dashboard · ⚡ buffered writes · 🛡️ Docker/CI/K8s safe boot
|
|
37
39
|
|
|
38
40
|
## Requirements
|
|
39
41
|
|
|
40
42
|
- Ruby 3.2+
|
|
41
43
|
- Rails 8.0+
|
|
42
44
|
- Solid Queue (configured with `connects_to` in all environments)
|
|
45
|
+
- Solid Cache (optional — enable with `config.observe_cache = true`)
|
|
46
|
+
- Solid Cable (optional — enable with `config.observe_cable = true`)
|
|
43
47
|
|
|
44
48
|
> **Note:** Ensure Solid Queue is configured with `connects_to` in all environments, not just production. See [Troubleshooting](#troubleshooting) if you encounter database connection issues.
|
|
45
49
|
|
|
@@ -75,7 +79,7 @@ That's it. You now have access to queue status, job listing, retry, and discard
|
|
|
75
79
|
|
|
76
80
|
Store event history, metrics, and storage snapshots in a dedicated observer database. This gives you everything in real-time mode plus long-term event tracking, buffered writes, and retention-based cleanup. The install generator defaults to SQLite; the database can use any Rails-supported adapter for record persistence, and storage-size monitoring is implemented for SQLite, PostgreSQL/PostGIS, MySQL, and Trilogy. See [Database Setup](#database-setup-persistence-mode) below.
|
|
77
81
|
|
|
78
|
-
>
|
|
82
|
+
> **⚠️ PostgreSQL / MySQL hosts:** The generator writes a self-contained `solid_observer_queue` block with `adapter: sqlite3`. **Review `config/database.yml` before running `db:create`** — do not merge `<<: *default` into the observer entry, as that pulls the host adapter in and `db:create` will fail. See [Multi-adapter setup](#multi-adapter-setup) for the correct pattern.
|
|
79
83
|
|
|
80
84
|
```bash
|
|
81
85
|
bin/rails solid_observer:install:migrations
|
|
@@ -173,6 +177,14 @@ Configuration:
|
|
|
173
177
|
|
|
174
178
|
SolidObserver ships with a zero-dependency Web UI (no asset pipeline, no JS framework) at `/solid_observer`.
|
|
175
179
|
|
|
180
|
+
### Views & Routes
|
|
181
|
+
|
|
182
|
+
- **Unified Home Dashboard** (`/solid_observer/`) — Overall worst-of health score banner, per-component status cards (Queue, Cache, Cable), and a unified cross-component activity feed.
|
|
183
|
+
- **Queue Overview** (`/solid_observer/queue`) — Dedicated Queue metrics, job browser (ready/scheduled/claimed/failed), events log, time-range trends, and operational controls.
|
|
184
|
+
- **Cache Dashboard** (`/solid_observer/cache_dashboard`) — SolidCache hit rates, ops/sec, error rates, storage footprint, and prune/clear controls.
|
|
185
|
+
- **Cable Dashboard** (`/solid_observer/cable_dashboard`) — SolidCable broadcast volume, rejection rates, backlog trends, and message trim controls.
|
|
186
|
+
- **Cross-Component Trace Timeline** (`/solid_observer/traces/:correlation_id`) — Component-tagged event sequence across Queue, Cache, and Cable for a specific correlation ID.
|
|
187
|
+
|
|
176
188
|
### Mount
|
|
177
189
|
|
|
178
190
|
The install generator mounts it for you. To mount manually:
|
|
@@ -182,6 +194,18 @@ The install generator mounts it for you. To mount manually:
|
|
|
182
194
|
mount SolidObserver::Engine, at: "/solid_observer"
|
|
183
195
|
```
|
|
184
196
|
|
|
197
|
+
Production dashboard exposure is the host application's responsibility. If you
|
|
198
|
+
enable and mount the dashboard in production, wrap the mount in your existing
|
|
199
|
+
admin authentication/authorization constraint (example shown for apps that
|
|
200
|
+
already provide an `authenticate` route helper):
|
|
201
|
+
|
|
202
|
+
```ruby
|
|
203
|
+
# config/routes.rb
|
|
204
|
+
authenticate :user, ->(user) { user.admin? } do
|
|
205
|
+
mount SolidObserver::Engine, at: "/solid_observer"
|
|
206
|
+
end
|
|
207
|
+
```
|
|
208
|
+
|
|
185
209
|
### Auth Configuration
|
|
186
210
|
|
|
187
211
|
```ruby
|
|
@@ -207,6 +231,7 @@ For production hardening (fail-loud on missing env var vs. fail-open), see [Conf
|
|
|
207
231
|
- **API-only apps** — the Web UI works without manual configuration. The engine ships its own Cookies/Session/Flash middleware stack scoped to `/solid_observer/*`.
|
|
208
232
|
- **Host-app callbacks are not inherited.** Use `ui_username` / `ui_password` for built-in HTTP Basic Auth.
|
|
209
233
|
- **Auth misconfiguration is fail-open, but loud.** If `ui_username` is set but `ui_password` resolves to `nil`, the UI ships unauthenticated and logs a boot `WARNING`.
|
|
234
|
+
- **HTTP Basic Auth requires both credentials.** Setting only `ui_username` or only `ui_password` disables built-in auth; protect production mounts at the host-app route boundary.
|
|
210
235
|
|
|
211
236
|
See the full component breakdown in [Components](#components) below.
|
|
212
237
|
|
|
@@ -270,16 +295,51 @@ bin/rails solid_observer:cache:clear # Clear all SolidCache entries (with confi
|
|
|
270
295
|
bin/rails solid_observer:cache:prune # Prune expired SolidCache entries
|
|
271
296
|
```
|
|
272
297
|
|
|
298
|
+
### Solid Cable Observability
|
|
299
|
+
|
|
300
|
+
Cable observability is optional. Enable with `config.observe_cable = true`. Requires SolidCable in the host app. SolidObserver does not add any SolidCable dependency.
|
|
301
|
+
|
|
302
|
+
Cable dashboard (`/solid_observer/cable_dashboard`) shows broadcast/rejection trends, a stability indicator (hybrid: event + backlog signals), and recent safe events. Storages page includes Cable telemetry rows. Guarded trim controls: UI button for ≤1,000 trimmable messages; `solid_observer:cable:trim` Rake task for larger backlogs. **Broadcasting names are stored as SHA256 digests** — raw names are never persisted.
|
|
303
|
+
|
|
304
|
+
**Enabling Cable observability:**
|
|
305
|
+
|
|
306
|
+
```ruby
|
|
307
|
+
SolidObserver.configure do |config|
|
|
308
|
+
config.observe_cable = true # default: false
|
|
309
|
+
# config.cable_sampling_rate = 0.1 # Sample 10% of broadcast events (default: 0.1)
|
|
310
|
+
# config.cable_rejection_threshold = 0.05 # Rejection rate threshold for Degraded (default: 0.05)
|
|
311
|
+
# config.cable_backlog_threshold = 0.10 # Backlog ratio threshold (default: 0.10)
|
|
312
|
+
# config.cable_error_threshold = 0.0 # Error rate threshold (default: 0.0)
|
|
313
|
+
end
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
`solid_cable_enabled?` is `true` when both `observe_cable = true` and SolidCable is available in the host app.
|
|
317
|
+
|
|
318
|
+
**CLI commands (Solid Cable):**
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
bin/rails solid_observer:cable:trim # Trim expired Cable messages (with confirmation)
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
> Cable dashboard screenshots are not yet available — capture from a host app with SolidCable configured.
|
|
325
|
+
|
|
273
326
|
### Storage
|
|
274
327
|
|
|
275
|
-
The Storage page aggregates per-component health rows: Queue Observer database, Cache Observer,
|
|
328
|
+
The Storage page aggregates per-component health rows: Queue Observer database, Cache Observer, SolidCache table sizes, and Cable telemetry. Each row reports size, record counts, and a status indicator.
|
|
276
329
|
|
|
277
330
|
<p align="center">
|
|
278
|
-
<a href=".github/assets/dash_8.png"><img src=".github/assets/dash_8.png" alt="Component health — Queue Observer + Cache Observer + SolidCache" width="700"></a>
|
|
331
|
+
<a href=".github/assets/dash_8.png"><img src=".github/assets/dash_8.png" alt="Component health — Queue Observer + Cache Observer + SolidCache + Cable telemetry" width="700"></a>
|
|
279
332
|
</p>
|
|
280
333
|
|
|
281
334
|
For adapter notes and multi-adapter setup, see [Database Setup](#database-setup-persistence-mode) below.
|
|
282
335
|
|
|
336
|
+
### Storage unavailable diagnostics
|
|
337
|
+
|
|
338
|
+
The dashboard is admin/developer-facing. When SolidObserver storage is not
|
|
339
|
+
reachable, the current default 503 error page includes the raw exception class
|
|
340
|
+
and message so maintainers can diagnose adapter, credential, migration, or
|
|
341
|
+
database availability problems. Do not expose the dashboard to untrusted users.
|
|
342
|
+
|
|
283
343
|
## Configuration
|
|
284
344
|
|
|
285
345
|
<details>
|
|
@@ -300,6 +360,13 @@ SolidObserver.configure do |config|
|
|
|
300
360
|
# Enable cache monitoring (default: false; requires SolidCache in host app)
|
|
301
361
|
config.observe_cache = true
|
|
302
362
|
|
|
363
|
+
# Enable cable monitoring (default: false; requires SolidCable in host app)
|
|
364
|
+
config.observe_cable = true
|
|
365
|
+
config.cable_sampling_rate = 0.1 # Sample 10% of broadcast events
|
|
366
|
+
config.cable_rejection_threshold = 0.05 # Rejection rate → Degraded stability
|
|
367
|
+
config.cable_backlog_threshold = 0.10 # Backlog ratio threshold
|
|
368
|
+
config.cable_error_threshold = 0.0 # Error rate threshold
|
|
369
|
+
|
|
303
370
|
# Data Retention (persistence mode only)
|
|
304
371
|
config.event_retention = 30.days # Keep events for 30 days
|
|
305
372
|
config.metrics_retention = 90.days # Keep metrics for 90 days
|
|
@@ -341,7 +408,10 @@ SolidObserver.configure do |config|
|
|
|
341
408
|
end
|
|
342
409
|
```
|
|
343
410
|
|
|
344
|
-
When configured, all job events will include your correlation ID
|
|
411
|
+
When configured, all job, cache, and cable events will include your correlation ID. You can query the full cross-component event timeline for any request or job:
|
|
412
|
+
|
|
413
|
+
- **CLI**: `bin/rails "solid_observer:trace[CORRELATION_ID,LIMIT]"`
|
|
414
|
+
- **Web UI**: `/solid_observer/traces/:correlation_id`
|
|
345
415
|
|
|
346
416
|
### Production hardening (recommended)
|
|
347
417
|
|
|
@@ -370,6 +440,7 @@ end
|
|
|
370
440
|
| Command | Description |
|
|
371
441
|
|---------|-------------|
|
|
372
442
|
| `solid_observer:status` | Show queue status overview |
|
|
443
|
+
| `solid_observer:health` | Display aggregate and per-component health status |
|
|
373
444
|
| `solid_observer:jobs:list[status,queue,class,limit]` | List jobs with optional filters |
|
|
374
445
|
| `solid_observer:jobs:show[ID]` | Show job details |
|
|
375
446
|
| `solid_observer:jobs:retry[ID]` | Retry a failed job |
|
|
@@ -386,6 +457,8 @@ end
|
|
|
386
457
|
| `solid_observer:storage:purge` | Delete ALL SolidObserver data |
|
|
387
458
|
| `solid_observer:cache:clear` | Clear all SolidCache entries (with confirmation) |
|
|
388
459
|
| `solid_observer:cache:prune` | Prune expired SolidCache entries |
|
|
460
|
+
| `solid_observer:cable:trim` | Trim expired Solid Cable messages |
|
|
461
|
+
| `solid_observer:trace[CORRELATION_ID,LIMIT]` | Show cross-component trace timeline |
|
|
389
462
|
|
|
390
463
|
> **Note:** Storage commands manage **SolidObserver's storage** (event logs, metrics, snapshots) — not Solid Queue's jobs. Cache commands operate on SolidCache entries in the host app's cache store.
|
|
391
464
|
|
|
@@ -423,16 +496,18 @@ The example below is what `rails generate solid_observer:install` produces:
|
|
|
423
496
|
```yaml
|
|
424
497
|
# config/database.yml (generator default)
|
|
425
498
|
solid_observer_queue:
|
|
426
|
-
<<: *default
|
|
427
499
|
adapter: sqlite3
|
|
500
|
+
pool: 5
|
|
501
|
+
timeout: 5000
|
|
428
502
|
database: storage/<%= Rails.env %>_solid_observer_queue.sqlite3
|
|
503
|
+
migrations_paths: db/solid_observer_migrate
|
|
429
504
|
```
|
|
430
505
|
|
|
431
506
|
> **Note:** SQLite is the generator default, not a requirement. The `solid_observer_queue` database can use any Rails-supported adapter for record persistence. Adapter-native **storage-size monitoring** is currently implemented for SQLite, PostgreSQL/PostGIS, MySQL, and Trilogy. On other adapters, the size query returns `nil` and the engine logs a single `[SolidObserver] Unknown adapter for DatabaseSize: …` warning — record persistence still works.
|
|
432
507
|
|
|
433
508
|
### Multi-adapter setup
|
|
434
509
|
|
|
435
|
-
If your host application uses PostgreSQL and you want SolidObserver to use SQLite, keep the `solid_observer_queue` block **self-contained** — do not rely on `<<: *default`. The generator
|
|
510
|
+
If your host application uses PostgreSQL and you want SolidObserver to use SQLite, keep the `solid_observer_queue` block **self-contained** — do not rely on `<<: *default`. The generator produces a self-contained block with explicit `adapter: sqlite3` and `migrations_paths`, which is safe on any host adapter. On a PostgreSQL host, merging `<<: *default` without an explicit adapter override pulls the PG adapter in and fails at `db:create`. For multi-adapter connections, omit the merge entirely:
|
|
436
511
|
|
|
437
512
|
```yaml
|
|
438
513
|
# config/database.yml
|
|
@@ -473,9 +548,9 @@ gem "sqlite3", "~> 2.0"
|
|
|
473
548
|
| v0.1.0 | Solid Queue monitoring, CLI tools | ✅ Released |
|
|
474
549
|
| v0.1.1 | Real-time mode (no migrations needed) | ✅ Released |
|
|
475
550
|
| v0.3.0 | Web UI dashboard + stability hardening | ✅ Released |
|
|
476
|
-
| v0.4.0 | Solid Cache monitoring | ✅
|
|
477
|
-
| v0.5.0 | Solid Cable monitoring |
|
|
478
|
-
| v0.6.0 | Cross-component correlation, health scores |
|
|
551
|
+
| v0.4.0 | Solid Cache monitoring | ✅ Released |
|
|
552
|
+
| v0.5.0 | Solid Cable monitoring | ✅ Released |
|
|
553
|
+
| v0.6.0 | Cross-component trace correlation, health scores, unified dashboard | ✅ Current |
|
|
479
554
|
| v0.7.0 | Alerting & notifications | 🔜 Planned |
|
|
480
555
|
| v1.0.0 | Production stable release | 🎯 Goal |
|
|
481
556
|
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
var checkbox, rangeSelect, refreshBtn, helpBtn, helpPanel, helpWrapper, freshnessEl;
|
|
11
11
|
var hoverActive = false;
|
|
12
12
|
var sparks = {};
|
|
13
|
-
var url
|
|
13
|
+
var url;
|
|
14
14
|
var inFlight = false;
|
|
15
15
|
var timerId = null;
|
|
16
16
|
var lastFullSnapshot = null;
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
var wrapper = document.querySelector("[data-so-live]");
|
|
22
22
|
if (!wrapper) return;
|
|
23
23
|
|
|
24
|
+
url = wrapper.getAttribute("data-so-poll-url") || "/solid_observer/poll_data";
|
|
25
|
+
|
|
24
26
|
checkbox = wrapper.querySelector('[data-so-live-toggle]');
|
|
25
27
|
if (!checkbox) return;
|
|
26
28
|
|
|
@@ -6,6 +6,7 @@ module SolidObserver
|
|
|
6
6
|
[
|
|
7
7
|
*([PG::ConnectionBad] if defined?(PG::ConnectionBad)),
|
|
8
8
|
*([Mysql2::Error::ConnectionError] if defined?(Mysql2::Error::ConnectionError)),
|
|
9
|
+
*([Trilogy::Error] if defined?(Trilogy::Error)),
|
|
9
10
|
*([SQLite3::CantOpenException] if defined?(SQLite3::CantOpenException))
|
|
10
11
|
]
|
|
11
12
|
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "dashboard_controller"
|
|
4
|
+
|
|
5
|
+
module SolidObserver
|
|
6
|
+
# :reek:TooManyInstanceVariables
|
|
7
|
+
class CableDashboardController < DashboardController
|
|
8
|
+
CABLE_STORAGE_COMPONENTS = %w[cable_observer solid_cable].freeze
|
|
9
|
+
|
|
10
|
+
def index
|
|
11
|
+
@component = "cable"
|
|
12
|
+
assign_cable_dashboard
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
# :reek:TooManyStatements
|
|
18
|
+
def assign_cable_dashboard
|
|
19
|
+
unless SolidObserver.config.solid_cable_enabled?
|
|
20
|
+
@cable_dashboard_available = false
|
|
21
|
+
@storage_components = []
|
|
22
|
+
@recent_events = []
|
|
23
|
+
return
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
range = SolidObserver::Services::CableStats.parse_range(request_range_param)
|
|
27
|
+
window = SolidObserver::Services::CableStats.range_duration(range)
|
|
28
|
+
stats = SolidObserver::Services::CableStats.call(window: window)
|
|
29
|
+
|
|
30
|
+
@cable_dashboard_available = true
|
|
31
|
+
@range = range
|
|
32
|
+
@stats = stats
|
|
33
|
+
@storage_components = cable_storage_components
|
|
34
|
+
@recent_events = recent_events(window)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def cable_storage_components
|
|
38
|
+
SolidObserver::Services::StorageInfoSnapshot.call.select do |snapshot|
|
|
39
|
+
CABLE_STORAGE_COMPONENTS.include?(snapshot[:component])
|
|
40
|
+
end
|
|
41
|
+
rescue *SolidObserver::Services::StorageInfoSnapshot::CONNECTION_ERRORS, TypeError, NoMethodError
|
|
42
|
+
[]
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def recent_events(window)
|
|
46
|
+
current_time = Time.current
|
|
47
|
+
SolidObserver::CableEvent.where(recorded_at: (current_time - window)..current_time).recent(10)
|
|
48
|
+
rescue ActiveRecord::StatementInvalid
|
|
49
|
+
[]
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidObserver
|
|
4
|
+
class CableOperationsController < ApplicationController
|
|
5
|
+
def trim
|
|
6
|
+
redirect_with_result(SolidObserver::Services::CableOperations.trim)
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
def redirect_with_result(result)
|
|
12
|
+
flash_key = result[:ok] ? :notice : :alert
|
|
13
|
+
redirect_to cable_dashboard_path, flash_key => result[:message]
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -6,54 +6,47 @@ module SolidObserver
|
|
|
6
6
|
class CacheDashboardController < DashboardController
|
|
7
7
|
CACHE_STORAGE_COMPONENTS = %w[solid_cache cache_observer].freeze
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
range = SolidObserver::Services::CacheStats.parse_range(range_param)
|
|
14
|
-
window = SolidObserver::Services::CacheStats.range_duration(range)
|
|
15
|
-
stats = SolidObserver::Services::CacheStats.call(window: window)
|
|
16
|
-
|
|
17
|
-
{
|
|
18
|
-
cache_dashboard_available: true,
|
|
19
|
-
range: range,
|
|
20
|
-
stats: stats,
|
|
21
|
-
activity_trends: stats[:activity_trends],
|
|
22
|
-
stability: stats[:stability],
|
|
23
|
-
storage_components: cache_storage_components,
|
|
24
|
-
recent_events: recent_events(window)
|
|
25
|
-
}
|
|
26
|
-
end
|
|
9
|
+
def index
|
|
10
|
+
@component = "cache"
|
|
11
|
+
assign_cache_dashboard
|
|
12
|
+
end
|
|
27
13
|
|
|
28
|
-
|
|
14
|
+
private
|
|
29
15
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
16
|
+
def assign_cache_dashboard
|
|
17
|
+
unless SolidObserver.config.solid_cache_enabled?
|
|
18
|
+
@cache_dashboard_available = false
|
|
19
|
+
@storage_components = []
|
|
20
|
+
@recent_events = []
|
|
21
|
+
@activity_trends = SolidObserver::Services::CacheStats::ACTIVITY_TREND_EMPTY
|
|
22
|
+
@stability = SolidObserver::Services::CacheStats::STABILITY_EMPTY
|
|
23
|
+
return
|
|
38
24
|
end
|
|
39
25
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
26
|
+
range = SolidObserver::Services::CacheStats.parse_range(request_range_param)
|
|
27
|
+
window = SolidObserver::Services::CacheStats.range_duration(range)
|
|
28
|
+
stats = SolidObserver::Services::CacheStats.call(window: window)
|
|
29
|
+
|
|
30
|
+
@cache_dashboard_available = true
|
|
31
|
+
@range = range
|
|
32
|
+
@stats = stats
|
|
33
|
+
@activity_trends = stats[:activity_trends]
|
|
34
|
+
@stability = stats[:stability]
|
|
35
|
+
@storage_components = cache_storage_components
|
|
36
|
+
@recent_events = recent_events(window)
|
|
37
|
+
end
|
|
45
38
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
rescue ActiveRecord::StatementInvalid
|
|
50
|
-
[]
|
|
39
|
+
def cache_storage_components
|
|
40
|
+
SolidObserver::Services::StorageInfoSnapshot.call.select do |snapshot|
|
|
41
|
+
CACHE_STORAGE_COMPONENTS.include?(snapshot[:component])
|
|
51
42
|
end
|
|
52
43
|
end
|
|
53
44
|
|
|
54
|
-
def
|
|
55
|
-
|
|
56
|
-
|
|
45
|
+
def recent_events(window)
|
|
46
|
+
current_time = Time.current
|
|
47
|
+
SolidObserver::CacheEvent.where(recorded_at: (current_time - window)..current_time).recent(10)
|
|
48
|
+
rescue ActiveRecord::StatementInvalid
|
|
49
|
+
[]
|
|
57
50
|
end
|
|
58
51
|
end
|
|
59
52
|
end
|
|
@@ -11,15 +11,28 @@ module SolidObserver
|
|
|
11
11
|
|
|
12
12
|
def index
|
|
13
13
|
@component = selected_component
|
|
14
|
-
return assign_cache_dashboard if @component == "cache"
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
if @component == "queue" && SolidObserver.config.solid_queue_enabled?
|
|
16
|
+
assign_range_and_stats
|
|
17
|
+
load_persistence_data if persistence_mode?
|
|
18
|
+
elsif @component == "home"
|
|
19
|
+
@health = begin
|
|
20
|
+
Services::HealthScore.call
|
|
21
|
+
rescue => error
|
|
22
|
+
Rails.logger&.warn("Home HealthScore: #{error.class}: #{error.message}")
|
|
23
|
+
{overall: :degraded, components: {}}
|
|
24
|
+
end
|
|
25
|
+
@feed = begin
|
|
26
|
+
Services::UnifiedFeed.call
|
|
27
|
+
rescue => error
|
|
28
|
+
Rails.logger&.warn("Home UnifiedFeed: #{error.class}: #{error.message}")
|
|
29
|
+
[]
|
|
30
|
+
end
|
|
31
|
+
end
|
|
20
32
|
end
|
|
21
33
|
|
|
22
34
|
def live_poll
|
|
35
|
+
expires_in 1.day, public: true
|
|
23
36
|
send_file(
|
|
24
37
|
SolidObserver::Engine.root.join("app/assets/javascripts/solid_observer/live_poll.js"),
|
|
25
38
|
type: "application/javascript; charset=utf-8",
|
|
@@ -54,12 +67,6 @@ module SolidObserver
|
|
|
54
67
|
@recent_events = QueueEvent.recent(10)
|
|
55
68
|
end
|
|
56
69
|
|
|
57
|
-
def assign_cache_dashboard
|
|
58
|
-
SolidObserver::CacheDashboardController.cache_dashboard_assignments(range_param: request_range_param).each do |name, value|
|
|
59
|
-
instance_variable_set("@#{name}", value)
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
70
|
def request_range_param
|
|
64
71
|
request&.query_parameters&.[]("range") || request&.query_parameters&.[](:range)
|
|
65
72
|
end
|
|
@@ -104,17 +111,18 @@ module SolidObserver
|
|
|
104
111
|
""
|
|
105
112
|
end
|
|
106
113
|
requested = path_component if requested.empty?
|
|
114
|
+
return "queue" if requested == "queue" && SolidObserver.config.solid_queue_enabled?
|
|
107
115
|
return "cache" if requested == "cache" && SolidObserver.config.solid_cache_enabled?
|
|
108
116
|
|
|
109
|
-
"
|
|
117
|
+
"home"
|
|
110
118
|
end
|
|
111
119
|
|
|
112
120
|
def path_component
|
|
113
121
|
return "" unless request&.respond_to?(:path)
|
|
114
122
|
|
|
115
123
|
path = request&.path.to_s
|
|
116
|
-
return "cache" if path.end_with?("/cache")
|
|
117
124
|
return "queue" if path.end_with?("/queue")
|
|
125
|
+
return "cache" if path.end_with?("/cache")
|
|
118
126
|
|
|
119
127
|
""
|
|
120
128
|
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../helpers/solid_observer/application_helper"
|
|
4
|
+
|
|
5
|
+
module SolidObserver
|
|
6
|
+
class TracesController < ApplicationController
|
|
7
|
+
helper SolidObserver::ApplicationHelper
|
|
8
|
+
|
|
9
|
+
include RequirePersistenceMode
|
|
10
|
+
|
|
11
|
+
def show
|
|
12
|
+
result = SolidObserver::Queries::TraceQuery.new.call(correlation_id: params[:id])
|
|
13
|
+
@rows = result.rows
|
|
14
|
+
@unavailable_components = result.unavailable_components
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|