source_monitor 0.11.1 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. checksums.yaml +4 -4
  2. data/.claude/commands/rails-audit.md +77 -0
  3. data/.claude/commands/release.md +70 -47
  4. data/.claude/skills/sm-architecture/reference/module-map.md +32 -0
  5. data/.claude/skills/sm-domain-model/reference/model-graph.md +42 -1
  6. data/.claude/skills/sm-job/reference/job-conventions.md +46 -40
  7. data/.claude/skills/sm-upgrade/reference/version-history.md +19 -0
  8. data/CHANGELOG.md +57 -0
  9. data/CLAUDE.md +2 -2
  10. data/Gemfile.lock +7 -20
  11. data/RAILS_AUDIT.md +424 -0
  12. data/README.md +6 -3
  13. data/VERSION +1 -1
  14. data/app/assets/builds/source_monitor/application.css +4 -24
  15. data/app/assets/builds/source_monitor/application.js +57 -89
  16. data/app/assets/builds/source_monitor/application.js.map +4 -4
  17. data/app/assets/javascripts/source_monitor/application.js +3 -6
  18. data/app/assets/javascripts/source_monitor/controllers/dropdown_controller.js +6 -86
  19. data/app/assets/javascripts/source_monitor/controllers/filter_submit_controller.js +13 -0
  20. data/app/assets/javascripts/source_monitor/controllers/modal_controller.js +56 -0
  21. data/app/assets/javascripts/source_monitor/controllers/notification_controller.js +3 -13
  22. data/app/components/source_monitor/application_component.rb +10 -0
  23. data/app/components/source_monitor/filter_dropdown_component.rb +62 -0
  24. data/app/components/source_monitor/icon_component.rb +140 -0
  25. data/app/components/source_monitor/status_badge_component.html.erb +8 -0
  26. data/app/components/source_monitor/status_badge_component.rb +96 -0
  27. data/app/controllers/concerns/source_monitor/sanitizes_search_params.rb +4 -0
  28. data/app/controllers/concerns/source_monitor/set_source.rb +13 -0
  29. data/app/controllers/source_monitor/application_controller.rb +17 -0
  30. data/app/controllers/source_monitor/bulk_scrape_enablements_controller.rb +6 -10
  31. data/app/controllers/source_monitor/dashboard_controller.rb +5 -1
  32. data/app/controllers/source_monitor/import_history_dismissals_controller.rb +1 -1
  33. data/app/controllers/source_monitor/import_sessions_controller.rb +30 -9
  34. data/app/controllers/source_monitor/item_scrapes_controller.rb +70 -0
  35. data/app/controllers/source_monitor/items_controller.rb +2 -69
  36. data/app/controllers/source_monitor/source_bulk_scrapes_controller.rb +1 -4
  37. data/app/controllers/source_monitor/source_favicon_fetches_controller.rb +2 -12
  38. data/app/controllers/source_monitor/source_fetches_controller.rb +1 -6
  39. data/app/controllers/source_monitor/source_health_checks_controller.rb +9 -16
  40. data/app/controllers/source_monitor/source_health_resets_controller.rb +1 -6
  41. data/app/controllers/source_monitor/source_retries_controller.rb +1 -6
  42. data/app/controllers/source_monitor/source_scrape_tests_controller.rb +2 -4
  43. data/app/controllers/source_monitor/source_turbo_responses.rb +1 -3
  44. data/app/controllers/source_monitor/sources_controller.rb +15 -20
  45. data/app/helpers/source_monitor/application_helper.rb +15 -31
  46. data/app/helpers/source_monitor/health_badge_helper.rb +8 -0
  47. data/app/jobs/source_monitor/download_content_images_job.rb +1 -59
  48. data/app/jobs/source_monitor/favicon_fetch_job.rb +1 -58
  49. data/app/jobs/source_monitor/fetch_feed_job.rb +2 -52
  50. data/app/jobs/source_monitor/import_opml_job.rb +6 -145
  51. data/app/jobs/source_monitor/import_session_health_check_job.rb +15 -76
  52. data/app/jobs/source_monitor/item_cleanup_job.rb +5 -0
  53. data/app/jobs/source_monitor/log_cleanup_job.rb +13 -2
  54. data/app/jobs/source_monitor/schedule_fetches_job.rb +8 -0
  55. data/app/jobs/source_monitor/scrape_item_job.rb +6 -52
  56. data/app/jobs/source_monitor/source_health_check_job.rb +1 -72
  57. data/app/models/concerns/source_monitor/loggable.rb +12 -0
  58. data/app/models/source_monitor/fetch_log.rb +0 -8
  59. data/app/models/source_monitor/health_check_log.rb +0 -8
  60. data/app/models/source_monitor/import_history.rb +14 -0
  61. data/app/models/source_monitor/import_session.rb +2 -0
  62. data/app/models/source_monitor/item.rb +15 -0
  63. data/app/models/source_monitor/item_content.rb +4 -3
  64. data/app/models/source_monitor/scrape_log.rb +4 -6
  65. data/app/models/source_monitor/source.rb +28 -19
  66. data/app/presenters/source_monitor/base_presenter.rb +19 -0
  67. data/app/presenters/source_monitor/source_details_presenter.rb +61 -0
  68. data/app/presenters/source_monitor/sources_filter_presenter.rb +61 -0
  69. data/app/views/source_monitor/dashboard/_recent_activity.html.erb +3 -3
  70. data/app/views/source_monitor/dashboard/_stat_card.html.erb +2 -1
  71. data/app/views/source_monitor/dashboard/_stats.html.erb +5 -7
  72. data/app/views/source_monitor/items/_details.html.erb +11 -14
  73. data/app/views/source_monitor/items/index.html.erb +10 -35
  74. data/app/views/source_monitor/logs/index.html.erb +20 -41
  75. data/app/views/source_monitor/shared/_form_errors.html.erb +14 -0
  76. data/app/views/source_monitor/source_scrape_tests/_result.html.erb +1 -29
  77. data/app/views/source_monitor/source_scrape_tests/_result_content.html.erb +33 -0
  78. data/app/views/source_monitor/source_scrape_tests/show.html.erb +1 -29
  79. data/app/views/source_monitor/sources/_bulk_scrape_enable_modal.html.erb +2 -2
  80. data/app/views/source_monitor/sources/_bulk_scrape_modal.html.erb +7 -5
  81. data/app/views/source_monitor/sources/_details.html.erb +24 -52
  82. data/app/views/source_monitor/sources/_health_status_badge.html.erb +4 -6
  83. data/app/views/source_monitor/sources/_row.html.erb +7 -18
  84. data/app/views/source_monitor/sources/edit.html.erb +1 -10
  85. data/app/views/source_monitor/sources/index.html.erb +26 -46
  86. data/app/views/source_monitor/sources/new.html.erb +1 -10
  87. data/config/routes.rb +1 -1
  88. data/db/migrate/20260313120000_add_composite_indexes_to_log_tables.rb +14 -0
  89. data/db/migrate/20260314120000_align_health_status_default.rb +11 -0
  90. data/docs/setup.md +2 -2
  91. data/docs/upgrade.md +23 -0
  92. data/lib/source_monitor/analytics/sources_index_metrics.rb +15 -0
  93. data/lib/source_monitor/dashboard/queries/recent_activity_query.rb +10 -4
  94. data/lib/source_monitor/dashboard/turbo_broadcaster.rb +21 -5
  95. data/lib/source_monitor/favicons/fetcher.rb +86 -0
  96. data/lib/source_monitor/fetching/cloudflare_bypass.rb +14 -5
  97. data/lib/source_monitor/fetching/completion/event_publisher.rb +12 -0
  98. data/lib/source_monitor/fetching/completion/follow_up_handler.rb +15 -2
  99. data/lib/source_monitor/fetching/completion/retention_handler.rb +11 -3
  100. data/lib/source_monitor/fetching/feed_fetcher.rb +2 -21
  101. data/lib/source_monitor/fetching/fetch_runner.rb +12 -3
  102. data/lib/source_monitor/fetching/retry_orchestrator.rb +102 -0
  103. data/lib/source_monitor/fetching/stalled_fetch_reconciler.rb +9 -0
  104. data/lib/source_monitor/health/source_health_check_orchestrator.rb +95 -0
  105. data/lib/source_monitor/health.rb +1 -0
  106. data/lib/source_monitor/images/downloader.rb +6 -7
  107. data/lib/source_monitor/images/processor.rb +98 -0
  108. data/lib/source_monitor/import_sessions/health_check_updater.rb +95 -0
  109. data/lib/source_monitor/import_sessions/opml_importer.rb +163 -0
  110. data/lib/source_monitor/items/item_creator.rb +0 -21
  111. data/lib/source_monitor/logs/query.rb +20 -0
  112. data/lib/source_monitor/queries/scrape_candidates_query.rb +30 -0
  113. data/lib/source_monitor/queries.rb +7 -0
  114. data/lib/source_monitor/scheduler.rb +5 -0
  115. data/lib/source_monitor/scraping/bulk_result_presenter.rb +11 -8
  116. data/lib/source_monitor/scraping/runner.rb +52 -0
  117. data/lib/source_monitor/scraping/scheduler.rb +5 -0
  118. data/lib/source_monitor/scraping/state.rb +4 -2
  119. data/lib/source_monitor/security/parameter_sanitizer.rb +7 -0
  120. data/lib/source_monitor/version.rb +1 -1
  121. data/lib/source_monitor.rb +7 -0
  122. data/source_monitor.gemspec +1 -0
  123. metadata +47 -1
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- source_monitor (0.11.1)
4
+ source_monitor (0.12.1)
5
5
  cssbundling-rails (~> 1.4)
6
6
  faraday (~> 2.9)
7
7
  faraday-follow_redirects (~> 0.4)
@@ -16,6 +16,7 @@ PATH
16
16
  solid_cable (>= 3.0, < 4.0)
17
17
  solid_queue (>= 0.3, < 3.0)
18
18
  turbo-rails (~> 2.0)
19
+ view_component (>= 3.0, < 4.0)
19
20
 
20
21
  GEM
21
22
  remote: https://rubygems.org/
@@ -177,6 +178,7 @@ GEM
177
178
  net-smtp
178
179
  marcel (1.1.0)
179
180
  matrix (0.4.3)
181
+ method_source (1.1.0)
180
182
  mini_magick (5.3.1)
181
183
  logger
182
184
  mini_mime (1.1.5)
@@ -207,18 +209,7 @@ GEM
207
209
  racc (~> 1.4)
208
210
  nokogiri (1.19.0-arm-linux-musl)
209
211
  racc (~> 1.4)
210
- nokogiri (1.19.0-arm64-darwin)
211
- racc (~> 1.4)
212
- nokogiri (1.19.0-x86_64-darwin)
213
- racc (~> 1.4)
214
- nokogiri (1.19.0-x86_64-linux-gnu)
215
- racc (~> 1.4)
216
- nokogiri (1.19.0-x86_64-linux-musl)
217
- racc (~> 1.4)
218
212
  nokolexbor (0.6.2)
219
- nokolexbor (0.6.2-arm64-darwin)
220
- nokolexbor (0.6.2-x86_64-darwin)
221
- nokolexbor (0.6.2-x86_64-linux)
222
213
  ostruct (0.6.3)
223
214
  parallel (1.27.0)
224
215
  parser (3.3.10.1)
@@ -227,10 +218,6 @@ GEM
227
218
  pg (1.6.3)
228
219
  pg (1.6.3-aarch64-linux)
229
220
  pg (1.6.3-aarch64-linux-musl)
230
- pg (1.6.3-arm64-darwin)
231
- pg (1.6.3-x86_64-darwin)
232
- pg (1.6.3-x86_64-linux)
233
- pg (1.6.3-x86_64-linux-musl)
234
221
  pp (0.6.3)
235
222
  prettyprint
236
223
  prettyprint (0.2.0)
@@ -375,6 +362,10 @@ GEM
375
362
  uri (1.1.1)
376
363
  useragent (0.16.11)
377
364
  vcr (6.4.0)
365
+ view_component (3.24.0)
366
+ activesupport (>= 5.2.0, < 8.2)
367
+ concurrent-ruby (~> 1)
368
+ method_source (~> 1.0)
378
369
  webmock (3.26.1)
379
370
  addressable (>= 2.8.0)
380
371
  crack (>= 0.3.2)
@@ -394,11 +385,7 @@ PLATFORMS
394
385
  aarch64-linux-musl
395
386
  arm-linux-gnu
396
387
  arm-linux-musl
397
- arm64-darwin
398
388
  ruby
399
- x86_64-darwin
400
- x86_64-linux-gnu
401
- x86_64-linux-musl
402
389
 
403
390
  DEPENDENCIES
404
391
  brakeman
data/RAILS_AUDIT.md ADDED
@@ -0,0 +1,424 @@
1
+ # Rails Best Practices Audit — 2026-03-14
2
+
3
+ ## Executive Summary
4
+
5
+ 5 parallel agents audited the entire SourceMonitor engine codebase across models, controllers, services/jobs/pipeline, views/frontend, and testing layers.
6
+
7
+ | Severity | Remaining | Already Fixed |
8
+ |----------|-----------|---------------|
9
+ | **HIGH** | 6 | 0 |
10
+ | **MEDIUM** | 17 | 10 |
11
+ | **LOW** | 21 | 9 |
12
+ | **Total** | **44** | **19** |
13
+
14
+ **Overall verdict:** The codebase is well-structured for an engine of this complexity. Routes follow CRUD conventions, concerns are well-scoped, and the Hotwire frontend has solid fundamentals. Recent milestone work (phases 01-06) already resolved ~19 findings. The remaining gaps are primarily **business logic in jobs** (violates "shallow jobs" convention) and **duplicated logic** across pipeline layers.
15
+
16
+ > **Note:** 19 findings from the initial 63 were already addressed by commits in the recent ui-fixes-and-smart-scraping milestone. These are marked ~~strikethrough~~ below. The counts above reflect only unresolved findings.
17
+
18
+ ---
19
+
20
+ ## Table of Contents
21
+
22
+ - [HIGH Severity Findings](#high-severity-findings)
23
+ - [MEDIUM Severity Findings](#medium-severity-findings)
24
+ - [LOW Severity Findings](#low-severity-findings)
25
+ - [Top 10 Actions](#top-10-actions-prioritized-by-impacteffort-ratio)
26
+ - [Positive Observations](#positive-observations)
27
+
28
+ ---
29
+
30
+ ## HIGH Severity Findings
31
+
32
+ ### H1. LogCleanupJob orphans LogEntry records
33
+
34
+ - **File(s):** `app/jobs/source_monitor/log_cleanup_job.rb:42-49`
35
+ - **Current:** Uses `batch.delete_all` on FetchLog/ScrapeLog records. These models have `has_one :log_entry, dependent: :destroy`, but `delete_all` skips callbacks, orphaning LogEntry records.
36
+ - **Recommended:** Delete LogEntry records first by `loggable_type`/`loggable_id`, then delete the log records. Or use `destroy_in_batches`.
37
+ - **Rationale:** Orphaned LogEntry records accumulate over time, consuming disk space and corrupting the unified logs view. The `dependent: :destroy` declaration shows cascade was intended.
38
+ - **Effort:** short
39
+
40
+ ### H2. ImportOpmlJob contains 160 lines of business logic
41
+
42
+ - **File(s):** `app/jobs/source_monitor/import_opml_job.rb:14-157`
43
+ - **Current:** The job contains entry selection, deduplication, source creation, attribute building, broadcast logic, and error aggregation. This is multi-model orchestration (Source, ImportHistory, ImportSession) in a job.
44
+ - **Recommended:** Extract to `SourceMonitor::ImportSessions::OPMLImporter` service. The job becomes a 5-line delegation.
45
+ - **Rationale:** Violates "shallow jobs: only deserialization + delegation." Import logic cannot be invoked synchronously (console, tests) without going through ActiveJob. Spans 3+ models, qualifying for a service object.
46
+ - **Effort:** medium
47
+
48
+ ### H3. ScrapeItemJob contains rate-limiting, state management, and deferral logic
49
+
50
+ - **File(s):** `app/jobs/source_monitor/scrape_item_job.rb:14-57`
51
+ - **Current:** Checks scraping-enabled status, computes time-until-scrape-allowed, manages state transitions (`mark_processing!`, `mark_failed!`, `clear_inflight!`), and re-enqueues itself with a delay.
52
+ - **Recommended:** Move pre-flight checks and state management into `Scraping::Runner`. Job becomes a one-liner delegation.
53
+ - **Rationale:** Rate-limiting in `time_until_scrape_allowed` duplicates near-identical logic in `Scraping::Enqueuer#time_rate_limited?`. Two places to maintain the same business rule.
54
+ - **Effort:** medium
55
+
56
+ ### H4. DownloadContentImagesJob contains multi-model orchestration
57
+
58
+ - **File(s):** `app/jobs/source_monitor/download_content_images_job.rb:17-49`
59
+ - **Current:** Builds ItemContent, downloads images, creates ActiveStorage blobs, rewrites HTML, and updates the item.
60
+ - **Recommended:** Extract to `SourceMonitor::Images::Processor`. Job delegates with a single call.
61
+ - **Rationale:** Multi-model orchestration (Item, ItemContent, ActiveStorage::Blob) belongs in a pipeline class, not a job.
62
+ - **Effort:** short
63
+
64
+ ### H5. Scrape rate-limiting duplicated in two places
65
+
66
+ - **File(s):** `app/jobs/source_monitor/scrape_item_job.rb:47-57` and `lib/source_monitor/scraping/enqueuer.rb:129-143`
67
+ - **Current:** Both compute time since last scrape vs. `min_scrape_interval`. The Enqueuer defers the job, and the Job re-checks and defers again.
68
+ - **Recommended:** Remove the check from `ScrapeItemJob`. The Enqueuer already handles deferral at enqueue time. If race conditions are a concern, have the job delegate to a runner that calls the Enqueuer.
69
+ - **Rationale:** Redundant DB queries and divergence risk if one is updated without the other.
70
+ - **Effort:** quick
71
+
72
+ ### H6. `Source.destroy_all` in pagination tests is not parallel-safe
73
+
74
+ - **File(s):** `test/controllers/source_monitor/sources_controller_test.rb:252,264,276,286,297,311,322`
75
+ - **Current:** Seven tests call `Source.destroy_all` to get a clean slate for pagination counting. With thread-based parallelism, this can race with other threads.
76
+ - **Recommended:** Scope assertions to test-created records using a naming pattern or tracking IDs. Or use a dedicated test class with proper isolation.
77
+ - **Rationale:** Violates the project's own documented isolation rule in `TEST_CONVENTIONS.md` section 6.
78
+ - **Effort:** medium
79
+
80
+ ---
81
+
82
+ ## MEDIUM Severity Findings
83
+
84
+ ### Models & Concerns
85
+
86
+ #### M1. `health_status` default mismatch between model and database
87
+
88
+ - **File(s):** `app/models/source_monitor/source.rb:37` vs `db/schema.rb:384`
89
+ - **Current:** Model declares `attribute :health_status, :string, default: "working"` but schema has `default: "healthy"`. A Source created in Ruby gets `"working"`, one via raw SQL gets `"healthy"`.
90
+ - **Recommended:** Align the defaults. Add `validates :health_status, inclusion: { in: HEALTH_STATUS_VALUES }`.
91
+ - **Effort:** quick
92
+
93
+ #### M2. Missing `health_status` validation
94
+
95
+ - **File(s):** `app/models/source_monitor/source.rb:44-51`
96
+ - **Current:** `fetch_status` has an inclusion validation; `health_status` has none. Any arbitrary string can be stored.
97
+ - **Recommended:** Add `HEALTH_STATUS_VALUES = %w[healthy working declining failing].freeze` and `validates :health_status, inclusion: { in: HEALTH_STATUS_VALUES }`.
98
+ - **Effort:** quick
99
+
100
+ #### M3. `Item#soft_delete!` counter cache fragility
101
+
102
+ - **File(s):** `app/models/source_monitor/item.rb:69-83`
103
+ - **Current:** Manually calls `Source.decrement_counter(:items_count, source_id)` after `update_columns`. No corresponding `restore!` method to re-increment.
104
+ - **Recommended:** Add a `restore!` method for symmetry. Consider extracting soft-delete into a concern.
105
+ - **Effort:** short
106
+
107
+ #### M4. Duplicated `sync_log_entry` callback across 3 log models
108
+
109
+ - **File(s):** `app/models/source_monitor/fetch_log.rb:28`, `scrape_log.rb:20`, `health_check_log.rb:20`
110
+ - **Current:** All three define identical `after_save :sync_log_entry` callbacks.
111
+ - **Recommended:** Move into the `Loggable` concern.
112
+ - **Effort:** quick
113
+
114
+ ### Controllers & Routes
115
+
116
+ #### M5. No `rescue_from ActiveRecord::RecordNotFound`
117
+
118
+ - **File(s):** `app/controllers/source_monitor/application_controller.rb`
119
+ - **Current:** No rescue_from handlers. A missing record raises 500 in production.
120
+ - **Recommended:** Add a Turbo-aware RecordNotFound handler that renders a toast + 404.
121
+ - **Rationale:** As a mountable engine, SourceMonitor should handle its own common exceptions gracefully.
122
+ - **Effort:** short
123
+
124
+ #### M6. Duplicated `set_source` across 7 controllers
125
+
126
+ - **File(s):** `source_fetches_controller.rb`, `source_retries_controller.rb`, `source_bulk_scrapes_controller.rb`, `source_health_checks_controller.rb`, `source_health_resets_controller.rb`, `source_favicon_fetches_controller.rb`, `source_scrape_tests_controller.rb`
127
+ - **Current:** Each defines identical `def set_source; @source = Source.find(params[:source_id]); end`.
128
+ - **Recommended:** Extract to a `SetSource` concern.
129
+ - **Effort:** quick
130
+
131
+ #### M7. `fallback_user_id` creates users in host-app tables
132
+
133
+ - **File(s):** `app/controllers/source_monitor/import_sessions_controller.rb:244-276`
134
+ - **Current:** When no authenticated user exists, creates a "guest" user by introspecting column schema.
135
+ - **Recommended:** Guard behind `Rails.env.development?` or remove entirely. An engine should never create records in host-app tables.
136
+ - **Effort:** short
137
+
138
+ #### M8. ImportSessions controller concerns contain significant business logic
139
+
140
+ - **File(s):** `app/controllers/source_monitor/import_sessions/opml_parser.rb` (128 lines), `entry_annotation.rb` (187 lines), `health_check_management.rb` (112 lines), `bulk_configuration.rb` (106 lines)
141
+ - **Current:** XML parsing, URL validation, duplicate detection, job enqueueing, database locking — all in controller concerns.
142
+ - **Recommended:** Extract pure-domain parts to `lib/` or `app/services/` classes. Controller concerns become thin wrappers.
143
+ - **Effort:** large
144
+
145
+ #### ~~M9. `SourcesController#index` has 47 lines of query orchestration~~ RESOLVED
146
+
147
+ - ~~Resolved by `a6d7148` (extract sources index metrics) + `795b7b8` (SourcesFilterPresenter) + `cafefc2` (FilterDropdownComponent)~~
148
+
149
+ #### M10. `BulkScrapeEnablementsController` contains business logic
150
+
151
+ - **File(s):** `app/controllers/source_monitor/bulk_scrape_enablements_controller.rb:13-19`
152
+ - **Current:** `update_all` with field combination for enabling scraping is in the controller.
153
+ - **Recommended:** Extract to `Source.enable_scraping!(ids)` class method.
154
+ - **Effort:** quick
155
+
156
+ #### M11. Excessive `update_column` usage in ImportSessions flow
157
+
158
+ - **File(s):** `app/controllers/source_monitor/import_sessions_controller.rb` (11 calls)
159
+ - **Current:** Skips validations for `current_step` and `selected_source_ids` changes.
160
+ - **Recommended:** Encapsulate in model methods like `ImportSession#advance_to!(step)`.
161
+ - **Effort:** short
162
+
163
+ ### Services, Jobs & Pipeline
164
+
165
+ #### M12. FaviconFetchJob contains cooldown and attachment logic
166
+
167
+ - **File(s):** `app/jobs/source_monitor/favicon_fetch_job.rb:17-42`
168
+ - **Current:** Cooldown checking duplicated with `SourceUpdater#enqueue_favicon_fetch_if_needed`.
169
+ - **Recommended:** Extract to `Favicons::FetchService`. Consolidate cooldown in `Favicons::CooldownCheck`.
170
+ - **Effort:** short
171
+
172
+ #### M13. ImportSessionHealthCheckJob contains lock management
173
+
174
+ - **File(s):** `app/jobs/source_monitor/import_session_health_check_job.rb:18-63`
175
+ - **Current:** Acquires row lock, merges results, updates state, broadcasts.
176
+ - **Recommended:** Extract to `ImportSessions::HealthCheckUpdater`.
177
+ - **Effort:** short
178
+
179
+ #### M14. SourceHealthCheckJob contains broadcast and toast formatting
180
+
181
+ - **File(s):** `app/jobs/source_monitor/source_health_check_job.rb:29-83`
182
+ - **Current:** `toast_payload`, `broadcast_outcome`, `trigger_fetch_if_degraded` are all presentation/side-effect logic.
183
+ - **Recommended:** Move into `Health::SourceHealthCheckOrchestrator`.
184
+ - **Effort:** short
185
+
186
+ #### ~~M15. Inconsistent Result pattern across pipeline classes~~ PARTIALLY RESOLVED
187
+
188
+ - ~~`e03723d` added Result structs to completion handlers; `5bd538a` wired Result usage in FetchRunner~~
189
+ - **Remaining:** `FeedFetcher::Result` still lacks `success?`. No shared base `SourceMonitor::Result` class exists yet.
190
+ - **Effort:** medium
191
+
192
+ #### ~~M16. Retry logic split across 4 locations~~ RESOLVED
193
+
194
+ - ~~Resolved by `4ff8884` (extract FetchFeedJob retry orchestrator service)~~
195
+
196
+ #### M17. Swallowed exceptions in ensure/rescue blocks
197
+
198
+ - **File(s):** `feed_fetcher.rb:331`, `scraping/state.rb:68`, `source_health_check_job.rb:46-47`
199
+ - **Current:** `rescue StandardError => nil` silently swallows failures.
200
+ - **Recommended:** Add `Rails.logger.warn` in rescue blocks.
201
+ - **Effort:** quick
202
+
203
+ #### M18. StalledFetchReconciler uses fragile PG JSON operator on SolidQueue internals
204
+
205
+ - **File(s):** `lib/source_monitor/fetching/stalled_fetch_reconciler.rb:107`
206
+ - **Current:** `where("arguments::jsonb -> 'arguments' ->> 0 = ?", source.id.to_s)` — fragile if SolidQueue changes serialization.
207
+ - **Recommended:** Add version comment and regression test.
208
+ - **Effort:** quick
209
+
210
+ ### Views & Frontend
211
+
212
+ #### M19. Database queries executed in view templates
213
+
214
+ - **File(s):** `items/_details.html.erb:6-7`, `sources/_bulk_scrape_modal.html.erb:3-11`
215
+ - **Current:** `item.scrape_logs.order(...).limit(5)` and similar queries directly in ERB.
216
+ - **Recommended:** Move to controllers and pass as locals, or extend presenters.
217
+ - **Note:** `sources/_details.html.erb` was partially addressed by `SourceDetailsPresenter` (`7c2604b`), but items and bulk scrape modal still have inline queries.
218
+ - **Effort:** short
219
+
220
+ #### M20. StatusBadge markup duplicated 12+ times
221
+
222
+ - **File(s):** `sources/_row.html.erb`, `sources/_details.html.erb`, `dashboard/_recent_activity.html.erb`, `items/_details.html.erb`, `items/index.html.erb`, `logs/index.html.erb`
223
+ - **Current:** Hand-crafted `<span class="inline-flex items-center rounded-full ...">` with conditional spinners.
224
+ - **Recommended:** Create a `StatusBadgeComponent`.
225
+ - **Note:** `IconComponent` was added (`caa4e69`) for SVG icons, but status badges are a separate pattern that still needs extraction.
226
+ - **Effort:** medium
227
+
228
+ #### ~~M21. Missing SourceDetailsPresenter~~ PARTIALLY RESOLVED
229
+
230
+ - ~~`SourceDetailsPresenter` added in `7c2604b`~~
231
+ - **Remaining:** `ItemDetailsPresenter`, `FetchLogPresenter`, `ScrapeLogPresenter`, `SourceRowPresenter` still missing.
232
+ - **Effort:** medium (each)
233
+
234
+ #### M22. Modal missing `role="dialog"` and `aria-modal`
235
+
236
+ - **File(s):** `sources/_bulk_scrape_modal.html.erb`, `sources/_bulk_scrape_enable_modal.html.erb`
237
+ - **Current:** Modal panels are plain `<div>` elements.
238
+ - **Recommended:** Add `role="dialog"`, `aria-modal="true"`, `aria-labelledby` pointing to heading.
239
+ - **Effort:** quick
240
+
241
+ #### M23. Modal controller lacks focus trapping
242
+
243
+ - **File(s):** `app/assets/javascripts/source_monitor/controllers/modal_controller.js`
244
+ - **Current:** Handles Escape and backdrop click but doesn't trap focus. Users can Tab out.
245
+ - **Recommended:** Implement focus trapping with `inert` attribute on background elements.
246
+ - **Rationale:** WCAG 2.1 SC 2.4.3 requires meaningful focus order.
247
+ - **Effort:** medium
248
+
249
+ #### M24. Logs index missing Turbo Frame for filter/pagination
250
+
251
+ - **File(s):** `app/views/source_monitor/logs/index.html.erb`
252
+ - **Current:** No Turbo Frame wrapping. `form_with` uses `local: true` disabling Turbo. Full page reload on filter.
253
+ - **Recommended:** Wrap table+pagination in a Turbo Frame. Sources and Items both use this pattern.
254
+ - **Effort:** medium
255
+
256
+ #### M25. Button styles inconsistent across templates
257
+
258
+ - **File(s):** Virtually every template
259
+ - **Current:** 5-6 button variants with inconsistent `font-semibold` vs `font-medium`, padding variations.
260
+ - **Recommended:** Extract button variants to `@apply` CSS classes or a `ButtonComponent`.
261
+ - **Effort:** medium
262
+
263
+ #### M26. `ApplicationHelper` is 333 lines with mixed concerns
264
+
265
+ - **File(s):** `app/helpers/source_monitor/application_helper.rb`
266
+ - **Current:** 20+ methods spanning badges, favicons, pagination, URLs, formatting.
267
+ - **Recommended:** Split into focused helper modules: `StatusBadgeHelper`, `FaviconHelper`, `PaginationHelper`, `FetchIntervalHelper`, `ExternalLinkHelper`.
268
+ - **Note:** `SourcesFilterPresenter` (`795b7b8`) and `FilterDropdownComponent` (`cafefc2`) extracted some filter logic, but the helper itself is still large.
269
+ - **Effort:** medium
270
+
271
+ ### Testing
272
+
273
+ #### ~~M27. `create_item!` factory underused~~ PARTIALLY RESOLVED
274
+
275
+ - ~~`18692f6` centralized factory helpers into ModelFactories module~~
276
+ - **Remaining:** Many test files still use manual `Item.create!` instead of the now-centralized `create_item!`. Migration to the shared factories is incomplete.
277
+ - **Effort:** medium
278
+
279
+ ---
280
+
281
+ ## LOW Severity Findings
282
+
283
+ ### Controllers & Routes
284
+
285
+ #### L1. `new` action delegates to `create` in ImportSessionsController (GET creates records)
286
+ - `import_sessions_controller.rb:35-37` — quick
287
+
288
+ #### L2. `BulkScrapeEnablementsController` accesses params without strong params wrapper
289
+ - `bulk_scrape_enablements_controller.rb:6` — quick
290
+
291
+ #### L3. `SourceScrapeTestsController#create` builds result hash inline (should be presenter)
292
+ - `source_scrape_tests_controller.rb:14-24` — short
293
+
294
+ #### L4. `SourceHealthChecksController` embeds Tailwind classes in controller
295
+ - `source_health_checks_controller.rb:25-31` — quick
296
+
297
+ #### L5. Inconsistent Turbo Stream response patterns (StreamResponder vs raw arrays)
298
+ - Multiple controllers — short
299
+
300
+ #### ~~L6. Broad `rescue StandardError` in action controllers~~ RESOLVED
301
+ - ~~Resolved by `6bcd0ac` and `19bb3b8` (transient vs fatal error classification in FaviconFetchJob and DownloadContentImagesJob) + `911c17e` (deadlock rescue + error logging)~~
302
+
303
+ #### L7. `SanitizesSearchParams` uses `to_unsafe_h` without documentation
304
+ - `concerns/source_monitor/sanitizes_search_params.rb:45` — quick
305
+
306
+ #### L8. `SourcesController#update` contains conditional job-enqueue logic
307
+ - `sources_controller.rb:94-109` — short
308
+
309
+ ### Models
310
+
311
+ #### L9. Ransacker subqueries could be extracted to `Source::SearchableAttributes` concern
312
+ - `source.rb:79-103` — short
313
+
314
+ #### L10. Missing `scraping_enabled` / `scraping_disabled` scopes
315
+ - `source.rb` — quick
316
+
317
+ #### L11. `ItemContent#compute_feed_word_count` reaches through association (minor Demeter violation)
318
+ - `item_content.rb:33-39` — quick
319
+
320
+ #### L12. `ImportHistory` missing chronological validation and JSONB attribute declarations
321
+ - `import_history.rb` — quick
322
+
323
+ ### Services, Jobs & Pipeline
324
+
325
+ #### ~~L13. `FetchFeedJob#should_run?` scheduling logic~~ PARTIALLY RESOLVED
326
+ - ~~`4ff8884` extracted retry orchestrator, reducing job logic. `should_run?` guard still exists but is simpler.~~
327
+
328
+ #### L14. Backward-compatibility forwarding methods in FeedFetcher (12) and ItemCreator (18)
329
+ - `feed_fetcher.rb:393-404`, `item_creator.rb:179-197` — medium
330
+
331
+ #### L15. FeedFetcher constants duplicated from AdaptiveInterval
332
+ - `feed_fetcher.rb:30-36` — quick
333
+
334
+ #### L16. Inconsistent logger guard pattern (20+ occurrences of full guard)
335
+ - Nearly every pipeline file — short
336
+
337
+ #### L17. `Images::Downloader` creates raw Faraday connection instead of `HTTP.client`
338
+ - `images/downloader.rb:46-58` — quick
339
+
340
+ #### L18. `Logs::Query` is good; `Scheduler` queries could be extracted
341
+ - `scheduler.rb:55-89`, `scraping/scheduler.rb:38-45` — short
342
+
343
+ #### L19. CloudflareBypass tries all 4 user agents sequentially (could cause 60s+ fetch)
344
+ - `fetching/cloudflare_bypass.rb:39-49` — quick
345
+
346
+ ### Views & Frontend
347
+
348
+ #### L20. Items and Logs index pagination not using shared `_pagination.html.erb` partial
349
+ - `items/index.html.erb:124-146`, `logs/index.html.erb:186-209` — short
350
+
351
+ #### L21. Scrape test result markup duplicated between show page and modal
352
+ - `source_scrape_tests/show.html.erb:8-56`, `_result.html.erb:12-60` — quick
353
+
354
+ #### L22. Card panel pattern repeated ~20 times (potential `PanelComponent`)
355
+ - Various dashboard/sources/items views — medium
356
+
357
+ #### L23. Error display duplicated across new/edit views (should be `_form_errors` partial)
358
+ - `sources/new.html.erb:4-13`, `edit.html.erb:4-13` — quick
359
+
360
+ #### ~~L24. Dropdown controller registers global click listener eagerly~~ RESOLVED
361
+ - ~~Resolved by `491fae1` (simplify dropdown controller and remove JS globals)~~
362
+
363
+ #### ~~L25. Notification controller has dead `applyLevelDelay()` method~~ RESOLVED
364
+ - ~~Resolved by `15e7d53` (remove dead JS error delay override and document constants)~~
365
+
366
+ #### ~~L26. Dismiss button SVG missing `aria-label`, should use `IconComponent`~~ PARTIALLY RESOLVED
367
+ - ~~`4c56789` replaced inline SVGs with IconComponent. Check if this specific dismiss button was included.~~
368
+
369
+ #### L27. `FilterDropdownComponent` uses inline `onchange` instead of Stimulus action
370
+ - `filter_dropdown_component.rb:48,55` — short
371
+
372
+ ### Testing
373
+
374
+ #### L28. Duplicated `configure_authentication` helper across 4 test files
375
+ - 4 test files — quick
376
+
377
+ #### ~~L29. Duplicated SolidQueue table purge logic~~ PARTIALLY RESOLVED
378
+ - ~~`54617b8` created SystemTestHelpers module with shared purge method. Some lib tests may still inline it.~~
379
+
380
+ #### L30. No test files for FetchLogsController, ScrapeLogsController, ImportHistory model
381
+ - Missing test files — short
382
+
383
+ ---
384
+
385
+ ## Top 10 Actions (prioritized by impact/effort ratio)
386
+
387
+ | # | Finding | Severity | Effort | Category |
388
+ |---|---------|----------|--------|----------|
389
+ | 1 | **H1** — Fix LogCleanupJob orphaned LogEntry records | HIGH | short | Data integrity |
390
+ | 2 | **H5** — Remove duplicated scrape rate-limiting from ScrapeItemJob | HIGH | quick | DRY |
391
+ | 3 | **M6** — Extract `set_source` into shared concern | MEDIUM | quick | DRY |
392
+ | 4 | **M1+M2** — Align `health_status` default + add validation | MEDIUM | quick | Correctness |
393
+ | 5 | **M4** — Move `sync_log_entry` callback into Loggable concern | MEDIUM | quick | DRY |
394
+ | 6 | **M5** — Add `rescue_from RecordNotFound` | MEDIUM | short | Robustness |
395
+ | 7 | **H4** — Extract DownloadContentImagesJob orchestration | HIGH | short | Convention |
396
+ | 8 | **H6** — Fix pagination test parallel-safety | HIGH | medium | Test reliability |
397
+ | 9 | **H2** — Extract ImportOpmlJob business logic to service | HIGH | medium | Convention |
398
+ | 10 | **M20** — Create StatusBadgeComponent | MEDIUM | medium | DRY/Consistency |
399
+
400
+ > **Previously in Top 10, now resolved:** M9 (SourcesController#index metrics — extracted to presenters), M16 (retry logic consolidation — extracted to RetryOrchestrator service)
401
+
402
+ ---
403
+
404
+ ## Positive Observations
405
+
406
+ The audit identified many areas where the codebase excels:
407
+
408
+ - **CRUD route design** is textbook — every action is a resource (`source_fetches`, `source_retries`, etc.)
409
+ - **Loggable concern** is exemplary single-purpose shared behavior
410
+ - **Boolean usage** correctly follows state-as-records convention (all booleans are technical flags)
411
+ - **Association declarations** include `inverse_of`, `dependent: :destroy`, and thorough indexing
412
+ - **Factory helpers** (`ModelFactories`) have good defaults with `SecureRandom` for parallel safety
413
+ - **VCR/WebMock separation** is clean (VCR for real feeds, WebMock for controlled scenarios)
414
+ - **Source turbo responses** concern is well-focused on response rendering
415
+ - **Table styling** is consistent across all views
416
+ - **Test conventions** are documented in `TEST_CONVENTIONS.md` with clear guidance
417
+ - **Thread-safe config reset** with `SourceMonitor.reset_configuration!`
418
+ - **Active Storage guarding** with `if defined?(ActiveStorage)` checks
419
+ - **Strong params** via `Sources::Params.sanitize` with explicit allowlist
420
+ - **Pipeline architecture** — all service objects are justified multi-model orchestrators
421
+
422
+ ---
423
+
424
+ *Generated by `/rails-audit` command. Re-run to refresh findings.*
data/README.md CHANGED
@@ -9,8 +9,8 @@ SourceMonitor is a production-ready Rails 8 mountable engine for ingesting, norm
9
9
  In your host Rails app:
10
10
 
11
11
  ```bash
12
- bundle add source_monitor --version "~> 0.11.0"
13
- # or add `gem "source_monitor", "~> 0.11.0"` manually, then run:
12
+ bundle add source_monitor --version "~> 0.12.0"
13
+ # or add `gem "source_monitor", "~> 0.12.0"` manually, then run:
14
14
  bundle install
15
15
  ```
16
16
 
@@ -25,6 +25,9 @@ This exposes `bin/source_monitor` (via Bundler binstubs) so you can run the guid
25
25
  - Extensible scraper adapters (Readability included) with per-source settings and structured result metadata
26
26
  - Declarative configuration DSL covering queues, HTTP, retention, events, model extensions, authentication, and realtime transports
27
27
  - First-class observability through ActiveSupport notifications and `SourceMonitor::Metrics` counters/gauges
28
+ - ViewComponent UI primitives: `StatusBadgeComponent` and `IconComponent` for consistent badge and icon rendering in custom views
29
+ - Presenter layer: `SourceDetailsPresenter` and `SourcesFilterPresenter` for view-specific formatting without coupling controllers to display logic
30
+ - Shallow delegation service layer: five background jobs (ScrapeItemJob, DownloadContentImagesJob, FaviconFetchJob, SourceHealthCheckJob, ImportSessionHealthCheckJob) extracted to dedicated service classes, keeping job bodies to deserialization + delegation only
28
31
 
29
32
  ## Requirements
30
33
  - Ruby 4.0+ (we recommend [rbenv](https://github.com/rbenv/rbenv) for local development, but use whatever Ruby version manager suits your environment—asdf, chruby, rvm, or container-based workflows all work fine)
@@ -43,7 +46,7 @@ This exposes `bin/source_monitor` (via Bundler binstubs) so you can run the guid
43
46
  Before running any SourceMonitor commands inside your host app, add the gem and install dependencies:
44
47
 
45
48
  ```bash
46
- bundle add source_monitor --version "~> 0.11.0"
49
+ bundle add source_monitor --version "~> 0.12.0"
47
50
  # or edit your Gemfile, then run
48
51
  bundle install
49
52
  ```
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.11.1
1
+ 0.12.1
@@ -626,6 +626,10 @@ video {
626
626
  pointer-events: auto;
627
627
  }
628
628
 
629
+ .fm-admin .visible {
630
+ visibility: visible;
631
+ }
632
+
629
633
  .fm-admin .static {
630
634
  position: static;
631
635
  }
@@ -705,10 +709,6 @@ video {
705
709
  margin-bottom: 0.25rem;
706
710
  }
707
711
 
708
- .fm-admin .mb-1 {
709
- margin-bottom: 0.25rem;
710
- }
711
-
712
712
  .fm-admin .ml-0\.5 {
713
713
  margin-left: 0.125rem;
714
714
  }
@@ -717,14 +717,6 @@ video {
717
717
  margin-left: 0.5rem;
718
718
  }
719
719
 
720
- .fm-admin .ml-3 {
721
- margin-left: 0.75rem;
722
- }
723
-
724
- .fm-admin .ml-4 {
725
- margin-left: 1rem;
726
- }
727
-
728
720
  .fm-admin .mr-1 {
729
721
  margin-right: 0.25rem;
730
722
  }
@@ -809,10 +801,6 @@ video {
809
801
  height: 1rem;
810
802
  }
811
803
 
812
- .fm-admin .h-5 {
813
- height: 1.25rem;
814
- }
815
-
816
804
  .fm-admin .h-8 {
817
805
  height: 2rem;
818
806
  }
@@ -861,10 +849,6 @@ video {
861
849
  width: 10rem;
862
850
  }
863
851
 
864
- .fm-admin .w-5 {
865
- width: 1.25rem;
866
- }
867
-
868
852
  .fm-admin .w-8 {
869
853
  width: 2rem;
870
854
  }
@@ -1885,10 +1869,6 @@ video {
1885
1869
  opacity: 0;
1886
1870
  }
1887
1871
 
1888
- .fm-admin .opacity-25 {
1889
- opacity: 0.25;
1890
- }
1891
-
1892
1872
  .fm-admin .opacity-60 {
1893
1873
  opacity: 0.6;
1894
1874
  }