source_monitor 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/.rubocop.yml +12 -0
  4. data/.ruby-version +1 -0
  5. data/AGENTS.md +132 -0
  6. data/CHANGELOG.md +66 -0
  7. data/CONTRIBUTING.md +31 -0
  8. data/Gemfile +30 -0
  9. data/Gemfile.lock +411 -0
  10. data/MIT-LICENSE +20 -0
  11. data/README.md +108 -0
  12. data/Rakefile +8 -0
  13. data/app/assets/builds/.keep +0 -0
  14. data/app/assets/config/source_monitor_manifest.js +4 -0
  15. data/app/assets/images/source_monitor/.keep +0 -0
  16. data/app/assets/javascripts/source_monitor/application.js +20 -0
  17. data/app/assets/javascripts/source_monitor/controllers/async_submit_controller.js +36 -0
  18. data/app/assets/javascripts/source_monitor/controllers/dropdown_controller.js +109 -0
  19. data/app/assets/javascripts/source_monitor/controllers/modal_controller.js +56 -0
  20. data/app/assets/javascripts/source_monitor/controllers/notification_controller.js +53 -0
  21. data/app/assets/javascripts/source_monitor/turbo_actions.js +13 -0
  22. data/app/assets/stylesheets/source_monitor/application.tailwind.css +13 -0
  23. data/app/assets/svgs/source_monitor/.keep +0 -0
  24. data/app/controllers/concerns/.keep +0 -0
  25. data/app/controllers/concerns/source_monitor/sanitizes_search_params.rb +81 -0
  26. data/app/controllers/source_monitor/application_controller.rb +62 -0
  27. data/app/controllers/source_monitor/dashboard_controller.rb +27 -0
  28. data/app/controllers/source_monitor/fetch_logs_controller.rb +9 -0
  29. data/app/controllers/source_monitor/health_controller.rb +10 -0
  30. data/app/controllers/source_monitor/items_controller.rb +116 -0
  31. data/app/controllers/source_monitor/logs_controller.rb +15 -0
  32. data/app/controllers/source_monitor/scrape_logs_controller.rb +9 -0
  33. data/app/controllers/source_monitor/source_bulk_scrapes_controller.rb +35 -0
  34. data/app/controllers/source_monitor/source_fetches_controller.rb +22 -0
  35. data/app/controllers/source_monitor/source_health_checks_controller.rb +34 -0
  36. data/app/controllers/source_monitor/source_health_resets_controller.rb +27 -0
  37. data/app/controllers/source_monitor/source_retries_controller.rb +22 -0
  38. data/app/controllers/source_monitor/source_turbo_responses.rb +115 -0
  39. data/app/controllers/source_monitor/sources_controller.rb +179 -0
  40. data/app/helpers/source_monitor/application_helper.rb +327 -0
  41. data/app/jobs/source_monitor/application_job.rb +13 -0
  42. data/app/jobs/source_monitor/fetch_feed_job.rb +117 -0
  43. data/app/jobs/source_monitor/item_cleanup_job.rb +48 -0
  44. data/app/jobs/source_monitor/log_cleanup_job.rb +47 -0
  45. data/app/jobs/source_monitor/schedule_fetches_job.rb +29 -0
  46. data/app/jobs/source_monitor/scrape_item_job.rb +47 -0
  47. data/app/jobs/source_monitor/source_health_check_job.rb +77 -0
  48. data/app/mailers/source_monitor/application_mailer.rb +17 -0
  49. data/app/models/concerns/.keep +0 -0
  50. data/app/models/concerns/source_monitor/loggable.rb +18 -0
  51. data/app/models/source_monitor/application_record.rb +5 -0
  52. data/app/models/source_monitor/fetch_log.rb +31 -0
  53. data/app/models/source_monitor/health_check_log.rb +28 -0
  54. data/app/models/source_monitor/item.rb +102 -0
  55. data/app/models/source_monitor/item_content.rb +11 -0
  56. data/app/models/source_monitor/log_entry.rb +56 -0
  57. data/app/models/source_monitor/scrape_log.rb +31 -0
  58. data/app/models/source_monitor/source.rb +115 -0
  59. data/app/views/layouts/source_monitor/application.html.erb +54 -0
  60. data/app/views/source_monitor/dashboard/_fetch_schedule.html.erb +90 -0
  61. data/app/views/source_monitor/dashboard/_job_metrics.html.erb +82 -0
  62. data/app/views/source_monitor/dashboard/_recent_activity.html.erb +39 -0
  63. data/app/views/source_monitor/dashboard/_stat_card.html.erb +6 -0
  64. data/app/views/source_monitor/dashboard/_stats.html.erb +9 -0
  65. data/app/views/source_monitor/dashboard/index.html.erb +48 -0
  66. data/app/views/source_monitor/fetch_logs/show.html.erb +90 -0
  67. data/app/views/source_monitor/items/_details.html.erb +234 -0
  68. data/app/views/source_monitor/items/_details_wrapper.html.erb +3 -0
  69. data/app/views/source_monitor/items/index.html.erb +147 -0
  70. data/app/views/source_monitor/items/show.html.erb +3 -0
  71. data/app/views/source_monitor/logs/index.html.erb +208 -0
  72. data/app/views/source_monitor/scrape_logs/show.html.erb +73 -0
  73. data/app/views/source_monitor/shared/_toast.html.erb +34 -0
  74. data/app/views/source_monitor/sources/_bulk_scrape_form.html.erb +64 -0
  75. data/app/views/source_monitor/sources/_bulk_scrape_modal.html.erb +53 -0
  76. data/app/views/source_monitor/sources/_details.html.erb +302 -0
  77. data/app/views/source_monitor/sources/_details_wrapper.html.erb +3 -0
  78. data/app/views/source_monitor/sources/_empty_state_row.html.erb +5 -0
  79. data/app/views/source_monitor/sources/_fetch_interval_heatmap.html.erb +46 -0
  80. data/app/views/source_monitor/sources/_form.html.erb +143 -0
  81. data/app/views/source_monitor/sources/_health_status_badge.html.erb +46 -0
  82. data/app/views/source_monitor/sources/_row.html.erb +102 -0
  83. data/app/views/source_monitor/sources/edit.html.erb +28 -0
  84. data/app/views/source_monitor/sources/index.html.erb +153 -0
  85. data/app/views/source_monitor/sources/new.html.erb +22 -0
  86. data/app/views/source_monitor/sources/show.html.erb +3 -0
  87. data/config/coverage_baseline.json +2010 -0
  88. data/config/initializers/feedjira.rb +19 -0
  89. data/config/routes.rb +18 -0
  90. data/config/tailwind.config.js +17 -0
  91. data/db/migrate/20241008120000_create_source_monitor_sources.rb +40 -0
  92. data/db/migrate/20241008121000_create_source_monitor_items.rb +44 -0
  93. data/db/migrate/20241008122000_create_source_monitor_fetch_logs.rb +32 -0
  94. data/db/migrate/20241008123000_create_source_monitor_scrape_logs.rb +25 -0
  95. data/db/migrate/20251008183000_change_fetch_interval_to_minutes.rb +23 -0
  96. data/db/migrate/20251009090000_create_source_monitor_item_contents.rb +38 -0
  97. data/db/migrate/20251009103000_add_feed_content_readability_to_sources.rb +5 -0
  98. data/db/migrate/20251010090000_add_adaptive_fetching_toggle_to_sources.rb +7 -0
  99. data/db/migrate/20251010123000_add_deleted_at_to_source_monitor_items.rb +8 -0
  100. data/db/migrate/20251010153000_add_type_to_source_monitor_sources.rb +8 -0
  101. data/db/migrate/20251010154500_add_fetch_status_to_source_monitor_sources.rb +9 -0
  102. data/db/migrate/20251010160000_create_solid_cable_messages.rb +16 -0
  103. data/db/migrate/20251011090000_add_fetch_retry_state_to_sources.rb +14 -0
  104. data/db/migrate/20251012090000_add_health_fields_to_sources.rb +17 -0
  105. data/db/migrate/20251012100000_optimize_source_monitor_database_performance.rb +13 -0
  106. data/db/migrate/20251014064947_add_not_null_constraints_to_items.rb +30 -0
  107. data/db/migrate/20251014171659_add_performance_indexes.rb +29 -0
  108. data/db/migrate/20251014172525_add_fetch_status_check_constraint.rb +18 -0
  109. data/db/migrate/20251015100000_create_source_monitor_log_entries.rb +89 -0
  110. data/db/migrate/20251022100000_create_source_monitor_health_check_logs.rb +22 -0
  111. data/db/migrate/20251108120116_refresh_fetch_status_constraint.rb +29 -0
  112. data/docs/configuration.md +170 -0
  113. data/docs/deployment.md +63 -0
  114. data/docs/gh-cli-workflow.md +44 -0
  115. data/docs/installation.md +144 -0
  116. data/docs/troubleshooting.md +76 -0
  117. data/eslint.config.mjs +27 -0
  118. data/lib/generators/source_monitor/install/install_generator.rb +59 -0
  119. data/lib/generators/source_monitor/install/templates/source_monitor.rb.tt +155 -0
  120. data/lib/source_monitor/analytics/source_activity_rates.rb +53 -0
  121. data/lib/source_monitor/analytics/source_fetch_interval_distribution.rb +57 -0
  122. data/lib/source_monitor/analytics/sources_index_metrics.rb +92 -0
  123. data/lib/source_monitor/assets/bundler.rb +49 -0
  124. data/lib/source_monitor/assets.rb +6 -0
  125. data/lib/source_monitor/configuration.rb +654 -0
  126. data/lib/source_monitor/dashboard/queries.rb +356 -0
  127. data/lib/source_monitor/dashboard/quick_action.rb +7 -0
  128. data/lib/source_monitor/dashboard/quick_actions_presenter.rb +26 -0
  129. data/lib/source_monitor/dashboard/recent_activity.rb +30 -0
  130. data/lib/source_monitor/dashboard/recent_activity_presenter.rb +77 -0
  131. data/lib/source_monitor/dashboard/turbo_broadcaster.rb +87 -0
  132. data/lib/source_monitor/dashboard/upcoming_fetch_schedule.rb +126 -0
  133. data/lib/source_monitor/engine.rb +107 -0
  134. data/lib/source_monitor/events.rb +110 -0
  135. data/lib/source_monitor/feedjira_extensions.rb +103 -0
  136. data/lib/source_monitor/fetching/advisory_lock.rb +54 -0
  137. data/lib/source_monitor/fetching/completion/event_publisher.rb +22 -0
  138. data/lib/source_monitor/fetching/completion/follow_up_handler.rb +37 -0
  139. data/lib/source_monitor/fetching/completion/retention_handler.rb +30 -0
  140. data/lib/source_monitor/fetching/feed_fetcher.rb +627 -0
  141. data/lib/source_monitor/fetching/fetch_error.rb +88 -0
  142. data/lib/source_monitor/fetching/fetch_runner.rb +142 -0
  143. data/lib/source_monitor/fetching/retry_policy.rb +85 -0
  144. data/lib/source_monitor/fetching/stalled_fetch_reconciler.rb +146 -0
  145. data/lib/source_monitor/health/source_health_check.rb +100 -0
  146. data/lib/source_monitor/health/source_health_monitor.rb +210 -0
  147. data/lib/source_monitor/health/source_health_reset.rb +68 -0
  148. data/lib/source_monitor/health.rb +46 -0
  149. data/lib/source_monitor/http.rb +85 -0
  150. data/lib/source_monitor/instrumentation.rb +52 -0
  151. data/lib/source_monitor/items/item_creator.rb +601 -0
  152. data/lib/source_monitor/items/retention_pruner.rb +146 -0
  153. data/lib/source_monitor/items/retention_strategies/destroy.rb +26 -0
  154. data/lib/source_monitor/items/retention_strategies/soft_delete.rb +50 -0
  155. data/lib/source_monitor/items/retention_strategies.rb +9 -0
  156. data/lib/source_monitor/jobs/cleanup_options.rb +85 -0
  157. data/lib/source_monitor/jobs/fetch_failure_subscriber.rb +129 -0
  158. data/lib/source_monitor/jobs/solid_queue_metrics.rb +199 -0
  159. data/lib/source_monitor/jobs/visibility.rb +133 -0
  160. data/lib/source_monitor/logs/entry_sync.rb +69 -0
  161. data/lib/source_monitor/logs/filter_set.rb +163 -0
  162. data/lib/source_monitor/logs/query.rb +81 -0
  163. data/lib/source_monitor/logs/table_presenter.rb +161 -0
  164. data/lib/source_monitor/metrics.rb +77 -0
  165. data/lib/source_monitor/model_extensions.rb +109 -0
  166. data/lib/source_monitor/models/sanitizable.rb +76 -0
  167. data/lib/source_monitor/models/url_normalizable.rb +84 -0
  168. data/lib/source_monitor/pagination/paginator.rb +90 -0
  169. data/lib/source_monitor/realtime/adapter.rb +97 -0
  170. data/lib/source_monitor/realtime/broadcaster.rb +237 -0
  171. data/lib/source_monitor/realtime.rb +17 -0
  172. data/lib/source_monitor/release/changelog.rb +59 -0
  173. data/lib/source_monitor/release/runner.rb +73 -0
  174. data/lib/source_monitor/scheduler.rb +82 -0
  175. data/lib/source_monitor/scrapers/base.rb +105 -0
  176. data/lib/source_monitor/scrapers/fetchers/http_fetcher.rb +97 -0
  177. data/lib/source_monitor/scrapers/parsers/readability_parser.rb +101 -0
  178. data/lib/source_monitor/scrapers/readability.rb +156 -0
  179. data/lib/source_monitor/scraping/bulk_result_presenter.rb +85 -0
  180. data/lib/source_monitor/scraping/bulk_source_scraper.rb +233 -0
  181. data/lib/source_monitor/scraping/enqueuer.rb +125 -0
  182. data/lib/source_monitor/scraping/item_scraper/adapter_resolver.rb +44 -0
  183. data/lib/source_monitor/scraping/item_scraper/persistence.rb +189 -0
  184. data/lib/source_monitor/scraping/item_scraper.rb +84 -0
  185. data/lib/source_monitor/scraping/scheduler.rb +43 -0
  186. data/lib/source_monitor/scraping/state.rb +79 -0
  187. data/lib/source_monitor/security/authentication.rb +85 -0
  188. data/lib/source_monitor/security/parameter_sanitizer.rb +42 -0
  189. data/lib/source_monitor/sources/turbo_stream_presenter.rb +54 -0
  190. data/lib/source_monitor/turbo_streams/stream_responder.rb +95 -0
  191. data/lib/source_monitor/version.rb +3 -0
  192. data/lib/source_monitor.rb +149 -0
  193. data/lib/tasks/recover_stalled_fetches.rake +16 -0
  194. data/lib/tasks/source_monitor_assets.rake +28 -0
  195. data/lib/tasks/source_monitor_tasks.rake +29 -0
  196. data/lib/tasks/test_smoke.rake +12 -0
  197. data/package-lock.json +3997 -0
  198. data/package.json +29 -0
  199. data/postcss.config.js +6 -0
  200. data/source_monitor.gemspec +46 -0
  201. data/stylelint.config.js +12 -0
  202. metadata +469 -0
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "source_monitor/models/sanitizable"
4
+ require "source_monitor/models/url_normalizable"
5
+
6
+ module SourceMonitor
7
+ class Source < ApplicationRecord
8
+ include SourceMonitor::Models::Sanitizable
9
+ include SourceMonitor::Models::UrlNormalizable
10
+
11
+ FETCH_STATUS_VALUES = %w[idle queued fetching failed].freeze
12
+
13
+ has_many :all_items, class_name: "SourceMonitor::Item", inverse_of: :source, dependent: :destroy
14
+ has_many :items, -> { active }, class_name: "SourceMonitor::Item", inverse_of: :source
15
+ has_many :fetch_logs, class_name: "SourceMonitor::FetchLog", inverse_of: :source, dependent: :destroy
16
+ has_many :scrape_logs, class_name: "SourceMonitor::ScrapeLog", inverse_of: :source, dependent: :destroy
17
+ has_many :health_check_logs, class_name: "SourceMonitor::HealthCheckLog", inverse_of: :source, dependent: :destroy
18
+ has_many :log_entries, class_name: "SourceMonitor::LogEntry", inverse_of: :source, dependent: :destroy
19
+
20
+ # Scopes for common source states
21
+ scope :active, -> { where(active: true) }
22
+ scope :failed, lambda {
23
+ failure = arel_table[:failure_count].gt(0)
24
+ error_present = arel_table[:last_error].not_eq(nil)
25
+ error_time_present = arel_table[:last_error_at].not_eq(nil)
26
+ where(failure.or(error_present).or(error_time_present))
27
+ }
28
+ scope :healthy, -> { active.where(failure_count: 0, last_error: nil, last_error_at: nil) }
29
+
30
+ # Use Rails attribute API for default values instead of after_initialize callbacks
31
+ attribute :scrape_settings, default: -> { {} }
32
+ attribute :custom_headers, default: -> { {} }
33
+ attribute :metadata, default: -> { {} }
34
+ attribute :fetch_status, :string, default: "idle"
35
+ attribute :health_status, :string, default: "healthy"
36
+
37
+ sanitizes_string_attributes :name, :feed_url, :website_url, :scraper_adapter
38
+ sanitizes_hash_attributes :scrape_settings, :custom_headers, :metadata
39
+ normalizes_urls :feed_url, :website_url
40
+ validates_url_format :feed_url, :website_url
41
+
42
+ validates :name, presence: true
43
+ validates :feed_url, presence: true, uniqueness: { case_sensitive: false }
44
+ validates :fetch_interval_minutes, numericality: { greater_than: 0 }
45
+ validates :scraper_adapter, presence: true
46
+ validates :items_retention_days, numericality: { allow_nil: true, only_integer: true, greater_than_or_equal_to: 0 }
47
+ validates :max_items, numericality: { allow_nil: true, only_integer: true, greater_than_or_equal_to: 0 }
48
+ validates :fetch_status, inclusion: { in: FETCH_STATUS_VALUES }
49
+ validates :fetch_retry_attempt, numericality: { greater_than_or_equal_to: 0, only_integer: true }
50
+
51
+ validate :health_auto_pause_threshold_within_bounds
52
+
53
+ SourceMonitor::ModelExtensions.register(self, :source)
54
+
55
+ class << self
56
+ # Convert scope to class method to make reference_time parameter explicit
57
+ # Scopes with internal variables should be class methods per Rails best practices
58
+ def due_for_fetch(reference_time: Time.current)
59
+ active.where(arel_table[:next_fetch_at].eq(nil).or(arel_table[:next_fetch_at].lteq(reference_time)))
60
+ end
61
+
62
+ def ransackable_attributes(_auth_object = nil)
63
+ %w[name feed_url website_url created_at fetch_interval_minutes items_count last_fetched_at]
64
+ end
65
+
66
+ def ransackable_associations(_auth_object = nil)
67
+ []
68
+ end
69
+ end
70
+
71
+ def fetch_interval_minutes=(value)
72
+ self[:fetch_interval_minutes] = value.presence && value.to_i
73
+ end
74
+
75
+ def fetch_interval_hours=(value)
76
+ self.fetch_interval_minutes = (value.to_f * 60).round if value.present?
77
+ end
78
+
79
+ def fetch_interval_hours
80
+ return 0 unless fetch_interval_minutes
81
+
82
+ fetch_interval_minutes.to_f / 60.0
83
+ end
84
+
85
+ def fetch_circuit_open?
86
+ fetch_circuit_until.present? && fetch_circuit_until.future?
87
+ end
88
+
89
+ def fetch_retry_attempt
90
+ value = super
91
+ value.present? ? value : 0
92
+ end
93
+
94
+ def auto_paused?
95
+ auto_paused_until.present? && auto_paused_until.future?
96
+ end
97
+
98
+ def reset_items_counter!
99
+ # Recalculate items_count from actual active (non-deleted) items
100
+ actual_count = items.count
101
+ update_columns(items_count: actual_count)
102
+ end
103
+
104
+ private
105
+
106
+ def health_auto_pause_threshold_within_bounds
107
+ return if health_auto_pause_threshold.nil?
108
+
109
+ value = health_auto_pause_threshold.to_f
110
+ return if value >= 0 && value <= 1
111
+
112
+ errors.add(:health_auto_pause_threshold, "must be between 0 and 1")
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,54 @@
1
+ <!DOCTYPE html>
2
+ <html>
3
+ <head>
4
+ <title><%= content_for?(:title) ? yield(:title) : "SourceMonitor" %></title>
5
+ <%= csrf_meta_tags %>
6
+ <%= csp_meta_tag %>
7
+
8
+ <%= yield :head %>
9
+
10
+ <%= source_monitor_stylesheet_bundle_tag %>
11
+ <%= turbo_include_tags %>
12
+ <%= source_monitor_javascript_bundle_tag %>
13
+ </head>
14
+ <body class="fm-admin">
15
+ <%= turbo_stream_from "source_monitor_notifications" %>
16
+ <div class="pointer-events-none fixed inset-x-0 top-4 z-50 flex justify-end px-6">
17
+ <div id="source_monitor_notifications" class="flex w-full max-w-sm flex-col gap-3"></div>
18
+ </div>
19
+ <div id="source_monitor_redirects" class="hidden" aria-hidden="true"></div>
20
+ <div class="flex min-h-screen flex-col">
21
+ <header class="bg-slate-800 text-white shadow">
22
+ <div class="mx-auto flex w-full max-w-6xl lg:max-w-7xl 2xl:max-w-[110rem] items-center justify-between px-6 py-4">
23
+ <div class="text-lg font-semibold tracking-tight">SourceMonitor</div>
24
+ <% nav_items = [
25
+ ["Dashboard", source_monitor.dashboard_path],
26
+ ["Sources", source_monitor.sources_path],
27
+ ["Items", source_monitor.items_path],
28
+ ["Logs", source_monitor.logs_path]
29
+ ] %>
30
+ <nav class="flex gap-2">
31
+ <% nav_items.each do |label, path| %>
32
+ <% active = current_page?(path) %>
33
+ <%= link_to label, path, class: [
34
+ "inline-flex items-center rounded-md px-3 py-2 text-sm font-medium transition-colors",
35
+ active ? "bg-slate-900 text-white shadow" : "text-slate-200 hover:bg-slate-700 hover:text-white"
36
+ ].join(" ") %>
37
+ <% end %>
38
+ </nav>
39
+ </div>
40
+ </header>
41
+
42
+ <main class="mx-auto flex w-full max-w-6xl lg:max-w-7xl 2xl:max-w-[110rem] flex-1 flex-col px-6 py-8">
43
+ <%= yield %>
44
+ </main>
45
+
46
+ <footer class="bg-slate-900 py-4">
47
+ <div class="mx-auto flex w-full max-w-6xl lg:max-w-7xl 2xl:max-w-[110rem] justify-between px-6 text-xs text-slate-300">
48
+ <span>&copy; <%= Time.current.year %> SourceMonitor</span>
49
+ <span>v<%= SourceMonitor::VERSION %></span>
50
+ </div>
51
+ </footer>
52
+ </div>
53
+ </body>
54
+ </html>
@@ -0,0 +1,90 @@
1
+ <div id="source_monitor_dashboard_fetch_schedule" class="space-y-4">
2
+ <div>
3
+ <h2 class="text-lg font-medium text-slate-900">Upcoming Fetch Schedule</h2>
4
+ <p class="mt-1 text-xs text-slate-500">
5
+ Windows are relative to <%= reference_time.present? ? format_schedule_time(reference_time) : "the current time" %> and update automatically as jobs finish.
6
+ </p>
7
+ </div>
8
+
9
+ <% if groups.present? %>
10
+ <% groups.each do |group| %>
11
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
12
+ <div class="flex items-center justify-between gap-3 border-b border-slate-200 px-5 py-4">
13
+ <div>
14
+ <h3 class="text-sm font-semibold text-slate-900"><%= group.label %></h3>
15
+ <% if (window_label = fetch_schedule_window_label(group)).present? %>
16
+ <p class="mt-1 text-xs text-slate-500"><%= window_label %></p>
17
+ <% elsif group.include_unscheduled %>
18
+ <p class="mt-1 text-xs text-slate-500">Includes sources scheduled beyond four hours or without a next fetch timestamp.</p>
19
+ <% end %>
20
+ </div>
21
+ <span class="inline-flex items-center rounded-full bg-slate-100 px-3 py-1 text-xs font-semibold text-slate-600">
22
+ <%= pluralize(group.sources.size, "source") %>
23
+ </span>
24
+ </div>
25
+
26
+ <% if group.sources.any? %>
27
+ <div class="overflow-x-auto px-5 py-4">
28
+ <table class="min-w-full divide-y divide-slate-200 text-left text-sm">
29
+ <thead class="text-xs font-semibold uppercase tracking-wide text-slate-500">
30
+ <tr>
31
+ <th scope="col" class="px-4 py-2">Source</th>
32
+ <th scope="col" class="px-4 py-2">Status</th>
33
+ <th scope="col" class="px-4 py-2">Next Fetch</th>
34
+ <th scope="col" class="px-4 py-2">Interval</th>
35
+ </tr>
36
+ </thead>
37
+ <tbody class="divide-y divide-slate-100 text-slate-700">
38
+ <% group.sources.each do |source| %>
39
+ <% badge = async_status_badge(source.fetch_status) %>
40
+ <tr>
41
+ <td class="px-4 py-3">
42
+ <div class="flex flex-col">
43
+ <%= link_to source.name, source_monitor.source_path(source), class: "text-sm font-semibold text-blue-600 hover:text-blue-500" %>
44
+ <span class="text-xs text-slate-400"><%= source.feed_url %></span>
45
+ </div>
46
+ </td>
47
+ <td class="px-4 py-3">
48
+ <span class="inline-flex items-center gap-1 rounded-full px-3 py-1 text-xs font-semibold <%= badge[:classes] %>">
49
+ <% if badge[:show_spinner] %>
50
+ <%= loading_spinner_svg(css_class: "h-3 w-3 animate-spin text-current") %>
51
+ <% end %>
52
+ <%= badge[:label] %>
53
+ </span>
54
+ </td>
55
+ <td class="px-4 py-3">
56
+ <% if source.next_fetch_at.present? %>
57
+ <div class="text-sm font-medium text-slate-900"><%= format_schedule_time(source.next_fetch_at) %></div>
58
+ <% if source.next_fetch_at.future? %>
59
+ <div class="text-xs text-slate-400">in <%= distance_of_time_in_words(reference_time, source.next_fetch_at) %></div>
60
+ <% else %>
61
+ <div class="text-xs text-rose-500"><%= distance_of_time_in_words(source.next_fetch_at, reference_time) %> overdue</div>
62
+ <% end %>
63
+ <% else %>
64
+ <div class="text-sm font-medium text-slate-500">Not scheduled</div>
65
+ <% end %>
66
+ </td>
67
+ <td class="px-4 py-3">
68
+ <div class="text-sm font-medium text-slate-900"><%= human_fetch_interval(source.fetch_interval_minutes) %></div>
69
+ <% if source.respond_to?(:adaptive_fetching_enabled?) && source.adaptive_fetching_enabled? %>
70
+ <div class="text-xs text-slate-400">Adaptive</div>
71
+ <% else %>
72
+ <div class="text-xs text-slate-400">Fixed</div>
73
+ <% end %>
74
+ </td>
75
+ </tr>
76
+ <% end %>
77
+ </tbody>
78
+ </table>
79
+ </div>
80
+ <% else %>
81
+ <div class="px-5 py-6 text-sm text-slate-500">No sources scheduled in this window.</div>
82
+ <% end %>
83
+ </div>
84
+ <% end %>
85
+ <% else %>
86
+ <div class="rounded-lg border border-dashed border-slate-200 bg-white px-5 py-6 text-sm text-slate-500 shadow-sm">
87
+ No upcoming fetches scheduled. Activate sources or trigger a fetch to populate the schedule.
88
+ </div>
89
+ <% end %>
90
+ </div>
@@ -0,0 +1,82 @@
1
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
2
+ <div class="border-b border-slate-200 px-5 py-4">
3
+ <h2 class="text-lg font-medium">Job Queues</h2>
4
+ <p class="mt-1 text-xs text-slate-500">ActiveJob adapter: <span class="font-semibold text-slate-700"><%= adapter.presence || "unknown" %></span></p>
5
+ </div>
6
+
7
+ <div class="divide-y divide-slate-100">
8
+ <% available_metrics = metrics.select { |metric| metric[:summary].available } %>
9
+ <% if available_metrics.any? %>
10
+ <% metrics.each do |metric| %>
11
+ <% summary = metric[:summary] %>
12
+ <div class="px-5 py-4">
13
+ <div class="flex items-center justify-between">
14
+ <div>
15
+ <p class="text-sm font-medium text-slate-900"><%= metric[:role].to_s.titleize %> queue</p>
16
+ <p class="text-xs text-slate-500"><%= metric[:queue_name] %></p>
17
+ </div>
18
+ <div class="flex items-center gap-2">
19
+ <% unless summary.total_count.zero? %>
20
+ <span class="inline-flex items-center rounded-full bg-slate-100 px-3 py-1 text-xs font-semibold text-slate-600">Total: <%= summary.total_count %></span>
21
+ <% end %>
22
+ <% if summary.paused %>
23
+ <span class="inline-flex items-center rounded-full bg-amber-100 px-3 py-1 text-xs font-semibold text-amber-600">Paused</span>
24
+ <% end %>
25
+ </div>
26
+ </div>
27
+
28
+ <% if summary.available %>
29
+ <dl class="mt-3 grid gap-2 text-xs text-slate-500">
30
+ <div class="flex justify-between">
31
+ <dt>Ready</dt>
32
+ <dd><%= summary.ready_count %></dd>
33
+ </div>
34
+ <div class="flex justify-between">
35
+ <dt>Scheduled</dt>
36
+ <dd><%= summary.scheduled_count %></dd>
37
+ </div>
38
+ <div class="flex justify-between">
39
+ <dt>Failed</dt>
40
+ <dd><%= summary.failed_count %></dd>
41
+ </div>
42
+ <div class="flex justify-between">
43
+ <dt>Recurring Tasks</dt>
44
+ <dd><%= summary.recurring_count %></dd>
45
+ </div>
46
+ <div class="flex justify-between">
47
+ <dt>Last enqueued</dt>
48
+ <dd><%= summary.last_enqueued_at.present? ? "#{time_ago_in_words(summary.last_enqueued_at)} ago" : "Never" %></dd>
49
+ </div>
50
+ <div class="flex justify-between">
51
+ <dt>Last started</dt>
52
+ <dd><%= summary.last_started_at.present? ? "#{time_ago_in_words(summary.last_started_at)} ago" : "Never" %></dd>
53
+ </div>
54
+ <div class="flex justify-between">
55
+ <dt>Last finished</dt>
56
+ <dd><%= summary.last_finished_at.present? ? "#{time_ago_in_words(summary.last_finished_at)} ago" : "Never" %></dd>
57
+ </div>
58
+ </dl>
59
+ <% if summary.total_count.zero? && summary.recurring_count.zero? %>
60
+ <p class="mt-3 text-xs text-slate-500">No jobs queued for this role yet.</p>
61
+ <% end %>
62
+ <% else %>
63
+ <p class="mt-3 text-xs text-slate-500">Solid Queue metrics are unavailable. Ensure Solid Queue migrations have been installed.</p>
64
+ <% end %>
65
+ </div>
66
+ <% end %>
67
+ <% else %>
68
+ <div class="px-5 py-6 text-sm text-slate-500">Solid Queue metrics are unavailable. Confirm Solid Queue is installed and migrations have been run.</div>
69
+ <% end %>
70
+ </div>
71
+ <% if mission_control_enabled %>
72
+ <div class="border-t border-slate-200 bg-slate-50 px-5 py-4 text-xs text-slate-600">
73
+ <p class="font-semibold text-slate-700">Mission Control</p>
74
+ <% if mission_control_path.present? %>
75
+ <p class="mt-1">View detailed queue metrics in Mission Control.</p>
76
+ <%= link_to "Open Mission Control", mission_control_path, class: "mt-2 inline-flex items-center rounded-md bg-blue-600 px-3 py-1 text-xs font-semibold text-white shadow hover:bg-blue-500" %>
77
+ <% else %>
78
+ <p class="mt-1">Mission Control integration is enabled. Configure <code>SourceMonitor.config.mission_control_dashboard_path</code> with a host route to show a direct link.</p>
79
+ <% end %>
80
+ </div>
81
+ <% end %>
82
+ </div>
@@ -0,0 +1,39 @@
1
+ <div id="source_monitor_dashboard_recent_activity" class="rounded-lg border border-slate-200 bg-white shadow-sm">
2
+ <div class="border-b border-slate-200 px-5 py-4">
3
+ <h2 class="text-lg font-medium">Recent Activity</h2>
4
+ <p class="mt-1 text-xs text-slate-500">Latest fetch, scrape, and item events across sources.</p>
5
+ </div>
6
+ <div class="divide-y divide-slate-100">
7
+ <% if recent_activity.any? %>
8
+ <% recent_activity.each do |event| %>
9
+ <div class="flex items-center justify-between px-5 py-4">
10
+ <div>
11
+ <div class="flex items-center gap-2">
12
+ <div class="text-sm font-medium text-slate-900">
13
+ <% if event[:path].present? %>
14
+ <%= link_to event[:label], event[:path], data: { turbo_frame: "_top" }, class: "text-slate-900 hover:text-blue-600" %>
15
+ <% else %>
16
+ <%= event[:label] %>
17
+ <% end %>
18
+ </div>
19
+ <span class="inline-flex items-center rounded-full bg-slate-100 px-2 py-0.5 text-[11px] font-semibold uppercase tracking-wide text-slate-600"><%= event[:type].to_s.humanize %></span>
20
+ </div>
21
+ <div class="mt-1 text-xs text-slate-500">
22
+ <%= event[:description].presence || "No additional details recorded." %>
23
+ </div>
24
+ </div>
25
+ <div class="text-right">
26
+ <span class="inline-flex items-center rounded-full px-3 py-1 text-xs font-semibold <%= event[:status] == :success ? "bg-green-100 text-green-700" : "bg-rose-100 text-rose-700" %>">
27
+ <%= event[:status] == :success ? "Success" : "Failure" %>
28
+ </span>
29
+ <div class="mt-1 text-xs text-slate-400">
30
+ <%= event[:time]&.strftime("%b %d, %H:%M") || "Unknown" %>
31
+ </div>
32
+ </div>
33
+ </div>
34
+ <% end %>
35
+ <% else %>
36
+ <div class="px-5 py-6 text-sm text-slate-500">No recent activity yet. Fetches, scrapes, and new items will appear here.</div>
37
+ <% end %>
38
+ </div>
39
+ </div>
@@ -0,0 +1,6 @@
1
+ <% value = (stat_card[:value] || 0) %>
2
+ <div class="rounded-lg border border-slate-200 bg-white p-5 shadow-sm">
3
+ <dt class="text-xs font-medium uppercase tracking-wide text-slate-500"><%= stat_card[:label] %></dt>
4
+ <dd class="mt-2 text-3xl font-semibold text-slate-900"><%= value %></dd>
5
+ <p class="mt-1 text-xs text-slate-500"><%= stat_card[:caption] %></p>
6
+ </div>
@@ -0,0 +1,9 @@
1
+ <div id="source_monitor_dashboard_stats" class="grid gap-5 sm:grid-cols-2 xl:grid-cols-5">
2
+ <%= render partial: "stat_card", collection: [
3
+ { label: "Sources", value: stats[:total_sources], caption: "Total registered" },
4
+ { label: "Active", value: stats[:active_sources], caption: "Fetching on schedule" },
5
+ { label: "Failures", value: stats[:failed_sources], caption: "Require attention" },
6
+ { label: "Items", value: stats[:total_items], caption: "Stored entries" },
7
+ { label: "Fetches Today", value: stats[:fetches_today], caption: "Completed runs" }
8
+ ], locals: { stats: stats } %>
9
+ </div>
@@ -0,0 +1,48 @@
1
+ <%= content_for :title, "Dashboard · SourceMonitor" %>
2
+ <%= turbo_stream_from SourceMonitor::Dashboard::TurboBroadcaster::STREAM_NAME %>
3
+
4
+ <div class="space-y-8">
5
+ <section>
6
+ <h1 class="text-2xl font-semibold text-slate-900">Overview</h1>
7
+ <p class="mt-1 text-sm text-slate-600">Current status of feed ingestion and scraping.</p>
8
+
9
+ <div class="mt-6">
10
+ <%= render "stats", stats: @stats %>
11
+ </div>
12
+ </section>
13
+
14
+ <section class="grid gap-6 lg:grid-cols-3">
15
+ <div class="lg:col-span-2 space-y-6">
16
+ <%= render "recent_activity", recent_activity: @recent_activity %>
17
+ <%= render "fetch_schedule", groups: @fetch_schedule_groups, reference_time: @fetch_schedule_reference_time %>
18
+ </div>
19
+
20
+ <div class="space-y-6">
21
+ <%= render "job_metrics",
22
+ metrics: @job_metrics,
23
+ adapter: @job_adapter,
24
+ mission_control_enabled: @mission_control_enabled,
25
+ mission_control_path: @mission_control_dashboard_path %>
26
+
27
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
28
+ <div class="border-b border-slate-200 px-5 py-4">
29
+ <h2 class="text-lg font-medium">Quick Actions</h2>
30
+ <p class="mt-1 text-xs text-slate-500">Jump to common workflows.</p>
31
+ </div>
32
+ <ul class="divide-y divide-slate-100">
33
+ <% @quick_actions.each do |action| %>
34
+ <li class="px-5 py-4">
35
+ <div class="flex items-center justify-between">
36
+ <div>
37
+ <div class="text-sm font-medium text-slate-900"><%= action[:label] %></div>
38
+ <div class="text-xs text-slate-500"><%= action[:description] %></div>
39
+ </div>
40
+ <%= link_to "Go", action[:path], class: "inline-flex items-center rounded-md bg-blue-600 px-3 py-1 text-xs font-semibold text-white shadow hover:bg-blue-500" %>
41
+ </div>
42
+ </li>
43
+ <% end %>
44
+ </ul>
45
+ </div>
46
+ </div>
47
+ </section>
48
+ </div>
@@ -0,0 +1,90 @@
1
+ <div class="space-y-6">
2
+ <div class="flex flex-wrap items-start justify-between gap-4">
3
+ <div>
4
+ <h1 class="text-3xl font-semibold text-slate-900">Fetch Log</h1>
5
+ <p class="mt-1 text-sm text-slate-500">
6
+ Started <%= @log.started_at&.strftime("%b %d, %Y %H:%M:%S %Z") || "Unknown" %> ·
7
+ <%= @log.success? ? "Success" : "Failure" %>
8
+ </p>
9
+ </div>
10
+ <div class="flex gap-3">
11
+ <%= link_to "Back to Logs", source_monitor.logs_path(log_type: "fetch"), class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50" %>
12
+ <%= link_to "View Source", source_monitor.source_path(@log.source), class: "inline-flex items-center rounded-md bg-slate-800 px-3 py-2 text-sm font-medium text-white hover:bg-slate-700" %>
13
+ </div>
14
+ </div>
15
+
16
+ <div class="grid gap-6 lg:grid-cols-2">
17
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
18
+ <div class="border-b border-slate-200 px-5 py-4">
19
+ <h2 class="text-lg font-medium">Summary</h2>
20
+ </div>
21
+ <dl class="divide-y divide-slate-100">
22
+ <% summary = {
23
+ "Source" => link_to(@log.source.name, source_monitor.source_path(@log.source)),
24
+ "HTTP Status" => @log.http_status || "—",
25
+ "Success" => @log.success? ? "Yes" : "No",
26
+ "Duration (ms)" => @log.duration_ms || "—",
27
+ "Items Created" => @log.items_created,
28
+ "Items Updated" => @log.items_updated,
29
+ "Items Failed" => @log.items_failed,
30
+ "Started At" => @log.started_at&.strftime("%b %d, %Y %H:%M:%S %Z") || "—",
31
+ "Completed At" => @log.completed_at&.strftime("%b %d, %Y %H:%M:%S %Z") || "—",
32
+ "Job ID" => @log.job_id || "—"
33
+ } %>
34
+ <% summary.each do |label, value| %>
35
+ <div class="flex items-center justify-between px-5 py-3">
36
+ <dt class="text-sm font-medium text-slate-600"><%= label %></dt>
37
+ <dd class="text-sm text-slate-900 text-right"><%= value %></dd>
38
+ </div>
39
+ <% end %>
40
+ </dl>
41
+ </div>
42
+
43
+ <div class="space-y-6">
44
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
45
+ <div class="border-b border-slate-200 px-5 py-4">
46
+ <h2 class="text-lg font-medium">Error Details</h2>
47
+ </div>
48
+ <div class="px-5 py-4 text-sm text-slate-700">
49
+ <% if @log.success? %>
50
+ <p class="text-slate-500">No errors recorded.</p>
51
+ <% else %>
52
+ <% if @log.error_class.present? %>
53
+ <p class="font-medium text-rose-600"><%= @log.error_class %></p>
54
+ <% end %>
55
+ <p class="mt-2 whitespace-pre-wrap break-words text-xs text-slate-600"><%= @log.error_message.presence || "No error message captured." %></p>
56
+ <% if @log.error_backtrace.present? %>
57
+ <pre class="mt-3 whitespace-pre-wrap break-words text-[11px] leading-relaxed text-slate-600"><%= @log.error_backtrace %></pre>
58
+ <% end %>
59
+ <% end %>
60
+ </div>
61
+ </div>
62
+
63
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
64
+ <div class="border-b border-slate-200 px-5 py-4">
65
+ <h2 class="text-lg font-medium">HTTP Headers</h2>
66
+ </div>
67
+ <div class="px-5 py-4 text-sm text-slate-700">
68
+ <% if @log.http_response_headers.present? %>
69
+ <pre class="whitespace-pre-wrap break-words text-xs text-slate-600"><%= JSON.pretty_generate(@log.http_response_headers) %></pre>
70
+ <% else %>
71
+ <p class="text-slate-500">No response headers recorded.</p>
72
+ <% end %>
73
+ </div>
74
+ </div>
75
+
76
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
77
+ <div class="border-b border-slate-200 px-5 py-4">
78
+ <h2 class="text-lg font-medium">Metadata</h2>
79
+ </div>
80
+ <div class="px-5 py-4 text-sm text-slate-700">
81
+ <% if @log.metadata.present? %>
82
+ <pre class="whitespace-pre-wrap break-words text-xs text-slate-600"><%= JSON.pretty_generate(@log.metadata) %></pre>
83
+ <% else %>
84
+ <p class="text-slate-500">No metadata recorded.</p>
85
+ <% end %>
86
+ </div>
87
+ </div>
88
+ </div>
89
+ </div>
90
+ </div>