source_monitor 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +16 -0
  3. data/.rubocop.yml +12 -0
  4. data/.ruby-version +1 -0
  5. data/AGENTS.md +132 -0
  6. data/CHANGELOG.md +66 -0
  7. data/CONTRIBUTING.md +31 -0
  8. data/Gemfile +30 -0
  9. data/Gemfile.lock +411 -0
  10. data/MIT-LICENSE +20 -0
  11. data/README.md +108 -0
  12. data/Rakefile +8 -0
  13. data/app/assets/builds/.keep +0 -0
  14. data/app/assets/config/source_monitor_manifest.js +4 -0
  15. data/app/assets/images/source_monitor/.keep +0 -0
  16. data/app/assets/javascripts/source_monitor/application.js +20 -0
  17. data/app/assets/javascripts/source_monitor/controllers/async_submit_controller.js +36 -0
  18. data/app/assets/javascripts/source_monitor/controllers/dropdown_controller.js +109 -0
  19. data/app/assets/javascripts/source_monitor/controllers/modal_controller.js +56 -0
  20. data/app/assets/javascripts/source_monitor/controllers/notification_controller.js +53 -0
  21. data/app/assets/javascripts/source_monitor/turbo_actions.js +13 -0
  22. data/app/assets/stylesheets/source_monitor/application.tailwind.css +13 -0
  23. data/app/assets/svgs/source_monitor/.keep +0 -0
  24. data/app/controllers/concerns/.keep +0 -0
  25. data/app/controllers/concerns/source_monitor/sanitizes_search_params.rb +81 -0
  26. data/app/controllers/source_monitor/application_controller.rb +62 -0
  27. data/app/controllers/source_monitor/dashboard_controller.rb +27 -0
  28. data/app/controllers/source_monitor/fetch_logs_controller.rb +9 -0
  29. data/app/controllers/source_monitor/health_controller.rb +10 -0
  30. data/app/controllers/source_monitor/items_controller.rb +116 -0
  31. data/app/controllers/source_monitor/logs_controller.rb +15 -0
  32. data/app/controllers/source_monitor/scrape_logs_controller.rb +9 -0
  33. data/app/controllers/source_monitor/source_bulk_scrapes_controller.rb +35 -0
  34. data/app/controllers/source_monitor/source_fetches_controller.rb +22 -0
  35. data/app/controllers/source_monitor/source_health_checks_controller.rb +34 -0
  36. data/app/controllers/source_monitor/source_health_resets_controller.rb +27 -0
  37. data/app/controllers/source_monitor/source_retries_controller.rb +22 -0
  38. data/app/controllers/source_monitor/source_turbo_responses.rb +115 -0
  39. data/app/controllers/source_monitor/sources_controller.rb +179 -0
  40. data/app/helpers/source_monitor/application_helper.rb +327 -0
  41. data/app/jobs/source_monitor/application_job.rb +13 -0
  42. data/app/jobs/source_monitor/fetch_feed_job.rb +117 -0
  43. data/app/jobs/source_monitor/item_cleanup_job.rb +48 -0
  44. data/app/jobs/source_monitor/log_cleanup_job.rb +47 -0
  45. data/app/jobs/source_monitor/schedule_fetches_job.rb +29 -0
  46. data/app/jobs/source_monitor/scrape_item_job.rb +47 -0
  47. data/app/jobs/source_monitor/source_health_check_job.rb +77 -0
  48. data/app/mailers/source_monitor/application_mailer.rb +17 -0
  49. data/app/models/concerns/.keep +0 -0
  50. data/app/models/concerns/source_monitor/loggable.rb +18 -0
  51. data/app/models/source_monitor/application_record.rb +5 -0
  52. data/app/models/source_monitor/fetch_log.rb +31 -0
  53. data/app/models/source_monitor/health_check_log.rb +28 -0
  54. data/app/models/source_monitor/item.rb +102 -0
  55. data/app/models/source_monitor/item_content.rb +11 -0
  56. data/app/models/source_monitor/log_entry.rb +56 -0
  57. data/app/models/source_monitor/scrape_log.rb +31 -0
  58. data/app/models/source_monitor/source.rb +115 -0
  59. data/app/views/layouts/source_monitor/application.html.erb +54 -0
  60. data/app/views/source_monitor/dashboard/_fetch_schedule.html.erb +90 -0
  61. data/app/views/source_monitor/dashboard/_job_metrics.html.erb +82 -0
  62. data/app/views/source_monitor/dashboard/_recent_activity.html.erb +39 -0
  63. data/app/views/source_monitor/dashboard/_stat_card.html.erb +6 -0
  64. data/app/views/source_monitor/dashboard/_stats.html.erb +9 -0
  65. data/app/views/source_monitor/dashboard/index.html.erb +48 -0
  66. data/app/views/source_monitor/fetch_logs/show.html.erb +90 -0
  67. data/app/views/source_monitor/items/_details.html.erb +234 -0
  68. data/app/views/source_monitor/items/_details_wrapper.html.erb +3 -0
  69. data/app/views/source_monitor/items/index.html.erb +147 -0
  70. data/app/views/source_monitor/items/show.html.erb +3 -0
  71. data/app/views/source_monitor/logs/index.html.erb +208 -0
  72. data/app/views/source_monitor/scrape_logs/show.html.erb +73 -0
  73. data/app/views/source_monitor/shared/_toast.html.erb +34 -0
  74. data/app/views/source_monitor/sources/_bulk_scrape_form.html.erb +64 -0
  75. data/app/views/source_monitor/sources/_bulk_scrape_modal.html.erb +53 -0
  76. data/app/views/source_monitor/sources/_details.html.erb +302 -0
  77. data/app/views/source_monitor/sources/_details_wrapper.html.erb +3 -0
  78. data/app/views/source_monitor/sources/_empty_state_row.html.erb +5 -0
  79. data/app/views/source_monitor/sources/_fetch_interval_heatmap.html.erb +46 -0
  80. data/app/views/source_monitor/sources/_form.html.erb +143 -0
  81. data/app/views/source_monitor/sources/_health_status_badge.html.erb +46 -0
  82. data/app/views/source_monitor/sources/_row.html.erb +102 -0
  83. data/app/views/source_monitor/sources/edit.html.erb +28 -0
  84. data/app/views/source_monitor/sources/index.html.erb +153 -0
  85. data/app/views/source_monitor/sources/new.html.erb +22 -0
  86. data/app/views/source_monitor/sources/show.html.erb +3 -0
  87. data/config/coverage_baseline.json +2010 -0
  88. data/config/initializers/feedjira.rb +19 -0
  89. data/config/routes.rb +18 -0
  90. data/config/tailwind.config.js +17 -0
  91. data/db/migrate/20241008120000_create_source_monitor_sources.rb +40 -0
  92. data/db/migrate/20241008121000_create_source_monitor_items.rb +44 -0
  93. data/db/migrate/20241008122000_create_source_monitor_fetch_logs.rb +32 -0
  94. data/db/migrate/20241008123000_create_source_monitor_scrape_logs.rb +25 -0
  95. data/db/migrate/20251008183000_change_fetch_interval_to_minutes.rb +23 -0
  96. data/db/migrate/20251009090000_create_source_monitor_item_contents.rb +38 -0
  97. data/db/migrate/20251009103000_add_feed_content_readability_to_sources.rb +5 -0
  98. data/db/migrate/20251010090000_add_adaptive_fetching_toggle_to_sources.rb +7 -0
  99. data/db/migrate/20251010123000_add_deleted_at_to_source_monitor_items.rb +8 -0
  100. data/db/migrate/20251010153000_add_type_to_source_monitor_sources.rb +8 -0
  101. data/db/migrate/20251010154500_add_fetch_status_to_source_monitor_sources.rb +9 -0
  102. data/db/migrate/20251010160000_create_solid_cable_messages.rb +16 -0
  103. data/db/migrate/20251011090000_add_fetch_retry_state_to_sources.rb +14 -0
  104. data/db/migrate/20251012090000_add_health_fields_to_sources.rb +17 -0
  105. data/db/migrate/20251012100000_optimize_source_monitor_database_performance.rb +13 -0
  106. data/db/migrate/20251014064947_add_not_null_constraints_to_items.rb +30 -0
  107. data/db/migrate/20251014171659_add_performance_indexes.rb +29 -0
  108. data/db/migrate/20251014172525_add_fetch_status_check_constraint.rb +18 -0
  109. data/db/migrate/20251015100000_create_source_monitor_log_entries.rb +89 -0
  110. data/db/migrate/20251022100000_create_source_monitor_health_check_logs.rb +22 -0
  111. data/db/migrate/20251108120116_refresh_fetch_status_constraint.rb +29 -0
  112. data/docs/configuration.md +170 -0
  113. data/docs/deployment.md +63 -0
  114. data/docs/gh-cli-workflow.md +44 -0
  115. data/docs/installation.md +144 -0
  116. data/docs/troubleshooting.md +76 -0
  117. data/eslint.config.mjs +27 -0
  118. data/lib/generators/source_monitor/install/install_generator.rb +59 -0
  119. data/lib/generators/source_monitor/install/templates/source_monitor.rb.tt +155 -0
  120. data/lib/source_monitor/analytics/source_activity_rates.rb +53 -0
  121. data/lib/source_monitor/analytics/source_fetch_interval_distribution.rb +57 -0
  122. data/lib/source_monitor/analytics/sources_index_metrics.rb +92 -0
  123. data/lib/source_monitor/assets/bundler.rb +49 -0
  124. data/lib/source_monitor/assets.rb +6 -0
  125. data/lib/source_monitor/configuration.rb +654 -0
  126. data/lib/source_monitor/dashboard/queries.rb +356 -0
  127. data/lib/source_monitor/dashboard/quick_action.rb +7 -0
  128. data/lib/source_monitor/dashboard/quick_actions_presenter.rb +26 -0
  129. data/lib/source_monitor/dashboard/recent_activity.rb +30 -0
  130. data/lib/source_monitor/dashboard/recent_activity_presenter.rb +77 -0
  131. data/lib/source_monitor/dashboard/turbo_broadcaster.rb +87 -0
  132. data/lib/source_monitor/dashboard/upcoming_fetch_schedule.rb +126 -0
  133. data/lib/source_monitor/engine.rb +107 -0
  134. data/lib/source_monitor/events.rb +110 -0
  135. data/lib/source_monitor/feedjira_extensions.rb +103 -0
  136. data/lib/source_monitor/fetching/advisory_lock.rb +54 -0
  137. data/lib/source_monitor/fetching/completion/event_publisher.rb +22 -0
  138. data/lib/source_monitor/fetching/completion/follow_up_handler.rb +37 -0
  139. data/lib/source_monitor/fetching/completion/retention_handler.rb +30 -0
  140. data/lib/source_monitor/fetching/feed_fetcher.rb +627 -0
  141. data/lib/source_monitor/fetching/fetch_error.rb +88 -0
  142. data/lib/source_monitor/fetching/fetch_runner.rb +142 -0
  143. data/lib/source_monitor/fetching/retry_policy.rb +85 -0
  144. data/lib/source_monitor/fetching/stalled_fetch_reconciler.rb +146 -0
  145. data/lib/source_monitor/health/source_health_check.rb +100 -0
  146. data/lib/source_monitor/health/source_health_monitor.rb +210 -0
  147. data/lib/source_monitor/health/source_health_reset.rb +68 -0
  148. data/lib/source_monitor/health.rb +46 -0
  149. data/lib/source_monitor/http.rb +85 -0
  150. data/lib/source_monitor/instrumentation.rb +52 -0
  151. data/lib/source_monitor/items/item_creator.rb +601 -0
  152. data/lib/source_monitor/items/retention_pruner.rb +146 -0
  153. data/lib/source_monitor/items/retention_strategies/destroy.rb +26 -0
  154. data/lib/source_monitor/items/retention_strategies/soft_delete.rb +50 -0
  155. data/lib/source_monitor/items/retention_strategies.rb +9 -0
  156. data/lib/source_monitor/jobs/cleanup_options.rb +85 -0
  157. data/lib/source_monitor/jobs/fetch_failure_subscriber.rb +129 -0
  158. data/lib/source_monitor/jobs/solid_queue_metrics.rb +199 -0
  159. data/lib/source_monitor/jobs/visibility.rb +133 -0
  160. data/lib/source_monitor/logs/entry_sync.rb +69 -0
  161. data/lib/source_monitor/logs/filter_set.rb +163 -0
  162. data/lib/source_monitor/logs/query.rb +81 -0
  163. data/lib/source_monitor/logs/table_presenter.rb +161 -0
  164. data/lib/source_monitor/metrics.rb +77 -0
  165. data/lib/source_monitor/model_extensions.rb +109 -0
  166. data/lib/source_monitor/models/sanitizable.rb +76 -0
  167. data/lib/source_monitor/models/url_normalizable.rb +84 -0
  168. data/lib/source_monitor/pagination/paginator.rb +90 -0
  169. data/lib/source_monitor/realtime/adapter.rb +97 -0
  170. data/lib/source_monitor/realtime/broadcaster.rb +237 -0
  171. data/lib/source_monitor/realtime.rb +17 -0
  172. data/lib/source_monitor/release/changelog.rb +59 -0
  173. data/lib/source_monitor/release/runner.rb +73 -0
  174. data/lib/source_monitor/scheduler.rb +82 -0
  175. data/lib/source_monitor/scrapers/base.rb +105 -0
  176. data/lib/source_monitor/scrapers/fetchers/http_fetcher.rb +97 -0
  177. data/lib/source_monitor/scrapers/parsers/readability_parser.rb +101 -0
  178. data/lib/source_monitor/scrapers/readability.rb +156 -0
  179. data/lib/source_monitor/scraping/bulk_result_presenter.rb +85 -0
  180. data/lib/source_monitor/scraping/bulk_source_scraper.rb +233 -0
  181. data/lib/source_monitor/scraping/enqueuer.rb +125 -0
  182. data/lib/source_monitor/scraping/item_scraper/adapter_resolver.rb +44 -0
  183. data/lib/source_monitor/scraping/item_scraper/persistence.rb +189 -0
  184. data/lib/source_monitor/scraping/item_scraper.rb +84 -0
  185. data/lib/source_monitor/scraping/scheduler.rb +43 -0
  186. data/lib/source_monitor/scraping/state.rb +79 -0
  187. data/lib/source_monitor/security/authentication.rb +85 -0
  188. data/lib/source_monitor/security/parameter_sanitizer.rb +42 -0
  189. data/lib/source_monitor/sources/turbo_stream_presenter.rb +54 -0
  190. data/lib/source_monitor/turbo_streams/stream_responder.rb +95 -0
  191. data/lib/source_monitor/version.rb +3 -0
  192. data/lib/source_monitor.rb +149 -0
  193. data/lib/tasks/recover_stalled_fetches.rake +16 -0
  194. data/lib/tasks/source_monitor_assets.rake +28 -0
  195. data/lib/tasks/source_monitor_tasks.rake +29 -0
  196. data/lib/tasks/test_smoke.rake +12 -0
  197. data/package-lock.json +3997 -0
  198. data/package.json +29 -0
  199. data/postcss.config.js +6 -0
  200. data/source_monitor.gemspec +46 -0
  201. data/stylelint.config.js +12 -0
  202. metadata +469 -0
@@ -0,0 +1,73 @@
1
+ <div class="space-y-6">
2
+ <div class="flex flex-wrap items-start justify-between gap-4">
3
+ <div>
4
+ <h1 class="text-3xl font-semibold text-slate-900">Scrape Log</h1>
5
+ <p class="mt-1 text-sm text-slate-500">
6
+ Started <%= @log.started_at&.strftime("%b %d, %Y %H:%M:%S %Z") || "Unknown" %> ·
7
+ <%= @log.success? ? "Success" : "Failure" %>
8
+ </p>
9
+ </div>
10
+ <div class="flex gap-3">
11
+ <%= link_to "Back to Logs", source_monitor.logs_path(log_type: "scrape"), class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50" %>
12
+ <%= link_to "View Item", source_monitor.item_path(@log.item), class: "inline-flex items-center rounded-md bg-slate-800 px-3 py-2 text-sm font-medium text-white hover:bg-slate-700" %>
13
+ </div>
14
+ </div>
15
+
16
+ <div class="grid gap-6 lg:grid-cols-2">
17
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
18
+ <div class="border-b border-slate-200 px-5 py-4">
19
+ <h2 class="text-lg font-medium">Summary</h2>
20
+ </div>
21
+ <dl class="divide-y divide-slate-100">
22
+ <% summary = {
23
+ "Item" => link_to(@log.item.title.presence || "(untitled)", source_monitor.item_path(@log.item)),
24
+ "Source" => link_to(@log.source.name, source_monitor.source_path(@log.source)),
25
+ "Scraper Adapter" => @log.scraper_adapter || "—",
26
+ "HTTP Status" => @log.http_status || "—",
27
+ "Success" => @log.success? ? "Yes" : "No",
28
+ "Duration (ms)" => @log.duration_ms || "—",
29
+ "Content Length" => @log.content_length || "—",
30
+ "Started At" => @log.started_at&.strftime("%b %d, %Y %H:%M:%S %Z") || "—",
31
+ "Completed At" => @log.completed_at&.strftime("%b %d, %Y %H:%M:%S %Z") || "—"
32
+ } %>
33
+ <% summary.each do |label, value| %>
34
+ <div class="flex items-center justify-between px-5 py-3">
35
+ <dt class="text-sm font-medium text-slate-600"><%= label %></dt>
36
+ <dd class="text-sm text-slate-900 text-right"><%= value %></dd>
37
+ </div>
38
+ <% end %>
39
+ </dl>
40
+ </div>
41
+
42
+ <div class="space-y-6">
43
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
44
+ <div class="border-b border-slate-200 px-5 py-4">
45
+ <h2 class="text-lg font-medium">Error Details</h2>
46
+ </div>
47
+ <div class="px-5 py-4 text-sm text-slate-700">
48
+ <% if @log.success? %>
49
+ <p class="text-slate-500">No errors recorded.</p>
50
+ <% else %>
51
+ <% if @log.error_class.present? %>
52
+ <p class="font-medium text-rose-600"><%= @log.error_class %></p>
53
+ <% end %>
54
+ <p class="mt-2 whitespace-pre-wrap break-words text-xs text-slate-600"><%= @log.error_message.presence || "No error message captured." %></p>
55
+ <% end %>
56
+ </div>
57
+ </div>
58
+
59
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
60
+ <div class="border-b border-slate-200 px-5 py-4">
61
+ <h2 class="text-lg font-medium">Metadata</h2>
62
+ </div>
63
+ <div class="px-5 py-4 text-sm text-slate-700">
64
+ <% if @log.metadata.present? %>
65
+ <pre class="whitespace-pre-wrap break-words text-xs text-slate-600"><%= JSON.pretty_generate(@log.metadata) %></pre>
66
+ <% else %>
67
+ <p class="text-slate-500">No metadata recorded.</p>
68
+ <% end %>
69
+ </div>
70
+ </div>
71
+ </div>
72
+ </div>
73
+ </div>
@@ -0,0 +1,34 @@
1
+ <% level_key = local_assigns.fetch(:level, :info).to_s %>
2
+ <% delay_ms = local_assigns.fetch(:delay_ms, 5000) %>
3
+ <% title = local_assigns[:title] %>
4
+ <% message = local_assigns[:message] %>
5
+ <% palette = {
6
+ "success" => "border-emerald-200 bg-emerald-50 text-emerald-800",
7
+ "error" => "border-rose-200 bg-rose-50 text-rose-800",
8
+ "info" => "border-blue-200 bg-blue-50 text-blue-800",
9
+ "warning" => "border-amber-200 bg-amber-50 text-amber-800"
10
+ } %>
11
+ <% classes = palette.fetch(level_key, palette["info"]) %>
12
+
13
+ <div
14
+ data-controller="notification"
15
+ data-notification-delay-value="<%= delay_ms %>"
16
+ class="pointer-events-auto w-full max-w-md rounded-lg border px-4 py-3 shadow-lg transition duration-300 <%= classes %>"
17
+ >
18
+ <div class="flex items-start justify-between gap-3">
19
+ <div class="flex-1">
20
+ <% if title.present? %>
21
+ <p class="text-sm font-semibold"><%= title %></p>
22
+ <% end %>
23
+ <p class="mt-1 text-sm leading-snug"><%= message %></p>
24
+ </div>
25
+ <button
26
+ type="button"
27
+ class="text-xs font-semibold uppercase tracking-wide opacity-60 transition hover:opacity-100"
28
+ data-action="notification#hide"
29
+ aria-label="Dismiss notification"
30
+ >
31
+ Close
32
+ </button>
33
+ </div>
34
+ </div>
@@ -0,0 +1,64 @@
1
+ <% counts = (local_assigns[:counts] || {}).with_indifferent_access %>
2
+ <% selected = (local_assigns[:selected] || :current).to_sym %>
3
+ <% scraping_enabled = source.scraping_enabled? %>
4
+ <% options = [
5
+ { value: :current, label: "Current view", count: counts[:current] || 0 },
6
+ { value: :unscraped, label: "Unscraped items", count: counts[:unscraped] || 0 },
7
+ { value: :all, label: "All items", count: counts[:all] || 0 }
8
+ ] %>
9
+
10
+ <div data-testid="bulk-scrape-form">
11
+ <% if scraping_enabled %>
12
+ <%= form_with url: source_monitor.source_bulk_scrape_path(source),
13
+ method: :post,
14
+ scope: :bulk_scrape,
15
+ data: {
16
+ controller: "async-submit",
17
+ action: "turbo:submit-start->async-submit#start turbo:submit-end->async-submit#finish"
18
+ },
19
+ class: "space-y-6" do |form| %>
20
+ <fieldset class="space-y-4" role="radiogroup">
21
+ <legend class="text-xs font-semibold uppercase tracking-wider text-slate-500">Scrape scope</legend>
22
+ <div class="space-y-2">
23
+ <% options.each do |option| %>
24
+ <% value = option[:value] %>
25
+ <% is_selected = (selected == value) %>
26
+ <% count_label = option[:count].to_i %>
27
+ <label class="relative block" data-testid="bulk-scrape-option-<%= value %>">
28
+ <%= form.radio_button :selection,
29
+ value,
30
+ checked: is_selected,
31
+ class: "peer absolute left-4 top-1/2 h-4 w-4 -translate-y-1/2 text-blue-600 focus:ring-blue-500",
32
+ data: { testid: "bulk-scrape-radio-#{value}" } %>
33
+ <div class="flex items-center justify-between gap-3 rounded-lg border border-slate-200 bg-white px-4 py-3 pl-11 text-sm font-medium text-slate-600 transition peer-checked:border-blue-500 peer-checked:bg-blue-50 peer-checked:text-blue-700">
34
+ <span><%= option[:label] %></span>
35
+ <span class="rounded-full bg-slate-100 px-2 py-0.5 text-xs font-semibold text-slate-600 peer-checked:bg-blue-100 peer-checked:text-blue-700"><%= pluralize(count_label, "item") %></span>
36
+ </div>
37
+ </label>
38
+ <% end %>
39
+ </div>
40
+ </fieldset>
41
+
42
+ <div class="flex items-center justify-end gap-3">
43
+ <button
44
+ type="button"
45
+ class="inline-flex items-center rounded-md border border-slate-200 px-4 py-2 text-sm font-medium text-slate-600 hover:bg-slate-100"
46
+ data-action="modal#close"
47
+ >
48
+ Cancel
49
+ </button>
50
+ <%= form.submit "Scrape Items",
51
+ class: "inline-flex items-center rounded-md bg-blue-600 px-4 py-2 text-sm font-medium text-white shadow hover:bg-blue-500 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2",
52
+ data: {
53
+ "async-submit-target": "button",
54
+ "async-submit-loading-text-value": "Enqueuing..."
55
+ } %>
56
+ </div>
57
+ <% end %>
58
+ <% else %>
59
+ <div class="rounded-lg border border-slate-200 bg-slate-50 px-4 py-3 text-xs text-slate-500">
60
+ <p class="font-semibold text-slate-700">Scraping is disabled for this source.</p>
61
+ <p class="mt-1">Enable scraping to queue bulk scrapes.</p>
62
+ </div>
63
+ <% end %>
64
+ </div>
@@ -0,0 +1,53 @@
1
+ <% selected = (local_assigns[:selected] || :current).to_sym %>
2
+ <% preview_limit = local_assigns[:preview_limit] || SourceMonitor::Scraping::BulkSourceScraper::DEFAULT_PREVIEW_LIMIT %>
3
+ <% preview_items_relation = SourceMonitor::Item.where(source_id: source.id)
4
+ .order(Arel.sql("published_at DESC NULLS LAST, created_at DESC"))
5
+ .limit(preview_limit) %>
6
+ <% preview_items = preview_items_relation.to_a %>
7
+ <% counts = SourceMonitor::Scraping::BulkSourceScraper.selection_counts(
8
+ source: source,
9
+ preview_items: preview_items,
10
+ preview_limit: preview_limit
11
+ ) %>
12
+
13
+ <div data-controller="modal" data-modal-open-class="flex" class="relative">
14
+ <button
15
+ type="button"
16
+ class="inline-flex items-center rounded-md bg-blue-600 px-3 py-2 text-sm font-medium text-white shadow hover:bg-blue-500 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2"
17
+ data-action="modal#open"
18
+ data-testid="bulk-scrape-button"
19
+ >
20
+ Bulk Scrape
21
+ </button>
22
+
23
+ <div
24
+ data-modal-target="panel"
25
+ class="hidden fixed inset-0 z-50 items-center justify-center"
26
+ data-testid="bulk-scrape-modal"
27
+ >
28
+ <div class="absolute inset-0 bg-slate-900/40" data-action="click->modal#close"></div>
29
+ <div class="relative z-10 w-full max-w-xl overflow-hidden rounded-lg bg-white shadow-xl">
30
+ <div class="flex items-start justify-between border-b border-slate-200 px-6 py-4">
31
+ <div>
32
+ <h2 class="text-lg font-semibold text-slate-900">Bulk Scrape Items</h2>
33
+ <p class="mt-1 text-xs text-slate-500">Queue scraping jobs for this source without leaving the page.</p>
34
+ </div>
35
+ <button
36
+ type="button"
37
+ class="inline-flex h-8 w-8 items-center justify-center rounded-full text-slate-500 hover:bg-slate-100 hover:text-slate-700"
38
+ data-action="modal#close"
39
+ aria-label="Close bulk scrape modal"
40
+ >
41
+ <span aria-hidden="true">&times;</span>
42
+ </button>
43
+ </div>
44
+
45
+ <div class="px-6 py-5">
46
+ <%= render "source_monitor/sources/bulk_scrape_form",
47
+ source: source,
48
+ counts: counts,
49
+ selected: selected %>
50
+ </div>
51
+ </div>
52
+ </div>
53
+ </div>
@@ -0,0 +1,302 @@
1
+ <% source = local_assigns.fetch(:source) %>
2
+ <% recent_fetch_logs = source.fetch_logs.order(started_at: :desc).limit(5) %>
3
+ <% recent_scrape_logs = source.scrape_logs.order(started_at: :desc).limit(5) %>
4
+ <% preview_limit = SourceMonitor::Scraping::BulkSourceScraper::DEFAULT_PREVIEW_LIMIT %>
5
+ <% items = source.items.recent.limit(preview_limit) %>
6
+ <% fetch_status = async_status_badge(source.fetch_status) %>
7
+ <% health_status_override = local_assigns[:health_status_override] %>
8
+
9
+ <div class="space-y-8">
10
+ <div class="flex items-start justify-between">
11
+ <div>
12
+ <h1 class="text-3xl font-semibold text-slate-900"><%= source.name %></h1>
13
+ <div class="mt-2 flex flex-wrap items-center gap-2 text-xs">
14
+ <span data-testid="fetch-status-badge" class="inline-flex items-center rounded-full px-3 py-1 font-semibold <%= fetch_status[:classes] %>">
15
+ <%= loading_spinner_svg if fetch_status[:show_spinner] %>
16
+ <%= fetch_status[:label] %>
17
+ </span>
18
+ <% if source.last_fetch_started_at.present? %>
19
+ <span class="rounded-full bg-slate-100 px-3 py-1 font-semibold text-slate-600">
20
+ Started <%= time_ago_in_words(source.last_fetch_started_at) %> ago
21
+ </span>
22
+ <% elsif source.last_fetched_at.present? %>
23
+ <span class="rounded-full bg-slate-100 px-3 py-1 font-semibold text-slate-600">
24
+ Last fetched <%= time_ago_in_words(source.last_fetched_at) %> ago
25
+ </span>
26
+ <% end %>
27
+ </div>
28
+ <p class="mt-2 text-sm text-slate-500">Feed URL: <%= source.feed_url %></p>
29
+ </div>
30
+ <div class="flex flex-wrap items-center justify-end gap-3">
31
+ <% fetch_disabled = %w[queued fetching].include?(source.fetch_status) %>
32
+ <%= button_to "Fetch Now",
33
+ source_monitor.source_fetch_path(source),
34
+ method: :post,
35
+ class: [
36
+ "inline-flex items-center rounded-md px-3 py-2 text-sm font-medium text-white shadow",
37
+ fetch_disabled ? "cursor-not-allowed bg-slate-400" : "bg-blue-600 hover:bg-blue-500"
38
+ ].join(" "),
39
+ disabled: fetch_disabled,
40
+ data: {
41
+ "async-submit-target": "button",
42
+ "async-submit-loading-text-value": "Enqueuing..."
43
+ },
44
+ form: {
45
+ class: "inline",
46
+ data: {
47
+ controller: "async-submit",
48
+ action: "turbo:submit-start->async-submit#start turbo:submit-end->async-submit#finish"
49
+ }
50
+ } %>
51
+ <%= render "source_monitor/sources/bulk_scrape_modal",
52
+ source: source,
53
+ selected: (@bulk_scrape_selection || :current),
54
+ preview_limit: preview_limit %>
55
+ <% if source.fetch_circuit_open? %>
56
+ <%= button_to "Retry Now",
57
+ source_monitor.source_retry_path(source),
58
+ method: :post,
59
+ class: "inline-flex items-center rounded-md bg-amber-600 px-3 py-2 text-sm font-medium text-white shadow hover:bg-amber-500",
60
+ data: {
61
+ "async-submit-target": "button",
62
+ "async-submit-loading-text-value": "Retrying..."
63
+ },
64
+ form: {
65
+ class: "inline",
66
+ data: {
67
+ controller: "async-submit",
68
+ action: "turbo:submit-start->async-submit#start turbo:submit-end->async-submit#finish"
69
+ }
70
+ } %>
71
+ <% end %>
72
+ <%= link_to "Edit", source_monitor.edit_source_path(source), class: "inline-flex items-center rounded-md bg-slate-800 px-3 py-2 text-sm font-medium text-white shadow hover:bg-slate-700" %>
73
+ <%= form_with url: source_monitor.source_path(source), method: :delete, data: { turbo_confirm: "Delete this source and remove all associated data?", turbo_frame: "_top" }, class: "inline" do %>
74
+ <%= hidden_field_tag :redirect_to, source_monitor.sources_path %>
75
+ <button type="submit" class="inline-flex items-center rounded-md border border-rose-400 px-3 py-2 text-sm font-medium text-rose-600 hover:bg-rose-50">
76
+ Delete
77
+ </button>
78
+ <% end %>
79
+ </div>
80
+ </div>
81
+
82
+ <% if source.fetch_circuit_open? %>
83
+ <div class="rounded-lg border border-amber-200 bg-amber-50 px-6 py-4 shadow-sm">
84
+ <div class="flex flex-wrap items-center justify-between gap-4">
85
+ <div>
86
+ <h2 class="text-lg font-medium text-amber-900">Circuit breaker open</h2>
87
+ <p class="mt-1 text-xs text-amber-700">
88
+ Recent failures paused automatic fetches until <%= source.fetch_circuit_until&.strftime("%b %d, %Y %H:%M %Z") || "later" %>. Use Retry Now to bypass the cooldown.
89
+ </p>
90
+ </div>
91
+ <span class="inline-flex items-center rounded-full bg-amber-600/10 px-3 py-1 text-xs font-semibold text-amber-700">
92
+ Failures: <%= source.failure_count %>
93
+ </span>
94
+ </div>
95
+ </div>
96
+ <% end %>
97
+
98
+ <% if source.items_retention_days.present? || source.max_items.present? %>
99
+ <div class="rounded-lg border border-blue-200 bg-blue-50 px-6 py-4 shadow-sm">
100
+ <div class="flex flex-wrap items-center justify-between gap-4">
101
+ <div>
102
+ <h2 class="text-lg font-medium text-slate-900">Retention Policy Active</h2>
103
+ <p class="mt-1 text-xs text-slate-600">
104
+ SourceMonitor prunes items for this source immediately after each fetch so stored data stays within the configured limits.
105
+ </p>
106
+ </div>
107
+ <div class="flex gap-2">
108
+ <% if source.items_retention_days.present? %>
109
+ <span class="inline-flex items-center rounded-full bg-blue-600/10 px-3 py-1 text-xs font-semibold text-blue-700">
110
+ <%= pluralize(source.items_retention_days, "day") %> window
111
+ </span>
112
+ <% end %>
113
+ <% if source.max_items.present? %>
114
+ <span class="inline-flex items-center rounded-full bg-blue-600/10 px-3 py-1 text-xs font-semibold text-blue-700">
115
+ Max <%= number_with_delimiter(source.max_items) %> items
116
+ </span>
117
+ <% end %>
118
+ </div>
119
+ </div>
120
+ </div>
121
+ <% end %>
122
+
123
+ <div class="grid gap-6 lg:grid-cols-3 lg:items-start">
124
+ <div class="space-y-6 lg:col-span-1">
125
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
126
+ <div class="border-b border-slate-200 px-5 py-4">
127
+ <h2 class="text-lg font-medium">Source Details</h2>
128
+ </div>
129
+ <dl class="divide-y divide-slate-100">
130
+ <% interval_hours = number_with_precision(source.fetch_interval_minutes / 60.0, precision: 2)
131
+ circuit_state =
132
+ if source.fetch_circuit_open?
133
+ until_time = source.fetch_circuit_until&.strftime("%b %d, %Y %H:%M %Z") || "unknown"
134
+ "Open until #{until_time}"
135
+ else
136
+ "Closed"
137
+ end
138
+
139
+ details = {
140
+ "Website" => (source.website_url.presence || "—"),
141
+ "Fetch interval" => "#{source.fetch_interval_minutes} minutes (~#{interval_hours} hours)",
142
+ "Adaptive interval" => source.adaptive_fetching_enabled? ? "Auto" : "Fixed",
143
+ "Scraper" => source.scraper_adapter,
144
+ "Feed content" => source.feed_content_readability_enabled? ? "Readability" : "Raw",
145
+ "Active" => source.active? ? "Yes" : "No",
146
+ "Scraping" => source.scraping_enabled? ? "Enabled" : "Disabled",
147
+ "Auto scrape" => source.auto_scrape? ? "Enabled" : "Disabled",
148
+ "Requires JS" => source.requires_javascript? ? "Yes" : "No",
149
+ "Failure count" => source.failure_count,
150
+ "Retry attempt" => source.fetch_retry_attempt,
151
+ "Circuit state" => circuit_state,
152
+ "Last error" => source.last_error.presence || "None",
153
+ "Items count" => source.items_count,
154
+ "Retention days" => source.items_retention_days || "—",
155
+ "Max items" => source.max_items || "—"
156
+ } %>
157
+ <% details.each do |label, value| %>
158
+ <div class="flex items-center justify-between px-5 py-3">
159
+ <dt class="text-sm font-medium text-slate-600"><%= label %></dt>
160
+ <dd class="text-sm text-slate-900"><%= value %></dd>
161
+ </div>
162
+ <% end %>
163
+ </dl>
164
+ </div>
165
+
166
+ <div class="rounded-lg border border-slate-200 bg-white px-6 py-5 shadow-sm">
167
+ <div class="flex items-center justify-between">
168
+ <h2 class="text-lg font-medium text-slate-900">Health Monitoring</h2>
169
+ <%= render "source_monitor/sources/health_status_badge",
170
+ source: source,
171
+ health_status_override: health_status_override %>
172
+ </div>
173
+ <dl class="mt-4 divide-y divide-slate-100">
174
+ <% rows = {
175
+ "Rolling success" => (source.rolling_success_rate ? number_to_percentage(source.rolling_success_rate * 100, precision: 0) : "—"),
176
+ "Auto-pause threshold" => begin
177
+ threshold = source.health_auto_pause_threshold.presence || SourceMonitor.config.health.auto_pause_threshold
178
+ threshold ? number_to_percentage(threshold.to_f * 100, precision: 0) : "—"
179
+ end,
180
+ "Auto-pause started" => (source.auto_paused_at&.strftime("%b %d, %Y %H:%M %Z") || "—"),
181
+ "Auto-pause until" => (source.auto_paused_until&.strftime("%b %d, %Y %H:%M %Z") || "—")
182
+ } %>
183
+ <% rows.each do |label, value| %>
184
+ <div class="flex items-center justify-between py-3 text-sm">
185
+ <dt class="font-medium text-slate-600"><%= label %></dt>
186
+ <dd class="text-slate-900"><%= value %></dd>
187
+ </div>
188
+ <% end %>
189
+ </dl>
190
+ </div>
191
+
192
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
193
+ <div class="border-b border-slate-200 px-5 py-4">
194
+ <h2 class="text-lg font-medium">Recent Fetches</h2>
195
+ </div>
196
+ <div class="divide-y divide-slate-100">
197
+ <% if recent_fetch_logs.any? %>
198
+ <% recent_fetch_logs.each do |log| %>
199
+ <div class="px-5 py-3 text-sm text-slate-700">
200
+ <div class="flex items-center justify-between">
201
+ <span>Started <%= log.started_at&.strftime("%b %d, %H:%M") || "Unknown" %></span>
202
+ <span class="ml-4 inline-flex items-center rounded-full px-2 py-0.5 text-xs font-semibold <%= log.success? ? "bg-green-100 text-green-700" : "bg-rose-100 text-rose-700" %>">
203
+ <%= log.success? ? "Success" : "Failure" %>
204
+ </span>
205
+ </div>
206
+ <p class="mt-1 text-xs text-slate-500"><%= log.items_created %> created · <%= log.items_updated %> updated · <%= log.items_failed %> failed</p>
207
+ </div>
208
+ <% end %>
209
+ <% else %>
210
+ <div class="px-5 py-4 text-sm text-slate-500">No fetch history yet.</div>
211
+ <% end %>
212
+ </div>
213
+ </div>
214
+
215
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
216
+ <div class="border-b border-slate-200 px-5 py-4">
217
+ <h2 class="text-lg font-medium">Recent Scrapes</h2>
218
+ </div>
219
+ <div class="divide-y divide-slate-100">
220
+ <% if recent_scrape_logs.any? %>
221
+ <% recent_scrape_logs.each do |log| %>
222
+ <div class="px-5 py-3 text-sm text-slate-700">
223
+ <div class="flex items-center justify-between">
224
+ <span>Started <%= log.started_at&.strftime("%b %d, %H:%M") || "Unknown" %></span>
225
+ <span class="ml-4 inline-flex items-center rounded-full px-2 py-0.5 text-xs font-semibold <%= log.success? ? "bg-green-100 text-green-700" : "bg-rose-100 text-rose-700" %>">
226
+ <%= log.success? ? "Success" : "Failure" %>
227
+ </span>
228
+ </div>
229
+ <p class="mt-1 text-xs text-slate-500"><%= log.scraper_adapter || "—" %></p>
230
+ </div>
231
+ <% end %>
232
+ <% else %>
233
+ <div class="px-5 py-4 text-sm text-slate-500">No scrape history yet.</div>
234
+ <% end %>
235
+ </div>
236
+ </div>
237
+ </div>
238
+
239
+ <div class="lg:col-span-2">
240
+ <div class="rounded-lg border border-slate-200 bg-white shadow-sm">
241
+ <div class="flex items-center justify-between border-b border-slate-200 px-5 py-4">
242
+ <h2 class="text-lg font-medium">Items</h2>
243
+ <%= link_to "View all items", source_monitor.items_path, class: "text-sm font-medium text-blue-600 hover:text-blue-500" %>
244
+ </div>
245
+ <% if items.any? %>
246
+ <div class="overflow-x-auto">
247
+ <table class="min-w-full divide-y divide-slate-200 text-left text-sm" data-testid="source-items-table">
248
+ <thead class="bg-slate-50 text-xs font-semibold uppercase tracking-wide text-slate-500">
249
+ <tr>
250
+ <th scope="col" class="px-5 py-3">Title</th>
251
+ <th scope="col" class="px-5 py-3">Categories</th>
252
+ <th scope="col" class="px-5 py-3">Tags</th>
253
+ <th scope="col" class="px-5 py-3">Published</th>
254
+ <th scope="col" class="px-5 py-3">Scrape Status</th>
255
+ </tr>
256
+ </thead>
257
+ <tbody class="divide-y divide-slate-100 text-slate-700">
258
+ <% items.each do |item| %>
259
+ <tr class="hover:bg-slate-50">
260
+ <td class="px-5 py-4">
261
+ <div class="font-medium text-slate-900">
262
+ <%= link_to item.title.presence || "(untitled)", source_monitor.item_path(item), class: "hover:text-blue-600" %>
263
+ </div>
264
+ <% if item.summary.present? %>
265
+ <div class="mt-1 text-xs text-slate-500"><%= item.summary %></div>
266
+ <% end %>
267
+ </td>
268
+ <% categories = Array(item.categories).filter_map { |value| value.to_s.strip.presence } %>
269
+ <% tags = Array(item.tags).filter_map { |value| value.to_s.strip.presence } %>
270
+ <td class="px-5 py-4 text-xs text-slate-500"><%= categories.present? ? categories.join(", ") : "—" %></td>
271
+ <td class="px-5 py-4 text-xs text-slate-500"><%= tags.present? ? tags.join(", ") : "—" %></td>
272
+ <td class="px-5 py-4 text-xs text-slate-500">
273
+ <%= item.published_at ? item.published_at.strftime("%b %d, %Y %H:%M") : "Unpublished" %>
274
+ </td>
275
+ <td class="px-5 py-4 text-xs">
276
+ <% scrape_badge = item_scrape_status_badge(item: item, source: source) %>
277
+ <span
278
+ class="inline-flex items-center gap-1 rounded-full px-3 py-1 font-semibold <%= scrape_badge[:classes] %>"
279
+ data-testid="item-scrape-status-badge"
280
+ data-status="<%= scrape_badge[:status] %>"
281
+ >
282
+ <%= loading_spinner_svg(css_class: "h-3.5 w-3.5 animate-spin text-blue-500") if scrape_badge[:show_spinner] %>
283
+ <%= scrape_badge[:label] %>
284
+ </span>
285
+ </td>
286
+ </tr>
287
+ <% end %>
288
+ </tbody>
289
+ </table>
290
+ </div>
291
+ <div class="border-t border-slate-200 px-5 py-3 text-xs text-slate-500">
292
+ Showing up to 10 most recent items.
293
+ </div>
294
+ <% else %>
295
+ <div class="px-5 py-6 text-sm text-slate-500">
296
+ No items for this source yet.
297
+ </div>
298
+ <% end %>
299
+ </div>
300
+ </div>
301
+ </div>
302
+ </div>
@@ -0,0 +1,3 @@
1
+ <div id="<%= dom_id(source, :details) %>">
2
+ <%= render "source_monitor/sources/details", source: source %>
3
+ </div>
@@ -0,0 +1,5 @@
1
+ <tr id="source_monitor_sources_empty_state">
2
+ <td colspan="7" class="px-6 py-6 text-center text-sm text-slate-500">
3
+ No sources yet. <%= link_to "Create the first source", source_monitor.new_source_path, class: "text-blue-600 hover:text-blue-500" %>.
4
+ </td>
5
+ </tr>
@@ -0,0 +1,46 @@
1
+ <% distribution = Array(local_assigns.fetch(:fetch_interval_distribution, [])) %>
2
+ <% selected_bucket = local_assigns[:selected_bucket] %>
3
+ <% search_params = local_assigns[:search_params] || {} %>
4
+ <% max_count = distribution.map(&:count).max.to_i %>
5
+
6
+ <div id="source_monitor_sources_heatmap" class="overflow-hidden rounded-lg border border-slate-200 bg-white shadow-sm">
7
+ <div class="border-b border-slate-200 bg-slate-50 px-6 py-4">
8
+ <h2 class="text-base font-semibold text-slate-900">Fetch Interval Distribution</h2>
9
+ <p class="mt-1 text-xs text-slate-500">Heatmap shows how many sources sit in each scheduling bucket.</p>
10
+ </div>
11
+ <div class="px-6 py-4" data-testid="fetch-interval-heatmap">
12
+ <table class="w-full table-fixed text-center text-sm">
13
+ <thead class="text-xs font-semibold uppercase tracking-wide text-slate-500">
14
+ <tr>
15
+ <% distribution.each do |bucket| %>
16
+ <th scope="col" class="px-2 pb-3"><%= bucket.label %></th>
17
+ <% end %>
18
+ </tr>
19
+ </thead>
20
+ <tbody>
21
+ <tr>
22
+ <% distribution.each do |bucket| %>
23
+ <% bucket_classes = heatmap_bucket_classes(bucket.count, max_count) %>
24
+ <% selected = bucket == selected_bucket %>
25
+ <% link_path = fetch_interval_bucket_path(bucket, search_params, selected: selected) %>
26
+ <% bucket_key = [bucket.min || 0, bucket.max || "plus"].join("-") %>
27
+ <% highlight_classes = selected ? "ring-2 ring-blue-500 shadow-lg" : "ring-1 ring-transparent hover:ring-blue-400 hover:shadow" %>
28
+ <td class="px-2">
29
+ <%= link_to link_path,
30
+ class: "block focus:outline-none group",
31
+ data: { turbo_frame: "_top" },
32
+ "data-testid" => "fetch-interval-bucket-#{bucket_key}",
33
+ aria: { label: selected ? "Show all sources" : "Filter sources by #{bucket.label}" } do %>
34
+ <div class="flex flex-col items-center justify-center gap-1 rounded-lg px-3 py-4 text-xs font-semibold transition <%= bucket_classes %> <%= highlight_classes %>"
35
+ title="<%= "#{bucket.count} sources between #{bucket.label}" %>">
36
+ <span class="text-base"><%= bucket.count %></span>
37
+ <span class="font-medium">sources</span>
38
+ </div>
39
+ <% end %>
40
+ </td>
41
+ <% end %>
42
+ </tr>
43
+ </tbody>
44
+ </table>
45
+ </div>
46
+ </div>