source_monitor 0.10.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/agent-memory/vbw-vbw-debugger/MEMORY.md +15 -0
- data/.claude/skills/sm-configuration-setting/reference/settings-catalog.md +3 -3
- data/.claude/skills/sm-configure/reference/configuration-reference.md +3 -3
- data/.claude/skills/sm-domain-model/SKILL.md +2 -2
- data/.claude/skills/sm-domain-model/reference/table-structure.md +3 -1
- data/.claude/skills/sm-engine-migration/SKILL.md +1 -1
- data/.claude/skills/sm-engine-migration/reference/migration-conventions.md +1 -1
- data/.claude/skills/sm-health-rule/SKILL.md +18 -21
- data/.claude/skills/sm-health-rule/reference/health-system.md +1 -1
- data/.claude/skills/sm-host-setup/reference/initializer-template.md +2 -2
- data/.claude/skills/sm-upgrade/reference/version-history.md +17 -12
- data/CHANGELOG.md +42 -0
- data/CLAUDE.md +2 -2
- data/Gemfile +1 -0
- data/Gemfile.lock +4 -1
- data/README.md +3 -3
- data/VERSION +1 -1
- data/app/assets/builds/source_monitor/application.css +132 -12
- data/app/assets/builds/source_monitor/application.js +25 -1
- data/app/assets/builds/source_monitor/application.js.map +2 -2
- data/app/assets/javascripts/source_monitor/controllers/modal_controller.js +8 -0
- data/app/assets/javascripts/source_monitor/controllers/select_all_controller.js +22 -2
- data/app/assets/stylesheets/source_monitor/application.tailwind.css +1 -1
- data/app/controllers/source_monitor/bulk_scrape_enablements_controller.rb +57 -0
- data/app/controllers/source_monitor/dashboard_controller.rb +10 -1
- data/app/controllers/source_monitor/import_history_dismissals_controller.rb +20 -0
- data/app/controllers/source_monitor/source_retries_controller.rb +10 -2
- data/app/controllers/source_monitor/source_scrape_tests_controller.rb +73 -0
- data/app/controllers/source_monitor/sources_controller.rb +51 -9
- data/app/helpers/source_monitor/application_helper.rb +24 -0
- data/app/helpers/source_monitor/health_badge_helper.rb +7 -20
- data/app/jobs/source_monitor/fetch_feed_job.rb +32 -3
- data/app/jobs/source_monitor/source_health_check_job.rb +1 -1
- data/app/models/source_monitor/fetch_log.rb +4 -0
- data/app/models/source_monitor/import_history.rb +2 -0
- data/app/models/source_monitor/source.rb +47 -2
- data/app/views/source_monitor/dashboard/_fetch_schedule.html.erb +94 -68
- data/app/views/source_monitor/dashboard/_scrape_recommendations.html.erb +17 -0
- data/app/views/source_monitor/dashboard/_stats.html.erb +19 -0
- data/app/views/source_monitor/dashboard/index.html.erb +7 -1
- data/app/views/source_monitor/import_sessions/health_check/_row.html.erb +2 -2
- data/app/views/source_monitor/shared/_pagination.html.erb +74 -0
- data/app/views/source_monitor/source_scrape_tests/_result.html.erb +81 -0
- data/app/views/source_monitor/source_scrape_tests/show.html.erb +60 -0
- data/app/views/source_monitor/sources/_bulk_scrape_enable_modal.html.erb +29 -0
- data/app/views/source_monitor/sources/_details.html.erb +19 -1
- data/app/views/source_monitor/sources/_empty_state_row.html.erb +1 -1
- data/app/views/source_monitor/sources/_import_history_panel.html.erb +12 -5
- data/app/views/source_monitor/sources/_row.html.erb +34 -6
- data/app/views/source_monitor/sources/index.html.erb +184 -132
- data/config/brakeman.ignore +11 -1
- data/config/routes.rb +5 -0
- data/db/migrate/20260305120000_add_dismissed_at_to_import_histories.rb +7 -0
- data/db/migrate/20260306233004_add_error_category_to_fetch_logs.rb +8 -0
- data/db/migrate/20260307120000_add_consecutive_fetch_failures_to_sources.rb +11 -0
- data/db/migrate/20260312120000_simplify_health_status_values.rb +20 -0
- data/docs/configuration.md +9 -1
- data/docs/troubleshooting.md +9 -0
- data/docs/upgrade.md +31 -0
- data/lib/generators/source_monitor/install/templates/source_monitor.rb.tt +2 -3
- data/lib/source_monitor/analytics/scrape_recommendations.rb +27 -0
- data/lib/source_monitor/configuration/health_settings.rb +0 -2
- data/lib/source_monitor/configuration/scraping_settings.rb +8 -1
- data/lib/source_monitor/dashboard/queries/stats_query.rb +12 -1
- data/lib/source_monitor/dashboard/queries.rb +6 -3
- data/lib/source_monitor/dashboard/recent_activity_presenter.rb +6 -5
- data/lib/source_monitor/dashboard/upcoming_fetch_schedule.rb +40 -54
- data/lib/source_monitor/favicons/discoverer.rb +16 -0
- data/lib/source_monitor/favicons/svg_converter.rb +60 -0
- data/lib/source_monitor/fetching/cloudflare_bypass.rb +79 -0
- data/lib/source_monitor/fetching/feed_fetcher/source_updater.rb +82 -2
- data/lib/source_monitor/fetching/feed_fetcher.rb +55 -1
- data/lib/source_monitor/fetching/fetch_error.rb +27 -0
- data/lib/source_monitor/fetching/fetch_runner.rb +4 -0
- data/lib/source_monitor/fetching/retry_policy.rb +4 -0
- data/lib/source_monitor/health/import_source_health_check.rb +3 -3
- data/lib/source_monitor/health/source_health_monitor.rb +9 -14
- data/lib/source_monitor/health/source_health_reset.rb +1 -1
- data/lib/source_monitor/pagination/paginator.rb +18 -1
- data/lib/source_monitor/version.rb +1 -1
- data/lib/source_monitor.rb +3 -0
- metadata +17 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
<div id="source_monitor_dashboard_fetch_schedule" class="space-y-
|
|
1
|
+
<div id="source_monitor_dashboard_fetch_schedule" class="space-y-6">
|
|
2
2
|
<div>
|
|
3
3
|
<h2 class="text-lg font-medium text-slate-900">Upcoming Fetch Schedule</h2>
|
|
4
4
|
<p class="mt-1 text-xs text-slate-500">
|
|
@@ -8,79 +8,105 @@
|
|
|
8
8
|
|
|
9
9
|
<% if groups.present? %>
|
|
10
10
|
<% groups.each do |group| %>
|
|
11
|
-
|
|
12
|
-
<div class="
|
|
13
|
-
<div>
|
|
14
|
-
<
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
11
|
+
<%= turbo_frame_tag "source_monitor_schedule_#{group.key}", class: "block" do %>
|
|
12
|
+
<div class="rounded-lg border border-slate-200 bg-white shadow-sm">
|
|
13
|
+
<div class="flex items-center justify-between gap-3 border-b border-slate-200 px-5 py-4">
|
|
14
|
+
<div>
|
|
15
|
+
<h3 class="text-sm font-semibold text-slate-900"><%= group.label %></h3>
|
|
16
|
+
<% if (window_label = fetch_schedule_window_label(group)).present? %>
|
|
17
|
+
<p class="mt-1 text-xs text-slate-500"><%= window_label %></p>
|
|
18
|
+
<% elsif group.include_unscheduled %>
|
|
19
|
+
<p class="mt-1 text-xs text-slate-500">Includes sources scheduled beyond four hours or without a next fetch timestamp.</p>
|
|
20
|
+
<% end %>
|
|
21
|
+
</div>
|
|
22
|
+
<span class="inline-flex items-center rounded-full bg-slate-100 px-3 py-1 text-xs font-semibold text-slate-600">
|
|
23
|
+
<%= pluralize(group.sources.size, "source") %>
|
|
24
|
+
</span>
|
|
20
25
|
</div>
|
|
21
|
-
<span class="inline-flex items-center rounded-full bg-slate-100 px-3 py-1 text-xs font-semibold text-slate-600">
|
|
22
|
-
<%= pluralize(group.sources.size, "source") %>
|
|
23
|
-
</span>
|
|
24
|
-
</div>
|
|
25
26
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
<tr>
|
|
31
|
-
<th scope="col" class="w-[45%] px-4 py-2">Source</th>
|
|
32
|
-
<th scope="col" class="w-[15%] px-4 py-2">Status</th>
|
|
33
|
-
<th scope="col" class="w-[22%] px-4 py-2">Next Fetch</th>
|
|
34
|
-
<th scope="col" class="w-[18%] px-4 py-2">Interval</th>
|
|
35
|
-
</tr>
|
|
36
|
-
</thead>
|
|
37
|
-
<tbody class="divide-y divide-slate-100 text-slate-700">
|
|
38
|
-
<% group.sources.each do |source| %>
|
|
39
|
-
<% badge = async_status_badge(source.fetch_status) %>
|
|
27
|
+
<% if group.sources.any? %>
|
|
28
|
+
<div class="overflow-x-auto px-5 py-4">
|
|
29
|
+
<table class="min-w-full table-fixed divide-y divide-slate-200 text-left text-sm">
|
|
30
|
+
<thead class="text-xs font-semibold uppercase tracking-wide text-slate-500">
|
|
40
31
|
<tr>
|
|
41
|
-
<
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
32
|
+
<th scope="col" class="w-[45%] px-4 py-2">Source</th>
|
|
33
|
+
<th scope="col" class="w-[15%] px-4 py-2">Status</th>
|
|
34
|
+
<th scope="col" class="w-[22%] px-4 py-2">Next Fetch</th>
|
|
35
|
+
<th scope="col" class="w-[18%] px-4 py-2">Interval</th>
|
|
36
|
+
</tr>
|
|
37
|
+
</thead>
|
|
38
|
+
<tbody class="divide-y divide-slate-100 text-slate-700">
|
|
39
|
+
<% group.sources.each do |source| %>
|
|
40
|
+
<% badge = async_status_badge(source.fetch_status) %>
|
|
41
|
+
<tr>
|
|
42
|
+
<td class="px-4 py-3">
|
|
43
|
+
<div class="flex flex-col">
|
|
44
|
+
<%= link_to source.name, source_monitor.source_path(source), class: "text-sm font-semibold text-blue-600 hover:text-blue-500", data: { turbo_frame: "_top" } %>
|
|
45
|
+
<span class="text-xs text-slate-400"><%= source.feed_url %></span>
|
|
46
|
+
</div>
|
|
47
|
+
</td>
|
|
48
|
+
<td class="px-4 py-3">
|
|
49
|
+
<span class="inline-flex items-center gap-1 rounded-full px-3 py-1 text-xs font-semibold <%= badge[:classes] %>">
|
|
50
|
+
<% if badge[:show_spinner] %>
|
|
51
|
+
<%= loading_spinner_svg(css_class: "h-3 w-3 animate-spin text-current") %>
|
|
52
|
+
<% end %>
|
|
53
|
+
<%= badge[:label] %>
|
|
54
|
+
</span>
|
|
55
|
+
</td>
|
|
56
|
+
<td class="px-4 py-3">
|
|
57
|
+
<% if source.next_fetch_at.present? %>
|
|
58
|
+
<div class="text-sm font-medium text-slate-900"><%= format_schedule_time(source.next_fetch_at) %></div>
|
|
59
|
+
<% if source.next_fetch_at.future? %>
|
|
60
|
+
<div class="text-xs text-slate-400">in <%= distance_of_time_in_words(reference_time, source.next_fetch_at) %></div>
|
|
61
|
+
<% else %>
|
|
62
|
+
<div class="text-xs text-rose-500"><%= distance_of_time_in_words(source.next_fetch_at, reference_time) %> overdue</div>
|
|
63
|
+
<% end %>
|
|
64
|
+
<% else %>
|
|
65
|
+
<div class="text-sm font-medium text-slate-500">Not scheduled</div>
|
|
51
66
|
<% end %>
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
<div class="text-sm font-medium text-slate-900"><%= format_schedule_time(source.next_fetch_at) %></div>
|
|
58
|
-
<% if source.next_fetch_at.future? %>
|
|
59
|
-
<div class="text-xs text-slate-400">in <%= distance_of_time_in_words(reference_time, source.next_fetch_at) %></div>
|
|
67
|
+
</td>
|
|
68
|
+
<td class="px-4 py-3">
|
|
69
|
+
<div class="text-sm font-medium text-slate-900"><%= human_fetch_interval(source.fetch_interval_minutes) %></div>
|
|
70
|
+
<% if source.respond_to?(:adaptive_fetching_enabled?) && source.adaptive_fetching_enabled? %>
|
|
71
|
+
<div class="text-xs text-slate-400">Adaptive</div>
|
|
60
72
|
<% else %>
|
|
61
|
-
<div class="text-xs text-
|
|
73
|
+
<div class="text-xs text-slate-400">Fixed</div>
|
|
62
74
|
<% end %>
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
75
|
+
</td>
|
|
76
|
+
</tr>
|
|
77
|
+
<% end %>
|
|
78
|
+
</tbody>
|
|
79
|
+
</table>
|
|
80
|
+
</div>
|
|
81
|
+
|
|
82
|
+
<% if group.has_previous_page || group.has_next_page %>
|
|
83
|
+
<div class="flex items-center justify-between border-t border-slate-200 px-5 py-3">
|
|
84
|
+
<div>
|
|
85
|
+
<% if group.has_previous_page %>
|
|
86
|
+
<%= link_to source_monitor.dashboard_path(schedule_pages: schedule_pages.merge(group.key => group.page - 1)),
|
|
87
|
+
class: "inline-flex items-center gap-1 rounded-md bg-white px-3 py-1.5 text-xs font-semibold text-slate-700 ring-1 ring-slate-300 hover:bg-slate-50",
|
|
88
|
+
data: { turbo_frame: "source_monitor_schedule_#{group.key}" } do %>
|
|
89
|
+
← Previous
|
|
90
|
+
<% end %>
|
|
91
|
+
<% end %>
|
|
92
|
+
</div>
|
|
93
|
+
<span class="text-xs text-slate-500">Page <%= group.page %></span>
|
|
94
|
+
<div>
|
|
95
|
+
<% if group.has_next_page %>
|
|
96
|
+
<%= link_to source_monitor.dashboard_path(schedule_pages: schedule_pages.merge(group.key => group.page + 1)),
|
|
97
|
+
class: "inline-flex items-center gap-1 rounded-md bg-white px-3 py-1.5 text-xs font-semibold text-slate-700 ring-1 ring-slate-300 hover:bg-slate-50",
|
|
98
|
+
data: { turbo_frame: "source_monitor_schedule_#{group.key}" } do %>
|
|
99
|
+
Next →
|
|
100
|
+
<% end %>
|
|
101
|
+
<% end %>
|
|
102
|
+
</div>
|
|
103
|
+
</div>
|
|
104
|
+
<% end %>
|
|
105
|
+
<% else %>
|
|
106
|
+
<div class="px-5 py-6 text-sm text-slate-500">No sources scheduled in this window.</div>
|
|
107
|
+
<% end %>
|
|
108
|
+
</div>
|
|
109
|
+
<% end %>
|
|
84
110
|
<% end %>
|
|
85
111
|
<% else %>
|
|
86
112
|
<div class="rounded-lg border border-dashed border-slate-200 bg-white px-5 py-6 text-sm text-slate-500 shadow-sm">
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
<div class="rounded-lg border border-slate-200 bg-white shadow-sm">
|
|
2
|
+
<div class="border-b border-slate-200 px-5 py-4">
|
|
3
|
+
<h2 class="text-lg font-medium">Scrape Recommendations</h2>
|
|
4
|
+
<p class="mt-1 text-xs text-slate-500">Sources with low feed word counts.</p>
|
|
5
|
+
</div>
|
|
6
|
+
<div class="px-5 py-4">
|
|
7
|
+
<p class="text-3xl font-bold text-blue-600"><%= count %></p>
|
|
8
|
+
<p class="mt-1 text-sm text-slate-600">
|
|
9
|
+
<%= "source".pluralize(count) %> could benefit from scraping
|
|
10
|
+
</p>
|
|
11
|
+
<div class="mt-4">
|
|
12
|
+
<%= link_to "View Candidates",
|
|
13
|
+
source_monitor.sources_path(q: { avg_feed_words_lt: threshold, scraping_enabled_eq: false, active_eq: true }),
|
|
14
|
+
class: "inline-flex items-center rounded-md bg-blue-600 px-3 py-1 text-xs font-semibold text-white shadow hover:bg-blue-500" %>
|
|
15
|
+
</div>
|
|
16
|
+
</div>
|
|
17
|
+
</div>
|
|
@@ -7,3 +7,22 @@
|
|
|
7
7
|
{ label: "Fetches Today", value: stats[:fetches_today], caption: "Completed runs" }
|
|
8
8
|
], locals: { stats: stats } %>
|
|
9
9
|
</div>
|
|
10
|
+
|
|
11
|
+
<% if stats[:health_distribution]&.values&.any? { |v| v > 0 } %>
|
|
12
|
+
<div id="source_monitor_dashboard_health_distribution" class="mt-4 flex flex-wrap items-center gap-2">
|
|
13
|
+
<%
|
|
14
|
+
health_colors = {
|
|
15
|
+
"working" => "bg-green-100 text-green-700",
|
|
16
|
+
"declining" => "bg-yellow-100 text-yellow-700",
|
|
17
|
+
"improving" => "bg-sky-100 text-sky-700",
|
|
18
|
+
"failing" => "bg-rose-100 text-rose-700"
|
|
19
|
+
}
|
|
20
|
+
%>
|
|
21
|
+
<% stats[:health_distribution].each do |status, count| %>
|
|
22
|
+
<% next if count == 0 %>
|
|
23
|
+
<span class="inline-flex items-center gap-1 rounded-full px-3 py-1 text-xs font-semibold <%= health_colors[status] %>">
|
|
24
|
+
<%= status.capitalize %> <span class="font-bold"><%= count %></span>
|
|
25
|
+
</span>
|
|
26
|
+
<% end %>
|
|
27
|
+
</div>
|
|
28
|
+
<% end %>
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
<section class="grid gap-6 lg:grid-cols-3">
|
|
15
15
|
<div class="lg:col-span-2 space-y-6">
|
|
16
16
|
<%= render "recent_activity", recent_activity: @recent_activity %>
|
|
17
|
-
<%= render "fetch_schedule", groups: @fetch_schedule_groups, reference_time: @fetch_schedule_reference_time %>
|
|
17
|
+
<%= render "fetch_schedule", groups: @fetch_schedule_groups, reference_time: @fetch_schedule_reference_time, schedule_pages: @schedule_pages %>
|
|
18
18
|
</div>
|
|
19
19
|
|
|
20
20
|
<div class="space-y-6">
|
|
@@ -24,6 +24,12 @@
|
|
|
24
24
|
mission_control_enabled: @mission_control_enabled,
|
|
25
25
|
mission_control_path: @mission_control_dashboard_path %>
|
|
26
26
|
|
|
27
|
+
<% if @scrape_candidates_count.to_i > 0 %>
|
|
28
|
+
<%= render "scrape_recommendations",
|
|
29
|
+
count: @scrape_candidates_count,
|
|
30
|
+
threshold: @scrape_recommendation_threshold %>
|
|
31
|
+
<% end %>
|
|
32
|
+
|
|
27
33
|
<div class="rounded-lg border border-slate-200 bg-white shadow-sm">
|
|
28
34
|
<div class="border-b border-slate-200 px-5 py-4">
|
|
29
35
|
<h2 class="text-lg font-medium">Quick Actions</h2>
|
|
@@ -19,10 +19,10 @@
|
|
|
19
19
|
<td class="px-4 py-3 align-top">
|
|
20
20
|
<% status = entry[:health_status].presence || "pending" %>
|
|
21
21
|
<% case status %>
|
|
22
|
-
<% when "
|
|
22
|
+
<% when "working" %>
|
|
23
23
|
<span class="inline-flex items-center gap-2 rounded-full bg-green-100 px-2 py-1 text-xs font-semibold text-green-800">
|
|
24
24
|
<span class="inline-block h-2 w-2 rounded-full bg-green-500" aria-hidden="true"></span>
|
|
25
|
-
|
|
25
|
+
Working
|
|
26
26
|
</span>
|
|
27
27
|
<% when "unhealthy" %>
|
|
28
28
|
<div class="space-y-1">
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
<%# Shared pagination partial
|
|
2
|
+
Locals:
|
|
3
|
+
paginator_result - Pagination::Result with total_count/total_pages
|
|
4
|
+
base_path - URL path (e.g., source_monitor.sources_path)
|
|
5
|
+
extra_params - Hash of additional query params to preserve (default: {})
|
|
6
|
+
turbo_frame - Optional Turbo Frame target (default: nil)
|
|
7
|
+
%>
|
|
8
|
+
<% extra_params ||= {} %>
|
|
9
|
+
<% turbo_frame ||= nil %>
|
|
10
|
+
<% current_page = paginator_result.page %>
|
|
11
|
+
<% total_pages = paginator_result.total_pages %>
|
|
12
|
+
<% frame_data = turbo_frame ? { turbo_frame: turbo_frame } : {} %>
|
|
13
|
+
<% pagination_url = ->(pg) { "#{base_path}?#{extra_params.merge(page: pg).to_query}" } %>
|
|
14
|
+
|
|
15
|
+
<div class="flex flex-col items-center gap-3 border-t border-slate-200 px-6 py-4 sm:flex-row sm:justify-between">
|
|
16
|
+
<div class="text-xs text-slate-500">
|
|
17
|
+
Page <%= current_page %> of <%= total_pages %>
|
|
18
|
+
</div>
|
|
19
|
+
|
|
20
|
+
<% if total_pages > 1 %>
|
|
21
|
+
<div class="flex items-center gap-2">
|
|
22
|
+
<%# Previous button %>
|
|
23
|
+
<% if paginator_result.has_previous_page? %>
|
|
24
|
+
<%= link_to "Previous", pagination_url.call(paginator_result.previous_page),
|
|
25
|
+
class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50",
|
|
26
|
+
data: frame_data %>
|
|
27
|
+
<% else %>
|
|
28
|
+
<span class="inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-300">Previous</span>
|
|
29
|
+
<% end %>
|
|
30
|
+
|
|
31
|
+
<%# Page numbers %>
|
|
32
|
+
<% pagination_page_numbers(current_page: current_page, total_pages: total_pages).each do |page_item| %>
|
|
33
|
+
<% if page_item == :gap %>
|
|
34
|
+
<span class="px-1 text-sm text-slate-400">…</span>
|
|
35
|
+
<% elsif page_item == current_page %>
|
|
36
|
+
<span class="inline-flex items-center rounded-md bg-blue-600 px-3 py-2 text-sm font-medium text-white"><%= page_item %></span>
|
|
37
|
+
<% else %>
|
|
38
|
+
<%= link_to page_item, pagination_url.call(page_item),
|
|
39
|
+
class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50",
|
|
40
|
+
data: frame_data %>
|
|
41
|
+
<% end %>
|
|
42
|
+
<% end %>
|
|
43
|
+
|
|
44
|
+
<%# Next button %>
|
|
45
|
+
<% if paginator_result.has_next_page? %>
|
|
46
|
+
<%= link_to "Next", pagination_url.call(paginator_result.next_page),
|
|
47
|
+
class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50",
|
|
48
|
+
data: frame_data %>
|
|
49
|
+
<% else %>
|
|
50
|
+
<span class="inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-300">Next</span>
|
|
51
|
+
<% end %>
|
|
52
|
+
|
|
53
|
+
<%# Jump-to-page form %>
|
|
54
|
+
<%= form_with url: base_path, method: :get, class: "ml-2 flex items-center gap-1", data: frame_data do |f| %>
|
|
55
|
+
<% extra_params.each do |key, value| %>
|
|
56
|
+
<% if value.is_a?(Hash) %>
|
|
57
|
+
<% value.each do |sub_key, sub_value| %>
|
|
58
|
+
<%= hidden_field_tag "#{key}[#{sub_key}]", sub_value %>
|
|
59
|
+
<% end %>
|
|
60
|
+
<% else %>
|
|
61
|
+
<%= hidden_field_tag key, value %>
|
|
62
|
+
<% end %>
|
|
63
|
+
<% end %>
|
|
64
|
+
<%= number_field_tag :page, nil,
|
|
65
|
+
min: 1,
|
|
66
|
+
max: total_pages,
|
|
67
|
+
placeholder: "#",
|
|
68
|
+
class: "w-14 rounded-md border border-slate-300 px-2 py-1.5 text-sm text-center focus:border-blue-500 focus:ring-1 focus:ring-blue-500",
|
|
69
|
+
"aria-label": "Jump to page" %>
|
|
70
|
+
<%= f.submit "Go", class: "rounded-md border border-slate-300 px-2 py-1.5 text-sm font-medium text-slate-700 hover:bg-slate-50 cursor-pointer" %>
|
|
71
|
+
<% end %>
|
|
72
|
+
</div>
|
|
73
|
+
<% end %>
|
|
74
|
+
</div>
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
<div id="scrape_test_modal_<%= source.id %>"
|
|
2
|
+
data-controller="modal"
|
|
3
|
+
data-modal-open-class="flex"
|
|
4
|
+
data-modal-auto-open-value="true"
|
|
5
|
+
data-modal-remove-on-close-value="true">
|
|
6
|
+
<div
|
|
7
|
+
data-modal-target="panel"
|
|
8
|
+
class="hidden fixed inset-0 z-50 items-center justify-center"
|
|
9
|
+
data-testid="scrape-test-modal"
|
|
10
|
+
>
|
|
11
|
+
<div class="absolute inset-0 bg-slate-900/40" data-action="click->modal#close"></div>
|
|
12
|
+
<div class="relative z-10 w-full max-w-lg overflow-hidden rounded-xl bg-white shadow-xl">
|
|
13
|
+
<div class="flex items-start justify-between border-b border-slate-200 px-6 py-4">
|
|
14
|
+
<div>
|
|
15
|
+
<h2 class="text-lg font-semibold text-slate-900">Scrape Test Result</h2>
|
|
16
|
+
<p class="mt-1 text-xs text-slate-500">
|
|
17
|
+
Tested item: "<%= test_result[:item].title.to_s.truncate(60) %>"
|
|
18
|
+
</p>
|
|
19
|
+
</div>
|
|
20
|
+
<button
|
|
21
|
+
type="button"
|
|
22
|
+
class="inline-flex h-8 w-8 items-center justify-center rounded-full text-slate-500 hover:bg-slate-100 hover:text-slate-700"
|
|
23
|
+
data-action="modal#close"
|
|
24
|
+
aria-label="Close scrape test modal"
|
|
25
|
+
>
|
|
26
|
+
<span aria-hidden="true">×</span>
|
|
27
|
+
</button>
|
|
28
|
+
</div>
|
|
29
|
+
|
|
30
|
+
<div class="px-6 py-5">
|
|
31
|
+
<div class="grid grid-cols-2 gap-6">
|
|
32
|
+
<div>
|
|
33
|
+
<dt class="text-xs font-medium uppercase tracking-wide text-slate-500">Feed Word Count</dt>
|
|
34
|
+
<dd class="mt-1 text-2xl font-semibold text-slate-900"><%= test_result[:feed_word_count] || "N/A" %></dd>
|
|
35
|
+
</div>
|
|
36
|
+
<div>
|
|
37
|
+
<dt class="text-xs font-medium uppercase tracking-wide text-slate-500">Scraped Word Count</dt>
|
|
38
|
+
<dd class="mt-1 text-2xl font-semibold text-slate-900"><%= test_result[:scraped_word_count] || "N/A" %></dd>
|
|
39
|
+
</div>
|
|
40
|
+
</div>
|
|
41
|
+
|
|
42
|
+
<% if test_result[:improvement] && test_result[:improvement] != 0 %>
|
|
43
|
+
<div class="mt-4">
|
|
44
|
+
<% color = test_result[:improvement] > 0 ? "text-green-600" : "text-amber-600" %>
|
|
45
|
+
<span class="text-sm font-medium <%= color %>">
|
|
46
|
+
<%= test_result[:improvement] > 0 ? "+" : "" %><%= test_result[:improvement] %>% word count change
|
|
47
|
+
</span>
|
|
48
|
+
</div>
|
|
49
|
+
<% end %>
|
|
50
|
+
|
|
51
|
+
<% if test_result[:scrape_result]&.success? %>
|
|
52
|
+
<div class="mt-4 rounded-md bg-green-50 px-3 py-2 text-sm text-green-700">
|
|
53
|
+
Scrape successful. Enabling scraping for this source would capture more content.
|
|
54
|
+
</div>
|
|
55
|
+
<% else %>
|
|
56
|
+
<div class="mt-4 rounded-md bg-amber-50 px-3 py-2 text-sm text-amber-700">
|
|
57
|
+
Scrape had issues: <%= test_result[:scrape_result]&.message || "Unknown error" %>
|
|
58
|
+
</div>
|
|
59
|
+
<% end %>
|
|
60
|
+
</div>
|
|
61
|
+
|
|
62
|
+
<div class="flex items-center justify-end gap-3 border-t border-slate-200 px-6 py-4">
|
|
63
|
+
<button
|
|
64
|
+
type="button"
|
|
65
|
+
class="inline-flex items-center rounded-md border border-slate-300 bg-white px-4 py-2 text-sm font-medium text-slate-700 shadow-sm hover:bg-slate-50"
|
|
66
|
+
data-action="modal#close"
|
|
67
|
+
>
|
|
68
|
+
Cancel
|
|
69
|
+
</button>
|
|
70
|
+
<% unless source.scraping_enabled? %>
|
|
71
|
+
<%= form_with url: source_monitor.source_path(source), method: :patch, data: { turbo_frame: "_top" } do |f| %>
|
|
72
|
+
<%= f.hidden_field :scraping_enabled, value: "true", name: "source[scraping_enabled]" %>
|
|
73
|
+
<%= f.hidden_field :auto_scrape, value: "true", name: "source[auto_scrape]" %>
|
|
74
|
+
<%= f.submit "Enable Auto-Scraping",
|
|
75
|
+
class: "inline-flex items-center rounded-md bg-violet-600 px-4 py-2 text-sm font-medium text-white shadow-sm hover:bg-violet-500 focus:outline-none focus:ring-2 focus:ring-violet-500 focus:ring-offset-2 cursor-pointer" %>
|
|
76
|
+
<% end %>
|
|
77
|
+
<% end %>
|
|
78
|
+
</div>
|
|
79
|
+
</div>
|
|
80
|
+
</div>
|
|
81
|
+
</div>
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
<div class="mx-auto max-w-3xl space-y-6 py-8">
|
|
2
|
+
<div class="flex items-center justify-between">
|
|
3
|
+
<h1 class="text-2xl font-semibold text-slate-900">Scrape Test Result</h1>
|
|
4
|
+
<%= link_to "Back to Source", source_monitor.source_path(@source), class: "text-sm font-medium text-blue-600 hover:text-blue-500" %>
|
|
5
|
+
</div>
|
|
6
|
+
|
|
7
|
+
<% if @test_result %>
|
|
8
|
+
<div class="rounded-xl border border-slate-200 bg-white shadow-sm">
|
|
9
|
+
<div class="border-b border-slate-200 px-6 py-4">
|
|
10
|
+
<h2 class="text-lg font-semibold text-slate-900">Scrape Test Result</h2>
|
|
11
|
+
<p class="mt-1 text-xs text-slate-500">
|
|
12
|
+
Tested item: "<%= @test_result[:item].title.to_s.truncate(60) %>"
|
|
13
|
+
</p>
|
|
14
|
+
</div>
|
|
15
|
+
<div class="px-6 py-5">
|
|
16
|
+
<div class="grid grid-cols-2 gap-6">
|
|
17
|
+
<div>
|
|
18
|
+
<dt class="text-xs font-medium uppercase tracking-wide text-slate-500">Feed Word Count</dt>
|
|
19
|
+
<dd class="mt-1 text-2xl font-semibold text-slate-900"><%= @test_result[:feed_word_count] || "N/A" %></dd>
|
|
20
|
+
</div>
|
|
21
|
+
<div>
|
|
22
|
+
<dt class="text-xs font-medium uppercase tracking-wide text-slate-500">Scraped Word Count</dt>
|
|
23
|
+
<dd class="mt-1 text-2xl font-semibold text-slate-900"><%= @test_result[:scraped_word_count] || "N/A" %></dd>
|
|
24
|
+
</div>
|
|
25
|
+
</div>
|
|
26
|
+
|
|
27
|
+
<% if @test_result[:improvement] && @test_result[:improvement] != 0 %>
|
|
28
|
+
<div class="mt-4">
|
|
29
|
+
<% color = @test_result[:improvement] > 0 ? "text-green-600" : "text-amber-600" %>
|
|
30
|
+
<span class="text-sm font-medium <%= color %>">
|
|
31
|
+
<%= @test_result[:improvement] > 0 ? "+" : "" %><%= @test_result[:improvement] %>% word count change
|
|
32
|
+
</span>
|
|
33
|
+
</div>
|
|
34
|
+
<% end %>
|
|
35
|
+
|
|
36
|
+
<% if @test_result[:scrape_result]&.success? %>
|
|
37
|
+
<div class="mt-4 rounded-md bg-green-50 px-3 py-2 text-sm text-green-700">
|
|
38
|
+
Scrape successful. Enabling scraping for this source would capture more content.
|
|
39
|
+
</div>
|
|
40
|
+
<% else %>
|
|
41
|
+
<div class="mt-4 rounded-md bg-amber-50 px-3 py-2 text-sm text-amber-700">
|
|
42
|
+
Scrape had issues: <%= @test_result[:scrape_result]&.message || "Unknown error" %>
|
|
43
|
+
</div>
|
|
44
|
+
<% end %>
|
|
45
|
+
</div>
|
|
46
|
+
|
|
47
|
+
<% unless @source.scraping_enabled? %>
|
|
48
|
+
<div class="flex items-center justify-end gap-3 border-t border-slate-200 px-6 py-4">
|
|
49
|
+
<%= form_with url: source_monitor.source_path(@source), method: :patch do |f| %>
|
|
50
|
+
<%= f.hidden_field :scraping_enabled, value: "true", name: "source[scraping_enabled]" %>
|
|
51
|
+
<%= f.submit "Enable Auto-Scraping",
|
|
52
|
+
class: "inline-flex items-center rounded-md bg-violet-600 px-4 py-2 text-sm font-medium text-white shadow-sm hover:bg-violet-500 cursor-pointer" %>
|
|
53
|
+
<% end %>
|
|
54
|
+
</div>
|
|
55
|
+
<% end %>
|
|
56
|
+
</div>
|
|
57
|
+
<% else %>
|
|
58
|
+
<p class="text-sm text-slate-500">No test result available.</p>
|
|
59
|
+
<% end %>
|
|
60
|
+
</div>
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
<div data-controller="modal" class="relative">
|
|
2
|
+
<div data-modal-target="panel" class="hidden fixed inset-0 z-50 flex items-center justify-center bg-black/50" data-action="click->modal#backdrop">
|
|
3
|
+
<div class="w-full max-w-md rounded-lg bg-white shadow-xl" data-action="click->modal#stop">
|
|
4
|
+
<div class="border-b border-slate-200 px-6 py-4">
|
|
5
|
+
<h3 class="text-lg font-semibold text-slate-900">Enable Scraping</h3>
|
|
6
|
+
</div>
|
|
7
|
+
<div class="px-6 py-4">
|
|
8
|
+
<p class="text-sm text-slate-700">
|
|
9
|
+
This will enable scraping for the selected sources using the default scraper adapter.
|
|
10
|
+
Each source's items will be scraped on their next scheduled run.
|
|
11
|
+
</p>
|
|
12
|
+
<p class="mt-3 text-sm font-medium text-amber-700">
|
|
13
|
+
This action will modify the selected sources' configuration.
|
|
14
|
+
</p>
|
|
15
|
+
</div>
|
|
16
|
+
<div class="flex justify-end gap-3 border-t border-slate-200 px-6 py-4">
|
|
17
|
+
<button type="button"
|
|
18
|
+
data-action="modal#close"
|
|
19
|
+
class="rounded-md border border-slate-200 px-4 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50">
|
|
20
|
+
Cancel
|
|
21
|
+
</button>
|
|
22
|
+
<button type="submit"
|
|
23
|
+
class="rounded-md bg-violet-600 px-4 py-2 text-sm font-semibold text-white shadow hover:bg-violet-500">
|
|
24
|
+
Confirm Enable
|
|
25
|
+
</button>
|
|
26
|
+
</div>
|
|
27
|
+
</div>
|
|
28
|
+
</div>
|
|
29
|
+
</div>
|
|
@@ -40,7 +40,17 @@
|
|
|
40
40
|
</span>
|
|
41
41
|
</div>
|
|
42
42
|
<div>
|
|
43
|
-
<
|
|
43
|
+
<div class="flex items-center gap-2">
|
|
44
|
+
<h1 class="text-3xl font-semibold text-slate-900"><%= source.name %></h1>
|
|
45
|
+
<% if source.last_error.to_s.match?(/\bblocked\b/i) %>
|
|
46
|
+
<% blocked_detail = source.last_error.to_s.match(/blocked by (\w+)/i) %>
|
|
47
|
+
<span class="inline-flex items-center rounded-full bg-rose-100 px-3 py-1 text-xs font-semibold text-rose-700"
|
|
48
|
+
title="<%= blocked_detail ? "#{blocked_detail[1].capitalize} Blocked" : "Blocked" %>"
|
|
49
|
+
data-testid="source-blocked-badge">
|
|
50
|
+
Blocked
|
|
51
|
+
</span>
|
|
52
|
+
<% end %>
|
|
53
|
+
</div>
|
|
44
54
|
<div class="mt-2 flex flex-wrap items-center gap-2 text-xs">
|
|
45
55
|
<% if source.last_fetch_started_at.present? %>
|
|
46
56
|
<span class="rounded-full bg-slate-100 px-3 py-1 font-semibold text-slate-600">
|
|
@@ -98,6 +108,14 @@
|
|
|
98
108
|
}
|
|
99
109
|
} %>
|
|
100
110
|
<% end %>
|
|
111
|
+
<% unless source.scraping_enabled? %>
|
|
112
|
+
<%= button_to "Test Scrape",
|
|
113
|
+
source_monitor.source_scrape_test_path(source),
|
|
114
|
+
method: :post,
|
|
115
|
+
class: "inline-flex items-center rounded-md border border-violet-200 bg-violet-50 px-3 py-1.5 text-sm font-medium text-violet-700 hover:bg-violet-100",
|
|
116
|
+
data: { turbo_stream: true },
|
|
117
|
+
title: "Scrape a recent item to compare feed vs scraped word count" %>
|
|
118
|
+
<% end %>
|
|
101
119
|
<%= link_to "Edit", source_monitor.edit_source_path(source), class: "inline-flex items-center rounded-md bg-slate-800 px-3 py-2 text-sm font-medium text-white shadow hover:bg-slate-700" %>
|
|
102
120
|
<%= form_with url: source_monitor.source_path(source), method: :delete, data: { turbo_confirm: "Delete this source and remove all associated data?", turbo_frame: "_top" }, class: "inline" do %>
|
|
103
121
|
<%= hidden_field_tag :redirect_to, source_monitor.sources_path %>
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<tr id="source_monitor_sources_empty_state">
|
|
2
|
-
<td colspan="
|
|
2
|
+
<td colspan="10" class="px-6 py-6 text-center text-sm text-slate-500">
|
|
3
3
|
No sources yet. <%= link_to "Create the first source", source_monitor.new_source_path, class: "text-blue-600 hover:text-blue-500" %>.
|
|
4
4
|
</td>
|
|
5
5
|
</tr>
|
|
@@ -12,11 +12,18 @@
|
|
|
12
12
|
Completed <%= time_ago_in_words(latest.completed_at || latest.created_at) %> ago.
|
|
13
13
|
</p>
|
|
14
14
|
</div>
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
<
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
<div class="flex items-start gap-3">
|
|
16
|
+
<% if latest.completed? && latest.duration_ms %>
|
|
17
|
+
<div class="text-right text-sm text-slate-600">
|
|
18
|
+
<p>Duration: <%= latest.duration_ms %> ms</p>
|
|
19
|
+
</div>
|
|
20
|
+
<% end %>
|
|
21
|
+
<%= button_to import_history_dismissal_path(import_history_id: latest.id), method: :post, data: { turbo_stream: true }, class: "inline-flex items-center justify-center rounded-md p-1 text-blue-400 hover:bg-blue-100 hover:text-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500", title: "Dismiss" do %>
|
|
22
|
+
<svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4" viewBox="0 0 20 20" fill="currentColor">
|
|
23
|
+
<path fill-rule="evenodd" d="M4.293 4.293a1 1 0 011.414 0L10 8.586l4.293-4.293a1 1 0 111.414 1.414L11.414 10l4.293 4.293a1 1 0 01-1.414 1.414L10 11.414l-4.293 4.293a1 1 0 01-1.414-1.414L8.586 10 4.293 5.707a1 1 0 010-1.414z" clip-rule="evenodd" />
|
|
24
|
+
</svg>
|
|
25
|
+
<% end %>
|
|
26
|
+
</div>
|
|
20
27
|
</div>
|
|
21
28
|
|
|
22
29
|
<% if latest.failed_sources.present? %>
|