html2rss 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +41 -18
- data/html2rss.gemspec +1 -1
- data/lib/html2rss/auto_source/README.md +57 -0
- data/lib/html2rss/auto_source/cleanup.rb +121 -56
- data/lib/html2rss/auto_source/scraper.rb +13 -0
- data/lib/html2rss/auto_source.rb +34 -8
- data/lib/html2rss/capture/README.md +61 -0
- data/lib/html2rss/capture.rb +120 -117
- data/lib/html2rss/cli.rb +35 -17
- data/lib/html2rss/config/schema.rb +12 -0
- data/lib/html2rss/config/validator.rb +31 -8
- data/lib/html2rss/config.rb +28 -0
- data/lib/html2rss/error.rb +24 -6
- data/lib/html2rss/feed_pipeline/README.md +42 -0
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +22 -9
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +11 -0
- data/lib/html2rss/feed_pipeline.rb +30 -12
- data/lib/html2rss/html/article_extractor/category_extractor.rb +36 -22
- data/lib/html2rss/html/article_extractor/date_extractor.rb +5 -3
- data/lib/html2rss/html/article_extractor.rb +95 -17
- data/lib/html2rss/html/article_rules/category.rb +28 -11
- data/lib/html2rss/html/article_rules/date.rb +60 -6
- data/lib/html2rss/html/article_rules/description.rb +122 -0
- data/lib/html2rss/html/card_walk.rb +42 -0
- data/lib/html2rss/html/feed_link.rb +34 -0
- data/lib/html2rss/html/navigator.rb +18 -0
- data/lib/html2rss/html/sst_article_extractor.rb +119 -32
- data/lib/html2rss/link_destination/path_classifier.rb +49 -35
- data/lib/html2rss/mcp/config_argument.rb +42 -0
- data/lib/html2rss/mcp/contract.rb +173 -0
- data/lib/html2rss/mcp/inspect.rb +241 -0
- data/lib/html2rss/mcp/outcome.rb +188 -0
- data/lib/html2rss/mcp/server.rb +253 -409
- data/lib/html2rss/request_service/botasaurus_contract.rb +193 -102
- data/lib/html2rss/request_service/botasaurus_strategy.rb +15 -8
- data/lib/html2rss/request_service/compressed_body.rb +109 -0
- data/lib/html2rss/request_service/faraday_strategy.rb +3 -1
- data/lib/html2rss/request_service/policy.rb +1 -1
- data/lib/html2rss/request_service/response.rb +57 -6
- data/lib/html2rss/request_service.rb +6 -1
- data/lib/html2rss/selectors.rb +2 -1
- data/lib/html2rss/status.rb +27 -11
- data/lib/html2rss/url.rb +20 -0
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +30 -6
- data/schema/html2rss-config.schema.json +26 -15
- metadata +15 -4
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
module MCP
|
|
5
|
+
Outcome = Data.define(:ok, :next_step, :guidance, :payload)
|
|
6
|
+
|
|
7
|
+
##
|
|
8
|
+
# Typed MCP tool result. Owns next-step policy and guidance copy so the
|
|
9
|
+
# protocol adapter does not branch on quality heuristics.
|
|
10
|
+
class Outcome
|
|
11
|
+
# Matches {ConfigArgument} XOR {ArgumentError} messages.
|
|
12
|
+
XOR_ERROR = /exactly one of config or yaml/
|
|
13
|
+
NextStep = Data.define(:name, :guidance)
|
|
14
|
+
|
|
15
|
+
##
|
|
16
|
+
# Closed set of agent next actions. Invalid names cannot be constructed.
|
|
17
|
+
class NextStep
|
|
18
|
+
# Wire names for +next_step+.
|
|
19
|
+
NAMES = %i[done inspect_url validate_config apply_config scrape_url capture_config read_runtime].freeze
|
|
20
|
+
# Default guidance copy keyed by {NAMES}.
|
|
21
|
+
GUIDANCE = {
|
|
22
|
+
done: 'Done. Read payload for the result.',
|
|
23
|
+
inspect_url: 'Call inspect_url next. Read payload for recon (final_url, status, ' \
|
|
24
|
+
'scheme_downgrade, alternate_feeds).',
|
|
25
|
+
validate_config: 'Call validate_config with payload.yaml or a config hash (XOR, not both).',
|
|
26
|
+
apply_config: 'Call apply_config next. Confirm payload.item_count before shipping.',
|
|
27
|
+
scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus.',
|
|
28
|
+
capture_config: 'Call capture_config for a reusable YAML draft, then follow next_step.',
|
|
29
|
+
read_runtime: 'Read html2rss://runtime. Set BOTASAURUS_SCRAPER_URL on the MCP process ' \
|
|
30
|
+
'if botasaurus_configured is false.'
|
|
31
|
+
}.freeze
|
|
32
|
+
|
|
33
|
+
##
|
|
34
|
+
# @param name [Symbol, String]
|
|
35
|
+
# @param guidance [String, nil]
|
|
36
|
+
def initialize(name:, guidance: nil)
|
|
37
|
+
step = name.to_sym
|
|
38
|
+
raise ArgumentError, "unknown next_step: #{name.inspect}" unless NAMES.include?(step)
|
|
39
|
+
|
|
40
|
+
super(name: step, guidance: (guidance || GUIDANCE.fetch(step)).freeze)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
class << self
|
|
44
|
+
NAMES.each { |step| define_method(step) { new(name: step) } }
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
##
|
|
49
|
+
# @param ok [Boolean]
|
|
50
|
+
# @param next_step [NextStep]
|
|
51
|
+
# @param guidance [String]
|
|
52
|
+
# @param payload [Hash]
|
|
53
|
+
def initialize(ok:, next_step:, guidance:, payload:) # rubocop:disable Naming/MethodParameterName -- +ok+ is the envelope field
|
|
54
|
+
raise ArgumentError, 'next_step must be a NextStep' unless next_step.is_a?(NextStep)
|
|
55
|
+
raise ArgumentError, 'payload must be a Hash' unless payload.is_a?(Hash)
|
|
56
|
+
|
|
57
|
+
super(ok: !!ok, next_step:, guidance: guidance.to_s.freeze, payload: payload.dup.freeze)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
##
|
|
61
|
+
# @return [Hash{Symbol => Object}] envelope for {Contract.response}
|
|
62
|
+
def to_h
|
|
63
|
+
{ ok:, next_step: next_step.name.to_s, guidance:, payload: }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
class << self
|
|
67
|
+
##
|
|
68
|
+
# @param items [Array]
|
|
69
|
+
# @param requested_strategy [String, Symbol]
|
|
70
|
+
# @param channel_title [String, nil]
|
|
71
|
+
# @param admission_drops [Hash]
|
|
72
|
+
# @param botasaurus_configured [Boolean]
|
|
73
|
+
# @return [Outcome]
|
|
74
|
+
def scrape(items:, requested_strategy:, channel_title:, botasaurus_configured:, admission_drops: {})
|
|
75
|
+
next_step = scrape_next_step(items.empty?, botasaurus_configured:)
|
|
76
|
+
new(ok: true, next_step:, guidance: next_step.guidance,
|
|
77
|
+
payload: scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:))
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
##
|
|
81
|
+
# @param payload [Hash] inspect recon Hash
|
|
82
|
+
# @return [Outcome]
|
|
83
|
+
def inspect(payload:)
|
|
84
|
+
next_step = inspect_next_step(payload)
|
|
85
|
+
new(ok: true, next_step:, guidance: next_step.guidance, payload:)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
##
|
|
89
|
+
# @param yaml [String]
|
|
90
|
+
# @param articles_count [Integer]
|
|
91
|
+
# @param has_selectors [Boolean]
|
|
92
|
+
# @param channel_title [String, nil]
|
|
93
|
+
# @param requested_strategy [String, Symbol]
|
|
94
|
+
# @param segment_strategy [Symbol, String, nil]
|
|
95
|
+
# @param selected_strategy [Symbol, String, nil]
|
|
96
|
+
# @param admission_drops [Hash]
|
|
97
|
+
# @return [Outcome]
|
|
98
|
+
def capture(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
|
|
99
|
+
segment_strategy: nil, selected_strategy: nil, admission_drops: {})
|
|
100
|
+
next_step = capture_next_step(articles_count:, has_selectors:)
|
|
101
|
+
new(ok: true, next_step:, guidance: next_step.guidance, payload: capture_payload(
|
|
102
|
+
yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:,
|
|
103
|
+
segment_strategy:, selected_strategy:, admission_drops:
|
|
104
|
+
))
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
##
|
|
108
|
+
# @param errors [Hash, nil] schema errors; +nil+ means success
|
|
109
|
+
# @return [Outcome]
|
|
110
|
+
def validate(errors:)
|
|
111
|
+
ok = errors.nil?
|
|
112
|
+
next_step = ok ? NextStep.apply_config : NextStep.validate_config
|
|
113
|
+
new(ok:, next_step:, guidance: next_step.guidance, payload: ok ? {} : { errors: })
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
##
|
|
117
|
+
# @param rss [String]
|
|
118
|
+
# @param item_count [Integer]
|
|
119
|
+
# @param empty [Boolean] {FeedResult#empty?} (ship gate); defaults to zero items
|
|
120
|
+
# @return [Outcome]
|
|
121
|
+
def apply(rss:, item_count:, empty: item_count.zero?)
|
|
122
|
+
ok = !empty
|
|
123
|
+
next_step = ok ? NextStep.done : NextStep.inspect_url
|
|
124
|
+
new(ok:, next_step:, guidance: next_step.guidance, payload: { rss:, item_count: })
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
##
|
|
128
|
+
# @param error [Exception]
|
|
129
|
+
# @return [Outcome]
|
|
130
|
+
def from_error(error)
|
|
131
|
+
next_step = next_step_for_error(error)
|
|
132
|
+
new(ok: false, next_step:, guidance: next_step.guidance,
|
|
133
|
+
payload: { class: error.class.name, message: error.message })
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
private
|
|
137
|
+
|
|
138
|
+
def scrape_next_step(empty, botasaurus_configured:)
|
|
139
|
+
return NextStep.done unless empty
|
|
140
|
+
return NextStep.read_runtime unless botasaurus_configured
|
|
141
|
+
|
|
142
|
+
NextStep.inspect_url
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:)
|
|
146
|
+
{
|
|
147
|
+
items:, total: items.size, requested_strategy: requested_strategy.to_s, channel_title:,
|
|
148
|
+
**(admission_drops.any? ? { admission_drops: } : {})
|
|
149
|
+
}
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def inspect_next_step(payload)
|
|
153
|
+
return NextStep.done if Array(payload[:alternate_feeds]).any?
|
|
154
|
+
return NextStep.capture_config if payload[:articles_count].to_i.positive?
|
|
155
|
+
|
|
156
|
+
NextStep.scrape_url
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def capture_next_step(articles_count:, has_selectors:)
|
|
160
|
+
articles_count.positive? && has_selectors ? NextStep.validate_config : NextStep.inspect_url
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
|
|
164
|
+
segment_strategy:, selected_strategy:, admission_drops:)
|
|
165
|
+
{
|
|
166
|
+
yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
|
|
167
|
+
**(segment_strategy ? { segment_strategy: segment_strategy.to_s } : {}),
|
|
168
|
+
**(selected_strategy ? { selected_strategy: selected_strategy.to_s } : {}),
|
|
169
|
+
**(admission_drops.any? ? { admission_drops: } : {})
|
|
170
|
+
}
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def next_step_for_error(error)
|
|
174
|
+
case error
|
|
175
|
+
when RequestService::BotasaurusConfigurationError then NextStep.read_runtime
|
|
176
|
+
when Contract::UnpublishedRequestError then NextStep.validate_config
|
|
177
|
+
when ArgumentError then argument_error_next_step(error)
|
|
178
|
+
else NextStep.inspect_url
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def argument_error_next_step(error)
|
|
183
|
+
XOR_ERROR.match?(error.message) ? NextStep.validate_config : NextStep.inspect_url
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|