html2rss 0.26.0 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +41 -18
  3. data/html2rss.gemspec +1 -1
  4. data/lib/html2rss/auto_source/README.md +57 -0
  5. data/lib/html2rss/auto_source/cleanup.rb +121 -56
  6. data/lib/html2rss/auto_source/scraper.rb +13 -0
  7. data/lib/html2rss/auto_source.rb +34 -8
  8. data/lib/html2rss/capture/README.md +61 -0
  9. data/lib/html2rss/capture.rb +120 -117
  10. data/lib/html2rss/cli.rb +35 -17
  11. data/lib/html2rss/config/schema.rb +12 -0
  12. data/lib/html2rss/config/validator.rb +33 -8
  13. data/lib/html2rss/config.rb +28 -0
  14. data/lib/html2rss/error.rb +24 -6
  15. data/lib/html2rss/feed_pipeline/README.md +42 -0
  16. data/lib/html2rss/feed_pipeline/auto_fallback.rb +22 -9
  17. data/lib/html2rss/feed_pipeline/strategy_plan.rb +11 -0
  18. data/lib/html2rss/feed_pipeline.rb +30 -12
  19. data/lib/html2rss/html/article_extractor/category_extractor.rb +36 -22
  20. data/lib/html2rss/html/article_extractor/date_extractor.rb +5 -3
  21. data/lib/html2rss/html/article_extractor.rb +95 -17
  22. data/lib/html2rss/html/article_rules/category.rb +28 -11
  23. data/lib/html2rss/html/article_rules/date.rb +60 -6
  24. data/lib/html2rss/html/article_rules/description.rb +122 -0
  25. data/lib/html2rss/html/card_walk.rb +42 -0
  26. data/lib/html2rss/html/feed_link.rb +34 -0
  27. data/lib/html2rss/html/navigator.rb +18 -0
  28. data/lib/html2rss/html/sst_article_extractor.rb +119 -32
  29. data/lib/html2rss/link_destination/path_classifier.rb +49 -35
  30. data/lib/html2rss/mcp/config_argument.rb +42 -0
  31. data/lib/html2rss/mcp/contract.rb +173 -0
  32. data/lib/html2rss/mcp/inspect.rb +241 -0
  33. data/lib/html2rss/mcp/outcome.rb +188 -0
  34. data/lib/html2rss/mcp/server.rb +253 -409
  35. data/lib/html2rss/request_service/botasaurus_contract.rb +193 -102
  36. data/lib/html2rss/request_service/botasaurus_strategy.rb +15 -8
  37. data/lib/html2rss/request_service/compressed_body.rb +109 -0
  38. data/lib/html2rss/request_service/faraday_strategy.rb +3 -1
  39. data/lib/html2rss/request_service/policy.rb +1 -1
  40. data/lib/html2rss/request_service/response.rb +57 -6
  41. data/lib/html2rss/request_service.rb +6 -1
  42. data/lib/html2rss/selectors.rb +2 -1
  43. data/lib/html2rss/status.rb +27 -11
  44. data/lib/html2rss/url.rb +20 -0
  45. data/lib/html2rss/version.rb +1 -1
  46. data/lib/html2rss.rb +30 -6
  47. data/schema/html2rss-config.schema.json +40 -16
  48. metadata +15 -4
@@ -0,0 +1,188 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ module MCP
5
+ Outcome = Data.define(:ok, :next_step, :guidance, :payload)
6
+
7
+ ##
8
+ # Typed MCP tool result. Owns next-step policy and guidance copy so the
9
+ # protocol adapter does not branch on quality heuristics.
10
+ class Outcome
11
+ # Matches {ConfigArgument} XOR {ArgumentError} messages.
12
+ XOR_ERROR = /exactly one of config or yaml/
13
+ NextStep = Data.define(:name, :guidance)
14
+
15
+ ##
16
+ # Closed set of agent next actions. Invalid names cannot be constructed.
17
+ class NextStep
18
+ # Wire names for +next_step+.
19
+ NAMES = %i[done inspect_url validate_config apply_config scrape_url capture_config read_runtime].freeze
20
+ # Default guidance copy keyed by {NAMES}.
21
+ GUIDANCE = {
22
+ done: 'Done. Read payload for the result.',
23
+ inspect_url: 'Call inspect_url next. Read payload for recon (final_url, status, ' \
24
+ 'scheme_downgrade, alternate_feeds).',
25
+ validate_config: 'Call validate_config with payload.yaml or a config hash (XOR, not both).',
26
+ apply_config: 'Call apply_config next. Confirm payload.item_count before shipping.',
27
+ scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus.',
28
+ capture_config: 'Call capture_config for a reusable YAML draft, then follow next_step.',
29
+ read_runtime: 'Read html2rss://runtime. Set BOTASAURUS_SCRAPER_URL on the MCP process ' \
30
+ 'if botasaurus_configured is false.'
31
+ }.freeze
32
+
33
+ ##
34
+ # @param name [Symbol, String]
35
+ # @param guidance [String, nil]
36
+ def initialize(name:, guidance: nil)
37
+ step = name.to_sym
38
+ raise ArgumentError, "unknown next_step: #{name.inspect}" unless NAMES.include?(step)
39
+
40
+ super(name: step, guidance: (guidance || GUIDANCE.fetch(step)).freeze)
41
+ end
42
+
43
+ class << self
44
+ NAMES.each { |step| define_method(step) { new(name: step) } }
45
+ end
46
+ end
47
+
48
+ ##
49
+ # @param ok [Boolean]
50
+ # @param next_step [NextStep]
51
+ # @param guidance [String]
52
+ # @param payload [Hash]
53
+ def initialize(ok:, next_step:, guidance:, payload:) # rubocop:disable Naming/MethodParameterName -- +ok+ is the envelope field
54
+ raise ArgumentError, 'next_step must be a NextStep' unless next_step.is_a?(NextStep)
55
+ raise ArgumentError, 'payload must be a Hash' unless payload.is_a?(Hash)
56
+
57
+ super(ok: !!ok, next_step:, guidance: guidance.to_s.freeze, payload: payload.dup.freeze)
58
+ end
59
+
60
+ ##
61
+ # @return [Hash{Symbol => Object}] envelope for {Contract.response}
62
+ def to_h
63
+ { ok:, next_step: next_step.name.to_s, guidance:, payload: }
64
+ end
65
+
66
+ class << self
67
+ ##
68
+ # @param items [Array]
69
+ # @param requested_strategy [String, Symbol]
70
+ # @param channel_title [String, nil]
71
+ # @param admission_drops [Hash]
72
+ # @param botasaurus_configured [Boolean]
73
+ # @return [Outcome]
74
+ def scrape(items:, requested_strategy:, channel_title:, botasaurus_configured:, admission_drops: {})
75
+ next_step = scrape_next_step(items.empty?, botasaurus_configured:)
76
+ new(ok: true, next_step:, guidance: next_step.guidance,
77
+ payload: scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:))
78
+ end
79
+
80
+ ##
81
+ # @param payload [Hash] inspect recon Hash
82
+ # @return [Outcome]
83
+ def inspect(payload:)
84
+ next_step = inspect_next_step(payload)
85
+ new(ok: true, next_step:, guidance: next_step.guidance, payload:)
86
+ end
87
+
88
+ ##
89
+ # @param yaml [String]
90
+ # @param articles_count [Integer]
91
+ # @param has_selectors [Boolean]
92
+ # @param channel_title [String, nil]
93
+ # @param requested_strategy [String, Symbol]
94
+ # @param segment_strategy [Symbol, String, nil]
95
+ # @param selected_strategy [Symbol, String, nil]
96
+ # @param admission_drops [Hash]
97
+ # @return [Outcome]
98
+ def capture(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
99
+ segment_strategy: nil, selected_strategy: nil, admission_drops: {})
100
+ next_step = capture_next_step(articles_count:, has_selectors:)
101
+ new(ok: true, next_step:, guidance: next_step.guidance, payload: capture_payload(
102
+ yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:,
103
+ segment_strategy:, selected_strategy:, admission_drops:
104
+ ))
105
+ end
106
+
107
+ ##
108
+ # @param errors [Hash, nil] schema errors; +nil+ means success
109
+ # @return [Outcome]
110
+ def validate(errors:)
111
+ ok = errors.nil?
112
+ next_step = ok ? NextStep.apply_config : NextStep.validate_config
113
+ new(ok:, next_step:, guidance: next_step.guidance, payload: ok ? {} : { errors: })
114
+ end
115
+
116
+ ##
117
+ # @param rss [String]
118
+ # @param item_count [Integer]
119
+ # @param empty [Boolean] {FeedResult#empty?} (ship gate); defaults to zero items
120
+ # @return [Outcome]
121
+ def apply(rss:, item_count:, empty: item_count.zero?)
122
+ ok = !empty
123
+ next_step = ok ? NextStep.done : NextStep.inspect_url
124
+ new(ok:, next_step:, guidance: next_step.guidance, payload: { rss:, item_count: })
125
+ end
126
+
127
+ ##
128
+ # @param error [Exception]
129
+ # @return [Outcome]
130
+ def from_error(error)
131
+ next_step = next_step_for_error(error)
132
+ new(ok: false, next_step:, guidance: next_step.guidance,
133
+ payload: { class: error.class.name, message: error.message })
134
+ end
135
+
136
+ private
137
+
138
+ def scrape_next_step(empty, botasaurus_configured:)
139
+ return NextStep.done unless empty
140
+ return NextStep.read_runtime unless botasaurus_configured
141
+
142
+ NextStep.inspect_url
143
+ end
144
+
145
+ def scrape_payload(items:, requested_strategy:, channel_title:, admission_drops:)
146
+ {
147
+ items:, total: items.size, requested_strategy: requested_strategy.to_s, channel_title:,
148
+ **(admission_drops.any? ? { admission_drops: } : {})
149
+ }
150
+ end
151
+
152
+ def inspect_next_step(payload)
153
+ return NextStep.done if Array(payload[:alternate_feeds]).any?
154
+ return NextStep.capture_config if payload[:articles_count].to_i.positive?
155
+
156
+ NextStep.scrape_url
157
+ end
158
+
159
+ def capture_next_step(articles_count:, has_selectors:)
160
+ articles_count.positive? && has_selectors ? NextStep.validate_config : NextStep.inspect_url
161
+ end
162
+
163
+ def capture_payload(yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy:, # rubocop:disable Metrics/ParameterLists
164
+ segment_strategy:, selected_strategy:, admission_drops:)
165
+ {
166
+ yaml:, articles_count:, has_selectors:, channel_title:, requested_strategy: requested_strategy.to_s,
167
+ **(segment_strategy ? { segment_strategy: segment_strategy.to_s } : {}),
168
+ **(selected_strategy ? { selected_strategy: selected_strategy.to_s } : {}),
169
+ **(admission_drops.any? ? { admission_drops: } : {})
170
+ }
171
+ end
172
+
173
+ def next_step_for_error(error)
174
+ case error
175
+ when RequestService::BotasaurusConfigurationError then NextStep.read_runtime
176
+ when Contract::UnpublishedRequestError then NextStep.validate_config
177
+ when ArgumentError then argument_error_next_step(error)
178
+ else NextStep.inspect_url
179
+ end
180
+ end
181
+
182
+ def argument_error_next_step(error)
183
+ XOR_ERROR.match?(error.message) ? NextStep.validate_config : NextStep.inspect_url
184
+ end
185
+ end
186
+ end
187
+ end
188
+ end